1use super::*;
2
3use std::collections::{BTreeMap, BTreeSet, HashSet};
4use std::path::{Path, PathBuf};
5use std::sync::atomic::{AtomicBool, Ordering};
6use std::time::Duration;
7
8use bamboo_config::{
9 ensure_provider_mcp_migration_ready, AtomicJsonStore, ConfigDirectoryWatcher,
10 ConfigSectionEvent, ConfigStoreError, McpSection, ProviderConfigs, SectionId,
11 SectionSourceKind, SectionStatus,
12};
13use bamboo_mcp::{McpConfig, McpServerManager, TransportConfig};
14use chrono::{DateTime, Utc};
15use serde::Serialize;
16use serde_json::Value;
17
18#[cfg(test)]
19struct InitialMcpApplyTestHook {
20 before: Box<dyn FnOnce() + Send + 'static>,
21 after: Box<dyn FnOnce() + Send + 'static>,
22}
23
24#[cfg(test)]
25fn initial_mcp_apply_test_hooks(
26) -> &'static std::sync::Mutex<std::collections::HashMap<PathBuf, InitialMcpApplyTestHook>> {
27 static HOOKS: std::sync::OnceLock<
28 std::sync::Mutex<std::collections::HashMap<PathBuf, InitialMcpApplyTestHook>>,
29 > = std::sync::OnceLock::new();
30 HOOKS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
31}
32
33#[cfg(test)]
34fn set_initial_mcp_apply_test_hook(
35 data_dir: &Path,
36 before: impl FnOnce() + Send + 'static,
37 after: impl FnOnce() + Send + 'static,
38) {
39 initial_mcp_apply_test_hooks()
40 .lock()
41 .unwrap_or_else(|poisoned| poisoned.into_inner())
42 .insert(
43 data_dir.to_path_buf(),
44 InitialMcpApplyTestHook {
45 before: Box::new(before),
46 after: Box::new(after),
47 },
48 );
49}
50
51#[cfg(test)]
52struct InitialMcpApplyTestCompletion(Option<Box<dyn FnOnce() + Send + 'static>>);
53
54#[cfg(test)]
55impl Drop for InitialMcpApplyTestCompletion {
56 fn drop(&mut self) {
57 if let Some(after) = self.0.take() {
58 after();
59 }
60 }
61}
62
63#[cfg(test)]
64fn begin_initial_mcp_apply_test_hook(data_dir: &Path) -> InitialMcpApplyTestCompletion {
65 let hook = initial_mcp_apply_test_hooks()
66 .lock()
67 .unwrap_or_else(|poisoned| poisoned.into_inner())
68 .remove(data_dir);
69 let Some(hook) = hook else {
70 return InitialMcpApplyTestCompletion(None);
71 };
72 (hook.before)();
73 InitialMcpApplyTestCompletion(Some(hook.after))
74}
75
76#[cfg(test)]
77struct ClusterAfterCommitBeforeAdoptionTestHook {
78 expected_revision: u64,
79 hook: Box<dyn FnOnce(&Path) + Send + 'static>,
80}
81
82#[cfg(test)]
83fn cluster_after_commit_before_adoption_test_hooks() -> &'static std::sync::Mutex<
84 std::collections::HashMap<PathBuf, ClusterAfterCommitBeforeAdoptionTestHook>,
85> {
86 static HOOKS: std::sync::OnceLock<
87 std::sync::Mutex<
88 std::collections::HashMap<PathBuf, ClusterAfterCommitBeforeAdoptionTestHook>,
89 >,
90 > = std::sync::OnceLock::new();
91 HOOKS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
92}
93
94#[cfg(test)]
95fn set_cluster_after_commit_before_adoption_test_hook(
96 data_dir: &Path,
97 expected_revision: u64,
98 hook: impl FnOnce(&Path) + Send + 'static,
99) {
100 cluster_after_commit_before_adoption_test_hooks()
101 .lock()
102 .unwrap_or_else(|poisoned| poisoned.into_inner())
103 .insert(
104 data_dir.to_path_buf(),
105 ClusterAfterCommitBeforeAdoptionTestHook {
106 expected_revision,
107 hook: Box::new(hook),
108 },
109 );
110}
111
112#[cfg(test)]
113fn run_cluster_after_commit_before_adoption_test_hook(data_dir: &Path, expected_revision: u64) {
114 let hook = {
115 let mut hooks = cluster_after_commit_before_adoption_test_hooks()
116 .lock()
117 .unwrap_or_else(|poisoned| poisoned.into_inner());
118 if hooks
119 .get(data_dir)
120 .is_some_and(|hook| hook.expected_revision == expected_revision)
121 {
122 hooks.remove(data_dir)
123 } else {
124 None
125 }
126 };
127 if let Some(hook) = hook {
128 (hook.hook)(data_dir);
129 }
130}
131
132#[cfg(test)]
133type CredentialCommitTestHook = Box<dyn FnOnce() + Send + 'static>;
134#[cfg(test)]
135type CredentialCommitTestHooks =
136 std::sync::Mutex<std::collections::HashMap<(PathBuf, SectionId), CredentialCommitTestHook>>;
137
138#[cfg(test)]
139fn credential_after_commit_before_live_test_hooks() -> &'static CredentialCommitTestHooks {
140 static HOOKS: std::sync::OnceLock<CredentialCommitTestHooks> = std::sync::OnceLock::new();
141 HOOKS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
142}
143
144#[cfg(test)]
145fn set_credential_after_commit_before_live_test_hook(
146 data_dir: &Path,
147 section: SectionId,
148 hook: impl FnOnce() + Send + 'static,
149) {
150 credential_after_commit_before_live_test_hooks()
151 .lock()
152 .unwrap_or_else(|poisoned| poisoned.into_inner())
153 .insert((data_dir.to_path_buf(), section), Box::new(hook));
154}
155
156#[cfg(test)]
157fn run_credential_after_commit_before_live_test_hook(data_dir: &Path, section: SectionId) {
158 let hook = credential_after_commit_before_live_test_hooks()
159 .lock()
160 .unwrap_or_else(|poisoned| poisoned.into_inner())
161 .remove(&(data_dir.to_path_buf(), section));
162 if let Some(hook) = hook {
163 hook();
164 }
165}
166
167#[cfg(test)]
168type GenericBeforeEventTestHook = Box<dyn FnOnce() + Send + 'static>;
169#[cfg(test)]
170type GenericBeforeEventTestHooks =
171 std::sync::Mutex<std::collections::HashMap<PathBuf, GenericBeforeEventTestHook>>;
172
173#[cfg(test)]
174fn generic_before_event_test_hooks() -> &'static GenericBeforeEventTestHooks {
175 static HOOKS: std::sync::OnceLock<GenericBeforeEventTestHooks> = std::sync::OnceLock::new();
176 HOOKS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
177}
178
179#[cfg(test)]
180fn set_generic_before_event_test_hook(data_dir: &Path, hook: impl FnOnce() + Send + 'static) {
181 generic_before_event_test_hooks()
182 .lock()
183 .unwrap_or_else(|poisoned| poisoned.into_inner())
184 .insert(data_dir.to_path_buf(), Box::new(hook));
185}
186
187#[cfg(test)]
188fn run_generic_before_event_test_hook(data_dir: &Path) {
189 let hook = generic_before_event_test_hooks()
190 .lock()
191 .unwrap_or_else(|poisoned| poisoned.into_inner())
192 .remove(data_dir);
193 if let Some(hook) = hook {
194 hook();
195 }
196}
197
198#[cfg(test)]
199type GenericBeforeProviderPublishTestHook = Box<dyn FnOnce() + Send + 'static>;
200#[cfg(test)]
201type GenericBeforeProviderPublishTestHooks =
202 std::sync::Mutex<std::collections::HashMap<PathBuf, GenericBeforeProviderPublishTestHook>>;
203
204#[cfg(test)]
205fn generic_before_provider_publish_test_hooks() -> &'static GenericBeforeProviderPublishTestHooks {
206 static HOOKS: std::sync::OnceLock<GenericBeforeProviderPublishTestHooks> =
207 std::sync::OnceLock::new();
208 HOOKS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
209}
210
211#[cfg(test)]
212fn set_generic_before_provider_publish_test_hook(
213 data_dir: &Path,
214 hook: impl FnOnce() + Send + 'static,
215) {
216 generic_before_provider_publish_test_hooks()
217 .lock()
218 .unwrap_or_else(|poisoned| poisoned.into_inner())
219 .insert(data_dir.to_path_buf(), Box::new(hook));
220}
221
222#[cfg(test)]
223fn run_generic_before_provider_publish_test_hook(data_dir: &Path) {
224 let hook = generic_before_provider_publish_test_hooks()
225 .lock()
226 .unwrap_or_else(|poisoned| poisoned.into_inner())
227 .remove(data_dir);
228 if let Some(hook) = hook {
229 hook();
230 }
231}
232
233#[cfg(test)]
234type ResetAfterDeleteTestHook = Box<dyn FnOnce() + Send + 'static>;
235#[cfg(test)]
236type ResetAfterDeleteTestHooks =
237 std::sync::Mutex<std::collections::HashMap<PathBuf, ResetAfterDeleteTestHook>>;
238
239#[cfg(test)]
240fn reset_after_delete_test_hooks() -> &'static ResetAfterDeleteTestHooks {
241 static HOOKS: std::sync::OnceLock<ResetAfterDeleteTestHooks> = std::sync::OnceLock::new();
242 HOOKS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
243}
244
245#[cfg(test)]
246fn set_reset_after_delete_test_hook(data_dir: &Path, hook: impl FnOnce() + Send + 'static) {
247 reset_after_delete_test_hooks()
248 .lock()
249 .unwrap_or_else(|poisoned| poisoned.into_inner())
250 .insert(data_dir.to_path_buf(), Box::new(hook));
251}
252
253#[cfg(test)]
254fn run_reset_after_delete_test_hook(data_dir: &Path) {
255 let hook = reset_after_delete_test_hooks()
256 .lock()
257 .unwrap_or_else(|poisoned| poisoned.into_inner())
258 .remove(data_dir);
259 if let Some(hook) = hook {
260 hook();
261 }
262}
263
264struct FacadeRuntimeMaterialization {
265 config: Config,
266 failures: BTreeSet<SectionId>,
267}
268
269fn materialize_facade_effective_config(
270 facade: &bamboo_config::ConfigFacade,
271 data_dir: &Path,
272) -> FacadeRuntimeMaterialization {
273 let mut config = facade.effective_config();
274 let mut failures = BTreeSet::new();
275 if let Err(error) = config.hydrate_proxy_auth_from_store(data_dir) {
276 tracing::warn!(error = %error, "proxy auth credential hydration unavailable");
277 config.proxy_auth = None;
278 failures.insert(SectionId::Core);
279 }
280 if let Err(error) = config.hydrate_provider_credentials_from_store(data_dir) {
281 tracing::warn!(error = %error, "provider credential hydration unavailable");
282 failures.insert(SectionId::Providers);
283 }
284 if let Err(error) = config.hydrate_mcp_credentials_from_store(data_dir) {
285 tracing::warn!(error = %error, "MCP credential hydration unavailable");
286 failures.insert(SectionId::Mcp);
287 }
288 if let Err(error) = config.hydrate_env_var_credentials_from_store(data_dir) {
289 tracing::warn!(error = %error, "env credential hydration unavailable");
290 for entry in &mut config.env_vars {
291 if entry.secret {
292 entry.value.clear();
293 }
294 }
295 failures.insert(SectionId::Env);
296 }
297 if let Err(error) = config.hydrate_cluster_credentials_from_store(data_dir) {
298 tracing::warn!(error = %error, "cluster credential hydration unavailable");
299 failures.insert(SectionId::ClusterFabric);
300 }
301 if let Err(error) = config.hydrate_notification_credentials_from_store(data_dir) {
302 tracing::warn!(error = %error, "notification credential hydration unavailable");
303 config.notifications.ntfy.token = None;
304 config.notifications.bark.device_key = None;
305 failures.insert(SectionId::Notifications);
306 }
307 if let Err(error) = config.hydrate_connect_credentials_from_store(data_dir) {
308 tracing::warn!(error = %error, "connect credential hydration unavailable");
309 for platform in &mut config.connect.platforms {
310 platform.token = None;
311 platform.app_secret = None;
312 }
313 failures.insert(SectionId::Connect);
314 }
315 if let Err(error) = config.hydrate_access_control_credentials_from_store(data_dir) {
316 tracing::warn!(error = %error, "access-control credential hydration unavailable");
317 config.clear_access_control_runtime_verifiers();
318 failures.insert(SectionId::AccessControl);
319 }
320 if config
321 .access_control
322 .as_ref()
323 .is_some_and(|access| access.repair_required)
324 {
325 failures.insert(SectionId::AccessControl);
326 }
327 if let Some(broker) = config.subagents_mut().broker.as_mut() {
328 if let Err(error) = broker.hydrate_credential_from_store(data_dir) {
329 tracing::warn!(error = %error, "external broker credential hydration unavailable");
330 broker.token.clear();
331 failures.insert(SectionId::Subagents);
332 }
333 }
334 config.apply_runtime_env_overrides();
335 FacadeRuntimeMaterialization { config, failures }
336}
337
338pub(super) fn load_facade_effective_config(
339 facade: &bamboo_config::ConfigFacade,
340 data_dir: &Path,
341) -> Config {
342 let materialized = materialize_facade_effective_config(facade, data_dir);
343 for section in &materialized.failures {
344 facade.registry().mark_runtime_degraded(
345 *section,
346 "configuration runtime credential repair is required",
347 );
348 }
349 materialized.config
350}
351
352fn load_committed_effective_config(data_dir: &Path) -> Result<Config, ConfigStoreError> {
353 if bamboo_config::modular_authority_boundary_present(data_dir)? {
354 let facade = bamboo_config::ConfigFacade::open_or_migrate(data_dir)?;
355 let config = load_facade_effective_config(&facade, data_dir);
356 Ok(config)
357 } else {
358 Ok(Config::from_data_dir_without_publish(Some(
359 data_dir.to_path_buf(),
360 )))
361 }
362}
363
364#[derive(Debug, Clone, Serialize)]
366pub struct ConfigLiveHealth {
367 pub revision: u64,
368 pub loaded_at: DateTime<Utc>,
369 pub source_path: PathBuf,
370 pub source_kind: SectionSourceKind,
371 pub status: SectionStatus,
372 pub last_error: Option<String>,
373}
374
375pub struct ConfigWatcherRuntime {
377 stop: Arc<AtomicBool>,
378 watcher_task: Option<std::thread::JoinHandle<()>>,
379 apply_task: Option<tokio::task::JoinHandle<()>>,
380}
381
382struct ConfigPathChanges {
383 paths: Vec<PathBuf>,
384 initial_mcp_revision: Option<u64>,
385 startup_legacy_root: Option<bamboo_config::LegacyRootReconciliationOutcome>,
386 startup_recoveries: BTreeMap<SectionId, u64>,
387 legacy_root_retry_attempt: u8,
388}
389
390struct ConfigRuntimeEffectContext {
394 app_data_dir: PathBuf,
395 config_facade: Option<Arc<bamboo_config::ConfigFacade>>,
396 provider_registry: Arc<bamboo_llm::ProviderRegistry>,
397 provider: Arc<RwLock<Arc<dyn LLMProvider>>>,
398 mcp_manager: Arc<McpServerManager>,
399 account_sink: Arc<bamboo_engine::events::AccountEventSink>,
400 config_live_health: Arc<std::sync::RwLock<ConfigLiveHealth>>,
401 mcp_config_live_health: Arc<std::sync::RwLock<ConfigLiveHealth>>,
402}
403
404impl ConfigWatcherRuntime {
405 #[allow(clippy::too_many_arguments)]
406 pub fn start(
407 data_dir: PathBuf,
408 config: Arc<RwLock<Config>>,
409 config_facade: Option<Arc<bamboo_config::ConfigFacade>>,
410 config_io_lock: Arc<tokio::sync::Mutex<()>>,
411 provider_registry: Arc<bamboo_llm::ProviderRegistry>,
412 provider: Arc<RwLock<Arc<dyn LLMProvider>>>,
413 mcp_manager: Arc<McpServerManager>,
414 account_sink: Arc<bamboo_engine::events::AccountEventSink>,
415 ) -> (
416 Self,
417 Arc<std::sync::RwLock<ConfigLiveHealth>>,
418 Arc<std::sync::RwLock<ConfigLiveHealth>>,
419 ) {
420 let provider_store = AtomicJsonStore::new(data_dir.join("providers.json"), 1);
421 let provider_health = Arc::new(std::sync::RwLock::new(initial_provider_health(
422 &provider_store,
423 )));
424 let mcp_store = AtomicJsonStore::new(data_dir.join("mcp.json"), 1);
425 let mcp_health = Arc::new(std::sync::RwLock::new(initial_mcp_health(&mcp_store)));
426 let stop = Arc::new(AtomicBool::new(false));
427 let watcher = match ConfigDirectoryWatcher::watch(&data_dir, Duration::from_millis(120)) {
428 Ok(watcher) => watcher,
429 Err(error) => {
430 tracing::warn!(error = %error, "live configuration watcher could not start");
431 {
432 let mut value = provider_health
433 .write()
434 .unwrap_or_else(|poisoned| poisoned.into_inner());
435 value.status = SectionStatus::Degraded;
436 value.last_error = Some("configuration watcher unavailable".to_string());
437 }
438 {
439 let mut value = mcp_health
440 .write()
441 .unwrap_or_else(|poisoned| poisoned.into_inner());
442 value.status = SectionStatus::Degraded;
443 value.last_error = Some("configuration watcher unavailable".to_string());
444 }
445 return (
446 Self {
447 stop,
448 watcher_task: None,
449 apply_task: None,
450 },
451 provider_health,
452 mcp_health,
453 );
454 }
455 };
456
457 let self_write_marker = watcher.self_write_marker();
461 let startup_legacy_root = config_facade
462 .as_ref()
463 .and_then(|facade| facade.take_startup_legacy_root_reconciliation());
464 let (changes_tx, mut changes_rx) =
465 tokio::sync::mpsc::unbounded_channel::<ConfigPathChanges>();
466 let initial_changes = changes_tx.clone();
467 let worker_stop = stop.clone();
468 let watcher_task = std::thread::spawn(move || {
469 while !worker_stop.load(Ordering::Relaxed) {
470 match watcher.recv_timeout(Duration::from_millis(250)) {
471 Ok(paths) => {
472 if changes_tx
473 .send(ConfigPathChanges {
474 paths,
475 initial_mcp_revision: None,
476 startup_legacy_root: None,
477 startup_recoveries: BTreeMap::new(),
478 legacy_root_retry_attempt: 0,
479 })
480 .is_err()
481 {
482 break;
483 }
484 }
485 Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {}
486 Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break,
487 }
488 }
489 });
490
491 let initial_mcp_path = data_dir.join("mcp.json");
496 let mut initial_paths = Vec::new();
497 let initial_mcp_revision = if initial_mcp_path.exists() {
498 initial_paths.push(initial_mcp_path);
499 Some(
500 config_facade
501 .as_ref()
502 .map(|facade| facade.registry().mcp.snapshot().revision)
503 .unwrap_or_else(|| {
504 mcp_health
505 .read()
506 .unwrap_or_else(|poisoned| poisoned.into_inner())
507 .revision
508 }),
509 )
510 } else {
511 None
512 };
513 let rejected = bamboo_config::legacy_root_rejected_sections(&data_dir);
514 let startup_recoveries = match (config_facade.as_ref(), rejected.as_ref()) {
515 (Some(facade), Ok(rejected)) => {
516 durable_invalid_recoveries(&account_sink, facade, rejected)
517 }
518 _ => BTreeMap::new(),
519 };
520 if let (Some(facade), Ok(rejected)) = (config_facade.as_ref(), rejected.as_ref()) {
521 if let Ok(health) = facade.registry().health() {
522 initial_paths.extend(
523 health
524 .into_iter()
525 .filter(|health| {
526 health.status != SectionStatus::Healthy
527 && !rejected.contains(&health.section)
528 })
529 .map(|health| data_dir.join(health.section.descriptor().file_name)),
530 );
531 }
532 }
533 initial_paths.extend(
534 startup_recoveries
535 .keys()
536 .map(|id| data_dir.join(id.descriptor().file_name)),
537 );
538 let legacy_root_needs_reconciliation = config_facade.is_some();
543 if legacy_root_needs_reconciliation {
544 initial_paths.push(data_dir.join("config.json"));
545 }
546 if !initial_paths.is_empty() {
547 let _ = initial_changes.send(ConfigPathChanges {
548 paths: initial_paths,
549 initial_mcp_revision,
550 startup_legacy_root,
551 startup_recoveries,
552 legacy_root_retry_attempt: 0,
553 });
554 }
555
556 let apply_provider_health = provider_health.clone();
557 let apply_mcp_health = mcp_health.clone();
558 let catchup_changes = initial_changes.clone();
559 let apply_task = tokio::spawn(async move {
560 let mut reported_root_runtime_failures = BTreeSet::<(SectionId, u64)>::new();
561 while let Some(mut changes) = changes_rx.recv().await {
562 let mut watched_sections = config_facade
563 .as_ref()
564 .map(|_| {
565 changes
566 .paths
567 .iter()
568 .filter_map(|path| {
569 path.file_name()
570 .and_then(|name| name.to_str())
571 .and_then(SectionId::from_file_name)
572 })
573 .collect::<BTreeSet<_>>()
574 })
575 .unwrap_or_default();
576 let direct_watched_sections = watched_sections.clone();
577 let legacy_root_watched = config_facade.is_some()
578 && changes.paths.iter().any(|path| {
579 matches!(
580 path.file_name().and_then(|name| name.to_str()),
581 Some(
582 "config.json"
583 | "config.json.bak"
584 | "config.json.bak.1"
585 | "config.json.bak.2"
586 )
587 )
588 });
589 let mut provider_watched = changes.paths.iter().any(|path| {
590 path.file_name().and_then(|name| name.to_str()) == Some("providers.json")
591 });
592 let mut mcp_watched = changes.paths.iter().any(|path| {
593 path.file_name().and_then(|name| name.to_str()) == Some("mcp.json")
594 });
595 if !provider_watched
596 && !mcp_watched
597 && watched_sections.is_empty()
598 && !legacy_root_watched
599 {
600 continue;
601 }
602
603 #[cfg(test)]
604 let _initial_mcp_apply_completion = changes
605 .initial_mcp_revision
606 .map(|_| begin_initial_mcp_apply_test_hook(&data_dir));
607
608 let _io = config_io_lock.lock().await;
612 let mut synthetic_root_events = BTreeMap::<SectionId, ConfigSectionEvent>::new();
613 let mut pending_root_publications =
614 BTreeMap::<SectionId, ConfigSectionEvent>::new();
615 let startup_root_batch = changes.startup_legacy_root.is_some();
616 let mut requeue_legacy_root = false;
617 let mut retry_legacy_root_publication = false;
618 let mut canonical_root_rejections = BTreeSet::new();
619 if legacy_root_watched {
620 let reconciliation = match changes.startup_legacy_root.take() {
621 Some(outcome) => Ok(Some(outcome)),
622 None => {
623 let reconcile_facade = config_facade
624 .as_ref()
625 .expect("legacy root watching requires a facade")
626 .clone();
627 tokio::task::spawn_blocking(move || {
628 reconcile_facade.reconcile_reappeared_legacy_root()
629 })
630 .await
631 .map_err(|_| {
632 ConfigStoreError::Validation(
633 "legacy root reconciliation task failed".to_string(),
634 )
635 })
636 .and_then(|result| result)
637 }
638 };
639 match reconciliation {
640 Ok(Some(outcome)) if !outcome.duplicate => {
641 requeue_legacy_root = outcome.partial || startup_root_batch;
642 for event in &outcome.committed {
643 let section = match event {
644 ConfigSectionEvent::Changed { section, .. }
645 | ConfigSectionEvent::Invalid { section, .. }
646 | ConfigSectionEvent::Recovered { section, .. } => section,
647 };
648 if let Some(id) = SectionId::from_name(section) {
649 watched_sections.insert(id);
650 synthetic_root_events.insert(id, event.clone());
651 if matches!(
652 event,
653 ConfigSectionEvent::Changed { .. }
654 | ConfigSectionEvent::Recovered { .. }
655 ) {
656 pending_root_publications.insert(id, event.clone());
657 }
658 }
659 }
660 if let Some(facade) = config_facade.as_ref() {
661 for id in &outcome.recovered {
662 queue_legacy_root_recovery(
663 facade,
664 *id,
665 None,
666 &mut watched_sections,
667 &mut synthetic_root_events,
668 );
669 }
670 for rejection in outcome.rejected {
671 canonical_root_rejections.insert(rejection.section);
672 if rejection.reason
673 == bamboo_config::LegacyRootRejectionReason::RevisionConflict
674 {
675 watched_sections.insert(rejection.section);
676 }
677 if let Some(event) = legacy_root_rejection_event(
678 facade,
679 rejection.section,
680 rejection.reason.diagnostic(),
681 startup_root_batch,
682 ) {
683 publish_registry_event(&account_sink, &event).await;
684 }
685 }
686 }
687 if outcome.partial {
688 tracing::warn!(
689 "legacy config root changed during reconciliation; awaiting the newer generation"
690 );
691 }
692 }
693 Ok(Some(outcome)) => {
694 requeue_legacy_root = outcome.partial || startup_root_batch;
695 if let Some(facade) = config_facade.as_ref() {
702 for id in &outcome.recovered {
703 queue_legacy_root_recovery(
704 facade,
705 *id,
706 None,
707 &mut watched_sections,
708 &mut synthetic_root_events,
709 );
710 }
711 for event in &outcome.committed {
712 let (section, revision) = match event {
713 ConfigSectionEvent::Changed { section, revision }
714 | ConfigSectionEvent::Recovered { section, revision } => {
715 (section, *revision)
716 }
717 ConfigSectionEvent::Invalid { .. } => continue,
718 };
719 let Some(id) = SectionId::from_name(section) else {
720 continue;
721 };
722 watched_sections.insert(id);
723 pending_root_publications.insert(id, event.clone());
724 if facade.registry().envelope_value(id).is_ok_and(|envelope| {
725 envelope.revision == revision
726 && envelope.status == SectionStatus::Healthy
727 }) {
728 synthetic_root_events.insert(id, event.clone());
729 }
730 }
731 for rejection in outcome.rejected {
732 canonical_root_rejections.insert(rejection.section);
733 if rejection.reason
734 == bamboo_config::LegacyRootRejectionReason::RevisionConflict
735 {
736 watched_sections.insert(rejection.section);
737 }
738 if let Some(event) = legacy_root_rejection_event(
739 facade,
740 rejection.section,
741 rejection.reason.diagnostic(),
742 startup_root_batch,
743 ) {
744 publish_registry_event(&account_sink, &event).await;
745 }
746 }
747 }
748 }
749 Ok(None) => {
750 if bamboo_config::modular_authority_boundary_present(&data_dir)
751 .unwrap_or(false)
752 {
753 if let Some(facade) = config_facade.as_ref() {
754 if let Some(event) = facade.registry().mark_runtime_degraded(
755 SectionId::Core,
756 "completed modular configuration reconciliation is unavailable",
757 ) {
758 publish_registry_event(&account_sink, &event).await;
759 }
760 }
761 }
762 }
763 Err(_) => {
764 if let Some(facade) = config_facade.as_ref() {
765 if let Some(event) = facade.registry().mark_runtime_degraded(
766 SectionId::Core,
767 "legacy config root reconciliation is unavailable",
768 ) {
769 publish_registry_event(&account_sink, &event).await;
770 }
771 }
772 }
773 }
774 if let Some(facade) = config_facade.as_ref() {
775 for (id, event) in pending_root_publications.clone() {
776 let revision = config_section_event_revision(&event);
777 if !pending_root_publication_matches_fresh_durable(
778 &data_dir, facade, id, revision,
779 ) {
780 pending_root_publications.remove(&id);
787 synthetic_root_events.remove(&id);
788 watched_sections.insert(id);
789 requeue_legacy_root = true;
790 }
791 }
792 }
793 provider_watched |= watched_sections.contains(&SectionId::Providers);
794 mcp_watched |= watched_sections.contains(&SectionId::Mcp);
795 }
796 if let Some(facade) = config_facade.as_ref() {
797 for (id, revision) in std::mem::take(&mut changes.startup_recoveries) {
798 if !canonical_root_rejections.contains(&id) {
799 queue_legacy_root_recovery(
800 facade,
801 id,
802 Some(revision),
803 &mut watched_sections,
804 &mut synthetic_root_events,
805 );
806 }
807 }
808 provider_watched |= watched_sections.contains(&SectionId::Providers);
809 mcp_watched |= watched_sections.contains(&SectionId::Mcp);
810 }
811 let catchup_paths = synthetic_root_events
817 .keys()
818 .filter(|id| startup_root_batch || direct_watched_sections.contains(id))
819 .map(|id| data_dir.join(id.descriptor().file_name))
820 .collect::<Vec<_>>();
821 let ordinary_watched = watched_sections
822 .iter()
823 .copied()
824 .filter(|id| !matches!(id, SectionId::Providers | SectionId::Mcp))
825 .collect::<Vec<_>>();
826 if let Some(facade) = config_facade.as_ref() {
827 retry_legacy_root_publication |= reload_and_apply_ordinary_sections(
828 &data_dir,
829 &config,
830 facade,
831 &account_sink,
832 ordinary_watched,
833 OrdinarySectionReloadState {
834 synthetic_events: &mut synthetic_root_events,
835 pending_root_publications: &mut pending_root_publications,
836 reported_root_runtime_failures: &mut reported_root_runtime_failures,
837 },
838 )
839 .await;
840 }
841 if provider_watched {
842 if let Some(facade) = config_facade.as_ref() {
843 wait_for_section_file_settle(&data_dir, SectionId::Providers).await;
844 if let Some(observed) = synthetic_root_events
845 .remove(&SectionId::Providers)
846 .or_else(|| facade.registry().reload_if_changed(SectionId::Providers))
847 {
848 if let Some(event) = pending_root_publication_event(
849 &data_dir,
850 facade,
851 &account_sink,
852 &pending_root_publications,
853 SectionId::Providers,
854 observed,
855 ) {
856 if matches!(event, ConfigSectionEvent::Invalid { .. }) {
857 publish_section_failure(
858 &apply_provider_health,
859 &account_sink,
860 "providers",
861 facade.registry().providers.snapshot().status,
862 "provider section is invalid; retaining last-known-good runtime"
863 .to_string(),
864 )
865 .await;
866 } else {
867 self_write_marker.mark_self_write(provider_store.path());
868 let materialized =
869 materialize_facade_effective_config(facade, &data_dir);
870 if materialized.failures.contains(&SectionId::Providers) {
871 retry_legacy_root_publication |=
872 publish_staged_facade_section_failure(
873 &data_dir,
874 facade,
875 SectionId::Providers,
876 "provider credential hydration failed; retaining last-known-good runtime",
877 StagedFacadeSectionFailureContext {
878 health: &apply_provider_health,
879 account_sink: &account_sink,
880 section: "providers",
881 pending_root_publications: &pending_root_publications,
882 },
883 )
884 .await;
885 } else {
886 let mut candidate = config.read().await.clone();
887 apply_runtime_section(
888 SectionId::Providers,
889 &materialized.config,
890 &mut candidate,
891 );
892 match prepare_provider_candidate(candidate, &data_dir).await
893 {
894 Ok((candidate, registry, next_provider)) => {
895 let mut live_config = config.write().await;
896 let mut live_provider = provider.write().await;
897 let recovered =
898 section_is_unhealthy(&apply_provider_health);
899 candidate.publish_env_vars();
900 *live_config = candidate;
901 provider_registry.replace_with(registry);
902 *live_provider = next_provider;
903 drop(live_provider);
904 drop(live_config);
905 let revision =
906 facade.registry().providers.snapshot().revision;
907 if pending_root_publications
908 .get(&SectionId::Providers)
909 .is_some_and(|event| {
910 config_section_event_revision(event)
911 == revision
912 })
913 {
914 set_live_health_revision(
915 &apply_provider_health,
916 revision,
917 Some((
918 data_dir.join("providers.json"),
919 SectionSourceKind::File,
920 )),
921 );
922 retry_legacy_root_publication |=
923 publish_registry_event_with_root_ack(
924 &data_dir,
925 &account_sink,
926 &mut pending_root_publications,
927 SectionId::Providers,
928 &event,
929 )
930 .await;
931 } else if matches!(
932 event,
933 ConfigSectionEvent::Recovered { .. }
934 ) {
935 set_live_health_revision(
936 &apply_provider_health,
937 revision,
938 Some((
939 data_dir.join("providers.json"),
940 SectionSourceKind::File,
941 )),
942 );
943 publish_registry_event(&account_sink, &event)
944 .await;
945 } else {
946 publish_section_success(
947 &apply_provider_health,
948 &account_sink,
949 "providers",
950 data_dir.join("providers.json"),
951 recovered,
952 Some(revision),
953 )
954 .await;
955 }
956 }
957 Err(_) => {
958 retry_legacy_root_publication |=
959 publish_staged_facade_section_failure(
960 &data_dir,
961 facade,
962 SectionId::Providers,
963 "provider runtime initialization failed; retaining last-known-good runtime",
964 StagedFacadeSectionFailureContext {
965 health: &apply_provider_health,
966 account_sink: &account_sink,
967 section: "providers",
968 pending_root_publications: &pending_root_publications,
969 },
970 )
971 .await;
972 }
973 }
974 }
975 }
976 } else {
977 retry_legacy_root_publication = true;
978 }
979 }
980 } else {
981 let current_config = config.read().await.clone();
982 let current_revision = apply_provider_health
983 .read()
984 .unwrap_or_else(|poisoned| poisoned.into_inner())
985 .revision;
986 let result = load_and_prepare_provider_candidate(
987 &provider_store,
988 current_revision,
989 current_config,
990 )
991 .await;
992 match result {
993 Ok(candidate) if candidate.unchanged => {}
994 Ok(candidate) => {
995 if candidate.normalized_external_revision {
996 self_write_marker.mark_self_write(provider_store.path());
997 }
998 let mut live_config = config.write().await;
999 let mut live_provider = provider.write().await;
1000 let recovered = section_is_unhealthy(&apply_provider_health);
1001 candidate.config.publish_env_vars();
1002 *live_config = candidate.config;
1003 provider_registry.replace_with(candidate.registry);
1004 *live_provider = candidate.provider;
1005 drop(live_provider);
1006 drop(live_config);
1007
1008 publish_section_success(
1009 &apply_provider_health,
1010 &account_sink,
1011 "providers",
1012 data_dir.join("providers.json"),
1013 recovered,
1014 Some(candidate.revision),
1015 )
1016 .await;
1017 }
1018 Err(error) => {
1019 publish_section_failure(
1020 &apply_provider_health,
1021 &account_sink,
1022 "providers",
1023 candidate_error_status(&error.kind),
1024 error.message,
1025 )
1026 .await
1027 }
1028 }
1029 }
1030 }
1031
1032 if mcp_watched {
1033 if let Some(facade) = config_facade.as_ref() {
1034 wait_for_section_file_settle(&data_dir, SectionId::Mcp).await;
1035 let startup_root_mcp = synthetic_root_events.contains_key(&SectionId::Mcp);
1036 let synthetic = synthetic_root_events.remove(&SectionId::Mcp);
1037 let reloaded = synthetic
1038 .is_none()
1039 .then(|| facade.registry().reload_if_changed(SectionId::Mcp))
1040 .flatten();
1041 let forced_initial_mcp = synthetic.is_none()
1042 && reloaded.is_none()
1043 && changes.initial_mcp_revision.is_some_and(|revision| {
1044 facade.registry().mcp.snapshot().revision == revision
1045 });
1046 let event = synthetic.or(reloaded).or_else(|| {
1047 forced_initial_mcp.then(|| ConfigSectionEvent::Changed {
1048 section: "mcp".to_string(),
1049 revision: facade.registry().mcp.snapshot().revision,
1050 })
1051 });
1052 if let Some(observed) = event {
1053 if let Some(event) = pending_root_publication_event(
1054 &data_dir,
1055 facade,
1056 &account_sink,
1057 &pending_root_publications,
1058 SectionId::Mcp,
1059 observed,
1060 ) {
1061 if matches!(event, ConfigSectionEvent::Invalid { .. }) {
1062 publish_section_failure(
1063 &apply_mcp_health,
1064 &account_sink,
1065 "mcp",
1066 facade.registry().mcp.snapshot().status,
1067 "MCP section is invalid; retaining last-known-good runtime"
1068 .to_string(),
1069 )
1070 .await;
1071 } else {
1072 self_write_marker.mark_self_write(mcp_store.path());
1073 let materialized =
1074 materialize_facade_effective_config(facade, &data_dir);
1075 if materialized.failures.contains(&SectionId::Mcp) {
1076 retry_legacy_root_publication |=
1077 publish_staged_facade_section_failure(
1078 &data_dir,
1079 facade,
1080 SectionId::Mcp,
1081 "MCP credential hydration failed; retaining last-known-good runtime",
1082 StagedFacadeSectionFailureContext {
1083 health: &apply_mcp_health,
1084 account_sink: &account_sink,
1085 section: "mcp",
1086 pending_root_publications: &pending_root_publications,
1087 },
1088 )
1089 .await;
1090 } else {
1091 let next_mcp = materialized.config.mcp.clone();
1092 let publish_config = config.clone();
1093 match mcp_manager
1094 .reconcile_from_config_transactional_after(
1095 &materialized.config.mcp,
1096 || async move {
1097 publish_config.write().await.mcp = next_mcp;
1098 Ok(())
1099 },
1100 )
1101 .await
1102 {
1103 Ok(()) => {
1104 let recovered =
1105 section_is_unhealthy(&apply_mcp_health);
1106 let snapshot = facade.registry().mcp.snapshot();
1107 let revision = snapshot.revision;
1108 if forced_initial_mcp
1109 && !startup_root_mcp
1110 && !pending_root_publications
1111 .contains_key(&SectionId::Mcp)
1112 && snapshot.status == SectionStatus::Healthy
1113 {
1114 set_live_health_revision(
1120 &apply_mcp_health,
1121 revision,
1122 Some((
1123 data_dir.join("mcp.json"),
1124 SectionSourceKind::File,
1125 )),
1126 );
1127 } else {
1128 if pending_root_publications
1129 .get(&SectionId::Mcp)
1130 .is_some_and(|event| {
1131 config_section_event_revision(event)
1132 == revision
1133 })
1134 {
1135 set_live_health_revision(
1136 &apply_mcp_health,
1137 revision,
1138 Some((
1139 data_dir.join("mcp.json"),
1140 SectionSourceKind::File,
1141 )),
1142 );
1143 retry_legacy_root_publication |=
1144 publish_registry_event_with_root_ack(
1145 &data_dir,
1146 &account_sink,
1147 &mut pending_root_publications,
1148 SectionId::Mcp,
1149 &event,
1150 )
1151 .await;
1152 } else if matches!(
1153 event,
1154 ConfigSectionEvent::Recovered { .. }
1155 ) {
1156 set_live_health_revision(
1157 &apply_mcp_health,
1158 revision,
1159 Some((
1160 data_dir.join("mcp.json"),
1161 SectionSourceKind::File,
1162 )),
1163 );
1164 publish_registry_event(
1165 &account_sink,
1166 &event,
1167 )
1168 .await;
1169 } else {
1170 publish_section_success(
1171 &apply_mcp_health,
1172 &account_sink,
1173 "mcp",
1174 data_dir.join("mcp.json"),
1175 recovered,
1176 Some(revision),
1177 )
1178 .await;
1179 }
1180 }
1181 }
1182 Err(_) => {
1183 retry_legacy_root_publication |=
1184 publish_staged_facade_section_failure(
1185 &data_dir,
1186 facade,
1187 SectionId::Mcp,
1188 "MCP runtime initialization failed; retaining last-known-good runtime",
1189 StagedFacadeSectionFailureContext {
1190 health: &apply_mcp_health,
1191 account_sink: &account_sink,
1192 section: "mcp",
1193 pending_root_publications: &pending_root_publications,
1194 },
1195 )
1196 .await;
1197 }
1198 }
1199 }
1200 }
1201 } else {
1202 retry_legacy_root_publication = true;
1203 }
1204 }
1205 } else {
1206 let current_config = config.read().await.clone();
1207 let current_revision = apply_mcp_health
1208 .read()
1209 .unwrap_or_else(|poisoned| poisoned.into_inner())
1210 .revision;
1211 let force_initial_mcp =
1212 changes.initial_mcp_revision == Some(current_revision);
1213 let result = load_and_validate_mcp_candidate(
1214 &mcp_store,
1215 current_revision,
1216 current_config,
1217 force_initial_mcp,
1218 )
1219 .await;
1220 match result {
1221 Ok(candidate) if candidate.unchanged => {}
1222 Ok(candidate) => {
1223 if candidate.normalized_external_revision
1224 || candidate.source_kind == SectionSourceKind::Backup
1225 {
1226 self_write_marker.mark_self_write(mcp_store.path());
1231 }
1232 let next_mcp = candidate.config.mcp.clone();
1233 let publish_config = config.clone();
1234 match mcp_manager
1235 .reconcile_from_config_transactional_after(
1236 &candidate.config.mcp,
1237 || async move {
1238 publish_config.write().await.mcp = next_mcp;
1239 Ok(())
1240 },
1241 )
1242 .await
1243 {
1244 Ok(()) => {
1245 let recovered = section_is_unhealthy(&apply_mcp_health);
1246 if candidate.source_kind == SectionSourceKind::Backup {
1247 publish_mcp_backup_lkg(
1248 &apply_mcp_health,
1249 &account_sink,
1250 candidate.source_path,
1251 candidate.revision,
1252 )
1253 .await;
1254 } else {
1255 publish_section_success(
1256 &apply_mcp_health,
1257 &account_sink,
1258 "mcp",
1259 data_dir.join("mcp.json"),
1260 recovered,
1261 Some(candidate.revision),
1262 )
1263 .await;
1264 }
1265 }
1266 Err(_) => publish_section_failure(
1267 &apply_mcp_health,
1268 &account_sink,
1269 "mcp",
1270 SectionStatus::Degraded,
1271 "MCP runtime initialization failed; retaining last-known-good runtime"
1272 .to_string(),
1273 )
1274 .await,
1275 }
1276 }
1277 Err(error) => {
1278 publish_section_failure(
1279 &apply_mcp_health,
1280 &account_sink,
1281 "mcp",
1282 candidate_error_status(&error.kind),
1283 error.message,
1284 )
1285 .await
1286 }
1287 }
1288 }
1289 }
1290 if !catchup_paths.is_empty() {
1291 let _ = catchup_changes.send(ConfigPathChanges {
1292 paths: catchup_paths,
1293 initial_mcp_revision: None,
1294 startup_legacy_root: None,
1295 startup_recoveries: BTreeMap::new(),
1296 legacy_root_retry_attempt: 0,
1297 });
1298 }
1299 if requeue_legacy_root && pending_root_publications.is_empty() {
1300 let _ = catchup_changes.send(ConfigPathChanges {
1301 paths: vec![data_dir.join("config.json")],
1302 initial_mcp_revision: None,
1303 startup_legacy_root: None,
1304 startup_recoveries: BTreeMap::new(),
1305 legacy_root_retry_attempt: 0,
1306 });
1307 }
1308 if retry_legacy_root_publication {
1309 let retry_changes = catchup_changes.clone();
1310 let retry_path = data_dir.join("config.json");
1311 let retry_attempt = changes.legacy_root_retry_attempt.saturating_add(1);
1312 let delay = Duration::from_millis(50_u64 << retry_attempt.min(5));
1313 tokio::spawn(async move {
1314 tokio::time::sleep(delay).await;
1315 let _ = retry_changes.send(ConfigPathChanges {
1316 paths: vec![retry_path],
1317 initial_mcp_revision: None,
1318 startup_legacy_root: None,
1319 startup_recoveries: BTreeMap::new(),
1320 legacy_root_retry_attempt: retry_attempt,
1321 });
1322 });
1323 }
1324 }
1325 });
1326
1327 (
1328 Self {
1329 stop,
1330 watcher_task: Some(watcher_task),
1331 apply_task: Some(apply_task),
1332 },
1333 provider_health,
1334 mcp_health,
1335 )
1336 }
1337}
1338
1339fn durable_invalid_recoveries(
1340 account_sink: &bamboo_engine::events::AccountEventSink,
1341 facade: &bamboo_config::ConfigFacade,
1342 rejected: &BTreeSet<SectionId>,
1343) -> BTreeMap<SectionId, u64> {
1344 let Ok(durable_facade) = bamboo_config::ConfigFacade::open(facade.data_dir()) else {
1345 return BTreeMap::new();
1346 };
1347 let mut latest = BTreeMap::<SectionId, (bool, u64)>::new();
1348 if let Ok(events) = bamboo_engine::events::journal::read_since(account_sink.events_dir(), 0) {
1349 for change in events {
1350 let state = match change.event {
1351 AgentEvent::ConfigInvalid { section, revision } => Some((section, true, revision)),
1352 AgentEvent::ConfigChanged { section, revision }
1353 | AgentEvent::ConfigRecovered { section, revision } => {
1354 Some((section, false, revision))
1355 }
1356 _ => None,
1357 };
1358 if let Some((section, invalid, revision)) = state {
1359 if let Some(id) = SectionId::from_name(§ion) {
1360 latest.insert(id, (invalid, revision));
1361 }
1362 }
1363 }
1364 }
1365 latest
1366 .into_iter()
1367 .filter_map(|(id, (invalid, invalid_revision))| {
1368 if !invalid || rejected.contains(&id) {
1369 return None;
1370 }
1371 let envelope = durable_facade.registry().envelope_value(id).ok()?;
1372 (envelope.revision >= invalid_revision
1373 && envelope.status == SectionStatus::Healthy
1374 && envelope.source_kind == SectionSourceKind::File)
1375 .then_some((id, envelope.revision))
1376 })
1377 .collect()
1378}
1379
1380fn queue_legacy_root_recovery(
1381 facade: &bamboo_config::ConfigFacade,
1382 id: SectionId,
1383 minimum_revision: Option<u64>,
1384 watched_sections: &mut BTreeSet<SectionId>,
1385 synthetic_events: &mut BTreeMap<SectionId, ConfigSectionEvent>,
1386) {
1387 watched_sections.insert(id);
1388 let _ = facade.registry().reload(id);
1391 let Ok(envelope) = facade.registry().envelope_value(id) else {
1392 return;
1393 };
1394 if envelope.status != SectionStatus::Healthy
1395 || envelope.source_kind != SectionSourceKind::File
1396 || minimum_revision.is_some_and(|minimum| envelope.revision < minimum)
1397 {
1398 return;
1399 }
1400 synthetic_events
1401 .entry(id)
1402 .or_insert_with(|| ConfigSectionEvent::Recovered {
1403 section: id.descriptor().name.to_string(),
1404 revision: envelope.revision,
1405 });
1406}
1407
1408fn pending_root_publication_matches_fresh_durable(
1409 data_dir: &Path,
1410 process_facade: &bamboo_config::ConfigFacade,
1411 id: SectionId,
1412 revision: u64,
1413) -> bool {
1414 let Ok(process) = process_facade.registry().envelope_value(id) else {
1415 return false;
1416 };
1417 if process.revision != revision {
1418 return false;
1419 }
1420 let event = ConfigSectionEvent::Changed {
1421 section: id.descriptor().name.to_string(),
1422 revision,
1423 };
1424 bamboo_config::legacy_root_publication_matches_snapshot(data_dir, &event, &process.data)
1425 .unwrap_or(false)
1426}
1427
1428async fn wait_for_section_file_settle(data_dir: &Path, id: SectionId) {
1429 let path = data_dir.join(id.descriptor().file_name);
1430 for _ in 0..3 {
1431 if path.exists() {
1432 return;
1433 }
1434 tokio::time::sleep(Duration::from_millis(50)).await;
1435 }
1436}
1437
1438struct OrdinarySectionReloadState<'a> {
1444 synthetic_events: &'a mut BTreeMap<SectionId, ConfigSectionEvent>,
1445 pending_root_publications: &'a mut BTreeMap<SectionId, ConfigSectionEvent>,
1446 reported_root_runtime_failures: &'a mut BTreeSet<(SectionId, u64)>,
1447}
1448
1449async fn reload_and_apply_ordinary_sections(
1450 data_dir: &Path,
1451 config: &Arc<RwLock<Config>>,
1452 facade: &bamboo_config::ConfigFacade,
1453 account_sink: &bamboo_engine::events::AccountEventSink,
1454 sections: impl IntoIterator<Item = SectionId>,
1455 state: OrdinarySectionReloadState<'_>,
1456) -> bool {
1457 let OrdinarySectionReloadState {
1458 synthetic_events,
1459 pending_root_publications,
1460 reported_root_runtime_failures,
1461 } = state;
1462 let mut retry_legacy_root_publication = false;
1463 let mut publishable = Vec::new();
1464 for id in sections {
1465 wait_for_section_file_settle(data_dir, id).await;
1466 let Some(event) = synthetic_events
1467 .remove(&id)
1468 .or_else(|| facade.registry().reload_if_changed(id))
1469 else {
1470 continue;
1471 };
1472 let Some(event) = pending_root_publication_event(
1473 data_dir,
1474 facade,
1475 account_sink,
1476 pending_root_publications,
1477 id,
1478 event,
1479 ) else {
1480 retry_legacy_root_publication = true;
1481 continue;
1482 };
1483 if matches!(event, ConfigSectionEvent::Invalid { .. }) {
1484 publish_registry_event(account_sink, &event).await;
1485 } else {
1486 publishable.push((id, event));
1487 }
1488 }
1489 if publishable.is_empty() {
1490 return retry_legacy_root_publication;
1491 }
1492
1493 let materialized = materialize_facade_effective_config(facade, data_dir);
1494 let mut current = config.read().await.clone();
1495 let mut applied = Vec::new();
1496 for (id, event) in publishable {
1497 if materialized.failures.contains(&id) {
1498 let revision = facade
1499 .registry()
1500 .envelope_value(id)
1501 .map(|envelope| envelope.revision)
1502 .unwrap_or_default();
1503 let invalid = facade.registry().mark_runtime_degraded(
1504 id,
1505 "configuration runtime hydration failed; retaining last-known-good runtime",
1506 );
1507 if let Some(invalid) = invalid {
1508 if pending_root_publications
1509 .get(&id)
1510 .is_some_and(|event| config_section_event_revision(event) == revision)
1511 {
1512 let _ =
1513 confirm_legacy_root_runtime_failure(data_dir, account_sink, &invalid).await;
1514 } else if reported_root_runtime_failures.insert((id, revision)) {
1515 publish_registry_event(account_sink, &invalid).await;
1516 }
1517 }
1518 retry_legacy_root_publication |= pending_root_publications.contains_key(&id);
1519 continue;
1520 }
1521 apply_runtime_section(id, &materialized.config, &mut current);
1522 applied.push((id, event));
1523 }
1524 if applied.is_empty() {
1525 return retry_legacy_root_publication;
1526 }
1527
1528 let publishes_env = applied.iter().any(|(id, _)| *id == SectionId::Env);
1529 let enforcement_newly_off = !config.read().await.plugin_trust.enforcement_is_off()
1530 && current.plugin_trust.enforcement_is_off();
1531 *config.write().await = current.clone();
1532 if publishes_env {
1533 current.publish_env_vars();
1534 }
1535 if enforcement_newly_off {
1536 warn_plugin_trust_enforcement_off();
1537 }
1538 for (id, event) in applied {
1539 retry_legacy_root_publication |= publish_registry_event_with_root_ack(
1540 data_dir,
1541 account_sink,
1542 pending_root_publications,
1543 id,
1544 &event,
1545 )
1546 .await;
1547 reported_root_runtime_failures.retain(|(failed_id, _)| *failed_id != id);
1548 }
1549 retry_legacy_root_publication
1550}
1551
1552fn pending_root_publication_event(
1553 data_dir: &Path,
1554 facade: &bamboo_config::ConfigFacade,
1555 account_sink: &bamboo_engine::events::AccountEventSink,
1556 pending: &BTreeMap<SectionId, ConfigSectionEvent>,
1557 id: SectionId,
1558 observed: ConfigSectionEvent,
1559) -> Option<ConfigSectionEvent> {
1560 let Some(pending_event) = pending.get(&id) else {
1561 return Some(observed);
1562 };
1563 let revision = config_section_event_revision(pending_event);
1564 let Ok(envelope) = facade.registry().envelope_value(id) else {
1565 return None;
1566 };
1567 if envelope.revision != revision || envelope.status != SectionStatus::Healthy {
1568 return None;
1569 }
1570 let pending = ConfigSectionEvent::Changed {
1571 section: id.descriptor().name.to_string(),
1572 revision,
1573 };
1574 if account_sink.latest_config_transition_is_invalid(id.descriptor().name, revision) {
1575 let invalid = ConfigSectionEvent::Invalid {
1576 section: id.descriptor().name.to_string(),
1577 revision,
1578 };
1579 match bamboo_config::mark_legacy_root_publication_runtime_degraded(data_dir, &invalid) {
1580 Ok(true) => {}
1581 Ok(false) => return None,
1582 Err(error) => {
1583 tracing::warn!(
1584 %error,
1585 section = id.descriptor().name,
1586 revision,
1587 "failed to repair canonical root runtime-degraded proof from the durable journal"
1588 );
1589 return None;
1590 }
1591 }
1592 }
1593 bamboo_config::legacy_root_publication_success_event(data_dir, &pending, &envelope.data)
1594 .ok()
1595 .flatten()
1596}
1597
1598async fn publish_registry_event_with_root_ack(
1599 data_dir: &Path,
1600 account_sink: &bamboo_engine::events::AccountEventSink,
1601 pending: &mut BTreeMap<SectionId, ConfigSectionEvent>,
1602 id: SectionId,
1603 event: &ConfigSectionEvent,
1604) -> bool {
1605 if matches!(event, ConfigSectionEvent::Invalid { .. }) {
1606 publish_registry_event(account_sink, event).await;
1607 return false;
1608 }
1609 if pending.get(&id).is_none_or(|pending_event| {
1610 config_section_event_revision(pending_event) != config_section_event_revision(event)
1611 }) {
1612 publish_registry_event(account_sink, event).await;
1613 return false;
1614 }
1615 let durable = account_sink
1616 .record_confirmed(None, ®istry_agent_event(event))
1617 .await;
1618 if durable
1619 && bamboo_config::acknowledge_legacy_root_publication(data_dir, event).unwrap_or(false)
1620 {
1621 pending.remove(&id);
1622 false
1623 } else {
1624 true
1625 }
1626}
1627
1628pub(super) async fn publish_registry_event(
1629 account_sink: &bamboo_engine::events::AccountEventSink,
1630 event: &ConfigSectionEvent,
1631) {
1632 let event = registry_agent_event(event);
1633 if !account_sink.record_confirmed(None, &event).await {
1634 tracing::warn!("configuration event could not be confirmed in the account journal");
1635 }
1636}
1637
1638async fn confirm_legacy_root_runtime_failure(
1639 data_dir: &Path,
1640 account_sink: &bamboo_engine::events::AccountEventSink,
1641 event: &ConfigSectionEvent,
1642) -> bool {
1643 if !account_sink
1644 .record_confirmed(None, ®istry_agent_event(event))
1645 .await
1646 {
1647 return false;
1648 }
1649 match bamboo_config::mark_legacy_root_publication_runtime_degraded(data_dir, event) {
1650 Ok(marked) => marked,
1651 Err(error) => {
1652 tracing::warn!(
1653 %error,
1654 "failed to mark canonical root publication runtime-degraded"
1655 );
1656 false
1657 }
1658 }
1659}
1660
1661fn legacy_root_rejection_event(
1662 facade: &bamboo_config::ConfigFacade,
1663 id: SectionId,
1664 diagnostic: &str,
1665 force_publication: bool,
1666) -> Option<ConfigSectionEvent> {
1667 let already_reported = facade.registry().envelope_value(id).is_ok_and(|envelope| {
1668 envelope.status == SectionStatus::Degraded
1669 && envelope.last_error.as_deref() == Some(diagnostic)
1670 });
1671 if already_reported && !force_publication {
1672 return None;
1673 }
1674 facade
1675 .registry()
1676 .mark_runtime_degraded(id, diagnostic)
1677 .or_else(|| {
1678 force_publication
1681 .then(|| {
1682 facade.registry().envelope_value(id).ok().map(|envelope| {
1683 ConfigSectionEvent::Invalid {
1684 section: id.descriptor().name.to_string(),
1685 revision: envelope.revision,
1686 }
1687 })
1688 })
1689 .flatten()
1690 })
1691}
1692
1693fn config_section_event_revision(event: &ConfigSectionEvent) -> u64 {
1694 match event {
1695 ConfigSectionEvent::Changed { revision, .. }
1696 | ConfigSectionEvent::Invalid { revision, .. }
1697 | ConfigSectionEvent::Recovered { revision, .. } => *revision,
1698 }
1699}
1700
1701fn registry_agent_event(event: &ConfigSectionEvent) -> AgentEvent {
1702 match event {
1703 ConfigSectionEvent::Changed { section, revision } => AgentEvent::ConfigChanged {
1704 section: section.clone(),
1705 revision: *revision,
1706 },
1707 ConfigSectionEvent::Invalid { section, revision } => AgentEvent::ConfigInvalid {
1708 section: section.clone(),
1709 revision: *revision,
1710 },
1711 ConfigSectionEvent::Recovered { section, revision } => AgentEvent::ConfigRecovered {
1712 section: section.clone(),
1713 revision: *revision,
1714 },
1715 }
1716}
1717
1718async fn publish_exact_facade_events(
1719 account_sink: &bamboo_engine::events::AccountEventSink,
1720 events: &[ConfigSectionEvent],
1721) -> Result<(), AppError> {
1722 for event in events {
1723 let durable = account_sink
1724 .record_confirmed(None, ®istry_agent_event(event))
1725 .await;
1726 if !durable {
1727 return Err(AppError::InternalError(anyhow::anyhow!(
1728 "committed configuration event could not be confirmed in the account journal"
1729 )));
1730 }
1731 if matches!(event, ConfigSectionEvent::Invalid { .. }) {
1732 return Err(AppError::InternalError(anyhow::anyhow!(
1733 "committed configuration section became invalid before publication"
1734 )));
1735 }
1736 }
1737 Ok(())
1738}
1739
1740struct InstalledCredentialSectionCommit {
1741 events: Vec<ConfigSectionEvent>,
1742 metadata: bamboo_config::CredentialSectionRuntimeMetadata,
1743 section: Option<bamboo_config::SectionEnvelope<Value>>,
1744}
1745
1746pub(crate) struct ExactCredentialSectionSnapshot {
1747 pub config: Config,
1748 pub section: bamboo_config::SectionEnvelope<Value>,
1749 pub metadata: bamboo_config::CredentialSectionRuntimeMetadata,
1750}
1751
1752fn map_exact_credential_store_error(error: ConfigStoreError) -> AppError {
1753 match error {
1754 ConfigStoreError::Conflict { expected, actual } => {
1755 AppError::ConfigConflict { expected, actual }
1756 }
1757 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
1758 ConfigStoreError::CommitIndeterminate(message) => AppError::InternalError(anyhow::anyhow!(
1759 "configuration commit outcome is indeterminate: {message}"
1760 )),
1761 ConfigStoreError::Io(error) => AppError::StorageError(error),
1762 ConfigStoreError::Json(_) => {
1763 AppError::BadRequest("configuration document is invalid".to_string())
1764 }
1765 ConfigStoreError::Watch(error) => {
1766 AppError::InternalError(anyhow::anyhow!("configuration watch failed: {error}"))
1767 }
1768 }
1769}
1770
1771async fn install_exact_credential_section_mutation_base(
1772 data_dir: PathBuf,
1773 section: SectionId,
1774 expected_revision: u64,
1775 target: &mut Config,
1776) -> Result<bamboo_config::CredentialSectionRuntimeMetadata, AppError> {
1777 let exact = tokio::task::spawn_blocking(move || {
1778 bamboo_config::read_exact_credential_section_snapshot(
1779 data_dir,
1780 section,
1781 Some(expected_revision),
1782 )
1783 })
1784 .await
1785 .map_err(|error| {
1786 AppError::InternalError(anyhow::anyhow!(
1787 "{} exact mutation snapshot task failed: {error}",
1788 section.descriptor().name
1789 ))
1790 })?
1791 .map_err(map_exact_credential_store_error)?;
1792 Ok(exact.install_into(target))
1793}
1794
1795fn read_credential_runtime_metadata(
1796 data_dir: &std::path::Path,
1797) -> Result<bamboo_config::CredentialSectionRuntimeMetadata, ConfigStoreError> {
1798 let (credential_statuses, credential_health) =
1799 bamboo_config::CredentialStore::open(data_dir).statuses_with_health()?;
1800 Ok(bamboo_config::CredentialSectionRuntimeMetadata {
1801 credential_statuses,
1802 credential_health,
1803 })
1804}
1805
1806fn install_credential_section_commit(
1807 commit: bamboo_config::CredentialSectionTransactionCommit,
1808 target: &mut Config,
1809) -> Result<InstalledCredentialSectionCommit, ConfigStoreError> {
1810 let bamboo_config::CredentialSectionTransactionCommit {
1811 revision: _,
1812 section_adoption,
1813 credential_adoption,
1814 section,
1815 runtime,
1816 } = commit;
1817 let section = section?;
1818 let metadata = runtime?.install_into(target);
1819 let mut events = Vec::new();
1820 if let Some(adoption) = credential_adoption {
1821 if let Some(event) = adoption? {
1822 events.push(event);
1823 }
1824 }
1825 if let Some(adoption) = section_adoption {
1826 events.push(adoption?);
1827 }
1828 Ok(InstalledCredentialSectionCommit {
1829 events,
1830 metadata,
1831 section: Some(section),
1832 })
1833}
1834
1835fn install_facade_config_commit(
1836 commit: bamboo_config::FacadeConfigCommit,
1837 target: &mut Config,
1838) -> Result<Vec<ConfigSectionEvent>, ConfigStoreError> {
1839 let bamboo_config::FacadeConfigCommit {
1840 section_adoption,
1841 runtime,
1842 } = commit;
1843 if let Some(runtime) = runtime? {
1844 runtime.install_into(target);
1845 }
1846 section_adoption
1847 .map(|adoption| adoption.map(|event| vec![event]))
1848 .unwrap_or_else(|| Ok(Vec::new()))
1849}
1850
1851fn apply_runtime_section(id: SectionId, source: &Config, target: &mut Config) {
1852 match id {
1853 SectionId::Core => {
1854 target.http_proxy = source.http_proxy.clone();
1855 target.https_proxy = source.https_proxy.clone();
1856 target.proxy_auth = source.proxy_auth.clone();
1857 target.proxy_auth_encrypted = None;
1858 target.proxy_auth_credential_ref = source.proxy_auth_credential_ref.clone();
1859 target.headless_auth = source.headless_auth;
1860 target.server = source.server.clone();
1861 target.default_work_area = source.default_work_area.clone();
1862 target.run_budget = source.run_budget;
1863 target.stream_timeout = source.stream_timeout;
1864 target.extra = source.extra.clone();
1865 }
1866 SectionId::Providers => {
1867 target.provider = source.provider.clone();
1868 target.defaults = source.defaults.clone();
1869 target.provider_instances = source.provider_instances.clone();
1870 target.default_provider_instance = source.default_provider_instance.clone();
1871 target.features = source.features.clone();
1872 *target.providers_mut() = source.providers().clone();
1873 }
1874 SectionId::Mcp => target.mcp = source.mcp.clone(),
1875 SectionId::ToolsSkills => {
1876 target.tools = source.tools.clone();
1877 target.skills = source.skills.clone();
1878 target.plugin_trust = source.plugin_trust.clone();
1879 }
1880 SectionId::Memory => *target.memory_mut() = source.memory().clone(),
1881 SectionId::Subagents => {
1882 let runtime_broker = target.subagents().broker.clone();
1883 *target.subagents_mut() = source.subagents().clone();
1884 if target.subagents().broker.is_none() {
1885 target.subagents_mut().broker = runtime_broker;
1886 }
1887 }
1888 SectionId::Notifications => target.notifications = source.notifications.clone(),
1889 SectionId::Connect => target.connect = source.connect.clone(),
1890 SectionId::ClusterFabric => target.cluster_fabric = source.cluster_fabric.clone(),
1891 SectionId::Env => target.env_vars = source.env_vars.clone(),
1892 SectionId::AccessControl => target.access_control = source.access_control.clone(),
1893 SectionId::Hooks => target.hooks = source.hooks.clone(),
1894 SectionId::ModelPolicy => {
1895 target.keyword_masking = source.keyword_masking.clone();
1896 target.anthropic_model_mapping = source.anthropic_model_mapping.clone();
1897 target.gemini_model_mapping = source.gemini_model_mapping.clone();
1898 }
1899 SectionId::ModelLimits | SectionId::Credentials => {}
1900 }
1901}
1902
1903fn restore_authoritative_cluster_fabric(
1908 facade: Option<&std::sync::Arc<bamboo_config::ConfigFacade>>,
1909 candidate: &mut Config,
1910) {
1911 if let Some(facade) = facade {
1912 candidate.cluster_fabric = facade.registry().cluster_fabric.snapshot().data.0.clone();
1913 }
1914}
1915
1916fn preserve_runtime_broker(new_config: &mut Config, previous: &Config) {
1920 if new_config.subagents().broker.is_none() {
1921 new_config.subagents_mut().broker = previous.subagents().broker.clone();
1922 }
1923}
1924
1925fn section_is_unhealthy(health: &std::sync::RwLock<ConfigLiveHealth>) -> bool {
1926 health
1927 .read()
1928 .unwrap_or_else(|poisoned| poisoned.into_inner())
1929 .status
1930 != SectionStatus::Healthy
1931}
1932
1933async fn publish_section_success(
1934 health: &std::sync::RwLock<ConfigLiveHealth>,
1935 account_sink: &bamboo_engine::events::AccountEventSink,
1936 section: &str,
1937 source_path: PathBuf,
1938 recovered: bool,
1939 revision: Option<u64>,
1940) {
1941 let revision = match revision {
1942 Some(revision) => set_live_health_revision(
1943 health,
1944 revision,
1945 Some((source_path, SectionSourceKind::File)),
1946 ),
1947 None => update_live_health(
1948 health,
1949 SectionStatus::Healthy,
1950 None,
1951 true,
1952 Some((source_path, SectionSourceKind::File)),
1953 ),
1954 };
1955 let event = if recovered {
1956 AgentEvent::ConfigRecovered {
1957 section: section.to_string(),
1958 revision,
1959 }
1960 } else {
1961 AgentEvent::ConfigChanged {
1962 section: section.to_string(),
1963 revision,
1964 }
1965 };
1966 if !account_sink.record_confirmed(None, &event).await {
1967 tracing::warn!(
1968 section,
1969 revision,
1970 "configuration success event was not durable"
1971 );
1972 }
1973}
1974
1975async fn publish_section_failure(
1976 health: &std::sync::RwLock<ConfigLiveHealth>,
1977 account_sink: &bamboo_engine::events::AccountEventSink,
1978 section: &str,
1979 status: SectionStatus,
1980 message: String,
1981) {
1982 let duplicate = {
1983 let health = health
1984 .read()
1985 .unwrap_or_else(|poisoned| poisoned.into_inner());
1986 health.status == status && health.last_error.as_deref() == Some(message.as_str())
1987 };
1988 let revision = update_live_health(health, status, Some(message), false, None);
1989 if duplicate {
1990 return;
1991 }
1992 let event = AgentEvent::ConfigInvalid {
1993 section: section.to_string(),
1994 revision,
1995 };
1996 if !account_sink.record_confirmed(None, &event).await {
1997 tracing::warn!(
1998 section,
1999 revision,
2000 "configuration failure event was not durable"
2001 );
2002 }
2003}
2004
2005struct StagedFacadeSectionFailureContext<'a> {
2006 health: &'a std::sync::RwLock<ConfigLiveHealth>,
2007 account_sink: &'a bamboo_engine::events::AccountEventSink,
2008 section: &'a str,
2009 pending_root_publications: &'a BTreeMap<SectionId, ConfigSectionEvent>,
2010}
2011
2012async fn publish_staged_facade_section_failure(
2013 data_dir: &Path,
2014 facade: &bamboo_config::ConfigFacade,
2015 id: SectionId,
2016 message: &str,
2017 context: StagedFacadeSectionFailureContext<'_>,
2018) -> bool {
2019 let StagedFacadeSectionFailureContext {
2020 health,
2021 account_sink,
2022 section,
2023 pending_root_publications,
2024 } = context;
2025 let event = facade
2026 .registry()
2027 .mark_runtime_degraded(id, message)
2028 .expect("every facade section exposes runtime health");
2029 let exact_pending = matches!(
2030 &event,
2031 ConfigSectionEvent::Invalid { revision, .. }
2032 if pending_root_publications
2033 .get(&id)
2034 .is_some_and(|event| config_section_event_revision(event) == *revision)
2035 );
2036 if exact_pending {
2037 update_live_health(
2041 health,
2042 SectionStatus::Degraded,
2043 Some(message.to_string()),
2044 false,
2045 None,
2046 );
2047 if !confirm_legacy_root_runtime_failure(data_dir, account_sink, &event).await {
2048 tracing::warn!(
2049 section,
2050 "root runtime failure was not confirmed against its canonical publication"
2051 );
2052 }
2053 } else {
2054 publish_section_failure(
2055 health,
2056 account_sink,
2057 section,
2058 SectionStatus::Degraded,
2059 message.to_string(),
2060 )
2061 .await;
2062 }
2063 exact_pending
2064}
2065
2066async fn publish_mcp_backup_lkg(
2067 health: &std::sync::RwLock<ConfigLiveHealth>,
2068 account_sink: &bamboo_engine::events::AccountEventSink,
2069 source_path: PathBuf,
2070 revision: u64,
2071) {
2072 {
2073 let mut health = health
2074 .write()
2075 .unwrap_or_else(|poisoned| poisoned.into_inner());
2076 health.revision = revision;
2077 health.loaded_at = Utc::now();
2078 health.source_path = source_path;
2079 health.source_kind = SectionSourceKind::Backup;
2080 health.status = SectionStatus::Degraded;
2081 health.last_error =
2082 Some("primary MCP section invalid; running last-known-good backup runtime".to_string());
2083 }
2084 let event = AgentEvent::ConfigInvalid {
2085 section: "mcp".to_string(),
2086 revision,
2087 };
2088 if !account_sink.record_confirmed(None, &event).await {
2089 tracing::warn!(revision, "MCP backup health event was not durable");
2090 }
2091}
2092
2093fn initial_provider_health(store: &AtomicJsonStore<ProviderConfigs>) -> ConfigLiveHealth {
2094 if ensure_provider_mcp_migration_ready(store.path().parent().unwrap_or_else(|| Path::new(".")))
2095 .is_err()
2096 {
2097 return ConfigLiveHealth {
2098 revision: 0,
2099 loaded_at: Utc::now(),
2100 source_path: store.path().to_path_buf(),
2101 source_kind: SectionSourceKind::File,
2102 status: SectionStatus::Degraded,
2103 last_error: Some("provider/MCP credential migration is pending".to_string()),
2104 };
2105 }
2106 match store.load_validated_allowing_unversioned(|_| Ok(())) {
2107 Ok(Some(stored)) => ConfigLiveHealth {
2108 revision: stored.revision,
2109 loaded_at: Utc::now(),
2110 source_path: stored.source_path,
2111 source_kind: if stored.recovered_from_backup {
2112 SectionSourceKind::Backup
2113 } else {
2114 SectionSourceKind::File
2115 },
2116 status: if stored.recovered_from_backup {
2117 SectionStatus::Degraded
2118 } else {
2119 SectionStatus::Healthy
2120 },
2121 last_error: stored.recovered_from_backup.then(|| {
2122 "primary provider section invalid; using last-known-good backup".to_string()
2123 }),
2124 },
2125 Ok(None) => ConfigLiveHealth {
2126 revision: 0,
2127 loaded_at: Utc::now(),
2128 source_path: store.path().to_path_buf(),
2129 source_kind: SectionSourceKind::Default,
2130 status: SectionStatus::Missing,
2131 last_error: None,
2132 },
2133 Err(_) => ConfigLiveHealth {
2134 revision: 0,
2135 loaded_at: Utc::now(),
2136 source_path: store.path().to_path_buf(),
2137 source_kind: SectionSourceKind::File,
2138 status: SectionStatus::Invalid,
2139 last_error: Some("provider section could not be parsed or read".to_string()),
2140 },
2141 }
2142}
2143
2144fn initial_mcp_health(store: &AtomicJsonStore<McpConfig>) -> ConfigLiveHealth {
2145 if ensure_provider_mcp_migration_ready(store.path().parent().unwrap_or_else(|| Path::new(".")))
2146 .is_err()
2147 {
2148 return ConfigLiveHealth {
2149 revision: 0,
2150 loaded_at: Utc::now(),
2151 source_path: store.path().to_path_buf(),
2152 source_kind: SectionSourceKind::File,
2153 status: SectionStatus::Degraded,
2154 last_error: Some("provider/MCP credential migration is pending".to_string()),
2155 };
2156 }
2157 match store.load_validated(validate_mcp_config) {
2158 Ok(Some(stored)) => ConfigLiveHealth {
2159 revision: 0,
2162 loaded_at: Utc::now(),
2163 source_path: store.path().to_path_buf(),
2164 source_kind: if stored.recovered_from_backup {
2165 SectionSourceKind::Backup
2166 } else {
2167 SectionSourceKind::File
2168 },
2169 status: SectionStatus::Degraded,
2170 last_error: Some(if stored.recovered_from_backup {
2171 "primary MCP section invalid; runtime initialization pending from backup"
2172 .to_string()
2173 } else {
2174 "MCP runtime initialization pending".to_string()
2175 }),
2176 },
2177 Ok(None) => ConfigLiveHealth {
2178 revision: 0,
2179 loaded_at: Utc::now(),
2180 source_path: store.path().to_path_buf(),
2181 source_kind: SectionSourceKind::Default,
2182 status: SectionStatus::Missing,
2183 last_error: None,
2184 },
2185 Err(_) => ConfigLiveHealth {
2186 revision: 0,
2187 loaded_at: Utc::now(),
2188 source_path: store.path().to_path_buf(),
2189 source_kind: SectionSourceKind::File,
2190 status: SectionStatus::Invalid,
2191 last_error: Some("MCP section could not be parsed or validated".to_string()),
2192 },
2193 }
2194}
2195
2196impl Drop for ConfigWatcherRuntime {
2197 fn drop(&mut self) {
2198 self.stop.store(true, Ordering::Relaxed);
2199 if let Some(task) = self.apply_task.take() {
2200 task.abort();
2201 }
2202 if let Some(task) = self.watcher_task.take() {
2203 let _ = task.join();
2204 }
2205 }
2206}
2207
2208async fn load_and_prepare_provider_candidate(
2209 store: &AtomicJsonStore<ProviderConfigs>,
2210 current_revision: u64,
2211 candidate_config: Config,
2212) -> Result<ProviderCandidate, ProviderCandidateError> {
2213 ensure_provider_mcp_migration_ready(store.path().parent().unwrap_or_else(|| Path::new(".")))
2214 .map_err(|_| {
2215 ProviderCandidateError::invalid(
2216 "provider/MCP credential migration is pending; retaining last-known-good runtime",
2217 )
2218 })?;
2219 for _ in 0..3 {
2222 if store.path().exists() {
2223 break;
2224 }
2225 tokio::time::sleep(Duration::from_millis(50)).await;
2226 }
2227 if !store.path().exists() {
2228 return Err(ProviderCandidateError::missing());
2229 }
2230 let stored = store
2231 .load_validated_for_reload_allowing_unversioned(
2232 current_revision,
2233 candidate_config.providers(),
2234 validate_provider_config,
2235 )
2236 .map_err(|_| {
2237 if store.path().exists() {
2238 ProviderCandidateError::invalid("provider section is invalid")
2239 } else {
2240 ProviderCandidateError::missing()
2241 }
2242 })?
2243 .ok_or_else(ProviderCandidateError::missing)?;
2244 if stored.recovered_from_backup {
2245 return Err(ProviderCandidateError::invalid(
2246 "primary provider section is invalid; retaining last-known-good runtime",
2247 ));
2248 }
2249 let unchanged = stored.revision == current_revision
2250 && serde_json::to_value(&stored.data).ok()
2251 == serde_json::to_value(candidate_config.providers()).ok();
2252 let mut candidate_config = candidate_config;
2253 *candidate_config.providers_mut() = stored.data;
2254 let (candidate_config, registry, provider) = prepare_provider_candidate(
2255 candidate_config,
2256 store
2257 .path()
2258 .parent()
2259 .unwrap_or_else(|| std::path::Path::new(".")),
2260 )
2261 .await?;
2262 Ok(ProviderCandidate {
2263 config: candidate_config,
2264 registry,
2265 provider,
2266 revision: stored.revision,
2267 normalized_external_revision: stored.normalized_external_revision,
2268 unchanged,
2269 })
2270}
2271
2272async fn prepare_provider_candidate(
2273 mut candidate_config: Config,
2274 data_dir: &std::path::Path,
2275) -> Result<(Config, bamboo_llm::ProviderRegistry, Arc<dyn LLMProvider>), ProviderCandidateError> {
2276 candidate_config.hydrate_provider_api_keys_from_encrypted();
2277 candidate_config
2278 .hydrate_provider_credentials_from_store(data_dir)
2279 .map_err(|_| ProviderCandidateError::invalid("provider credential is unavailable"))?;
2280 let candidate_registry =
2281 bamboo_llm::ProviderRegistry::from_config(&candidate_config, data_dir.to_path_buf())
2282 .await
2283 .map_err(|_| ProviderCandidateError::runtime())?;
2284 let candidate_provider = candidate_registry
2285 .get_default()
2286 .ok_or_else(ProviderCandidateError::runtime)?;
2287 Ok((candidate_config, candidate_registry, candidate_provider))
2288}
2289
2290struct ProviderCandidate {
2291 config: Config,
2292 registry: bamboo_llm::ProviderRegistry,
2293 provider: Arc<dyn LLMProvider>,
2294 revision: u64,
2295 normalized_external_revision: bool,
2296 unchanged: bool,
2297}
2298
2299async fn load_and_validate_mcp_candidate(
2300 store: &AtomicJsonStore<McpConfig>,
2301 current_revision: u64,
2302 mut candidate_config: Config,
2303 allow_startup_backup: bool,
2304) -> Result<McpCandidate, ProviderCandidateError> {
2305 ensure_provider_mcp_migration_ready(store.path().parent().unwrap_or_else(|| Path::new(".")))
2306 .map_err(|_| {
2307 ProviderCandidateError::invalid(
2308 "provider/MCP credential migration is pending; retaining last-known-good runtime",
2309 )
2310 })?;
2311 for _ in 0..3 {
2312 if store.path().exists() {
2313 break;
2314 }
2315 tokio::time::sleep(Duration::from_millis(50)).await;
2316 }
2317 if !store.path().exists() {
2318 return Err(ProviderCandidateError::missing_section(
2319 "MCP section is missing",
2320 ));
2321 }
2322 let current_document = mcp_durable_comparison_document(&candidate_config.mcp);
2323 let stored = store
2324 .load_validated_for_reload(current_revision, ¤t_document, validate_mcp_config)
2325 .map_err(|_| ProviderCandidateError::invalid("MCP section is invalid"))?
2326 .ok_or_else(|| ProviderCandidateError::missing_section("MCP section is missing"))?;
2327 if stored.recovered_from_backup && !allow_startup_backup {
2328 return Err(ProviderCandidateError::invalid(
2329 "primary MCP section is invalid; retaining last-known-good runtime",
2330 ));
2331 }
2332 let unchanged = !allow_startup_backup
2333 && stored.revision == current_revision
2334 && serde_json::to_value(&stored.data).ok() == serde_json::to_value(¤t_document).ok();
2335 candidate_config.mcp = stored.data;
2336 candidate_config.hydrate_mcp_secrets_from_encrypted();
2337 candidate_config
2338 .hydrate_mcp_credentials_from_store(
2339 store
2340 .path()
2341 .parent()
2342 .unwrap_or_else(|| std::path::Path::new(".")),
2343 )
2344 .map_err(|_| ProviderCandidateError::invalid("MCP credential is unavailable"))?;
2345 Ok(McpCandidate {
2346 config: candidate_config,
2347 revision: stored.revision,
2348 source_kind: if stored.recovered_from_backup {
2349 SectionSourceKind::Backup
2350 } else {
2351 SectionSourceKind::File
2352 },
2353 source_path: stored.source_path,
2354 normalized_external_revision: stored.normalized_external_revision,
2355 unchanged,
2356 })
2357}
2358
2359struct McpCandidate {
2360 config: Config,
2361 revision: u64,
2362 source_kind: SectionSourceKind,
2363 source_path: PathBuf,
2364 normalized_external_revision: bool,
2365 unchanged: bool,
2366}
2367
2368fn mcp_durable_comparison_document(config: &McpConfig) -> McpConfig {
2372 let mut document = config.clone();
2373 for server in &mut document.servers {
2374 match &mut server.transport {
2375 TransportConfig::Stdio(config) => {
2376 config.env.retain(|name, _| {
2377 !config.env_encrypted.contains_key(name)
2378 && !config.env_credential_refs.contains_key(name)
2379 });
2380 }
2381 TransportConfig::Sse(config) => clear_paired_header_plaintext(&mut config.headers),
2382 TransportConfig::StreamableHttp(config) => {
2383 clear_paired_header_plaintext(&mut config.headers)
2384 }
2385 }
2386 }
2387 document
2388}
2389
2390fn clear_paired_header_plaintext(headers: &mut [bamboo_mcp::HeaderConfig]) {
2391 for header in headers {
2392 if header.value_encrypted.is_some() || header.credential_ref.is_some() {
2393 header.value.clear();
2394 }
2395 }
2396}
2397
2398fn validate_mcp_config(config: &McpConfig) -> Result<(), String> {
2399 let mut ids = std::collections::HashSet::new();
2400 for server in &config.servers {
2401 if server.id.trim().is_empty() {
2402 return Err("MCP server id cannot be empty".to_string());
2403 }
2404 if !ids.insert(server.id.as_str()) {
2405 return Err(format!("duplicate MCP server id '{}'", server.id));
2406 }
2407 if server.request_timeout_ms == 0 || server.healthcheck_interval_ms == 0 {
2408 return Err(format!(
2409 "MCP server '{}' timeouts must be non-zero",
2410 server.id
2411 ));
2412 }
2413 match &server.transport {
2414 TransportConfig::Stdio(stdio) if stdio.command.trim().is_empty() => {
2415 return Err(format!(
2416 "MCP stdio server '{}' command cannot be empty",
2417 server.id
2418 ));
2419 }
2420 TransportConfig::Sse(sse) if sse.url.trim().is_empty() => {
2421 return Err(format!(
2422 "MCP SSE server '{}' URL cannot be empty",
2423 server.id
2424 ));
2425 }
2426 TransportConfig::StreamableHttp(http) if http.url.trim().is_empty() => {
2427 return Err(format!(
2428 "MCP HTTP server '{}' URL cannot be empty",
2429 server.id
2430 ));
2431 }
2432 _ => {}
2433 }
2434 match &server.transport {
2435 TransportConfig::Stdio(stdio) => {
2436 if !stdio.env_encrypted.is_empty()
2437 || stdio.env.iter().any(|(name, value)| {
2438 !value.is_empty() && !stdio.env_credential_refs.contains_key(name)
2439 })
2440 {
2441 return Err(format!(
2442 "MCP server '{}' contains a secret outside the credential store",
2443 server.id
2444 ));
2445 }
2446 for raw in stdio.env_credential_refs.values() {
2447 bamboo_config::CredentialRef::parse(raw.clone())
2448 .map_err(|_| "MCP credential reference is invalid".to_string())?;
2449 }
2450 }
2451 TransportConfig::Sse(config) => validate_header_refs(&server.id, &config.headers)?,
2452 TransportConfig::StreamableHttp(config) => {
2453 validate_header_refs(&server.id, &config.headers)?
2454 }
2455 }
2456 }
2457 Ok(())
2458}
2459
2460fn validate_provider_config(providers: &ProviderConfigs) -> Result<(), String> {
2461 macro_rules! validate {
2462 ($field:ident) => {
2463 if let Some(provider) = &providers.$field {
2464 if provider.api_key_encrypted.is_some()
2465 || (!provider.api_key.trim().is_empty()
2466 && !provider.api_key_from_env
2467 && provider.credential_ref.is_none())
2468 {
2469 return Err("provider secret is outside the credential store".to_string());
2470 }
2471 }
2472 };
2473 }
2474 validate!(openai);
2475 validate!(anthropic);
2476 validate!(gemini);
2477 if let Some(provider) = &providers.bodhi {
2478 if provider.api_key_encrypted.is_some()
2479 || (!provider.api_key.trim().is_empty() && provider.credential_ref.is_none())
2480 {
2481 return Err("provider secret is outside the credential store".to_string());
2482 }
2483 }
2484 Ok(())
2485}
2486
2487fn validate_header_refs(
2488 server_id: &str,
2489 headers: &[bamboo_mcp::HeaderConfig],
2490) -> Result<(), String> {
2491 for header in headers {
2492 if header.value_encrypted.is_some()
2493 || (!header.value.is_empty() && header.credential_ref.is_none())
2494 {
2495 return Err(format!(
2496 "MCP server '{server_id}' contains a secret outside the credential store"
2497 ));
2498 }
2499 if let Some(raw) = &header.credential_ref {
2500 bamboo_config::CredentialRef::parse(raw.clone())
2501 .map_err(|_| "MCP credential reference is invalid".to_string())?;
2502 }
2503 }
2504 Ok(())
2505}
2506
2507enum ProviderCandidateErrorKind {
2508 Missing,
2509 InvalidDocument,
2510 Runtime,
2511}
2512
2513struct ProviderCandidateError {
2514 kind: ProviderCandidateErrorKind,
2515 message: String,
2516}
2517
2518impl ProviderCandidateError {
2519 fn missing() -> Self {
2520 Self {
2521 kind: ProviderCandidateErrorKind::Missing,
2522 message: "provider section is missing".to_string(),
2523 }
2524 }
2525
2526 fn invalid(message: &str) -> Self {
2527 Self {
2528 kind: ProviderCandidateErrorKind::InvalidDocument,
2529 message: message.to_string(),
2530 }
2531 }
2532
2533 fn missing_section(message: &str) -> Self {
2534 Self {
2535 kind: ProviderCandidateErrorKind::Missing,
2536 message: message.to_string(),
2537 }
2538 }
2539
2540 fn runtime() -> Self {
2541 Self {
2542 kind: ProviderCandidateErrorKind::Runtime,
2543 message: "provider runtime initialization failed".to_string(),
2544 }
2545 }
2546}
2547
2548fn candidate_error_status(kind: &ProviderCandidateErrorKind) -> SectionStatus {
2549 match kind {
2550 ProviderCandidateErrorKind::Missing => SectionStatus::Missing,
2551 ProviderCandidateErrorKind::InvalidDocument => SectionStatus::Invalid,
2552 ProviderCandidateErrorKind::Runtime => SectionStatus::Degraded,
2553 }
2554}
2555
2556fn update_live_health(
2557 health: &std::sync::RwLock<ConfigLiveHealth>,
2558 status: SectionStatus,
2559 last_error: Option<String>,
2560 advance_revision: bool,
2561 source: Option<(PathBuf, SectionSourceKind)>,
2562) -> u64 {
2563 let mut health = health
2564 .write()
2565 .unwrap_or_else(|poisoned| poisoned.into_inner());
2566 if advance_revision {
2567 health.revision = health.revision.saturating_add(1);
2568 }
2569 health.loaded_at = Utc::now();
2570 health.status = status;
2571 health.last_error = last_error;
2572 if let Some((source_path, source_kind)) = source {
2573 health.source_path = source_path;
2574 health.source_kind = source_kind;
2575 }
2576 health.revision
2577}
2578
2579fn set_live_health_revision(
2580 health: &std::sync::RwLock<ConfigLiveHealth>,
2581 revision: u64,
2582 source: Option<(PathBuf, SectionSourceKind)>,
2583) -> u64 {
2584 let mut health = health
2585 .write()
2586 .unwrap_or_else(|poisoned| poisoned.into_inner());
2587 health.revision = revision;
2588 health.loaded_at = Utc::now();
2589 health.status = SectionStatus::Healthy;
2590 health.last_error = None;
2591 if let Some((source_path, source_kind)) = source {
2592 health.source_path = source_path;
2593 health.source_kind = source_kind;
2594 }
2595 revision
2596}
2597
2598fn set_live_health_from_snapshot<T>(
2599 health: &std::sync::RwLock<ConfigLiveHealth>,
2600 snapshot: &bamboo_config::SectionSnapshot<T>,
2601) {
2602 let mut health = health
2603 .write()
2604 .unwrap_or_else(|poisoned| poisoned.into_inner());
2605 health.revision = snapshot.revision;
2606 health.loaded_at = snapshot.loaded_at;
2607 health.source_path = snapshot.source_path.clone();
2608 health.source_kind = snapshot.source_kind;
2609 health.status = snapshot.status;
2610 health.last_error = snapshot.last_error.clone();
2611}
2612
2613#[derive(Debug)]
2614pub(crate) enum ConfigSectionMutationError {
2615 Store(ConfigStoreError),
2616 Invalid(String),
2617 Runtime(String),
2618}
2619
2620pub(crate) enum CredentialBackedResetCommit {
2621 Section(bamboo_config::SectionEnvelope<Value>),
2622 Cluster(Box<bamboo_server_tools::FabricCommitSnapshot>),
2623}
2624
2625impl AppState {
2626 #[cfg(test)]
2627 pub(crate) fn stop_config_watcher_for_test(&mut self) {
2628 self.config_watcher.stop.store(true, Ordering::Relaxed);
2629 if let Some(task) = self.config_watcher.apply_task.take() {
2630 task.abort();
2631 }
2632 if let Some(task) = self.config_watcher.watcher_task.take() {
2633 let _ = task.join();
2634 }
2635 }
2636
2637 #[cfg(test)]
2638 pub(crate) async fn reload_ordinary_section_for_test(&self, id: SectionId) {
2639 let _io = self.config_io_lock.lock().await;
2640 let facade = self
2641 .config_facade
2642 .as_ref()
2643 .expect("test ordinary reload requires the modular facade");
2644 let mut synthetic_events = BTreeMap::new();
2645 let mut pending_root_publications = BTreeMap::new();
2646 let mut reported_root_runtime_failures = BTreeSet::new();
2647 reload_and_apply_ordinary_sections(
2648 &self.app_data_dir,
2649 &self.config,
2650 facade,
2651 &self.account_sink,
2652 std::iter::once(id),
2653 OrdinarySectionReloadState {
2654 synthetic_events: &mut synthetic_events,
2655 pending_root_publications: &mut pending_root_publications,
2656 reported_root_runtime_failures: &mut reported_root_runtime_failures,
2657 },
2658 )
2659 .await;
2660 }
2661
2662 pub(crate) async fn read_exact_credential_section(
2667 &self,
2668 section: SectionId,
2669 ) -> Result<ExactCredentialSectionSnapshot, AppError> {
2670 let _io = self.config_io_lock.lock().await;
2671 let data_dir = self.app_data_dir.clone();
2672 let exact = tokio::task::spawn_blocking(move || {
2673 bamboo_config::read_exact_credential_section_snapshot(data_dir, section, None)
2674 })
2675 .await
2676 .map_err(|error| {
2677 AppError::InternalError(anyhow::anyhow!(
2678 "{} exact read snapshot task failed: {error}",
2679 section.descriptor().name
2680 ))
2681 })?
2682 .map_err(map_exact_credential_store_error)?;
2683 let envelope = exact.section.clone();
2684 let mut config = Config::default();
2685 let metadata = exact.install_into(&mut config);
2686 Ok(ExactCredentialSectionSnapshot {
2687 config,
2688 section: envelope,
2689 metadata,
2690 })
2691 }
2692
2693 pub(crate) async fn put_ordinary_section(
2697 &self,
2698 id: SectionId,
2699 expected_revision: u64,
2700 candidate: Value,
2701 ) -> Result<bamboo_config::SectionEnvelope<Value>, ConfigSectionMutationError> {
2702 if matches!(
2703 id,
2704 SectionId::Providers
2705 | SectionId::Mcp
2706 | SectionId::Credentials
2707 | SectionId::ClusterFabric
2708 ) {
2709 return Err(ConfigSectionMutationError::Invalid(
2710 "this section requires its dedicated endpoint".to_string(),
2711 ));
2712 }
2713 let _io = self.config_io_lock.lock().await;
2714 ensure_provider_mcp_migration_ready(&self.app_data_dir)
2715 .map_err(ConfigSectionMutationError::Store)?;
2716 let facade = self.config_facade.as_ref().ok_or_else(|| {
2717 ConfigSectionMutationError::Invalid(
2718 "typed section writes require the modular configuration facade".to_string(),
2719 )
2720 })?;
2721 let current = if id == SectionId::Core {
2722 let data_dir = self.app_data_dir.clone();
2723 let exact = tokio::task::spawn_blocking(move || {
2724 bamboo_config::read_exact_credential_section_snapshot(
2725 data_dir,
2726 SectionId::Core,
2727 Some(expected_revision),
2728 )
2729 })
2730 .await
2731 .map_err(|error| {
2732 ConfigSectionMutationError::Runtime(format!(
2733 "Core exact inventory snapshot task failed: {error}"
2734 ))
2735 })?
2736 .map_err(ConfigSectionMutationError::Store)?;
2737 if let Some(reference) = exact
2738 .section
2739 .data
2740 .get("proxy_auth_credential_ref")
2741 .and_then(Value::as_str)
2742 {
2743 let configured = exact
2744 .credential_statuses
2745 .iter()
2746 .any(|status| status.credential_ref.as_str() == reference && status.configured);
2747 if !configured {
2748 return Err(ConfigSectionMutationError::Invalid(
2749 "the active Core proxy credential is invalid; explicitly replace or clear it through the proxy-auth API"
2750 .to_string(),
2751 ));
2752 }
2753 }
2754 exact.section
2755 } else {
2756 facade
2757 .registry()
2758 .envelope_value(id)
2759 .map_err(ConfigSectionMutationError::Store)?
2760 };
2761 if credential_reference_inventory(¤t.data)
2762 != credential_reference_inventory(&candidate)
2763 {
2764 return Err(ConfigSectionMutationError::Invalid(
2765 "credential references are server-managed; use the credential or domain API"
2766 .to_string(),
2767 ));
2768 }
2769
2770 let (event, committed) = if id == SectionId::Core {
2771 bamboo_config::commit_core_metadata_from_durable_base(
2777 &self.app_data_dir,
2778 facade,
2779 expected_revision,
2780 candidate,
2781 )
2782 } else {
2783 facade
2784 .registry()
2785 .commit_value_with_envelope(id, expected_revision, candidate)
2786 }
2787 .map_err(ConfigSectionMutationError::Store)?;
2788 let materialized = materialize_facade_effective_config(facade, &self.app_data_dir);
2789 if materialized.failures.contains(&id) {
2790 let message =
2791 "configuration runtime hydration failed; retaining last-known-good runtime"
2792 .to_string();
2793 if let Some(invalid) = facade.registry().mark_runtime_degraded(id, message.clone()) {
2794 publish_registry_event(&self.account_sink, &invalid).await;
2795 }
2796 return Err(ConfigSectionMutationError::Runtime(message));
2797 }
2798
2799 let mut live = self.config.read().await.clone();
2800 let enforcement_newly_off = id == SectionId::ToolsSkills
2801 && !live.plugin_trust.enforcement_is_off()
2802 && materialized.config.plugin_trust.enforcement_is_off();
2803 apply_runtime_section(id, &materialized.config, &mut live);
2804 if id == SectionId::Env {
2805 live.publish_env_vars();
2806 }
2807 *self.config.write().await = live;
2808 if enforcement_newly_off {
2809 warn_plugin_trust_enforcement_off();
2810 }
2811 publish_registry_event(&self.account_sink, &event).await;
2812 Ok(committed)
2813 }
2814
2815 pub(crate) async fn put_provider_section(
2818 &self,
2819 expected_revision: u64,
2820 mut providers: ProviderConfigs,
2821 ) -> Result<u64, ConfigSectionMutationError> {
2822 let _io = self.config_io_lock.lock().await;
2823 ensure_provider_mcp_migration_ready(&self.app_data_dir)
2824 .map_err(ConfigSectionMutationError::Store)?;
2825 let current = self.config.read().await.clone();
2826 retain_provider_credentials(current.providers(), &mut providers);
2827 let mut candidate = current;
2828 *candidate.providers_mut() = providers.clone();
2829 let (candidate, registry, provider) =
2830 match prepare_provider_candidate(candidate, &self.app_data_dir).await {
2831 Ok(prepared) => prepared,
2832 Err(_) => {
2833 let message =
2834 "provider runtime initialization failed; retaining last-known-good runtime"
2835 .to_string();
2836 publish_section_failure(
2837 &self.config_live_health,
2838 &self.account_sink,
2839 "providers",
2840 SectionStatus::Degraded,
2841 message.clone(),
2842 )
2843 .await;
2844 return Err(ConfigSectionMutationError::Runtime(message));
2845 }
2846 };
2847
2848 let durable_providers = provider_durable_document(&providers)?;
2849 let mut live_config = self.config.write().await;
2853 let mut live_provider = self.provider.write().await;
2854 let (revision, source_path) = if let Some(facade) = self.config_facade.as_ref() {
2855 let mut section = facade.registry().providers.snapshot().data.as_ref().clone();
2856 section.providers = durable_providers;
2857 let event = facade
2858 .registry()
2859 .providers
2860 .commit(expected_revision, section)
2861 .map_err(ConfigSectionMutationError::Store)?;
2862 let ConfigSectionEvent::Changed { revision, .. } = event else {
2863 unreachable!("a successful section commit is changed")
2864 };
2865 (
2866 revision,
2867 facade.registry().providers.snapshot().source_path.clone(),
2868 )
2869 } else {
2870 let store = AtomicJsonStore::new(self.app_data_dir.join("providers.json"), 1);
2871 let revision = store
2872 .commit_allowing_unversioned(
2873 expected_revision,
2874 durable_providers,
2875 validate_provider_config,
2876 )
2877 .map_err(ConfigSectionMutationError::Store)?;
2878 (revision, store.path().to_path_buf())
2879 };
2880
2881 candidate.publish_env_vars();
2882 *live_config = candidate;
2883 self.provider_registry.replace_with(registry);
2884 *live_provider = provider;
2885 publish_section_success(
2886 &self.config_live_health,
2887 &self.account_sink,
2888 "providers",
2889 source_path,
2890 section_is_unhealthy(&self.config_live_health),
2891 Some(revision),
2892 )
2893 .await;
2894 Ok(revision)
2895 }
2896
2897 pub(crate) async fn put_provider_settings<F>(
2902 &self,
2903 expected_revision: u64,
2904 update: F,
2905 ) -> Result<u64, ConfigSectionMutationError>
2906 where
2907 F: FnOnce(
2908 &Config,
2909 &mut Config,
2910 )
2911 -> Result<(BTreeSet<String>, BTreeSet<String>), ConfigSectionMutationError>
2912 + Send
2913 + 'static,
2914 {
2915 let config_io_lock = self.config_io_lock.clone();
2916 let config = self.config.clone();
2917 let app_data_dir = self.app_data_dir.clone();
2918 let config_facade = self.config_facade.clone();
2919 let account_sink = self.account_sink.clone();
2920 let provider_registry = self.provider_registry.clone();
2921 let provider = self.provider.clone();
2922 let config_live_health = self.config_live_health.clone();
2923 let transaction = tokio::spawn(async move {
2924 let _io = config_io_lock.lock().await;
2925 ensure_provider_mcp_migration_ready(&app_data_dir)
2926 .map_err(ConfigSectionMutationError::Store)?;
2927 let facade = config_facade.as_ref().ok_or_else(|| {
2928 ConfigSectionMutationError::Invalid(
2929 "provider settings require the modular configuration facade".to_string(),
2930 )
2931 })?;
2932 let current = config.read().await.clone();
2933 let mut candidate = current.clone();
2934 let (provider_intents, provider_instance_intents) = update(¤t, &mut candidate)?;
2935
2936 let (candidate, registry, candidate_provider) =
2937 match prepare_provider_candidate(candidate, &app_data_dir).await {
2938 Ok(prepared) => prepared,
2939 Err(_) => {
2940 let message =
2941 "provider runtime initialization failed; retaining last-known-good runtime"
2942 .to_string();
2943 publish_section_failure(
2944 &config_live_health,
2945 &account_sink,
2946 "providers",
2947 SectionStatus::Degraded,
2948 message.clone(),
2949 )
2950 .await;
2951 return Err(ConfigSectionMutationError::Runtime(message));
2952 }
2953 };
2954
2955 let mut live_config = config.write().await;
2960 let mut live_provider = provider.write().await;
2961 let transaction_dir = app_data_dir.clone();
2962 let commit_facade = facade.clone();
2963 let (mut committed, commit) = tokio::task::spawn_blocking(move || {
2964 let mut durable_candidate = candidate;
2965 let commit =
2966 bamboo_config::persist_provider_credential_transaction_at_revision_with_adoption(
2967 &transaction_dir,
2968 &mut durable_candidate,
2969 &provider_intents,
2970 &provider_instance_intents,
2971 expected_revision,
2972 commit_facade.as_ref(),
2973 )?;
2974 Ok::<_, ConfigStoreError>((durable_candidate, commit))
2975 })
2976 .await
2977 .map_err(|error| {
2978 ConfigSectionMutationError::Runtime(format!(
2979 "provider settings transaction task failed: {error}"
2980 ))
2981 })?
2982 .map_err(ConfigSectionMutationError::Store)?;
2983
2984 let installed = install_credential_section_commit(commit, &mut committed)
2985 .map_err(ConfigSectionMutationError::Store)?;
2986 committed.publish_env_vars();
2987 *live_config = committed;
2988 provider_registry.replace_with(registry);
2989 *live_provider = candidate_provider;
2990 publish_exact_facade_events(&account_sink, &installed.events)
2991 .await
2992 .map_err(|error| ConfigSectionMutationError::Runtime(error.to_string()))?;
2993 let provider_snapshot = facade.registry().providers.snapshot();
2994 set_live_health_revision(
2995 &config_live_health,
2996 provider_snapshot.revision,
2997 Some((
2998 provider_snapshot.source_path.clone(),
2999 SectionSourceKind::File,
3000 )),
3001 );
3002 Ok(provider_snapshot.revision)
3003 });
3004 transaction.await.map_err(|error| {
3005 ConfigSectionMutationError::Runtime(format!(
3006 "provider settings transaction task failed: {error}"
3007 ))
3008 })?
3009 }
3010
3011 pub(crate) async fn reset_provider_section(
3016 &self,
3017 expected_revision: u64,
3018 ) -> Result<u64, ConfigSectionMutationError> {
3019 let config_io_lock = self.config_io_lock.clone();
3020 let config = self.config.clone();
3021 let app_data_dir = self.app_data_dir.clone();
3022 let config_facade = self.config_facade.clone();
3023 let account_sink = self.account_sink.clone();
3024 let provider_registry = self.provider_registry.clone();
3025 let provider = self.provider.clone();
3026 let config_live_health = self.config_live_health.clone();
3027 let transaction = tokio::spawn(async move {
3028 let _io = config_io_lock.lock().await;
3029 ensure_provider_mcp_migration_ready(&app_data_dir)
3030 .map_err(ConfigSectionMutationError::Store)?;
3031 let facade = config_facade.as_ref().ok_or_else(|| {
3032 ConfigSectionMutationError::Invalid(
3033 "provider reset requires the modular configuration facade".to_string(),
3034 )
3035 })?;
3036 let current = config.read().await.clone();
3037 let provider_intents = BTreeSet::from([
3038 "openai".to_string(),
3039 "anthropic".to_string(),
3040 "gemini".to_string(),
3041 "bodhi".to_string(),
3042 ]);
3043 let provider_instance_intents = current.provider_instances.keys().cloned().collect();
3044 let mut candidate = current;
3045 apply_runtime_section(SectionId::Providers, &Config::default(), &mut candidate);
3046
3047 let transaction_dir = app_data_dir.clone();
3048 let commit_facade = facade.clone();
3049 let (mut candidate, commit) = tokio::task::spawn_blocking(move || {
3050 let commit =
3051 bamboo_config::persist_provider_reset_credential_transaction_at_revision_with_adoption(
3052 &transaction_dir,
3053 &mut candidate,
3054 &provider_intents,
3055 &provider_instance_intents,
3056 expected_revision,
3057 commit_facade.as_ref(),
3058 )?;
3059 Ok::<_, ConfigStoreError>((candidate, commit))
3060 })
3061 .await
3062 .map_err(|error| {
3063 ConfigSectionMutationError::Runtime(format!(
3064 "provider reset transaction task failed: {error}"
3065 ))
3066 })?
3067 .map_err(ConfigSectionMutationError::Store)?;
3068
3069 let installed = install_credential_section_commit(commit, &mut candidate)
3070 .map_err(ConfigSectionMutationError::Store)?;
3071 *config.write().await = candidate.clone();
3072 let provider_snapshot = facade.registry().providers.snapshot();
3073 set_live_health_revision(
3074 &config_live_health,
3075 provider_snapshot.revision,
3076 Some((
3077 provider_snapshot.source_path.clone(),
3078 SectionSourceKind::File,
3079 )),
3080 );
3081 publish_exact_facade_events(&account_sink, &installed.events)
3082 .await
3083 .map_err(|error| ConfigSectionMutationError::Runtime(error.to_string()))?;
3084
3085 let runtime_failure = match bamboo_llm::ProviderRegistry::from_config(
3086 &candidate,
3087 app_data_dir,
3088 )
3089 .await
3090 {
3091 Ok(registry) => {
3092 if let Some(candidate_provider) = registry.get_default() {
3093 provider_registry.replace_with(registry);
3094 *provider.write().await = candidate_provider;
3095 None
3096 } else {
3097 Some(
3098 "provider reset committed; default provider is not initialized"
3099 .to_string(),
3100 )
3101 }
3102 }
3103 Err(error) => {
3104 tracing::warn!(error = %error, "provider reset committed but runtime initialization failed");
3105 Some("provider reset committed; retaining last-known-good runtime".to_string())
3106 }
3107 };
3108 if let Some(message) = runtime_failure {
3109 publish_section_failure(
3110 &config_live_health,
3111 &account_sink,
3112 "providers",
3113 SectionStatus::Degraded,
3114 message,
3115 )
3116 .await;
3117 }
3118 Ok(provider_snapshot.revision)
3119 });
3120 transaction.await.map_err(|error| {
3121 ConfigSectionMutationError::Runtime(format!(
3122 "provider reset transaction task failed: {error}"
3123 ))
3124 })?
3125 }
3126
3127 pub(crate) async fn put_mcp_section(
3131 &self,
3132 expected_revision: u64,
3133 mut candidate: McpConfig,
3134 ) -> Result<u64, ConfigSectionMutationError> {
3135 let _io = self.config_io_lock.lock().await;
3136 ensure_provider_mcp_migration_ready(&self.app_data_dir)
3137 .map_err(ConfigSectionMutationError::Store)?;
3138 retain_mcp_credentials(
3139 &self.config.read().await.mcp,
3140 &mut candidate,
3141 &BTreeSet::new(),
3142 );
3143 validate_mcp_config(&candidate).map_err(ConfigSectionMutationError::Invalid)?;
3144 let mut hydration_config = Config::default();
3145 hydration_config.mcp = candidate;
3146 hydration_config
3147 .hydrate_mcp_credentials_from_store(&self.app_data_dir)
3148 .map_err(|_| {
3149 ConfigSectionMutationError::Invalid(
3150 "referenced MCP credential is unavailable".to_string(),
3151 )
3152 })?;
3153 let candidate = hydration_config.mcp.clone();
3154 let mut revision = None;
3155 let mut store_error = None;
3156 let durable_candidate = credential_ref_mcp_document(&candidate)?;
3157 let mut next_config = candidate.clone();
3158 retain_mcp_credential_refs(&durable_candidate, &mut next_config);
3159 let result = self
3160 .mcp_manager
3161 .reconcile_from_config_transactional_after(&candidate, || async {
3162 let mut live_config = self.config.write().await;
3166 let commit = if let Some(facade) = self.config_facade.as_ref() {
3167 facade
3168 .registry()
3169 .mcp
3170 .commit(expected_revision, McpSection(durable_candidate))
3171 .map(|event| match event {
3172 ConfigSectionEvent::Changed { revision, .. } => revision,
3173 _ => unreachable!("a successful section commit is changed"),
3174 })
3175 } else {
3176 AtomicJsonStore::new(self.app_data_dir.join("mcp.json"), 1).commit(
3177 expected_revision,
3178 durable_candidate,
3179 validate_mcp_config,
3180 )
3181 };
3182 match commit {
3183 Ok(committed) => {
3184 live_config.mcp = next_config;
3185 revision = Some(committed);
3186 Ok(())
3187 }
3188 Err(error) => {
3189 store_error = Some(error);
3190 Err(bamboo_mcp::McpError::InvalidConfig(
3191 "MCP section durable commit failed".to_string(),
3192 ))
3193 }
3194 }
3195 })
3196 .await;
3197 if let Some(error) = store_error {
3198 return Err(ConfigSectionMutationError::Store(error));
3199 }
3200 if result.is_err() {
3201 let message =
3202 "MCP runtime initialization failed; retaining last-known-good runtime".to_string();
3203 publish_section_failure(
3204 &self.mcp_config_live_health,
3205 &self.account_sink,
3206 "mcp",
3207 SectionStatus::Degraded,
3208 message.clone(),
3209 )
3210 .await;
3211 return Err(ConfigSectionMutationError::Runtime(message));
3212 }
3213 let revision = revision.expect("successful MCP reconcile commits a revision");
3214 publish_section_success(
3215 &self.mcp_config_live_health,
3216 &self.account_sink,
3217 "mcp",
3218 self.app_data_dir.join("mcp.json"),
3219 section_is_unhealthy(&self.mcp_config_live_health),
3220 Some(revision),
3221 )
3222 .await;
3223 Ok(revision)
3224 }
3225
3226 pub(crate) async fn put_mcp_settings(
3230 &self,
3231 expected_revision: u64,
3232 candidate: McpConfig,
3233 credential_intents: BTreeSet<bamboo_config::CredentialRef>,
3234 ) -> Result<u64, ConfigSectionMutationError> {
3235 if credential_intents.is_empty() {
3236 return self.put_mcp_section(expected_revision, candidate).await;
3237 }
3238
3239 let config_io_lock = self.config_io_lock.clone();
3240 let config = self.config.clone();
3241 let app_data_dir = self.app_data_dir.clone();
3242 let config_facade = self.config_facade.clone();
3243 let account_sink = self.account_sink.clone();
3244 let mcp_manager = self.mcp_manager.clone();
3245 let mcp_config_live_health = self.mcp_config_live_health.clone();
3246 let transaction = tokio::spawn(async move {
3247 let _io = config_io_lock.lock().await;
3248 ensure_provider_mcp_migration_ready(&app_data_dir)
3249 .map_err(ConfigSectionMutationError::Store)?;
3250 let facade = config_facade.as_ref().ok_or_else(|| {
3251 ConfigSectionMutationError::Invalid(
3252 "MCP settings require the modular configuration facade".to_string(),
3253 )
3254 })?;
3255 let current = config.read().await.clone();
3256 let mut runtime_candidate = candidate;
3257 materialize_mcp_touched_replacements(&mut runtime_candidate, &credential_intents)
3258 .map_err(ConfigSectionMutationError::Invalid)?;
3259 retain_mcp_credentials(¤t.mcp, &mut runtime_candidate, &credential_intents);
3260 validate_mcp_config(&runtime_candidate).map_err(ConfigSectionMutationError::Invalid)?;
3261
3262 let mut transaction_error = None;
3263 let mut commit_events = Vec::new();
3264 let transaction_dir = app_data_dir.clone();
3265 let commit_facade = facade.clone();
3266 let mut durable_candidate = current;
3267 durable_candidate.mcp = runtime_candidate.clone();
3268 let result = mcp_manager
3269 .reconcile_from_config_transactional_after(&runtime_candidate, || async {
3270 let mut live_config = config.write().await;
3271 let commit = tokio::task::spawn_blocking(move || {
3272 let commit =
3273 bamboo_config::persist_mcp_credential_transaction_at_revision_with_adoption(
3274 &transaction_dir,
3275 &mut durable_candidate,
3276 &credential_intents,
3277 expected_revision,
3278 commit_facade.as_ref(),
3279 )?;
3280 Ok::<_, ConfigStoreError>((durable_candidate, commit))
3281 })
3282 .await;
3283 match commit {
3284 Ok(Ok((mut committed, commit))) => {
3285 let installed =
3286 match install_credential_section_commit(commit, &mut committed) {
3287 Ok(installed) => installed,
3288 Err(error) => {
3289 transaction_error =
3290 Some(ConfigSectionMutationError::Store(error));
3291 return Err(bamboo_mcp::McpError::InvalidConfig(
3292 "MCP settings process adoption failed".to_string(),
3293 ));
3294 }
3295 };
3296 *live_config = committed;
3297 commit_events = installed.events;
3298 Ok(())
3299 }
3300 Ok(Err(error)) => {
3301 transaction_error = Some(ConfigSectionMutationError::Store(error));
3302 Err(bamboo_mcp::McpError::InvalidConfig(
3303 "MCP settings durable transaction failed".to_string(),
3304 ))
3305 }
3306 Err(error) => {
3307 transaction_error = Some(ConfigSectionMutationError::Runtime(format!(
3308 "MCP settings transaction task failed: {error}"
3309 )));
3310 Err(bamboo_mcp::McpError::InvalidConfig(
3311 "MCP settings durable transaction failed".to_string(),
3312 ))
3313 }
3314 }
3315 })
3316 .await;
3317 if let Some(error) = transaction_error {
3318 return Err(error);
3319 }
3320 if result.is_err() {
3321 let message =
3322 "MCP runtime initialization failed; retaining last-known-good runtime"
3323 .to_string();
3324 publish_section_failure(
3325 &mcp_config_live_health,
3326 &account_sink,
3327 "mcp",
3328 SectionStatus::Degraded,
3329 message.clone(),
3330 )
3331 .await;
3332 return Err(ConfigSectionMutationError::Runtime(message));
3333 }
3334
3335 publish_exact_facade_events(&account_sink, &commit_events)
3336 .await
3337 .map_err(|error| ConfigSectionMutationError::Runtime(error.to_string()))?;
3338 let snapshot = facade.registry().mcp.snapshot();
3339 set_live_health_revision(
3340 &mcp_config_live_health,
3341 snapshot.revision,
3342 Some((snapshot.source_path.clone(), SectionSourceKind::File)),
3343 );
3344 Ok(snapshot.revision)
3345 });
3346 transaction.await.map_err(|error| {
3347 ConfigSectionMutationError::Runtime(format!(
3348 "MCP settings transaction task failed: {error}"
3349 ))
3350 })?
3351 }
3352
3353 pub(crate) async fn update_legacy_mcp_config<F>(
3363 &self,
3364 force_restart: BTreeSet<String>,
3365 update: F,
3366 ) -> Result<Config, AppError>
3367 where
3368 F: FnOnce(&mut McpConfig) -> Result<(), AppError>,
3369 {
3370 let io = self.config_io_lock.clone().lock_owned().await;
3373 let (mut candidate_config, expected_revision, credential_intents) = {
3374 ensure_provider_mcp_migration_ready(&self.app_data_dir)
3375 .map_err(map_exact_credential_store_error)?;
3376 let facade = self.config_facade.as_ref().ok_or_else(|| {
3377 AppError::BadRequest(
3378 "legacy MCP mutations require the modular configuration facade".to_string(),
3379 )
3380 })?;
3381 let current = self.config.read().await.clone();
3382 reject_if_recovery_pending(¤t)?;
3383 let mut candidate = current.mcp.clone();
3384 update(&mut candidate)?;
3385 let credential_intents =
3386 normalize_legacy_mcp_credentials(¤t.mcp, &mut candidate)?;
3387 validate_mcp_config(&candidate).map_err(AppError::BadRequest)?;
3388 let expected_revision = facade.registry().mcp.snapshot().revision;
3389 let mut candidate_config = current;
3390 candidate_config.mcp = candidate;
3391 (candidate_config, expected_revision, credential_intents)
3392 };
3393
3394 let config = self.config.clone();
3395 let app_data_dir = self.app_data_dir.clone();
3396 let config_facade = self.config_facade.clone();
3397 let account_sink = self.account_sink.clone();
3398 let mcp_manager = self.mcp_manager.clone();
3399 let mcp_config_live_health = self.mcp_config_live_health.clone();
3400 let transaction = tokio::spawn(async move {
3401 let _io = io;
3402 let facade = config_facade.expect("validated modular configuration facade");
3403 let runtime_candidate = candidate_config.mcp.clone();
3404 let force_replacements = force_restart.into_iter().collect();
3405 let durable_document =
3406 credential_ref_mcp_document(&runtime_candidate).map_err(map_mcp_section_error)?;
3407 bamboo_config::validate_mcp_section(&durable_document).map_err(AppError::BadRequest)?;
3408 let current_document = facade.registry().mcp.snapshot();
3409 let metadata_changed = serde_json::to_value(¤t_document.data.0)
3410 .map_err(AppError::SerializationError)?
3411 != serde_json::to_value(&durable_document).map_err(AppError::SerializationError)?;
3412 let credential_transaction = !credential_intents.is_empty();
3413 let mut transaction_error = None;
3414 let mut commit_events = Vec::new();
3415 let mut published_config = None;
3416 let commit_dir = app_data_dir.clone();
3417 let commit_facade = facade.clone();
3418 let result = mcp_manager
3419 .reconcile_from_config_transactional_after_forcing(
3420 &runtime_candidate,
3421 &force_replacements,
3422 || async {
3423 let mut live_config = config.write().await;
3427 if credential_transaction {
3428 let commit = tokio::task::spawn_blocking(move || {
3429 let commit = bamboo_config::persist_mcp_credential_transaction_at_revision_with_adoption(
3430 &commit_dir,
3431 &mut candidate_config,
3432 &credential_intents,
3433 expected_revision,
3434 commit_facade.as_ref(),
3435 )?;
3436 Ok::<_, ConfigStoreError>((candidate_config, commit))
3437 })
3438 .await;
3439 match commit {
3440 Ok(Ok((mut committed, commit))) => {
3441 #[cfg(test)]
3442 run_credential_after_commit_before_live_test_hook(
3443 &app_data_dir,
3444 SectionId::Mcp,
3445 );
3446 match install_credential_section_commit(commit, &mut committed) {
3447 Ok(installed) => {
3448 *live_config = committed.clone();
3449 commit_events = installed.events;
3450 published_config = Some(committed);
3451 }
3452 Err(error) => {
3453 transaction_error =
3454 Some(map_exact_credential_store_error(error));
3455 return Err(bamboo_mcp::McpError::InvalidConfig(
3456 "MCP process adoption failed".to_string(),
3457 ));
3458 }
3459 }
3460 }
3461 Ok(Err(error)) => {
3462 transaction_error =
3463 Some(map_exact_credential_store_error(error));
3464 return Err(bamboo_mcp::McpError::InvalidConfig(
3465 "MCP durable transaction failed".to_string(),
3466 ));
3467 }
3468 Err(error) => {
3469 transaction_error = Some(AppError::InternalError(
3470 anyhow::anyhow!(
3471 "MCP credential transaction task failed: {error}"
3472 ),
3473 ));
3474 return Err(bamboo_mcp::McpError::InvalidConfig(
3475 "MCP durable transaction failed".to_string(),
3476 ));
3477 }
3478 }
3479 } else {
3480 if metadata_changed {
3481 match facade
3482 .registry()
3483 .mcp
3484 .commit(expected_revision, McpSection(durable_document))
3485 {
3486 Ok(event) => commit_events.push(event),
3487 Err(error) => {
3488 transaction_error =
3489 Some(map_exact_credential_store_error(error));
3490 return Err(bamboo_mcp::McpError::InvalidConfig(
3491 "MCP durable commit failed".to_string(),
3492 ));
3493 }
3494 }
3495 }
3496 candidate_config.mcp = runtime_candidate.clone();
3497 *live_config = candidate_config.clone();
3498 published_config = Some(candidate_config);
3499 }
3500 Ok(())
3501 },
3502 )
3503 .await;
3504
3505 if let Some(error) = transaction_error {
3506 return Err(error);
3507 }
3508 if result.is_err() {
3509 tracing::warn!("legacy MCP runtime staging failed before durable commit");
3510 let message =
3511 "MCP runtime initialization failed before commit; retaining last-known-good generation"
3512 .to_string();
3513 return Err(AppError::InternalError(anyhow::anyhow!(message)));
3514 }
3515
3516 publish_exact_facade_events(&account_sink, &commit_events).await?;
3520 let snapshot = facade.registry().mcp.snapshot();
3521 set_live_health_revision(
3522 &mcp_config_live_health,
3523 snapshot.revision,
3524 Some((snapshot.source_path.clone(), SectionSourceKind::File)),
3525 );
3526 Ok::<_, AppError>(
3527 published_config.expect("successful MCP transaction publishes config"),
3528 )
3529 });
3530 transaction.await.map_err(|error| {
3531 AppError::InternalError(anyhow::anyhow!(
3532 "legacy MCP config transaction task failed: {error}"
3533 ))
3534 })?
3535 }
3536
3537 pub(crate) async fn reset_mcp_section(
3541 &self,
3542 expected_revision: u64,
3543 ) -> Result<u64, ConfigSectionMutationError> {
3544 let _io = self.config_io_lock.lock().await;
3545 ensure_provider_mcp_migration_ready(&self.app_data_dir)
3546 .map_err(ConfigSectionMutationError::Store)?;
3547 let facade = self.config_facade.as_ref().ok_or_else(|| {
3548 ConfigSectionMutationError::Invalid(
3549 "MCP reset requires the modular configuration facade".to_string(),
3550 )
3551 })?;
3552 let candidate_mcp = McpConfig::default();
3553 let mut candidate_config = self.config.read().await.clone();
3554 candidate_config.mcp = candidate_mcp.clone();
3555 let mut committed = false;
3556 let mut commit_events = Vec::new();
3557 let mut store_error = None;
3558 let data_dir = self.app_data_dir.clone();
3559 let result = self
3560 .mcp_manager
3561 .reconcile_from_config_transactional_after(&candidate_mcp, || async {
3562 let mut live_config = self.config.write().await;
3563 match bamboo_config::persist_mcp_reset_credential_transaction_at_revision_with_adoption(
3564 &data_dir,
3565 &mut candidate_config,
3566 expected_revision,
3567 facade.as_ref(),
3568 ) {
3569 Ok(commit) => {
3570 match install_credential_section_commit(commit, &mut candidate_config) {
3571 Ok(installed) => {
3572 *live_config = candidate_config.clone();
3573 commit_events = installed.events;
3574 committed = true;
3575 }
3576 Err(error) => {
3577 store_error = Some(error);
3578 return Err(bamboo_mcp::McpError::InvalidConfig(
3579 "MCP reset process adoption failed".to_string(),
3580 ));
3581 }
3582 }
3583 Ok(())
3584 }
3585 Err(error) => {
3586 store_error = Some(error);
3587 Err(bamboo_mcp::McpError::InvalidConfig(
3588 "MCP reset durable commit failed".to_string(),
3589 ))
3590 }
3591 }
3592 })
3593 .await;
3594 if let Some(error) = store_error {
3595 return Err(ConfigSectionMutationError::Store(error));
3596 }
3597 if result.is_err() || !committed {
3598 let message =
3599 "MCP reset runtime initialization failed; retaining last-known-good runtime"
3600 .to_string();
3601 publish_section_failure(
3602 &self.mcp_config_live_health,
3603 &self.account_sink,
3604 "mcp",
3605 SectionStatus::Degraded,
3606 message.clone(),
3607 )
3608 .await;
3609 return Err(ConfigSectionMutationError::Runtime(message));
3610 }
3611 publish_exact_facade_events(&self.account_sink, &commit_events)
3612 .await
3613 .map_err(|error| ConfigSectionMutationError::Runtime(error.to_string()))?;
3614 let revision = facade.registry().mcp.snapshot().revision;
3615 publish_section_success(
3616 &self.mcp_config_live_health,
3617 &self.account_sink,
3618 "mcp",
3619 self.app_data_dir.join("mcp.json"),
3620 section_is_unhealthy(&self.mcp_config_live_health),
3621 Some(revision),
3622 )
3623 .await;
3624 Ok(revision)
3625 }
3626
3627 pub(crate) async fn reset_credential_backed_section(
3632 &self,
3633 id: SectionId,
3634 expected_revision: u64,
3635 ) -> Result<CredentialBackedResetCommit, ConfigSectionMutationError> {
3636 if !matches!(
3637 id,
3638 SectionId::Core
3639 | SectionId::Notifications
3640 | SectionId::Connect
3641 | SectionId::Env
3642 | SectionId::ClusterFabric
3643 | SectionId::AccessControl
3644 ) {
3645 return Err(ConfigSectionMutationError::Invalid(
3646 "section is not a credential-backed reset domain".to_string(),
3647 ));
3648 }
3649 let config_io_lock = self.config_io_lock.clone();
3650 let config = self.config.clone();
3651 let app_data_dir = self.app_data_dir.clone();
3652 let config_facade = self.config_facade.clone();
3653 let account_sink = self.account_sink.clone();
3654 let provider_registry = self.provider_registry.clone();
3655 let provider = self.provider.clone();
3656 let mcp_manager = self.mcp_manager.clone();
3657 let deployed_registry = self.fabric_deployer.registry();
3658 let transaction = tokio::spawn(async move {
3659 let _io = config_io_lock.lock().await;
3660 if id == SectionId::ClusterFabric {
3661 let deployed = deployed_registry.lock().await;
3662 if let Some(node_id) = deployed.keys().find_map(|key| {
3663 let (source, node_id) = bamboo_server_tools::registry_keys::split(key);
3664 (source == "node").then(|| node_id.to_string())
3665 }) {
3666 return Err(ConfigSectionMutationError::Invalid(format!(
3667 "node '{node_id}' is deployed; stop it before resetting cluster-fabric"
3668 )));
3669 }
3670 }
3671 ensure_provider_mcp_migration_ready(&app_data_dir)
3672 .map_err(ConfigSectionMutationError::Store)?;
3673 let facade = config_facade.as_ref().ok_or_else(|| {
3674 ConfigSectionMutationError::Invalid(
3675 "section reset requires the modular configuration facade".to_string(),
3676 )
3677 })?;
3678 let mut candidate = config.read().await.clone();
3679 apply_runtime_section(id, &Config::default(), &mut candidate);
3680 let transaction_dir = app_data_dir.clone();
3681 let commit_facade = facade.clone();
3682 let (mut candidate, revision, cluster_commit, section_commit) =
3683 tokio::task::spawn_blocking(move || {
3684 if id == SectionId::ClusterFabric {
3685 let commit =
3686 bamboo_config::persist_cluster_fabric_reset_at_revision_with_adoption(
3687 &transaction_dir,
3688 &mut candidate,
3689 expected_revision,
3690 commit_facade.as_ref(),
3691 |_, _| {},
3692 )?;
3693 let revision = commit.revision;
3694 Ok::<_, ConfigStoreError>((candidate, revision, Some(commit), None))
3695 } else {
3696 let commit =
3697 bamboo_config::persist_credential_backed_section_reset_at_revision_with_adoption(
3698 &transaction_dir,
3699 &mut candidate,
3700 id,
3701 expected_revision,
3702 commit_facade.as_ref(),
3703 )?;
3704 let revision = commit.revision;
3705 Ok((candidate, revision, None, Some(commit)))
3706 }
3707 })
3708 .await
3709 .map_err(|error| {
3710 ConfigSectionMutationError::Runtime(format!(
3711 "section reset transaction task failed: {error}"
3712 ))
3713 })?
3714 .map_err(ConfigSectionMutationError::Store)?;
3715
3716 let cluster_runtime = match cluster_commit {
3717 Some(commit) => {
3718 let bamboo_config::ClusterFabricTransactionCommit {
3719 revision: _,
3720 adoption,
3721 credential_adoption,
3722 committed_recovery,
3723 runtime,
3724 } = commit;
3725 let runtime = match runtime {
3726 Ok(bamboo_config::ClusterFabricRuntimeSnapshot {
3727 cluster_fabric,
3728 credential_statuses,
3729 credential_health,
3730 }) => {
3731 candidate.cluster_fabric = cluster_fabric;
3732 Ok((credential_statuses, credential_health))
3733 }
3734 Err(error) if revision == expected_revision => {
3735 return Err(ConfigSectionMutationError::Store(error));
3738 }
3739 Err(error) => {
3740 candidate.clear_cluster_runtime_credentials();
3745 Err(error)
3746 }
3747 };
3748 Some((adoption, credential_adoption, committed_recovery, runtime))
3749 }
3750 None => None,
3751 };
3752 let (section_events, exact_section) = match section_commit {
3753 Some(commit) => {
3754 let installed = install_credential_section_commit(commit, &mut candidate)
3755 .map_err(ConfigSectionMutationError::Store)?;
3756 (installed.events, installed.section)
3757 }
3758 None => (Vec::new(), None),
3759 };
3760 if id == SectionId::Env {
3761 candidate.publish_env_vars();
3762 }
3763 *config.write().await = candidate.clone();
3764 if id != SectionId::ClusterFabric {
3765 publish_exact_facade_events(&account_sink, §ion_events)
3766 .await
3767 .map_err(|error| ConfigSectionMutationError::Runtime(error.to_string()))?;
3768 }
3769 let commit = if id == SectionId::ClusterFabric {
3770 let (cluster_adoption, credential_adoption, committed_recovery, cluster_runtime) =
3771 cluster_runtime.expect("cluster reset captures an exact runtime");
3772 let event = match cluster_adoption {
3773 Some(Ok(event)) => Some(event),
3774 Some(Err(error)) => {
3775 return Err(ConfigSectionMutationError::Runtime(format!(
3776 "cluster reset committed at revision {revision} but process adoption failed: {error}"
3777 )));
3778 }
3779 None if revision == expected_revision => None,
3780 None => {
3781 return Err(ConfigSectionMutationError::Runtime(format!(
3782 "cluster reset committed at revision {revision} without a process adoption result"
3783 )));
3784 }
3785 };
3786 let section = facade
3787 .registry()
3788 .envelope_value(SectionId::ClusterFabric)
3789 .map_err(|error| {
3790 if revision == expected_revision {
3791 ConfigSectionMutationError::Store(error)
3792 } else {
3793 ConfigSectionMutationError::Runtime(format!(
3794 "cluster reset committed at revision {revision} but its exact envelope is unavailable: {error}"
3795 ))
3796 }
3797 })?;
3798 if section.revision != revision {
3799 return Err(ConfigSectionMutationError::Runtime(format!(
3800 "cluster reset committed at revision {revision} but facade retained revision {}",
3801 section.revision
3802 )));
3803 }
3804 if let Some(event) = event.as_ref() {
3805 publish_registry_event(&account_sink, event).await;
3806 }
3807 if let Err(error) = committed_recovery {
3808 return Err(ConfigSectionMutationError::Runtime(format!(
3809 "cluster reset committed at revision {revision} but transaction recovery failed: {error}"
3810 )));
3811 }
3812 if let Some(Err(error)) = credential_adoption {
3813 return Err(ConfigSectionMutationError::Runtime(format!(
3814 "cluster reset committed at revision {revision} but credential facade adoption failed: {error}"
3815 )));
3816 }
3817 let (credential_statuses, credential_health) =
3818 cluster_runtime.map_err(|error| {
3819 ConfigSectionMutationError::Runtime(format!(
3820 "cluster reset committed at revision {revision} but could not materialize its exact runtime credentials: {error}"
3821 ))
3822 })?;
3823 CredentialBackedResetCommit::Cluster(Box::new(
3824 bamboo_server_tools::FabricCommitSnapshot {
3825 config: candidate.clone(),
3826 section,
3827 credential_statuses,
3828 credential_health,
3829 },
3830 ))
3831 } else {
3832 let section = exact_section.ok_or_else(|| {
3833 ConfigSectionMutationError::Runtime(format!(
3834 "{} reset committed at revision {revision} without its exact envelope",
3835 id.descriptor().name
3836 ))
3837 })?;
3838 if section.revision != revision {
3839 return Err(ConfigSectionMutationError::Runtime(format!(
3840 "{} reset committed at revision {revision} but captured revision {}",
3841 id.descriptor().name,
3842 section.revision
3843 )));
3844 }
3845 CredentialBackedResetCommit::Section(section)
3846 };
3847
3848 if id == SectionId::Core {
3849 match bamboo_llm::ProviderRegistry::from_config(&candidate, app_data_dir.clone())
3850 .await
3851 {
3852 Ok(registry) => {
3853 if let Some(candidate_provider) = registry.get_default() {
3854 provider_registry.replace_with(registry);
3855 *provider.write().await = candidate_provider;
3856 }
3857 }
3858 Err(error) => {
3859 tracing::warn!(error = %error, "core reset committed but provider reload failed");
3860 }
3861 }
3862 mcp_manager.reconcile_from_config(&candidate.mcp).await;
3863 }
3864
3865 Ok(commit)
3866 });
3867 transaction.await.map_err(|error| {
3868 ConfigSectionMutationError::Runtime(format!(
3869 "section reset transaction task failed: {error}"
3870 ))
3871 })?
3872 }
3873}
3874
3875fn credential_reference_inventory(value: &Value) -> std::collections::BTreeMap<String, Value> {
3876 fn collect(value: &Value, path: &str, output: &mut std::collections::BTreeMap<String, Value>) {
3877 match value {
3878 Value::Object(object) => {
3879 for (key, value) in object {
3880 let child_path =
3881 format!("{path}/{}", key.replace('~', "~0").replace('/', "~1"));
3882 let normalized = key
3883 .chars()
3884 .filter(|ch| ch.is_ascii_alphanumeric())
3885 .flat_map(char::to_lowercase)
3886 .collect::<String>();
3887 if normalized == "credentialref"
3888 || normalized.ends_with("credentialref")
3889 || normalized.ends_with("credentialrefs")
3890 {
3891 output.insert(child_path, value.clone());
3892 } else {
3893 collect(value, &child_path, output);
3894 }
3895 }
3896 }
3897 Value::Array(values) => {
3898 for (index, value) in values.iter().enumerate() {
3899 collect(value, &format!("{path}/{index}"), output);
3900 }
3901 }
3902 _ => {}
3903 }
3904 }
3905
3906 let mut output = std::collections::BTreeMap::new();
3907 collect(value, "", &mut output);
3908 output
3909}
3910
3911fn provider_durable_document(
3912 providers: &ProviderConfigs,
3913) -> Result<ProviderConfigs, ConfigSectionMutationError> {
3914 let mut document = providers.clone();
3915 macro_rules! sanitize {
3916 ($field:ident) => {
3917 if let Some(provider) = document.$field.as_mut() {
3918 provider.api_key.clear();
3919 provider.api_key_encrypted = None;
3920 }
3921 };
3922 }
3923 sanitize!(openai);
3924 sanitize!(anthropic);
3925 sanitize!(gemini);
3926 if let Some(provider) = document.bodhi.as_mut() {
3927 provider.api_key.clear();
3928 provider.api_key_encrypted = None;
3929 }
3930 validate_provider_config(&document).map_err(ConfigSectionMutationError::Invalid)?;
3931 Ok(document)
3932}
3933
3934fn retain_provider_credentials(current: &ProviderConfigs, candidate: &mut ProviderConfigs) {
3935 candidate.extra = current.extra.clone();
3936 macro_rules! retain {
3937 ($field:ident) => {
3938 if let (Some(current), Some(candidate)) = (¤t.$field, &mut candidate.$field) {
3939 candidate.api_key = current.api_key.clone();
3940 candidate.api_key_encrypted = current.api_key_encrypted.clone();
3941 if candidate.credential_ref.is_none() {
3942 candidate.credential_ref = current.credential_ref.clone();
3943 }
3944 if candidate.credential_ref != current.credential_ref {
3945 candidate.api_key.clear();
3946 candidate.api_key_encrypted = None;
3947 }
3948 candidate.api_key_from_env = current.api_key_from_env;
3949 candidate.request_overrides = current.request_overrides.clone();
3950 candidate.extra = current.extra.clone();
3951 }
3952 };
3953 }
3954 retain!(openai);
3955 retain!(anthropic);
3956 retain!(gemini);
3957 if let (Some(current), Some(candidate)) = (¤t.bodhi, &mut candidate.bodhi) {
3958 candidate.api_key = current.api_key.clone();
3959 candidate.api_key_encrypted = current.api_key_encrypted.clone();
3960 if candidate.credential_ref.is_none() {
3961 candidate.credential_ref = current.credential_ref.clone();
3962 }
3963 if candidate.credential_ref != current.credential_ref {
3964 candidate.api_key.clear();
3965 candidate.api_key_encrypted = None;
3966 }
3967 candidate.extra = current.extra.clone();
3968 }
3969 if let (Some(current), Some(candidate)) = (¤t.copilot, &mut candidate.copilot) {
3970 candidate.request_overrides = current.request_overrides.clone();
3971 candidate.extra = current.extra.clone();
3972 }
3973}
3974
3975fn retain_mcp_credentials(
3976 current: &McpConfig,
3977 candidate: &mut McpConfig,
3978 touched: &BTreeSet<bamboo_config::CredentialRef>,
3979) {
3980 for candidate_server in &mut candidate.servers {
3981 let Some(current_server) = current
3982 .servers
3983 .iter()
3984 .find(|server| server.id == candidate_server.id)
3985 else {
3986 continue;
3987 };
3988 if let (TransportConfig::Stdio(current), TransportConfig::Stdio(candidate)) =
3989 (¤t_server.transport, &mut candidate_server.transport)
3990 {
3991 if candidate.env.is_empty()
3992 && candidate.env_encrypted.is_empty()
3993 && candidate.env_credential_refs.is_empty()
3994 {
3995 for (name, reference) in ¤t.env_credential_refs {
3996 if mcp_credential_ref_is_touched(Some(reference), touched) {
3997 continue;
3998 }
3999 candidate
4000 .env_credential_refs
4001 .insert(name.clone(), reference.clone());
4002 if let Some(value) = current.env.get(name) {
4003 candidate.env.insert(name.clone(), value.clone());
4004 }
4005 if let Some(value) = current.env_encrypted.get(name) {
4006 candidate.env_encrypted.insert(name.clone(), value.clone());
4007 }
4008 }
4009 } else {
4010 for (name, reference) in ¤t.env_credential_refs {
4011 if candidate.env_credential_refs.get(name) != Some(reference) {
4012 continue;
4013 }
4014 if candidate.env.get(name).is_none_or(|value| value.is_empty()) {
4015 if let Some(value) = current.env.get(name) {
4016 candidate.env.insert(name.clone(), value.clone());
4017 }
4018 }
4019 }
4020 }
4021 }
4022 match (¤t_server.transport, &mut candidate_server.transport) {
4023 (TransportConfig::Sse(current), TransportConfig::Sse(candidate)) => {
4024 retain_mcp_header_credentials(¤t.headers, &mut candidate.headers)
4025 }
4026 (
4027 TransportConfig::StreamableHttp(current),
4028 TransportConfig::StreamableHttp(candidate),
4029 ) => retain_mcp_header_credentials(¤t.headers, &mut candidate.headers),
4030 _ => {}
4031 }
4032 }
4033}
4034
4035fn materialize_mcp_touched_replacements(
4036 candidate: &mut McpConfig,
4037 touched: &BTreeSet<bamboo_config::CredentialRef>,
4038) -> Result<(), String> {
4039 let mut replacements = BTreeMap::<bamboo_config::CredentialRef, String>::new();
4040 for server in &candidate.servers {
4041 match &server.transport {
4042 TransportConfig::Stdio(stdio) => {
4043 for (name, raw_reference) in &stdio.env_credential_refs {
4044 let reference = bamboo_config::CredentialRef::parse(raw_reference.clone())
4045 .map_err(|_| "MCP credential reference is invalid".to_string())?;
4046 if let Some(value) = stdio
4047 .env
4048 .get(name)
4049 .filter(|value| touched.contains(&reference) && !value.is_empty())
4050 {
4051 insert_mcp_replacement(&mut replacements, reference, value)?;
4052 }
4053 }
4054 }
4055 TransportConfig::Sse(http) => {
4056 collect_mcp_header_replacements(&http.headers, touched, &mut replacements)?
4057 }
4058 TransportConfig::StreamableHttp(http) => {
4059 collect_mcp_header_replacements(&http.headers, touched, &mut replacements)?
4060 }
4061 }
4062 }
4063 if replacements.is_empty() {
4064 return Ok(());
4065 }
4066 for server in &mut candidate.servers {
4067 match &mut server.transport {
4068 TransportConfig::Stdio(stdio) => {
4069 for (name, raw_reference) in &stdio.env_credential_refs {
4070 let reference = bamboo_config::CredentialRef::parse(raw_reference.clone())
4071 .map_err(|_| "MCP credential reference is invalid".to_string())?;
4072 if let Some(value) = replacements.get(&reference) {
4073 stdio.env.insert(name.clone(), value.clone());
4074 }
4075 }
4076 }
4077 TransportConfig::Sse(http) => {
4078 apply_mcp_header_replacements(&mut http.headers, &replacements)?
4079 }
4080 TransportConfig::StreamableHttp(http) => {
4081 apply_mcp_header_replacements(&mut http.headers, &replacements)?
4082 }
4083 }
4084 }
4085 Ok(())
4086}
4087
4088fn collect_mcp_header_replacements(
4089 headers: &[bamboo_mcp::HeaderConfig],
4090 touched: &BTreeSet<bamboo_config::CredentialRef>,
4091 replacements: &mut BTreeMap<bamboo_config::CredentialRef, String>,
4092) -> Result<(), String> {
4093 for header in headers {
4094 let Some(raw_reference) = header.credential_ref.as_ref() else {
4095 continue;
4096 };
4097 let reference = bamboo_config::CredentialRef::parse(raw_reference.clone())
4098 .map_err(|_| "MCP credential reference is invalid".to_string())?;
4099 if touched.contains(&reference) && !header.value.is_empty() {
4100 insert_mcp_replacement(replacements, reference, &header.value)?;
4101 }
4102 }
4103 Ok(())
4104}
4105
4106fn insert_mcp_replacement(
4107 replacements: &mut BTreeMap<bamboo_config::CredentialRef, String>,
4108 reference: bamboo_config::CredentialRef,
4109 value: &str,
4110) -> Result<(), String> {
4111 match replacements.get(&reference) {
4112 Some(existing) if existing != value => {
4113 Err("MCP updates assign conflicting values to one credential reference".to_string())
4114 }
4115 Some(_) => Ok(()),
4116 None => {
4117 replacements.insert(reference, value.to_string());
4118 Ok(())
4119 }
4120 }
4121}
4122
4123fn apply_mcp_header_replacements(
4124 headers: &mut [bamboo_mcp::HeaderConfig],
4125 replacements: &BTreeMap<bamboo_config::CredentialRef, String>,
4126) -> Result<(), String> {
4127 for header in headers {
4128 let Some(raw_reference) = header.credential_ref.as_ref() else {
4129 continue;
4130 };
4131 let reference = bamboo_config::CredentialRef::parse(raw_reference.clone())
4132 .map_err(|_| "MCP credential reference is invalid".to_string())?;
4133 if let Some(value) = replacements.get(&reference) {
4134 header.value = value.clone();
4135 }
4136 }
4137 Ok(())
4138}
4139
4140fn mcp_credential_ref_is_touched(
4141 raw_reference: Option<&String>,
4142 touched: &BTreeSet<bamboo_config::CredentialRef>,
4143) -> bool {
4144 raw_reference
4145 .and_then(|raw| bamboo_config::CredentialRef::parse(raw.clone()).ok())
4146 .is_some_and(|reference| touched.contains(&reference))
4147}
4148
4149fn retain_mcp_header_credentials(
4150 current: &[bamboo_mcp::HeaderConfig],
4151 candidate: &mut [bamboo_mcp::HeaderConfig],
4152) {
4153 for candidate_header in candidate {
4154 let Some(current_header) = current
4155 .iter()
4156 .find(|header| header.name == candidate_header.name)
4157 else {
4158 continue;
4159 };
4160 if candidate_header.credential_ref == current_header.credential_ref
4161 && candidate_header.value.is_empty()
4162 {
4163 candidate_header.value = current_header.value.clone();
4164 candidate_header.value_encrypted = current_header.value_encrypted.clone();
4165 }
4166 }
4167}
4168
4169fn credential_ref_mcp_document(
4170 runtime: &McpConfig,
4171) -> Result<McpConfig, ConfigSectionMutationError> {
4172 let mut document = runtime.clone();
4173 for server in &mut document.servers {
4174 match &mut server.transport {
4175 TransportConfig::Stdio(config) => {
4176 config.env_encrypted.clear();
4177 config.env.retain(|name, value| {
4178 !(value.is_empty() || config.env_credential_refs.contains_key(name))
4179 });
4180 if !config.env.is_empty() {
4181 return Err(ConfigSectionMutationError::Invalid(
4182 "MCP secret requires a credential reference".to_string(),
4183 ));
4184 }
4185 }
4186 TransportConfig::Sse(config) => reference_headers(&mut config.headers)?,
4187 TransportConfig::StreamableHttp(config) => reference_headers(&mut config.headers)?,
4188 }
4189 }
4190 Ok(document)
4191}
4192
4193fn retain_mcp_credential_refs(document: &McpConfig, runtime: &mut McpConfig) {
4194 for runtime_server in &mut runtime.servers {
4195 let Some(document_server) = document
4196 .servers
4197 .iter()
4198 .find(|server| server.id == runtime_server.id)
4199 else {
4200 continue;
4201 };
4202 match (&document_server.transport, &mut runtime_server.transport) {
4203 (TransportConfig::Stdio(document), TransportConfig::Stdio(runtime)) => {
4204 runtime.env_encrypted.clear();
4205 runtime.env_credential_refs = document.env_credential_refs.clone();
4206 }
4207 (TransportConfig::Sse(document), TransportConfig::Sse(runtime)) => {
4208 copy_header_ciphertext(&document.headers, &mut runtime.headers);
4209 }
4210 (
4211 TransportConfig::StreamableHttp(document),
4212 TransportConfig::StreamableHttp(runtime),
4213 ) => copy_header_ciphertext(&document.headers, &mut runtime.headers),
4214 _ => {}
4215 }
4216 }
4217}
4218
4219fn copy_header_ciphertext(
4220 document: &[bamboo_mcp::HeaderConfig],
4221 runtime: &mut [bamboo_mcp::HeaderConfig],
4222) {
4223 for runtime_header in runtime {
4224 if let Some(document_header) = document
4225 .iter()
4226 .find(|header| header.name == runtime_header.name)
4227 {
4228 runtime_header.value_encrypted = None;
4229 runtime_header.credential_ref = document_header.credential_ref.clone();
4230 }
4231 }
4232}
4233
4234fn reference_headers(
4235 headers: &mut [bamboo_mcp::HeaderConfig],
4236) -> Result<(), ConfigSectionMutationError> {
4237 for header in headers {
4238 if !header.value.is_empty() && header.credential_ref.is_none() {
4239 return Err(ConfigSectionMutationError::Invalid(
4240 "MCP secret requires a credential reference".to_string(),
4241 ));
4242 }
4243 header.value.clear();
4244 header.value_encrypted = None;
4245 }
4246 Ok(())
4247}
4248
4249fn mcp_credential_refs(
4257 config: &McpConfig,
4258) -> Result<BTreeSet<bamboo_config::CredentialRef>, ConfigSectionMutationError> {
4259 let mut references = BTreeSet::new();
4260 for server in &config.servers {
4261 match &server.transport {
4262 TransportConfig::Stdio(stdio) => {
4263 for raw in stdio.env_credential_refs.values() {
4264 references.insert(bamboo_config::CredentialRef::parse(raw.clone()).map_err(
4265 |_| {
4266 ConfigSectionMutationError::Invalid(
4267 "MCP credential reference is invalid".to_string(),
4268 )
4269 },
4270 )?);
4271 }
4272 }
4273 TransportConfig::Sse(http) => collect_mcp_header_refs(&http.headers, &mut references)?,
4274 TransportConfig::StreamableHttp(http) => {
4275 collect_mcp_header_refs(&http.headers, &mut references)?
4276 }
4277 }
4278 }
4279 Ok(references)
4280}
4281
4282fn collect_mcp_header_refs(
4283 headers: &[bamboo_mcp::HeaderConfig],
4284 output: &mut BTreeSet<bamboo_config::CredentialRef>,
4285) -> Result<(), ConfigSectionMutationError> {
4286 for raw in headers
4287 .iter()
4288 .filter_map(|header| header.credential_ref.as_ref())
4289 {
4290 output.insert(
4291 bamboo_config::CredentialRef::parse(raw.clone()).map_err(|_| {
4292 ConfigSectionMutationError::Invalid(
4293 "MCP credential reference is invalid".to_string(),
4294 )
4295 })?,
4296 );
4297 }
4298 Ok(())
4299}
4300
4301fn normalize_legacy_mcp_credentials(
4302 current: &McpConfig,
4303 candidate: &mut McpConfig,
4304) -> Result<BTreeSet<bamboo_config::CredentialRef>, AppError> {
4305 let current_refs = mcp_credential_refs(current).map_err(map_mcp_section_error)?;
4306 let mut intents = BTreeSet::new();
4307
4308 for candidate_server in &mut candidate.servers {
4309 let current_server = current
4310 .servers
4311 .iter()
4312 .find(|server| server.id == candidate_server.id);
4313 match &mut candidate_server.transport {
4314 TransportConfig::Stdio(candidate_stdio) => {
4315 if !candidate_stdio.env_encrypted.is_empty() {
4316 return Err(AppError::BadRequest(
4317 "MCP ciphertext is server-managed and cannot be supplied".to_string(),
4318 ));
4319 }
4320 let current_stdio = current_server.and_then(|server| match &server.transport {
4321 TransportConfig::Stdio(stdio) => Some(stdio),
4322 _ => None,
4323 });
4324 for (name, incoming_reference) in &candidate_stdio.env_credential_refs {
4325 let current_reference = current_stdio
4326 .and_then(|stdio| stdio.env_credential_refs.get(name))
4327 .map(String::as_str);
4328 if current_reference != Some(incoming_reference.as_str()) {
4329 return Err(AppError::BadRequest(
4330 "MCP credential references are server-managed and cannot be supplied"
4331 .to_string(),
4332 ));
4333 }
4334 }
4335 let incoming = std::mem::take(&mut candidate_stdio.env);
4336 candidate_stdio.env_credential_refs.clear();
4337 for (name, value) in incoming {
4338 let current_reference = current_stdio
4339 .and_then(|stdio| stdio.env_credential_refs.get(&name))
4340 .map(|raw| {
4341 bamboo_config::CredentialRef::parse(raw.clone()).map_err(|_| {
4342 AppError::BadRequest(
4343 "MCP credential reference is invalid".to_string(),
4344 )
4345 })
4346 })
4347 .transpose()?;
4348 let current_value = current_stdio.and_then(|stdio| stdio.env.get(&name));
4349 let (value, reference, touched) =
4350 if bamboo_config::patch::is_masked_api_key(&value) {
4351 let reference = current_reference.ok_or_else(|| {
4352 AppError::BadRequest(
4353 "masked MCP credential has no existing value".to_string(),
4354 )
4355 })?;
4356 let value = current_value.cloned().ok_or_else(|| {
4357 AppError::BadRequest(
4358 "referenced MCP credential is unavailable".to_string(),
4359 )
4360 })?;
4361 (value, reference, false)
4362 } else if value.is_empty() {
4363 if let Some(reference) = current_reference {
4364 intents.insert(reference);
4365 }
4366 continue;
4367 } else {
4368 let reference = current_reference.map_or_else(
4369 || {
4370 bamboo_config::credential_ref(
4371 "mcp",
4372 &candidate_server.id,
4373 &format!("env_{name}"),
4374 )
4375 .map_err(map_exact_credential_store_error)
4376 },
4377 Ok,
4378 )?;
4379 let touched = current_value != Some(&value);
4380 (value, reference, touched)
4381 };
4382 if touched {
4383 intents.insert(reference.clone());
4384 }
4385 candidate_stdio.env.insert(name.clone(), value);
4386 candidate_stdio
4387 .env_credential_refs
4388 .insert(name, reference.as_str().to_string());
4389 }
4390 }
4391 TransportConfig::Sse(candidate_http) => normalize_legacy_mcp_headers(
4392 &candidate_server.id,
4393 current_server.and_then(|server| match &server.transport {
4394 TransportConfig::Sse(http) => Some(http.headers.as_slice()),
4395 _ => None,
4396 }),
4397 &mut candidate_http.headers,
4398 &mut intents,
4399 )?,
4400 TransportConfig::StreamableHttp(candidate_http) => normalize_legacy_mcp_headers(
4401 &candidate_server.id,
4402 current_server.and_then(|server| match &server.transport {
4403 TransportConfig::StreamableHttp(http) => Some(http.headers.as_slice()),
4404 _ => None,
4405 }),
4406 &mut candidate_http.headers,
4407 &mut intents,
4408 )?,
4409 }
4410 }
4411
4412 let candidate_refs = mcp_credential_refs(candidate).map_err(map_mcp_section_error)?;
4413 intents.extend(current_refs.symmetric_difference(&candidate_refs).cloned());
4414 Ok(intents)
4415}
4416
4417fn normalize_legacy_mcp_headers(
4418 server_id: &str,
4419 current: Option<&[bamboo_mcp::HeaderConfig]>,
4420 candidate: &mut [bamboo_mcp::HeaderConfig],
4421 intents: &mut BTreeSet<bamboo_config::CredentialRef>,
4422) -> Result<(), AppError> {
4423 for header in candidate {
4424 if header.value_encrypted.is_some() {
4425 return Err(AppError::BadRequest(
4426 "MCP ciphertext is server-managed and cannot be supplied".to_string(),
4427 ));
4428 }
4429 let current_header =
4430 current.and_then(|headers| headers.iter().find(|current| current.name == header.name));
4431 if header.credential_ref.as_deref().is_some_and(|incoming| {
4432 current_header.and_then(|current| current.credential_ref.as_deref()) != Some(incoming)
4433 }) {
4434 return Err(AppError::BadRequest(
4435 "MCP credential references are server-managed and cannot be supplied".to_string(),
4436 ));
4437 }
4438 let current_reference = current_header
4439 .and_then(|current| current.credential_ref.as_ref())
4440 .map(|raw| {
4441 bamboo_config::CredentialRef::parse(raw.clone()).map_err(|_| {
4442 AppError::BadRequest("MCP credential reference is invalid".to_string())
4443 })
4444 })
4445 .transpose()?;
4446 let current_value = current_header.map(|current| ¤t.value);
4447 if bamboo_config::patch::is_masked_api_key(&header.value) {
4448 let reference = current_reference.ok_or_else(|| {
4449 AppError::BadRequest("masked MCP credential has no existing value".to_string())
4450 })?;
4451 header.value = current_value.cloned().ok_or_else(|| {
4452 AppError::BadRequest("referenced MCP credential is unavailable".to_string())
4453 })?;
4454 header.credential_ref = Some(reference.as_str().to_string());
4455 } else if header.value.is_empty() {
4456 if let Some(reference) = current_reference {
4457 intents.insert(reference);
4458 }
4459 header.credential_ref = None;
4460 } else {
4461 let reference = current_reference.map_or_else(
4462 || {
4463 bamboo_config::credential_ref(
4464 "mcp",
4465 server_id,
4466 &format!("header_{}", header.name),
4467 )
4468 .map_err(map_exact_credential_store_error)
4469 },
4470 Ok,
4471 )?;
4472 if current_value != Some(&header.value) {
4473 intents.insert(reference.clone());
4474 }
4475 header.credential_ref = Some(reference.as_str().to_string());
4476 }
4477 header.value_encrypted = None;
4478 }
4479 Ok(())
4480}
4481
4482fn map_mcp_section_error(error: ConfigSectionMutationError) -> AppError {
4483 match error {
4484 ConfigSectionMutationError::Store(error) => map_exact_credential_store_error(error),
4485 ConfigSectionMutationError::Invalid(message)
4486 | ConfigSectionMutationError::Runtime(message) => AppError::BadRequest(message),
4487 }
4488}
4489
4490impl AppState {
4491 pub async fn reload_provider(&self) -> Result<(), bamboo_llm::LLMError> {
4523 let _io = self.config_io_lock.lock().await;
4528 let config = self.config.read().await.clone();
4529 let candidate_registry =
4530 bamboo_llm::ProviderRegistry::from_config(&config, self.app_data_dir.clone()).await?;
4531 let default_provider_name = candidate_registry.default_provider_name();
4532 tracing::info!(
4533 default_provider = %default_provider_name,
4534 legacy_provider = %config.provider,
4535 has_provider_instances = config.has_provider_instances(),
4536 "Reloading provider runtime from current config"
4537 );
4538
4539 let new_provider = candidate_registry.get_default().ok_or_else(|| {
4540 let message = if config.has_provider_instances() {
4541 format!(
4542 "Default provider instance '{}' is not available or failed to initialize",
4543 default_provider_name
4544 )
4545 } else {
4546 format!(
4547 "Provider '{}' is not available or failed to initialize",
4548 config.provider
4549 )
4550 };
4551 bamboo_llm::LLMError::Auth(message)
4552 })?;
4553
4554 #[cfg(test)]
4555 run_generic_before_provider_publish_test_hook(&self.app_data_dir);
4556 let mut provider = self.provider.write().await;
4557 self.provider_registry.replace_with(candidate_registry);
4558 *provider = new_provider;
4559
4560 tracing::info!(
4561 default_provider = %default_provider_name,
4562 "Provider reloaded successfully"
4563 );
4564 Ok(())
4565 }
4566
4567 pub async fn reload_config(&self) -> Config {
4597 let _io = self.config_io_lock.lock().await;
4602 let mut config = self.config.write().await;
4603 let mut new_config = self
4604 .config_facade
4605 .as_ref()
4606 .map(|facade| load_facade_effective_config(facade, &self.app_data_dir))
4607 .unwrap_or_else(|| {
4608 Config::from_data_dir_without_publish(Some(self.app_data_dir.clone()))
4609 });
4610 preserve_runtime_broker(&mut new_config, &config);
4611 new_config.publish_env_vars();
4612 *config = new_config.clone();
4613 new_config
4614 }
4615
4616 pub async fn reload_config_and_runtime(&self) -> Result<Config, AppError> {
4624 let io = self.config_io_lock.clone().lock_owned().await;
4625 let app_data_dir = self.app_data_dir.clone();
4626 let config_facade = self.config_facade.clone();
4627 let config = self.config.clone();
4628 let provider_registry = self.provider_registry.clone();
4629 let provider = self.provider.clone();
4630 let mcp_manager = self.mcp_manager.clone();
4631 let account_sink = self.account_sink.clone();
4632 let config_live_health = self.config_live_health.clone();
4633 let mcp_config_live_health = self.mcp_config_live_health.clone();
4634 let transaction = tokio::spawn(async move {
4635 let _io = io;
4636 let mut new_config = config_facade
4637 .as_ref()
4638 .map(|facade| load_facade_effective_config(facade, &app_data_dir))
4639 .unwrap_or_else(|| {
4640 Config::from_data_dir_without_publish(Some(app_data_dir.clone()))
4641 });
4642 {
4643 let previous = config.read().await;
4644 preserve_runtime_broker(&mut new_config, &previous);
4645 }
4646 if let Err(error) = bamboo_llm::validate_provider_config(&new_config) {
4647 tracing::warn!("reloaded provider config is invalid");
4648 let message =
4649 "provider configuration is invalid; retaining last-known-good generation"
4650 .to_string();
4651 if let Some(facade) = config_facade.as_ref() {
4652 if let Some(event) = facade
4653 .registry()
4654 .mark_runtime_degraded(SectionId::Providers, message.clone())
4655 {
4656 publish_registry_event(&account_sink, &event).await;
4657 }
4658 }
4659 publish_section_failure(
4660 &config_live_health,
4661 &account_sink,
4662 "providers",
4663 SectionStatus::Invalid,
4664 message,
4665 )
4666 .await;
4667 return Err(AppError::BadRequest(format!(
4668 "Invalid configuration: {error}"
4669 )));
4670 }
4671 new_config.publish_env_vars();
4672 *config.write().await = new_config.clone();
4673 Self::apply_config_effects_owned(
4674 new_config.clone(),
4675 ConfigUpdateEffects {
4676 reload_provider: bamboo_config::patch::ReloadMode::Strict,
4677 reconcile_mcp: bamboo_config::patch::ReloadMode::Strict,
4678 },
4679 ConfigRuntimeEffectContext {
4680 app_data_dir,
4681 config_facade,
4682 provider_registry,
4683 provider,
4684 mcp_manager,
4685 account_sink,
4686 config_live_health,
4687 mcp_config_live_health,
4688 },
4689 )
4690 .await?;
4691 Ok::<_, AppError>(new_config)
4692 });
4693 transaction.await.map_err(|error| {
4694 AppError::InternalError(anyhow::anyhow!(
4695 "config/runtime reload transaction task failed: {error}"
4696 ))
4697 })?
4698 }
4699
4700 pub async fn reset_legacy_config_and_runtime(&self) -> Result<Config, AppError> {
4713 if self.config_facade.is_some() {
4714 return Err(AppError::BadRequest(
4715 "full config reset spans multiple revisioned sections and is disabled without a recoverable manifest; reset sections individually through the typed section API"
4716 .to_string(),
4717 ));
4718 }
4719
4720 let io = self.config_io_lock.clone().lock_owned().await;
4721 let app_data_dir = self.app_data_dir.clone();
4722 let config = self.config.clone();
4723 let provider_registry = self.provider_registry.clone();
4724 let provider = self.provider.clone();
4725 let mcp_manager = self.mcp_manager.clone();
4726 let account_sink = self.account_sink.clone();
4727 let config_live_health = self.config_live_health.clone();
4728 let mcp_config_live_health = self.mcp_config_live_health.clone();
4729 let transaction = tokio::spawn(async move {
4730 let _io = io;
4731 let mut deletion_error = None;
4732 for path in [
4733 app_data_dir.join("config.json"),
4734 app_data_dir.join("model_limits.json"),
4735 app_data_dir.join("connect.json"),
4736 app_data_dir.join("connect.json.bak"),
4737 ] {
4738 let result = match tokio::fs::try_exists(&path).await {
4739 Ok(true) => tokio::fs::remove_file(&path).await,
4740 Ok(false) => Ok(()),
4741 Err(error) => Err(error),
4742 };
4743 if let Err(error) = result {
4744 tracing::warn!(
4745 file = %path.display(),
4746 "failed to delete one legacy config artifact during reset"
4747 );
4748 deletion_error.get_or_insert(error);
4749 }
4750 }
4751 #[cfg(test)]
4752 run_reset_after_delete_test_hook(&app_data_dir);
4753
4754 let mut new_config = Config::from_data_dir_without_publish(Some(app_data_dir.clone()));
4755 {
4756 let previous = config.read().await;
4757 preserve_runtime_broker(&mut new_config, &previous);
4758 }
4759 new_config.publish_env_vars();
4760 *config.write().await = new_config.clone();
4761 Self::apply_config_effects_owned(
4762 new_config.clone(),
4763 ConfigUpdateEffects {
4764 reload_provider: bamboo_config::patch::ReloadMode::BestEffort,
4765 reconcile_mcp: bamboo_config::patch::ReloadMode::BestEffort,
4766 },
4767 ConfigRuntimeEffectContext {
4768 app_data_dir,
4769 config_facade: None,
4770 provider_registry,
4771 provider,
4772 mcp_manager,
4773 account_sink,
4774 config_live_health,
4775 mcp_config_live_health,
4776 },
4777 )
4778 .await?;
4779 match deletion_error {
4780 Some(error) => Err(AppError::StorageError(error)),
4781 None => Ok(new_config),
4782 }
4783 });
4784 transaction.await.map_err(|error| {
4785 AppError::InternalError(anyhow::anyhow!(
4786 "legacy config reset transaction task failed: {error}"
4787 ))
4788 })?
4789 }
4790
4791 async fn persist_config_snapshot(
4792 data_dir: PathBuf,
4793 config_facade: Option<Arc<bamboo_config::ConfigFacade>>,
4794 config: Config,
4795 ) -> Result<Option<bamboo_config::FacadeConfigCommit>, AppError> {
4796 if let Some(facade) = config_facade {
4797 tokio::task::spawn_blocking(move || {
4798 let result = bamboo_config::persist_facade_effective_config_with_adoption(
4799 &data_dir,
4800 &config,
4801 facade.as_ref(),
4802 );
4803 #[cfg(test)]
4804 if result.is_ok() {
4805 run_generic_before_event_test_hook(&data_dir);
4806 }
4807 result
4808 })
4809 .await
4810 .map_err(|error| {
4811 AppError::InternalError(anyhow::anyhow!("Config save task failed: {error}"))
4812 })?
4813 .map(Some)
4814 .map_err(map_exact_credential_store_error)
4815 } else {
4816 tokio::task::spawn_blocking(move || {
4817 let result = config.save_to_dir(data_dir.clone());
4818 #[cfg(test)]
4819 if result.is_ok() {
4820 run_generic_before_event_test_hook(&data_dir);
4821 }
4822 result
4823 })
4824 .await
4825 .map_err(|error| {
4826 AppError::InternalError(anyhow::anyhow!("Config save task failed: {error}"))
4827 })?
4828 .map_err(|error| {
4829 AppError::InternalError(anyhow::anyhow!("Failed to save config: {error}"))
4830 })?;
4831 Ok(None)
4832 }
4833 }
4834
4835 pub async fn update_config<F>(
4842 &self,
4843 update: F,
4844 effects: ConfigUpdateEffects,
4845 ) -> Result<Config, AppError>
4846 where
4847 F: FnOnce(&mut Config) -> Result<(), AppError>,
4848 {
4849 self.update_config_with_forced_mcp_replacements(update, effects, HashSet::new())
4850 .await
4851 }
4852
4853 pub(crate) async fn update_config_with_forced_mcp_replacements<F>(
4854 &self,
4855 update: F,
4856 effects: ConfigUpdateEffects,
4857 forced_mcp_replacements: HashSet<String>,
4858 ) -> Result<Config, AppError>
4859 where
4860 F: FnOnce(&mut Config) -> Result<(), AppError>,
4861 {
4862 let io = self.config_io_lock.clone().lock_owned().await;
4866 let (mut snapshot, live_base, enforcement_newly_off) = {
4867 let cfg = self.config.read().await;
4868 reject_if_recovery_pending(&cfg)?;
4874 let was_off = cfg.plugin_trust.enforcement_is_off();
4875 let live_base = cfg.clone();
4876 let mut candidate = cfg.clone();
4877 restore_authoritative_cluster_fabric(self.config_facade.as_ref(), &mut candidate);
4878 update(&mut candidate)?;
4879 restore_authoritative_cluster_fabric(self.config_facade.as_ref(), &mut candidate);
4881 if self.config_facade.is_none() {
4882 candidate.assign_connect_platform_ids();
4883 candidate.refresh_encrypted_secrets().map_err(|e| {
4884 AppError::InternalError(anyhow::anyhow!(
4885 "Failed to refresh encrypted secrets: {e}"
4886 ))
4887 })?;
4888 }
4889 let newly_off = !was_off && candidate.plugin_trust.enforcement_is_off();
4890 (candidate, live_base, newly_off)
4891 };
4892 if enforcement_newly_off {
4893 warn_plugin_trust_enforcement_off();
4894 }
4895 let config = self.config.clone();
4896 let app_data_dir = self.app_data_dir.clone();
4897 let config_facade = self.config_facade.clone();
4898 let account_sink = self.account_sink.clone();
4899 let provider_registry = self.provider_registry.clone();
4900 let provider = self.provider.clone();
4901 let mcp_manager = self.mcp_manager.clone();
4902 let config_live_health = self.config_live_health.clone();
4903 let mcp_config_live_health = self.mcp_config_live_health.clone();
4904 let transaction = tokio::spawn(async move {
4908 let snapshot = {
4915 let _io = io;
4916 let commit = Self::persist_config_snapshot(
4917 app_data_dir.clone(),
4918 config_facade.clone(),
4919 snapshot.clone(),
4920 )
4921 .await?;
4922 let events = match commit {
4923 Some(commit) => {
4924 let mut published = live_base;
4925 let events =
4926 install_facade_config_commit(commit, &mut published).map_err(|e| {
4927 AppError::InternalError(anyhow::anyhow!(
4928 "failed to install committed configuration section: {e}"
4929 ))
4930 })?;
4931 snapshot = published;
4932 events
4933 }
4934 None => Vec::new(),
4935 };
4936 {
4937 let mut cfg = config.write().await;
4938 preserve_runtime_broker(&mut snapshot, &cfg);
4939 snapshot.publish_env_vars();
4940 *cfg = snapshot.clone();
4941 }
4942 publish_exact_facade_events(&account_sink, &events).await?;
4946 Self::apply_config_effects_owned_after_forcing(
4947 snapshot.clone(),
4948 effects,
4949 ConfigRuntimeEffectContext {
4950 app_data_dir,
4951 config_facade,
4952 provider_registry,
4953 provider,
4954 mcp_manager,
4955 account_sink,
4956 config_live_health,
4957 mcp_config_live_health,
4958 },
4959 forced_mcp_replacements,
4960 )
4961 .await?;
4962 snapshot
4963 };
4964 Ok::<_, AppError>(snapshot)
4965 });
4966 transaction.await.map_err(|error| {
4967 AppError::InternalError(anyhow::anyhow!(
4968 "config update transaction task failed: {error}"
4969 ))
4970 })?
4971 }
4972
4973 pub async fn update_config_with_provider_credentials<F>(
4976 &self,
4977 update: F,
4978 provider_intents: std::collections::BTreeSet<String>,
4979 provider_instance_intents: std::collections::BTreeSet<String>,
4980 effects: ConfigUpdateEffects,
4981 ) -> Result<Config, AppError>
4982 where
4983 F: FnOnce(&mut Config) -> Result<(), AppError>,
4984 {
4985 self.update_config_with_provider_credentials_inner(
4986 update,
4987 provider_intents,
4988 provider_instance_intents,
4989 effects,
4990 false,
4991 )
4992 .await
4993 }
4994
4995 pub(crate) async fn update_provider_metadata<F>(
4999 &self,
5000 update: F,
5001 effects: ConfigUpdateEffects,
5002 ) -> Result<Config, AppError>
5003 where
5004 F: FnOnce(&mut Config) -> Result<(), AppError>,
5005 {
5006 self.update_config_with_provider_credentials_inner(
5007 update,
5008 BTreeSet::new(),
5009 BTreeSet::new(),
5010 effects,
5011 true,
5012 )
5013 .await
5014 }
5015
5016 async fn update_config_with_provider_credentials_inner<F>(
5017 &self,
5018 update: F,
5019 provider_intents: BTreeSet<String>,
5020 provider_instance_intents: BTreeSet<String>,
5021 effects: ConfigUpdateEffects,
5022 exact_provider_metadata: bool,
5023 ) -> Result<Config, AppError>
5024 where
5025 F: FnOnce(&mut Config) -> Result<(), AppError>,
5026 {
5027 let exact_provider_metadata = exact_provider_metadata
5028 && provider_intents.is_empty()
5029 && provider_instance_intents.is_empty()
5030 && self.config_facade.is_some();
5031 if provider_intents.is_empty()
5032 && provider_instance_intents.is_empty()
5033 && !exact_provider_metadata
5034 {
5035 return self.update_config(update, effects).await;
5036 }
5037 let io = self.config_io_lock.clone().lock_owned().await;
5038 let config_facade = self.config_facade.clone();
5039 let (mut candidate, live_base, enforcement_newly_off, provider_expected_revision) = {
5040 let cfg = self.config.read().await;
5041 reject_if_recovery_pending(&cfg)?;
5042 let was_off = cfg.plugin_trust.enforcement_is_off();
5043 let live_base = cfg.clone();
5044 let mut candidate = cfg.clone();
5045 restore_authoritative_cluster_fabric(config_facade.as_ref(), &mut candidate);
5046 update(&mut candidate)?;
5047 restore_authoritative_cluster_fabric(config_facade.as_ref(), &mut candidate);
5049 let mut non_provider_candidate = candidate.clone();
5054 apply_runtime_section(SectionId::Providers, &cfg, &mut non_provider_candidate);
5055 let mut comparison_base = cfg.clone();
5056 restore_authoritative_cluster_fabric(config_facade.as_ref(), &mut comparison_base);
5057 let mut changed =
5058 bamboo_config::changed_facade_sections(&comparison_base, &non_provider_candidate)
5059 .map_err(|_| {
5060 AppError::InternalError(anyhow::anyhow!(
5061 "failed to compare modular configuration sections"
5062 ))
5063 })?;
5064 if serde_json::to_value(cfg.subagents()).ok()
5065 == serde_json::to_value(candidate.subagents()).ok()
5066 {
5067 changed.retain(|section| *section != SectionId::Subagents);
5068 }
5069 if let Some(other) = changed
5070 .into_iter()
5071 .find(|section| *section != SectionId::Providers)
5072 {
5073 return Err(AppError::BadRequest(format!(
5074 "provider credential updates cannot be combined with {} changes; split the request",
5075 other.descriptor().name
5076 )));
5077 }
5078 if config_facade.is_none() {
5079 candidate.assign_connect_platform_ids();
5080 candidate.refresh_encrypted_secrets().map_err(|error| {
5081 AppError::InternalError(anyhow::anyhow!(
5082 "Failed to refresh encrypted secrets: {error}"
5083 ))
5084 })?;
5085 }
5086 let newly_off = !was_off && candidate.plugin_trust.enforcement_is_off();
5087 let provider_expected_revision = exact_provider_metadata.then(|| {
5088 config_facade
5089 .as_ref()
5090 .expect("exact provider metadata requires the modular facade")
5091 .registry()
5092 .providers
5093 .snapshot()
5094 .revision
5095 });
5096 (candidate, live_base, newly_off, provider_expected_revision)
5097 };
5098 let config = self.config.clone();
5099 let app_data_dir = self.app_data_dir.clone();
5100 let account_sink = self.account_sink.clone();
5101 let provider_registry = self.provider_registry.clone();
5102 let provider = self.provider.clone();
5103 let mcp_manager = self.mcp_manager.clone();
5104 let config_live_health = self.config_live_health.clone();
5105 let mcp_config_live_health = self.mcp_config_live_health.clone();
5106 let transaction = tokio::spawn(async move {
5107 let snapshot = {
5108 let _io = io;
5109 let data_dir = app_data_dir.clone();
5110 let commit_facade = config_facade.clone();
5111 let (candidate, commit) = tokio::task::spawn_blocking(move || {
5112 let result = if let Some(facade) = commit_facade {
5113 let commit = if let Some(expected_revision) = provider_expected_revision {
5114 bamboo_config::persist_provider_credential_transaction_at_revision_with_adoption(
5115 &data_dir,
5116 &mut candidate,
5117 &provider_intents,
5118 &provider_instance_intents,
5119 expected_revision,
5120 facade.as_ref(),
5121 )?
5122 } else {
5123 bamboo_config::persist_provider_instance_credential_transaction_with_adoption(
5124 &data_dir,
5125 &mut candidate,
5126 &provider_intents,
5127 &provider_instance_intents,
5128 facade.as_ref(),
5129 )?
5130 };
5131 Ok::<_, ConfigStoreError>((candidate, Some(commit)))
5132 } else {
5133 bamboo_config::persist_provider_instance_credential_transaction(
5134 &data_dir,
5135 &mut candidate,
5136 &provider_intents,
5137 &provider_instance_intents,
5138 )?;
5139 Ok((load_committed_effective_config(&data_dir)?, None))
5140 };
5141 #[cfg(test)]
5142 run_generic_before_event_test_hook(&data_dir);
5143 result
5144 })
5145 .await
5146 .map_err(|error| {
5147 AppError::InternalError(anyhow::anyhow!(
5148 "provider credential transaction task failed: {error}"
5149 ))
5150 })?
5151 .map_err(|error| match error {
5152 ConfigStoreError::Conflict { expected, actual } => {
5153 AppError::ConfigConflict { expected, actual }
5154 }
5155 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
5156 ConfigStoreError::CommitIndeterminate(message) => AppError::InternalError(
5157 anyhow::anyhow!("configuration commit outcome is indeterminate: {message}"),
5158 ),
5159 ConfigStoreError::Io(error) => AppError::StorageError(error),
5160 ConfigStoreError::Json(_) => {
5161 AppError::BadRequest("configuration document is invalid".to_string())
5162 }
5163 ConfigStoreError::Watch(error) => AppError::InternalError(anyhow::anyhow!(
5164 "configuration watch failed: {error}"
5165 )),
5166 })?;
5167 let (mut snapshot, events) = match commit {
5168 Some(commit) => {
5169 let mut published = live_base;
5170 let installed = install_credential_section_commit(commit, &mut published)
5171 .map_err(|error| {
5172 AppError::InternalError(anyhow::anyhow!(
5173 "provider process adoption failed: {error}"
5174 ))
5175 })?;
5176 (published, installed.events)
5177 }
5178 None => (candidate, Vec::new()),
5179 };
5180 {
5181 let mut cfg = config.write().await;
5182 preserve_runtime_broker(&mut snapshot, &cfg);
5183 snapshot.publish_env_vars();
5184 *cfg = snapshot.clone();
5185 }
5186 publish_exact_facade_events(&account_sink, &events).await?;
5187 if enforcement_newly_off {
5188 warn_plugin_trust_enforcement_off();
5189 }
5190 Self::apply_config_effects_owned(
5191 snapshot.clone(),
5192 effects,
5193 ConfigRuntimeEffectContext {
5194 app_data_dir,
5195 config_facade,
5196 provider_registry,
5197 provider,
5198 mcp_manager,
5199 account_sink,
5200 config_live_health,
5201 mcp_config_live_health,
5202 },
5203 )
5204 .await?;
5205 snapshot
5206 };
5207 Ok::<_, AppError>(snapshot)
5208 });
5209 transaction.await.map_err(|error| {
5210 AppError::InternalError(anyhow::anyhow!(
5211 "provider config transaction task failed: {error}"
5212 ))
5213 })?
5214 }
5215
5216 pub async fn update_env_var_credentials<F>(
5220 &self,
5221 expected_revision: u64,
5222 mut env_intents: std::collections::BTreeSet<String>,
5223 full_replace: bool,
5224 update: F,
5225 ) -> Result<
5226 (
5227 Config,
5228 u64,
5229 bamboo_config::CredentialSectionRuntimeMetadata,
5230 Option<bamboo_config::SectionEnvelope<Value>>,
5231 ),
5232 AppError,
5233 >
5234 where
5235 F: FnOnce(&mut Config) -> Result<(), AppError> + Send + 'static,
5236 {
5237 let config_io_lock = self.config_io_lock.clone();
5238 let config = self.config.clone();
5239 let app_data_dir = self.app_data_dir.clone();
5240 let account_sink = self.account_sink.clone();
5241 let config_facade = self.config_facade.clone();
5242 let transaction = tokio::spawn(async move {
5243 let _io = config_io_lock.lock().await;
5244 let live_base = {
5245 let current = config.read().await;
5246 reject_if_recovery_pending(¤t)?;
5247 current.clone()
5248 };
5249 let mut candidate = live_base.clone();
5250 if config_facade.is_some() {
5251 install_exact_credential_section_mutation_base(
5252 app_data_dir.clone(),
5253 SectionId::Env,
5254 expected_revision,
5255 &mut candidate,
5256 )
5257 .await?;
5258 }
5259 if full_replace {
5260 env_intents.extend(candidate.env_vars.iter().map(|entry| entry.name.clone()));
5261 }
5262 update(&mut candidate)?;
5263 if config_facade.is_none() {
5264 candidate.assign_connect_platform_ids();
5265 }
5266 let transaction_dir = app_data_dir.clone();
5267 let commit_facade = config_facade.clone();
5268 let (candidate, revision, commit) = tokio::task::spawn_blocking(move || {
5269 if let Some(facade) = commit_facade {
5270 let commit =
5271 bamboo_config::persist_env_var_credential_transaction_at_revision_with_adoption(
5272 &transaction_dir,
5273 &mut candidate,
5274 &env_intents,
5275 expected_revision,
5276 facade.as_ref(),
5277 )?;
5278 #[cfg(test)]
5279 run_credential_after_commit_before_live_test_hook(
5280 &transaction_dir,
5281 SectionId::Env,
5282 );
5283 let revision = commit.revision;
5284 Ok::<_, ConfigStoreError>((candidate, revision, Some(commit)))
5285 } else {
5286 let revision =
5287 bamboo_config::persist_env_var_credential_transaction_at_revision(
5288 &transaction_dir,
5289 &mut candidate,
5290 &env_intents,
5291 expected_revision,
5292 )?;
5293 Ok((
5294 load_committed_effective_config(&transaction_dir)?,
5295 revision,
5296 None,
5297 ))
5298 }
5299 })
5300 .await
5301 .map_err(|error| {
5302 AppError::InternalError(anyhow::anyhow!(
5303 "env credential transaction task failed: {error}"
5304 ))
5305 })?
5306 .map_err(|error| match error {
5307 ConfigStoreError::Conflict { expected, actual } => {
5308 AppError::ConfigConflict { expected, actual }
5309 }
5310 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
5311 ConfigStoreError::CommitIndeterminate(message) => AppError::InternalError(
5312 anyhow::anyhow!("configuration commit outcome is indeterminate: {message}"),
5313 ),
5314 ConfigStoreError::Io(error) => AppError::StorageError(error),
5315 ConfigStoreError::Json(_) => {
5316 AppError::BadRequest("configuration document is invalid".to_string())
5317 }
5318 ConfigStoreError::Watch(error) => {
5319 AppError::InternalError(anyhow::anyhow!("configuration watch failed: {error}"))
5320 }
5321 })?;
5322 let (published, installed) = match commit {
5323 Some(commit) => {
5324 let mut published = live_base;
5325 let installed = install_credential_section_commit(commit, &mut published)
5326 .map_err(|error| {
5327 AppError::InternalError(anyhow::anyhow!(
5328 "env process adoption failed: {error}"
5329 ))
5330 })?;
5331 (published, installed)
5332 }
5333 None => (
5334 candidate,
5335 InstalledCredentialSectionCommit {
5336 events: Vec::new(),
5337 metadata: read_credential_runtime_metadata(&app_data_dir).map_err(
5338 |error| {
5339 AppError::InternalError(anyhow::anyhow!(
5340 "env credential status unavailable after commit: {error}"
5341 ))
5342 },
5343 )?,
5344 section: None,
5345 },
5346 ),
5347 };
5348 published.publish_env_vars();
5349 *config.write().await = published.clone();
5350 publish_exact_facade_events(&account_sink, &installed.events).await?;
5351 let section = installed.section;
5352 Ok::<_, AppError>((published, revision, installed.metadata, section))
5353 });
5354 transaction.await.map_err(|error| {
5355 AppError::InternalError(anyhow::anyhow!(
5356 "env credential transaction task failed: {error}"
5357 ))
5358 })?
5359 }
5360
5361 pub async fn update_notification_credentials<F>(
5366 &self,
5367 expected_revision: u64,
5368 secret_intents: std::collections::BTreeSet<String>,
5369 reset_domain: bool,
5370 update: F,
5371 ) -> Result<
5372 (
5373 Config,
5374 u64,
5375 bamboo_config::CredentialSectionRuntimeMetadata,
5376 Option<bamboo_config::SectionEnvelope<Value>>,
5377 ),
5378 AppError,
5379 >
5380 where
5381 F: FnOnce(&mut Config) -> Result<(), AppError> + Send + 'static,
5382 {
5383 let config_io_lock = self.config_io_lock.clone();
5384 let config = self.config.clone();
5385 let app_data_dir = self.app_data_dir.clone();
5386 let account_sink = self.account_sink.clone();
5387 let config_facade = self.config_facade.clone();
5388 let transaction = tokio::spawn(async move {
5389 let _io = config_io_lock.lock().await;
5390 let live_base = {
5391 let current = config.read().await;
5392 reject_if_recovery_pending(¤t)?;
5393 current.clone()
5394 };
5395 let mut candidate = live_base.clone();
5396 if config_facade.is_some() {
5397 install_exact_credential_section_mutation_base(
5398 app_data_dir.clone(),
5399 SectionId::Notifications,
5400 expected_revision,
5401 &mut candidate,
5402 )
5403 .await?;
5404 }
5405 update(&mut candidate)?;
5406 let transaction_dir = app_data_dir.clone();
5407 let commit_facade = config_facade.clone();
5408 let (candidate, revision, commit) = tokio::task::spawn_blocking(move || {
5409 if let Some(facade) = commit_facade {
5410 let commit =
5411 bamboo_config::persist_notification_credential_transaction_at_revision_with_reset_and_adoption(
5412 &transaction_dir,
5413 &mut candidate,
5414 &secret_intents,
5415 reset_domain,
5416 expected_revision,
5417 facade.as_ref(),
5418 )?;
5419 let revision = commit.revision;
5420 Ok::<_, ConfigStoreError>((candidate, revision, Some(commit)))
5421 } else {
5422 let revision =
5423 bamboo_config::persist_notification_credential_transaction_at_revision_with_reset(
5424 &transaction_dir,
5425 &mut candidate,
5426 &secret_intents,
5427 reset_domain,
5428 expected_revision,
5429 )?;
5430 Ok((
5431 load_committed_effective_config(&transaction_dir)?,
5432 revision,
5433 None,
5434 ))
5435 }
5436 })
5437 .await
5438 .map_err(|error| {
5439 AppError::InternalError(anyhow::anyhow!(
5440 "notification credential transaction task failed: {error}"
5441 ))
5442 })?
5443 .map_err(|error| match error {
5444 ConfigStoreError::Conflict { expected, actual } => {
5445 AppError::ConfigConflict { expected, actual }
5446 }
5447 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
5448 ConfigStoreError::CommitIndeterminate(message) => {
5449 AppError::InternalError(anyhow::anyhow!(
5450 "configuration commit outcome is indeterminate: {message}"
5451 ))
5452 }
5453 ConfigStoreError::Io(error) => AppError::StorageError(error),
5454 ConfigStoreError::Json(_) => {
5455 AppError::BadRequest("configuration document is invalid".to_string())
5456 }
5457 ConfigStoreError::Watch(error) => {
5458 AppError::InternalError(anyhow::anyhow!("configuration watch failed: {error}"))
5459 }
5460 })?;
5461 let (published, installed) = match commit {
5462 Some(commit) => {
5463 let mut published = live_base;
5464 let installed = install_credential_section_commit(commit, &mut published)
5465 .map_err(|error| {
5466 AppError::InternalError(anyhow::anyhow!(
5467 "notification process adoption failed: {error}"
5468 ))
5469 })?;
5470 (published, installed)
5471 }
5472 None => (
5473 candidate,
5474 InstalledCredentialSectionCommit {
5475 events: Vec::new(),
5476 metadata: read_credential_runtime_metadata(&app_data_dir).map_err(
5477 |error| {
5478 AppError::InternalError(anyhow::anyhow!(
5479 "notification credential status unavailable after commit: {error}"
5480 ))
5481 },
5482 )?,
5483 section: None,
5484 },
5485 ),
5486 };
5487 *config.write().await = published.clone();
5488 publish_exact_facade_events(&account_sink, &installed.events).await?;
5489 let section = installed.section;
5490 Ok::<_, AppError>((published, revision, installed.metadata, section))
5491 });
5492 transaction.await.map_err(|error| {
5493 AppError::InternalError(anyhow::anyhow!(
5494 "notification credential transaction task failed: {error}"
5495 ))
5496 })?
5497 }
5498
5499 pub async fn update_connect_credentials<F>(
5504 &self,
5505 expected_revision: u64,
5506 secret_intents: bamboo_config::patch::ConnectSecretIntents,
5507 update: F,
5508 ) -> Result<
5509 (
5510 Config,
5511 u64,
5512 bamboo_config::CredentialSectionRuntimeMetadata,
5513 Option<bamboo_config::SectionEnvelope<Value>>,
5514 ),
5515 AppError,
5516 >
5517 where
5518 F: FnOnce(&mut Config) -> Result<(), AppError> + Send + 'static,
5519 {
5520 let config_io_lock = self.config_io_lock.clone();
5521 let config = self.config.clone();
5522 let app_data_dir = self.app_data_dir.clone();
5523 let account_sink = self.account_sink.clone();
5524 let config_facade = self.config_facade.clone();
5525 let transaction = tokio::spawn(async move {
5526 let _io = config_io_lock.lock().await;
5527 let live_base = {
5528 let current = config.read().await;
5529 reject_if_recovery_pending(¤t)?;
5530 current.clone()
5531 };
5532 let mut candidate = live_base.clone();
5533 if config_facade.is_some() {
5534 install_exact_credential_section_mutation_base(
5535 app_data_dir.clone(),
5536 SectionId::Connect,
5537 expected_revision,
5538 &mut candidate,
5539 )
5540 .await?;
5541 }
5542 update(&mut candidate)?;
5543 candidate.assign_connect_platform_ids();
5544 let transaction_dir = app_data_dir.clone();
5545 let commit_facade = config_facade.clone();
5546 let (candidate, revision, commit) = tokio::task::spawn_blocking(move || {
5547 if let Some(facade) = commit_facade {
5548 let commit =
5549 bamboo_config::persist_connect_credential_transaction_at_revision_with_adoption(
5550 &transaction_dir,
5551 &mut candidate,
5552 &secret_intents,
5553 expected_revision,
5554 facade.as_ref(),
5555 )?;
5556 let revision = commit.revision;
5557 Ok::<_, ConfigStoreError>((candidate, revision, Some(commit)))
5558 } else {
5559 let revision =
5560 bamboo_config::persist_connect_credential_transaction_at_revision(
5561 &transaction_dir,
5562 &mut candidate,
5563 &secret_intents,
5564 expected_revision,
5565 )?;
5566 Ok((
5567 load_committed_effective_config(&transaction_dir)?,
5568 revision,
5569 None,
5570 ))
5571 }
5572 })
5573 .await
5574 .map_err(|error| {
5575 AppError::InternalError(anyhow::anyhow!(
5576 "connect credential transaction task failed: {error}"
5577 ))
5578 })?
5579 .map_err(|error| match error {
5580 ConfigStoreError::Conflict { expected, actual } => {
5581 AppError::ConfigConflict { expected, actual }
5582 }
5583 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
5584 ConfigStoreError::CommitIndeterminate(message) => AppError::InternalError(
5585 anyhow::anyhow!("configuration commit outcome is indeterminate: {message}"),
5586 ),
5587 ConfigStoreError::Io(error) => AppError::StorageError(error),
5588 ConfigStoreError::Json(_) => {
5589 AppError::BadRequest("configuration document is invalid".to_string())
5590 }
5591 ConfigStoreError::Watch(error) => {
5592 AppError::InternalError(anyhow::anyhow!("configuration watch failed: {error}"))
5593 }
5594 })?;
5595 let (published, installed) = match commit {
5596 Some(commit) => {
5597 let mut published = live_base;
5598 let installed = install_credential_section_commit(commit, &mut published)
5599 .map_err(|error| {
5600 AppError::InternalError(anyhow::anyhow!(
5601 "connect process adoption failed: {error}"
5602 ))
5603 })?;
5604 (published, installed)
5605 }
5606 None => (
5607 candidate,
5608 InstalledCredentialSectionCommit {
5609 events: Vec::new(),
5610 metadata: read_credential_runtime_metadata(&app_data_dir).map_err(
5611 |error| {
5612 AppError::InternalError(anyhow::anyhow!(
5613 "connect credential status unavailable after commit: {error}"
5614 ))
5615 },
5616 )?,
5617 section: None,
5618 },
5619 ),
5620 };
5621 *config.write().await = published.clone();
5622 publish_exact_facade_events(&account_sink, &installed.events).await?;
5623 let section = installed.section;
5624 Ok::<_, AppError>((published, revision, installed.metadata, section))
5625 });
5626 transaction.await.map_err(|error| {
5627 AppError::InternalError(anyhow::anyhow!(
5628 "connect credential transaction task failed: {error}"
5629 ))
5630 })?
5631 }
5632
5633 pub async fn update_access_control_credentials<F>(
5636 &self,
5637 expected_revision: u64,
5638 password_intent: bool,
5639 device_intents: std::collections::BTreeSet<String>,
5640 update: F,
5641 ) -> Result<
5642 (
5643 Config,
5644 u64,
5645 bamboo_config::CredentialSectionRuntimeMetadata,
5646 Option<bamboo_config::SectionEnvelope<Value>>,
5647 ),
5648 AppError,
5649 >
5650 where
5651 F: FnOnce(&mut Config) -> Result<(), AppError> + Send + 'static,
5652 {
5653 let config_io_lock = self.config_io_lock.clone();
5654 let config = self.config.clone();
5655 let app_data_dir = self.app_data_dir.clone();
5656 let account_sink = self.account_sink.clone();
5657 let config_facade = self.config_facade.clone();
5658 let transaction = tokio::spawn(async move {
5659 let _io = config_io_lock.lock().await;
5660 let live_base = {
5661 let current = config.read().await;
5662 reject_if_recovery_pending(¤t)?;
5663 current.clone()
5664 };
5665 let mut candidate = live_base.clone();
5666 if config_facade.is_some() {
5667 install_exact_credential_section_mutation_base(
5668 app_data_dir.clone(),
5669 SectionId::AccessControl,
5670 expected_revision,
5671 &mut candidate,
5672 )
5673 .await?;
5674 }
5675 update(&mut candidate)?;
5676 let transaction_dir = app_data_dir.clone();
5677 let commit_facade = config_facade.clone();
5678 let (candidate, revision, commit) = tokio::task::spawn_blocking(move || {
5679 if let Some(facade) = commit_facade {
5680 let commit =
5681 bamboo_config::persist_access_control_credential_transaction_at_revision_with_adoption(
5682 &transaction_dir,
5683 &mut candidate,
5684 password_intent,
5685 &device_intents,
5686 expected_revision,
5687 facade.as_ref(),
5688 )?;
5689 let revision = commit.revision;
5690 Ok::<_, ConfigStoreError>((candidate, revision, Some(commit)))
5691 } else {
5692 let revision =
5693 bamboo_config::persist_access_control_credential_transaction_at_revision(
5694 &transaction_dir,
5695 &mut candidate,
5696 password_intent,
5697 &device_intents,
5698 expected_revision,
5699 )?;
5700 Ok((
5701 load_committed_effective_config(&transaction_dir)?,
5702 revision,
5703 None,
5704 ))
5705 }
5706 })
5707 .await
5708 .map_err(|error| {
5709 AppError::InternalError(anyhow::anyhow!(
5710 "access-control credential transaction task failed: {error}"
5711 ))
5712 })?
5713 .map_err(|error| match error {
5714 ConfigStoreError::Conflict { expected, actual } => {
5715 AppError::ConfigConflict { expected, actual }
5716 }
5717 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
5718 ConfigStoreError::CommitIndeterminate(message) => AppError::InternalError(
5719 anyhow::anyhow!("configuration commit outcome is indeterminate: {message}"),
5720 ),
5721 ConfigStoreError::Io(error) => AppError::StorageError(error),
5722 ConfigStoreError::Json(_) => {
5723 AppError::BadRequest("configuration document is invalid".to_string())
5724 }
5725 ConfigStoreError::Watch(error) => {
5726 AppError::InternalError(anyhow::anyhow!("configuration watch failed: {error}"))
5727 }
5728 })?;
5729 let (published, installed) = match commit {
5730 Some(commit) => {
5731 let mut published = live_base;
5732 let installed = install_credential_section_commit(commit, &mut published)
5733 .map_err(|error| {
5734 AppError::InternalError(anyhow::anyhow!(
5735 "access-control process adoption failed: {error}"
5736 ))
5737 })?;
5738 (published, installed)
5739 }
5740 None => (
5741 candidate,
5742 InstalledCredentialSectionCommit {
5743 events: Vec::new(),
5744 metadata: read_credential_runtime_metadata(&app_data_dir).map_err(
5745 |error| {
5746 AppError::InternalError(anyhow::anyhow!(
5747 "access-control credential status unavailable after commit: {error}"
5748 ))
5749 },
5750 )?,
5751 section: None,
5752 },
5753 ),
5754 };
5755 *config.write().await = published.clone();
5756 publish_exact_facade_events(&account_sink, &installed.events).await?;
5757 let section = installed.section;
5758 Ok::<_, AppError>((published, revision, installed.metadata, section))
5759 });
5760 transaction.await.map_err(|error| {
5761 AppError::InternalError(anyhow::anyhow!(
5762 "access-control credential transaction task failed: {error}"
5763 ))
5764 })?
5765 }
5766
5767 pub async fn update_cluster_fabric_credentials<F>(
5772 &self,
5773 expected_revision: u64,
5774 node_intents: std::collections::BTreeMap<
5775 String,
5776 bamboo_config::ClusterNodeCredentialIntents,
5777 >,
5778 update: F,
5779 ) -> Result<bamboo_server_tools::FabricCommitSnapshot, AppError>
5780 where
5781 F: FnOnce(&mut Config) -> Result<(), AppError> + Send + 'static,
5782 {
5783 self.update_cluster_fabric_credentials_guarded(
5784 expected_revision,
5785 node_intents,
5786 None,
5787 update,
5788 )
5789 .await
5790 }
5791
5792 pub(crate) async fn delete_cluster_node_credentials<F>(
5797 &self,
5798 expected_revision: u64,
5799 node_id: String,
5800 node_intents: std::collections::BTreeMap<
5801 String,
5802 bamboo_config::ClusterNodeCredentialIntents,
5803 >,
5804 update: F,
5805 ) -> Result<bamboo_server_tools::FabricCommitSnapshot, AppError>
5806 where
5807 F: FnOnce(&mut Config) -> Result<(), AppError> + Send + 'static,
5808 {
5809 self.update_cluster_fabric_credentials_guarded(
5810 expected_revision,
5811 node_intents,
5812 Some(node_id),
5813 update,
5814 )
5815 .await
5816 }
5817
5818 async fn update_cluster_fabric_credentials_guarded<F>(
5819 &self,
5820 expected_revision: u64,
5821 node_intents: std::collections::BTreeMap<
5822 String,
5823 bamboo_config::ClusterNodeCredentialIntents,
5824 >,
5825 required_stopped_node: Option<String>,
5826 update: F,
5827 ) -> Result<bamboo_server_tools::FabricCommitSnapshot, AppError>
5828 where
5829 F: FnOnce(&mut Config) -> Result<(), AppError> + Send + 'static,
5830 {
5831 let config_io_lock = self.config_io_lock.clone();
5832 let config = self.config.clone();
5833 let app_data_dir = self.app_data_dir.clone();
5834 let account_sink = self.account_sink.clone();
5835 let config_facade = self.config_facade.clone();
5836 let deployed_registry = self.fabric_deployer.registry();
5837 let transaction = tokio::spawn(async move {
5838 let _io = config_io_lock.lock().await;
5839 if let Some(node_id) = required_stopped_node.as_deref() {
5840 let deployed = deployed_registry.lock().await;
5841 if deployed.contains_key(&bamboo_server_tools::registry_keys::node_key(node_id)) {
5842 return Err(AppError::BadRequest(format!(
5843 "node '{node_id}' is deployed; stop it before deleting it"
5844 )));
5845 }
5846 }
5847 let facade = config_facade.as_ref().ok_or_else(|| {
5848 AppError::BadRequest(
5849 "cluster mutations require the modular configuration facade".to_string(),
5850 )
5851 })?;
5852 let mut candidate = {
5853 let current = config.read().await;
5854 reject_if_recovery_pending(¤t)?;
5855 current.clone()
5856 };
5857 let snapshot_dir = app_data_dir.clone();
5870 let exact = tokio::task::spawn_blocking(move || {
5871 bamboo_config::read_exact_cluster_fabric_snapshot(&snapshot_dir, None)
5872 })
5873 .await
5874 .map_err(|error| {
5875 AppError::InternalError(anyhow::anyhow!("cluster snapshot task failed: {error}"))
5876 })?
5877 .map_err(|error| match error {
5878 ConfigStoreError::Conflict { expected, actual } => {
5879 AppError::ConfigConflict { expected, actual }
5880 }
5881 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
5882 ConfigStoreError::CommitIndeterminate(message) => AppError::InternalError(
5883 anyhow::anyhow!("configuration commit outcome is indeterminate: {message}"),
5884 ),
5885 ConfigStoreError::Io(error) => AppError::StorageError(error),
5886 ConfigStoreError::Json(_) => {
5887 AppError::BadRequest("configuration document is invalid".to_string())
5888 }
5889 ConfigStoreError::Watch(error) => {
5890 AppError::InternalError(anyhow::anyhow!("configuration watch failed: {error}"))
5891 }
5892 })?;
5893 if exact.section.revision != expected_revision {
5894 return Err(AppError::ConfigConflict {
5895 expected: expected_revision,
5896 actual: exact.section.revision,
5897 });
5898 }
5899 if exact.section.status != SectionStatus::Healthy
5900 || exact.section.source_kind != SectionSourceKind::File
5901 || exact.credential_health.status == SectionStatus::Degraded
5902 {
5903 return Err(AppError::BadRequest(
5904 "revision-bound cluster mutations require healthy primary authorities"
5905 .to_string(),
5906 ));
5907 }
5908 candidate.cluster_fabric = exact.cluster_fabric;
5909 update(&mut candidate)?;
5910 let transaction_dir = app_data_dir.clone();
5911 let commit_facade = facade.clone();
5912 let (mut candidate, commit) = tokio::task::spawn_blocking(move || {
5913 let commit =
5914 bamboo_config::persist_cluster_fabric_credential_transaction_with_adoption(
5915 &transaction_dir,
5916 &mut candidate,
5917 &node_intents,
5918 expected_revision,
5919 commit_facade.as_ref(),
5920 |_, _| {
5921 #[cfg(test)]
5922 run_cluster_after_commit_before_adoption_test_hook(
5923 &transaction_dir,
5924 expected_revision,
5925 );
5926 },
5927 )?;
5928 Ok::<_, ConfigStoreError>((candidate, commit))
5929 })
5930 .await
5931 .map_err(|error| {
5932 AppError::InternalError(anyhow::anyhow!(
5933 "cluster credential transaction task failed: {error}"
5934 ))
5935 })?
5936 .map_err(|error| match error {
5937 ConfigStoreError::Conflict { expected, actual } => {
5938 AppError::ConfigConflict { expected, actual }
5939 }
5940 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
5941 ConfigStoreError::CommitIndeterminate(message) => AppError::InternalError(
5942 anyhow::anyhow!("configuration commit outcome is indeterminate: {message}"),
5943 ),
5944 ConfigStoreError::Io(error) => AppError::StorageError(error),
5945 ConfigStoreError::Json(_) => {
5946 AppError::BadRequest("configuration document is invalid".to_string())
5947 }
5948 ConfigStoreError::Watch(error) => {
5949 AppError::InternalError(anyhow::anyhow!("configuration watch failed: {error}"))
5950 }
5951 })?;
5952 let bamboo_config::ClusterFabricTransactionCommit {
5953 revision,
5954 adoption,
5955 credential_adoption,
5956 committed_recovery,
5957 runtime,
5958 } = commit;
5959 let runtime = match runtime {
5960 Ok(bamboo_config::ClusterFabricRuntimeSnapshot {
5961 cluster_fabric,
5962 credential_statuses,
5963 credential_health,
5964 }) => {
5965 candidate.cluster_fabric = cluster_fabric;
5966 Ok((credential_statuses, credential_health))
5967 }
5968 Err(error) if revision == expected_revision => {
5969 return Err(AppError::InternalError(anyhow::anyhow!(
5970 "cluster configuration at revision {revision} could not materialize its exact runtime credentials: {error}"
5971 )));
5972 }
5973 Err(error) => {
5974 candidate.clear_cluster_runtime_credentials();
5975 Err(error)
5976 }
5977 };
5978 *config.write().await = candidate.clone();
5979 let event = match adoption {
5980 Some(Ok(event)) => Some(event),
5981 Some(Err(error)) => {
5982 return Err(AppError::InternalError(anyhow::anyhow!(
5983 "cluster configuration committed at revision {} but process adoption failed: {error}",
5984 revision
5985 )));
5986 }
5987 None if revision == expected_revision => None,
5988 None => {
5989 return Err(AppError::InternalError(anyhow::anyhow!(
5990 "cluster configuration committed at revision {} without a process adoption result",
5991 revision
5992 )));
5993 }
5994 };
5995 let section = facade
5996 .registry()
5997 .envelope_value(SectionId::ClusterFabric)
5998 .map_err(|error| {
5999 AppError::InternalError(anyhow::anyhow!(
6000 "committed cluster section envelope is unavailable: {error}"
6001 ))
6002 })?;
6003 if section.revision != revision {
6004 return Err(AppError::InternalError(anyhow::anyhow!(
6005 "cluster configuration committed at revision {} but facade retained revision {}",
6006 revision,
6007 section.revision
6008 )));
6009 }
6010 if let Some(event) = event.as_ref() {
6011 publish_registry_event(&account_sink, event).await;
6012 }
6013 if let Err(error) = committed_recovery {
6014 return Err(AppError::InternalError(anyhow::anyhow!(
6015 "cluster configuration committed at revision {revision} but transaction recovery failed: {error}"
6016 )));
6017 }
6018 if let Some(Err(error)) = credential_adoption {
6019 return Err(AppError::InternalError(anyhow::anyhow!(
6020 "cluster configuration committed at revision {revision} but credential facade adoption failed: {error}"
6021 )));
6022 }
6023 let (credential_statuses, credential_health) = runtime.map_err(|error| {
6024 AppError::InternalError(anyhow::anyhow!(
6025 "cluster configuration committed at revision {revision} but could not materialize its exact runtime credentials: {error}"
6026 ))
6027 })?;
6028 Ok::<_, AppError>(bamboo_server_tools::FabricCommitSnapshot {
6029 config: candidate,
6030 section,
6031 credential_statuses,
6032 credential_health,
6033 })
6034 });
6035 transaction.await.map_err(|error| {
6036 AppError::InternalError(anyhow::anyhow!(
6037 "cluster credential transaction task failed: {error}"
6038 ))
6039 })?
6040 }
6041
6042 pub async fn update_proxy_auth_credential(
6046 &self,
6047 auth: Option<bamboo_config::ProxyAuth>,
6048 expected_revision: u64,
6049 effects: ConfigUpdateEffects,
6050 ) -> Result<
6051 (
6052 Config,
6053 u64,
6054 bamboo_config::CredentialStatus,
6055 bamboo_config::CredentialStoreHealth,
6056 Option<bamboo_config::SectionEnvelope<Value>>,
6057 ),
6058 AppError,
6059 > {
6060 self.update_core_with_proxy_credential(expected_revision, effects, move |candidate| {
6061 candidate.proxy_auth = auth;
6062 })
6063 .await
6064 }
6065
6066 async fn update_core_with_proxy_credential<F>(
6067 &self,
6068 expected_revision: u64,
6069 effects: ConfigUpdateEffects,
6070 update: F,
6071 ) -> Result<
6072 (
6073 Config,
6074 u64,
6075 bamboo_config::CredentialStatus,
6076 bamboo_config::CredentialStoreHealth,
6077 Option<bamboo_config::SectionEnvelope<Value>>,
6078 ),
6079 AppError,
6080 >
6081 where
6082 F: FnOnce(&mut Config) + Send + 'static,
6083 {
6084 let config_io_lock = self.config_io_lock.clone();
6085 let config = self.config.clone();
6086 let app_data_dir = self.app_data_dir.clone();
6087 let credential_store = self.credential_store.clone();
6088 let provider_registry = self.provider_registry.clone();
6089 let provider = self.provider.clone();
6090 let mcp_manager = self.mcp_manager.clone();
6091 let config_live_health = self.config_live_health.clone();
6092 let mcp_config_live_health = self.mcp_config_live_health.clone();
6093 let config_facade = self.config_facade.clone();
6094 let account_sink = self.account_sink.clone();
6095
6096 let transaction = tokio::spawn(async move {
6101 let _io = config_io_lock.lock().await;
6102 let live_base = {
6103 let cfg = config.read().await;
6104 reject_if_recovery_pending(&cfg)?;
6105 cfg.clone()
6106 };
6107 let mut candidate = live_base.clone();
6108 if config_facade.is_some() {
6109 install_exact_credential_section_mutation_base(
6110 app_data_dir.clone(),
6111 SectionId::Core,
6112 expected_revision,
6113 &mut candidate,
6114 )
6115 .await?;
6116 }
6117 update(&mut candidate);
6118 if config_facade.is_none() {
6119 candidate.assign_connect_platform_ids();
6120 candidate.refresh_encrypted_secrets().map_err(|error| {
6121 AppError::InternalError(anyhow::anyhow!(
6122 "Failed to refresh encrypted secrets: {error}"
6123 ))
6124 })?;
6125 }
6126 let transaction_dir = app_data_dir.clone();
6127 let status_reference =
6128 candidate
6129 .proxy_auth_credential_ref
6130 .clone()
6131 .unwrap_or_else(|| {
6132 bamboo_config::CredentialRef::parse("proxy.default.auth")
6133 .expect("canonical proxy credential reference is valid")
6134 });
6135 let commit_facade = config_facade.clone();
6136 let (candidate, revision, reference, commit) =
6137 tokio::task::spawn_blocking(move || {
6138 if let Some(facade) = commit_facade {
6139 let commit =
6140 bamboo_config::persist_proxy_auth_credential_transaction_at_revision_with_adoption(
6141 &transaction_dir,
6142 &mut candidate,
6143 expected_revision,
6144 facade.as_ref(),
6145 )?;
6146 let revision = commit.revision;
6147 Ok::<_, ConfigStoreError>((
6148 candidate,
6149 revision,
6150 status_reference,
6151 Some(commit),
6152 ))
6153 } else {
6154 let revision =
6155 bamboo_config::persist_proxy_auth_credential_transaction_at_revision(
6156 &transaction_dir,
6157 &mut candidate,
6158 expected_revision,
6159 )?;
6160 Ok((
6161 load_committed_effective_config(&transaction_dir)?,
6162 revision,
6163 status_reference,
6164 None,
6165 ))
6166 }
6167 })
6168 .await
6169 .map_err(|error| {
6170 AppError::InternalError(anyhow::anyhow!(
6171 "proxy credential transaction task failed: {error}"
6172 ))
6173 })?
6174 .map_err(|error| match error {
6175 ConfigStoreError::Conflict { expected, actual } => {
6176 AppError::ConfigConflict { expected, actual }
6177 }
6178 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
6179 ConfigStoreError::CommitIndeterminate(message) => AppError::InternalError(
6180 anyhow::anyhow!("configuration commit outcome is indeterminate: {message}"),
6181 ),
6182 ConfigStoreError::Io(error) => AppError::StorageError(error),
6183 ConfigStoreError::Json(_) => {
6184 AppError::BadRequest("configuration document is invalid".to_string())
6185 }
6186 ConfigStoreError::Watch(error) => {
6187 AppError::InternalError(anyhow::anyhow!("configuration watch failed: {error}"))
6188 }
6189 })?;
6190 let (published, installed) = match commit {
6191 Some(commit) => {
6192 let mut published = live_base;
6193 let installed = install_credential_section_commit(commit, &mut published)
6194 .map_err(|error| {
6195 AppError::InternalError(anyhow::anyhow!(
6196 "proxy process adoption failed: {error}"
6197 ))
6198 })?;
6199 (published, Some(installed))
6200 }
6201 None => (candidate, None),
6202 };
6203 let section = installed
6204 .as_ref()
6205 .and_then(|installed| installed.section.clone());
6206
6207 published.publish_env_vars();
6211 *config.write().await = published.clone();
6212
6213 if let Some(installed) = installed.as_ref() {
6214 publish_exact_facade_events(&account_sink, &installed.events).await?;
6215 }
6216
6217 Self::apply_config_effects_owned(
6218 published.clone(),
6219 effects,
6220 ConfigRuntimeEffectContext {
6221 app_data_dir,
6222 config_facade,
6223 provider_registry,
6224 provider,
6225 mcp_manager,
6226 account_sink,
6227 config_live_health,
6228 mcp_config_live_health,
6229 },
6230 )
6231 .await?;
6232
6233 let (status, health) = if let Some(installed) = installed {
6234 (
6235 installed.metadata.status(&reference),
6236 installed.metadata.credential_health,
6237 )
6238 } else {
6239 credential_store
6240 .status_with_health(&reference)
6241 .map_err(|error| match error {
6242 ConfigStoreError::Conflict { expected, actual } => {
6243 AppError::ConfigConflict { expected, actual }
6244 }
6245 ConfigStoreError::Validation(_)
6246 | ConfigStoreError::CommitIndeterminate(_)
6247 | ConfigStoreError::Json(_) => AppError::InternalError(anyhow::anyhow!(
6248 "credential store validation failed"
6249 )),
6250 ConfigStoreError::Io(error) => AppError::StorageError(error),
6251 ConfigStoreError::Watch(error) => AppError::InternalError(anyhow::anyhow!(
6252 "configuration watch failed: {error}"
6253 )),
6254 })?
6255 };
6256 Ok::<_, AppError>((published, revision, status, health, section))
6257 });
6258 transaction.await.map_err(|error| {
6259 AppError::InternalError(anyhow::anyhow!(
6260 "proxy credential mutation task failed: {error}"
6261 ))
6262 })?
6263 }
6264
6265 pub async fn replace_config(
6267 &self,
6268 mut new_config: Config,
6269 effects: ConfigUpdateEffects,
6270 ) -> Result<Config, AppError> {
6271 if self.config_facade.is_none() {
6277 new_config.assign_connect_platform_ids();
6278 new_config.refresh_encrypted_secrets().map_err(|e| {
6282 AppError::InternalError(anyhow::anyhow!("Failed to refresh encrypted secrets: {e}"))
6283 })?;
6284 }
6285
6286 let io = self.config_io_lock.clone().lock_owned().await;
6287 restore_authoritative_cluster_fabric(self.config_facade.as_ref(), &mut new_config);
6288 let (was_off, live_base) = {
6289 let cfg = self.config.read().await;
6290 reject_if_recovery_pending(&cfg)?;
6293 (cfg.plugin_trust.enforcement_is_off(), cfg.clone())
6294 };
6295 let config = self.config.clone();
6296 let app_data_dir = self.app_data_dir.clone();
6297 let config_facade = self.config_facade.clone();
6298 let account_sink = self.account_sink.clone();
6299 let provider_registry = self.provider_registry.clone();
6300 let provider = self.provider.clone();
6301 let mcp_manager = self.mcp_manager.clone();
6302 let config_live_health = self.config_live_health.clone();
6303 let mcp_config_live_health = self.mcp_config_live_health.clone();
6304 let transaction = tokio::spawn(async move {
6305 let new_config = {
6310 let _io = io;
6311 let commit = Self::persist_config_snapshot(
6312 app_data_dir.clone(),
6313 config_facade.clone(),
6314 new_config.clone(),
6315 )
6316 .await?;
6317 let mut published = if commit.is_some() {
6318 live_base
6319 } else {
6320 new_config
6321 };
6322 let events = match commit {
6323 Some(commit) => {
6324 install_facade_config_commit(commit, &mut published).map_err(|error| {
6325 AppError::InternalError(anyhow::anyhow!(
6326 "failed to install committed configuration section: {error}"
6327 ))
6328 })?
6329 }
6330 None => Vec::new(),
6331 };
6332 let enforcement_newly_off = !was_off && published.plugin_trust.enforcement_is_off();
6333 {
6334 let mut current = config.write().await;
6335 preserve_runtime_broker(&mut published, ¤t);
6336 published.publish_env_vars();
6337 *current = published.clone();
6338 }
6339 if enforcement_newly_off {
6342 warn_plugin_trust_enforcement_off();
6343 }
6344 publish_exact_facade_events(&account_sink, &events).await?;
6345 Self::apply_config_effects_owned(
6346 published.clone(),
6347 effects,
6348 ConfigRuntimeEffectContext {
6349 app_data_dir,
6350 config_facade,
6351 provider_registry,
6352 provider,
6353 mcp_manager,
6354 account_sink,
6355 config_live_health,
6356 mcp_config_live_health,
6357 },
6358 )
6359 .await?;
6360 published
6361 };
6362 Ok::<_, AppError>(new_config)
6363 });
6364 transaction.await.map_err(|error| {
6365 AppError::InternalError(anyhow::anyhow!(
6366 "config replacement transaction task failed: {error}"
6367 ))
6368 })?
6369 }
6370
6371 async fn apply_config_effects_owned(
6372 new_config: Config,
6373 effects: ConfigUpdateEffects,
6374 context: ConfigRuntimeEffectContext,
6375 ) -> Result<(), AppError> {
6376 Self::apply_config_effects_owned_after_forcing(new_config, effects, context, HashSet::new())
6377 .await
6378 }
6379
6380 async fn apply_config_effects_owned_after_forcing(
6381 new_config: Config,
6382 effects: ConfigUpdateEffects,
6383 context: ConfigRuntimeEffectContext,
6384 forced_mcp_replacements: HashSet<String>,
6385 ) -> Result<(), AppError> {
6386 let ConfigRuntimeEffectContext {
6387 app_data_dir,
6388 config_facade,
6389 provider_registry,
6390 provider,
6391 mcp_manager,
6392 account_sink,
6393 config_live_health,
6394 mcp_config_live_health,
6395 } = context;
6396 let mut provider_failure = None;
6401 if !matches!(
6402 effects.reload_provider,
6403 bamboo_config::patch::ReloadMode::None
6404 ) {
6405 let candidate = async {
6406 let candidate_registry =
6407 bamboo_llm::ProviderRegistry::from_config(&new_config, app_data_dir.clone())
6408 .await?;
6409 let default_provider_name = candidate_registry.default_provider_name();
6410 let candidate_provider = candidate_registry.get_default().ok_or_else(|| {
6411 let message = if new_config.has_provider_instances() {
6412 format!(
6413 "Default provider instance '{}' is not available or failed to initialize",
6414 default_provider_name
6415 )
6416 } else {
6417 format!(
6418 "Provider '{}' is not available or failed to initialize",
6419 new_config.provider
6420 )
6421 };
6422 bamboo_llm::LLMError::Auth(message)
6423 })?;
6424 Ok::<_, bamboo_llm::LLMError>((
6425 candidate_registry,
6426 candidate_provider,
6427 default_provider_name,
6428 ))
6429 }
6430 .await;
6431
6432 match candidate {
6433 Ok((candidate_registry, candidate_provider, default_provider_name)) => {
6434 #[cfg(test)]
6435 run_generic_before_provider_publish_test_hook(&app_data_dir);
6436 {
6437 let mut live_provider = provider.write().await;
6440 provider_registry.replace_with(candidate_registry);
6441 *live_provider = candidate_provider;
6442 }
6443 if let Some(facade) = config_facade.as_ref() {
6444 let snapshot = facade.registry().providers.snapshot();
6445 set_live_health_revision(
6446 &config_live_health,
6447 snapshot.revision,
6448 Some((snapshot.source_path.clone(), snapshot.source_kind)),
6449 );
6450 } else {
6451 update_live_health(
6452 &config_live_health,
6453 SectionStatus::Healthy,
6454 None,
6455 true,
6456 Some((app_data_dir.join("config.json"), SectionSourceKind::File)),
6457 );
6458 }
6459 tracing::info!(
6460 default_provider = %default_provider_name,
6461 "Provider reloaded successfully"
6462 );
6463 }
6464 Err(_) => {
6465 tracing::warn!("committed provider generation could not start");
6466 let message =
6467 "provider runtime initialization failed; retaining last-known-good runtime"
6468 .to_string();
6469 if let Some(facade) = config_facade.as_ref() {
6470 if let Some(event) = facade
6471 .registry()
6472 .mark_runtime_degraded(SectionId::Providers, message.clone())
6473 {
6474 let snapshot = facade.registry().providers.snapshot();
6475 set_live_health_from_snapshot(&config_live_health, &snapshot);
6476 publish_registry_event(&account_sink, &event).await;
6477 }
6478 } else {
6479 publish_section_failure(
6480 &config_live_health,
6481 &account_sink,
6482 "providers",
6483 SectionStatus::Degraded,
6484 message.clone(),
6485 )
6486 .await;
6487 }
6488 if matches!(
6489 effects.reload_provider,
6490 bamboo_config::patch::ReloadMode::Strict
6491 ) {
6492 provider_failure = Some(AppError::InternalError(anyhow::anyhow!(message)));
6493 }
6494 }
6495 }
6496 }
6497
6498 let mut mcp_failure = None;
6499 if !matches!(
6500 effects.reconcile_mcp,
6501 bamboo_config::patch::ReloadMode::None
6502 ) {
6503 match mcp_manager
6504 .reconcile_from_config_transactional_after_forcing(
6505 &new_config.mcp,
6506 &forced_mcp_replacements,
6507 || async { Ok(()) },
6508 )
6509 .await
6510 {
6511 Ok(()) => {
6512 if let Some(facade) = config_facade.as_ref() {
6513 let snapshot = facade.registry().mcp.snapshot();
6514 set_live_health_revision(
6515 &mcp_config_live_health,
6516 snapshot.revision,
6517 Some((snapshot.source_path.clone(), snapshot.source_kind)),
6518 );
6519 } else {
6520 update_live_health(
6521 &mcp_config_live_health,
6522 SectionStatus::Healthy,
6523 None,
6524 true,
6525 Some((app_data_dir.join("config.json"), SectionSourceKind::File)),
6526 );
6527 }
6528 }
6529 Err(_) => {
6530 tracing::warn!("committed MCP generation could not start");
6531 let message =
6532 "MCP runtime initialization failed; retaining last-known-good runtime"
6533 .to_string();
6534 if let Some(facade) = config_facade.as_ref() {
6535 if let Some(event) = facade
6536 .registry()
6537 .mark_runtime_degraded(SectionId::Mcp, message.clone())
6538 {
6539 let snapshot = facade.registry().mcp.snapshot();
6540 set_live_health_from_snapshot(&mcp_config_live_health, &snapshot);
6541 publish_registry_event(&account_sink, &event).await;
6542 }
6543 } else {
6544 publish_section_failure(
6545 &mcp_config_live_health,
6546 &account_sink,
6547 "mcp",
6548 SectionStatus::Degraded,
6549 message.clone(),
6550 )
6551 .await;
6552 }
6553 if matches!(
6554 effects.reconcile_mcp,
6555 bamboo_config::patch::ReloadMode::Strict
6556 ) {
6557 mcp_failure = Some(AppError::InternalError(anyhow::anyhow!(message)));
6558 }
6559 }
6560 }
6561 }
6562
6563 provider_failure.or(mcp_failure).map_or(Ok(()), Err)
6564 }
6565
6566 pub async fn confirm_config_recovery(&self, accept: bool) -> Result<Config, AppError> {
6582 let _io = self.config_io_lock.lock().await;
6583
6584 if !accept {
6585 let cfg = self.config.read().await;
6586 return match cfg.recovery_status() {
6587 Some(_) => Ok(cfg.clone()),
6588 None => Err(AppError::BadRequest(
6589 "No pending config-corruption recovery to resolve".to_string(),
6590 )),
6591 };
6592 }
6593
6594 let mut candidate = {
6595 let cfg = self.config.read().await;
6596 match cfg.recovery_status() {
6597 Some(_) => cfg.clone(),
6598 None => {
6599 return Err(AppError::BadRequest(
6600 "No pending config-corruption recovery to resolve".to_string(),
6601 ))
6602 }
6603 }
6604 };
6605
6606 let data_dir = self.app_data_dir.clone();
6607 candidate = tokio::task::spawn_blocking(move || {
6608 candidate
6609 .confirm_recovery_and_save_to_dir(data_dir)
6610 .map(|_| candidate)
6611 })
6612 .await
6613 .map_err(|e| {
6614 AppError::InternalError(anyhow::anyhow!("Config recovery-confirm task failed: {e}"))
6615 })?
6616 .map_err(|e| {
6617 AppError::InternalError(anyhow::anyhow!("Failed to save recovered config: {e}"))
6618 })?;
6619
6620 {
6621 let mut cfg = self.config.write().await;
6622 *cfg = candidate.clone();
6623 cfg.publish_env_vars();
6624 }
6625
6626 Ok(candidate)
6627 }
6628}
6629
6630fn reject_if_recovery_pending(cfg: &Config) -> Result<(), AppError> {
6638 if let Some(status) = cfg.recovery_status() {
6639 if !status.confirmed {
6640 return Err(AppError::ConfigRecoveryPending(format!(
6641 "config.json was recovered from corruption ({:?}) and is awaiting \
6642 confirmation; confirm or reject the recovery (see /bamboo/config/recovery-status \
6643 and /bamboo/config/recovery/confirm) before changing settings",
6644 status.source
6645 )));
6646 }
6647 }
6648 Ok(())
6649}
6650
6651pub(crate) fn warn_plugin_trust_enforcement_off() {
6661 tracing::warn!(
6662 "plugin_trust.enforcement is OFF — plugin installs from ANY URL are accepted \
6663 without host/signature/checksum verification (config.json plugin_trust.enforcement)"
6664 );
6665}
6666
6667#[cfg(test)]
6668mod live_reload_tests {
6669 use super::*;
6670 use bamboo_agent_core::{Message, ToolSchema};
6671 use bamboo_llm::{LLMError, LLMStream};
6672 use bamboo_mcp::{McpServerConfig, ReconnectConfig, StdioConfig};
6673
6674 struct WorkingProvider;
6675
6676 fn stop_config_watcher(state: &mut AppState) {
6677 state.config_watcher.stop.store(true, Ordering::Relaxed);
6678 if let Some(task) = state.config_watcher.apply_task.take() {
6679 task.abort();
6680 }
6681 if let Some(task) = state.config_watcher.watcher_task.take() {
6682 task.join().unwrap();
6683 }
6684 }
6685
6686 fn restart_config_watcher(state: &mut AppState) {
6687 let (runtime, provider_health, mcp_health) = ConfigWatcherRuntime::start(
6688 state.app_data_dir.clone(),
6689 state.config.clone(),
6690 state.config_facade.clone(),
6691 state.config_io_lock.clone(),
6692 state.provider_registry.clone(),
6693 state.provider.clone(),
6694 state.mcp_manager.clone(),
6695 state.account_sink.clone(),
6696 );
6697 state.config_watcher = runtime;
6698 state.config_live_health = provider_health;
6699 state.mcp_config_live_health = mcp_health;
6700 }
6701
6702 async fn insert_registry_worker(state: &AppState, key: String, worker_id: &str) {
6703 #[cfg(unix)]
6704 let child = tokio::process::Command::new("/bin/sleep")
6705 .arg("30")
6706 .spawn()
6707 .unwrap();
6708 #[cfg(windows)]
6709 let child = tokio::process::Command::new("cmd")
6710 .args(["/C", "timeout", "/T", "30", "/NOBREAK"])
6711 .spawn()
6712 .unwrap();
6713 state.fabric_deployer.registry().lock().await.insert(
6714 key,
6715 bamboo_server_tools::Deployed {
6716 env: "test".to_string(),
6717 handle: bamboo_broker::DeployedAgent::from_parts(worker_id, child, None),
6718 },
6719 );
6720 }
6721
6722 fn disabled_mcp_config(id: &str) -> McpConfig {
6723 McpConfig {
6724 version: 1,
6725 servers: vec![McpServerConfig {
6726 id: id.to_string(),
6727 name: None,
6728 enabled: false,
6729 transport: TransportConfig::Stdio(StdioConfig {
6730 command: "unused-disabled-command".to_string(),
6731 args: vec![],
6732 cwd: None,
6733 env: std::collections::HashMap::new(),
6734 env_encrypted: std::collections::HashMap::new(),
6735 env_credential_refs: std::collections::HashMap::new(),
6736 startup_timeout_ms: 100,
6737 }),
6738 request_timeout_ms: 100,
6739 healthcheck_interval_ms: 100,
6740 reconnect: ReconnectConfig::default(),
6741 allowed_tools: vec![],
6742 denied_tools: vec![],
6743 }],
6744 }
6745 }
6746
6747 fn working_stdio_mcp_config(dir: &Path, id: &str, secret: Option<&str>) -> McpConfig {
6748 let script = dir.join(format!("{id}-mcp-fixture.py"));
6749 std::fs::write(
6750 &script,
6751 r#"import json
6752import sys
6753
6754for line in sys.stdin:
6755 request = json.loads(line)
6756 request_id = request.get("id")
6757 if request_id is None:
6758 continue
6759 if request.get("method") == "server/discover":
6760 print(json.dumps({
6761 "jsonrpc": "2.0",
6762 "id": request_id,
6763 "error": {"code": -32601, "message": "Method not found"},
6764 }), flush=True)
6765 continue
6766 if request.get("method") == "initialize":
6767 result = {
6768 "protocolVersion": "2024-11-05",
6769 "capabilities": {"tools": {"listChanged": False}},
6770 "serverInfo": {"name": "config-generation-fixture", "version": "1.0.0"},
6771 }
6772 elif request.get("method") == "tools/list":
6773 result = {"tools": []}
6774 else:
6775 result = {}
6776 print(json.dumps({"jsonrpc": "2.0", "id": request_id, "result": result}), flush=True)
6777"#,
6778 )
6779 .unwrap();
6780 let python = ["python3", "python"]
6781 .into_iter()
6782 .find(|command| {
6783 std::process::Command::new(command)
6784 .arg("--version")
6785 .output()
6786 .is_ok_and(|output| output.status.success())
6787 })
6788 .expect("a Python interpreter is required for the MCP ordering fixture");
6789 let mut env = std::collections::HashMap::new();
6790 if let Some(secret) = secret {
6791 env.insert("TOKEN".to_string(), secret.to_string());
6792 }
6793 McpConfig {
6794 version: 1,
6795 servers: vec![McpServerConfig {
6796 id: id.to_string(),
6797 name: None,
6798 enabled: true,
6799 transport: TransportConfig::Stdio(StdioConfig {
6800 command: python.to_string(),
6801 args: vec![script.to_string_lossy().into_owned()],
6802 cwd: None,
6803 env,
6804 env_encrypted: std::collections::HashMap::new(),
6805 env_credential_refs: std::collections::HashMap::new(),
6806 startup_timeout_ms: 2_000,
6807 }),
6808 request_timeout_ms: 2_000,
6809 healthcheck_interval_ms: 10_000,
6810 reconnect: ReconnectConfig {
6811 enabled: false,
6812 ..Default::default()
6813 },
6814 allowed_tools: vec![],
6815 denied_tools: vec![],
6816 }],
6817 }
6818 }
6819
6820 #[tokio::test]
6821 async fn config_update_preserves_runtime_broker() {
6822 let dir = tempfile::tempdir().unwrap();
6823 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
6824 let expected = state
6825 .config
6826 .read()
6827 .await
6828 .subagents()
6829 .broker
6830 .clone()
6831 .expect("AppState embeds a runtime broker");
6832
6833 let updated = state
6834 .update_config(
6835 |config| {
6836 config.subagents_mut().max_concurrent = Some(3);
6837 Ok(())
6838 },
6839 ConfigUpdateEffects::default(),
6840 )
6841 .await
6842 .unwrap();
6843
6844 assert_eq!(updated.subagents().broker.as_ref(), Some(&expected));
6845 assert_eq!(
6846 state.config.read().await.subagents().broker.as_ref(),
6847 Some(&expected)
6848 );
6849 }
6850
6851 #[test]
6852 fn preserve_runtime_broker_keeps_explicit_broker() {
6853 let previous_broker = bamboo_config::BrokerClientConfig {
6854 endpoint: "ws://127.0.0.1:41001".to_string(),
6855 token: "previous".to_string(),
6856 token_encrypted: None,
6857 credential_ref: None,
6858 configured: false,
6859 };
6860 let explicit_broker = bamboo_config::BrokerClientConfig {
6861 endpoint: "wss://broker.example.test".to_string(),
6862 token: "explicit".to_string(),
6863 token_encrypted: None,
6864 credential_ref: None,
6865 configured: true,
6866 };
6867 let mut previous = Config::default();
6868 previous.subagents_mut().broker = Some(previous_broker);
6869 let mut incoming = Config::default();
6870 incoming.subagents_mut().broker = Some(explicit_broker.clone());
6871
6872 preserve_runtime_broker(&mut incoming, &previous);
6873
6874 assert_eq!(incoming.subagents().broker.as_ref(), Some(&explicit_broker));
6875 }
6876
6877 fn mcp_document_bytes(revision: u64, config: &McpConfig) -> Vec<u8> {
6878 serde_json::to_vec_pretty(&serde_json::json!({
6879 "schema_version": 1,
6880 "revision": revision,
6881 "data": config,
6882 }))
6883 .unwrap()
6884 }
6885
6886 #[test]
6887 fn legacy_mcp_rejects_client_owned_stdio_and_header_credential_refs() {
6888 let mut stdio_current = disabled_mcp_config("stdio-server");
6889 let stdio_reference =
6890 bamboo_config::credential_ref("mcp", "stdio-server", "env_TOKEN").unwrap();
6891 let TransportConfig::Stdio(stdio) = &mut stdio_current.servers[0].transport else {
6892 unreachable!()
6893 };
6894 stdio
6895 .env
6896 .insert("TOKEN".to_string(), "existing-secret".to_string());
6897 stdio
6898 .env_credential_refs
6899 .insert("TOKEN".to_string(), stdio_reference.as_str().to_string());
6900 let mut stdio_candidate = stdio_current.clone();
6901 let TransportConfig::Stdio(stdio) = &mut stdio_candidate.servers[0].transport else {
6902 unreachable!()
6903 };
6904 stdio
6905 .env_credential_refs
6906 .insert("TOKEN".to_string(), "mcp.foreign.env_token".to_string());
6907 let error = normalize_legacy_mcp_credentials(&stdio_current, &mut stdio_candidate)
6908 .expect_err("an arbitrary stdio credential ref must be rejected");
6909 assert!(matches!(
6910 error,
6911 AppError::BadRequest(message)
6912 if message == "MCP credential references are server-managed and cannot be supplied"
6913 ));
6914
6915 let header_reference =
6916 bamboo_config::credential_ref("mcp", "http-server", "header_Authorization").unwrap();
6917 let http_current = McpConfig {
6918 version: 1,
6919 servers: vec![McpServerConfig {
6920 id: "http-server".to_string(),
6921 name: None,
6922 enabled: false,
6923 transport: TransportConfig::Sse(bamboo_mcp::SseConfig {
6924 url: "https://example.test/sse".to_string(),
6925 headers: vec![bamboo_mcp::HeaderConfig {
6926 name: "Authorization".to_string(),
6927 value: "existing-secret".to_string(),
6928 value_encrypted: None,
6929 credential_ref: Some(header_reference.as_str().to_string()),
6930 }],
6931 connect_timeout_ms: 100,
6932 }),
6933 request_timeout_ms: 100,
6934 healthcheck_interval_ms: 100,
6935 reconnect: ReconnectConfig::default(),
6936 allowed_tools: vec![],
6937 denied_tools: vec![],
6938 }],
6939 };
6940 let mut http_candidate = http_current.clone();
6941 let TransportConfig::Sse(http) = &mut http_candidate.servers[0].transport else {
6942 unreachable!()
6943 };
6944 http.headers[0].credential_ref = Some("mcp.foreign.header_authorization".to_string());
6945 let error = normalize_legacy_mcp_credentials(&http_current, &mut http_candidate)
6946 .expect_err("an arbitrary header credential ref must be rejected");
6947 assert!(matches!(
6948 error,
6949 AppError::BadRequest(message)
6950 if message == "MCP credential references are server-managed and cannot be supplied"
6951 ));
6952 }
6953
6954 #[test]
6955 fn touched_shared_mcp_refs_stage_replacements_and_preserve_surviving_clears() {
6956 let shared =
6957 bamboo_config::CredentialRef::parse("mcp.shared.env_token".to_string()).unwrap();
6958 let mut current = disabled_mcp_config("first");
6959 let mut second = current.servers[0].clone();
6960 second.id = "second".to_string();
6961 current.servers.push(second);
6962 for server in &mut current.servers {
6963 let TransportConfig::Stdio(stdio) = &mut server.transport else {
6964 unreachable!()
6965 };
6966 stdio
6967 .env
6968 .insert("TOKEN".to_string(), "old-shared-secret".to_string());
6969 stdio
6970 .env_credential_refs
6971 .insert("TOKEN".to_string(), shared.as_str().to_string());
6972 }
6973 let touched = BTreeSet::from([shared]);
6974
6975 let mut replace = current.clone();
6976 for server in &mut replace.servers {
6977 let TransportConfig::Stdio(stdio) = &mut server.transport else {
6978 unreachable!()
6979 };
6980 stdio.env.get_mut("TOKEN").unwrap().clear();
6981 }
6982 let TransportConfig::Stdio(first) = &mut replace.servers[0].transport else {
6983 unreachable!()
6984 };
6985 first
6986 .env
6987 .insert("TOKEN".to_string(), "new-shared-secret".to_string());
6988 materialize_mcp_touched_replacements(&mut replace, &touched).unwrap();
6989 retain_mcp_credentials(¤t, &mut replace, &touched);
6990 for server in &replace.servers {
6991 let TransportConfig::Stdio(stdio) = &server.transport else {
6992 unreachable!()
6993 };
6994 assert_eq!(stdio.env["TOKEN"], "new-shared-secret");
6995 }
6996
6997 let mut clear_one = current.clone();
6998 for server in &mut clear_one.servers {
6999 let TransportConfig::Stdio(stdio) = &mut server.transport else {
7000 unreachable!()
7001 };
7002 stdio.env.get_mut("TOKEN").unwrap().clear();
7003 }
7004 let TransportConfig::Stdio(first) = &mut clear_one.servers[0].transport else {
7005 unreachable!()
7006 };
7007 first.env.remove("TOKEN");
7008 first.env_credential_refs.remove("TOKEN");
7009 materialize_mcp_touched_replacements(&mut clear_one, &touched).unwrap();
7010 retain_mcp_credentials(¤t, &mut clear_one, &touched);
7011 let TransportConfig::Stdio(first) = &clear_one.servers[0].transport else {
7012 unreachable!()
7013 };
7014 assert!(!first.env.contains_key("TOKEN"));
7015 assert!(!first.env_credential_refs.contains_key("TOKEN"));
7016 let TransportConfig::Stdio(second) = &clear_one.servers[1].transport else {
7017 unreachable!()
7018 };
7019 assert_eq!(second.env["TOKEN"], "old-shared-secret");
7020 assert_eq!(second.env_credential_refs["TOKEN"], "mcp.shared.env_token");
7021
7022 let header_ref =
7023 bamboo_config::CredentialRef::parse("mcp.shared.header_token".to_string()).unwrap();
7024 let current_http = McpConfig {
7025 version: 1,
7026 servers: vec![McpServerConfig {
7027 id: "http".to_string(),
7028 name: None,
7029 enabled: false,
7030 transport: TransportConfig::Sse(bamboo_mcp::SseConfig {
7031 url: "https://example.test/sse".to_string(),
7032 headers: vec![bamboo_mcp::HeaderConfig {
7033 name: "Authorization".to_string(),
7034 value: "old-header-secret".to_string(),
7035 value_encrypted: None,
7036 credential_ref: Some(header_ref.as_str().to_string()),
7037 }],
7038 connect_timeout_ms: 100,
7039 }),
7040 request_timeout_ms: 100,
7041 healthcheck_interval_ms: 100,
7042 reconnect: ReconnectConfig::default(),
7043 allowed_tools: vec![],
7044 denied_tools: vec![],
7045 }],
7046 };
7047 let mut delete_all_headers = current_http.clone();
7048 let TransportConfig::Sse(candidate) = &mut delete_all_headers.servers[0].transport else {
7049 unreachable!()
7050 };
7051 candidate.headers.clear();
7052 let touched = BTreeSet::from([header_ref]);
7053 materialize_mcp_touched_replacements(&mut delete_all_headers, &touched).unwrap();
7054 retain_mcp_credentials(¤t_http, &mut delete_all_headers, &touched);
7055 let TransportConfig::Sse(candidate) = &delete_all_headers.servers[0].transport else {
7056 unreachable!()
7057 };
7058 assert!(candidate.headers.is_empty());
7059 }
7060
7061 fn install_unrecoverable_pending_provider_migration(dir: &Path) {
7062 let transaction_id = uuid::Uuid::new_v4().to_string();
7063 std::fs::write(
7064 dir.join("config.json"),
7065 br#"{"providers":{"openai":{"model":"root-lkg"}}}"#,
7066 )
7067 .unwrap();
7068 std::fs::write(
7069 dir.join("providers.json"),
7070 br#"{"schema_version":1,"revision":2,"data":{"openai":{"model":"partial-must-not-load","credential_ref":"provider.openai.api_key"}}}"#,
7071 )
7072 .unwrap();
7073 std::fs::write(
7074 dir.join("config-credential-migration.json"),
7075 serde_json::to_vec_pretty(&serde_json::json!({
7076 "version": 1,
7077 "transaction_id": transaction_id.clone(),
7078 "stage_dir": format!(".config-credential-stage-v1-{transaction_id}"),
7079 "state": "pending",
7080 "files": [
7081 {
7082 "name": "credentials.json",
7083 "staged_name": "credentials.json",
7084 "sha256": "0".repeat(64),
7085 "sensitive": true
7086 },
7087 {
7088 "name": "providers.json",
7089 "staged_name": "providers.json",
7090 "sha256": "1".repeat(64),
7091 "original_sha256": "2".repeat(64),
7092 "migration_generation": 2,
7093 "sensitive": false
7094 }
7095 ]
7096 }))
7097 .unwrap(),
7098 )
7099 .unwrap();
7100 }
7101
7102 async fn wait_for_mcp_health(
7103 state: &AppState,
7104 status: SectionStatus,
7105 minimum_revision: u64,
7106 ) -> ConfigLiveHealth {
7107 match tokio::time::timeout(Duration::from_secs(4), async {
7108 loop {
7109 let health = state
7110 .mcp_config_live_health
7111 .read()
7112 .unwrap_or_else(|poisoned| poisoned.into_inner())
7113 .clone();
7114 if health.status == status && health.revision >= minimum_revision {
7115 break health;
7116 }
7117 tokio::time::sleep(Duration::from_millis(20)).await;
7118 }
7119 })
7120 .await
7121 {
7122 Ok(health) => health,
7123 Err(_) => panic!(
7124 "MCP health transition timed out: {:?}",
7125 state
7126 .mcp_config_live_health
7127 .read()
7128 .unwrap_or_else(|poisoned| poisoned.into_inner())
7129 .clone()
7130 ),
7131 }
7132 }
7133
7134 async fn next_config_event(
7135 feed: &mut tokio::sync::broadcast::Receiver<Arc<bamboo_engine::events::ChangeEvent>>,
7136 expected_section: &str,
7137 ) -> AgentEvent {
7138 tokio::time::timeout(Duration::from_secs(3), async {
7139 loop {
7140 let envelope = feed.recv().await.expect("account feed remains open");
7141 match &envelope.event {
7142 AgentEvent::ConfigChanged { section, .. }
7143 | AgentEvent::ConfigInvalid { section, .. }
7144 | AgentEvent::ConfigRecovered { section, .. }
7145 if section == expected_section =>
7146 {
7147 break envelope.event.clone();
7148 }
7149 _ => {}
7150 }
7151 }
7152 })
7153 .await
7154 .expect("config event timed out")
7155 }
7156
7157 async fn next_mcp_config_event(
7158 feed: &mut tokio::sync::broadcast::Receiver<Arc<bamboo_engine::events::ChangeEvent>>,
7159 ) -> AgentEvent {
7160 next_config_event(feed, "mcp").await
7161 }
7162
7163 async fn wait_for_root_outbox_to_clear(data_dir: &Path) {
7164 tokio::time::timeout(Duration::from_secs(6), async {
7165 loop {
7166 if !bamboo_config::has_pending_legacy_root_publications(data_dir).unwrap() {
7167 break;
7168 }
7169 tokio::time::sleep(Duration::from_millis(20)).await;
7170 }
7171 })
7172 .await
7173 .expect("legacy root outbox did not clear");
7174 }
7175
7176 #[tokio::test]
7177 async fn compatibility_update_cannot_reintroduce_an_unrevisioned_cluster_mutation() {
7178 let _key = bamboo_config::encryption::set_test_encryption_key([0x70; 32]);
7179 let dir = tempfile::tempdir().unwrap();
7180 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
7181 state
7182 .update_cluster_fabric_credentials(
7183 0,
7184 std::collections::BTreeMap::from([(
7185 "owned-node".to_string(),
7186 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
7187 )]),
7188 |config| {
7189 config.cluster_fabric.nodes.push(bamboo_config::Node {
7190 id: "owned-node".to_string(),
7191 label: "revisioned-label".to_string(),
7192 placement: bamboo_config::NodePlacement::Local,
7193 trust_level: bamboo_config::TrustLevel::Trusted,
7194 deploy: bamboo_config::DeployProfile::default(),
7195 state: None,
7196 enabled: true,
7197 });
7198 Ok(())
7199 },
7200 )
7201 .await
7202 .unwrap();
7203 let cluster_path = dir.path().join("cluster-fabric.json");
7204 let cluster_before = std::fs::read(&cluster_path).unwrap();
7205
7206 let updated = state
7207 .update_config(
7208 |config| {
7209 config.server.port = 21_000;
7210 config.cluster_fabric.node_mut("owned-node").unwrap().label =
7211 "unrevisioned-label".to_string();
7212 Ok(())
7213 },
7214 ConfigUpdateEffects::default(),
7215 )
7216 .await
7217 .unwrap();
7218
7219 assert_eq!(updated.server.port, 21_000);
7220 assert_eq!(
7221 updated.cluster_fabric.node("owned-node").unwrap().label,
7222 "revisioned-label"
7223 );
7224 assert_eq!(
7225 state
7226 .config_facade
7227 .as_ref()
7228 .unwrap()
7229 .registry()
7230 .cluster_fabric
7231 .snapshot()
7232 .revision,
7233 1
7234 );
7235 assert_eq!(std::fs::read(cluster_path).unwrap(), cluster_before);
7236 }
7237
7238 #[tokio::test]
7239 async fn stopped_watcher_compatibility_writers_install_only_their_owned_section() {
7240 let _key = bamboo_config::encryption::set_test_encryption_key([0x72; 32]);
7241 let dir = tempfile::tempdir().unwrap();
7242 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
7243 state
7244 .update_cluster_fabric_credentials(
7245 0,
7246 BTreeMap::from([(
7247 "shared-node".to_string(),
7248 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
7249 )]),
7250 |config| {
7251 config.cluster_fabric.nodes.push(bamboo_config::Node {
7252 id: "shared-node".to_string(),
7253 label: "generation-one".to_string(),
7254 placement: bamboo_config::NodePlacement::Local,
7255 trust_level: bamboo_config::TrustLevel::Trusted,
7256 deploy: bamboo_config::DeployProfile::default(),
7257 state: None,
7258 enabled: true,
7259 });
7260 Ok(())
7261 },
7262 )
7263 .await
7264 .unwrap();
7265 stop_config_watcher(&mut state);
7266 let state = Arc::new(state);
7267
7268 let external = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
7269 let mut external_candidate = external.effective_config();
7270 external_candidate
7271 .cluster_fabric
7272 .node_mut("shared-node")
7273 .unwrap()
7274 .label = "external-generation-two".to_string();
7275 assert_eq!(
7276 bamboo_config::persist_cluster_fabric_credential_transaction_at_revision(
7277 dir.path(),
7278 &mut external_candidate,
7279 &BTreeMap::new(),
7280 1,
7281 )
7282 .unwrap(),
7283 2
7284 );
7285 let cluster_path = dir.path().join("cluster-fabric.json");
7286 let cluster_r2 = std::fs::read(&cluster_path).unwrap();
7287 assert_eq!(
7288 state
7289 .config_facade
7290 .as_ref()
7291 .unwrap()
7292 .registry()
7293 .cluster_fabric
7294 .snapshot()
7295 .revision,
7296 1
7297 );
7298 assert_eq!(
7299 state
7300 .config
7301 .read()
7302 .await
7303 .cluster_fabric
7304 .node("shared-node")
7305 .unwrap()
7306 .label,
7307 "generation-one"
7308 );
7309
7310 let baseline_seq = state.account_sink.latest_seq();
7311 let mut core_feed = state.account_sink.subscribe();
7312 let mut cluster_feed = state.account_sink.subscribe();
7313 let stale_runtime = state.config.read().await;
7314 let updating = {
7315 let state = state.clone();
7316 tokio::spawn(async move {
7317 state
7318 .update_config(
7319 |config| {
7320 config.server.port = 23_332;
7321 Ok(())
7322 },
7323 ConfigUpdateEffects::default(),
7324 )
7325 .await
7326 })
7327 };
7328 assert!(
7329 tokio::time::timeout(
7330 Duration::from_millis(100),
7331 next_config_event(&mut core_feed, "core"),
7332 )
7333 .await
7334 .is_err(),
7335 "core event became observable while the old AppState snapshot was held"
7336 );
7337 assert_ne!(stale_runtime.server.port, 23_332);
7338 drop(stale_runtime);
7339 let published = updating.await.unwrap().unwrap();
7340 assert!(matches!(
7341 next_config_event(&mut core_feed, "core").await,
7342 AgentEvent::ConfigChanged { section, .. } if section == "core"
7343 ));
7344 assert_eq!(state.config.read().await.server.port, 23_332);
7345 assert!(
7346 tokio::time::timeout(
7347 Duration::from_millis(300),
7348 next_config_event(&mut cluster_feed, "cluster-fabric"),
7349 )
7350 .await
7351 .is_err(),
7352 "an unrelated compatibility update published a cluster event"
7353 );
7354 assert_eq!(
7355 published.cluster_fabric.node("shared-node").unwrap().label,
7356 "generation-one"
7357 );
7358 assert_eq!(std::fs::read(&cluster_path).unwrap(), cluster_r2);
7359 assert_eq!(
7360 state
7361 .config_facade
7362 .as_ref()
7363 .unwrap()
7364 .registry()
7365 .cluster_fabric
7366 .snapshot()
7367 .revision,
7368 1,
7369 "an unrelated compatibility update must not catch up cluster"
7370 );
7371 assert_eq!(
7372 state
7373 .config
7374 .read()
7375 .await
7376 .cluster_fabric
7377 .node("shared-node")
7378 .unwrap()
7379 .label,
7380 "generation-one"
7381 );
7382 tokio::time::sleep(Duration::from_millis(100)).await;
7383 let events = bamboo_engine::events::journal::read_since(
7384 state.account_sink.events_dir(),
7385 baseline_seq,
7386 )
7387 .unwrap();
7388 assert_eq!(
7389 events
7390 .iter()
7391 .filter(|event| matches!(
7392 &event.event,
7393 AgentEvent::ConfigChanged { section, .. } if section == "core"
7394 ))
7395 .count(),
7396 1
7397 );
7398 assert!(!events.iter().any(|event| matches!(
7399 &event.event,
7400 AgentEvent::ConfigChanged { section, .. } if section == "cluster-fabric"
7401 )));
7402
7403 let external = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
7404 let mut external_candidate = external.effective_config();
7405 external_candidate
7406 .cluster_fabric
7407 .node_mut("shared-node")
7408 .unwrap()
7409 .label = "external-generation-three".to_string();
7410 assert_eq!(
7411 bamboo_config::persist_cluster_fabric_credential_transaction_at_revision(
7412 dir.path(),
7413 &mut external_candidate,
7414 &BTreeMap::new(),
7415 2,
7416 )
7417 .unwrap(),
7418 3
7419 );
7420 let cluster_r3 = std::fs::read(&cluster_path).unwrap();
7421 assert_eq!(
7422 state
7423 .config_facade
7424 .as_ref()
7425 .unwrap()
7426 .registry()
7427 .cluster_fabric
7428 .snapshot()
7429 .revision,
7430 1,
7431 "the stopped watcher must remain stale before replace_config"
7432 );
7433
7434 let mut replacement = state.config.read().await.clone();
7435 replacement.server.port = 23_333;
7436 let baseline_seq = state.account_sink.latest_seq();
7437 let mut core_feed = state.account_sink.subscribe();
7438 let mut cluster_feed = state.account_sink.subscribe();
7439 let stale_runtime = state.config.read().await;
7440 let replacing = {
7441 let state = state.clone();
7442 tokio::spawn(async move {
7443 state
7444 .replace_config(replacement, ConfigUpdateEffects::default())
7445 .await
7446 })
7447 };
7448 assert!(
7449 tokio::time::timeout(
7450 Duration::from_millis(100),
7451 next_config_event(&mut core_feed, "core"),
7452 )
7453 .await
7454 .is_err(),
7455 "replacement event became observable while the old AppState snapshot was held"
7456 );
7457 assert_ne!(stale_runtime.server.port, 23_333);
7458 drop(stale_runtime);
7459 let published = replacing.await.unwrap().unwrap();
7460 assert!(matches!(
7461 next_config_event(&mut core_feed, "core").await,
7462 AgentEvent::ConfigChanged { section, .. } if section == "core"
7463 ));
7464 assert_eq!(state.config.read().await.server.port, 23_333);
7465 assert!(
7466 tokio::time::timeout(
7467 Duration::from_millis(300),
7468 next_config_event(&mut cluster_feed, "cluster-fabric"),
7469 )
7470 .await
7471 .is_err(),
7472 "an unrelated compatibility replacement published a cluster event"
7473 );
7474 assert_eq!(published.server.port, 23_333);
7475 assert_eq!(
7476 published.cluster_fabric.node("shared-node").unwrap().label,
7477 "generation-one"
7478 );
7479 assert_eq!(std::fs::read(&cluster_path).unwrap(), cluster_r3);
7480 assert_eq!(
7481 state
7482 .config_facade
7483 .as_ref()
7484 .unwrap()
7485 .registry()
7486 .cluster_fabric
7487 .snapshot()
7488 .revision,
7489 1
7490 );
7491 assert_eq!(
7492 state
7493 .config
7494 .read()
7495 .await
7496 .cluster_fabric
7497 .node("shared-node")
7498 .unwrap()
7499 .label,
7500 "generation-one"
7501 );
7502 tokio::time::sleep(Duration::from_millis(100)).await;
7503 let events = bamboo_engine::events::journal::read_since(
7504 state.account_sink.events_dir(),
7505 baseline_seq,
7506 )
7507 .unwrap();
7508 assert_eq!(
7509 events
7510 .iter()
7511 .filter(|event| matches!(
7512 &event.event,
7513 AgentEvent::ConfigChanged { section, .. } if section == "core"
7514 ))
7515 .count(),
7516 1
7517 );
7518 assert!(!events.iter().any(|event| matches!(
7519 &event.event,
7520 AgentEvent::ConfigChanged { section, .. } if section == "cluster-fabric"
7521 )));
7522 }
7523
7524 #[tokio::test]
7525 async fn exact_notification_publication_installs_only_its_owned_runtime_section() {
7526 let dir = tempfile::tempdir().unwrap();
7527 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
7528 {
7529 let mut live = state.config.write().await;
7530 live.connect
7531 .platforms
7532 .push(bamboo_config::ConnectPlatformConfig {
7533 id: None,
7534 project_id: None,
7535 platform_type: "runtime-sentinel".to_string(),
7536 token: None,
7537 token_encrypted: None,
7538 token_credential_ref: None,
7539 token_configured: false,
7540 app_id: None,
7541 app_secret: None,
7542 app_secret_encrypted: None,
7543 app_secret_credential_ref: None,
7544 app_secret_configured: false,
7545 domain: None,
7546 allow_from: Vec::new(),
7547 admin_from: Vec::new(),
7548 });
7549 live.providers_mut().openai = Some(bamboo_config::OpenAIConfig {
7550 api_key: "runtime-provider-sentinel".to_string(),
7551 ..Default::default()
7552 });
7553 }
7554 let connect_before = std::fs::read(dir.path().join("connect.json")).unwrap();
7555 let (published, revision, _, section) = state
7556 .update_notification_credentials(0, BTreeSet::new(), false, |candidate| {
7557 candidate.notifications.ntfy.enabled = true;
7558 candidate.notifications.ntfy.topic = "owned-notification".to_string();
7559 candidate.assign_connect_platform_ids();
7563 candidate
7564 .providers_mut()
7565 .openai
7566 .as_mut()
7567 .unwrap()
7568 .api_key
7569 .clear();
7570 Ok(())
7571 })
7572 .await
7573 .unwrap();
7574
7575 assert_eq!(revision, 1);
7576 assert_eq!(section.unwrap().revision, 1);
7577 assert_eq!(published.notifications.ntfy.topic, "owned-notification");
7578 assert!(published.connect.platforms[0].id.is_none());
7579 assert_eq!(
7580 published.providers().openai.as_ref().unwrap().api_key,
7581 "runtime-provider-sentinel"
7582 );
7583 let live = state.config.read().await;
7584 assert!(live.connect.platforms[0].id.is_none());
7585 assert_eq!(
7586 live.providers().openai.as_ref().unwrap().api_key,
7587 "runtime-provider-sentinel"
7588 );
7589 drop(live);
7590 assert_eq!(
7591 std::fs::read(dir.path().join("connect.json")).unwrap(),
7592 connect_before
7593 );
7594 assert_eq!(
7595 bamboo_config::ConfigFacade::open(dir.path())
7596 .unwrap()
7597 .registry()
7598 .connect
7599 .snapshot()
7600 .revision,
7601 0
7602 );
7603 }
7604
7605 #[tokio::test]
7606 async fn generic_update_cannot_forge_exact_core_credential_binding() {
7607 let dir = tempfile::tempdir().unwrap();
7608 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
7609 let core_before = std::fs::read(dir.path().join("core.json")).unwrap();
7610 let error = state
7611 .update_config(
7612 |candidate| {
7613 candidate.proxy_auth_credential_ref =
7614 Some(bamboo_config::CredentialRef::parse("proxy.default.auth").unwrap());
7615 Ok(())
7616 },
7617 ConfigUpdateEffects::default(),
7618 )
7619 .await
7620 .unwrap_err();
7621 assert!(matches!(error, AppError::BadRequest(_)));
7622 assert!(error.to_string().contains("credential bindings"));
7623 assert_eq!(
7624 std::fs::read(dir.path().join("core.json")).unwrap(),
7625 core_before
7626 );
7627 assert!(state
7628 .config
7629 .read()
7630 .await
7631 .proxy_auth_credential_ref
7632 .is_none());
7633 assert_eq!(
7634 state
7635 .config_facade
7636 .as_ref()
7637 .unwrap()
7638 .registry()
7639 .core
7640 .snapshot()
7641 .revision,
7642 0
7643 );
7644 }
7645
7646 #[tokio::test]
7647 async fn env_credential_commit_installs_owned_runtime_before_exact_events() {
7648 let _key = bamboo_config::encryption::set_test_encryption_key([0x73; 32]);
7649 let dir = tempfile::tempdir().unwrap();
7650 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
7651 state
7652 .update_cluster_fabric_credentials(
7653 0,
7654 BTreeMap::from([(
7655 "shared-node".to_string(),
7656 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
7657 )]),
7658 |config| {
7659 config.cluster_fabric.nodes.push(bamboo_config::Node {
7660 id: "shared-node".to_string(),
7661 label: "generation-one".to_string(),
7662 placement: bamboo_config::NodePlacement::Local,
7663 trust_level: bamboo_config::TrustLevel::Trusted,
7664 deploy: bamboo_config::DeployProfile::default(),
7665 state: None,
7666 enabled: true,
7667 });
7668 Ok(())
7669 },
7670 )
7671 .await
7672 .unwrap();
7673 stop_config_watcher(&mut state);
7674 let state = Arc::new(state);
7675
7676 let external = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
7677 let mut external_candidate = external.effective_config();
7678 external_candidate
7679 .cluster_fabric
7680 .node_mut("shared-node")
7681 .unwrap()
7682 .label = "external-generation-two".to_string();
7683 bamboo_config::persist_cluster_fabric_credential_transaction_at_revision(
7684 dir.path(),
7685 &mut external_candidate,
7686 &BTreeMap::new(),
7687 1,
7688 )
7689 .unwrap();
7690 let cluster_path = dir.path().join("cluster-fabric.json");
7691 let cluster_r2 = std::fs::read(&cluster_path).unwrap();
7692 let expected_revision = state
7693 .config_facade
7694 .as_ref()
7695 .unwrap()
7696 .registry()
7697 .env
7698 .snapshot()
7699 .revision;
7700
7701 let (reached_tx, reached_rx) = std::sync::mpsc::channel();
7702 let (release_tx, release_rx) = std::sync::mpsc::channel();
7703 set_credential_after_commit_before_live_test_hook(dir.path(), SectionId::Env, move || {
7704 reached_tx.send(()).unwrap();
7705 release_rx.recv().unwrap();
7706 });
7707 let baseline_seq = state.account_sink.latest_seq();
7708 let mut credential_feed = state.account_sink.subscribe();
7709 let mut env_feed = state.account_sink.subscribe();
7710 let mut cluster_feed = state.account_sink.subscribe();
7711 let updating = {
7712 let state = state.clone();
7713 tokio::spawn(async move {
7714 state
7715 .update_env_var_credentials(
7716 expected_revision,
7717 BTreeSet::from(["TOKEN".to_string()]),
7718 false,
7719 |config| {
7720 config.env_vars.push(bamboo_config::EnvVarEntry {
7721 name: "TOKEN".to_string(),
7722 value: "exact-secret".to_string(),
7723 secret: true,
7724 value_encrypted: None,
7725 credential_ref: None,
7726 configured: true,
7727 description: None,
7728 });
7729 Ok(())
7730 },
7731 )
7732 .await
7733 })
7734 };
7735 tokio::task::spawn_blocking(move || reached_rx.recv().unwrap())
7736 .await
7737 .unwrap();
7738 let stale_runtime = state.config.read().await;
7739 release_tx.send(()).unwrap();
7740 for (feed, section) in [
7741 (&mut credential_feed, "credentials"),
7742 (&mut env_feed, "env"),
7743 (&mut cluster_feed, "cluster-fabric"),
7744 ] {
7745 assert!(
7746 tokio::time::timeout(Duration::from_millis(100), next_config_event(feed, section),)
7747 .await
7748 .is_err(),
7749 "{section} event became observable before the owned runtime install"
7750 );
7751 }
7752 assert!(
7753 stale_runtime
7754 .env_vars
7755 .iter()
7756 .all(|entry| entry.name != "TOKEN"),
7757 "the held runtime must still be the pre-commit env generation"
7758 );
7759 assert_eq!(
7760 stale_runtime
7761 .cluster_fabric
7762 .node("shared-node")
7763 .unwrap()
7764 .label,
7765 "generation-one"
7766 );
7767 drop(stale_runtime);
7768
7769 let (published, revision, _, _) = updating.await.unwrap().unwrap();
7770 assert!(revision > expected_revision);
7771 assert!(published
7772 .env_vars
7773 .iter()
7774 .any(|entry| entry.name == "TOKEN" && entry.value == "exact-secret"));
7775 assert_eq!(
7776 published.cluster_fabric.node("shared-node").unwrap().label,
7777 "generation-one"
7778 );
7779 assert!(matches!(
7780 next_config_event(&mut credential_feed, "credentials").await,
7781 AgentEvent::ConfigChanged { section, .. } if section == "credentials"
7782 ));
7783 assert!(matches!(
7784 next_config_event(&mut env_feed, "env").await,
7785 AgentEvent::ConfigChanged { section, .. } if section == "env"
7786 ));
7787 assert!(tokio::time::timeout(
7788 Duration::from_millis(300),
7789 next_config_event(&mut cluster_feed, "cluster-fabric"),
7790 )
7791 .await
7792 .is_err());
7793 assert_eq!(std::fs::read(cluster_path).unwrap(), cluster_r2);
7794 assert_eq!(
7795 state
7796 .config_facade
7797 .as_ref()
7798 .unwrap()
7799 .registry()
7800 .cluster_fabric
7801 .snapshot()
7802 .revision,
7803 1
7804 );
7805 tokio::time::sleep(Duration::from_millis(100)).await;
7806 let events = bamboo_engine::events::journal::read_since(
7807 state.account_sink.events_dir(),
7808 baseline_seq,
7809 )
7810 .unwrap();
7811 assert_eq!(
7812 events
7813 .iter()
7814 .filter(|event| matches!(
7815 &event.event,
7816 AgentEvent::ConfigChanged { section, .. } if section == "credentials"
7817 ))
7818 .count(),
7819 1
7820 );
7821 assert_eq!(
7822 events
7823 .iter()
7824 .filter(|event| matches!(
7825 &event.event,
7826 AgentEvent::ConfigChanged { section, .. } if section == "env"
7827 ))
7828 .count(),
7829 1
7830 );
7831 assert!(!events.iter().any(|event| matches!(
7832 &event.event,
7833 AgentEvent::ConfigChanged { section, .. } if section == "cluster-fabric"
7834 )));
7835 }
7836
7837 #[tokio::test]
7838 async fn env_mutation_returns_its_captured_envelope_after_a_later_section_commit() {
7839 let _key = bamboo_config::encryption::set_test_encryption_key([0x74; 32]);
7840 let dir = tempfile::tempdir().unwrap();
7841 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
7842 stop_config_watcher(&mut state);
7843 let state = Arc::new(state);
7844
7845 let (reached_tx, reached_rx) = std::sync::mpsc::channel();
7846 let (release_tx, release_rx) = std::sync::mpsc::channel();
7847 set_credential_after_commit_before_live_test_hook(dir.path(), SectionId::Env, move || {
7848 reached_tx.send(()).unwrap();
7849 release_rx.recv().unwrap();
7850 });
7851 let updating = {
7852 let state = state.clone();
7853 tokio::spawn(async move {
7854 state
7855 .update_env_var_credentials(
7856 0,
7857 BTreeSet::from(["TOKEN".to_string()]),
7858 false,
7859 |config| {
7860 config.env_vars.push(bamboo_config::EnvVarEntry {
7861 name: "TOKEN".to_string(),
7862 value: "first-secret".to_string(),
7863 secret: true,
7864 value_encrypted: None,
7865 credential_ref: None,
7866 configured: true,
7867 description: Some("first generation".to_string()),
7868 });
7869 Ok(())
7870 },
7871 )
7872 .await
7873 })
7874 };
7875 tokio::task::spawn_blocking(move || reached_rx.recv().unwrap())
7876 .await
7877 .unwrap();
7878
7879 let external_dir = dir.path().to_path_buf();
7880 let process_facade = state.config_facade.clone().unwrap();
7881 let later = tokio::task::spawn_blocking(move || {
7882 let external = bamboo_config::ConfigFacade::open(&external_dir).unwrap();
7883 let mut candidate = external.effective_config();
7884 candidate.env_vars[0].description = Some("later generation".to_string());
7885 bamboo_config::persist_env_var_credential_transaction_at_revision_with_adoption(
7886 &external_dir,
7887 &mut candidate,
7888 &BTreeSet::from(["TOKEN".to_string()]),
7889 1,
7890 process_facade.as_ref(),
7891 )
7892 .unwrap()
7893 })
7894 .await
7895 .unwrap();
7896 assert_eq!(later.revision, 2);
7897 assert_eq!(later.section.unwrap().revision, 2);
7898 release_tx.send(()).unwrap();
7899
7900 let (_, revision, _, section) = updating.await.unwrap().unwrap();
7901 let section = section.expect("modular mutation returns its exact section");
7902 assert_eq!(revision, 1);
7903 assert_eq!(section.revision, 1);
7904 assert_eq!(section.data[0]["description"], "first generation");
7905 assert_eq!(
7906 state
7907 .config_facade
7908 .as_ref()
7909 .unwrap()
7910 .registry()
7911 .env
7912 .snapshot()
7913 .revision,
7914 2,
7915 "the process facade advanced, but the response retained its own commit"
7916 );
7917 }
7918
7919 #[tokio::test]
7920 async fn cluster_commit_installs_runtime_before_one_authoritative_event() {
7921 let _key = bamboo_config::encryption::set_test_encryption_key([0x71; 32]);
7922 let dir = tempfile::tempdir().unwrap();
7923 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
7924 let revision = state
7925 .config_facade
7926 .as_ref()
7927 .unwrap()
7928 .registry()
7929 .cluster_fabric
7930 .snapshot()
7931 .revision;
7932 let baseline_seq = state.account_sink.latest_seq();
7933 let mut feed = state.account_sink.subscribe();
7934 let runtime = state.config.clone();
7935 let observer = tokio::spawn(async move {
7936 tokio::time::timeout(Duration::from_secs(3), async move {
7937 loop {
7938 let event = feed.recv().await.unwrap();
7939 match &event.event {
7940 AgentEvent::ConfigChanged { section, .. } if section == "credentials" => {
7941 panic!("cluster mutation published an internal credential event")
7942 }
7943 AgentEvent::ConfigChanged { section, revision }
7944 if section == "cluster-fabric" =>
7945 {
7946 assert!(
7947 runtime
7948 .read()
7949 .await
7950 .cluster_fabric
7951 .node("event-node")
7952 .is_some(),
7953 "event observer saw the old runtime snapshot"
7954 );
7955 return *revision;
7956 }
7957 _ => {}
7958 }
7959 }
7960 })
7961 .await
7962 .expect("cluster event timed out")
7963 });
7964
7965 let node = bamboo_config::Node {
7966 id: "event-node".to_string(),
7967 label: "event-node".to_string(),
7968 placement: bamboo_config::NodePlacement::Local,
7969 trust_level: bamboo_config::TrustLevel::Trusted,
7970 deploy: bamboo_config::DeployProfile::default(),
7971 state: None,
7972 enabled: true,
7973 };
7974 let committed = state
7975 .update_cluster_fabric_credentials(
7976 revision,
7977 BTreeMap::from([(
7978 "event-node".to_string(),
7979 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
7980 )]),
7981 move |config| {
7982 config.cluster_fabric.nodes.push(node);
7983 Ok(())
7984 },
7985 )
7986 .await
7987 .unwrap();
7988 let committed = committed.section.revision;
7989 assert_eq!(committed, revision + 1);
7990 assert_eq!(observer.await.unwrap(), committed);
7991
7992 tokio::time::sleep(Duration::from_millis(100)).await;
7993 let events = bamboo_engine::events::journal::read_since(
7994 state.account_sink.events_dir(),
7995 baseline_seq,
7996 )
7997 .unwrap();
7998 let cluster_events = events
7999 .iter()
8000 .filter(|event| {
8001 matches!(
8002 &event.event,
8003 AgentEvent::ConfigChanged { section, revision: event_revision }
8004 if section == "cluster-fabric" && *event_revision == committed
8005 )
8006 })
8007 .count();
8008 let credential_events = events
8009 .iter()
8010 .filter(|event| {
8011 matches!(
8012 &event.event,
8013 AgentEvent::ConfigChanged { section, .. } if section == "credentials"
8014 )
8015 })
8016 .count();
8017 assert_eq!(cluster_events, 1);
8018 assert_eq!(credential_events, 0);
8019 }
8020
8021 #[tokio::test]
8022 async fn stale_process_cluster_candidate_rebases_on_exact_durable_client_generation() {
8023 let dir = tempfile::tempdir().unwrap();
8024 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
8025 state
8026 .update_cluster_fabric_credentials(
8027 0,
8028 BTreeMap::from([(
8029 "shared-node".to_string(),
8030 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
8031 )]),
8032 |config| {
8033 config.cluster_fabric.nodes.push(bamboo_config::Node {
8034 id: "shared-node".to_string(),
8035 label: "generation-one".to_string(),
8036 placement: bamboo_config::NodePlacement::Local,
8037 trust_level: bamboo_config::TrustLevel::Trusted,
8038 deploy: bamboo_config::DeployProfile::default(),
8039 state: None,
8040 enabled: true,
8041 });
8042 Ok(())
8043 },
8044 )
8045 .await
8046 .unwrap();
8047 stop_config_watcher(&mut state);
8048
8049 let external = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
8050 let mut external_candidate = external.effective_config();
8051 external_candidate
8052 .cluster_fabric
8053 .clusters
8054 .push(bamboo_config::Cluster {
8055 name: "external-cluster".to_string(),
8056 description: Some("durable-r2-field".to_string()),
8057 node_ids: vec!["shared-node".to_string()],
8058 });
8059 assert_eq!(
8060 bamboo_config::persist_cluster_fabric_credential_transaction_at_revision(
8061 dir.path(),
8062 &mut external_candidate,
8063 &BTreeMap::new(),
8064 1,
8065 )
8066 .unwrap(),
8067 2
8068 );
8069 assert_eq!(
8070 state
8071 .config_facade
8072 .as_ref()
8073 .unwrap()
8074 .registry()
8075 .cluster_fabric
8076 .snapshot()
8077 .revision,
8078 1
8079 );
8080 assert!(
8081 state
8082 .config
8083 .read()
8084 .await
8085 .cluster_fabric
8086 .cluster("external-cluster")
8087 .is_none(),
8088 "the process runtime is intentionally stale at r1"
8089 );
8090
8091 let committed = state
8092 .update_cluster_fabric_credentials(2, BTreeMap::new(), |config| {
8093 config.cluster_fabric.node_mut("shared-node").unwrap().label =
8094 "client-r3-edit".to_string();
8095 Ok(())
8096 })
8097 .await
8098 .unwrap();
8099 assert_eq!(committed.section.revision, 3);
8100 assert_eq!(
8101 committed
8102 .config
8103 .cluster_fabric
8104 .node("shared-node")
8105 .unwrap()
8106 .label,
8107 "client-r3-edit"
8108 );
8109 assert_eq!(
8110 committed
8111 .config
8112 .cluster_fabric
8113 .cluster("external-cluster")
8114 .unwrap()
8115 .description
8116 .as_deref(),
8117 Some("durable-r2-field")
8118 );
8119 assert_eq!(
8120 state
8121 .config_facade
8122 .as_ref()
8123 .unwrap()
8124 .registry()
8125 .cluster_fabric
8126 .snapshot()
8127 .revision,
8128 3,
8129 "compound adoption must safely catch the stale r1 facade up to r3"
8130 );
8131
8132 let runtime_before_conflict = state.config.read().await.cluster_fabric.clone();
8133 let conflict = state
8134 .update_cluster_fabric_credentials(2, BTreeMap::new(), |config| {
8135 config.cluster_fabric.nodes.clear();
8136 config.cluster_fabric.clusters.clear();
8137 Ok(())
8138 })
8139 .await;
8140 assert!(matches!(
8141 conflict,
8142 Err(AppError::ConfigConflict {
8143 expected: 2,
8144 actual: 3
8145 })
8146 ));
8147 assert_eq!(
8148 state.config.read().await.cluster_fabric,
8149 runtime_before_conflict,
8150 "a durable CAS conflict must not overwrite the process runtime"
8151 );
8152 }
8153
8154 #[tokio::test]
8155 async fn stale_process_cluster_noop_catches_up_exact_durable_generation() {
8156 let dir = tempfile::tempdir().unwrap();
8157 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
8158 stop_config_watcher(&mut state);
8159
8160 let external = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
8161 let mut external_candidate = external.effective_config();
8162 external_candidate
8163 .cluster_fabric
8164 .nodes
8165 .push(bamboo_config::Node {
8166 id: "external-node".to_string(),
8167 label: "external-r1".to_string(),
8168 placement: bamboo_config::NodePlacement::Local,
8169 trust_level: bamboo_config::TrustLevel::Trusted,
8170 deploy: bamboo_config::DeployProfile::default(),
8171 state: None,
8172 enabled: true,
8173 });
8174 assert_eq!(
8175 bamboo_config::persist_cluster_fabric_credential_transaction_at_revision(
8176 dir.path(),
8177 &mut external_candidate,
8178 &BTreeMap::from([(
8179 "external-node".to_string(),
8180 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
8181 )]),
8182 0,
8183 )
8184 .unwrap(),
8185 1
8186 );
8187 assert_eq!(
8188 state
8189 .config_facade
8190 .as_ref()
8191 .unwrap()
8192 .registry()
8193 .cluster_fabric
8194 .snapshot()
8195 .revision,
8196 0
8197 );
8198 let baseline_seq = state.account_sink.latest_seq();
8199
8200 let committed = state
8201 .update_cluster_fabric_credentials(1, BTreeMap::new(), |_| Ok(()))
8202 .await
8203 .unwrap();
8204 assert_eq!(committed.section.revision, 1);
8205 assert_eq!(
8206 committed
8207 .config
8208 .cluster_fabric
8209 .node("external-node")
8210 .unwrap()
8211 .label,
8212 "external-r1"
8213 );
8214 assert_eq!(
8215 state
8216 .config_facade
8217 .as_ref()
8218 .unwrap()
8219 .registry()
8220 .cluster_fabric
8221 .snapshot()
8222 .revision,
8223 1
8224 );
8225 assert_eq!(
8226 state
8227 .config
8228 .read()
8229 .await
8230 .cluster_fabric
8231 .node("external-node")
8232 .unwrap()
8233 .label,
8234 "external-r1"
8235 );
8236
8237 tokio::time::sleep(Duration::from_millis(100)).await;
8238 let events = bamboo_engine::events::journal::read_since(
8239 state.account_sink.events_dir(),
8240 baseline_seq,
8241 )
8242 .unwrap();
8243 let revisions = events
8244 .iter()
8245 .filter_map(|event| match &event.event {
8246 AgentEvent::ConfigChanged { section, revision } if section == "cluster-fabric" => {
8247 Some(*revision)
8248 }
8249 _ => None,
8250 })
8251 .collect::<Vec<_>>();
8252 assert_eq!(revisions, vec![1]);
8253 }
8254
8255 #[tokio::test]
8256 async fn stale_process_cluster_reset_noop_catches_up_exact_durable_generation() {
8257 let dir = tempfile::tempdir().unwrap();
8258 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
8259 stop_config_watcher(&mut state);
8260
8261 let external = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
8262 let mut external_candidate = external.effective_config();
8263 external_candidate
8264 .cluster_fabric
8265 .nodes
8266 .push(bamboo_config::Node {
8267 id: "reset-node".to_string(),
8268 label: "reset-node".to_string(),
8269 placement: bamboo_config::NodePlacement::Local,
8270 trust_level: bamboo_config::TrustLevel::Trusted,
8271 deploy: bamboo_config::DeployProfile::default(),
8272 state: None,
8273 enabled: true,
8274 });
8275 assert_eq!(
8276 bamboo_config::persist_cluster_fabric_credential_transaction_at_revision(
8277 dir.path(),
8278 &mut external_candidate,
8279 &BTreeMap::from([(
8280 "reset-node".to_string(),
8281 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
8282 )]),
8283 0,
8284 )
8285 .unwrap(),
8286 1
8287 );
8288 let reset_facade = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
8289 let mut reset_candidate = reset_facade.effective_config();
8290 reset_candidate.cluster_fabric = bamboo_config::ClusterFabricConfig::default();
8291 let external_reset = bamboo_config::persist_cluster_fabric_reset_at_revision_with_adoption(
8292 dir.path(),
8293 &mut reset_candidate,
8294 1,
8295 &reset_facade,
8296 |_, _| {},
8297 )
8298 .unwrap();
8299 assert_eq!(external_reset.revision, 2);
8300 assert_eq!(
8301 state
8302 .config_facade
8303 .as_ref()
8304 .unwrap()
8305 .registry()
8306 .cluster_fabric
8307 .snapshot()
8308 .revision,
8309 0
8310 );
8311 let baseline_seq = state.account_sink.latest_seq();
8312
8313 let committed = state
8314 .reset_credential_backed_section(SectionId::ClusterFabric, 2)
8315 .await
8316 .unwrap();
8317 let CredentialBackedResetCommit::Cluster(committed) = committed else {
8318 panic!("cluster reset must return its exact snapshot")
8319 };
8320 assert_eq!(committed.section.revision, 2);
8321 assert!(committed.config.cluster_fabric.nodes.is_empty());
8322 assert_eq!(
8323 state
8324 .config_facade
8325 .as_ref()
8326 .unwrap()
8327 .registry()
8328 .cluster_fabric
8329 .snapshot()
8330 .revision,
8331 2
8332 );
8333 assert!(state.config.read().await.cluster_fabric.nodes.is_empty());
8334
8335 tokio::time::sleep(Duration::from_millis(100)).await;
8336 let events = bamboo_engine::events::journal::read_since(
8337 state.account_sink.events_dir(),
8338 baseline_seq,
8339 )
8340 .unwrap();
8341 let revisions = events
8342 .iter()
8343 .filter_map(|event| match &event.event {
8344 AgentEvent::ConfigChanged { section, revision } if section == "cluster-fabric" => {
8345 Some(*revision)
8346 }
8347 _ => None,
8348 })
8349 .collect::<Vec<_>>();
8350 assert_eq!(revisions, vec![2]);
8351 }
8352
8353 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8354 async fn generic_events_follow_serialized_local_commit_order() {
8355 let dir = tempfile::tempdir().unwrap();
8356 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
8357 stop_config_watcher(&mut state);
8358 let state = Arc::new(state);
8359 let baseline_seq = state.account_sink.latest_seq();
8360 let (reached_tx, reached_rx) = std::sync::mpsc::sync_channel(0);
8361 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
8362 set_generic_before_event_test_hook(dir.path(), move || {
8363 reached_tx.send(()).unwrap();
8364 release_rx.recv().unwrap();
8365 });
8366
8367 let first = {
8368 let state = state.clone();
8369 tokio::spawn(async move {
8370 state
8371 .update_config(
8372 |config| {
8373 config.server.port = 22_231;
8374 Ok(())
8375 },
8376 ConfigUpdateEffects::default(),
8377 )
8378 .await
8379 })
8380 };
8381 tokio::task::spawn_blocking(move || reached_rx.recv().unwrap())
8382 .await
8383 .unwrap();
8384 let second = {
8385 let state = state.clone();
8386 tokio::spawn(async move {
8387 state
8388 .update_config(
8389 |config| {
8390 config.server.port = 22_232;
8391 Ok(())
8392 },
8393 ConfigUpdateEffects::default(),
8394 )
8395 .await
8396 })
8397 };
8398 tokio::time::sleep(Duration::from_millis(100)).await;
8399 assert!(
8400 !second.is_finished(),
8401 "the later writer must remain behind the first writer's event"
8402 );
8403 let events = bamboo_engine::events::journal::read_since(
8404 state.account_sink.events_dir(),
8405 baseline_seq,
8406 )
8407 .unwrap();
8408 assert!(
8409 events.iter().all(|event| !matches!(
8410 &event.event,
8411 AgentEvent::ConfigChanged { section, .. } if section == "core"
8412 )),
8413 "neither local commit can publish while the first owns config_io_lock"
8414 );
8415
8416 release_tx.send(()).unwrap();
8417 assert_eq!(first.await.unwrap().unwrap().server.port, 22_231);
8418 assert_eq!(second.await.unwrap().unwrap().server.port, 22_232);
8419 tokio::time::timeout(Duration::from_secs(3), async {
8420 loop {
8421 let events = bamboo_engine::events::journal::read_since(
8422 state.account_sink.events_dir(),
8423 baseline_seq,
8424 )
8425 .unwrap();
8426 let revisions = events
8427 .iter()
8428 .filter_map(|event| match &event.event {
8429 AgentEvent::ConfigChanged { section, revision } if section == "core" => {
8430 Some(*revision)
8431 }
8432 _ => None,
8433 })
8434 .collect::<Vec<_>>();
8435 if revisions.len() == 2 {
8436 break revisions;
8437 }
8438 tokio::time::sleep(Duration::from_millis(20)).await;
8439 }
8440 })
8441 .await
8442 .map(|revisions| assert_eq!(revisions, vec![1, 2]))
8443 .expect("both serialized core events must reach the journal");
8444 }
8445
8446 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8447 async fn generic_runtime_effects_finish_before_later_config_writer() {
8448 let dir = tempfile::tempdir().unwrap();
8449 let script = dir.path().join("mcp-fixture.py");
8450 std::fs::write(
8451 &script,
8452 r#"import json
8453import sys
8454
8455for line in sys.stdin:
8456 request = json.loads(line)
8457 request_id = request.get("id")
8458 if request_id is None:
8459 continue
8460 if request.get("method") == "server/discover":
8461 print(json.dumps({
8462 "jsonrpc": "2.0",
8463 "id": request_id,
8464 "error": {"code": -32601, "message": "Method not found"},
8465 }), flush=True)
8466 continue
8467 if request.get("method") == "initialize":
8468 result = {
8469 "protocolVersion": "2024-11-05",
8470 "capabilities": {"tools": {"listChanged": False}},
8471 "serverInfo": {"name": "config-order-fixture", "version": "1.0.0"},
8472 }
8473 elif request.get("method") == "tools/list":
8474 result = {"tools": []}
8475 else:
8476 result = {}
8477 print(json.dumps({"jsonrpc": "2.0", "id": request_id, "result": result}), flush=True)
8478"#,
8479 )
8480 .unwrap();
8481 let python = ["python3", "python"]
8482 .into_iter()
8483 .find(|command| {
8484 std::process::Command::new(command)
8485 .arg("--version")
8486 .output()
8487 .is_ok_and(|output| output.status.success())
8488 })
8489 .expect("a Python interpreter is required for the MCP ordering fixture");
8490
8491 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
8492 stop_config_watcher(&mut state);
8493 let state = Arc::new(state);
8494 let held_provider = state.provider.write().await;
8495 let (provider_ready_tx, provider_ready_rx) = std::sync::mpsc::channel();
8496 set_generic_before_provider_publish_test_hook(dir.path(), move || {
8497 provider_ready_tx.send(()).unwrap();
8498 });
8499
8500 let first = {
8501 let state = state.clone();
8502 tokio::spawn(async move {
8503 state
8504 .update_config(
8505 |config| {
8506 config.provider = "copilot".to_string();
8507 Ok(())
8508 },
8509 ConfigUpdateEffects {
8510 reload_provider: bamboo_config::patch::ReloadMode::Strict,
8511 reconcile_mcp: bamboo_config::patch::ReloadMode::Strict,
8512 },
8513 )
8514 .await
8515 })
8516 };
8517 tokio::task::spawn_blocking(move || provider_ready_rx.recv().unwrap())
8518 .await
8519 .unwrap();
8520 assert!(
8521 state.config_io_lock.try_lock().is_err(),
8522 "the first writer must retain config_io_lock until its runtime effects finish"
8523 );
8524 assert!(
8525 !first.is_finished(),
8526 "the first writer must still be waiting to publish its provider"
8527 );
8528
8529 let later_mcp = McpConfig {
8530 version: 1,
8531 servers: vec![McpServerConfig {
8532 id: "later-winner".to_string(),
8533 name: None,
8534 enabled: true,
8535 transport: TransportConfig::Stdio(StdioConfig {
8536 command: python.to_string(),
8537 args: vec![script.to_string_lossy().into_owned()],
8538 cwd: None,
8539 env: std::collections::HashMap::new(),
8540 env_encrypted: std::collections::HashMap::new(),
8541 env_credential_refs: std::collections::HashMap::new(),
8542 startup_timeout_ms: 2_000,
8543 }),
8544 request_timeout_ms: 2_000,
8545 healthcheck_interval_ms: 10_000,
8546 reconnect: ReconnectConfig {
8547 enabled: false,
8548 ..Default::default()
8549 },
8550 allowed_tools: vec![],
8551 denied_tools: vec![],
8552 }],
8553 };
8554 let second = {
8555 let state = state.clone();
8556 tokio::spawn(async move {
8557 state
8558 .update_config(
8559 move |config| {
8560 config.mcp = later_mcp;
8561 Ok(())
8562 },
8563 ConfigUpdateEffects {
8564 reload_provider: bamboo_config::patch::ReloadMode::None,
8565 reconcile_mcp: bamboo_config::patch::ReloadMode::Strict,
8566 },
8567 )
8568 .await
8569 })
8570 };
8571
8572 drop(held_provider);
8573 first.await.unwrap().unwrap();
8574 let published = second.await.unwrap().unwrap();
8575 assert_eq!(published.mcp.servers[0].id, "later-winner");
8576 assert_eq!(state.config.read().await.mcp.servers[0].id, "later-winner");
8577 assert_eq!(
8578 state.mcp_manager.list_servers(),
8579 vec!["later-winner".to_string()],
8580 "the later durable config generation must remain the final runtime generation"
8581 );
8582 state.mcp_manager.shutdown_all().await;
8583 }
8584
8585 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8586 async fn direct_provider_reload_cannot_publish_after_later_config_generation() {
8587 let dir = tempfile::tempdir().unwrap();
8588 let mut initial = Config::default();
8589 initial.provider = "openai".to_string();
8590 initial.providers_mut().openai = Some(bamboo_config::OpenAIConfig {
8591 api_key: "first-generation-key".to_string(),
8592 base_url: Some("http://127.0.0.1:1/v1".to_string()),
8593 ..Default::default()
8594 });
8595 let mut state = AppState::new_with_provider(
8596 dir.path().to_path_buf(),
8597 initial,
8598 Arc::new(WorkingProvider),
8599 )
8600 .await
8601 .unwrap();
8602 stop_config_watcher(&mut state);
8603 let state = Arc::new(state);
8604 let quiesced = tokio::time::timeout(Duration::from_secs(5), state.config_io_lock.lock())
8605 .await
8606 .expect("startup config work must quiesce");
8607 drop(quiesced);
8608 let (reload_ready_tx, reload_ready_rx) = tokio::sync::oneshot::channel();
8609 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
8610 set_generic_before_provider_publish_test_hook(dir.path(), move || {
8611 let _ = reload_ready_tx.send(());
8612 release_rx.recv().unwrap();
8613 });
8614
8615 let reload = {
8616 let state = state.clone();
8617 tokio::spawn(async move { state.reload_provider().await })
8618 };
8619 tokio::time::timeout(Duration::from_secs(5), reload_ready_rx)
8620 .await
8621 .expect("direct reload reaches provider publication hook")
8622 .unwrap();
8623 assert!(state.config_io_lock.try_lock().is_err());
8624
8625 let mut later_instance: bamboo_config::ProviderInstanceConfig =
8626 serde_json::from_value(serde_json::json!({
8627 "provider_type": "openai",
8628 "base_url": "http://127.0.0.1:1/v1",
8629 "enabled": true
8630 }))
8631 .unwrap();
8632 later_instance.api_key = "later-generation-key".to_string();
8633 let (later_started_tx, later_started_rx) = tokio::sync::oneshot::channel();
8634 let later = {
8635 let state = state.clone();
8636 tokio::spawn(async move {
8637 let _ = later_started_tx.send(());
8638 state
8639 .update_config_with_provider_credentials(
8640 move |config| {
8641 config
8642 .provider_instances
8643 .insert("later-winner".to_string(), later_instance);
8644 config.default_provider_instance = Some("later-winner".to_string());
8645 Ok(())
8646 },
8647 BTreeSet::new(),
8648 BTreeSet::from(["later-winner".to_string()]),
8649 ConfigUpdateEffects {
8650 reload_provider: bamboo_config::patch::ReloadMode::Strict,
8651 reconcile_mcp: bamboo_config::patch::ReloadMode::None,
8652 },
8653 )
8654 .await
8655 })
8656 };
8657 later_started_rx.await.unwrap();
8658 assert!(!later.is_finished());
8659
8660 release_tx.send(()).unwrap();
8661 tokio::time::timeout(Duration::from_secs(10), async {
8662 reload.await.unwrap().unwrap();
8663 later.await.unwrap().unwrap();
8664 })
8665 .await
8666 .expect("serialized provider generations finish");
8667 assert_eq!(
8668 state
8669 .config
8670 .read()
8671 .await
8672 .default_provider_instance
8673 .as_deref(),
8674 Some("later-winner")
8675 );
8676 assert_eq!(
8677 state.provider_registry.default_provider_name(),
8678 "later-winner"
8679 );
8680 let registry_default = state.provider_registry.get_default().unwrap();
8681 let live_provider = state.provider.read().await.clone();
8682 assert!(
8683 Arc::ptr_eq(®istry_default, &live_provider),
8684 "registry default and reloadable provider handle must publish one generation"
8685 );
8686 }
8687
8688 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8689 async fn combined_reload_cannot_publish_captured_mcp_after_later_generation() {
8690 let dir = tempfile::tempdir().unwrap();
8691 let mut initial = Config::default();
8692 initial.provider = "openai".to_string();
8693 initial.providers_mut().openai = Some(bamboo_config::OpenAIConfig {
8694 api_key: "combined-reload-key".to_string(),
8695 base_url: Some("http://127.0.0.1:1/v1".to_string()),
8696 ..Default::default()
8697 });
8698 let mut state = AppState::new_with_provider(
8699 dir.path().to_path_buf(),
8700 initial,
8701 Arc::new(WorkingProvider),
8702 )
8703 .await
8704 .unwrap();
8705 stop_config_watcher(&mut state);
8706 state
8707 .update_config_with_provider_credentials(
8708 |_| Ok(()),
8709 BTreeSet::from(["openai".to_string()]),
8710 BTreeSet::new(),
8711 ConfigUpdateEffects::default(),
8712 )
8713 .await
8714 .unwrap();
8715 let first_mcp = working_stdio_mcp_config(dir.path(), "captured-first", None);
8716 state
8717 .update_config(
8718 move |config| {
8719 config.mcp = first_mcp;
8720 Ok(())
8721 },
8722 ConfigUpdateEffects::default(),
8723 )
8724 .await
8725 .unwrap();
8726 let state = Arc::new(state);
8727 let (reload_ready_tx, reload_ready_rx) = tokio::sync::oneshot::channel();
8728 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
8729 set_generic_before_provider_publish_test_hook(dir.path(), move || {
8730 let _ = reload_ready_tx.send(());
8731 release_rx.recv().unwrap();
8732 });
8733
8734 let reload = {
8735 let state = state.clone();
8736 tokio::spawn(async move { state.reload_config_and_runtime().await })
8737 };
8738 tokio::time::timeout(Duration::from_secs(5), reload_ready_rx)
8739 .await
8740 .expect("combined reload reaches provider publication hook")
8741 .unwrap();
8742 assert!(state.config_io_lock.try_lock().is_err());
8743
8744 let later_mcp = working_stdio_mcp_config(dir.path(), "later-winner", None);
8745 let (later_started_tx, later_started_rx) = tokio::sync::oneshot::channel();
8746 let later = {
8747 let state = state.clone();
8748 tokio::spawn(async move {
8749 let _ = later_started_tx.send(());
8750 state
8751 .update_config(
8752 move |config| {
8753 config.mcp = later_mcp;
8754 Ok(())
8755 },
8756 ConfigUpdateEffects {
8757 reload_provider: bamboo_config::patch::ReloadMode::None,
8758 reconcile_mcp: bamboo_config::patch::ReloadMode::Strict,
8759 },
8760 )
8761 .await
8762 })
8763 };
8764 later_started_rx.await.unwrap();
8765 assert!(!later.is_finished());
8766
8767 release_tx.send(()).unwrap();
8768 tokio::time::timeout(Duration::from_secs(10), async {
8769 reload.await.unwrap().unwrap();
8770 later.await.unwrap().unwrap();
8771 })
8772 .await
8773 .expect("serialized config/runtime generations finish");
8774 assert_eq!(state.config.read().await.mcp.servers[0].id, "later-winner");
8775 assert_eq!(
8776 state.mcp_manager.list_servers(),
8777 vec!["later-winner".to_string()]
8778 );
8779 state.mcp_manager.shutdown_all().await;
8780 }
8781
8782 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8783 async fn legacy_mcp_credentials_round_trip_without_plaintext_in_durable_or_events() {
8784 let dir = tempfile::tempdir().unwrap();
8785 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
8786 stop_config_watcher(&mut state);
8787 let state = Arc::new(state);
8788 let baseline_seq = state.account_sink.latest_seq();
8789 let secret = "legacy-mcp-roundtrip-secret";
8790 let mut candidate = disabled_mcp_config("credential-server");
8791 let TransportConfig::Stdio(stdio) = &mut candidate.servers[0].transport else {
8792 unreachable!()
8793 };
8794 stdio.env.insert("TOKEN".to_string(), secret.to_string());
8795
8796 state
8797 .update_legacy_mcp_config(BTreeSet::new(), move |mcp| {
8798 *mcp = candidate;
8799 Ok(())
8800 })
8801 .await
8802 .unwrap();
8803 let live = state.config.read().await.clone();
8804 let TransportConfig::Stdio(stdio) = &live.mcp.servers[0].transport else {
8805 unreachable!()
8806 };
8807 assert_eq!(stdio.env["TOKEN"], secret);
8808 let reference =
8809 bamboo_config::CredentialRef::parse(stdio.env_credential_refs["TOKEN"].clone())
8810 .unwrap();
8811 assert_eq!(
8812 state
8813 .credential_store
8814 .resolve(&reference)
8815 .unwrap()
8816 .unwrap()
8817 .expose(),
8818 secret
8819 );
8820 for path in [
8821 dir.path().join("mcp.json"),
8822 dir.path().join("credentials.json"),
8823 ] {
8824 let bytes = std::fs::read(path).unwrap();
8825 assert!(!String::from_utf8_lossy(&bytes).contains(secret));
8826 }
8827 let events = bamboo_engine::events::journal::read_since(
8828 state.account_sink.events_dir(),
8829 baseline_seq,
8830 )
8831 .unwrap();
8832 assert!(!format!("{events:?}").contains(secret));
8833
8834 state
8835 .update_legacy_mcp_config(BTreeSet::new(), |mcp| {
8836 let TransportConfig::Stdio(stdio) = &mut mcp.servers[0].transport else {
8837 unreachable!()
8838 };
8839 stdio
8840 .env
8841 .insert("TOKEN".to_string(), "****...****".to_string());
8842 Ok(())
8843 })
8844 .await
8845 .unwrap();
8846 let live = state.config.read().await.clone();
8847 let TransportConfig::Stdio(stdio) = &live.mcp.servers[0].transport else {
8848 unreachable!()
8849 };
8850 assert_eq!(stdio.env["TOKEN"], secret);
8851 assert_eq!(stdio.env_credential_refs["TOKEN"], reference.as_str());
8852 }
8853
8854 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8855 async fn legacy_mcp_cancelled_start_finishes_before_later_delete() {
8856 let dir = tempfile::tempdir().unwrap();
8857 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
8858 stop_config_watcher(&mut state);
8859 let state = Arc::new(state);
8860 let secret = "legacy-mcp-cancel-secret";
8861 let candidate = working_stdio_mcp_config(dir.path(), "cancelled-start", Some(secret));
8862 let reference =
8863 bamboo_config::credential_ref("mcp", "cancelled-start", "env_TOKEN").unwrap();
8864 let (commit_tx, commit_rx) = tokio::sync::oneshot::channel();
8865 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
8866 set_credential_after_commit_before_live_test_hook(dir.path(), SectionId::Mcp, move || {
8867 let _ = commit_tx.send(());
8868 release_rx.recv().unwrap();
8869 });
8870
8871 let operation = {
8872 let state = state.clone();
8873 tokio::spawn(async move {
8874 state
8875 .update_legacy_mcp_config(BTreeSet::new(), move |mcp| {
8876 *mcp = candidate;
8877 Ok(())
8878 })
8879 .await
8880 })
8881 };
8882 tokio::time::timeout(Duration::from_secs(5), commit_rx)
8883 .await
8884 .expect("legacy MCP write reaches durable-before-live hook")
8885 .unwrap();
8886 assert!(state.config_io_lock.try_lock().is_err());
8887 assert!(state.mcp_manager.list_servers().is_empty());
8888 assert!(
8889 !String::from_utf8_lossy(&std::fs::read(dir.path().join("mcp.json")).unwrap())
8890 .contains(secret)
8891 );
8892 operation.abort();
8893 assert!(operation.await.unwrap_err().is_cancelled());
8894
8895 let (delete_started_tx, delete_started_rx) = tokio::sync::oneshot::channel();
8896 let delete = {
8897 let state = state.clone();
8898 tokio::spawn(async move {
8899 let _ = delete_started_tx.send(());
8900 state
8901 .update_legacy_mcp_config(BTreeSet::new(), |mcp| {
8902 mcp.servers.clear();
8903 Ok(())
8904 })
8905 .await
8906 })
8907 };
8908 delete_started_rx.await.unwrap();
8909 assert!(!delete.is_finished());
8910 release_tx.send(()).unwrap();
8911 delete.await.unwrap().unwrap();
8912
8913 assert!(state.config.read().await.mcp.servers.is_empty());
8914 assert!(state.mcp_manager.list_servers().is_empty());
8915 assert!(state.mcp_manager.tool_index().all_aliases().is_empty());
8916 assert!(state
8917 .credential_store
8918 .resolve(&reference)
8919 .unwrap()
8920 .is_none());
8921 assert!(
8922 !String::from_utf8_lossy(&std::fs::read(dir.path().join("mcp.json")).unwrap())
8923 .contains(secret)
8924 );
8925 }
8926
8927 #[tokio::test]
8928 async fn legacy_mcp_rejects_secret_bearing_url_before_runtime_or_commit() {
8929 let dir = tempfile::tempdir().unwrap();
8930 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
8931 stop_config_watcher(&mut state);
8932 let before = std::fs::read(dir.path().join("mcp.json")).unwrap();
8933 let secret = "must-never-connect-or-log";
8934 let candidate = McpConfig {
8935 version: 1,
8936 servers: vec![McpServerConfig {
8937 id: "unsafe-url".to_string(),
8938 name: None,
8939 enabled: true,
8940 transport: TransportConfig::Sse(bamboo_mcp::SseConfig {
8941 url: format!("https://example.test/sse?token={secret}"),
8942 headers: vec![],
8943 connect_timeout_ms: 100,
8944 }),
8945 request_timeout_ms: 100,
8946 healthcheck_interval_ms: 100,
8947 reconnect: ReconnectConfig::default(),
8948 allowed_tools: vec![],
8949 denied_tools: vec![],
8950 }],
8951 };
8952 let error = state
8953 .update_legacy_mcp_config(BTreeSet::new(), move |mcp| {
8954 *mcp = candidate;
8955 Ok(())
8956 })
8957 .await
8958 .unwrap_err();
8959 assert!(matches!(error, AppError::BadRequest(_)));
8960 assert!(!error.to_string().contains(secret));
8961 assert_eq!(std::fs::read(dir.path().join("mcp.json")).unwrap(), before);
8962 assert!(state.config.read().await.mcp.servers.is_empty());
8963 assert!(state.mcp_manager.list_servers().is_empty());
8964 }
8965
8966 #[tokio::test]
8967 async fn legacy_mcp_start_failure_keeps_every_authority_on_the_previous_generation() {
8968 let dir = tempfile::tempdir().unwrap();
8969 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
8970 stop_config_watcher(&mut state);
8971 let disk_before = std::fs::read(dir.path().join("mcp.json")).unwrap();
8972 let config_before = state.config.read().await.clone();
8973 let facade_before = state
8974 .config_facade
8975 .as_ref()
8976 .unwrap()
8977 .registry()
8978 .mcp
8979 .snapshot();
8980 let health_before = state
8981 .mcp_config_live_health
8982 .read()
8983 .unwrap_or_else(|poisoned| poisoned.into_inner())
8984 .clone();
8985 let baseline_seq = state.account_sink.latest_seq();
8986 let mut failing = disabled_mcp_config("never-committed");
8987 failing.servers[0].enabled = true;
8988 let TransportConfig::Stdio(stdio) = &mut failing.servers[0].transport else {
8989 unreachable!()
8990 };
8991 stdio.command = "definitely-not-a-real-mcp-command-before-commit-736".to_string();
8992
8993 let error = state
8994 .update_legacy_mcp_config(BTreeSet::new(), move |mcp| {
8995 *mcp = failing;
8996 Ok(())
8997 })
8998 .await
8999 .expect_err("runtime staging must fail before the MCP durable boundary");
9000 assert!(matches!(error, AppError::InternalError(_)));
9001 assert_eq!(
9002 error.to_string(),
9003 "Internal server error: MCP runtime initialization failed before commit; retaining last-known-good generation"
9004 );
9005 assert_eq!(
9006 std::fs::read(dir.path().join("mcp.json")).unwrap(),
9007 disk_before
9008 );
9009 assert_eq!(
9010 serde_json::to_value(state.config.read().await.clone()).unwrap(),
9011 serde_json::to_value(config_before).unwrap()
9012 );
9013 assert!(state.mcp_manager.list_servers().is_empty());
9014 assert!(state.mcp_manager.tool_index().all_aliases().is_empty());
9015
9016 let facade_after = state
9017 .config_facade
9018 .as_ref()
9019 .unwrap()
9020 .registry()
9021 .mcp
9022 .snapshot();
9023 assert_eq!(facade_after.revision, facade_before.revision);
9024 assert_eq!(facade_after.loaded_at, facade_before.loaded_at);
9025 assert_eq!(facade_after.status, facade_before.status);
9026 let health_after = state
9027 .mcp_config_live_health
9028 .read()
9029 .unwrap_or_else(|poisoned| poisoned.into_inner())
9030 .clone();
9031 assert_eq!(health_after.revision, health_before.revision);
9032 assert_eq!(health_after.loaded_at, health_before.loaded_at);
9033 assert_eq!(health_after.status, health_before.status);
9034 assert_eq!(health_after.last_error, health_before.last_error);
9035 assert!(bamboo_engine::events::journal::read_since(
9036 state.account_sink.events_dir(),
9037 baseline_seq,
9038 )
9039 .unwrap()
9040 .into_iter()
9041 .all(|event| !matches!(
9042 event.event,
9043 AgentEvent::ConfigChanged { ref section, .. }
9044 | AgentEvent::ConfigInvalid { ref section, .. }
9045 | AgentEvent::ConfigRecovered { ref section, .. }
9046 if section == "mcp"
9047 )));
9048 }
9049
9050 #[tokio::test]
9051 async fn invalid_explicit_reload_retains_live_provider_generation_and_marks_health() {
9052 let dir = tempfile::tempdir().unwrap();
9053 let mut initial = Config::default();
9054 initial.server.port = 24_301;
9055 initial.save_to_dir(dir.path().to_path_buf()).unwrap();
9056 let injected: Arc<dyn LLMProvider> = Arc::new(WorkingProvider);
9057 let mut state = AppState::new_with_provider(
9058 dir.path().to_path_buf(),
9059 initial.clone(),
9060 injected.clone(),
9061 )
9062 .await
9063 .unwrap();
9064 stop_config_watcher(&mut state);
9065 let expected_provider = state.config.read().await.provider.clone();
9066
9067 let mut invalid = initial;
9068 invalid.provider = "unknown-invalid-provider".to_string();
9069 invalid.save_to_dir(dir.path().to_path_buf()).unwrap();
9070 let error = state.reload_config_and_runtime().await.unwrap_err();
9071 assert!(matches!(error, AppError::BadRequest(_)));
9072 assert_eq!(state.config.read().await.server.port, 24_301);
9073 assert_eq!(state.config.read().await.provider, expected_provider);
9074 let live_provider = state.provider.read().await.clone();
9075 assert!(Arc::ptr_eq(&live_provider, &injected));
9076 let health = state
9077 .config_live_health
9078 .read()
9079 .unwrap_or_else(|poisoned| poisoned.into_inner())
9080 .clone();
9081 assert_eq!(health.status, SectionStatus::Invalid);
9082 assert_eq!(
9083 health.last_error.as_deref(),
9084 Some("provider configuration is invalid; retaining last-known-good generation")
9085 );
9086 }
9087
9088 #[tokio::test]
9089 async fn committed_provider_start_failure_publishes_one_exact_revision() {
9090 let dir = tempfile::tempdir().unwrap();
9091 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9092 stop_config_watcher(&mut state);
9093 let baseline_seq = state.account_sink.latest_seq();
9094 let previous_provider = state.provider.read().await.clone();
9095 let previous_default = state.provider_registry.default_provider_name();
9096
9097 let published = state
9098 .update_config(
9099 |config| {
9100 config.provider = "unknown-runtime-provider".to_string();
9101 config.default_provider_instance = None;
9102 config.provider_instances.clear();
9103 Ok(())
9104 },
9105 ConfigUpdateEffects {
9106 reload_provider: bamboo_config::patch::ReloadMode::BestEffort,
9107 reconcile_mcp: bamboo_config::patch::ReloadMode::None,
9108 },
9109 )
9110 .await
9111 .expect("the durable provider generation commits with degraded runtime health");
9112 assert_eq!(published.provider, "unknown-runtime-provider");
9113 assert_eq!(
9114 state.config.read().await.provider,
9115 "unknown-runtime-provider"
9116 );
9117 assert_eq!(
9118 state.provider_registry.default_provider_name(),
9119 previous_default
9120 );
9121 assert!(Arc::ptr_eq(
9122 &state.provider.read().await.clone(),
9123 &previous_provider
9124 ));
9125
9126 let facade_snapshot = state
9127 .config_facade
9128 .as_ref()
9129 .unwrap()
9130 .registry()
9131 .providers
9132 .snapshot();
9133 let health = state
9134 .config_live_health
9135 .read()
9136 .unwrap_or_else(|poisoned| poisoned.into_inner())
9137 .clone();
9138 assert_eq!(facade_snapshot.revision, 1);
9139 assert_eq!(facade_snapshot.status, SectionStatus::Degraded);
9140 assert_eq!(health.revision, facade_snapshot.revision);
9141 assert_eq!(health.loaded_at, facade_snapshot.loaded_at);
9142 assert_eq!(health.source_path, facade_snapshot.source_path);
9143 assert_eq!(health.source_kind, facade_snapshot.source_kind);
9144 assert_eq!(health.status, facade_snapshot.status);
9145 assert_eq!(health.last_error, facade_snapshot.last_error);
9146
9147 let invalid_revisions = bamboo_engine::events::journal::read_since(
9148 state.account_sink.events_dir(),
9149 baseline_seq,
9150 )
9151 .unwrap()
9152 .into_iter()
9153 .filter_map(|event| match event.event {
9154 AgentEvent::ConfigInvalid { section, revision } if section == "providers" => {
9155 Some(revision)
9156 }
9157 _ => None,
9158 })
9159 .collect::<Vec<_>>();
9160 assert_eq!(invalid_revisions, vec![facade_snapshot.revision]);
9161 }
9162
9163 #[tokio::test]
9164 async fn committed_mcp_start_failure_publishes_one_exact_revision() {
9165 let dir = tempfile::tempdir().unwrap();
9166 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9167 stop_config_watcher(&mut state);
9168 let baseline_seq = state.account_sink.latest_seq();
9169 let mut failing = disabled_mcp_config("committed-but-unstartable");
9170 failing.servers[0].enabled = true;
9171 let TransportConfig::Stdio(stdio) = &mut failing.servers[0].transport else {
9172 unreachable!()
9173 };
9174 stdio.command = "definitely-not-a-real-mcp-command-736".to_string();
9175
9176 let published = state
9177 .update_config(
9178 move |config| {
9179 config.mcp = failing;
9180 Ok(())
9181 },
9182 ConfigUpdateEffects {
9183 reload_provider: bamboo_config::patch::ReloadMode::None,
9184 reconcile_mcp: bamboo_config::patch::ReloadMode::BestEffort,
9185 },
9186 )
9187 .await
9188 .expect("the durable MCP generation commits with degraded runtime health");
9189 assert_eq!(published.mcp.servers[0].id, "committed-but-unstartable");
9190 assert_eq!(
9191 state.config.read().await.mcp.servers[0].id,
9192 "committed-but-unstartable"
9193 );
9194 assert!(state.mcp_manager.list_servers().is_empty());
9195
9196 let facade_snapshot = state
9197 .config_facade
9198 .as_ref()
9199 .unwrap()
9200 .registry()
9201 .mcp
9202 .snapshot();
9203 let health = state
9204 .mcp_config_live_health
9205 .read()
9206 .unwrap_or_else(|poisoned| poisoned.into_inner())
9207 .clone();
9208 assert_eq!(facade_snapshot.revision, 1);
9209 assert_eq!(facade_snapshot.status, SectionStatus::Degraded);
9210 assert_eq!(health.revision, facade_snapshot.revision);
9211 assert_eq!(health.loaded_at, facade_snapshot.loaded_at);
9212 assert_eq!(health.source_path, facade_snapshot.source_path);
9213 assert_eq!(health.source_kind, facade_snapshot.source_kind);
9214 assert_eq!(health.status, facade_snapshot.status);
9215 assert_eq!(health.last_error, facade_snapshot.last_error);
9216
9217 let invalid_revisions = bamboo_engine::events::journal::read_since(
9218 state.account_sink.events_dir(),
9219 baseline_seq,
9220 )
9221 .unwrap()
9222 .into_iter()
9223 .filter_map(|event| match event.event {
9224 AgentEvent::ConfigInvalid { section, revision } if section == "mcp" => Some(revision),
9225 _ => None,
9226 })
9227 .collect::<Vec<_>>();
9228 assert_eq!(invalid_revisions, vec![facade_snapshot.revision]);
9229 }
9230
9231 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
9232 async fn cancelled_legacy_reset_finishes_deletion_and_runtime_publication() {
9233 let dir = tempfile::tempdir().unwrap();
9234 let mut initial = Config::default();
9235 initial.server.port = 24_302;
9236 initial.save_to_dir(dir.path().to_path_buf()).unwrap();
9237 std::fs::write(dir.path().join("config.json.bak"), b"recovery-marker").unwrap();
9238 std::fs::write(dir.path().join("model_limits.json"), b"{}").unwrap();
9239 std::fs::write(dir.path().join("connect.json"), b"{}").unwrap();
9240 std::fs::write(dir.path().join("connect.json.bak"), b"credential-backup").unwrap();
9241 let injected: Arc<dyn LLMProvider> = Arc::new(WorkingProvider);
9242 let mut state =
9243 AppState::new_with_provider(dir.path().to_path_buf(), initial, injected.clone())
9244 .await
9245 .unwrap();
9246 stop_config_watcher(&mut state);
9247 let state = Arc::new(state);
9248 let (deleted_tx, deleted_rx) = tokio::sync::oneshot::channel();
9249 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
9250 set_reset_after_delete_test_hook(dir.path(), move || {
9251 let _ = deleted_tx.send(());
9252 release_rx.recv().unwrap();
9253 });
9254
9255 let operation = {
9256 let state = state.clone();
9257 tokio::spawn(async move { state.reset_legacy_config_and_runtime().await })
9258 };
9259 tokio::time::timeout(Duration::from_secs(5), deleted_rx)
9260 .await
9261 .expect("reset reaches durable delete boundary")
9262 .unwrap();
9263 for path in [
9264 dir.path().join("config.json"),
9265 dir.path().join("model_limits.json"),
9266 dir.path().join("connect.json"),
9267 dir.path().join("connect.json.bak"),
9268 ] {
9269 assert!(!path.exists());
9270 }
9271 assert_eq!(
9272 std::fs::read(dir.path().join("config.json.bak")).unwrap(),
9273 b"recovery-marker"
9274 );
9275 assert_eq!(state.config.read().await.server.port, 24_302);
9276 operation.abort();
9277 assert!(operation.await.unwrap_err().is_cancelled());
9278
9279 release_tx.send(()).unwrap();
9280 let completed = tokio::time::timeout(Duration::from_secs(5), state.config_io_lock.lock())
9281 .await
9282 .expect("detached reset must finish live/runtime publication");
9283 drop(completed);
9284 assert_eq!(state.config.read().await.server.port, 9562);
9285 assert!(state.mcp_manager.list_servers().is_empty());
9286 let health = state
9287 .config_live_health
9288 .read()
9289 .unwrap_or_else(|poisoned| poisoned.into_inner())
9290 .clone();
9291 assert_eq!(
9292 health.status,
9293 SectionStatus::Degraded,
9294 "the committed default config has no usable Anthropic credential, so reset must report truthful runtime degradation"
9295 );
9296 assert_eq!(
9297 health.last_error.as_deref(),
9298 Some("provider runtime initialization failed; retaining last-known-good runtime")
9299 );
9300 assert!(Arc::ptr_eq(&state.provider.read().await.clone(), &injected));
9301 }
9302
9303 #[tokio::test]
9304 async fn legacy_reset_converges_runtime_after_partial_delete_failure() {
9305 let dir = tempfile::tempdir().unwrap();
9306 let mut initial = Config::default();
9307 initial.server.port = 24_303;
9308 initial.save_to_dir(dir.path().to_path_buf()).unwrap();
9309 std::fs::write(dir.path().join("model_limits.json"), b"{}").unwrap();
9310 let injected: Arc<dyn LLMProvider> = Arc::new(WorkingProvider);
9311 let mut state =
9312 AppState::new_with_provider(dir.path().to_path_buf(), initial, injected.clone())
9313 .await
9314 .unwrap();
9315 stop_config_watcher(&mut state);
9316
9317 std::fs::create_dir(dir.path().join("connect.json")).unwrap();
9321 std::fs::write(dir.path().join("connect.json.bak"), b"credential-backup").unwrap();
9322
9323 let error = state.reset_legacy_config_and_runtime().await.unwrap_err();
9324 assert!(matches!(error, AppError::StorageError(_)));
9325 assert!(!dir.path().join("config.json").exists());
9326 assert!(!dir.path().join("model_limits.json").exists());
9327 assert!(dir.path().join("connect.json").is_dir());
9328 assert!(!dir.path().join("connect.json.bak").exists());
9329 assert_eq!(state.config.read().await.server.port, 9562);
9330 assert!(state.mcp_manager.list_servers().is_empty());
9331 assert!(Arc::ptr_eq(&state.provider.read().await.clone(), &injected));
9332 }
9333
9334 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
9335 async fn generic_update_cancellation_after_commit_finishes_publication() {
9336 let dir = tempfile::tempdir().unwrap();
9337 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9338 stop_config_watcher(&mut state);
9339 let state = Arc::new(state);
9340 let mut feed = state.account_sink.subscribe();
9341 let (reached_tx, reached_rx) = std::sync::mpsc::sync_channel(0);
9342 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
9343 set_generic_before_event_test_hook(dir.path(), move || {
9344 reached_tx.send(()).unwrap();
9345 release_rx.recv().unwrap();
9346 });
9347
9348 let operation = {
9349 let state = state.clone();
9350 tokio::spawn(async move {
9351 state
9352 .update_config(
9353 |config| {
9354 config.server.port = 22_240;
9355 Ok(())
9356 },
9357 ConfigUpdateEffects::default(),
9358 )
9359 .await
9360 })
9361 };
9362 tokio::task::spawn_blocking(move || reached_rx.recv().unwrap())
9363 .await
9364 .unwrap();
9365 assert_eq!(
9366 state
9367 .config_facade
9368 .as_ref()
9369 .unwrap()
9370 .registry()
9371 .core
9372 .snapshot()
9373 .revision,
9374 1,
9375 "the abort boundary must follow durable commit and facade adoption"
9376 );
9377 operation.abort();
9378 assert!(operation.await.unwrap_err().is_cancelled());
9379 release_tx.send(()).unwrap();
9380 let converged = tokio::time::timeout(Duration::from_secs(5), state.config_io_lock.lock())
9381 .await
9382 .expect("detached generic update must finish live publication");
9383 drop(converged);
9384
9385 assert_eq!(state.config.read().await.server.port, 22_240);
9386 assert_eq!(
9387 bamboo_config::ConfigFacade::open(dir.path())
9388 .unwrap()
9389 .effective_config()
9390 .server
9391 .port,
9392 22_240
9393 );
9394 assert!(matches!(
9395 next_config_event(&mut feed, "core").await,
9396 AgentEvent::ConfigChanged {
9397 section,
9398 revision: 1
9399 } if section == "core"
9400 ));
9401 }
9402
9403 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
9404 async fn provider_update_cancellation_after_commit_finishes_publication() {
9405 let _key = bamboo_config::encryption::set_test_encryption_key([0x7d; 32]);
9406 let dir = tempfile::tempdir().unwrap();
9407 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9408 stop_config_watcher(&mut state);
9409 let state = Arc::new(state);
9410 let mut feed = state.account_sink.subscribe();
9411 let (reached_tx, reached_rx) = std::sync::mpsc::sync_channel(0);
9412 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
9413 set_generic_before_event_test_hook(dir.path(), move || {
9414 reached_tx.send(()).unwrap();
9415 release_rx.recv().unwrap();
9416 });
9417
9418 let operation = {
9419 let state = state.clone();
9420 tokio::spawn(async move {
9421 state
9422 .update_config_with_provider_credentials(
9423 |config| {
9424 config.provider = "openai".to_string();
9425 config.providers_mut().openai = Some(bamboo_config::OpenAIConfig {
9426 api_key: "cancellation-secret".to_string(),
9427 model: Some("cancellation-model".to_string()),
9428 ..Default::default()
9429 });
9430 Ok(())
9431 },
9432 BTreeSet::from(["openai".to_string()]),
9433 BTreeSet::new(),
9434 ConfigUpdateEffects::default(),
9435 )
9436 .await
9437 })
9438 };
9439 tokio::task::spawn_blocking(move || reached_rx.recv().unwrap())
9440 .await
9441 .unwrap();
9442 assert_eq!(
9443 state
9444 .config_facade
9445 .as_ref()
9446 .unwrap()
9447 .registry()
9448 .providers
9449 .snapshot()
9450 .revision,
9451 1,
9452 "the abort boundary must follow provider durable/facade adoption"
9453 );
9454 operation.abort();
9455 assert!(operation.await.unwrap_err().is_cancelled());
9456 release_tx.send(()).unwrap();
9457 let converged = tokio::time::timeout(Duration::from_secs(5), state.config_io_lock.lock())
9458 .await
9459 .expect("detached provider update must finish live publication");
9460 drop(converged);
9461
9462 assert_eq!(state.config.read().await.provider, "openai");
9463 let durable = bamboo_config::ConfigFacade::open(dir.path())
9464 .unwrap()
9465 .effective_config();
9466 assert_eq!(durable.provider, "openai");
9467 assert_eq!(
9468 durable
9469 .providers()
9470 .openai
9471 .as_ref()
9472 .unwrap()
9473 .model
9474 .as_deref(),
9475 Some("cancellation-model")
9476 );
9477 assert!(matches!(
9478 next_config_event(&mut feed, "providers").await,
9479 AgentEvent::ConfigChanged {
9480 section,
9481 revision: 1
9482 } if section == "providers"
9483 ));
9484 for file in ["providers.json", "credentials.json"] {
9485 assert!(
9486 !std::fs::read_to_string(dir.path().join(file))
9487 .unwrap()
9488 .contains("cancellation-secret"),
9489 "{file} must remain secret-free"
9490 );
9491 }
9492 }
9493
9494 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
9495 async fn provider_metadata_cancellation_after_commit_finishes_publication() {
9496 let _key = bamboo_config::encryption::set_test_encryption_key([0x6d; 32]);
9497 let dir = tempfile::tempdir().unwrap();
9498 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9499 stop_config_watcher(&mut state);
9500 let state = Arc::new(state);
9501 let instance_id = "metadata-cancellation".to_string();
9502 let instance_id_for_update = instance_id.clone();
9503 state
9504 .update_config_with_provider_credentials(
9505 move |config| {
9506 let instance = serde_json::from_value(serde_json::json!({
9507 "provider_type": "openai",
9508 "label": "Before cancellation",
9509 "api_key": "metadata-cancellation-secret"
9510 }))?;
9511 config
9512 .provider_instances
9513 .insert(instance_id_for_update.clone(), instance);
9514 config.default_provider_instance = Some(instance_id_for_update.clone());
9515 Ok(())
9516 },
9517 BTreeSet::new(),
9518 BTreeSet::from([instance_id.clone()]),
9519 ConfigUpdateEffects::default(),
9520 )
9521 .await
9522 .unwrap();
9523
9524 let core_path = dir.path().join("core.json");
9525 let core_before = std::fs::read(&core_path).unwrap();
9526 state.config.write().await.server.bind = "0.0.0.0".to_string();
9527 let mut feed = state.account_sink.subscribe();
9528 let (reached_tx, reached_rx) = std::sync::mpsc::sync_channel(0);
9529 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
9530 set_generic_before_event_test_hook(dir.path(), move || {
9531 reached_tx.send(()).unwrap();
9532 release_rx.recv().unwrap();
9533 });
9534
9535 let operation = {
9536 let state = state.clone();
9537 let instance_id = instance_id.clone();
9538 tokio::spawn(async move {
9539 state
9540 .update_provider_metadata(
9541 move |config| {
9542 config
9543 .provider_instances
9544 .get_mut(&instance_id)
9545 .unwrap()
9546 .label = Some("Committed after cancellation".to_string());
9547 Ok(())
9548 },
9549 ConfigUpdateEffects::default(),
9550 )
9551 .await
9552 })
9553 };
9554 tokio::task::spawn_blocking(move || reached_rx.recv().unwrap())
9555 .await
9556 .unwrap();
9557 assert_eq!(
9558 state
9559 .config_facade
9560 .as_ref()
9561 .unwrap()
9562 .registry()
9563 .providers
9564 .snapshot()
9565 .revision,
9566 2,
9567 "the abort boundary must follow provider metadata adoption"
9568 );
9569 operation.abort();
9570 assert!(operation.await.unwrap_err().is_cancelled());
9571 release_tx.send(()).unwrap();
9572 let converged = tokio::time::timeout(Duration::from_secs(5), state.config_io_lock.lock())
9573 .await
9574 .expect("detached provider metadata update must finish live publication");
9575 drop(converged);
9576
9577 let live = state.config.read().await;
9578 assert_eq!(
9579 live.provider_instances[&instance_id].label.as_deref(),
9580 Some("Committed after cancellation")
9581 );
9582 assert_eq!(live.server.bind, "0.0.0.0");
9583 drop(live);
9584 assert_eq!(std::fs::read(core_path).unwrap(), core_before);
9585 assert!(matches!(
9586 next_config_event(&mut feed, "providers").await,
9587 AgentEvent::ConfigChanged {
9588 section,
9589 revision: 2
9590 } if section == "providers"
9591 ));
9592 }
9593
9594 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
9595 async fn replace_config_cancellation_after_commit_finishes_publication() {
9596 let dir = tempfile::tempdir().unwrap();
9597 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9598 stop_config_watcher(&mut state);
9599 let state = Arc::new(state);
9600 let mut replacement = state.config.read().await.clone();
9601 replacement.server.port = 22_241;
9602 let mut feed = state.account_sink.subscribe();
9603 let (reached_tx, reached_rx) = std::sync::mpsc::sync_channel(0);
9604 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
9605 set_generic_before_event_test_hook(dir.path(), move || {
9606 reached_tx.send(()).unwrap();
9607 release_rx.recv().unwrap();
9608 });
9609
9610 let operation = {
9611 let state = state.clone();
9612 tokio::spawn(async move {
9613 state
9614 .replace_config(replacement, ConfigUpdateEffects::default())
9615 .await
9616 })
9617 };
9618 tokio::task::spawn_blocking(move || reached_rx.recv().unwrap())
9619 .await
9620 .unwrap();
9621 assert_eq!(
9622 state
9623 .config_facade
9624 .as_ref()
9625 .unwrap()
9626 .registry()
9627 .core
9628 .snapshot()
9629 .revision,
9630 1,
9631 "the abort boundary must follow replacement durable/facade adoption"
9632 );
9633 operation.abort();
9634 assert!(operation.await.unwrap_err().is_cancelled());
9635 release_tx.send(()).unwrap();
9636 let converged = tokio::time::timeout(Duration::from_secs(5), state.config_io_lock.lock())
9637 .await
9638 .expect("detached replacement must finish live publication");
9639 drop(converged);
9640
9641 assert_eq!(state.config.read().await.server.port, 22_241);
9642 assert_eq!(
9643 bamboo_config::ConfigFacade::open(dir.path())
9644 .unwrap()
9645 .effective_config()
9646 .server
9647 .port,
9648 22_241
9649 );
9650 assert!(matches!(
9651 next_config_event(&mut feed, "core").await,
9652 AgentEvent::ConfigChanged {
9653 section,
9654 revision: 1
9655 } if section == "core"
9656 ));
9657 }
9658
9659 #[tokio::test]
9660 async fn deployed_node_delete_and_cluster_reset_reject_before_commit_and_remain_stoppable() {
9661 let dir = tempfile::tempdir().unwrap();
9662 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9663 state
9664 .update_cluster_fabric_credentials(
9665 0,
9666 BTreeMap::from([(
9667 "live-node".to_string(),
9668 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
9669 )]),
9670 |config| {
9671 config.cluster_fabric.nodes.push(bamboo_config::Node {
9672 id: "live-node".to_string(),
9673 label: "live-node".to_string(),
9674 placement: bamboo_config::NodePlacement::Local,
9675 trust_level: bamboo_config::TrustLevel::Trusted,
9676 deploy: bamboo_config::DeployProfile::default(),
9677 state: Some(bamboo_config::NodeState {
9678 status: bamboo_config::NodeStatus::Running,
9679 worker_id: Some("live-worker".to_string()),
9680 ..Default::default()
9681 }),
9682 enabled: true,
9683 });
9684 Ok(())
9685 },
9686 )
9687 .await
9688 .unwrap();
9689 insert_registry_worker(
9690 &state,
9691 bamboo_server_tools::registry_keys::node_key("live-node"),
9692 "live-worker",
9693 )
9694 .await;
9695 let transaction_marker = dir.path().join("config-credential-migration.json");
9696 let marker_before_guard = std::fs::read(&transaction_marker).ok();
9697 bamboo_config::set_cluster_exact_commit_test_fault(
9698 dir.path().to_path_buf(),
9699 bamboo_config::ClusterExactCommitTestFault::AfterManifestRecoveryFailure,
9700 );
9701
9702 let delete = state
9703 .delete_cluster_node_credentials(
9704 1,
9705 "live-node".to_string(),
9706 BTreeMap::from([(
9707 "live-node".to_string(),
9708 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
9709 )]),
9710 |config| {
9711 config
9712 .cluster_fabric
9713 .nodes
9714 .retain(|node| node.id != "live-node");
9715 Ok(())
9716 },
9717 )
9718 .await;
9719 assert!(matches!(delete, Err(AppError::BadRequest(_))));
9720 let reset = state
9721 .reset_credential_backed_section(SectionId::ClusterFabric, 1)
9722 .await;
9723 assert!(matches!(reset, Err(ConfigSectionMutationError::Invalid(_))));
9724 assert_eq!(
9725 std::fs::read(&transaction_marker).ok(),
9726 marker_before_guard,
9727 "registry guards must reject before opening a new durable transaction"
9728 );
9729 assert_eq!(
9730 state
9731 .config_facade
9732 .as_ref()
9733 .unwrap()
9734 .registry()
9735 .cluster_fabric
9736 .snapshot()
9737 .revision,
9738 1
9739 );
9740 assert!(state
9741 .config
9742 .read()
9743 .await
9744 .cluster_fabric
9745 .node("live-node")
9746 .is_some());
9747 assert!(state
9748 .fabric_deployer
9749 .registry()
9750 .lock()
9751 .await
9752 .contains_key(&bamboo_server_tools::registry_keys::node_key("live-node")));
9753
9754 bamboo_config::clear_cluster_exact_commit_test_fault(dir.path());
9755 let stopped = state
9756 .fabric_deployer
9757 .stop_at_revision("live-node", 1)
9758 .await
9759 .unwrap();
9760 assert_eq!(stopped.snapshot.section.revision, 2);
9761 assert!(!state
9762 .fabric_deployer
9763 .registry()
9764 .lock()
9765 .await
9766 .contains_key(&bamboo_server_tools::registry_keys::node_key("live-node")));
9767 let deleted = state
9768 .delete_cluster_node_credentials(
9769 2,
9770 "live-node".to_string(),
9771 BTreeMap::from([(
9772 "live-node".to_string(),
9773 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
9774 )]),
9775 |config| {
9776 config
9777 .cluster_fabric
9778 .nodes
9779 .retain(|node| node.id != "live-node");
9780 Ok(())
9781 },
9782 )
9783 .await
9784 .unwrap();
9785 assert_eq!(deleted.section.revision, 3);
9786 assert!(deleted.config.cluster_fabric.node("live-node").is_none());
9787 }
9788
9789 #[tokio::test]
9790 async fn unrelated_agent_registry_entry_does_not_block_cluster_reset() {
9791 let dir = tempfile::tempdir().unwrap();
9792 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9793 state
9794 .update_cluster_fabric_credentials(
9795 0,
9796 BTreeMap::from([(
9797 "reset-node".to_string(),
9798 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
9799 )]),
9800 |config| {
9801 config.cluster_fabric.nodes.push(bamboo_config::Node {
9802 id: "reset-node".to_string(),
9803 label: "reset-node".to_string(),
9804 placement: bamboo_config::NodePlacement::Local,
9805 trust_level: bamboo_config::TrustLevel::Trusted,
9806 deploy: bamboo_config::DeployProfile::default(),
9807 state: None,
9808 enabled: true,
9809 });
9810 Ok(())
9811 },
9812 )
9813 .await
9814 .unwrap();
9815 let agent_key = bamboo_server_tools::registry_keys::agent_key("unrelated-agent");
9816 insert_registry_worker(&state, agent_key.clone(), "unrelated-agent").await;
9817
9818 state
9819 .reset_credential_backed_section(SectionId::ClusterFabric, 1)
9820 .await
9821 .unwrap();
9822 assert_eq!(
9823 state
9824 .config_facade
9825 .as_ref()
9826 .unwrap()
9827 .registry()
9828 .cluster_fabric
9829 .snapshot()
9830 .revision,
9831 2
9832 );
9833 assert!(state.config.read().await.cluster_fabric.nodes.is_empty());
9834 let unrelated = state
9835 .fabric_deployer
9836 .registry()
9837 .lock()
9838 .await
9839 .remove(&agent_key)
9840 .expect("agent registry entry must survive cluster reset");
9841 unrelated.handle.shutdown().await;
9842 }
9843
9844 #[tokio::test]
9845 async fn operator_cluster_crud_recovers_before_finish_and_converges_once() {
9846 let _key = bamboo_config::encryption::set_test_encryption_key([0x72; 32]);
9847 let dir = tempfile::tempdir().unwrap();
9848 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9849 let baseline_seq = state.account_sink.latest_seq();
9850 bamboo_config::set_cluster_exact_commit_test_fault(
9851 dir.path().to_path_buf(),
9852 bamboo_config::ClusterExactCommitTestFault::BeforeFinish,
9853 );
9854
9855 let committed = state
9856 .update_cluster_fabric_credentials(
9857 0,
9858 BTreeMap::from([(
9859 "recovered-crud-node".to_string(),
9860 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
9861 )]),
9862 |config| {
9863 config.cluster_fabric.nodes.push(bamboo_config::Node {
9864 id: "recovered-crud-node".to_string(),
9865 label: "recovered-crud-node".to_string(),
9866 placement: bamboo_config::NodePlacement::Local,
9867 trust_level: bamboo_config::TrustLevel::Trusted,
9868 deploy: bamboo_config::DeployProfile::default(),
9869 state: None,
9870 enabled: true,
9871 });
9872 Ok(())
9873 },
9874 )
9875 .await
9876 .expect("operator CRUD must recover the committed transaction");
9877 assert_eq!(committed.section.revision, 1);
9878 assert_eq!(
9879 committed
9880 .config
9881 .cluster_fabric
9882 .node("recovered-crud-node")
9883 .unwrap()
9884 .label,
9885 "recovered-crud-node"
9886 );
9887 assert_eq!(
9888 state
9889 .config
9890 .read()
9891 .await
9892 .cluster_fabric
9893 .node("recovered-crud-node")
9894 .unwrap()
9895 .label,
9896 "recovered-crud-node"
9897 );
9898 assert_eq!(
9899 state
9900 .config_facade
9901 .as_ref()
9902 .unwrap()
9903 .registry()
9904 .cluster_fabric
9905 .snapshot()
9906 .revision,
9907 1
9908 );
9909 let reopened = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
9910 assert_eq!(reopened.registry().cluster_fabric.snapshot().revision, 1);
9911 assert_eq!(
9912 reopened
9913 .effective_config()
9914 .cluster_fabric
9915 .node("recovered-crud-node")
9916 .unwrap()
9917 .label,
9918 "recovered-crud-node"
9919 );
9920 bamboo_config::ensure_provider_mcp_migration_ready(dir.path()).unwrap();
9921
9922 tokio::time::sleep(Duration::from_millis(100)).await;
9923 let events = bamboo_engine::events::journal::read_since(
9924 state.account_sink.events_dir(),
9925 baseline_seq,
9926 )
9927 .unwrap();
9928 let cluster_events = events
9929 .iter()
9930 .filter(|event| {
9931 matches!(
9932 &event.event,
9933 AgentEvent::ConfigChanged { section, revision }
9934 if section == "cluster-fabric" && *revision == 1
9935 )
9936 })
9937 .count();
9938 assert_eq!(cluster_events, 1);
9939 assert!(!events.iter().any(|event| {
9940 matches!(
9941 &event.event,
9942 AgentEvent::ConfigChanged { section, .. } if section == "credentials"
9943 )
9944 }));
9945 }
9946
9947 #[tokio::test]
9948 async fn cluster_replace_and_keep_noop_retain_the_exact_hydrated_runtime() {
9949 let _key = bamboo_config::encryption::set_test_encryption_key([0x73; 32]);
9950 let dir = tempfile::tempdir().unwrap();
9951 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9952 let baseline_seq = state.account_sink.latest_seq();
9953 let password_ref = bamboo_config::cluster_password_credential_ref("secret-node").unwrap();
9954 let password_from = |config: &Config| match &config
9955 .cluster_fabric
9956 .node("secret-node")
9957 .expect("secret node exists")
9958 .placement
9959 {
9960 bamboo_config::NodePlacement::Ssh(target) => match &target.auth {
9961 bamboo_config::SshAuth::Password { password, .. } => password.clone(),
9962 _ => panic!("expected password authentication"),
9963 },
9964 _ => panic!("expected SSH placement"),
9965 };
9966
9967 let replaced = state
9968 .update_cluster_fabric_credentials(
9969 0,
9970 BTreeMap::from([(
9971 "secret-node".to_string(),
9972 bamboo_config::ClusterNodeCredentialIntents {
9973 password: bamboo_config::ClusterCredentialAction::Replace(
9974 "exact-password".to_string(),
9975 ),
9976 private_key: bamboo_config::ClusterCredentialAction::Clear,
9977 passphrase: bamboo_config::ClusterCredentialAction::Clear,
9978 },
9979 )]),
9980 |config| {
9981 config.cluster_fabric.nodes.push(bamboo_config::Node {
9982 id: "secret-node".to_string(),
9983 label: "secret-node".to_string(),
9984 placement: bamboo_config::NodePlacement::Ssh(bamboo_config::SshTarget {
9985 host: "secret.example.test".to_string(),
9986 port: 22,
9987 username: "operator".to_string(),
9988 auth: bamboo_config::SshAuth::Password {
9989 password: String::new(),
9990 password_encrypted: None,
9991 },
9992 host_key_fingerprint: None,
9993 }),
9994 trust_level: bamboo_config::TrustLevel::Trusted,
9995 deploy: bamboo_config::DeployProfile::default(),
9996 state: None,
9997 enabled: true,
9998 });
9999 Ok(())
10000 },
10001 )
10002 .await
10003 .unwrap();
10004 assert_eq!(replaced.section.revision, 1);
10005 assert_eq!(password_from(&replaced.config), "exact-password");
10006 assert_eq!(
10007 password_from(&*state.config.read().await),
10008 "exact-password",
10009 "live runtime must install the under-lock hydrated candidate"
10010 );
10011 assert_eq!(replaced.credential_health.revision, 1);
10012 assert_eq!(replaced.credential_statuses.len(), 1);
10013 assert_eq!(replaced.credential_statuses[0].credential_ref, password_ref);
10014 assert!(replaced.credential_statuses[0].configured);
10015
10016 tokio::time::sleep(Duration::from_millis(500)).await;
10017 let replace_events = bamboo_engine::events::journal::read_since(
10018 state.account_sink.events_dir(),
10019 baseline_seq,
10020 )
10021 .unwrap();
10022 let cluster_revisions = replace_events
10023 .iter()
10024 .filter_map(|event| match &event.event {
10025 AgentEvent::ConfigChanged { section, revision } if section == "cluster-fabric" => {
10026 Some(*revision)
10027 }
10028 _ => None,
10029 })
10030 .collect::<Vec<_>>();
10031 assert_eq!(cluster_revisions, vec![1]);
10032 assert!(!replace_events.iter().any(|event| {
10033 matches!(
10034 &event.event,
10035 AgentEvent::ConfigChanged { section, .. }
10036 | AgentEvent::ConfigInvalid { section, .. }
10037 | AgentEvent::ConfigRecovered { section, .. }
10038 if section == "credentials"
10039 )
10040 }));
10041
10042 let noop_baseline_seq = state.account_sink.latest_seq();
10043 let kept = state
10044 .update_cluster_fabric_credentials(
10045 1,
10046 BTreeMap::from([(
10047 "secret-node".to_string(),
10048 bamboo_config::ClusterNodeCredentialIntents {
10049 password: bamboo_config::ClusterCredentialAction::Keep,
10050 private_key: bamboo_config::ClusterCredentialAction::Clear,
10051 passphrase: bamboo_config::ClusterCredentialAction::Clear,
10052 },
10053 )]),
10054 |config| {
10055 let node = config
10056 .cluster_fabric
10057 .node_mut("secret-node")
10058 .expect("secret node exists");
10059 let bamboo_config::NodePlacement::Ssh(target) = &mut node.placement else {
10060 panic!("expected SSH placement")
10061 };
10062 let bamboo_config::SshAuth::Password {
10063 password,
10064 password_encrypted,
10065 } = &mut target.auth
10066 else {
10067 panic!("expected password authentication")
10068 };
10069 password.clear();
10070 *password_encrypted = None;
10071 Ok(())
10072 },
10073 )
10074 .await
10075 .unwrap();
10076 assert_eq!(kept.section.revision, 1);
10077 assert_eq!(kept.credential_health.revision, 1);
10078 assert_eq!(password_from(&kept.config), "exact-password");
10079 assert_eq!(
10080 password_from(&*state.config.read().await),
10081 "exact-password",
10082 "semantic no-op must retain the exact credential snapshot"
10083 );
10084
10085 tokio::time::sleep(Duration::from_millis(500)).await;
10086 let noop_events = bamboo_engine::events::journal::read_since(
10087 state.account_sink.events_dir(),
10088 noop_baseline_seq,
10089 )
10090 .unwrap();
10091 assert!(!noop_events.iter().any(|event| {
10092 matches!(
10093 &event.event,
10094 AgentEvent::ConfigChanged { section, .. }
10095 | AgentEvent::ConfigInvalid { section, .. }
10096 | AgentEvent::ConfigRecovered { section, .. }
10097 if section == "cluster-fabric" || section == "credentials"
10098 )
10099 }));
10100 }
10101
10102 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
10103 async fn later_external_credential_winner_remains_observable_after_exact_cluster_commit() {
10104 let _key = bamboo_config::encryption::set_test_encryption_key([0x74; 32]);
10105 let dir = tempfile::tempdir().unwrap();
10106 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
10107 let baseline_seq = state.account_sink.latest_seq();
10108 let password_ref =
10109 bamboo_config::cluster_password_credential_ref("credential-race-node").unwrap();
10110 let external_ref = password_ref.clone();
10111 let (external_done_tx, external_done_rx) = std::sync::mpsc::sync_channel(1);
10112 set_cluster_after_commit_before_adoption_test_hook(dir.path(), 0, move |data_dir| {
10113 let data_dir = data_dir.to_path_buf();
10114 let (started_tx, started_rx) = std::sync::mpsc::sync_channel(0);
10115 std::thread::spawn(move || {
10116 started_tx.send(()).unwrap();
10117 let result = bamboo_config::CredentialStore::open(&data_dir).replace(
10118 external_ref,
10119 "later-external-password",
10120 bamboo_config::CredentialSource::User,
10121 1,
10122 );
10123 external_done_tx.send(result).unwrap();
10124 });
10125 started_rx
10126 .recv_timeout(Duration::from_secs(5))
10127 .expect("external credential writer must launch under the commit lock");
10128 });
10129
10130 let committed = state
10131 .update_cluster_fabric_credentials(
10132 0,
10133 BTreeMap::from([(
10134 "credential-race-node".to_string(),
10135 bamboo_config::ClusterNodeCredentialIntents {
10136 password: bamboo_config::ClusterCredentialAction::Replace(
10137 "exact-commit-password".to_string(),
10138 ),
10139 private_key: bamboo_config::ClusterCredentialAction::Clear,
10140 passphrase: bamboo_config::ClusterCredentialAction::Clear,
10141 },
10142 )]),
10143 |config| {
10144 config.cluster_fabric.nodes.push(bamboo_config::Node {
10145 id: "credential-race-node".to_string(),
10146 label: "credential-race-node".to_string(),
10147 placement: bamboo_config::NodePlacement::Ssh(bamboo_config::SshTarget {
10148 host: "race.example.test".to_string(),
10149 port: 22,
10150 username: "operator".to_string(),
10151 auth: bamboo_config::SshAuth::Password {
10152 password: String::new(),
10153 password_encrypted: None,
10154 },
10155 host_key_fingerprint: None,
10156 }),
10157 trust_level: bamboo_config::TrustLevel::Trusted,
10158 deploy: bamboo_config::DeployProfile::default(),
10159 state: None,
10160 enabled: true,
10161 });
10162 Ok(())
10163 },
10164 )
10165 .await
10166 .unwrap();
10167 let committed_password = match &committed
10168 .config
10169 .cluster_fabric
10170 .node("credential-race-node")
10171 .unwrap()
10172 .placement
10173 {
10174 bamboo_config::NodePlacement::Ssh(target) => match &target.auth {
10175 bamboo_config::SshAuth::Password { password, .. } => password,
10176 _ => panic!("expected password authentication"),
10177 },
10178 _ => panic!("expected SSH placement"),
10179 };
10180 assert_eq!(committed.section.revision, 1);
10181 assert_eq!(committed.credential_health.revision, 1);
10182 assert_eq!(committed_password, "exact-commit-password");
10183
10184 let external_revision = tokio::task::spawn_blocking(move || {
10185 external_done_rx
10186 .recv_timeout(Duration::from_secs(10))
10187 .expect("external credential writer must complete")
10188 .unwrap()
10189 .0
10190 })
10191 .await
10192 .unwrap();
10193 assert_eq!(external_revision, 2);
10194
10195 tokio::time::timeout(Duration::from_secs(5), async {
10196 loop {
10197 let facade_revision = state
10198 .config_facade
10199 .as_ref()
10200 .unwrap()
10201 .registry()
10202 .credentials
10203 .snapshot()
10204 .revision;
10205 let events = bamboo_engine::events::journal::read_since(
10206 state.account_sink.events_dir(),
10207 baseline_seq,
10208 )
10209 .unwrap();
10210 let saw_external_event = events.iter().any(|event| {
10211 matches!(
10212 &event.event,
10213 AgentEvent::ConfigChanged { section, revision }
10214 if section == "credentials" && *revision == 2
10215 )
10216 });
10217 if facade_revision == 2 && saw_external_event {
10218 break;
10219 }
10220 tokio::time::sleep(Duration::from_millis(20)).await;
10221 }
10222 })
10223 .await
10224 .expect("watcher must expose the later credential revision");
10225 tokio::time::sleep(Duration::from_millis(250)).await;
10226
10227 let runtime_password = match &state
10228 .config
10229 .read()
10230 .await
10231 .cluster_fabric
10232 .node("credential-race-node")
10233 .unwrap()
10234 .placement
10235 {
10236 bamboo_config::NodePlacement::Ssh(target) => match &target.auth {
10237 bamboo_config::SshAuth::Password { password, .. } => password.clone(),
10238 _ => panic!("expected password authentication"),
10239 },
10240 _ => panic!("expected SSH placement"),
10241 };
10242 assert_eq!(
10243 runtime_password, "exact-commit-password",
10244 "a status-only credential event must not rewrite the exact cluster runtime"
10245 );
10246 let credential_dir = dir.path().to_path_buf();
10247 let durable_password = tokio::task::spawn_blocking(move || {
10248 bamboo_config::CredentialStore::open(credential_dir)
10249 .resolve(&password_ref)
10250 .unwrap()
10251 .unwrap()
10252 .expose()
10253 .to_string()
10254 })
10255 .await
10256 .unwrap();
10257 assert_eq!(durable_password, "later-external-password");
10258
10259 let events = bamboo_engine::events::journal::read_since(
10260 state.account_sink.events_dir(),
10261 baseline_seq,
10262 )
10263 .unwrap();
10264 let relevant = events
10265 .iter()
10266 .filter_map(|event| match &event.event {
10267 AgentEvent::ConfigChanged { section, revision }
10268 if section == "cluster-fabric" || section == "credentials" =>
10269 {
10270 Some((section.as_str(), *revision))
10271 }
10272 _ => None,
10273 })
10274 .collect::<Vec<_>>();
10275 assert_eq!(
10276 relevant,
10277 vec![("cluster-fabric", 1), ("credentials", 2)],
10278 "the exact cluster event must precede the genuine later credential winner"
10279 );
10280 }
10281
10282 #[tokio::test]
10283 async fn changed_cluster_commit_publishes_secret_free_runtime_before_materialization_error() {
10284 let _key = bamboo_config::encryption::set_test_encryption_key([0x75; 32]);
10285 let dir = tempfile::tempdir().unwrap();
10286 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
10287 let password_ref =
10288 bamboo_config::cluster_password_credential_ref("corrupt-secret-node").unwrap();
10289 state
10290 .update_cluster_fabric_credentials(
10291 0,
10292 BTreeMap::from([(
10293 "corrupt-secret-node".to_string(),
10294 bamboo_config::ClusterNodeCredentialIntents {
10295 password: bamboo_config::ClusterCredentialAction::Replace(
10296 "initial-password".to_string(),
10297 ),
10298 private_key: bamboo_config::ClusterCredentialAction::Clear,
10299 passphrase: bamboo_config::ClusterCredentialAction::Clear,
10300 },
10301 )]),
10302 |config| {
10303 config.cluster_fabric.nodes.push(bamboo_config::Node {
10304 id: "corrupt-secret-node".to_string(),
10305 label: "before-corruption".to_string(),
10306 placement: bamboo_config::NodePlacement::Ssh(bamboo_config::SshTarget {
10307 host: "corrupt.example.test".to_string(),
10308 port: 22,
10309 username: "operator".to_string(),
10310 auth: bamboo_config::SshAuth::Password {
10311 password: String::new(),
10312 password_encrypted: None,
10313 },
10314 host_key_fingerprint: None,
10315 }),
10316 trust_level: bamboo_config::TrustLevel::Trusted,
10317 deploy: bamboo_config::DeployProfile::default(),
10318 state: None,
10319 enabled: true,
10320 });
10321 Ok(())
10322 },
10323 )
10324 .await
10325 .unwrap();
10326
10327 let credentials_path = dir.path().join("credentials.json");
10328 let mut document: Value =
10329 serde_json::from_slice(&std::fs::read(&credentials_path).unwrap()).unwrap();
10330 document["data"]["entries"][password_ref.as_str()]["ciphertext"] =
10331 Value::String("corrupt-ciphertext".to_string());
10332 std::fs::write(
10333 &credentials_path,
10334 serde_json::to_vec_pretty(&document).unwrap(),
10335 )
10336 .unwrap();
10337 tokio::time::sleep(Duration::from_millis(300)).await;
10338
10339 let noop_baseline_seq = state.account_sink.latest_seq();
10340 let noop = state
10341 .update_cluster_fabric_credentials(1, BTreeMap::new(), |_| Ok(()))
10342 .await;
10343 match noop {
10344 Err(AppError::InternalError(_)) => {}
10345 Err(error) => panic!("no-op materialization error was misclassified: {error}"),
10346 Ok(_) => panic!("corrupt credential unexpectedly materialized"),
10347 }
10348 let runtime = state.config.read().await;
10349 let node = runtime.cluster_fabric.node("corrupt-secret-node").unwrap();
10350 let bamboo_config::NodePlacement::Ssh(target) = &node.placement else {
10351 panic!("expected SSH placement")
10352 };
10353 let bamboo_config::SshAuth::Password { password, .. } = &target.auth else {
10354 panic!("expected password authentication")
10355 };
10356 assert_eq!(
10357 password, "initial-password",
10358 "a true no-op materialization failure must preserve the old runtime"
10359 );
10360 drop(runtime);
10361 assert_eq!(
10362 state
10363 .config_facade
10364 .as_ref()
10365 .unwrap()
10366 .registry()
10367 .cluster_fabric
10368 .snapshot()
10369 .revision,
10370 1
10371 );
10372 let noop_events = bamboo_engine::events::journal::read_since(
10373 state.account_sink.events_dir(),
10374 noop_baseline_seq,
10375 )
10376 .unwrap();
10377 assert!(!noop_events.iter().any(|event| {
10378 matches!(
10379 &event.event,
10380 AgentEvent::ConfigChanged { section, .. } if section == "cluster-fabric"
10381 )
10382 }));
10383
10384 let baseline_seq = state.account_sink.latest_seq();
10385 let result = state
10386 .update_cluster_fabric_credentials(1, BTreeMap::new(), |config| {
10387 config
10388 .cluster_fabric
10389 .node_mut("corrupt-secret-node")
10390 .unwrap()
10391 .label = "committed-metadata".to_string();
10392 Ok(())
10393 })
10394 .await;
10395 match result {
10396 Err(AppError::InternalError(_)) => {}
10397 Err(error) => panic!("post-commit materialization error was misclassified: {error}"),
10398 Ok(_) => panic!("corrupt credential unexpectedly materialized"),
10399 }
10400
10401 let runtime = state.config.read().await;
10402 let node = runtime.cluster_fabric.node("corrupt-secret-node").unwrap();
10403 assert_eq!(node.label, "committed-metadata");
10404 let bamboo_config::NodePlacement::Ssh(target) = &node.placement else {
10405 panic!("expected SSH placement")
10406 };
10407 let bamboo_config::SshAuth::Password {
10408 password,
10409 password_encrypted,
10410 } = &target.auth
10411 else {
10412 panic!("expected password authentication")
10413 };
10414 assert!(password.is_empty());
10415 assert!(password_encrypted.is_none());
10416 drop(runtime);
10417
10418 let section = state
10419 .config_facade
10420 .as_ref()
10421 .unwrap()
10422 .registry()
10423 .cluster_fabric
10424 .snapshot();
10425 assert_eq!(section.revision, 2);
10426 assert_eq!(
10427 section.data.0.node("corrupt-secret-node").unwrap().label,
10428 "committed-metadata"
10429 );
10430 let events = bamboo_engine::events::journal::read_since(
10431 state.account_sink.events_dir(),
10432 baseline_seq,
10433 )
10434 .unwrap();
10435 let cluster_revisions = events
10436 .iter()
10437 .filter_map(|event| match &event.event {
10438 AgentEvent::ConfigChanged { section, revision } if section == "cluster-fabric" => {
10439 Some(*revision)
10440 }
10441 _ => None,
10442 })
10443 .collect::<Vec<_>>();
10444 assert_eq!(cluster_revisions, vec![2]);
10445 }
10446
10447 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
10448 async fn cluster_commit_adopts_exact_revision_before_later_external_winner() {
10449 let _key = bamboo_config::encryption::set_test_encryption_key([0x72; 32]);
10450 let dir = tempfile::tempdir().unwrap();
10451 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
10452 let baseline_seq = state.account_sink.latest_seq();
10453 let (external_done_tx, external_done_rx) = std::sync::mpsc::sync_channel(1);
10454 set_cluster_after_commit_before_adoption_test_hook(dir.path(), 0, move |data_dir| {
10455 let data_dir = data_dir.to_path_buf();
10456 let (started_tx, started_rx) = std::sync::mpsc::sync_channel(0);
10457 std::thread::spawn(move || {
10458 started_tx.send(()).unwrap();
10459 let external = bamboo_config::ConfigFacade::open(&data_dir).unwrap();
10460 let mut winner = external.effective_config();
10461 winner.cluster_fabric.node_mut("race-node").unwrap().label =
10462 "external-winner".to_string();
10463 let result =
10464 bamboo_config::persist_cluster_fabric_credential_transaction_at_revision(
10465 &data_dir,
10466 &mut winner,
10467 &BTreeMap::new(),
10468 1,
10469 );
10470 external_done_tx.send(result).unwrap();
10471 });
10472 started_rx
10473 .recv_timeout(Duration::from_secs(5))
10474 .expect("external writer must launch after the durable commit");
10475 });
10476
10477 let committed = state
10478 .update_cluster_fabric_credentials(
10479 0,
10480 BTreeMap::from([(
10481 "race-node".to_string(),
10482 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
10483 )]),
10484 |config| {
10485 config.cluster_fabric.nodes.push(bamboo_config::Node {
10486 id: "race-node".to_string(),
10487 label: "api-commit".to_string(),
10488 placement: bamboo_config::NodePlacement::Local,
10489 trust_level: bamboo_config::TrustLevel::Trusted,
10490 deploy: bamboo_config::DeployProfile::default(),
10491 state: None,
10492 enabled: true,
10493 });
10494 Ok(())
10495 },
10496 )
10497 .await
10498 .unwrap();
10499 assert_eq!(committed.section.revision, 1);
10500 assert_eq!(
10501 committed
10502 .config
10503 .cluster_fabric
10504 .node("race-node")
10505 .unwrap()
10506 .label,
10507 "api-commit",
10508 "the response must remain bound to its exact committed candidate"
10509 );
10510 assert_eq!(
10511 tokio::task::spawn_blocking(move || {
10512 external_done_rx
10513 .recv_timeout(Duration::from_secs(10))
10514 .expect("later external winner must complete")
10515 .unwrap()
10516 })
10517 .await
10518 .unwrap(),
10519 2
10520 );
10521
10522 tokio::time::timeout(Duration::from_secs(5), async {
10523 loop {
10524 let facade_revision = state
10525 .config_facade
10526 .as_ref()
10527 .unwrap()
10528 .registry()
10529 .cluster_fabric
10530 .snapshot()
10531 .revision;
10532 let runtime_label = state
10533 .config
10534 .read()
10535 .await
10536 .cluster_fabric
10537 .node("race-node")
10538 .map(|node| node.label.clone());
10539 if facade_revision == 2 && runtime_label.as_deref() == Some("external-winner") {
10540 break;
10541 }
10542 tokio::time::sleep(Duration::from_millis(20)).await;
10543 }
10544 })
10545 .await
10546 .expect("watcher must apply the later external revision");
10547
10548 let events = bamboo_engine::events::journal::read_since(
10549 state.account_sink.events_dir(),
10550 baseline_seq,
10551 )
10552 .unwrap();
10553 let revisions = events
10554 .iter()
10555 .filter_map(|event| match &event.event {
10556 AgentEvent::ConfigChanged { section, revision } if section == "cluster-fabric" => {
10557 Some(*revision)
10558 }
10559 _ => None,
10560 })
10561 .collect::<Vec<_>>();
10562 assert_eq!(
10563 revisions,
10564 vec![1, 2],
10565 "the exact API event must precede the later watcher winner exactly once"
10566 );
10567 assert!(!events.iter().any(|event| {
10568 matches!(
10569 &event.event,
10570 AgentEvent::ConfigChanged { section, .. } if section == "credentials"
10571 )
10572 }));
10573 }
10574
10575 async fn wait_for_facade_health(
10576 state: &AppState,
10577 id: SectionId,
10578 status: SectionStatus,
10579 revision: u64,
10580 ) -> bamboo_config::SectionHealth {
10581 tokio::time::timeout(Duration::from_secs(4), async {
10582 loop {
10583 let health = state
10584 .config_facade
10585 .as_ref()
10586 .expect("production state owns a facade")
10587 .registry()
10588 .health()
10589 .unwrap()
10590 .into_iter()
10591 .find(|health| health.section == id)
10592 .unwrap();
10593 if health.status == status && health.revision == revision {
10594 break health;
10595 }
10596 tokio::time::sleep(Duration::from_millis(20)).await;
10597 }
10598 })
10599 .await
10600 .expect("facade health transition timed out")
10601 }
10602
10603 #[test]
10604 fn initial_provider_health_validates_primary_and_backup() {
10605 let dir = tempfile::tempdir().unwrap();
10606 let store = AtomicJsonStore::new(dir.path().join("providers.json"), 1);
10607 let missing = initial_provider_health(&store);
10608 assert_eq!(missing.status, SectionStatus::Missing);
10609 assert_eq!(missing.source_kind, SectionSourceKind::Default);
10610
10611 std::fs::write(dir.path().join("providers.json"), b"{broken").unwrap();
10612 let invalid = initial_provider_health(&store);
10613 assert_eq!(invalid.status, SectionStatus::Invalid);
10614 assert_eq!(invalid.source_kind, SectionSourceKind::File);
10615
10616 std::fs::write(dir.path().join("providers.json.bak"), b"{}").unwrap();
10617 let recovered = initial_provider_health(&store);
10618 assert_eq!(recovered.status, SectionStatus::Degraded);
10619 assert_eq!(recovered.source_kind, SectionSourceKind::Backup);
10620 assert!(recovered
10621 .last_error
10622 .as_deref()
10623 .unwrap()
10624 .contains("last-known-good backup"));
10625
10626 std::fs::write(dir.path().join("providers.json"), b"{}").unwrap();
10627 let healthy = initial_provider_health(&store);
10628 assert_eq!(healthy.status, SectionStatus::Healthy);
10629 assert_eq!(healthy.source_kind, SectionSourceKind::File);
10630 }
10631
10632 #[tokio::test]
10633 async fn unrecoverable_pending_manifest_never_publishes_partial_provider_state() {
10634 let _key = bamboo_config::encryption::set_test_encryption_key([0x6c; 32]);
10635 let dir = tempfile::tempdir().unwrap();
10636 install_unrecoverable_pending_provider_migration(dir.path());
10637
10638 let loaded = Config::from_data_dir_without_env(Some(dir.path().to_path_buf()));
10639 assert_eq!(
10640 loaded.providers().openai.as_ref().unwrap().model.as_deref(),
10641 Some("root-lkg")
10642 );
10643 let store = AtomicJsonStore::new(dir.path().join("providers.json"), 1);
10644 let health = initial_provider_health(&store);
10645 assert_eq!(health.status, SectionStatus::Degraded);
10646 assert!(health
10647 .last_error
10648 .as_deref()
10649 .unwrap()
10650 .contains("migration is pending"));
10651 let error = match load_and_prepare_provider_candidate(&store, 0, loaded).await {
10652 Ok(_) => panic!("pending migration must reject provider candidate"),
10653 Err(error) => error,
10654 };
10655 assert!(error.message.contains("retaining last-known-good runtime"));
10656 assert!(!error.message.contains("partial-must-not-load"));
10657
10658 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
10659 assert_eq!(
10660 state
10661 .config
10662 .read()
10663 .await
10664 .providers()
10665 .openai
10666 .as_ref()
10667 .unwrap()
10668 .model
10669 .as_deref(),
10670 Some("root-lkg")
10671 );
10672 assert_eq!(
10673 state
10674 .config_live_health
10675 .read()
10676 .unwrap_or_else(|poisoned| poisoned.into_inner())
10677 .status,
10678 SectionStatus::Degraded
10679 );
10680 }
10681
10682 #[async_trait::async_trait]
10683 impl LLMProvider for WorkingProvider {
10684 async fn chat_stream(
10685 &self,
10686 _messages: &[Message],
10687 _tools: &[ToolSchema],
10688 _max_output_tokens: Option<u32>,
10689 _model: &str,
10690 ) -> Result<LLMStream, LLMError> {
10691 Err(LLMError::Api("working-provider-marker".to_string()))
10692 }
10693 }
10694
10695 #[tokio::test]
10696 async fn cancelled_provider_put_cannot_commit_before_publication_guards() {
10697 let _key = bamboo_config::encryption::set_test_encryption_key([0x53; 32]);
10698 let dir = tempfile::tempdir().unwrap();
10699 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
10700 let secret = "provider-cancel-secret";
10701 let reference = bamboo_config::credential_ref("provider", "openai", "api_key").unwrap();
10702 bamboo_config::CredentialStore::open(dir.path())
10703 .replace(
10704 reference.clone(),
10705 secret,
10706 bamboo_config::CredentialSource::User,
10707 0,
10708 )
10709 .unwrap();
10710 {
10711 let mut config = state.config.write().await;
10712 config.provider = "openai".to_string();
10713 *config.providers_mut() = ProviderConfigs {
10714 openai: Some(bamboo_config::OpenAIConfig {
10715 api_key: secret.to_string(),
10716 credential_ref: Some(reference),
10717 ..Default::default()
10718 }),
10719 ..Default::default()
10720 };
10721 }
10722 let provider_lock = state.provider.clone();
10723 let held_provider = provider_lock.write().await;
10724 let providers_before = std::fs::read(dir.path().join("providers.json")).unwrap();
10725 let mut operation = Box::pin(state.put_provider_section(
10726 0,
10727 ProviderConfigs {
10728 openai: Some(bamboo_config::OpenAIConfig {
10729 model: Some("candidate-model".to_string()),
10730 ..Default::default()
10731 }),
10732 ..Default::default()
10733 },
10734 ));
10735
10736 assert!(
10737 tokio::time::timeout(Duration::from_millis(500), &mut operation)
10738 .await
10739 .is_err()
10740 );
10741 drop(operation);
10742 assert_eq!(
10743 std::fs::read(dir.path().join("providers.json")).unwrap(),
10744 providers_before,
10745 "cancellation while waiting for publication guards must precede durable commit"
10746 );
10747 drop(held_provider);
10748 }
10749
10750 #[test]
10751 fn cancelled_provider_settings_request_finishes_exact_commit_and_live_publication() {
10752 let _key = bamboo_config::encryption::set_test_encryption_key([0x71; 32]);
10753 let runtime = tokio::runtime::Builder::new_multi_thread()
10754 .worker_threads(2)
10755 .max_blocking_threads(1)
10756 .enable_all()
10757 .build()
10758 .unwrap();
10759 runtime.block_on(async {
10760 let dir = tempfile::tempdir().unwrap();
10761 let state = Arc::new(AppState::new(dir.path().to_path_buf()).await.unwrap());
10762 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
10763
10764 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
10765 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
10766 let blocker = tokio::task::spawn_blocking(move || {
10767 let _ = started_tx.send(());
10768 release_rx.recv().unwrap();
10769 });
10770 started_rx.await.unwrap();
10771
10772 let operation_state = state.clone();
10773 let operation = tokio::spawn(async move {
10774 operation_state
10775 .put_provider_settings(0, |_current, candidate| {
10776 candidate.provider = "openai".to_string();
10777 candidate.providers_mut().openai = Some(bamboo_config::OpenAIConfig {
10778 api_key: "provider-settings-cancel-secret".to_string(),
10779 model: Some("provider-settings-cancel-model".to_string()),
10780 ..Default::default()
10781 });
10782 Ok((BTreeSet::from(["openai".to_string()]), BTreeSet::new()))
10783 })
10784 .await
10785 });
10786
10787 tokio::time::timeout(Duration::from_secs(1), async {
10788 loop {
10789 if state.config_io_lock.try_lock().is_err() {
10790 break;
10791 }
10792 assert!(!operation.is_finished());
10793 tokio::task::yield_now().await;
10794 }
10795 })
10796 .await
10797 .expect("provider settings mutation acquires the config IO lock");
10798 operation.abort();
10799 let _ = operation.await;
10800 release_tx.send(()).unwrap();
10801 blocker.await.unwrap();
10802
10803 tokio::time::timeout(Duration::from_secs(5), async {
10804 loop {
10805 let committed = std::fs::read(dir.path().join("providers.json"))
10806 .ok()
10807 .and_then(|bytes| serde_json::from_slice::<Value>(&bytes).ok())
10808 .is_some_and(|value| {
10809 value["revision"] == 1
10810 && value["data"]["openai"]["model"]
10811 == "provider-settings-cancel-model"
10812 });
10813 if committed {
10814 break;
10815 }
10816 tokio::task::yield_now().await;
10817 }
10818 })
10819 .await
10820 .expect("owned provider settings transaction completes after cancellation");
10821
10822 let converged =
10823 tokio::time::timeout(Duration::from_secs(5), state.config_io_lock.lock())
10824 .await
10825 .expect("owned provider runtime publication completes after cancellation");
10826 drop(converged);
10827 let live = state.config.read().await;
10828 let openai = live.providers().openai.as_ref().unwrap();
10829 assert_eq!(
10830 openai.model.as_deref(),
10831 Some("provider-settings-cancel-model")
10832 );
10833 assert_eq!(openai.api_key, "provider-settings-cancel-secret");
10834 drop(live);
10835 let providers = std::fs::read_to_string(dir.path().join("providers.json")).unwrap();
10836 let credentials = std::fs::read_to_string(dir.path().join("credentials.json")).unwrap();
10837 assert!(!providers.contains("provider-settings-cancel-secret"));
10838 assert!(!credentials.contains("provider-settings-cancel-secret"));
10839 });
10840 }
10841
10842 #[test]
10843 fn cancelled_proxy_update_cannot_leave_durable_state_ahead_of_live_snapshot() {
10844 let runtime = tokio::runtime::Builder::new_multi_thread()
10845 .worker_threads(2)
10846 .max_blocking_threads(1)
10847 .enable_all()
10848 .build()
10849 .unwrap();
10850 runtime.block_on(async {
10851 let dir = tempfile::tempdir().unwrap();
10852 let state = Arc::new(AppState::new(dir.path().to_path_buf()).await.unwrap());
10853 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
10854
10855 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
10859 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
10860 let blocker = tokio::task::spawn_blocking(move || {
10861 let _ = started_tx.send(());
10862 release_rx.recv().unwrap();
10863 });
10864 started_rx.await.unwrap();
10865
10866 let operation_state = state.clone();
10867 let operation = tokio::spawn(async move {
10868 operation_state
10869 .update_proxy_auth_credential(
10870 Some(bamboo_config::ProxyAuth {
10871 username: "cancel-user".to_string(),
10872 password: "cancel-secret".to_string(),
10873 }),
10874 0,
10875 ConfigUpdateEffects {
10876 reload_provider: bamboo_config::patch::ReloadMode::Strict,
10877 reconcile_mcp: bamboo_config::patch::ReloadMode::Strict,
10878 },
10879 )
10880 .await
10881 });
10882
10883 tokio::time::timeout(Duration::from_secs(1), async {
10888 loop {
10889 if state.config_io_lock.try_lock().is_err() {
10890 break;
10891 }
10892 assert!(!operation.is_finished());
10893 tokio::task::yield_now().await;
10894 }
10895 })
10896 .await
10897 .expect("proxy mutation acquires the config IO lock");
10898 operation.abort();
10899 let _ = operation.await;
10900 release_tx.send(()).unwrap();
10901 blocker.await.unwrap();
10902
10903 tokio::time::timeout(Duration::from_secs(5), async {
10904 loop {
10905 let credentials_ready = std::fs::read(dir.path().join("credentials.json"))
10906 .ok()
10907 .and_then(|bytes| serde_json::from_slice::<serde_json::Value>(&bytes).ok())
10908 .and_then(|value| value.get("revision").and_then(|value| value.as_u64()))
10909 == Some(1);
10910 let config_ready = std::fs::read(dir.path().join("core.json"))
10911 .ok()
10912 .and_then(|bytes| serde_json::from_slice::<serde_json::Value>(&bytes).ok())
10913 .and_then(|value| {
10914 value
10915 .get("data")
10916 .and_then(|value| value.get("proxy_auth_credential_ref"))
10917 .and_then(|value| value.as_str())
10918 .map(str::to_string)
10919 })
10920 .as_deref()
10921 == Some("proxy.default.auth");
10922 if credentials_ready && config_ready {
10923 break;
10924 }
10925 tokio::task::yield_now().await;
10926 }
10927 })
10928 .await
10929 .expect("owned durable transaction completes after caller cancellation");
10930
10931 let converged =
10935 tokio::time::timeout(Duration::from_secs(5), state.config_io_lock.lock())
10936 .await
10937 .expect("owned runtime convergence completes after cancellation");
10938 drop(converged);
10939
10940 let live = state.config.read().await;
10941 assert_eq!(
10942 live.proxy_auth_credential_ref
10943 .as_ref()
10944 .map(|reference| reference.as_str()),
10945 Some("proxy.default.auth")
10946 );
10947 let auth = live
10948 .proxy_auth
10949 .as_ref()
10950 .expect("durable proxy auth must be published despite cancellation");
10951 assert_eq!(auth.username, "cancel-user");
10952 assert_eq!(auth.password, "cancel-secret");
10953 drop(live);
10954
10955 let root = std::fs::read_to_string(dir.path().join("core.json")).unwrap();
10956 let credentials = std::fs::read_to_string(dir.path().join("credentials.json")).unwrap();
10957 assert!(!root.contains("cancel-secret"));
10958 assert!(!credentials.contains("cancel-secret"));
10959 });
10960 }
10961
10962 #[tokio::test]
10963 async fn missing_or_corrupt_referenced_credentials_reject_candidates_redacted() {
10964 let _key = bamboo_config::encryption::set_test_encryption_key([0x68; 32]);
10965 for corrupt_credentials in [false, true] {
10966 let dir = tempfile::tempdir().unwrap();
10967 let reference = bamboo_config::credential_ref("provider", "openai", "api_key").unwrap();
10968 let providers = ProviderConfigs {
10969 openai: Some(bamboo_config::OpenAIConfig {
10970 credential_ref: Some(reference),
10971 model: Some("candidate".to_string()),
10972 ..Default::default()
10973 }),
10974 ..Default::default()
10975 };
10976 let provider_store = AtomicJsonStore::new(dir.path().join("providers.json"), 1);
10977 provider_store
10978 .commit(0, providers, validate_provider_config)
10979 .unwrap();
10980 if corrupt_credentials {
10981 std::fs::write(dir.path().join("credentials.json"), b"{corrupt-secret").unwrap();
10982 }
10983 let error =
10984 match load_and_prepare_provider_candidate(&provider_store, 0, Config::default())
10985 .await
10986 {
10987 Ok(_) => panic!("unavailable credential must reject provider candidate"),
10988 Err(error) => error,
10989 };
10990 assert_eq!(error.message, "provider credential is unavailable");
10991 assert!(!error
10992 .message
10993 .contains(dir.path().to_string_lossy().as_ref()));
10994
10995 let mut mcp = disabled_mcp_config("credential-lkg");
10996 let TransportConfig::Stdio(stdio) = &mut mcp.servers[0].transport else {
10997 unreachable!()
10998 };
10999 stdio.env_credential_refs.insert(
11000 "TOKEN".to_string(),
11001 bamboo_config::credential_ref("mcp", "credential-lkg", "env_TOKEN")
11002 .unwrap()
11003 .as_str()
11004 .to_string(),
11005 );
11006 let mcp_store = AtomicJsonStore::new(dir.path().join("mcp.json"), 1);
11007 mcp_store.commit(0, mcp, validate_mcp_config).unwrap();
11008 let error = match load_and_validate_mcp_candidate(
11009 &mcp_store,
11010 0,
11011 Config::default(),
11012 false,
11013 )
11014 .await
11015 {
11016 Ok(_) => panic!("unavailable credential must reject MCP candidate"),
11017 Err(error) => error,
11018 };
11019 assert_eq!(error.message, "MCP credential is unavailable");
11020 assert!(!error.message.contains("TOKEN"));
11021 assert!(!error
11022 .message
11023 .contains(dir.path().to_string_lossy().as_ref()));
11024 }
11025 }
11026
11027 #[tokio::test]
11028 async fn typed_provider_put_switches_refs_and_rejects_missing_ref_without_mutation() {
11029 let _key = bamboo_config::encryption::set_test_encryption_key([0x6d; 32]);
11030 let dir = tempfile::tempdir().unwrap();
11031 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11032 let ref_a = bamboo_config::credential_ref("provider", "openai-a", "api_key").unwrap();
11033 let ref_b = bamboo_config::credential_ref("provider", "openai-b", "api_key").unwrap();
11034 let missing = bamboo_config::credential_ref("provider", "missing", "api_key").unwrap();
11035 state
11036 .credential_store
11037 .replace(
11038 ref_a.clone(),
11039 "provider-secret-a",
11040 bamboo_config::CredentialSource::User,
11041 0,
11042 )
11043 .unwrap();
11044 state
11045 .credential_store
11046 .replace(
11047 ref_b.clone(),
11048 "provider-secret-b",
11049 bamboo_config::CredentialSource::User,
11050 1,
11051 )
11052 .unwrap();
11053 {
11054 let mut config = state.config.write().await;
11055 config.provider = "openai".to_string();
11056 *config.providers_mut() = ProviderConfigs {
11057 openai: Some(bamboo_config::OpenAIConfig {
11058 api_key: "provider-secret-a".to_string(),
11059 credential_ref: Some(ref_a),
11060 ..Default::default()
11061 }),
11062 ..Default::default()
11063 };
11064 }
11065
11066 let revision = state
11067 .put_provider_section(
11068 0,
11069 ProviderConfigs {
11070 openai: Some(bamboo_config::OpenAIConfig {
11071 credential_ref: Some(ref_b.clone()),
11072 model: Some("switched".to_string()),
11073 ..Default::default()
11074 }),
11075 ..Default::default()
11076 },
11077 )
11078 .await
11079 .unwrap();
11080 assert_eq!(revision, 1);
11081 let runtime = state.config.read().await;
11082 let openai = runtime.providers().openai.as_ref().unwrap();
11083 assert_eq!(openai.credential_ref.as_ref(), Some(&ref_b));
11084 assert_eq!(openai.api_key, "provider-secret-b");
11085 drop(runtime);
11086 let disk_before = std::fs::read(dir.path().join("providers.json")).unwrap();
11087
11088 assert!(state
11089 .put_provider_section(
11090 1,
11091 ProviderConfigs {
11092 openai: Some(bamboo_config::OpenAIConfig {
11093 credential_ref: Some(missing),
11094 model: Some("must-not-publish".to_string()),
11095 ..Default::default()
11096 }),
11097 ..Default::default()
11098 },
11099 )
11100 .await
11101 .is_err());
11102 assert_eq!(
11103 std::fs::read(dir.path().join("providers.json")).unwrap(),
11104 disk_before
11105 );
11106 let runtime = state.config.read().await;
11107 let openai = runtime.providers().openai.as_ref().unwrap();
11108 assert_eq!(openai.credential_ref.as_ref(), Some(&ref_b));
11109 assert_eq!(openai.api_key, "provider-secret-b");
11110 }
11111
11112 #[tokio::test]
11113 async fn typed_mcp_put_switches_stdio_and_header_refs_atomically() {
11114 let _key = bamboo_config::encryption::set_test_encryption_key([0x6e; 32]);
11115 let dir = tempfile::tempdir().unwrap();
11116 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11117 let refs = [
11118 bamboo_config::credential_ref("mcp", "stdio-a", "env_TOKEN").unwrap(),
11119 bamboo_config::credential_ref("mcp", "header-a", "header_Authorization").unwrap(),
11120 bamboo_config::credential_ref("mcp", "stdio-b", "env_TOKEN").unwrap(),
11121 bamboo_config::credential_ref("mcp", "header-b", "header_Authorization").unwrap(),
11122 ];
11123 for (revision, (reference, value)) in refs
11124 .iter()
11125 .zip(["env-a", "header-a", "env-b", "header-b"])
11126 .enumerate()
11127 {
11128 state
11129 .credential_store
11130 .replace(
11131 reference.clone(),
11132 value,
11133 bamboo_config::CredentialSource::User,
11134 revision as u64,
11135 )
11136 .unwrap();
11137 }
11138 let make_config = |env_ref: &bamboo_config::CredentialRef,
11139 header_ref: &bamboo_config::CredentialRef| {
11140 McpConfig {
11141 version: 1,
11142 servers: vec![
11143 McpServerConfig {
11144 id: "switch-stdio".to_string(),
11145 name: None,
11146 enabled: false,
11147 transport: TransportConfig::Stdio(StdioConfig {
11148 command: "unused-disabled-command".to_string(),
11149 args: vec![],
11150 cwd: None,
11151 env: std::collections::HashMap::new(),
11152 env_encrypted: std::collections::HashMap::new(),
11153 env_credential_refs: std::collections::HashMap::from([(
11154 "TOKEN".to_string(),
11155 env_ref.as_str().to_string(),
11156 )]),
11157 startup_timeout_ms: 100,
11158 }),
11159 request_timeout_ms: 100,
11160 healthcheck_interval_ms: 100,
11161 reconnect: ReconnectConfig::default(),
11162 allowed_tools: vec![],
11163 denied_tools: vec![],
11164 },
11165 McpServerConfig {
11166 id: "switch-header".to_string(),
11167 name: None,
11168 enabled: false,
11169 transport: TransportConfig::Sse(bamboo_mcp::SseConfig {
11170 url: "https://example.test/sse".to_string(),
11171 headers: vec![bamboo_mcp::HeaderConfig {
11172 name: "Authorization".to_string(),
11173 value: String::new(),
11174 value_encrypted: None,
11175 credential_ref: Some(header_ref.as_str().to_string()),
11176 }],
11177 connect_timeout_ms: 100,
11178 }),
11179 request_timeout_ms: 100,
11180 healthcheck_interval_ms: 100,
11181 reconnect: ReconnectConfig::default(),
11182 allowed_tools: vec![],
11183 denied_tools: vec![],
11184 },
11185 ],
11186 }
11187 };
11188 let mut current = make_config(&refs[0], &refs[1]);
11189 if let TransportConfig::Stdio(stdio) = &mut current.servers[0].transport {
11190 stdio.env.insert("TOKEN".to_string(), "env-a".to_string());
11191 }
11192 if let TransportConfig::Sse(sse) = &mut current.servers[1].transport {
11193 sse.headers[0].value = "header-a".to_string();
11194 }
11195 state.config.write().await.mcp = current;
11196
11197 assert_eq!(
11198 state
11199 .put_mcp_section(0, make_config(&refs[2], &refs[3]))
11200 .await
11201 .unwrap(),
11202 1
11203 );
11204 let runtime = state.config.read().await;
11205 let TransportConfig::Stdio(stdio) = &runtime
11206 .mcp
11207 .servers
11208 .iter()
11209 .find(|server| server.id == "switch-stdio")
11210 .expect("stdio server")
11211 .transport
11212 else {
11213 panic!("stdio transport")
11214 };
11215 assert_eq!(stdio.env["TOKEN"], "env-b");
11216 let TransportConfig::Sse(sse) = &runtime
11217 .mcp
11218 .servers
11219 .iter()
11220 .find(|server| server.id == "switch-header")
11221 .expect("SSE server")
11222 .transport
11223 else {
11224 panic!("sse transport")
11225 };
11226 assert_eq!(sse.headers[0].value, "header-b");
11227 drop(runtime);
11228 let disk_before = std::fs::read(dir.path().join("mcp.json")).unwrap();
11229 let missing_env = bamboo_config::credential_ref("mcp", "missing", "env_TOKEN").unwrap();
11230 let missing_header =
11231 bamboo_config::credential_ref("mcp", "missing", "header_Authorization").unwrap();
11232 assert!(state
11233 .put_mcp_section(1, make_config(&missing_env, &missing_header))
11234 .await
11235 .is_err());
11236 assert_eq!(
11237 std::fs::read(dir.path().join("mcp.json")).unwrap(),
11238 disk_before
11239 );
11240 let runtime = state.config.read().await;
11241 let TransportConfig::Stdio(stdio) = &runtime
11242 .mcp
11243 .servers
11244 .iter()
11245 .find(|server| server.id == "switch-stdio")
11246 .expect("stdio server")
11247 .transport
11248 else {
11249 panic!("stdio transport")
11250 };
11251 assert_eq!(stdio.env["TOKEN"], "env-b");
11252 }
11253
11254 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
11255 async fn stale_initial_mcp_batch_does_not_reapply_after_typed_revision_advances() {
11256 let dir = tempfile::tempdir().unwrap();
11257 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11258 let make_server = |id: &str, transport: TransportConfig| McpServerConfig {
11259 id: id.to_string(),
11260 name: None,
11261 enabled: false,
11262 transport,
11263 request_timeout_ms: 100,
11264 healthcheck_interval_ms: 100,
11265 reconnect: ReconnectConfig::default(),
11266 allowed_tools: vec![],
11267 denied_tools: vec![],
11268 };
11269 let initial = McpConfig {
11270 version: 1,
11271 servers: vec![make_server(
11272 "initial",
11273 TransportConfig::Stdio(StdioConfig {
11274 command: "unused-initial-command".to_string(),
11275 args: vec![],
11276 cwd: None,
11277 env: std::collections::HashMap::new(),
11278 env_encrypted: std::collections::HashMap::new(),
11279 env_credential_refs: std::collections::HashMap::new(),
11280 startup_timeout_ms: 100,
11281 }),
11282 )],
11283 };
11284 assert_eq!(state.put_mcp_section(0, initial).await.unwrap(), 1);
11285 stop_config_watcher(&mut state);
11286
11287 let (reached_tx, reached_rx) = std::sync::mpsc::channel();
11288 let (release_tx, release_rx) = std::sync::mpsc::channel();
11289 let (done_tx, done_rx) = std::sync::mpsc::channel();
11290 set_initial_mcp_apply_test_hook(
11291 dir.path(),
11292 move || {
11293 reached_tx.send(()).unwrap();
11294 release_rx.recv().unwrap();
11295 },
11296 move || {
11297 done_tx.send(()).unwrap();
11298 },
11299 );
11300 restart_config_watcher(&mut state);
11301 tokio::task::spawn_blocking(move || reached_rx.recv().unwrap())
11302 .await
11303 .unwrap();
11304
11305 let latest = McpConfig {
11310 version: 1,
11311 servers: vec![
11312 make_server(
11313 "z-stdio",
11314 TransportConfig::Stdio(StdioConfig {
11315 command: "unused-latest-command".to_string(),
11316 args: vec![],
11317 cwd: None,
11318 env: std::collections::HashMap::new(),
11319 env_encrypted: std::collections::HashMap::new(),
11320 env_credential_refs: std::collections::HashMap::new(),
11321 startup_timeout_ms: 100,
11322 }),
11323 ),
11324 make_server(
11325 "a-sse",
11326 TransportConfig::Sse(bamboo_mcp::SseConfig {
11327 url: "https://example.test/sse".to_string(),
11328 headers: vec![],
11329 connect_timeout_ms: 100,
11330 }),
11331 ),
11332 ],
11333 };
11334 let baseline = state.account_sink.latest_seq();
11335 assert_eq!(state.put_mcp_section(1, latest).await.unwrap(), 2);
11336 release_tx.send(()).unwrap();
11337 tokio::task::spawn_blocking(move || done_rx.recv().unwrap())
11338 .await
11339 .unwrap();
11340
11341 let runtime = state.config.read().await;
11342 assert_eq!(
11343 runtime
11344 .mcp
11345 .servers
11346 .iter()
11347 .map(|server| server.id.as_str())
11348 .collect::<Vec<_>>(),
11349 vec!["z-stdio", "a-sse"],
11350 "the superseded startup generation must not reapply"
11351 );
11352 drop(runtime);
11353 let mcp_events =
11354 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), baseline)
11355 .unwrap()
11356 .into_iter()
11357 .filter(|change| {
11358 matches!(
11359 &change.event,
11360 AgentEvent::ConfigChanged { section, revision }
11361 | AgentEvent::ConfigRecovered { section, revision }
11362 if section == "mcp" && *revision == 2
11363 )
11364 })
11365 .count();
11366 assert_eq!(
11367 mcp_events, 1,
11368 "the startup batch must not emit a pseudo event"
11369 );
11370 }
11371
11372 #[tokio::test]
11373 async fn failed_candidate_keeps_existing_provider_registry_and_runtime() {
11374 let dir = tempfile::tempdir().unwrap();
11375 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11376 let working: Arc<dyn LLMProvider> = Arc::new(WorkingProvider);
11377 state
11378 .provider_registry
11379 .insert("working".to_string(), working.clone());
11380 state.provider_registry.set_default("working".to_string());
11381 *state.provider.write().await = working.clone();
11382 state.config.write().await.provider = "openai".to_string();
11383
11384 assert!(state.reload_provider().await.is_err());
11385 assert_eq!(state.provider_registry.default_provider_name(), "working");
11386 assert!(Arc::ptr_eq(
11387 &state.provider_registry.get_default().unwrap(),
11388 &working
11389 ));
11390 let live = state.provider.read().await;
11391 assert!(Arc::ptr_eq(&*live, &working));
11392 }
11393
11394 #[tokio::test]
11395 async fn provider_watcher_retains_lkg_on_invalid_and_recovers_after_repair() {
11396 let _key = bamboo_config::encryption::set_test_encryption_key([0x43; 32]);
11397 let dir = tempfile::tempdir().unwrap();
11398 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11399 let working: Arc<dyn LLMProvider> = Arc::new(WorkingProvider);
11400 state
11401 .provider_registry
11402 .insert("working".to_string(), working.clone());
11403 state.provider_registry.set_default("working".to_string());
11404 *state.provider.write().await = working.clone();
11405 state.config.write().await.provider = "openai".to_string();
11406 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
11407 let mut feed = state.account_sink.subscribe();
11408 let providers_path = dir.path().join("providers.json");
11409
11410 std::fs::write(&providers_path, b"{broken").unwrap();
11411 tokio::time::timeout(Duration::from_secs(3), async {
11412 loop {
11413 if state
11414 .config_live_health
11415 .read()
11416 .unwrap_or_else(|poisoned| poisoned.into_inner())
11417 .status
11418 == SectionStatus::Invalid
11419 {
11420 break;
11421 }
11422 tokio::time::sleep(Duration::from_millis(20)).await;
11423 }
11424 })
11425 .await
11426 .unwrap();
11427 assert_eq!(
11428 state
11429 .config_live_health
11430 .read()
11431 .unwrap_or_else(|poisoned| poisoned.into_inner())
11432 .revision,
11433 0,
11434 "invalid edits must not advance the LKG revision"
11435 );
11436 {
11437 let health = state
11438 .config_live_health
11439 .read()
11440 .unwrap_or_else(|poisoned| poisoned.into_inner());
11441 assert_eq!(health.status, SectionStatus::Invalid);
11442 assert_eq!(health.source_kind, SectionSourceKind::File);
11443 assert_eq!(health.source_path, providers_path);
11444 }
11445 assert!(Arc::ptr_eq(
11446 &state.provider_registry.get_default().unwrap(),
11447 &working
11448 ));
11449 let invalid = next_config_event(&mut feed, "providers").await;
11450 assert!(matches!(
11451 invalid,
11452 AgentEvent::ConfigInvalid { revision: 0, .. }
11453 ));
11454
11455 let reference = bamboo_config::credential_ref("provider", "openai", "api_key").unwrap();
11456 let credential_store = bamboo_config::CredentialStore::open(dir.path());
11457 let credential_revision = credential_store.revision().unwrap();
11458 credential_store
11459 .replace(
11460 reference.clone(),
11461 "watcher-test-key",
11462 bamboo_config::CredentialSource::User,
11463 credential_revision,
11464 )
11465 .unwrap();
11466 let providers = ProviderConfigs {
11467 openai: Some(bamboo_config::OpenAIConfig {
11468 credential_ref: Some(reference),
11469 ..Default::default()
11470 }),
11471 ..Default::default()
11472 };
11473 std::fs::write(
11474 &providers_path,
11475 serde_json::to_vec_pretty(&providers).unwrap(),
11476 )
11477 .unwrap();
11478 tokio::time::timeout(Duration::from_secs(3), async {
11479 loop {
11480 let health = state
11481 .config_live_health
11482 .read()
11483 .unwrap_or_else(|poisoned| poisoned.into_inner())
11484 .clone();
11485 if health.status == SectionStatus::Healthy && health.revision == 1 {
11486 break;
11487 }
11488 tokio::time::sleep(Duration::from_millis(20)).await;
11489 }
11490 })
11491 .await
11492 .unwrap();
11493 let recovered = next_config_event(&mut feed, "providers").await;
11494 assert!(matches!(
11495 recovered,
11496 AgentEvent::ConfigRecovered { revision: 1, .. }
11497 ));
11498 assert_eq!(state.provider_registry.default_provider_name(), "openai");
11499 }
11500
11501 #[tokio::test]
11502 async fn ordinary_section_watcher_updates_runtime_retains_lkg_and_recovers() {
11503 let _key = bamboo_config::encryption::set_test_encryption_key([0x44; 32]);
11504 let dir = tempfile::tempdir().unwrap();
11505 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11506 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
11507 let mut feed = state.account_sink.subscribe();
11508 let path = dir.path().join("core.json");
11509 let mut document: serde_json::Value =
11510 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
11511 document["revision"] = serde_json::json!(2);
11512 document["data"]["server"]["port"] = serde_json::json!(9876);
11513 std::fs::write(&path, serde_json::to_vec_pretty(&document).unwrap()).unwrap();
11514
11515 wait_for_facade_health(&state, SectionId::Core, SectionStatus::Healthy, 2).await;
11516 tokio::time::timeout(Duration::from_secs(3), async {
11517 loop {
11518 if state.config.read().await.server.port == 9876 {
11519 break;
11520 }
11521 tokio::time::sleep(Duration::from_millis(20)).await;
11522 }
11523 })
11524 .await
11525 .unwrap();
11526 assert!(matches!(
11527 next_config_event(&mut feed, "core").await,
11528 AgentEvent::ConfigChanged { revision: 2, .. }
11529 ));
11530
11531 std::fs::write(&path, b"{broken").unwrap();
11532 wait_for_facade_health(&state, SectionId::Core, SectionStatus::Invalid, 2).await;
11533 assert_eq!(state.config.read().await.server.port, 9876);
11534 assert!(matches!(
11535 next_config_event(&mut feed, "core").await,
11536 AgentEvent::ConfigInvalid { revision: 2, .. }
11537 ));
11538
11539 document["revision"] = serde_json::json!(3);
11540 document["data"]["server"]["port"] = serde_json::json!(9877);
11541 std::fs::write(&path, serde_json::to_vec_pretty(&document).unwrap()).unwrap();
11542 wait_for_facade_health(&state, SectionId::Core, SectionStatus::Healthy, 3).await;
11543 tokio::time::timeout(Duration::from_secs(3), async {
11544 loop {
11545 if state.config.read().await.server.port == 9877 {
11546 break;
11547 }
11548 tokio::time::sleep(Duration::from_millis(20)).await;
11549 }
11550 })
11551 .await
11552 .unwrap();
11553 assert!(matches!(
11554 next_config_event(&mut feed, "core").await,
11555 AgentEvent::ConfigRecovered { revision: 3, .. }
11556 ));
11557
11558 std::fs::remove_file(&path).unwrap();
11559 wait_for_facade_health(&state, SectionId::Core, SectionStatus::Missing, 3).await;
11560 assert_eq!(state.config.read().await.server.port, 9877);
11561 assert!(matches!(
11562 next_config_event(&mut feed, "core").await,
11563 AgentEvent::ConfigInvalid { revision: 3, .. }
11564 ));
11565
11566 document["revision"] = serde_json::json!(4);
11567 document["data"]["server"]["port"] = serde_json::json!(9878);
11568 let swap = dir.path().join("core.json.swap");
11569 std::fs::write(&swap, serde_json::to_vec_pretty(&document).unwrap()).unwrap();
11570 std::fs::rename(&swap, &path).unwrap();
11571 wait_for_facade_health(&state, SectionId::Core, SectionStatus::Healthy, 4).await;
11572 tokio::time::timeout(Duration::from_secs(3), async {
11573 loop {
11574 if state.config.read().await.server.port == 9878 {
11575 break;
11576 }
11577 tokio::time::sleep(Duration::from_millis(20)).await;
11578 }
11579 })
11580 .await
11581 .unwrap();
11582 assert!(matches!(
11583 next_config_event(&mut feed, "core").await,
11584 AgentEvent::ConfigRecovered { revision: 4, .. }
11585 ));
11586 }
11587
11588 #[tokio::test]
11589 async fn mcp_watcher_updates_lkg_rejects_invalid_and_recovers_after_atomic_replace() {
11590 let _key = bamboo_config::encryption::set_test_encryption_key([0x45; 32]);
11591 let dir = tempfile::tempdir().unwrap();
11592 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11593 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
11594 let mut feed = state.account_sink.subscribe();
11595 let path = dir.path().join("mcp.json");
11596
11597 std::fs::write(&path, mcp_document_bytes(2, &disabled_mcp_config("first"))).unwrap();
11598 let first = wait_for_mcp_health(&state, SectionStatus::Healthy, 2).await;
11599 assert_eq!(first.revision, 2);
11600 assert_eq!(state.config.read().await.mcp.servers[0].id, "first");
11601 assert!(matches!(
11602 next_mcp_config_event(&mut feed).await,
11603 AgentEvent::ConfigChanged { revision: 2, .. }
11604 ));
11605
11606 std::fs::write(&path, b"{broken").unwrap();
11607 let invalid = wait_for_mcp_health(&state, SectionStatus::Invalid, 2).await;
11608 assert_eq!(invalid.revision, 2, "invalid candidates cannot advance LKG");
11609 assert_eq!(state.config.read().await.mcp.servers[0].id, "first");
11610 assert!(matches!(
11611 next_mcp_config_event(&mut feed).await,
11612 AgentEvent::ConfigInvalid { revision: 2, .. }
11613 ));
11614
11615 let swap = dir.path().join("mcp.json.swap");
11620 std::fs::write(
11621 &swap,
11622 mcp_document_bytes(3, &disabled_mcp_config("intermediate")),
11623 )
11624 .unwrap();
11625 std::fs::rename(&swap, &path).unwrap();
11626 std::fs::write(
11627 &path,
11628 mcp_document_bytes(3, &disabled_mcp_config("recovered")),
11629 )
11630 .unwrap();
11631 let recovered = wait_for_mcp_health(&state, SectionStatus::Healthy, 3).await;
11632 assert_eq!(recovered.revision, 3, "rename burst should coalesce once");
11633 assert_eq!(state.config.read().await.mcp.servers[0].id, "recovered");
11634 assert!(matches!(
11635 next_mcp_config_event(&mut feed).await,
11636 AgentEvent::ConfigRecovered { revision: 3, .. }
11637 ));
11638
11639 std::fs::write(
11644 &path,
11645 mcp_document_bytes(3, &disabled_mcp_config("normalized")),
11646 )
11647 .unwrap();
11648 let normalized = wait_for_mcp_health(&state, SectionStatus::Healthy, 4).await;
11649 assert_eq!(normalized.revision, 4);
11650 assert_eq!(state.config.read().await.mcp.servers[0].id, "normalized");
11651 assert!(matches!(
11652 next_mcp_config_event(&mut feed).await,
11653 AgentEvent::ConfigChanged { revision: 4, .. }
11654 ));
11655 assert!(
11656 tokio::time::timeout(Duration::from_millis(500), feed.recv())
11657 .await
11658 .is_err()
11659 );
11660 let persisted: serde_json::Value =
11661 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
11662 assert_eq!(persisted["revision"], 4);
11663 }
11664
11665 #[tokio::test]
11666 async fn mcp_sidecar_present_at_startup_is_applied_through_runtime_transaction() {
11667 let _key = bamboo_config::encryption::set_test_encryption_key([0x47; 32]);
11668 let dir = tempfile::tempdir().unwrap();
11669 std::fs::write(
11670 dir.path().join("mcp.json"),
11671 mcp_document_bytes(1, &disabled_mcp_config("startup-sidecar")),
11672 )
11673 .unwrap();
11674
11675 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11676 let health = wait_for_mcp_health(&state, SectionStatus::Healthy, 1).await;
11677 assert_eq!(health.revision, 1);
11678 assert_eq!(health.source_kind, SectionSourceKind::File);
11679 assert_eq!(
11680 state.config.read().await.mcp.servers[0].id,
11681 "startup-sidecar"
11682 );
11683 }
11684
11685 #[tokio::test]
11686 async fn mcp_startup_uses_valid_backup_and_reports_degraded_invalid_health() {
11687 let _key = bamboo_config::encryption::set_test_encryption_key([0x48; 32]);
11688 let dir = tempfile::tempdir().unwrap();
11689 let path = dir.path().join("mcp.json");
11690 let store = AtomicJsonStore::new(&path, 1);
11691 store
11692 .commit(0, disabled_mcp_config("backup-lkg"), validate_mcp_config)
11693 .unwrap();
11694 store
11695 .commit(1, disabled_mcp_config("new-primary"), validate_mcp_config)
11696 .unwrap();
11697 std::fs::write(&path, b"{corrupt-primary").unwrap();
11698
11699 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11700 let health = wait_for_mcp_health(&state, SectionStatus::Degraded, 1).await;
11701 assert_eq!(health.revision, 1);
11702 assert_eq!(health.source_kind, SectionSourceKind::Backup);
11703 assert_eq!(health.source_path, path.with_extension("json.bak"));
11704 assert!(health
11705 .last_error
11706 .as_deref()
11707 .unwrap()
11708 .contains("last-known-good backup runtime"));
11709 assert_eq!(state.config.read().await.mcp.servers[0].id, "backup-lkg");
11710
11711 tokio::time::timeout(Duration::from_secs(3), async {
11712 loop {
11713 if state.account_sink.latest_seq() > 0 {
11714 break;
11715 }
11716 tokio::time::sleep(Duration::from_millis(20)).await;
11717 }
11718 })
11719 .await
11720 .unwrap();
11721 tokio::time::sleep(Duration::from_millis(500)).await;
11722 let events =
11723 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
11724 assert_eq!(
11725 events
11726 .iter()
11727 .filter(|event| matches!(
11728 &event.event,
11729 AgentEvent::ConfigInvalid { section, revision }
11730 if section == "mcp" && *revision == 1
11731 ))
11732 .count(),
11733 1
11734 );
11735 let stable_health = state
11736 .mcp_config_live_health
11737 .read()
11738 .unwrap_or_else(|poisoned| poisoned.into_inner())
11739 .clone();
11740 assert_eq!(stable_health.status, SectionStatus::Degraded);
11741 assert_eq!(stable_health.source_kind, SectionSourceKind::Backup);
11742 assert_eq!(stable_health.source_path, path.with_extension("json.bak"));
11743 }
11744
11745 #[tokio::test]
11746 async fn mcp_runtime_init_failure_marks_degraded_and_retains_lkg_config() {
11747 let _key = bamboo_config::encryption::set_test_encryption_key([0x46; 32]);
11748 let dir = tempfile::tempdir().unwrap();
11749 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11750 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
11751 let mut feed = state.account_sink.subscribe();
11752 let path = dir.path().join("mcp.json");
11753
11754 std::fs::write(
11755 &path,
11756 mcp_document_bytes(1, &disabled_mcp_config("last-known-good")),
11757 )
11758 .unwrap();
11759 wait_for_mcp_health(&state, SectionStatus::Healthy, 1).await;
11760 let _ = next_mcp_config_event(&mut feed).await;
11761
11762 let mut failing = disabled_mcp_config("candidate");
11763 failing.servers[0].enabled = true;
11764 if let TransportConfig::Stdio(stdio) = &mut failing.servers[0].transport {
11765 stdio.command = "definitely-not-a-real-mcp-command-597".to_string();
11766 }
11767 std::fs::write(&path, mcp_document_bytes(2, &failing)).unwrap();
11768
11769 let degraded = wait_for_mcp_health(&state, SectionStatus::Degraded, 1).await;
11770 assert_eq!(degraded.revision, 1);
11771 assert!(degraded
11772 .last_error
11773 .as_deref()
11774 .unwrap()
11775 .contains("last-known-good runtime"));
11776 assert_eq!(
11777 state.config.read().await.mcp.servers[0].id,
11778 "last-known-good"
11779 );
11780 assert!(state.mcp_manager.list_servers().is_empty());
11781 assert!(matches!(
11782 next_mcp_config_event(&mut feed).await,
11783 AgentEvent::ConfigInvalid { revision: 1, .. }
11784 ));
11785 }
11786
11787 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
11788 async fn stopped_root_commit_is_installed_before_one_confirmed_event_on_same_facade_restart() {
11789 let _key = bamboo_config::encryption::set_test_encryption_key([0x52; 32]);
11790 let dir = tempfile::tempdir().unwrap();
11791 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11792 stop_config_watcher(&mut state);
11793 std::fs::write(
11794 dir.path().join("config.json"),
11795 br#"{"server":{"port":25201}}"#,
11796 )
11797 .unwrap();
11798 let committed = state
11799 .config_facade
11800 .as_ref()
11801 .unwrap()
11802 .reconcile_reappeared_legacy_root()
11803 .unwrap()
11804 .unwrap();
11805 assert_eq!(committed.committed.len(), 1);
11806 assert_ne!(state.config.read().await.server.port, 25_201);
11807
11808 let mut feed = state.account_sink.subscribe();
11809 let config = state.config.clone();
11810 let read_guard = config.read().await;
11811 restart_config_watcher(&mut state);
11812 assert!(
11813 tokio::time::timeout(
11814 Duration::from_millis(350),
11815 next_config_event(&mut feed, "core")
11816 )
11817 .await
11818 .is_err(),
11819 "the account event must wait for the runtime write"
11820 );
11821 drop(read_guard);
11822
11823 assert!(matches!(
11824 next_config_event(&mut feed, "core").await,
11825 AgentEvent::ConfigChanged { revision: 1, .. }
11826 ));
11827 assert_eq!(state.config.read().await.server.port, 25_201);
11828 wait_for_root_outbox_to_clear(dir.path()).await;
11829 tokio::time::sleep(Duration::from_millis(250)).await;
11830 let events =
11831 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
11832 assert_eq!(
11833 events
11834 .iter()
11835 .filter(|event| matches!(
11836 &event.event,
11837 AgentEvent::ConfigChanged { section, revision }
11838 if section == "core" && *revision == 1
11839 ))
11840 .count(),
11841 1
11842 );
11843
11844 stop_config_watcher(&mut state);
11845 restart_config_watcher(&mut state);
11846 assert!(tokio::time::timeout(
11847 Duration::from_millis(500),
11848 next_config_event(&mut feed, "core")
11849 )
11850 .await
11851 .is_err());
11852 }
11853
11854 #[tokio::test]
11855 async fn pending_root_resolver_unavailable_keeps_lkg_silent_and_requests_retry() {
11856 let _key = bamboo_config::encryption::set_test_encryption_key([0x59; 32]);
11857 let dir = tempfile::tempdir().unwrap();
11858 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11859 stop_config_watcher(&mut state);
11860 let old_port = state.config.read().await.server.port;
11861 std::fs::write(
11862 dir.path().join("config.json"),
11863 br#"{"server":{"port":25901}}"#,
11864 )
11865 .unwrap();
11866 let committed = state
11867 .config_facade
11868 .as_ref()
11869 .unwrap()
11870 .reconcile_reappeared_legacy_root()
11871 .unwrap()
11872 .unwrap();
11873 let event = committed.committed[0].clone();
11874 let mut synthetic_events = BTreeMap::from([(SectionId::Core, event.clone())]);
11875 let mut pending_root_publications = BTreeMap::from([(SectionId::Core, event)]);
11876 let mut reported_root_runtime_failures = BTreeSet::new();
11877 let mut feed = state.account_sink.subscribe();
11878 std::fs::remove_file(dir.path().join("config-section-layout-completion.json")).unwrap();
11879
11880 let retry = reload_and_apply_ordinary_sections(
11881 dir.path(),
11882 &state.config,
11883 state.config_facade.as_ref().unwrap(),
11884 &state.account_sink,
11885 std::iter::once(SectionId::Core),
11886 OrdinarySectionReloadState {
11887 synthetic_events: &mut synthetic_events,
11888 pending_root_publications: &mut pending_root_publications,
11889 reported_root_runtime_failures: &mut reported_root_runtime_failures,
11890 },
11891 )
11892 .await;
11893
11894 assert!(retry);
11895 assert_eq!(state.config.read().await.server.port, old_port);
11896 assert!(
11897 tokio::time::timeout(Duration::from_millis(250), feed.recv())
11898 .await
11899 .is_err()
11900 );
11901 assert!(pending_root_publications.contains_key(&SectionId::Core));
11902 }
11903
11904 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
11905 async fn pending_root_then_new_root_survives_coalesced_mcp_noop_and_requeues() {
11906 let _key = bamboo_config::encryption::set_test_encryption_key([0x53; 32]);
11907 let dir = tempfile::tempdir().unwrap();
11908 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11909 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
11910 let mut feed = state.account_sink.subscribe();
11911 let io = state.config_io_lock.clone().lock_owned().await;
11912 std::fs::write(
11913 dir.path().join("config.json"),
11914 br#"{"server":{"port":25301}}"#,
11915 )
11916 .unwrap();
11917 let first = state
11918 .config_facade
11919 .as_ref()
11920 .unwrap()
11921 .reconcile_reappeared_legacy_root()
11922 .unwrap()
11923 .unwrap();
11924 assert_eq!(first.committed.len(), 1);
11925 std::fs::write(
11926 dir.path().join("config.json"),
11927 br#"{"server":{"port":25302}}"#,
11928 )
11929 .unwrap();
11930 let mcp_bytes = std::fs::read(dir.path().join("mcp.json")).unwrap();
11931 std::fs::write(dir.path().join("mcp.json"), mcp_bytes).unwrap();
11932 tokio::time::sleep(Duration::from_millis(300)).await;
11933 drop(io);
11934
11935 let first_event = next_config_event(&mut feed, "core").await;
11936 let second_event = next_config_event(&mut feed, "core").await;
11937 assert!(matches!(
11938 first_event,
11939 AgentEvent::ConfigChanged { revision: 1, .. }
11940 ));
11941 assert!(matches!(
11942 second_event,
11943 AgentEvent::ConfigChanged { revision: 2, .. }
11944 ));
11945 tokio::time::timeout(Duration::from_secs(5), async {
11946 loop {
11947 if state.config.read().await.server.port == 25_302 {
11948 break;
11949 }
11950 tokio::time::sleep(Duration::from_millis(20)).await;
11951 }
11952 })
11953 .await
11954 .unwrap();
11955 wait_for_root_outbox_to_clear(dir.path()).await;
11956 }
11957
11958 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
11959 async fn startup_handoff_always_catches_root_generation_written_before_watcher_registration() {
11960 let _key = bamboo_config::encryption::set_test_encryption_key([0x56; 32]);
11961 let dir = tempfile::tempdir().unwrap();
11962 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11963 stop_config_watcher(&mut state);
11964 std::fs::write(
11965 dir.path().join("config.json"),
11966 br#"{"server":{"port":25601}}"#,
11967 )
11968 .unwrap();
11969 let startup_facade =
11970 Arc::new(bamboo_config::ConfigFacade::open_or_migrate(dir.path()).unwrap());
11971 assert!(bamboo_config::has_pending_legacy_root_publications(dir.path()).unwrap());
11972 state.config_facade = Some(startup_facade);
11973 std::fs::write(
11974 dir.path().join("config.json"),
11975 br#"{"server":{"port":25602}}"#,
11976 )
11977 .unwrap();
11978 let mut feed = state.account_sink.subscribe();
11979
11980 restart_config_watcher(&mut state);
11981
11982 assert!(matches!(
11983 next_config_event(&mut feed, "core").await,
11984 AgentEvent::ConfigChanged { revision: 1, .. }
11985 ));
11986 assert!(matches!(
11987 next_config_event(&mut feed, "core").await,
11988 AgentEvent::ConfigChanged { revision: 2, .. }
11989 ));
11990 tokio::time::timeout(Duration::from_secs(5), async {
11991 loop {
11992 if state.config.read().await.server.port == 25_602 {
11993 break;
11994 }
11995 tokio::time::sleep(Duration::from_millis(20)).await;
11996 }
11997 })
11998 .await
11999 .unwrap();
12000 wait_for_root_outbox_to_clear(dir.path()).await;
12001 }
12002
12003 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
12004 async fn same_facade_restart_replays_one_rejection_and_one_lost_recovery() {
12005 let _key = bamboo_config::encryption::set_test_encryption_key([0x54; 32]);
12006 let dir = tempfile::tempdir().unwrap();
12007 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
12008 stop_config_watcher(&mut state);
12009 std::fs::write(
12010 dir.path().join("config.json"),
12011 br#"{"server":{"vendor_api_key":"never-persist-this"}}"#,
12012 )
12013 .unwrap();
12014 let rejected = state
12015 .config_facade
12016 .as_ref()
12017 .unwrap()
12018 .reconcile_reappeared_legacy_root()
12019 .unwrap()
12020 .unwrap();
12021 assert_eq!(rejected.rejected.len(), 1);
12022 assert_eq!(
12023 state
12024 .config_facade
12025 .as_ref()
12026 .unwrap()
12027 .registry()
12028 .core
12029 .snapshot()
12030 .status,
12031 SectionStatus::Healthy
12032 );
12033 let mut feed = state.account_sink.subscribe();
12034 restart_config_watcher(&mut state);
12035 assert!(matches!(
12036 next_config_event(&mut feed, "core").await,
12037 AgentEvent::ConfigInvalid { revision: 0, .. }
12038 ));
12039 stop_config_watcher(&mut state);
12040
12041 std::fs::write(dir.path().join("config.json"), b"{}").unwrap();
12042 let recovered = state
12043 .config_facade
12044 .as_ref()
12045 .unwrap()
12046 .reconcile_reappeared_legacy_root()
12047 .unwrap()
12048 .unwrap();
12049 assert_eq!(recovered.recovered, vec![SectionId::Core]);
12050 assert_eq!(
12051 state
12052 .config_facade
12053 .as_ref()
12054 .unwrap()
12055 .registry()
12056 .core
12057 .snapshot()
12058 .status,
12059 SectionStatus::Degraded
12060 );
12061 restart_config_watcher(&mut state);
12062 assert!(matches!(
12063 next_config_event(&mut feed, "core").await,
12064 AgentEvent::ConfigRecovered { revision: 0, .. }
12065 ));
12066 assert_eq!(
12067 state
12068 .config_facade
12069 .as_ref()
12070 .unwrap()
12071 .registry()
12072 .core
12073 .snapshot()
12074 .status,
12075 SectionStatus::Healthy
12076 );
12077
12078 stop_config_watcher(&mut state);
12079 restart_config_watcher(&mut state);
12080 assert!(tokio::time::timeout(
12081 Duration::from_millis(500),
12082 next_config_event(&mut feed, "core")
12083 )
12084 .await
12085 .is_err());
12086 let events =
12087 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
12088 assert_eq!(
12089 events
12090 .iter()
12091 .filter(|event| matches!(
12092 &event.event,
12093 AgentEvent::ConfigInvalid { section, .. } if section == "core"
12094 ))
12095 .count(),
12096 1
12097 );
12098 assert_eq!(
12099 events
12100 .iter()
12101 .filter(|event| matches!(
12102 &event.event,
12103 AgentEvent::ConfigRecovered { section, .. } if section == "core"
12104 ))
12105 .count(),
12106 1
12107 );
12108 }
12109
12110 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
12111 async fn degraded_root_mcp_is_carried_while_new_root_core_commits_then_recovers() {
12112 let _key = bamboo_config::encryption::set_test_encryption_key([0x57; 32]);
12113 let dir = tempfile::tempdir().unwrap();
12114 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
12115 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
12116 let failing = working_stdio_mcp_config(dir.path(), "root-carry", None);
12117 let script = match &failing.servers[0].transport {
12118 TransportConfig::Stdio(stdio) => PathBuf::from(&stdio.args[0]),
12119 _ => unreachable!(),
12120 };
12121 std::fs::remove_file(&script).unwrap();
12122 let mut feed = state.account_sink.subscribe();
12123 std::fs::write(
12124 dir.path().join("config.json"),
12125 serde_json::to_vec(&serde_json::json!({"mcp": failing})).unwrap(),
12126 )
12127 .unwrap();
12128 assert!(matches!(
12129 next_mcp_config_event(&mut feed).await,
12130 AgentEvent::ConfigInvalid { revision: 1, .. }
12131 ));
12132
12133 std::fs::write(
12134 dir.path().join("config.json"),
12135 br#"{"server":{"port":25701}}"#,
12136 )
12137 .unwrap();
12138 assert!(matches!(
12139 next_config_event(&mut feed, "core").await,
12140 AgentEvent::ConfigChanged { revision: 1, .. }
12141 ));
12142 tokio::time::timeout(Duration::from_secs(5), async {
12143 loop {
12144 if state.config.read().await.server.port == 25_701 {
12145 break;
12146 }
12147 tokio::time::sleep(Duration::from_millis(20)).await;
12148 }
12149 })
12150 .await
12151 .unwrap();
12152 assert!(bamboo_config::has_pending_legacy_root_publications(dir.path()).unwrap());
12153
12154 let repaired = working_stdio_mcp_config(dir.path(), "root-carry", None);
12155 assert_eq!(repaired.servers[0].id, "root-carry");
12156 assert!(matches!(
12157 next_mcp_config_event(&mut feed).await,
12158 AgentEvent::ConfigRecovered { revision: 1, .. }
12159 ));
12160 wait_for_root_outbox_to_clear(dir.path()).await;
12161 let events =
12162 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
12163 assert_eq!(
12164 events
12165 .iter()
12166 .filter(|event| matches!(
12167 &event.event,
12168 AgentEvent::ConfigInvalid { section, revision }
12169 if section == "mcp" && *revision == 1
12170 ))
12171 .count(),
12172 1
12173 );
12174 assert_eq!(
12175 events
12176 .iter()
12177 .filter(|event| matches!(
12178 &event.event,
12179 AgentEvent::ConfigRecovered { section, revision }
12180 if section == "mcp" && *revision == 1
12181 ))
12182 .count(),
12183 1
12184 );
12185 }
12186
12187 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
12188 async fn rejected_new_mcp_keeps_old_degraded_publication_dormant_until_clean_root() {
12189 let _key = bamboo_config::encryption::set_test_encryption_key([0x5b; 32]);
12190 let dir = tempfile::tempdir().unwrap();
12191 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
12192 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
12193 let failing = working_stdio_mcp_config(dir.path(), "root-dormant", None);
12194 let script = match &failing.servers[0].transport {
12195 TransportConfig::Stdio(stdio) => PathBuf::from(&stdio.args[0]),
12196 _ => unreachable!(),
12197 };
12198 std::fs::remove_file(&script).unwrap();
12199 let mut feed = state.account_sink.subscribe();
12200 std::fs::write(
12201 dir.path().join("config.json"),
12202 serde_json::to_vec(&serde_json::json!({"mcp": failing})).unwrap(),
12203 )
12204 .unwrap();
12205 assert!(matches!(
12206 next_mcp_config_event(&mut feed).await,
12207 AgentEvent::ConfigInvalid { revision: 1, .. }
12208 ));
12209
12210 std::fs::write(
12211 dir.path().join("config.json"),
12212 serde_json::to_vec(&serde_json::json!({
12213 "server": {"port": 25801},
12214 "mcpServers": {
12215 "rejected-next": {
12216 "command": "unused-rejected-command",
12217 "disabled": true,
12218 "access_token_value": "must-not-cross"
12219 }
12220 }
12221 }))
12222 .unwrap(),
12223 )
12224 .unwrap();
12225 assert!(matches!(
12226 next_config_event(&mut feed, "core").await,
12227 AgentEvent::ConfigChanged { revision: 1, .. }
12228 ));
12229 assert_eq!(state.config.read().await.server.port, 25_801);
12230 tokio::time::sleep(Duration::from_millis(500)).await;
12231 let dormant_events =
12232 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
12233 assert!(!dormant_events.iter().any(|event| matches!(
12234 &event.event,
12235 AgentEvent::ConfigRecovered { section, revision }
12236 if section == "mcp" && *revision == 1
12237 )));
12238 assert!(bamboo_config::has_pending_legacy_root_publications(dir.path()).unwrap());
12239
12240 std::fs::write(dir.path().join("config.json"), b"{}").unwrap();
12241 tokio::time::timeout(Duration::from_secs(5), async {
12242 loop {
12243 if !bamboo_config::legacy_root_rejected_sections(dir.path())
12244 .unwrap()
12245 .contains(&SectionId::Mcp)
12246 {
12247 break;
12248 }
12249 tokio::time::sleep(Duration::from_millis(20)).await;
12250 }
12251 })
12252 .await
12253 .unwrap();
12254 let repaired = working_stdio_mcp_config(dir.path(), "root-dormant", None);
12255 assert_eq!(repaired.servers[0].id, "root-dormant");
12256 assert!(matches!(
12257 next_mcp_config_event(&mut feed).await,
12258 AgentEvent::ConfigRecovered { revision: 1, .. }
12259 ));
12260 wait_for_root_outbox_to_clear(dir.path()).await;
12261 }
12262
12263 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
12264 async fn changed_before_ack_then_runtime_failure_recovers_with_exact_kind() {
12265 let _key = bamboo_config::encryption::set_test_encryption_key([0x58; 32]);
12266 let dir = tempfile::tempdir().unwrap();
12267 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
12268 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
12269 stop_config_watcher(&mut state);
12270 let failing = working_stdio_mcp_config(dir.path(), "root-crash-window", None);
12271 let script = match &failing.servers[0].transport {
12272 TransportConfig::Stdio(stdio) => PathBuf::from(&stdio.args[0]),
12273 _ => unreachable!(),
12274 };
12275 std::fs::remove_file(&script).unwrap();
12276 std::fs::write(
12277 dir.path().join("config.json"),
12278 serde_json::to_vec(&serde_json::json!({"mcp": failing})).unwrap(),
12279 )
12280 .unwrap();
12281 let committed = state
12282 .config_facade
12283 .as_ref()
12284 .unwrap()
12285 .reconcile_reappeared_legacy_root()
12286 .unwrap()
12287 .unwrap();
12288 assert_eq!(
12289 committed.committed,
12290 vec![ConfigSectionEvent::Changed {
12291 section: "mcp".to_string(),
12292 revision: 1,
12293 }]
12294 );
12295 assert!(
12296 state
12297 .account_sink
12298 .record_confirmed(None, ®istry_agent_event(&committed.committed[0]))
12299 .await
12300 );
12301 assert!(bamboo_config::has_pending_legacy_root_publications(dir.path()).unwrap());
12302
12303 let events_dir = state.account_sink.events_dir().to_path_buf();
12304 state.account_sink = bamboo_engine::events::AccountEventSink::new(events_dir).unwrap();
12305 tokio::task::yield_now().await;
12306 let mut feed = state.account_sink.subscribe();
12307 restart_config_watcher(&mut state);
12308 assert!(matches!(
12309 next_mcp_config_event(&mut feed).await,
12310 AgentEvent::ConfigInvalid { revision: 1, .. }
12311 ));
12312 let repaired = working_stdio_mcp_config(dir.path(), "root-crash-window", None);
12313 assert_eq!(repaired.servers[0].id, "root-crash-window");
12314 assert!(matches!(
12315 next_mcp_config_event(&mut feed).await,
12316 AgentEvent::ConfigRecovered { revision: 1, .. }
12317 ));
12318 wait_for_root_outbox_to_clear(dir.path()).await;
12319
12320 let events =
12321 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
12322 let transitions = events
12323 .iter()
12324 .filter_map(|event| match &event.event {
12325 AgentEvent::ConfigChanged { section, revision } if section == "mcp" => {
12326 Some(("changed", *revision))
12327 }
12328 AgentEvent::ConfigInvalid { section, revision } if section == "mcp" => {
12329 Some(("invalid", *revision))
12330 }
12331 AgentEvent::ConfigRecovered { section, revision } if section == "mcp" => {
12332 Some(("recovered", *revision))
12333 }
12334 _ => None,
12335 })
12336 .collect::<Vec<_>>();
12337 assert_eq!(
12338 transitions,
12339 vec![("changed", 1), ("invalid", 1), ("recovered", 1)]
12340 );
12341 }
12342
12343 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
12344 async fn invalid_journaled_before_canonical_mark_restarts_as_recovered_only() {
12345 let _key = bamboo_config::encryption::set_test_encryption_key([0x5b; 32]);
12346 let dir = tempfile::tempdir().unwrap();
12347 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
12348 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
12349 stop_config_watcher(&mut state);
12350 let candidate = disabled_mcp_config("root-invalid-mark-crash");
12351 std::fs::write(
12352 dir.path().join("config.json"),
12353 serde_json::to_vec(&serde_json::json!({"mcp": candidate})).unwrap(),
12354 )
12355 .unwrap();
12356 let committed = state
12357 .config_facade
12358 .as_ref()
12359 .unwrap()
12360 .reconcile_reappeared_legacy_root()
12361 .unwrap()
12362 .unwrap();
12363 assert_eq!(
12364 committed.committed,
12365 vec![ConfigSectionEvent::Changed {
12366 section: "mcp".to_string(),
12367 revision: 1,
12368 }]
12369 );
12370 let invalid = ConfigSectionEvent::Invalid {
12371 section: "mcp".to_string(),
12372 revision: 1,
12373 };
12374 assert!(
12375 state
12376 .account_sink
12377 .record_confirmed(None, ®istry_agent_event(&invalid))
12378 .await
12379 );
12380 let envelope = state
12381 .config_facade
12382 .as_ref()
12383 .unwrap()
12384 .registry()
12385 .envelope_value(SectionId::Mcp)
12386 .unwrap();
12387 assert!(matches!(
12388 bamboo_config::legacy_root_publication_success_event(
12389 dir.path(),
12390 &committed.committed[0],
12391 &envelope.data,
12392 )
12393 .unwrap(),
12394 Some(ConfigSectionEvent::Changed { revision: 1, .. })
12395 ));
12396
12397 let events_dir = state.account_sink.events_dir().to_path_buf();
12398 state.account_sink = bamboo_engine::events::AccountEventSink::new(events_dir).unwrap();
12399 assert!(state
12400 .account_sink
12401 .latest_config_transition_is_invalid("mcp", 1));
12402 let mut feed = state.account_sink.subscribe();
12403 restart_config_watcher(&mut state);
12404 assert!(matches!(
12405 next_mcp_config_event(&mut feed).await,
12406 AgentEvent::ConfigRecovered { revision: 1, .. }
12407 ));
12408 wait_for_root_outbox_to_clear(dir.path()).await;
12409
12410 let events =
12411 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
12412 let transitions = events
12413 .iter()
12414 .filter_map(|event| match &event.event {
12415 AgentEvent::ConfigChanged { section, revision } if section == "mcp" => {
12416 Some(("changed", *revision))
12417 }
12418 AgentEvent::ConfigInvalid { section, revision } if section == "mcp" => {
12419 Some(("invalid", *revision))
12420 }
12421 AgentEvent::ConfigRecovered { section, revision } if section == "mcp" => {
12422 Some(("recovered", *revision))
12423 }
12424 _ => None,
12425 })
12426 .collect::<Vec<_>>();
12427 assert_eq!(transitions, vec![("invalid", 1), ("recovered", 1)]);
12428 }
12429
12430 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
12431 async fn startup_lagging_mcp_facade_installs_and_acknowledges_root_publication() {
12432 let _key = bamboo_config::encryption::set_test_encryption_key([0x5a; 32]);
12433 let dir = tempfile::tempdir().unwrap();
12434 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
12435 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
12436 stop_config_watcher(&mut state);
12437 let external = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
12438 let candidate = disabled_mcp_config("startup-lag-root");
12439 std::fs::write(
12440 dir.path().join("config.json"),
12441 serde_json::to_vec(&serde_json::json!({"mcp": candidate})).unwrap(),
12442 )
12443 .unwrap();
12444 let committed = external
12445 .reconcile_reappeared_legacy_root()
12446 .unwrap()
12447 .unwrap();
12448 assert_eq!(
12449 committed.committed,
12450 vec![ConfigSectionEvent::Changed {
12451 section: "mcp".to_string(),
12452 revision: 1,
12453 }]
12454 );
12455 assert_eq!(
12456 state
12457 .config_facade
12458 .as_ref()
12459 .unwrap()
12460 .registry()
12461 .mcp
12462 .snapshot()
12463 .revision,
12464 0
12465 );
12466
12467 let mut feed = state.account_sink.subscribe();
12468 restart_config_watcher(&mut state);
12469 assert!(matches!(
12470 next_mcp_config_event(&mut feed).await,
12471 AgentEvent::ConfigChanged { revision: 1, .. }
12472 ));
12473 wait_for_root_outbox_to_clear(dir.path()).await;
12474 assert!(state
12475 .config
12476 .read()
12477 .await
12478 .mcp
12479 .servers
12480 .iter()
12481 .any(|server| server.id == "startup-lag-root"));
12482 }
12483
12484 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
12485 async fn persistent_root_mcp_runtime_failure_retries_without_invalid_event_storm() {
12486 let _key = bamboo_config::encryption::set_test_encryption_key([0x55; 32]);
12487 let dir = tempfile::tempdir().unwrap();
12488 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
12489 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
12490 let failing = working_stdio_mcp_config(dir.path(), "root-retry", None);
12491 let script = match &failing.servers[0].transport {
12492 TransportConfig::Stdio(stdio) => PathBuf::from(&stdio.args[0]),
12493 _ => unreachable!(),
12494 };
12495 std::fs::remove_file(&script).unwrap();
12496 let mut feed = state.account_sink.subscribe();
12497 std::fs::write(
12498 dir.path().join("config.json"),
12499 serde_json::to_vec(&serde_json::json!({"mcp": failing})).unwrap(),
12500 )
12501 .unwrap();
12502
12503 assert!(matches!(
12504 next_mcp_config_event(&mut feed).await,
12505 AgentEvent::ConfigInvalid { revision: 1, .. }
12506 ));
12507 tokio::time::sleep(Duration::from_secs(4)).await;
12508 let failed_events =
12509 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
12510 assert_eq!(
12511 failed_events
12512 .iter()
12513 .filter(|event| matches!(
12514 &event.event,
12515 AgentEvent::ConfigInvalid { section, .. } if section == "mcp"
12516 ))
12517 .count(),
12518 1
12519 );
12520 assert!(bamboo_config::has_pending_legacy_root_publications(dir.path()).unwrap());
12521
12522 let mut repaired = working_stdio_mcp_config(dir.path(), "root-retry-fixed", None);
12523 repaired.servers[0].id = "root-retry".to_string();
12524 std::fs::write(
12525 dir.path().join("config.json"),
12526 serde_json::to_vec(&serde_json::json!({"mcp": repaired})).unwrap(),
12527 )
12528 .unwrap();
12529 let repaired_root_event = next_mcp_config_event(&mut feed).await;
12530 let repaired_health = state
12531 .mcp_config_live_health
12532 .read()
12533 .unwrap_or_else(|poisoned| poisoned.into_inner())
12534 .clone();
12535 let repaired_snapshot = state
12536 .config_facade
12537 .as_ref()
12538 .unwrap()
12539 .registry()
12540 .mcp
12541 .snapshot();
12542 assert!(
12543 matches!(
12544 repaired_root_event,
12545 AgentEvent::ConfigChanged { revision: 2, .. }
12546 ),
12547 "unexpected repaired root event: {repaired_root_event:?}; health: {repaired_health:?}; typed: {:?}",
12548 repaired_snapshot.data
12549 );
12550 wait_for_root_outbox_to_clear(dir.path()).await;
12551 assert!(state.config.read().await.mcp.servers.iter().any(|server| {
12552 server.id == "root-retry"
12553 && matches!(
12554 &server.transport,
12555 TransportConfig::Stdio(stdio)
12556 if stdio.args.iter().any(|arg| arg.contains("root-retry-fixed"))
12557 )
12558 }));
12559 let events =
12560 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
12561 assert_eq!(
12562 events
12563 .iter()
12564 .filter(|event| matches!(
12565 &event.event,
12566 AgentEvent::ConfigInvalid { section, .. } if section == "mcp"
12567 ))
12568 .count(),
12569 1
12570 );
12571 assert_eq!(
12572 events
12573 .iter()
12574 .filter(|event| matches!(
12575 &event.event,
12576 AgentEvent::ConfigChanged { section, revision }
12577 if section == "mcp" && *revision == 2
12578 ))
12579 .count(),
12580 1
12581 );
12582 assert_eq!(
12583 events
12584 .iter()
12585 .filter(|event| matches!(
12586 &event.event,
12587 AgentEvent::ConfigChanged { section, revision }
12588 if section == "mcp" && *revision == 1
12589 ))
12590 .count(),
12591 0
12592 );
12593 }
12594}