1use super::*;
2
3use std::collections::{BTreeMap, BTreeSet, HashSet};
4use std::path::{Path, PathBuf};
5use std::sync::atomic::{AtomicBool, Ordering};
6use std::time::Duration;
7
8use bamboo_config::{
9 ensure_provider_mcp_migration_ready, AtomicJsonStore, ConfigDirectoryWatcher,
10 ConfigSectionEvent, ConfigStoreError, McpSection, ProviderConfigs, SectionId,
11 SectionSourceKind, SectionStatus,
12};
13use bamboo_mcp::{McpConfig, McpServerManager, TransportConfig};
14use chrono::{DateTime, Utc};
15use serde::Serialize;
16use serde_json::Value;
17
18#[cfg(test)]
19struct InitialMcpApplyTestHook {
20 before: Box<dyn FnOnce() + Send + 'static>,
21 after: Box<dyn FnOnce() + Send + 'static>,
22}
23
24#[cfg(test)]
25fn initial_mcp_apply_test_hooks(
26) -> &'static std::sync::Mutex<std::collections::HashMap<PathBuf, InitialMcpApplyTestHook>> {
27 static HOOKS: std::sync::OnceLock<
28 std::sync::Mutex<std::collections::HashMap<PathBuf, InitialMcpApplyTestHook>>,
29 > = std::sync::OnceLock::new();
30 HOOKS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
31}
32
33#[cfg(test)]
34fn set_initial_mcp_apply_test_hook(
35 data_dir: &Path,
36 before: impl FnOnce() + Send + 'static,
37 after: impl FnOnce() + Send + 'static,
38) {
39 initial_mcp_apply_test_hooks()
40 .lock()
41 .unwrap_or_else(|poisoned| poisoned.into_inner())
42 .insert(
43 data_dir.to_path_buf(),
44 InitialMcpApplyTestHook {
45 before: Box::new(before),
46 after: Box::new(after),
47 },
48 );
49}
50
51#[cfg(test)]
52struct InitialMcpApplyTestCompletion(Option<Box<dyn FnOnce() + Send + 'static>>);
53
54#[cfg(test)]
55impl Drop for InitialMcpApplyTestCompletion {
56 fn drop(&mut self) {
57 if let Some(after) = self.0.take() {
58 after();
59 }
60 }
61}
62
63#[cfg(test)]
64fn begin_initial_mcp_apply_test_hook(data_dir: &Path) -> InitialMcpApplyTestCompletion {
65 let hook = initial_mcp_apply_test_hooks()
66 .lock()
67 .unwrap_or_else(|poisoned| poisoned.into_inner())
68 .remove(data_dir);
69 let Some(hook) = hook else {
70 return InitialMcpApplyTestCompletion(None);
71 };
72 (hook.before)();
73 InitialMcpApplyTestCompletion(Some(hook.after))
74}
75
76#[cfg(test)]
77struct ClusterAfterCommitBeforeAdoptionTestHook {
78 expected_revision: u64,
79 hook: Box<dyn FnOnce(&Path) + Send + 'static>,
80}
81
82#[cfg(test)]
83fn cluster_after_commit_before_adoption_test_hooks() -> &'static std::sync::Mutex<
84 std::collections::HashMap<PathBuf, ClusterAfterCommitBeforeAdoptionTestHook>,
85> {
86 static HOOKS: std::sync::OnceLock<
87 std::sync::Mutex<
88 std::collections::HashMap<PathBuf, ClusterAfterCommitBeforeAdoptionTestHook>,
89 >,
90 > = std::sync::OnceLock::new();
91 HOOKS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
92}
93
94#[cfg(test)]
95fn set_cluster_after_commit_before_adoption_test_hook(
96 data_dir: &Path,
97 expected_revision: u64,
98 hook: impl FnOnce(&Path) + Send + 'static,
99) {
100 cluster_after_commit_before_adoption_test_hooks()
101 .lock()
102 .unwrap_or_else(|poisoned| poisoned.into_inner())
103 .insert(
104 data_dir.to_path_buf(),
105 ClusterAfterCommitBeforeAdoptionTestHook {
106 expected_revision,
107 hook: Box::new(hook),
108 },
109 );
110}
111
112#[cfg(test)]
113fn run_cluster_after_commit_before_adoption_test_hook(data_dir: &Path, expected_revision: u64) {
114 let hook = {
115 let mut hooks = cluster_after_commit_before_adoption_test_hooks()
116 .lock()
117 .unwrap_or_else(|poisoned| poisoned.into_inner());
118 if hooks
119 .get(data_dir)
120 .is_some_and(|hook| hook.expected_revision == expected_revision)
121 {
122 hooks.remove(data_dir)
123 } else {
124 None
125 }
126 };
127 if let Some(hook) = hook {
128 (hook.hook)(data_dir);
129 }
130}
131
132#[cfg(test)]
133type CredentialCommitTestHook = Box<dyn FnOnce() + Send + 'static>;
134#[cfg(test)]
135type CredentialCommitTestHooks =
136 std::sync::Mutex<std::collections::HashMap<(PathBuf, SectionId), CredentialCommitTestHook>>;
137
138#[cfg(test)]
139fn credential_after_commit_before_live_test_hooks() -> &'static CredentialCommitTestHooks {
140 static HOOKS: std::sync::OnceLock<CredentialCommitTestHooks> = std::sync::OnceLock::new();
141 HOOKS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
142}
143
144#[cfg(test)]
145fn set_credential_after_commit_before_live_test_hook(
146 data_dir: &Path,
147 section: SectionId,
148 hook: impl FnOnce() + Send + 'static,
149) {
150 credential_after_commit_before_live_test_hooks()
151 .lock()
152 .unwrap_or_else(|poisoned| poisoned.into_inner())
153 .insert((data_dir.to_path_buf(), section), Box::new(hook));
154}
155
156#[cfg(test)]
157fn run_credential_after_commit_before_live_test_hook(data_dir: &Path, section: SectionId) {
158 let hook = credential_after_commit_before_live_test_hooks()
159 .lock()
160 .unwrap_or_else(|poisoned| poisoned.into_inner())
161 .remove(&(data_dir.to_path_buf(), section));
162 if let Some(hook) = hook {
163 hook();
164 }
165}
166
167#[cfg(test)]
168type GenericBeforeEventTestHook = Box<dyn FnOnce() + Send + 'static>;
169#[cfg(test)]
170type GenericBeforeEventTestHooks =
171 std::sync::Mutex<std::collections::HashMap<PathBuf, GenericBeforeEventTestHook>>;
172
173#[cfg(test)]
174fn generic_before_event_test_hooks() -> &'static GenericBeforeEventTestHooks {
175 static HOOKS: std::sync::OnceLock<GenericBeforeEventTestHooks> = std::sync::OnceLock::new();
176 HOOKS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
177}
178
179#[cfg(test)]
180fn set_generic_before_event_test_hook(data_dir: &Path, hook: impl FnOnce() + Send + 'static) {
181 generic_before_event_test_hooks()
182 .lock()
183 .unwrap_or_else(|poisoned| poisoned.into_inner())
184 .insert(data_dir.to_path_buf(), Box::new(hook));
185}
186
187#[cfg(test)]
188fn run_generic_before_event_test_hook(data_dir: &Path) {
189 let hook = generic_before_event_test_hooks()
190 .lock()
191 .unwrap_or_else(|poisoned| poisoned.into_inner())
192 .remove(data_dir);
193 if let Some(hook) = hook {
194 hook();
195 }
196}
197
198#[cfg(test)]
199type GenericBeforeProviderPublishTestHook = Box<dyn FnOnce() + Send + 'static>;
200#[cfg(test)]
201type GenericBeforeProviderPublishTestHooks =
202 std::sync::Mutex<std::collections::HashMap<PathBuf, GenericBeforeProviderPublishTestHook>>;
203
204#[cfg(test)]
205fn generic_before_provider_publish_test_hooks() -> &'static GenericBeforeProviderPublishTestHooks {
206 static HOOKS: std::sync::OnceLock<GenericBeforeProviderPublishTestHooks> =
207 std::sync::OnceLock::new();
208 HOOKS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
209}
210
211#[cfg(test)]
212fn set_generic_before_provider_publish_test_hook(
213 data_dir: &Path,
214 hook: impl FnOnce() + Send + 'static,
215) {
216 generic_before_provider_publish_test_hooks()
217 .lock()
218 .unwrap_or_else(|poisoned| poisoned.into_inner())
219 .insert(data_dir.to_path_buf(), Box::new(hook));
220}
221
222#[cfg(test)]
223fn run_generic_before_provider_publish_test_hook(data_dir: &Path) {
224 let hook = generic_before_provider_publish_test_hooks()
225 .lock()
226 .unwrap_or_else(|poisoned| poisoned.into_inner())
227 .remove(data_dir);
228 if let Some(hook) = hook {
229 hook();
230 }
231}
232
233#[cfg(test)]
234type ResetAfterDeleteTestHook = Box<dyn FnOnce() + Send + 'static>;
235#[cfg(test)]
236type ResetAfterDeleteTestHooks =
237 std::sync::Mutex<std::collections::HashMap<PathBuf, ResetAfterDeleteTestHook>>;
238
239#[cfg(test)]
240fn reset_after_delete_test_hooks() -> &'static ResetAfterDeleteTestHooks {
241 static HOOKS: std::sync::OnceLock<ResetAfterDeleteTestHooks> = std::sync::OnceLock::new();
242 HOOKS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
243}
244
245#[cfg(test)]
246fn set_reset_after_delete_test_hook(data_dir: &Path, hook: impl FnOnce() + Send + 'static) {
247 reset_after_delete_test_hooks()
248 .lock()
249 .unwrap_or_else(|poisoned| poisoned.into_inner())
250 .insert(data_dir.to_path_buf(), Box::new(hook));
251}
252
253#[cfg(test)]
254fn run_reset_after_delete_test_hook(data_dir: &Path) {
255 let hook = reset_after_delete_test_hooks()
256 .lock()
257 .unwrap_or_else(|poisoned| poisoned.into_inner())
258 .remove(data_dir);
259 if let Some(hook) = hook {
260 hook();
261 }
262}
263
264struct FacadeRuntimeMaterialization {
265 config: Config,
266 failures: BTreeSet<SectionId>,
267}
268
269fn materialize_facade_effective_config(
270 facade: &bamboo_config::ConfigFacade,
271 data_dir: &Path,
272) -> FacadeRuntimeMaterialization {
273 let mut config = facade.effective_config();
274 let mut failures = BTreeSet::new();
275 if let Err(error) = config.hydrate_proxy_auth_from_store(data_dir) {
276 tracing::warn!(error = %error, "proxy auth credential hydration unavailable");
277 config.proxy_auth = None;
278 failures.insert(SectionId::Core);
279 }
280 if let Err(error) = config.hydrate_provider_credentials_from_store(data_dir) {
281 tracing::warn!(error = %error, "provider credential hydration unavailable");
282 failures.insert(SectionId::Providers);
283 }
284 if let Err(error) = config.hydrate_mcp_credentials_from_store(data_dir) {
285 tracing::warn!(error = %error, "MCP credential hydration unavailable");
286 failures.insert(SectionId::Mcp);
287 }
288 if let Err(error) = config.hydrate_env_var_credentials_from_store(data_dir) {
289 tracing::warn!(error = %error, "env credential hydration unavailable");
290 for entry in &mut config.env_vars {
291 if entry.secret {
292 entry.value.clear();
293 }
294 }
295 failures.insert(SectionId::Env);
296 }
297 if let Err(error) = config.hydrate_cluster_credentials_from_store(data_dir) {
298 tracing::warn!(error = %error, "cluster credential hydration unavailable");
299 failures.insert(SectionId::ClusterFabric);
300 }
301 if let Err(error) = config.hydrate_notification_credentials_from_store(data_dir) {
302 tracing::warn!(error = %error, "notification credential hydration unavailable");
303 config.notifications.ntfy.token = None;
304 config.notifications.bark.device_key = None;
305 failures.insert(SectionId::Notifications);
306 }
307 if let Err(error) = config.hydrate_connect_credentials_from_store(data_dir) {
308 tracing::warn!(error = %error, "connect credential hydration unavailable");
309 for platform in &mut config.connect.platforms {
310 platform.token = None;
311 platform.app_secret = None;
312 }
313 failures.insert(SectionId::Connect);
314 }
315 if let Err(error) = config.hydrate_access_control_credentials_from_store(data_dir) {
316 tracing::warn!(error = %error, "access-control credential hydration unavailable");
317 config.clear_access_control_runtime_verifiers();
318 failures.insert(SectionId::AccessControl);
319 }
320 if config
321 .access_control
322 .as_ref()
323 .is_some_and(|access| access.repair_required)
324 {
325 failures.insert(SectionId::AccessControl);
326 }
327 if let Some(broker) = config.subagents_mut().broker.as_mut() {
328 if let Err(error) = broker.hydrate_credential_from_store(data_dir) {
329 tracing::warn!(error = %error, "external broker credential hydration unavailable");
330 broker.token.clear();
331 failures.insert(SectionId::Subagents);
332 }
333 }
334 config.apply_runtime_env_overrides();
335 FacadeRuntimeMaterialization { config, failures }
336}
337
338pub(super) fn load_facade_effective_config(
339 facade: &bamboo_config::ConfigFacade,
340 data_dir: &Path,
341) -> Config {
342 let materialized = materialize_facade_effective_config(facade, data_dir);
343 for section in &materialized.failures {
344 facade.registry().mark_runtime_degraded(
345 *section,
346 "configuration runtime credential repair is required",
347 );
348 }
349 materialized.config
350}
351
352fn load_committed_effective_config(data_dir: &Path) -> Result<Config, ConfigStoreError> {
353 if bamboo_config::modular_authority_boundary_present(data_dir)? {
354 let facade = bamboo_config::ConfigFacade::open_or_migrate(data_dir)?;
355 let config = load_facade_effective_config(&facade, data_dir);
356 Ok(config)
357 } else {
358 Ok(Config::from_data_dir_without_publish(Some(
359 data_dir.to_path_buf(),
360 )))
361 }
362}
363
364#[derive(Debug, Clone, Serialize)]
366pub struct ConfigLiveHealth {
367 pub revision: u64,
368 pub loaded_at: DateTime<Utc>,
369 pub source_path: PathBuf,
370 pub source_kind: SectionSourceKind,
371 pub status: SectionStatus,
372 pub last_error: Option<String>,
373}
374
375pub struct ConfigWatcherRuntime {
377 stop: Arc<AtomicBool>,
378 watcher_task: Option<std::thread::JoinHandle<()>>,
379 apply_task: Option<tokio::task::JoinHandle<()>>,
380}
381
382struct ConfigPathChanges {
383 paths: Vec<PathBuf>,
384 initial_mcp_revision: Option<u64>,
385 startup_legacy_root: Option<bamboo_config::LegacyRootReconciliationOutcome>,
386 startup_recoveries: BTreeMap<SectionId, u64>,
387 legacy_root_retry_attempt: u8,
388}
389
390struct ConfigRuntimeEffectContext {
394 app_data_dir: PathBuf,
395 config_facade: Option<Arc<bamboo_config::ConfigFacade>>,
396 provider_registry: Arc<bamboo_llm::ProviderRegistry>,
397 provider: Arc<RwLock<Arc<dyn LLMProvider>>>,
398 mcp_manager: Arc<McpServerManager>,
399 account_sink: Arc<bamboo_engine::events::AccountEventSink>,
400 config_live_health: Arc<std::sync::RwLock<ConfigLiveHealth>>,
401 mcp_config_live_health: Arc<std::sync::RwLock<ConfigLiveHealth>>,
402}
403
404impl ConfigWatcherRuntime {
405 #[allow(clippy::too_many_arguments)]
406 pub fn start(
407 data_dir: PathBuf,
408 config: Arc<RwLock<Config>>,
409 config_facade: Option<Arc<bamboo_config::ConfigFacade>>,
410 config_io_lock: Arc<tokio::sync::Mutex<()>>,
411 provider_registry: Arc<bamboo_llm::ProviderRegistry>,
412 provider: Arc<RwLock<Arc<dyn LLMProvider>>>,
413 mcp_manager: Arc<McpServerManager>,
414 account_sink: Arc<bamboo_engine::events::AccountEventSink>,
415 ) -> (
416 Self,
417 Arc<std::sync::RwLock<ConfigLiveHealth>>,
418 Arc<std::sync::RwLock<ConfigLiveHealth>>,
419 ) {
420 let provider_store = AtomicJsonStore::new(data_dir.join("providers.json"), 1);
421 let provider_health = Arc::new(std::sync::RwLock::new(initial_provider_health(
422 &provider_store,
423 )));
424 let mcp_store = AtomicJsonStore::new(data_dir.join("mcp.json"), 1);
425 let mcp_health = Arc::new(std::sync::RwLock::new(initial_mcp_health(&mcp_store)));
426 let stop = Arc::new(AtomicBool::new(false));
427 let watcher = match ConfigDirectoryWatcher::watch(&data_dir, Duration::from_millis(120)) {
428 Ok(watcher) => watcher,
429 Err(error) => {
430 tracing::warn!(error = %error, "live configuration watcher could not start");
431 {
432 let mut value = provider_health
433 .write()
434 .unwrap_or_else(|poisoned| poisoned.into_inner());
435 value.status = SectionStatus::Degraded;
436 value.last_error = Some("configuration watcher unavailable".to_string());
437 }
438 {
439 let mut value = mcp_health
440 .write()
441 .unwrap_or_else(|poisoned| poisoned.into_inner());
442 value.status = SectionStatus::Degraded;
443 value.last_error = Some("configuration watcher unavailable".to_string());
444 }
445 return (
446 Self {
447 stop,
448 watcher_task: None,
449 apply_task: None,
450 },
451 provider_health,
452 mcp_health,
453 );
454 }
455 };
456
457 let self_write_marker = watcher.self_write_marker();
461 let startup_legacy_root = config_facade
462 .as_ref()
463 .and_then(|facade| facade.take_startup_legacy_root_reconciliation());
464 let (changes_tx, mut changes_rx) =
465 tokio::sync::mpsc::unbounded_channel::<ConfigPathChanges>();
466 let initial_changes = changes_tx.clone();
467 let worker_stop = stop.clone();
468 let watcher_task = std::thread::spawn(move || {
469 while !worker_stop.load(Ordering::Relaxed) {
470 match watcher.recv_timeout(Duration::from_millis(250)) {
471 Ok(paths) => {
472 if changes_tx
473 .send(ConfigPathChanges {
474 paths,
475 initial_mcp_revision: None,
476 startup_legacy_root: None,
477 startup_recoveries: BTreeMap::new(),
478 legacy_root_retry_attempt: 0,
479 })
480 .is_err()
481 {
482 break;
483 }
484 }
485 Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {}
486 Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break,
487 }
488 }
489 });
490
491 let initial_mcp_path = data_dir.join("mcp.json");
496 let mut initial_paths = Vec::new();
497 let initial_mcp_revision = if initial_mcp_path.exists() {
498 initial_paths.push(initial_mcp_path);
499 Some(
500 config_facade
501 .as_ref()
502 .map(|facade| facade.registry().mcp.snapshot().revision)
503 .unwrap_or_else(|| {
504 mcp_health
505 .read()
506 .unwrap_or_else(|poisoned| poisoned.into_inner())
507 .revision
508 }),
509 )
510 } else {
511 None
512 };
513 let rejected = bamboo_config::legacy_root_rejected_sections(&data_dir);
514 let startup_recoveries = match (config_facade.as_ref(), rejected.as_ref()) {
515 (Some(facade), Ok(rejected)) => {
516 durable_invalid_recoveries(&account_sink, facade, rejected)
517 }
518 _ => BTreeMap::new(),
519 };
520 if let (Some(facade), Ok(rejected)) = (config_facade.as_ref(), rejected.as_ref()) {
521 if let Ok(health) = facade.registry().health() {
522 initial_paths.extend(
523 health
524 .into_iter()
525 .filter(|health| {
526 health.status != SectionStatus::Healthy
527 && !rejected.contains(&health.section)
528 })
529 .map(|health| data_dir.join(health.section.descriptor().file_name)),
530 );
531 }
532 }
533 initial_paths.extend(
534 startup_recoveries
535 .keys()
536 .map(|id| data_dir.join(id.descriptor().file_name)),
537 );
538 let legacy_root_needs_reconciliation = config_facade.is_some();
543 if legacy_root_needs_reconciliation {
544 initial_paths.push(data_dir.join("config.json"));
545 }
546 if !initial_paths.is_empty() {
547 let _ = initial_changes.send(ConfigPathChanges {
548 paths: initial_paths,
549 initial_mcp_revision,
550 startup_legacy_root,
551 startup_recoveries,
552 legacy_root_retry_attempt: 0,
553 });
554 }
555
556 let apply_provider_health = provider_health.clone();
557 let apply_mcp_health = mcp_health.clone();
558 let catchup_changes = initial_changes.clone();
559 let apply_task = tokio::spawn(async move {
560 let mut reported_root_runtime_failures = BTreeSet::<(SectionId, u64)>::new();
561 while let Some(mut changes) = changes_rx.recv().await {
562 let mut watched_sections = config_facade
563 .as_ref()
564 .map(|_| {
565 changes
566 .paths
567 .iter()
568 .filter_map(|path| {
569 path.file_name()
570 .and_then(|name| name.to_str())
571 .and_then(SectionId::from_file_name)
572 })
573 .collect::<BTreeSet<_>>()
574 })
575 .unwrap_or_default();
576 let direct_watched_sections = watched_sections.clone();
577 let legacy_root_watched = config_facade.is_some()
578 && changes.paths.iter().any(|path| {
579 matches!(
580 path.file_name().and_then(|name| name.to_str()),
581 Some(
582 "config.json"
583 | "config.json.bak"
584 | "config.json.bak.1"
585 | "config.json.bak.2"
586 )
587 )
588 });
589 let mut provider_watched = changes.paths.iter().any(|path| {
590 path.file_name().and_then(|name| name.to_str()) == Some("providers.json")
591 });
592 let mut mcp_watched = changes.paths.iter().any(|path| {
593 path.file_name().and_then(|name| name.to_str()) == Some("mcp.json")
594 });
595 if !provider_watched
596 && !mcp_watched
597 && watched_sections.is_empty()
598 && !legacy_root_watched
599 {
600 continue;
601 }
602
603 #[cfg(test)]
604 let _initial_mcp_apply_completion = changes
605 .initial_mcp_revision
606 .map(|_| begin_initial_mcp_apply_test_hook(&data_dir));
607
608 let _io = config_io_lock.lock().await;
612 let mut synthetic_root_events = BTreeMap::<SectionId, ConfigSectionEvent>::new();
613 let mut pending_root_publications =
614 BTreeMap::<SectionId, ConfigSectionEvent>::new();
615 let startup_root_batch = changes.startup_legacy_root.is_some();
616 let mut requeue_legacy_root = false;
617 let mut retry_legacy_root_publication = false;
618 let mut canonical_root_rejections = BTreeSet::new();
619 if legacy_root_watched {
620 let reconciliation = match changes.startup_legacy_root.take() {
621 Some(outcome) => Ok(Some(outcome)),
622 None => {
623 let reconcile_facade = config_facade
624 .as_ref()
625 .expect("legacy root watching requires a facade")
626 .clone();
627 tokio::task::spawn_blocking(move || {
628 reconcile_facade.reconcile_reappeared_legacy_root()
629 })
630 .await
631 .map_err(|_| {
632 ConfigStoreError::Validation(
633 "legacy root reconciliation task failed".to_string(),
634 )
635 })
636 .and_then(|result| result)
637 }
638 };
639 match reconciliation {
640 Ok(Some(outcome)) if !outcome.duplicate => {
641 requeue_legacy_root = outcome.partial || startup_root_batch;
642 for event in &outcome.committed {
643 let section = match event {
644 ConfigSectionEvent::Changed { section, .. }
645 | ConfigSectionEvent::Invalid { section, .. }
646 | ConfigSectionEvent::Recovered { section, .. } => section,
647 };
648 if let Some(id) = SectionId::from_name(section) {
649 watched_sections.insert(id);
650 synthetic_root_events.insert(id, event.clone());
651 if matches!(
652 event,
653 ConfigSectionEvent::Changed { .. }
654 | ConfigSectionEvent::Recovered { .. }
655 ) {
656 pending_root_publications.insert(id, event.clone());
657 }
658 }
659 }
660 if let Some(facade) = config_facade.as_ref() {
661 for id in &outcome.recovered {
662 queue_legacy_root_recovery(
663 facade,
664 *id,
665 None,
666 &mut watched_sections,
667 &mut synthetic_root_events,
668 );
669 }
670 for rejection in outcome.rejected {
671 canonical_root_rejections.insert(rejection.section);
672 if rejection.reason
673 == bamboo_config::LegacyRootRejectionReason::RevisionConflict
674 {
675 watched_sections.insert(rejection.section);
676 }
677 if let Some(event) = legacy_root_rejection_event(
678 facade,
679 rejection.section,
680 rejection.reason.diagnostic(),
681 startup_root_batch,
682 ) {
683 publish_registry_event(&account_sink, &event).await;
684 }
685 }
686 }
687 if outcome.partial {
688 tracing::warn!(
689 "legacy config root changed during reconciliation; awaiting the newer generation"
690 );
691 }
692 }
693 Ok(Some(outcome)) => {
694 requeue_legacy_root = outcome.partial || startup_root_batch;
695 if let Some(facade) = config_facade.as_ref() {
702 for id in &outcome.recovered {
703 queue_legacy_root_recovery(
704 facade,
705 *id,
706 None,
707 &mut watched_sections,
708 &mut synthetic_root_events,
709 );
710 }
711 for event in &outcome.committed {
712 let (section, revision) = match event {
713 ConfigSectionEvent::Changed { section, revision }
714 | ConfigSectionEvent::Recovered { section, revision } => {
715 (section, *revision)
716 }
717 ConfigSectionEvent::Invalid { .. } => continue,
718 };
719 let Some(id) = SectionId::from_name(section) else {
720 continue;
721 };
722 watched_sections.insert(id);
723 pending_root_publications.insert(id, event.clone());
724 if facade.registry().envelope_value(id).is_ok_and(|envelope| {
725 envelope.revision == revision
726 && envelope.status == SectionStatus::Healthy
727 }) {
728 synthetic_root_events.insert(id, event.clone());
729 }
730 }
731 for rejection in outcome.rejected {
732 canonical_root_rejections.insert(rejection.section);
733 if rejection.reason
734 == bamboo_config::LegacyRootRejectionReason::RevisionConflict
735 {
736 watched_sections.insert(rejection.section);
737 }
738 if let Some(event) = legacy_root_rejection_event(
739 facade,
740 rejection.section,
741 rejection.reason.diagnostic(),
742 startup_root_batch,
743 ) {
744 publish_registry_event(&account_sink, &event).await;
745 }
746 }
747 }
748 }
749 Ok(None) => {
750 if bamboo_config::modular_authority_boundary_present(&data_dir)
751 .unwrap_or(false)
752 {
753 if let Some(facade) = config_facade.as_ref() {
754 if let Some(event) = facade.registry().mark_runtime_degraded(
755 SectionId::Core,
756 "completed modular configuration reconciliation is unavailable",
757 ) {
758 publish_registry_event(&account_sink, &event).await;
759 }
760 }
761 }
762 }
763 Err(_) => {
764 if let Some(facade) = config_facade.as_ref() {
765 if let Some(event) = facade.registry().mark_runtime_degraded(
766 SectionId::Core,
767 "legacy config root reconciliation is unavailable",
768 ) {
769 publish_registry_event(&account_sink, &event).await;
770 }
771 }
772 }
773 }
774 if let Some(facade) = config_facade.as_ref() {
775 for (id, event) in pending_root_publications.clone() {
776 let revision = config_section_event_revision(&event);
777 if !pending_root_publication_matches_fresh_durable(
778 &data_dir, facade, id, revision,
779 ) {
780 pending_root_publications.remove(&id);
787 synthetic_root_events.remove(&id);
788 watched_sections.insert(id);
789 requeue_legacy_root = true;
790 }
791 }
792 }
793 provider_watched |= watched_sections.contains(&SectionId::Providers);
794 mcp_watched |= watched_sections.contains(&SectionId::Mcp);
795 }
796 if let Some(facade) = config_facade.as_ref() {
797 for (id, revision) in std::mem::take(&mut changes.startup_recoveries) {
798 if !canonical_root_rejections.contains(&id) {
799 queue_legacy_root_recovery(
800 facade,
801 id,
802 Some(revision),
803 &mut watched_sections,
804 &mut synthetic_root_events,
805 );
806 }
807 }
808 provider_watched |= watched_sections.contains(&SectionId::Providers);
809 mcp_watched |= watched_sections.contains(&SectionId::Mcp);
810 }
811 let catchup_paths = synthetic_root_events
817 .keys()
818 .filter(|id| startup_root_batch || direct_watched_sections.contains(id))
819 .map(|id| data_dir.join(id.descriptor().file_name))
820 .collect::<Vec<_>>();
821 let ordinary_watched = watched_sections
822 .iter()
823 .copied()
824 .filter(|id| !matches!(id, SectionId::Providers | SectionId::Mcp))
825 .collect::<Vec<_>>();
826 if let Some(facade) = config_facade.as_ref() {
827 retry_legacy_root_publication |= reload_and_apply_ordinary_sections(
828 &data_dir,
829 &config,
830 facade,
831 &account_sink,
832 ordinary_watched,
833 OrdinarySectionReloadState {
834 synthetic_events: &mut synthetic_root_events,
835 pending_root_publications: &mut pending_root_publications,
836 reported_root_runtime_failures: &mut reported_root_runtime_failures,
837 },
838 )
839 .await;
840 }
841 if provider_watched {
842 if let Some(facade) = config_facade.as_ref() {
843 wait_for_section_file_settle(&data_dir, SectionId::Providers).await;
844 if let Some(observed) = synthetic_root_events
845 .remove(&SectionId::Providers)
846 .or_else(|| facade.registry().reload_if_changed(SectionId::Providers))
847 {
848 if let Some(event) = pending_root_publication_event(
849 &data_dir,
850 facade,
851 &account_sink,
852 &pending_root_publications,
853 SectionId::Providers,
854 observed,
855 ) {
856 if matches!(event, ConfigSectionEvent::Invalid { .. }) {
857 publish_section_failure(
858 &apply_provider_health,
859 &account_sink,
860 "providers",
861 facade.registry().providers.snapshot().status,
862 "provider section is invalid; retaining last-known-good runtime"
863 .to_string(),
864 )
865 .await;
866 } else {
867 self_write_marker.mark_self_write(provider_store.path());
868 let materialized =
869 materialize_facade_effective_config(facade, &data_dir);
870 if materialized.failures.contains(&SectionId::Providers) {
871 retry_legacy_root_publication |=
872 publish_staged_facade_section_failure(
873 &data_dir,
874 facade,
875 SectionId::Providers,
876 "provider credential hydration failed; retaining last-known-good runtime",
877 StagedFacadeSectionFailureContext {
878 health: &apply_provider_health,
879 account_sink: &account_sink,
880 section: "providers",
881 pending_root_publications: &pending_root_publications,
882 },
883 )
884 .await;
885 } else {
886 let mut candidate = config.read().await.clone();
887 apply_runtime_section(
888 SectionId::Providers,
889 &materialized.config,
890 &mut candidate,
891 );
892 match prepare_provider_candidate(candidate, &data_dir).await
893 {
894 Ok((candidate, registry, next_provider)) => {
895 let mut live_config = config.write().await;
896 let mut live_provider = provider.write().await;
897 let recovered =
898 section_is_unhealthy(&apply_provider_health);
899 candidate.publish_env_vars();
900 *live_config = candidate;
901 provider_registry.replace_with(registry);
902 *live_provider = next_provider;
903 drop(live_provider);
904 drop(live_config);
905 let revision =
906 facade.registry().providers.snapshot().revision;
907 if pending_root_publications
908 .get(&SectionId::Providers)
909 .is_some_and(|event| {
910 config_section_event_revision(event)
911 == revision
912 })
913 {
914 set_live_health_revision(
915 &apply_provider_health,
916 revision,
917 Some((
918 data_dir.join("providers.json"),
919 SectionSourceKind::File,
920 )),
921 );
922 retry_legacy_root_publication |=
923 publish_registry_event_with_root_ack(
924 &data_dir,
925 &account_sink,
926 &mut pending_root_publications,
927 SectionId::Providers,
928 &event,
929 )
930 .await;
931 } else if matches!(
932 event,
933 ConfigSectionEvent::Recovered { .. }
934 ) {
935 set_live_health_revision(
936 &apply_provider_health,
937 revision,
938 Some((
939 data_dir.join("providers.json"),
940 SectionSourceKind::File,
941 )),
942 );
943 publish_registry_event(&account_sink, &event)
944 .await;
945 } else {
946 publish_section_success(
947 &apply_provider_health,
948 &account_sink,
949 "providers",
950 data_dir.join("providers.json"),
951 recovered,
952 Some(revision),
953 )
954 .await;
955 }
956 }
957 Err(_) => {
958 retry_legacy_root_publication |=
959 publish_staged_facade_section_failure(
960 &data_dir,
961 facade,
962 SectionId::Providers,
963 "provider runtime initialization failed; retaining last-known-good runtime",
964 StagedFacadeSectionFailureContext {
965 health: &apply_provider_health,
966 account_sink: &account_sink,
967 section: "providers",
968 pending_root_publications: &pending_root_publications,
969 },
970 )
971 .await;
972 }
973 }
974 }
975 }
976 } else {
977 retry_legacy_root_publication = true;
978 }
979 }
980 } else {
981 let current_config = config.read().await.clone();
982 let current_revision = apply_provider_health
983 .read()
984 .unwrap_or_else(|poisoned| poisoned.into_inner())
985 .revision;
986 let result = load_and_prepare_provider_candidate(
987 &provider_store,
988 current_revision,
989 current_config,
990 )
991 .await;
992 match result {
993 Ok(candidate) if candidate.unchanged => {}
994 Ok(candidate) => {
995 if candidate.normalized_external_revision {
996 self_write_marker.mark_self_write(provider_store.path());
997 }
998 let mut live_config = config.write().await;
999 let mut live_provider = provider.write().await;
1000 let recovered = section_is_unhealthy(&apply_provider_health);
1001 candidate.config.publish_env_vars();
1002 *live_config = candidate.config;
1003 provider_registry.replace_with(candidate.registry);
1004 *live_provider = candidate.provider;
1005 drop(live_provider);
1006 drop(live_config);
1007
1008 publish_section_success(
1009 &apply_provider_health,
1010 &account_sink,
1011 "providers",
1012 data_dir.join("providers.json"),
1013 recovered,
1014 Some(candidate.revision),
1015 )
1016 .await;
1017 }
1018 Err(error) => {
1019 publish_section_failure(
1020 &apply_provider_health,
1021 &account_sink,
1022 "providers",
1023 candidate_error_status(&error.kind),
1024 error.message,
1025 )
1026 .await
1027 }
1028 }
1029 }
1030 }
1031
1032 if mcp_watched {
1033 if let Some(facade) = config_facade.as_ref() {
1034 wait_for_section_file_settle(&data_dir, SectionId::Mcp).await;
1035 let startup_root_mcp = synthetic_root_events.contains_key(&SectionId::Mcp);
1036 let synthetic = synthetic_root_events.remove(&SectionId::Mcp);
1037 let reloaded = synthetic
1038 .is_none()
1039 .then(|| facade.registry().reload_if_changed(SectionId::Mcp))
1040 .flatten();
1041 let forced_initial_mcp = synthetic.is_none()
1042 && reloaded.is_none()
1043 && changes.initial_mcp_revision.is_some_and(|revision| {
1044 facade.registry().mcp.snapshot().revision == revision
1045 });
1046 let event = synthetic.or(reloaded).or_else(|| {
1047 forced_initial_mcp.then(|| ConfigSectionEvent::Changed {
1048 section: "mcp".to_string(),
1049 revision: facade.registry().mcp.snapshot().revision,
1050 })
1051 });
1052 if let Some(observed) = event {
1053 if let Some(event) = pending_root_publication_event(
1054 &data_dir,
1055 facade,
1056 &account_sink,
1057 &pending_root_publications,
1058 SectionId::Mcp,
1059 observed,
1060 ) {
1061 if matches!(event, ConfigSectionEvent::Invalid { .. }) {
1062 publish_section_failure(
1063 &apply_mcp_health,
1064 &account_sink,
1065 "mcp",
1066 facade.registry().mcp.snapshot().status,
1067 "MCP section is invalid; retaining last-known-good runtime"
1068 .to_string(),
1069 )
1070 .await;
1071 } else {
1072 self_write_marker.mark_self_write(mcp_store.path());
1073 let materialized =
1074 materialize_facade_effective_config(facade, &data_dir);
1075 if materialized.failures.contains(&SectionId::Mcp) {
1076 retry_legacy_root_publication |=
1077 publish_staged_facade_section_failure(
1078 &data_dir,
1079 facade,
1080 SectionId::Mcp,
1081 "MCP credential hydration failed; retaining last-known-good runtime",
1082 StagedFacadeSectionFailureContext {
1083 health: &apply_mcp_health,
1084 account_sink: &account_sink,
1085 section: "mcp",
1086 pending_root_publications: &pending_root_publications,
1087 },
1088 )
1089 .await;
1090 } else {
1091 let next_mcp = materialized.config.mcp.clone();
1092 let publish_config = config.clone();
1093 match mcp_manager
1094 .reconcile_from_config_transactional_after(
1095 &materialized.config.mcp,
1096 || async move {
1097 publish_config.write().await.mcp = next_mcp;
1098 Ok(())
1099 },
1100 )
1101 .await
1102 {
1103 Ok(()) => {
1104 let recovered =
1105 section_is_unhealthy(&apply_mcp_health);
1106 let snapshot = facade.registry().mcp.snapshot();
1107 let revision = snapshot.revision;
1108 if forced_initial_mcp
1109 && !startup_root_mcp
1110 && !pending_root_publications
1111 .contains_key(&SectionId::Mcp)
1112 && snapshot.status == SectionStatus::Healthy
1113 {
1114 set_live_health_revision(
1120 &apply_mcp_health,
1121 revision,
1122 Some((
1123 data_dir.join("mcp.json"),
1124 SectionSourceKind::File,
1125 )),
1126 );
1127 } else {
1128 if pending_root_publications
1129 .get(&SectionId::Mcp)
1130 .is_some_and(|event| {
1131 config_section_event_revision(event)
1132 == revision
1133 })
1134 {
1135 set_live_health_revision(
1136 &apply_mcp_health,
1137 revision,
1138 Some((
1139 data_dir.join("mcp.json"),
1140 SectionSourceKind::File,
1141 )),
1142 );
1143 retry_legacy_root_publication |=
1144 publish_registry_event_with_root_ack(
1145 &data_dir,
1146 &account_sink,
1147 &mut pending_root_publications,
1148 SectionId::Mcp,
1149 &event,
1150 )
1151 .await;
1152 } else if matches!(
1153 event,
1154 ConfigSectionEvent::Recovered { .. }
1155 ) {
1156 set_live_health_revision(
1157 &apply_mcp_health,
1158 revision,
1159 Some((
1160 data_dir.join("mcp.json"),
1161 SectionSourceKind::File,
1162 )),
1163 );
1164 publish_registry_event(
1165 &account_sink,
1166 &event,
1167 )
1168 .await;
1169 } else {
1170 publish_section_success(
1171 &apply_mcp_health,
1172 &account_sink,
1173 "mcp",
1174 data_dir.join("mcp.json"),
1175 recovered,
1176 Some(revision),
1177 )
1178 .await;
1179 }
1180 }
1181 }
1182 Err(_) => {
1183 retry_legacy_root_publication |=
1184 publish_staged_facade_section_failure(
1185 &data_dir,
1186 facade,
1187 SectionId::Mcp,
1188 "MCP runtime initialization failed; retaining last-known-good runtime",
1189 StagedFacadeSectionFailureContext {
1190 health: &apply_mcp_health,
1191 account_sink: &account_sink,
1192 section: "mcp",
1193 pending_root_publications: &pending_root_publications,
1194 },
1195 )
1196 .await;
1197 }
1198 }
1199 }
1200 }
1201 } else {
1202 retry_legacy_root_publication = true;
1203 }
1204 }
1205 } else {
1206 let current_config = config.read().await.clone();
1207 let current_revision = apply_mcp_health
1208 .read()
1209 .unwrap_or_else(|poisoned| poisoned.into_inner())
1210 .revision;
1211 let force_initial_mcp =
1212 changes.initial_mcp_revision == Some(current_revision);
1213 let result = load_and_validate_mcp_candidate(
1214 &mcp_store,
1215 current_revision,
1216 current_config,
1217 force_initial_mcp,
1218 )
1219 .await;
1220 match result {
1221 Ok(candidate) if candidate.unchanged => {}
1222 Ok(candidate) => {
1223 if candidate.normalized_external_revision
1224 || candidate.source_kind == SectionSourceKind::Backup
1225 {
1226 self_write_marker.mark_self_write(mcp_store.path());
1231 }
1232 let next_mcp = candidate.config.mcp.clone();
1233 let publish_config = config.clone();
1234 match mcp_manager
1235 .reconcile_from_config_transactional_after(
1236 &candidate.config.mcp,
1237 || async move {
1238 publish_config.write().await.mcp = next_mcp;
1239 Ok(())
1240 },
1241 )
1242 .await
1243 {
1244 Ok(()) => {
1245 let recovered = section_is_unhealthy(&apply_mcp_health);
1246 if candidate.source_kind == SectionSourceKind::Backup {
1247 publish_mcp_backup_lkg(
1248 &apply_mcp_health,
1249 &account_sink,
1250 candidate.source_path,
1251 candidate.revision,
1252 )
1253 .await;
1254 } else {
1255 publish_section_success(
1256 &apply_mcp_health,
1257 &account_sink,
1258 "mcp",
1259 data_dir.join("mcp.json"),
1260 recovered,
1261 Some(candidate.revision),
1262 )
1263 .await;
1264 }
1265 }
1266 Err(_) => publish_section_failure(
1267 &apply_mcp_health,
1268 &account_sink,
1269 "mcp",
1270 SectionStatus::Degraded,
1271 "MCP runtime initialization failed; retaining last-known-good runtime"
1272 .to_string(),
1273 )
1274 .await,
1275 }
1276 }
1277 Err(error) => {
1278 publish_section_failure(
1279 &apply_mcp_health,
1280 &account_sink,
1281 "mcp",
1282 candidate_error_status(&error.kind),
1283 error.message,
1284 )
1285 .await
1286 }
1287 }
1288 }
1289 }
1290 if !catchup_paths.is_empty() {
1291 let _ = catchup_changes.send(ConfigPathChanges {
1292 paths: catchup_paths,
1293 initial_mcp_revision: None,
1294 startup_legacy_root: None,
1295 startup_recoveries: BTreeMap::new(),
1296 legacy_root_retry_attempt: 0,
1297 });
1298 }
1299 if requeue_legacy_root && pending_root_publications.is_empty() {
1300 let _ = catchup_changes.send(ConfigPathChanges {
1301 paths: vec![data_dir.join("config.json")],
1302 initial_mcp_revision: None,
1303 startup_legacy_root: None,
1304 startup_recoveries: BTreeMap::new(),
1305 legacy_root_retry_attempt: 0,
1306 });
1307 }
1308 if retry_legacy_root_publication {
1309 let retry_changes = catchup_changes.clone();
1310 let retry_path = data_dir.join("config.json");
1311 let retry_attempt = changes.legacy_root_retry_attempt.saturating_add(1);
1312 let delay = Duration::from_millis(50_u64 << retry_attempt.min(5));
1313 tokio::spawn(async move {
1314 tokio::time::sleep(delay).await;
1315 let _ = retry_changes.send(ConfigPathChanges {
1316 paths: vec![retry_path],
1317 initial_mcp_revision: None,
1318 startup_legacy_root: None,
1319 startup_recoveries: BTreeMap::new(),
1320 legacy_root_retry_attempt: retry_attempt,
1321 });
1322 });
1323 }
1324 }
1325 });
1326
1327 (
1328 Self {
1329 stop,
1330 watcher_task: Some(watcher_task),
1331 apply_task: Some(apply_task),
1332 },
1333 provider_health,
1334 mcp_health,
1335 )
1336 }
1337}
1338
1339fn durable_invalid_recoveries(
1340 account_sink: &bamboo_engine::events::AccountEventSink,
1341 facade: &bamboo_config::ConfigFacade,
1342 rejected: &BTreeSet<SectionId>,
1343) -> BTreeMap<SectionId, u64> {
1344 let Ok(durable_facade) = bamboo_config::ConfigFacade::open(facade.data_dir()) else {
1345 return BTreeMap::new();
1346 };
1347 let mut latest = BTreeMap::<SectionId, (bool, u64)>::new();
1348 if let Ok(events) = bamboo_engine::events::journal::read_since(account_sink.events_dir(), 0) {
1349 for change in events {
1350 let state = match change.event {
1351 AgentEvent::ConfigInvalid { section, revision } => Some((section, true, revision)),
1352 AgentEvent::ConfigChanged { section, revision }
1353 | AgentEvent::ConfigRecovered { section, revision } => {
1354 Some((section, false, revision))
1355 }
1356 _ => None,
1357 };
1358 if let Some((section, invalid, revision)) = state {
1359 if let Some(id) = SectionId::from_name(§ion) {
1360 latest.insert(id, (invalid, revision));
1361 }
1362 }
1363 }
1364 }
1365 latest
1366 .into_iter()
1367 .filter_map(|(id, (invalid, invalid_revision))| {
1368 if !invalid || rejected.contains(&id) {
1369 return None;
1370 }
1371 let envelope = durable_facade.registry().envelope_value(id).ok()?;
1372 (envelope.revision >= invalid_revision
1373 && envelope.status == SectionStatus::Healthy
1374 && envelope.source_kind == SectionSourceKind::File)
1375 .then_some((id, envelope.revision))
1376 })
1377 .collect()
1378}
1379
1380fn queue_legacy_root_recovery(
1381 facade: &bamboo_config::ConfigFacade,
1382 id: SectionId,
1383 minimum_revision: Option<u64>,
1384 watched_sections: &mut BTreeSet<SectionId>,
1385 synthetic_events: &mut BTreeMap<SectionId, ConfigSectionEvent>,
1386) {
1387 watched_sections.insert(id);
1388 let _ = facade.registry().reload(id);
1391 let Ok(envelope) = facade.registry().envelope_value(id) else {
1392 return;
1393 };
1394 if envelope.status != SectionStatus::Healthy
1395 || envelope.source_kind != SectionSourceKind::File
1396 || minimum_revision.is_some_and(|minimum| envelope.revision < minimum)
1397 {
1398 return;
1399 }
1400 synthetic_events
1401 .entry(id)
1402 .or_insert_with(|| ConfigSectionEvent::Recovered {
1403 section: id.descriptor().name.to_string(),
1404 revision: envelope.revision,
1405 });
1406}
1407
1408fn pending_root_publication_matches_fresh_durable(
1409 data_dir: &Path,
1410 process_facade: &bamboo_config::ConfigFacade,
1411 id: SectionId,
1412 revision: u64,
1413) -> bool {
1414 let Ok(process) = process_facade.registry().envelope_value(id) else {
1415 return false;
1416 };
1417 if process.revision != revision {
1418 return false;
1419 }
1420 let event = ConfigSectionEvent::Changed {
1421 section: id.descriptor().name.to_string(),
1422 revision,
1423 };
1424 bamboo_config::legacy_root_publication_matches_snapshot(data_dir, &event, &process.data)
1425 .unwrap_or(false)
1426}
1427
1428async fn wait_for_section_file_settle(data_dir: &Path, id: SectionId) {
1429 let path = data_dir.join(id.descriptor().file_name);
1430 for _ in 0..3 {
1431 if path.exists() {
1432 return;
1433 }
1434 tokio::time::sleep(Duration::from_millis(50)).await;
1435 }
1436}
1437
1438struct OrdinarySectionReloadState<'a> {
1444 synthetic_events: &'a mut BTreeMap<SectionId, ConfigSectionEvent>,
1445 pending_root_publications: &'a mut BTreeMap<SectionId, ConfigSectionEvent>,
1446 reported_root_runtime_failures: &'a mut BTreeSet<(SectionId, u64)>,
1447}
1448
1449async fn reload_and_apply_ordinary_sections(
1450 data_dir: &Path,
1451 config: &Arc<RwLock<Config>>,
1452 facade: &bamboo_config::ConfigFacade,
1453 account_sink: &bamboo_engine::events::AccountEventSink,
1454 sections: impl IntoIterator<Item = SectionId>,
1455 state: OrdinarySectionReloadState<'_>,
1456) -> bool {
1457 let OrdinarySectionReloadState {
1458 synthetic_events,
1459 pending_root_publications,
1460 reported_root_runtime_failures,
1461 } = state;
1462 let mut retry_legacy_root_publication = false;
1463 let mut publishable = Vec::new();
1464 for id in sections {
1465 wait_for_section_file_settle(data_dir, id).await;
1466 let Some(event) = synthetic_events
1467 .remove(&id)
1468 .or_else(|| facade.registry().reload_if_changed(id))
1469 else {
1470 continue;
1471 };
1472 let Some(event) = pending_root_publication_event(
1473 data_dir,
1474 facade,
1475 account_sink,
1476 pending_root_publications,
1477 id,
1478 event,
1479 ) else {
1480 retry_legacy_root_publication = true;
1481 continue;
1482 };
1483 if matches!(event, ConfigSectionEvent::Invalid { .. }) {
1484 publish_registry_event(account_sink, &event).await;
1485 } else {
1486 publishable.push((id, event));
1487 }
1488 }
1489 if publishable.is_empty() {
1490 return retry_legacy_root_publication;
1491 }
1492
1493 let materialized = materialize_facade_effective_config(facade, data_dir);
1494 let mut current = config.read().await.clone();
1495 let mut applied = Vec::new();
1496 for (id, event) in publishable {
1497 if materialized.failures.contains(&id) {
1498 let revision = facade
1499 .registry()
1500 .envelope_value(id)
1501 .map(|envelope| envelope.revision)
1502 .unwrap_or_default();
1503 let invalid = facade.registry().mark_runtime_degraded(
1504 id,
1505 "configuration runtime hydration failed; retaining last-known-good runtime",
1506 );
1507 if let Some(invalid) = invalid {
1508 if pending_root_publications
1509 .get(&id)
1510 .is_some_and(|event| config_section_event_revision(event) == revision)
1511 {
1512 let _ =
1513 confirm_legacy_root_runtime_failure(data_dir, account_sink, &invalid).await;
1514 } else if reported_root_runtime_failures.insert((id, revision)) {
1515 publish_registry_event(account_sink, &invalid).await;
1516 }
1517 }
1518 retry_legacy_root_publication |= pending_root_publications.contains_key(&id);
1519 continue;
1520 }
1521 apply_runtime_section(id, &materialized.config, &mut current);
1522 applied.push((id, event));
1523 }
1524 if applied.is_empty() {
1525 return retry_legacy_root_publication;
1526 }
1527
1528 let publishes_env = applied.iter().any(|(id, _)| *id == SectionId::Env);
1529 let enforcement_newly_off = !config.read().await.plugin_trust.enforcement_is_off()
1530 && current.plugin_trust.enforcement_is_off();
1531 *config.write().await = current.clone();
1532 if publishes_env {
1533 current.publish_env_vars();
1534 }
1535 if enforcement_newly_off {
1536 warn_plugin_trust_enforcement_off();
1537 }
1538 for (id, event) in applied {
1539 retry_legacy_root_publication |= publish_registry_event_with_root_ack(
1540 data_dir,
1541 account_sink,
1542 pending_root_publications,
1543 id,
1544 &event,
1545 )
1546 .await;
1547 reported_root_runtime_failures.retain(|(failed_id, _)| *failed_id != id);
1548 }
1549 retry_legacy_root_publication
1550}
1551
1552fn pending_root_publication_event(
1553 data_dir: &Path,
1554 facade: &bamboo_config::ConfigFacade,
1555 account_sink: &bamboo_engine::events::AccountEventSink,
1556 pending: &BTreeMap<SectionId, ConfigSectionEvent>,
1557 id: SectionId,
1558 observed: ConfigSectionEvent,
1559) -> Option<ConfigSectionEvent> {
1560 let Some(pending_event) = pending.get(&id) else {
1561 return Some(observed);
1562 };
1563 let revision = config_section_event_revision(pending_event);
1564 let Ok(envelope) = facade.registry().envelope_value(id) else {
1565 return None;
1566 };
1567 if envelope.revision != revision || envelope.status != SectionStatus::Healthy {
1568 return None;
1569 }
1570 let pending = ConfigSectionEvent::Changed {
1571 section: id.descriptor().name.to_string(),
1572 revision,
1573 };
1574 if account_sink.latest_config_transition_is_invalid(id.descriptor().name, revision) {
1575 let invalid = ConfigSectionEvent::Invalid {
1576 section: id.descriptor().name.to_string(),
1577 revision,
1578 };
1579 match bamboo_config::mark_legacy_root_publication_runtime_degraded(data_dir, &invalid) {
1580 Ok(true) => {}
1581 Ok(false) => return None,
1582 Err(error) => {
1583 tracing::warn!(
1584 %error,
1585 section = id.descriptor().name,
1586 revision,
1587 "failed to repair canonical root runtime-degraded proof from the durable journal"
1588 );
1589 return None;
1590 }
1591 }
1592 }
1593 bamboo_config::legacy_root_publication_success_event(data_dir, &pending, &envelope.data)
1594 .ok()
1595 .flatten()
1596}
1597
1598async fn publish_registry_event_with_root_ack(
1599 data_dir: &Path,
1600 account_sink: &bamboo_engine::events::AccountEventSink,
1601 pending: &mut BTreeMap<SectionId, ConfigSectionEvent>,
1602 id: SectionId,
1603 event: &ConfigSectionEvent,
1604) -> bool {
1605 if matches!(event, ConfigSectionEvent::Invalid { .. }) {
1606 publish_registry_event(account_sink, event).await;
1607 return false;
1608 }
1609 if pending.get(&id).is_none_or(|pending_event| {
1610 config_section_event_revision(pending_event) != config_section_event_revision(event)
1611 }) {
1612 publish_registry_event(account_sink, event).await;
1613 return false;
1614 }
1615 let durable = account_sink
1616 .record_confirmed(None, ®istry_agent_event(event))
1617 .await;
1618 if durable
1619 && bamboo_config::acknowledge_legacy_root_publication(data_dir, event).unwrap_or(false)
1620 {
1621 pending.remove(&id);
1622 false
1623 } else {
1624 true
1625 }
1626}
1627
1628pub(super) async fn publish_registry_event(
1629 account_sink: &bamboo_engine::events::AccountEventSink,
1630 event: &ConfigSectionEvent,
1631) {
1632 let event = registry_agent_event(event);
1633 if !account_sink.record_confirmed(None, &event).await {
1634 tracing::warn!("configuration event could not be confirmed in the account journal");
1635 }
1636}
1637
1638async fn confirm_legacy_root_runtime_failure(
1639 data_dir: &Path,
1640 account_sink: &bamboo_engine::events::AccountEventSink,
1641 event: &ConfigSectionEvent,
1642) -> bool {
1643 if !account_sink
1644 .record_confirmed(None, ®istry_agent_event(event))
1645 .await
1646 {
1647 return false;
1648 }
1649 match bamboo_config::mark_legacy_root_publication_runtime_degraded(data_dir, event) {
1650 Ok(marked) => marked,
1651 Err(error) => {
1652 tracing::warn!(
1653 %error,
1654 "failed to mark canonical root publication runtime-degraded"
1655 );
1656 false
1657 }
1658 }
1659}
1660
1661fn legacy_root_rejection_event(
1662 facade: &bamboo_config::ConfigFacade,
1663 id: SectionId,
1664 diagnostic: &str,
1665 force_publication: bool,
1666) -> Option<ConfigSectionEvent> {
1667 let already_reported = facade.registry().envelope_value(id).is_ok_and(|envelope| {
1668 envelope.status == SectionStatus::Degraded
1669 && envelope.last_error.as_deref() == Some(diagnostic)
1670 });
1671 if already_reported && !force_publication {
1672 return None;
1673 }
1674 facade
1675 .registry()
1676 .mark_runtime_degraded(id, diagnostic)
1677 .or_else(|| {
1678 force_publication
1681 .then(|| {
1682 facade.registry().envelope_value(id).ok().map(|envelope| {
1683 ConfigSectionEvent::Invalid {
1684 section: id.descriptor().name.to_string(),
1685 revision: envelope.revision,
1686 }
1687 })
1688 })
1689 .flatten()
1690 })
1691}
1692
1693fn config_section_event_revision(event: &ConfigSectionEvent) -> u64 {
1694 match event {
1695 ConfigSectionEvent::Changed { revision, .. }
1696 | ConfigSectionEvent::Invalid { revision, .. }
1697 | ConfigSectionEvent::Recovered { revision, .. } => *revision,
1698 }
1699}
1700
1701fn registry_agent_event(event: &ConfigSectionEvent) -> AgentEvent {
1702 match event {
1703 ConfigSectionEvent::Changed { section, revision } => AgentEvent::ConfigChanged {
1704 section: section.clone(),
1705 revision: *revision,
1706 },
1707 ConfigSectionEvent::Invalid { section, revision } => AgentEvent::ConfigInvalid {
1708 section: section.clone(),
1709 revision: *revision,
1710 },
1711 ConfigSectionEvent::Recovered { section, revision } => AgentEvent::ConfigRecovered {
1712 section: section.clone(),
1713 revision: *revision,
1714 },
1715 }
1716}
1717
1718async fn publish_exact_facade_events(
1719 account_sink: &bamboo_engine::events::AccountEventSink,
1720 events: &[ConfigSectionEvent],
1721) -> Result<(), AppError> {
1722 for event in events {
1723 let durable = account_sink
1724 .record_confirmed(None, ®istry_agent_event(event))
1725 .await;
1726 if !durable {
1727 return Err(AppError::InternalError(anyhow::anyhow!(
1728 "committed configuration event could not be confirmed in the account journal"
1729 )));
1730 }
1731 if matches!(event, ConfigSectionEvent::Invalid { .. }) {
1732 return Err(AppError::InternalError(anyhow::anyhow!(
1733 "committed configuration section became invalid before publication"
1734 )));
1735 }
1736 }
1737 Ok(())
1738}
1739
1740struct InstalledCredentialSectionCommit {
1741 events: Vec<ConfigSectionEvent>,
1742 metadata: bamboo_config::CredentialSectionRuntimeMetadata,
1743 section: Option<bamboo_config::SectionEnvelope<Value>>,
1744}
1745
1746pub(crate) struct ExactCredentialSectionSnapshot {
1747 pub config: Config,
1748 pub section: bamboo_config::SectionEnvelope<Value>,
1749 pub metadata: bamboo_config::CredentialSectionRuntimeMetadata,
1750}
1751
1752fn map_exact_credential_store_error(error: ConfigStoreError) -> AppError {
1753 match error {
1754 ConfigStoreError::Conflict { expected, actual } => {
1755 AppError::ConfigConflict { expected, actual }
1756 }
1757 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
1758 ConfigStoreError::CommitIndeterminate(message) => AppError::InternalError(anyhow::anyhow!(
1759 "configuration commit outcome is indeterminate: {message}"
1760 )),
1761 ConfigStoreError::Io(error) => AppError::StorageError(error),
1762 ConfigStoreError::Json(_) => {
1763 AppError::BadRequest("configuration document is invalid".to_string())
1764 }
1765 ConfigStoreError::Watch(error) => {
1766 AppError::InternalError(anyhow::anyhow!("configuration watch failed: {error}"))
1767 }
1768 }
1769}
1770
1771async fn install_exact_credential_section_mutation_base(
1772 data_dir: PathBuf,
1773 section: SectionId,
1774 expected_revision: u64,
1775 target: &mut Config,
1776) -> Result<bamboo_config::CredentialSectionRuntimeMetadata, AppError> {
1777 let exact = tokio::task::spawn_blocking(move || {
1778 bamboo_config::read_exact_credential_section_snapshot(
1779 data_dir,
1780 section,
1781 Some(expected_revision),
1782 )
1783 })
1784 .await
1785 .map_err(|error| {
1786 AppError::InternalError(anyhow::anyhow!(
1787 "{} exact mutation snapshot task failed: {error}",
1788 section.descriptor().name
1789 ))
1790 })?
1791 .map_err(map_exact_credential_store_error)?;
1792 Ok(exact.install_into(target))
1793}
1794
1795fn read_credential_runtime_metadata(
1796 data_dir: &std::path::Path,
1797) -> Result<bamboo_config::CredentialSectionRuntimeMetadata, ConfigStoreError> {
1798 let (credential_statuses, credential_health) =
1799 bamboo_config::CredentialStore::open(data_dir).statuses_with_health()?;
1800 Ok(bamboo_config::CredentialSectionRuntimeMetadata {
1801 credential_statuses,
1802 credential_health,
1803 })
1804}
1805
1806fn install_credential_section_commit(
1807 commit: bamboo_config::CredentialSectionTransactionCommit,
1808 target: &mut Config,
1809) -> Result<InstalledCredentialSectionCommit, ConfigStoreError> {
1810 let bamboo_config::CredentialSectionTransactionCommit {
1811 revision: _,
1812 section_adoption,
1813 credential_adoption,
1814 section,
1815 runtime,
1816 } = commit;
1817 let section = section?;
1818 let metadata = runtime?.install_into(target);
1819 let mut events = Vec::new();
1820 if let Some(adoption) = credential_adoption {
1821 if let Some(event) = adoption? {
1822 events.push(event);
1823 }
1824 }
1825 if let Some(adoption) = section_adoption {
1826 events.push(adoption?);
1827 }
1828 Ok(InstalledCredentialSectionCommit {
1829 events,
1830 metadata,
1831 section: Some(section),
1832 })
1833}
1834
1835fn install_facade_config_commit(
1836 commit: bamboo_config::FacadeConfigCommit,
1837 target: &mut Config,
1838) -> Result<Vec<ConfigSectionEvent>, ConfigStoreError> {
1839 let bamboo_config::FacadeConfigCommit {
1840 section_adoption,
1841 runtime,
1842 } = commit;
1843 if let Some(runtime) = runtime? {
1844 runtime.install_into(target);
1845 }
1846 section_adoption
1847 .map(|adoption| adoption.map(|event| vec![event]))
1848 .unwrap_or_else(|| Ok(Vec::new()))
1849}
1850
1851fn apply_runtime_section(id: SectionId, source: &Config, target: &mut Config) {
1852 match id {
1853 SectionId::Core => {
1854 target.http_proxy = source.http_proxy.clone();
1855 target.https_proxy = source.https_proxy.clone();
1856 target.proxy_auth = source.proxy_auth.clone();
1857 target.proxy_auth_encrypted = None;
1858 target.proxy_auth_credential_ref = source.proxy_auth_credential_ref.clone();
1859 target.headless_auth = source.headless_auth;
1860 target.server = source.server.clone();
1861 target.default_work_area = source.default_work_area.clone();
1862 target.run_budget = source.run_budget;
1863 target.stream_timeout = source.stream_timeout;
1864 target.extra = source.extra.clone();
1865 }
1866 SectionId::Providers => {
1867 target.provider = source.provider.clone();
1868 target.defaults = source.defaults.clone();
1869 target.provider_instances = source.provider_instances.clone();
1870 target.default_provider_instance = source.default_provider_instance.clone();
1871 target.features = source.features.clone();
1872 *target.providers_mut() = source.providers().clone();
1873 }
1874 SectionId::Mcp => target.mcp = source.mcp.clone(),
1875 SectionId::ToolsSkills => {
1876 target.tools = source.tools.clone();
1877 target.skills = source.skills.clone();
1878 target.plugin_trust = source.plugin_trust.clone();
1879 }
1880 SectionId::Memory => *target.memory_mut() = source.memory().clone(),
1881 SectionId::Subagents => {
1882 let runtime_broker = target.subagents().broker.clone();
1883 *target.subagents_mut() = source.subagents().clone();
1884 if target.subagents().broker.is_none() {
1885 target.subagents_mut().broker = runtime_broker;
1886 }
1887 }
1888 SectionId::Notifications => target.notifications = source.notifications.clone(),
1889 SectionId::Connect => target.connect = source.connect.clone(),
1890 SectionId::ClusterFabric => target.cluster_fabric = source.cluster_fabric.clone(),
1891 SectionId::Env => target.env_vars = source.env_vars.clone(),
1892 SectionId::AccessControl => target.access_control = source.access_control.clone(),
1893 SectionId::Hooks => target.hooks = source.hooks.clone(),
1894 SectionId::ModelPolicy => {
1895 target.keyword_masking = source.keyword_masking.clone();
1896 target.anthropic_model_mapping = source.anthropic_model_mapping.clone();
1897 target.gemini_model_mapping = source.gemini_model_mapping.clone();
1898 }
1899 SectionId::ModelLimits | SectionId::Credentials => {}
1900 }
1901}
1902
1903fn restore_authoritative_cluster_fabric(
1908 facade: Option<&std::sync::Arc<bamboo_config::ConfigFacade>>,
1909 candidate: &mut Config,
1910) {
1911 if let Some(facade) = facade {
1912 candidate.cluster_fabric = facade.registry().cluster_fabric.snapshot().data.0.clone();
1913 }
1914}
1915
1916fn preserve_runtime_broker(new_config: &mut Config, previous: &Config) {
1920 if new_config.subagents().broker.is_none() {
1921 new_config.subagents_mut().broker = previous.subagents().broker.clone();
1922 }
1923}
1924
1925fn section_is_unhealthy(health: &std::sync::RwLock<ConfigLiveHealth>) -> bool {
1926 health
1927 .read()
1928 .unwrap_or_else(|poisoned| poisoned.into_inner())
1929 .status
1930 != SectionStatus::Healthy
1931}
1932
1933async fn publish_section_success(
1934 health: &std::sync::RwLock<ConfigLiveHealth>,
1935 account_sink: &bamboo_engine::events::AccountEventSink,
1936 section: &str,
1937 source_path: PathBuf,
1938 recovered: bool,
1939 revision: Option<u64>,
1940) {
1941 let revision = match revision {
1942 Some(revision) => set_live_health_revision(
1943 health,
1944 revision,
1945 Some((source_path, SectionSourceKind::File)),
1946 ),
1947 None => update_live_health(
1948 health,
1949 SectionStatus::Healthy,
1950 None,
1951 true,
1952 Some((source_path, SectionSourceKind::File)),
1953 ),
1954 };
1955 let event = if recovered {
1956 AgentEvent::ConfigRecovered {
1957 section: section.to_string(),
1958 revision,
1959 }
1960 } else {
1961 AgentEvent::ConfigChanged {
1962 section: section.to_string(),
1963 revision,
1964 }
1965 };
1966 if !account_sink.record_confirmed(None, &event).await {
1967 tracing::warn!(
1968 section,
1969 revision,
1970 "configuration success event was not durable"
1971 );
1972 }
1973}
1974
1975async fn publish_section_failure(
1976 health: &std::sync::RwLock<ConfigLiveHealth>,
1977 account_sink: &bamboo_engine::events::AccountEventSink,
1978 section: &str,
1979 status: SectionStatus,
1980 message: String,
1981) {
1982 let duplicate = {
1983 let health = health
1984 .read()
1985 .unwrap_or_else(|poisoned| poisoned.into_inner());
1986 health.status == status && health.last_error.as_deref() == Some(message.as_str())
1987 };
1988 let revision = update_live_health(health, status, Some(message), false, None);
1989 if duplicate {
1990 return;
1991 }
1992 let event = AgentEvent::ConfigInvalid {
1993 section: section.to_string(),
1994 revision,
1995 };
1996 if !account_sink.record_confirmed(None, &event).await {
1997 tracing::warn!(
1998 section,
1999 revision,
2000 "configuration failure event was not durable"
2001 );
2002 }
2003}
2004
2005struct StagedFacadeSectionFailureContext<'a> {
2006 health: &'a std::sync::RwLock<ConfigLiveHealth>,
2007 account_sink: &'a bamboo_engine::events::AccountEventSink,
2008 section: &'a str,
2009 pending_root_publications: &'a BTreeMap<SectionId, ConfigSectionEvent>,
2010}
2011
2012async fn publish_staged_facade_section_failure(
2013 data_dir: &Path,
2014 facade: &bamboo_config::ConfigFacade,
2015 id: SectionId,
2016 message: &str,
2017 context: StagedFacadeSectionFailureContext<'_>,
2018) -> bool {
2019 let StagedFacadeSectionFailureContext {
2020 health,
2021 account_sink,
2022 section,
2023 pending_root_publications,
2024 } = context;
2025 let event = facade
2026 .registry()
2027 .mark_runtime_degraded(id, message)
2028 .expect("every facade section exposes runtime health");
2029 let exact_pending = matches!(
2030 &event,
2031 ConfigSectionEvent::Invalid { revision, .. }
2032 if pending_root_publications
2033 .get(&id)
2034 .is_some_and(|event| config_section_event_revision(event) == *revision)
2035 );
2036 if exact_pending {
2037 update_live_health(
2041 health,
2042 SectionStatus::Degraded,
2043 Some(message.to_string()),
2044 false,
2045 None,
2046 );
2047 if !confirm_legacy_root_runtime_failure(data_dir, account_sink, &event).await {
2048 tracing::warn!(
2049 section,
2050 "root runtime failure was not confirmed against its canonical publication"
2051 );
2052 }
2053 } else {
2054 publish_section_failure(
2055 health,
2056 account_sink,
2057 section,
2058 SectionStatus::Degraded,
2059 message.to_string(),
2060 )
2061 .await;
2062 }
2063 exact_pending
2064}
2065
2066async fn publish_mcp_backup_lkg(
2067 health: &std::sync::RwLock<ConfigLiveHealth>,
2068 account_sink: &bamboo_engine::events::AccountEventSink,
2069 source_path: PathBuf,
2070 revision: u64,
2071) {
2072 {
2073 let mut health = health
2074 .write()
2075 .unwrap_or_else(|poisoned| poisoned.into_inner());
2076 health.revision = revision;
2077 health.loaded_at = Utc::now();
2078 health.source_path = source_path;
2079 health.source_kind = SectionSourceKind::Backup;
2080 health.status = SectionStatus::Degraded;
2081 health.last_error =
2082 Some("primary MCP section invalid; running last-known-good backup runtime".to_string());
2083 }
2084 let event = AgentEvent::ConfigInvalid {
2085 section: "mcp".to_string(),
2086 revision,
2087 };
2088 if !account_sink.record_confirmed(None, &event).await {
2089 tracing::warn!(revision, "MCP backup health event was not durable");
2090 }
2091}
2092
2093fn initial_provider_health(store: &AtomicJsonStore<ProviderConfigs>) -> ConfigLiveHealth {
2094 if ensure_provider_mcp_migration_ready(store.path().parent().unwrap_or_else(|| Path::new(".")))
2095 .is_err()
2096 {
2097 return ConfigLiveHealth {
2098 revision: 0,
2099 loaded_at: Utc::now(),
2100 source_path: store.path().to_path_buf(),
2101 source_kind: SectionSourceKind::File,
2102 status: SectionStatus::Degraded,
2103 last_error: Some("provider/MCP credential migration is pending".to_string()),
2104 };
2105 }
2106 match store.load_validated_allowing_unversioned(|_| Ok(())) {
2107 Ok(Some(stored)) => ConfigLiveHealth {
2108 revision: stored.revision,
2109 loaded_at: Utc::now(),
2110 source_path: stored.source_path,
2111 source_kind: if stored.recovered_from_backup {
2112 SectionSourceKind::Backup
2113 } else {
2114 SectionSourceKind::File
2115 },
2116 status: if stored.recovered_from_backup {
2117 SectionStatus::Degraded
2118 } else {
2119 SectionStatus::Healthy
2120 },
2121 last_error: stored.recovered_from_backup.then(|| {
2122 "primary provider section invalid; using last-known-good backup".to_string()
2123 }),
2124 },
2125 Ok(None) => ConfigLiveHealth {
2126 revision: 0,
2127 loaded_at: Utc::now(),
2128 source_path: store.path().to_path_buf(),
2129 source_kind: SectionSourceKind::Default,
2130 status: SectionStatus::Missing,
2131 last_error: None,
2132 },
2133 Err(_) => ConfigLiveHealth {
2134 revision: 0,
2135 loaded_at: Utc::now(),
2136 source_path: store.path().to_path_buf(),
2137 source_kind: SectionSourceKind::File,
2138 status: SectionStatus::Invalid,
2139 last_error: Some("provider section could not be parsed or read".to_string()),
2140 },
2141 }
2142}
2143
2144fn initial_mcp_health(store: &AtomicJsonStore<McpConfig>) -> ConfigLiveHealth {
2145 if ensure_provider_mcp_migration_ready(store.path().parent().unwrap_or_else(|| Path::new(".")))
2146 .is_err()
2147 {
2148 return ConfigLiveHealth {
2149 revision: 0,
2150 loaded_at: Utc::now(),
2151 source_path: store.path().to_path_buf(),
2152 source_kind: SectionSourceKind::File,
2153 status: SectionStatus::Degraded,
2154 last_error: Some("provider/MCP credential migration is pending".to_string()),
2155 };
2156 }
2157 match store.load_validated(validate_mcp_config) {
2158 Ok(Some(stored)) => ConfigLiveHealth {
2159 revision: 0,
2162 loaded_at: Utc::now(),
2163 source_path: store.path().to_path_buf(),
2164 source_kind: if stored.recovered_from_backup {
2165 SectionSourceKind::Backup
2166 } else {
2167 SectionSourceKind::File
2168 },
2169 status: SectionStatus::Degraded,
2170 last_error: Some(if stored.recovered_from_backup {
2171 "primary MCP section invalid; runtime initialization pending from backup"
2172 .to_string()
2173 } else {
2174 "MCP runtime initialization pending".to_string()
2175 }),
2176 },
2177 Ok(None) => ConfigLiveHealth {
2178 revision: 0,
2179 loaded_at: Utc::now(),
2180 source_path: store.path().to_path_buf(),
2181 source_kind: SectionSourceKind::Default,
2182 status: SectionStatus::Missing,
2183 last_error: None,
2184 },
2185 Err(_) => ConfigLiveHealth {
2186 revision: 0,
2187 loaded_at: Utc::now(),
2188 source_path: store.path().to_path_buf(),
2189 source_kind: SectionSourceKind::File,
2190 status: SectionStatus::Invalid,
2191 last_error: Some("MCP section could not be parsed or validated".to_string()),
2192 },
2193 }
2194}
2195
2196impl Drop for ConfigWatcherRuntime {
2197 fn drop(&mut self) {
2198 self.stop.store(true, Ordering::Relaxed);
2199 if let Some(task) = self.apply_task.take() {
2200 task.abort();
2201 }
2202 if let Some(task) = self.watcher_task.take() {
2203 let _ = task.join();
2204 }
2205 }
2206}
2207
2208async fn load_and_prepare_provider_candidate(
2209 store: &AtomicJsonStore<ProviderConfigs>,
2210 current_revision: u64,
2211 candidate_config: Config,
2212) -> Result<ProviderCandidate, ProviderCandidateError> {
2213 ensure_provider_mcp_migration_ready(store.path().parent().unwrap_or_else(|| Path::new(".")))
2214 .map_err(|_| {
2215 ProviderCandidateError::invalid(
2216 "provider/MCP credential migration is pending; retaining last-known-good runtime",
2217 )
2218 })?;
2219 for _ in 0..3 {
2222 if store.path().exists() {
2223 break;
2224 }
2225 tokio::time::sleep(Duration::from_millis(50)).await;
2226 }
2227 if !store.path().exists() {
2228 return Err(ProviderCandidateError::missing());
2229 }
2230 let stored = store
2231 .load_validated_for_reload_allowing_unversioned(
2232 current_revision,
2233 candidate_config.providers(),
2234 validate_provider_config,
2235 )
2236 .map_err(|_| {
2237 if store.path().exists() {
2238 ProviderCandidateError::invalid("provider section is invalid")
2239 } else {
2240 ProviderCandidateError::missing()
2241 }
2242 })?
2243 .ok_or_else(ProviderCandidateError::missing)?;
2244 if stored.recovered_from_backup {
2245 return Err(ProviderCandidateError::invalid(
2246 "primary provider section is invalid; retaining last-known-good runtime",
2247 ));
2248 }
2249 let unchanged = stored.revision == current_revision
2250 && serde_json::to_value(&stored.data).ok()
2251 == serde_json::to_value(candidate_config.providers()).ok();
2252 let mut candidate_config = candidate_config;
2253 *candidate_config.providers_mut() = stored.data;
2254 let (candidate_config, registry, provider) = prepare_provider_candidate(
2255 candidate_config,
2256 store
2257 .path()
2258 .parent()
2259 .unwrap_or_else(|| std::path::Path::new(".")),
2260 )
2261 .await?;
2262 Ok(ProviderCandidate {
2263 config: candidate_config,
2264 registry,
2265 provider,
2266 revision: stored.revision,
2267 normalized_external_revision: stored.normalized_external_revision,
2268 unchanged,
2269 })
2270}
2271
2272async fn prepare_provider_candidate(
2273 mut candidate_config: Config,
2274 data_dir: &std::path::Path,
2275) -> Result<(Config, bamboo_llm::ProviderRegistry, Arc<dyn LLMProvider>), ProviderCandidateError> {
2276 candidate_config.hydrate_provider_api_keys_from_encrypted();
2277 candidate_config
2278 .hydrate_provider_credentials_from_store(data_dir)
2279 .map_err(|_| ProviderCandidateError::invalid("provider credential is unavailable"))?;
2280 let candidate_registry =
2281 bamboo_llm::ProviderRegistry::from_config(&candidate_config, data_dir.to_path_buf())
2282 .await
2283 .map_err(|_| ProviderCandidateError::runtime())?;
2284 let candidate_provider = candidate_registry
2285 .get_default()
2286 .ok_or_else(ProviderCandidateError::runtime)?;
2287 Ok((candidate_config, candidate_registry, candidate_provider))
2288}
2289
2290struct ProviderCandidate {
2291 config: Config,
2292 registry: bamboo_llm::ProviderRegistry,
2293 provider: Arc<dyn LLMProvider>,
2294 revision: u64,
2295 normalized_external_revision: bool,
2296 unchanged: bool,
2297}
2298
2299async fn load_and_validate_mcp_candidate(
2300 store: &AtomicJsonStore<McpConfig>,
2301 current_revision: u64,
2302 mut candidate_config: Config,
2303 allow_startup_backup: bool,
2304) -> Result<McpCandidate, ProviderCandidateError> {
2305 ensure_provider_mcp_migration_ready(store.path().parent().unwrap_or_else(|| Path::new(".")))
2306 .map_err(|_| {
2307 ProviderCandidateError::invalid(
2308 "provider/MCP credential migration is pending; retaining last-known-good runtime",
2309 )
2310 })?;
2311 for _ in 0..3 {
2312 if store.path().exists() {
2313 break;
2314 }
2315 tokio::time::sleep(Duration::from_millis(50)).await;
2316 }
2317 if !store.path().exists() {
2318 return Err(ProviderCandidateError::missing_section(
2319 "MCP section is missing",
2320 ));
2321 }
2322 let current_document = mcp_durable_comparison_document(&candidate_config.mcp);
2323 let stored = store
2324 .load_validated_for_reload(current_revision, ¤t_document, validate_mcp_config)
2325 .map_err(|_| ProviderCandidateError::invalid("MCP section is invalid"))?
2326 .ok_or_else(|| ProviderCandidateError::missing_section("MCP section is missing"))?;
2327 if stored.recovered_from_backup && !allow_startup_backup {
2328 return Err(ProviderCandidateError::invalid(
2329 "primary MCP section is invalid; retaining last-known-good runtime",
2330 ));
2331 }
2332 let unchanged = !allow_startup_backup
2333 && stored.revision == current_revision
2334 && serde_json::to_value(&stored.data).ok() == serde_json::to_value(¤t_document).ok();
2335 candidate_config.mcp = stored.data;
2336 candidate_config.hydrate_mcp_secrets_from_encrypted();
2337 candidate_config
2338 .hydrate_mcp_credentials_from_store(
2339 store
2340 .path()
2341 .parent()
2342 .unwrap_or_else(|| std::path::Path::new(".")),
2343 )
2344 .map_err(|_| ProviderCandidateError::invalid("MCP credential is unavailable"))?;
2345 Ok(McpCandidate {
2346 config: candidate_config,
2347 revision: stored.revision,
2348 source_kind: if stored.recovered_from_backup {
2349 SectionSourceKind::Backup
2350 } else {
2351 SectionSourceKind::File
2352 },
2353 source_path: stored.source_path,
2354 normalized_external_revision: stored.normalized_external_revision,
2355 unchanged,
2356 })
2357}
2358
2359struct McpCandidate {
2360 config: Config,
2361 revision: u64,
2362 source_kind: SectionSourceKind,
2363 source_path: PathBuf,
2364 normalized_external_revision: bool,
2365 unchanged: bool,
2366}
2367
2368fn mcp_durable_comparison_document(config: &McpConfig) -> McpConfig {
2372 let mut document = config.clone();
2373 for server in &mut document.servers {
2374 match &mut server.transport {
2375 TransportConfig::Stdio(config) => {
2376 config.env.retain(|name, _| {
2377 !config.env_encrypted.contains_key(name)
2378 && !config.env_credential_refs.contains_key(name)
2379 });
2380 }
2381 TransportConfig::Sse(config) => clear_paired_header_plaintext(&mut config.headers),
2382 TransportConfig::StreamableHttp(config) => {
2383 clear_paired_header_plaintext(&mut config.headers)
2384 }
2385 }
2386 }
2387 document
2388}
2389
2390fn clear_paired_header_plaintext(headers: &mut [bamboo_mcp::HeaderConfig]) {
2391 for header in headers {
2392 if header.value_encrypted.is_some() || header.credential_ref.is_some() {
2393 header.value.clear();
2394 }
2395 }
2396}
2397
2398fn validate_mcp_config(config: &McpConfig) -> Result<(), String> {
2399 let mut ids = std::collections::HashSet::new();
2400 for server in &config.servers {
2401 if server.id.trim().is_empty() {
2402 return Err("MCP server id cannot be empty".to_string());
2403 }
2404 if !ids.insert(server.id.as_str()) {
2405 return Err(format!("duplicate MCP server id '{}'", server.id));
2406 }
2407 if server.request_timeout_ms == 0 || server.healthcheck_interval_ms == 0 {
2408 return Err(format!(
2409 "MCP server '{}' timeouts must be non-zero",
2410 server.id
2411 ));
2412 }
2413 match &server.transport {
2414 TransportConfig::Stdio(stdio) if stdio.command.trim().is_empty() => {
2415 return Err(format!(
2416 "MCP stdio server '{}' command cannot be empty",
2417 server.id
2418 ));
2419 }
2420 TransportConfig::Sse(sse) if sse.url.trim().is_empty() => {
2421 return Err(format!(
2422 "MCP SSE server '{}' URL cannot be empty",
2423 server.id
2424 ));
2425 }
2426 TransportConfig::StreamableHttp(http) if http.url.trim().is_empty() => {
2427 return Err(format!(
2428 "MCP HTTP server '{}' URL cannot be empty",
2429 server.id
2430 ));
2431 }
2432 _ => {}
2433 }
2434 match &server.transport {
2435 TransportConfig::Stdio(stdio) => {
2436 if !stdio.env_encrypted.is_empty()
2437 || stdio.env.iter().any(|(name, value)| {
2438 !value.is_empty() && !stdio.env_credential_refs.contains_key(name)
2439 })
2440 {
2441 return Err(format!(
2442 "MCP server '{}' contains a secret outside the credential store",
2443 server.id
2444 ));
2445 }
2446 for raw in stdio.env_credential_refs.values() {
2447 bamboo_config::CredentialRef::parse(raw.clone())
2448 .map_err(|_| "MCP credential reference is invalid".to_string())?;
2449 }
2450 }
2451 TransportConfig::Sse(config) => validate_header_refs(&server.id, &config.headers)?,
2452 TransportConfig::StreamableHttp(config) => {
2453 validate_header_refs(&server.id, &config.headers)?
2454 }
2455 }
2456 }
2457 Ok(())
2458}
2459
2460fn validate_provider_config(providers: &ProviderConfigs) -> Result<(), String> {
2461 macro_rules! validate {
2462 ($field:ident) => {
2463 if let Some(provider) = &providers.$field {
2464 if provider.api_key_encrypted.is_some()
2465 || (!provider.api_key.trim().is_empty()
2466 && !provider.api_key_from_env
2467 && provider.credential_ref.is_none())
2468 {
2469 return Err("provider secret is outside the credential store".to_string());
2470 }
2471 }
2472 };
2473 }
2474 validate!(openai);
2475 validate!(anthropic);
2476 validate!(gemini);
2477 if let Some(provider) = &providers.bodhi {
2478 if provider.api_key_encrypted.is_some()
2479 || (!provider.api_key.trim().is_empty() && provider.credential_ref.is_none())
2480 {
2481 return Err("provider secret is outside the credential store".to_string());
2482 }
2483 }
2484 Ok(())
2485}
2486
2487fn validate_header_refs(
2488 server_id: &str,
2489 headers: &[bamboo_mcp::HeaderConfig],
2490) -> Result<(), String> {
2491 for header in headers {
2492 if header.value_encrypted.is_some()
2493 || (!header.value.is_empty() && header.credential_ref.is_none())
2494 {
2495 return Err(format!(
2496 "MCP server '{server_id}' contains a secret outside the credential store"
2497 ));
2498 }
2499 if let Some(raw) = &header.credential_ref {
2500 bamboo_config::CredentialRef::parse(raw.clone())
2501 .map_err(|_| "MCP credential reference is invalid".to_string())?;
2502 }
2503 }
2504 Ok(())
2505}
2506
2507enum ProviderCandidateErrorKind {
2508 Missing,
2509 InvalidDocument,
2510 Runtime,
2511}
2512
2513struct ProviderCandidateError {
2514 kind: ProviderCandidateErrorKind,
2515 message: String,
2516}
2517
2518impl ProviderCandidateError {
2519 fn missing() -> Self {
2520 Self {
2521 kind: ProviderCandidateErrorKind::Missing,
2522 message: "provider section is missing".to_string(),
2523 }
2524 }
2525
2526 fn invalid(message: &str) -> Self {
2527 Self {
2528 kind: ProviderCandidateErrorKind::InvalidDocument,
2529 message: message.to_string(),
2530 }
2531 }
2532
2533 fn missing_section(message: &str) -> Self {
2534 Self {
2535 kind: ProviderCandidateErrorKind::Missing,
2536 message: message.to_string(),
2537 }
2538 }
2539
2540 fn runtime() -> Self {
2541 Self {
2542 kind: ProviderCandidateErrorKind::Runtime,
2543 message: "provider runtime initialization failed".to_string(),
2544 }
2545 }
2546}
2547
2548fn candidate_error_status(kind: &ProviderCandidateErrorKind) -> SectionStatus {
2549 match kind {
2550 ProviderCandidateErrorKind::Missing => SectionStatus::Missing,
2551 ProviderCandidateErrorKind::InvalidDocument => SectionStatus::Invalid,
2552 ProviderCandidateErrorKind::Runtime => SectionStatus::Degraded,
2553 }
2554}
2555
2556fn update_live_health(
2557 health: &std::sync::RwLock<ConfigLiveHealth>,
2558 status: SectionStatus,
2559 last_error: Option<String>,
2560 advance_revision: bool,
2561 source: Option<(PathBuf, SectionSourceKind)>,
2562) -> u64 {
2563 let mut health = health
2564 .write()
2565 .unwrap_or_else(|poisoned| poisoned.into_inner());
2566 if advance_revision {
2567 health.revision = health.revision.saturating_add(1);
2568 }
2569 health.loaded_at = Utc::now();
2570 health.status = status;
2571 health.last_error = last_error;
2572 if let Some((source_path, source_kind)) = source {
2573 health.source_path = source_path;
2574 health.source_kind = source_kind;
2575 }
2576 health.revision
2577}
2578
2579fn set_live_health_revision(
2580 health: &std::sync::RwLock<ConfigLiveHealth>,
2581 revision: u64,
2582 source: Option<(PathBuf, SectionSourceKind)>,
2583) -> u64 {
2584 let mut health = health
2585 .write()
2586 .unwrap_or_else(|poisoned| poisoned.into_inner());
2587 health.revision = revision;
2588 health.loaded_at = Utc::now();
2589 health.status = SectionStatus::Healthy;
2590 health.last_error = None;
2591 if let Some((source_path, source_kind)) = source {
2592 health.source_path = source_path;
2593 health.source_kind = source_kind;
2594 }
2595 revision
2596}
2597
2598fn set_live_health_from_snapshot<T>(
2599 health: &std::sync::RwLock<ConfigLiveHealth>,
2600 snapshot: &bamboo_config::SectionSnapshot<T>,
2601) {
2602 let mut health = health
2603 .write()
2604 .unwrap_or_else(|poisoned| poisoned.into_inner());
2605 health.revision = snapshot.revision;
2606 health.loaded_at = snapshot.loaded_at;
2607 health.source_path = snapshot.source_path.clone();
2608 health.source_kind = snapshot.source_kind;
2609 health.status = snapshot.status;
2610 health.last_error = snapshot.last_error.clone();
2611}
2612
2613#[derive(Debug)]
2614pub(crate) enum ConfigSectionMutationError {
2615 Store(ConfigStoreError),
2616 Invalid(String),
2617 Runtime(String),
2618}
2619
2620pub(crate) enum CredentialBackedResetCommit {
2621 Section(bamboo_config::SectionEnvelope<Value>),
2622 Cluster(Box<bamboo_server_tools::FabricCommitSnapshot>),
2623}
2624
2625impl AppState {
2626 #[cfg(test)]
2627 pub(crate) fn stop_config_watcher_for_test(&mut self) {
2628 self.config_watcher.stop.store(true, Ordering::Relaxed);
2629 if let Some(task) = self.config_watcher.apply_task.take() {
2630 task.abort();
2631 }
2632 if let Some(task) = self.config_watcher.watcher_task.take() {
2633 let _ = task.join();
2634 }
2635 }
2636
2637 #[cfg(test)]
2638 pub(crate) async fn reload_ordinary_section_for_test(&self, id: SectionId) {
2639 let _io = self.config_io_lock.lock().await;
2640 let facade = self
2641 .config_facade
2642 .as_ref()
2643 .expect("test ordinary reload requires the modular facade");
2644 let mut synthetic_events = BTreeMap::new();
2645 let mut pending_root_publications = BTreeMap::new();
2646 let mut reported_root_runtime_failures = BTreeSet::new();
2647 reload_and_apply_ordinary_sections(
2648 &self.app_data_dir,
2649 &self.config,
2650 facade,
2651 &self.account_sink,
2652 std::iter::once(id),
2653 OrdinarySectionReloadState {
2654 synthetic_events: &mut synthetic_events,
2655 pending_root_publications: &mut pending_root_publications,
2656 reported_root_runtime_failures: &mut reported_root_runtime_failures,
2657 },
2658 )
2659 .await;
2660 }
2661
2662 pub(crate) async fn read_exact_credential_section(
2667 &self,
2668 section: SectionId,
2669 ) -> Result<ExactCredentialSectionSnapshot, AppError> {
2670 let _io = self.config_io_lock.lock().await;
2671 let data_dir = self.app_data_dir.clone();
2672 let exact = tokio::task::spawn_blocking(move || {
2673 bamboo_config::read_exact_credential_section_snapshot(data_dir, section, None)
2674 })
2675 .await
2676 .map_err(|error| {
2677 AppError::InternalError(anyhow::anyhow!(
2678 "{} exact read snapshot task failed: {error}",
2679 section.descriptor().name
2680 ))
2681 })?
2682 .map_err(map_exact_credential_store_error)?;
2683 let envelope = exact.section.clone();
2684 let mut config = Config::default();
2685 let metadata = exact.install_into(&mut config);
2686 Ok(ExactCredentialSectionSnapshot {
2687 config,
2688 section: envelope,
2689 metadata,
2690 })
2691 }
2692
2693 pub(crate) async fn put_ordinary_section(
2697 &self,
2698 id: SectionId,
2699 expected_revision: u64,
2700 candidate: Value,
2701 ) -> Result<bamboo_config::SectionEnvelope<Value>, ConfigSectionMutationError> {
2702 if matches!(
2703 id,
2704 SectionId::Providers
2705 | SectionId::Mcp
2706 | SectionId::Credentials
2707 | SectionId::ClusterFabric
2708 ) {
2709 return Err(ConfigSectionMutationError::Invalid(
2710 "this section requires its dedicated endpoint".to_string(),
2711 ));
2712 }
2713 let _io = self.config_io_lock.lock().await;
2714 ensure_provider_mcp_migration_ready(&self.app_data_dir)
2715 .map_err(ConfigSectionMutationError::Store)?;
2716 let facade = self.config_facade.as_ref().ok_or_else(|| {
2717 ConfigSectionMutationError::Invalid(
2718 "typed section writes require the modular configuration facade".to_string(),
2719 )
2720 })?;
2721 let current = if id == SectionId::Core {
2722 let data_dir = self.app_data_dir.clone();
2723 let exact = tokio::task::spawn_blocking(move || {
2724 bamboo_config::read_exact_credential_section_snapshot(
2725 data_dir,
2726 SectionId::Core,
2727 Some(expected_revision),
2728 )
2729 })
2730 .await
2731 .map_err(|error| {
2732 ConfigSectionMutationError::Runtime(format!(
2733 "Core exact inventory snapshot task failed: {error}"
2734 ))
2735 })?
2736 .map_err(ConfigSectionMutationError::Store)?;
2737 if let Some(reference) = exact
2738 .section
2739 .data
2740 .get("proxy_auth_credential_ref")
2741 .and_then(Value::as_str)
2742 {
2743 let configured = exact
2744 .credential_statuses
2745 .iter()
2746 .any(|status| status.credential_ref.as_str() == reference && status.configured);
2747 if !configured {
2748 return Err(ConfigSectionMutationError::Invalid(
2749 "the active Core proxy credential is invalid; explicitly replace or clear it through the proxy-auth API"
2750 .to_string(),
2751 ));
2752 }
2753 }
2754 exact.section
2755 } else {
2756 facade
2757 .registry()
2758 .envelope_value(id)
2759 .map_err(ConfigSectionMutationError::Store)?
2760 };
2761 if credential_reference_inventory(¤t.data)
2762 != credential_reference_inventory(&candidate)
2763 {
2764 return Err(ConfigSectionMutationError::Invalid(
2765 "credential references are server-managed; use the credential or domain API"
2766 .to_string(),
2767 ));
2768 }
2769
2770 let (event, committed) = if id == SectionId::Core {
2771 bamboo_config::commit_core_metadata_from_durable_base(
2777 &self.app_data_dir,
2778 facade,
2779 expected_revision,
2780 candidate,
2781 )
2782 } else {
2783 facade
2784 .registry()
2785 .commit_value_with_envelope(id, expected_revision, candidate)
2786 }
2787 .map_err(ConfigSectionMutationError::Store)?;
2788 let materialized = materialize_facade_effective_config(facade, &self.app_data_dir);
2789 if materialized.failures.contains(&id) {
2790 let message =
2791 "configuration runtime hydration failed; retaining last-known-good runtime"
2792 .to_string();
2793 if let Some(invalid) = facade.registry().mark_runtime_degraded(id, message.clone()) {
2794 publish_registry_event(&self.account_sink, &invalid).await;
2795 }
2796 return Err(ConfigSectionMutationError::Runtime(message));
2797 }
2798
2799 let mut live = self.config.read().await.clone();
2800 let enforcement_newly_off = id == SectionId::ToolsSkills
2801 && !live.plugin_trust.enforcement_is_off()
2802 && materialized.config.plugin_trust.enforcement_is_off();
2803 apply_runtime_section(id, &materialized.config, &mut live);
2804 if id == SectionId::Env {
2805 live.publish_env_vars();
2806 }
2807 *self.config.write().await = live;
2808 if enforcement_newly_off {
2809 warn_plugin_trust_enforcement_off();
2810 }
2811 publish_registry_event(&self.account_sink, &event).await;
2812 Ok(committed)
2813 }
2814
2815 pub(crate) async fn put_provider_section(
2818 &self,
2819 expected_revision: u64,
2820 mut providers: ProviderConfigs,
2821 ) -> Result<u64, ConfigSectionMutationError> {
2822 let _io = self.config_io_lock.lock().await;
2823 ensure_provider_mcp_migration_ready(&self.app_data_dir)
2824 .map_err(ConfigSectionMutationError::Store)?;
2825 let current = self.config.read().await.clone();
2826 retain_provider_credentials(current.providers(), &mut providers);
2827 let mut candidate = current;
2828 *candidate.providers_mut() = providers.clone();
2829 let (candidate, registry, provider) =
2830 match prepare_provider_candidate(candidate, &self.app_data_dir).await {
2831 Ok(prepared) => prepared,
2832 Err(_) => {
2833 let message =
2834 "provider runtime initialization failed; retaining last-known-good runtime"
2835 .to_string();
2836 publish_section_failure(
2837 &self.config_live_health,
2838 &self.account_sink,
2839 "providers",
2840 SectionStatus::Degraded,
2841 message.clone(),
2842 )
2843 .await;
2844 return Err(ConfigSectionMutationError::Runtime(message));
2845 }
2846 };
2847
2848 let durable_providers = provider_durable_document(&providers)?;
2849 let mut live_config = self.config.write().await;
2853 let mut live_provider = self.provider.write().await;
2854 let (revision, source_path) = if let Some(facade) = self.config_facade.as_ref() {
2855 let mut section = facade.registry().providers.snapshot().data.as_ref().clone();
2856 section.providers = durable_providers;
2857 let event = facade
2858 .registry()
2859 .providers
2860 .commit(expected_revision, section)
2861 .map_err(ConfigSectionMutationError::Store)?;
2862 let ConfigSectionEvent::Changed { revision, .. } = event else {
2863 unreachable!("a successful section commit is changed")
2864 };
2865 (
2866 revision,
2867 facade.registry().providers.snapshot().source_path.clone(),
2868 )
2869 } else {
2870 let store = AtomicJsonStore::new(self.app_data_dir.join("providers.json"), 1);
2871 let revision = store
2872 .commit_allowing_unversioned(
2873 expected_revision,
2874 durable_providers,
2875 validate_provider_config,
2876 )
2877 .map_err(ConfigSectionMutationError::Store)?;
2878 (revision, store.path().to_path_buf())
2879 };
2880
2881 candidate.publish_env_vars();
2882 *live_config = candidate;
2883 self.provider_registry.replace_with(registry);
2884 *live_provider = provider;
2885 publish_section_success(
2886 &self.config_live_health,
2887 &self.account_sink,
2888 "providers",
2889 source_path,
2890 section_is_unhealthy(&self.config_live_health),
2891 Some(revision),
2892 )
2893 .await;
2894 Ok(revision)
2895 }
2896
2897 pub(crate) async fn put_provider_settings<F>(
2902 &self,
2903 expected_revision: u64,
2904 update: F,
2905 ) -> Result<u64, ConfigSectionMutationError>
2906 where
2907 F: FnOnce(
2908 &Config,
2909 &mut Config,
2910 )
2911 -> Result<(BTreeSet<String>, BTreeSet<String>), ConfigSectionMutationError>
2912 + Send
2913 + 'static,
2914 {
2915 let config_io_lock = self.config_io_lock.clone();
2916 let config = self.config.clone();
2917 let app_data_dir = self.app_data_dir.clone();
2918 let config_facade = self.config_facade.clone();
2919 let account_sink = self.account_sink.clone();
2920 let provider_registry = self.provider_registry.clone();
2921 let provider = self.provider.clone();
2922 let config_live_health = self.config_live_health.clone();
2923 let transaction = tokio::spawn(async move {
2924 let _io = config_io_lock.lock().await;
2925 ensure_provider_mcp_migration_ready(&app_data_dir)
2926 .map_err(ConfigSectionMutationError::Store)?;
2927 let facade = config_facade.as_ref().ok_or_else(|| {
2928 ConfigSectionMutationError::Invalid(
2929 "provider settings require the modular configuration facade".to_string(),
2930 )
2931 })?;
2932 let current = config.read().await.clone();
2933 let mut candidate = current.clone();
2934 let (provider_intents, provider_instance_intents) = update(¤t, &mut candidate)?;
2935
2936 let (candidate, registry, candidate_provider) =
2937 match prepare_provider_candidate(candidate, &app_data_dir).await {
2938 Ok(prepared) => prepared,
2939 Err(_) => {
2940 let message =
2941 "provider runtime initialization failed; retaining last-known-good runtime"
2942 .to_string();
2943 publish_section_failure(
2944 &config_live_health,
2945 &account_sink,
2946 "providers",
2947 SectionStatus::Degraded,
2948 message.clone(),
2949 )
2950 .await;
2951 return Err(ConfigSectionMutationError::Runtime(message));
2952 }
2953 };
2954
2955 let mut live_config = config.write().await;
2960 let mut live_provider = provider.write().await;
2961 let transaction_dir = app_data_dir.clone();
2962 let commit_facade = facade.clone();
2963 let (mut committed, commit) = tokio::task::spawn_blocking(move || {
2964 let mut durable_candidate = candidate;
2965 let commit =
2966 bamboo_config::persist_provider_credential_transaction_at_revision_with_adoption(
2967 &transaction_dir,
2968 &mut durable_candidate,
2969 &provider_intents,
2970 &provider_instance_intents,
2971 expected_revision,
2972 commit_facade.as_ref(),
2973 )?;
2974 Ok::<_, ConfigStoreError>((durable_candidate, commit))
2975 })
2976 .await
2977 .map_err(|error| {
2978 ConfigSectionMutationError::Runtime(format!(
2979 "provider settings transaction task failed: {error}"
2980 ))
2981 })?
2982 .map_err(ConfigSectionMutationError::Store)?;
2983
2984 let installed = install_credential_section_commit(commit, &mut committed)
2985 .map_err(ConfigSectionMutationError::Store)?;
2986 committed.publish_env_vars();
2987 *live_config = committed;
2988 provider_registry.replace_with(registry);
2989 *live_provider = candidate_provider;
2990 publish_exact_facade_events(&account_sink, &installed.events)
2991 .await
2992 .map_err(|error| ConfigSectionMutationError::Runtime(error.to_string()))?;
2993 let provider_snapshot = facade.registry().providers.snapshot();
2994 set_live_health_revision(
2995 &config_live_health,
2996 provider_snapshot.revision,
2997 Some((
2998 provider_snapshot.source_path.clone(),
2999 SectionSourceKind::File,
3000 )),
3001 );
3002 Ok(provider_snapshot.revision)
3003 });
3004 transaction.await.map_err(|error| {
3005 ConfigSectionMutationError::Runtime(format!(
3006 "provider settings transaction task failed: {error}"
3007 ))
3008 })?
3009 }
3010
3011 pub(crate) async fn reset_provider_section(
3016 &self,
3017 expected_revision: u64,
3018 ) -> Result<u64, ConfigSectionMutationError> {
3019 let config_io_lock = self.config_io_lock.clone();
3020 let config = self.config.clone();
3021 let app_data_dir = self.app_data_dir.clone();
3022 let config_facade = self.config_facade.clone();
3023 let account_sink = self.account_sink.clone();
3024 let provider_registry = self.provider_registry.clone();
3025 let provider = self.provider.clone();
3026 let config_live_health = self.config_live_health.clone();
3027 let transaction = tokio::spawn(async move {
3028 let _io = config_io_lock.lock().await;
3029 ensure_provider_mcp_migration_ready(&app_data_dir)
3030 .map_err(ConfigSectionMutationError::Store)?;
3031 let facade = config_facade.as_ref().ok_or_else(|| {
3032 ConfigSectionMutationError::Invalid(
3033 "provider reset requires the modular configuration facade".to_string(),
3034 )
3035 })?;
3036 let current = config.read().await.clone();
3037 let provider_intents = BTreeSet::from([
3038 "openai".to_string(),
3039 "anthropic".to_string(),
3040 "gemini".to_string(),
3041 "bodhi".to_string(),
3042 ]);
3043 let provider_instance_intents = current.provider_instances.keys().cloned().collect();
3044 let mut candidate = current;
3045 apply_runtime_section(SectionId::Providers, &Config::default(), &mut candidate);
3046
3047 let transaction_dir = app_data_dir.clone();
3048 let commit_facade = facade.clone();
3049 let (mut candidate, commit) = tokio::task::spawn_blocking(move || {
3050 let commit =
3051 bamboo_config::persist_provider_reset_credential_transaction_at_revision_with_adoption(
3052 &transaction_dir,
3053 &mut candidate,
3054 &provider_intents,
3055 &provider_instance_intents,
3056 expected_revision,
3057 commit_facade.as_ref(),
3058 )?;
3059 Ok::<_, ConfigStoreError>((candidate, commit))
3060 })
3061 .await
3062 .map_err(|error| {
3063 ConfigSectionMutationError::Runtime(format!(
3064 "provider reset transaction task failed: {error}"
3065 ))
3066 })?
3067 .map_err(ConfigSectionMutationError::Store)?;
3068
3069 let installed = install_credential_section_commit(commit, &mut candidate)
3070 .map_err(ConfigSectionMutationError::Store)?;
3071 *config.write().await = candidate.clone();
3072 let provider_snapshot = facade.registry().providers.snapshot();
3073 set_live_health_revision(
3074 &config_live_health,
3075 provider_snapshot.revision,
3076 Some((
3077 provider_snapshot.source_path.clone(),
3078 SectionSourceKind::File,
3079 )),
3080 );
3081 publish_exact_facade_events(&account_sink, &installed.events)
3082 .await
3083 .map_err(|error| ConfigSectionMutationError::Runtime(error.to_string()))?;
3084
3085 let runtime_failure = match bamboo_llm::ProviderRegistry::from_config(
3086 &candidate,
3087 app_data_dir,
3088 )
3089 .await
3090 {
3091 Ok(registry) => {
3092 if let Some(candidate_provider) = registry.get_default() {
3093 provider_registry.replace_with(registry);
3094 *provider.write().await = candidate_provider;
3095 None
3096 } else {
3097 Some(
3098 "provider reset committed; default provider is not initialized"
3099 .to_string(),
3100 )
3101 }
3102 }
3103 Err(error) => {
3104 tracing::warn!(error = %error, "provider reset committed but runtime initialization failed");
3105 Some("provider reset committed; retaining last-known-good runtime".to_string())
3106 }
3107 };
3108 if let Some(message) = runtime_failure {
3109 publish_section_failure(
3110 &config_live_health,
3111 &account_sink,
3112 "providers",
3113 SectionStatus::Degraded,
3114 message,
3115 )
3116 .await;
3117 }
3118 Ok(provider_snapshot.revision)
3119 });
3120 transaction.await.map_err(|error| {
3121 ConfigSectionMutationError::Runtime(format!(
3122 "provider reset transaction task failed: {error}"
3123 ))
3124 })?
3125 }
3126
3127 pub(crate) async fn put_mcp_section(
3131 &self,
3132 expected_revision: u64,
3133 mut candidate: McpConfig,
3134 ) -> Result<u64, ConfigSectionMutationError> {
3135 let _io = self.config_io_lock.lock().await;
3136 ensure_provider_mcp_migration_ready(&self.app_data_dir)
3137 .map_err(ConfigSectionMutationError::Store)?;
3138 retain_mcp_credentials(
3139 &self.config.read().await.mcp,
3140 &mut candidate,
3141 &BTreeSet::new(),
3142 );
3143 validate_mcp_config(&candidate).map_err(ConfigSectionMutationError::Invalid)?;
3144 let mut hydration_config = Config::default();
3145 hydration_config.mcp = candidate;
3146 hydration_config
3147 .hydrate_mcp_credentials_from_store(&self.app_data_dir)
3148 .map_err(|_| {
3149 ConfigSectionMutationError::Invalid(
3150 "referenced MCP credential is unavailable".to_string(),
3151 )
3152 })?;
3153 let candidate = hydration_config.mcp.clone();
3154 let mut revision = None;
3155 let mut store_error = None;
3156 let durable_candidate = credential_ref_mcp_document(&candidate)?;
3157 let mut next_config = candidate.clone();
3158 retain_mcp_credential_refs(&durable_candidate, &mut next_config);
3159 let result = self
3160 .mcp_manager
3161 .reconcile_from_config_transactional_after(&candidate, || async {
3162 let mut live_config = self.config.write().await;
3166 let commit = if let Some(facade) = self.config_facade.as_ref() {
3167 facade
3168 .registry()
3169 .mcp
3170 .commit(expected_revision, McpSection(durable_candidate))
3171 .map(|event| match event {
3172 ConfigSectionEvent::Changed { revision, .. } => revision,
3173 _ => unreachable!("a successful section commit is changed"),
3174 })
3175 } else {
3176 AtomicJsonStore::new(self.app_data_dir.join("mcp.json"), 1).commit(
3177 expected_revision,
3178 durable_candidate,
3179 validate_mcp_config,
3180 )
3181 };
3182 match commit {
3183 Ok(committed) => {
3184 live_config.mcp = next_config;
3185 revision = Some(committed);
3186 Ok(())
3187 }
3188 Err(error) => {
3189 store_error = Some(error);
3190 Err(bamboo_mcp::McpError::InvalidConfig(
3191 "MCP section durable commit failed".to_string(),
3192 ))
3193 }
3194 }
3195 })
3196 .await;
3197 if let Some(error) = store_error {
3198 return Err(ConfigSectionMutationError::Store(error));
3199 }
3200 if result.is_err() {
3201 let message =
3202 "MCP runtime initialization failed; retaining last-known-good runtime".to_string();
3203 publish_section_failure(
3204 &self.mcp_config_live_health,
3205 &self.account_sink,
3206 "mcp",
3207 SectionStatus::Degraded,
3208 message.clone(),
3209 )
3210 .await;
3211 return Err(ConfigSectionMutationError::Runtime(message));
3212 }
3213 let revision = revision.expect("successful MCP reconcile commits a revision");
3214 publish_section_success(
3215 &self.mcp_config_live_health,
3216 &self.account_sink,
3217 "mcp",
3218 self.app_data_dir.join("mcp.json"),
3219 section_is_unhealthy(&self.mcp_config_live_health),
3220 Some(revision),
3221 )
3222 .await;
3223 Ok(revision)
3224 }
3225
3226 pub(crate) async fn put_mcp_settings(
3230 &self,
3231 expected_revision: u64,
3232 candidate: McpConfig,
3233 credential_intents: BTreeSet<bamboo_config::CredentialRef>,
3234 ) -> Result<u64, ConfigSectionMutationError> {
3235 if credential_intents.is_empty() {
3236 return self.put_mcp_section(expected_revision, candidate).await;
3237 }
3238
3239 let config_io_lock = self.config_io_lock.clone();
3240 let config = self.config.clone();
3241 let app_data_dir = self.app_data_dir.clone();
3242 let config_facade = self.config_facade.clone();
3243 let account_sink = self.account_sink.clone();
3244 let mcp_manager = self.mcp_manager.clone();
3245 let mcp_config_live_health = self.mcp_config_live_health.clone();
3246 let transaction = tokio::spawn(async move {
3247 let _io = config_io_lock.lock().await;
3248 ensure_provider_mcp_migration_ready(&app_data_dir)
3249 .map_err(ConfigSectionMutationError::Store)?;
3250 let facade = config_facade.as_ref().ok_or_else(|| {
3251 ConfigSectionMutationError::Invalid(
3252 "MCP settings require the modular configuration facade".to_string(),
3253 )
3254 })?;
3255 let current = config.read().await.clone();
3256 let mut runtime_candidate = candidate;
3257 materialize_mcp_touched_replacements(&mut runtime_candidate, &credential_intents)
3258 .map_err(ConfigSectionMutationError::Invalid)?;
3259 retain_mcp_credentials(¤t.mcp, &mut runtime_candidate, &credential_intents);
3260 validate_mcp_config(&runtime_candidate).map_err(ConfigSectionMutationError::Invalid)?;
3261
3262 let mut transaction_error = None;
3263 let mut commit_events = Vec::new();
3264 let transaction_dir = app_data_dir.clone();
3265 let commit_facade = facade.clone();
3266 let mut durable_candidate = current;
3267 durable_candidate.mcp = runtime_candidate.clone();
3268 let result = mcp_manager
3269 .reconcile_from_config_transactional_after(&runtime_candidate, || async {
3270 let mut live_config = config.write().await;
3271 let commit = tokio::task::spawn_blocking(move || {
3272 let commit =
3273 bamboo_config::persist_mcp_credential_transaction_at_revision_with_adoption(
3274 &transaction_dir,
3275 &mut durable_candidate,
3276 &credential_intents,
3277 expected_revision,
3278 commit_facade.as_ref(),
3279 )?;
3280 Ok::<_, ConfigStoreError>((durable_candidate, commit))
3281 })
3282 .await;
3283 match commit {
3284 Ok(Ok((mut committed, commit))) => {
3285 let installed =
3286 match install_credential_section_commit(commit, &mut committed) {
3287 Ok(installed) => installed,
3288 Err(error) => {
3289 transaction_error =
3290 Some(ConfigSectionMutationError::Store(error));
3291 return Err(bamboo_mcp::McpError::InvalidConfig(
3292 "MCP settings process adoption failed".to_string(),
3293 ));
3294 }
3295 };
3296 *live_config = committed;
3297 commit_events = installed.events;
3298 Ok(())
3299 }
3300 Ok(Err(error)) => {
3301 transaction_error = Some(ConfigSectionMutationError::Store(error));
3302 Err(bamboo_mcp::McpError::InvalidConfig(
3303 "MCP settings durable transaction failed".to_string(),
3304 ))
3305 }
3306 Err(error) => {
3307 transaction_error = Some(ConfigSectionMutationError::Runtime(format!(
3308 "MCP settings transaction task failed: {error}"
3309 )));
3310 Err(bamboo_mcp::McpError::InvalidConfig(
3311 "MCP settings durable transaction failed".to_string(),
3312 ))
3313 }
3314 }
3315 })
3316 .await;
3317 if let Some(error) = transaction_error {
3318 return Err(error);
3319 }
3320 if result.is_err() {
3321 let message =
3322 "MCP runtime initialization failed; retaining last-known-good runtime"
3323 .to_string();
3324 publish_section_failure(
3325 &mcp_config_live_health,
3326 &account_sink,
3327 "mcp",
3328 SectionStatus::Degraded,
3329 message.clone(),
3330 )
3331 .await;
3332 return Err(ConfigSectionMutationError::Runtime(message));
3333 }
3334
3335 publish_exact_facade_events(&account_sink, &commit_events)
3336 .await
3337 .map_err(|error| ConfigSectionMutationError::Runtime(error.to_string()))?;
3338 let snapshot = facade.registry().mcp.snapshot();
3339 set_live_health_revision(
3340 &mcp_config_live_health,
3341 snapshot.revision,
3342 Some((snapshot.source_path.clone(), SectionSourceKind::File)),
3343 );
3344 Ok(snapshot.revision)
3345 });
3346 transaction.await.map_err(|error| {
3347 ConfigSectionMutationError::Runtime(format!(
3348 "MCP settings transaction task failed: {error}"
3349 ))
3350 })?
3351 }
3352
3353 pub(crate) async fn update_legacy_mcp_config<F>(
3363 &self,
3364 force_restart: BTreeSet<String>,
3365 update: F,
3366 ) -> Result<Config, AppError>
3367 where
3368 F: FnOnce(&mut McpConfig) -> Result<(), AppError>,
3369 {
3370 let io = self.config_io_lock.clone().lock_owned().await;
3373 let (mut candidate_config, expected_revision, credential_intents) = {
3374 ensure_provider_mcp_migration_ready(&self.app_data_dir)
3375 .map_err(map_exact_credential_store_error)?;
3376 let facade = self.config_facade.as_ref().ok_or_else(|| {
3377 AppError::BadRequest(
3378 "legacy MCP mutations require the modular configuration facade".to_string(),
3379 )
3380 })?;
3381 let current = self.config.read().await.clone();
3382 reject_if_recovery_pending(¤t)?;
3383 let mut candidate = current.mcp.clone();
3384 update(&mut candidate)?;
3385 let credential_intents =
3386 normalize_legacy_mcp_credentials(¤t.mcp, &mut candidate)?;
3387 validate_mcp_config(&candidate).map_err(AppError::BadRequest)?;
3388 let expected_revision = facade.registry().mcp.snapshot().revision;
3389 let mut candidate_config = current;
3390 candidate_config.mcp = candidate;
3391 (candidate_config, expected_revision, credential_intents)
3392 };
3393
3394 let config = self.config.clone();
3395 let app_data_dir = self.app_data_dir.clone();
3396 let config_facade = self.config_facade.clone();
3397 let account_sink = self.account_sink.clone();
3398 let mcp_manager = self.mcp_manager.clone();
3399 let mcp_config_live_health = self.mcp_config_live_health.clone();
3400 let transaction = tokio::spawn(async move {
3401 let _io = io;
3402 let facade = config_facade.expect("validated modular configuration facade");
3403 let runtime_candidate = candidate_config.mcp.clone();
3404 let force_replacements = force_restart.into_iter().collect();
3405 let durable_document =
3406 credential_ref_mcp_document(&runtime_candidate).map_err(map_mcp_section_error)?;
3407 bamboo_config::validate_mcp_section(&durable_document).map_err(AppError::BadRequest)?;
3408 let current_document = facade.registry().mcp.snapshot();
3409 let metadata_changed = serde_json::to_value(¤t_document.data.0)
3410 .map_err(AppError::SerializationError)?
3411 != serde_json::to_value(&durable_document).map_err(AppError::SerializationError)?;
3412 let credential_transaction = !credential_intents.is_empty();
3413 let mut transaction_error = None;
3414 let mut commit_events = Vec::new();
3415 let mut published_config = None;
3416 let commit_dir = app_data_dir.clone();
3417 let commit_facade = facade.clone();
3418 let result = mcp_manager
3419 .reconcile_from_config_transactional_after_forcing(
3420 &runtime_candidate,
3421 &force_replacements,
3422 || async {
3423 let mut live_config = config.write().await;
3427 if credential_transaction {
3428 let commit = tokio::task::spawn_blocking(move || {
3429 let commit = bamboo_config::persist_mcp_credential_transaction_at_revision_with_adoption(
3430 &commit_dir,
3431 &mut candidate_config,
3432 &credential_intents,
3433 expected_revision,
3434 commit_facade.as_ref(),
3435 )?;
3436 Ok::<_, ConfigStoreError>((candidate_config, commit))
3437 })
3438 .await;
3439 match commit {
3440 Ok(Ok((mut committed, commit))) => {
3441 #[cfg(test)]
3442 run_credential_after_commit_before_live_test_hook(
3443 &app_data_dir,
3444 SectionId::Mcp,
3445 );
3446 match install_credential_section_commit(commit, &mut committed) {
3447 Ok(installed) => {
3448 *live_config = committed.clone();
3449 commit_events = installed.events;
3450 published_config = Some(committed);
3451 }
3452 Err(error) => {
3453 transaction_error =
3454 Some(map_exact_credential_store_error(error));
3455 return Err(bamboo_mcp::McpError::InvalidConfig(
3456 "MCP process adoption failed".to_string(),
3457 ));
3458 }
3459 }
3460 }
3461 Ok(Err(error)) => {
3462 transaction_error =
3463 Some(map_exact_credential_store_error(error));
3464 return Err(bamboo_mcp::McpError::InvalidConfig(
3465 "MCP durable transaction failed".to_string(),
3466 ));
3467 }
3468 Err(error) => {
3469 transaction_error = Some(AppError::InternalError(
3470 anyhow::anyhow!(
3471 "MCP credential transaction task failed: {error}"
3472 ),
3473 ));
3474 return Err(bamboo_mcp::McpError::InvalidConfig(
3475 "MCP durable transaction failed".to_string(),
3476 ));
3477 }
3478 }
3479 } else {
3480 if metadata_changed {
3481 match facade
3482 .registry()
3483 .mcp
3484 .commit(expected_revision, McpSection(durable_document))
3485 {
3486 Ok(event) => commit_events.push(event),
3487 Err(error) => {
3488 transaction_error =
3489 Some(map_exact_credential_store_error(error));
3490 return Err(bamboo_mcp::McpError::InvalidConfig(
3491 "MCP durable commit failed".to_string(),
3492 ));
3493 }
3494 }
3495 }
3496 candidate_config.mcp = runtime_candidate.clone();
3497 *live_config = candidate_config.clone();
3498 published_config = Some(candidate_config);
3499 }
3500 Ok(())
3501 },
3502 )
3503 .await;
3504
3505 if let Some(error) = transaction_error {
3506 return Err(error);
3507 }
3508 if result.is_err() {
3509 tracing::warn!("legacy MCP runtime staging failed before durable commit");
3510 let message =
3511 "MCP runtime initialization failed before commit; retaining last-known-good generation"
3512 .to_string();
3513 return Err(AppError::InternalError(anyhow::anyhow!(message)));
3514 }
3515
3516 publish_exact_facade_events(&account_sink, &commit_events).await?;
3520 let snapshot = facade.registry().mcp.snapshot();
3521 set_live_health_revision(
3522 &mcp_config_live_health,
3523 snapshot.revision,
3524 Some((snapshot.source_path.clone(), SectionSourceKind::File)),
3525 );
3526 Ok::<_, AppError>(
3527 published_config.expect("successful MCP transaction publishes config"),
3528 )
3529 });
3530 transaction.await.map_err(|error| {
3531 AppError::InternalError(anyhow::anyhow!(
3532 "legacy MCP config transaction task failed: {error}"
3533 ))
3534 })?
3535 }
3536
3537 pub(crate) async fn reset_mcp_section(
3541 &self,
3542 expected_revision: u64,
3543 ) -> Result<u64, ConfigSectionMutationError> {
3544 let _io = self.config_io_lock.lock().await;
3545 ensure_provider_mcp_migration_ready(&self.app_data_dir)
3546 .map_err(ConfigSectionMutationError::Store)?;
3547 let facade = self.config_facade.as_ref().ok_or_else(|| {
3548 ConfigSectionMutationError::Invalid(
3549 "MCP reset requires the modular configuration facade".to_string(),
3550 )
3551 })?;
3552 let candidate_mcp = McpConfig::default();
3553 let mut candidate_config = self.config.read().await.clone();
3554 candidate_config.mcp = candidate_mcp.clone();
3555 let mut committed = false;
3556 let mut commit_events = Vec::new();
3557 let mut store_error = None;
3558 let data_dir = self.app_data_dir.clone();
3559 let result = self
3560 .mcp_manager
3561 .reconcile_from_config_transactional_after(&candidate_mcp, || async {
3562 let mut live_config = self.config.write().await;
3563 match bamboo_config::persist_mcp_reset_credential_transaction_at_revision_with_adoption(
3564 &data_dir,
3565 &mut candidate_config,
3566 expected_revision,
3567 facade.as_ref(),
3568 ) {
3569 Ok(commit) => {
3570 match install_credential_section_commit(commit, &mut candidate_config) {
3571 Ok(installed) => {
3572 *live_config = candidate_config.clone();
3573 commit_events = installed.events;
3574 committed = true;
3575 }
3576 Err(error) => {
3577 store_error = Some(error);
3578 return Err(bamboo_mcp::McpError::InvalidConfig(
3579 "MCP reset process adoption failed".to_string(),
3580 ));
3581 }
3582 }
3583 Ok(())
3584 }
3585 Err(error) => {
3586 store_error = Some(error);
3587 Err(bamboo_mcp::McpError::InvalidConfig(
3588 "MCP reset durable commit failed".to_string(),
3589 ))
3590 }
3591 }
3592 })
3593 .await;
3594 if let Some(error) = store_error {
3595 return Err(ConfigSectionMutationError::Store(error));
3596 }
3597 if result.is_err() || !committed {
3598 let message =
3599 "MCP reset runtime initialization failed; retaining last-known-good runtime"
3600 .to_string();
3601 publish_section_failure(
3602 &self.mcp_config_live_health,
3603 &self.account_sink,
3604 "mcp",
3605 SectionStatus::Degraded,
3606 message.clone(),
3607 )
3608 .await;
3609 return Err(ConfigSectionMutationError::Runtime(message));
3610 }
3611 publish_exact_facade_events(&self.account_sink, &commit_events)
3612 .await
3613 .map_err(|error| ConfigSectionMutationError::Runtime(error.to_string()))?;
3614 let revision = facade.registry().mcp.snapshot().revision;
3615 publish_section_success(
3616 &self.mcp_config_live_health,
3617 &self.account_sink,
3618 "mcp",
3619 self.app_data_dir.join("mcp.json"),
3620 section_is_unhealthy(&self.mcp_config_live_health),
3621 Some(revision),
3622 )
3623 .await;
3624 Ok(revision)
3625 }
3626
3627 pub(crate) async fn reset_credential_backed_section(
3632 &self,
3633 id: SectionId,
3634 expected_revision: u64,
3635 ) -> Result<CredentialBackedResetCommit, ConfigSectionMutationError> {
3636 if !matches!(
3637 id,
3638 SectionId::Core
3639 | SectionId::Notifications
3640 | SectionId::Connect
3641 | SectionId::Env
3642 | SectionId::ClusterFabric
3643 | SectionId::AccessControl
3644 ) {
3645 return Err(ConfigSectionMutationError::Invalid(
3646 "section is not a credential-backed reset domain".to_string(),
3647 ));
3648 }
3649 let config_io_lock = self.config_io_lock.clone();
3650 let config = self.config.clone();
3651 let app_data_dir = self.app_data_dir.clone();
3652 let config_facade = self.config_facade.clone();
3653 let account_sink = self.account_sink.clone();
3654 let provider_registry = self.provider_registry.clone();
3655 let provider = self.provider.clone();
3656 let mcp_manager = self.mcp_manager.clone();
3657 let deployed_registry = self.fabric_deployer.registry();
3658 let transaction = tokio::spawn(async move {
3659 let _io = config_io_lock.lock().await;
3660 if id == SectionId::ClusterFabric {
3661 let deployed = deployed_registry.lock().await;
3662 if let Some(node_id) = deployed.keys().find_map(|key| {
3663 let (source, node_id) = bamboo_server_tools::registry_keys::split(key);
3664 (source == "node").then(|| node_id.to_string())
3665 }) {
3666 return Err(ConfigSectionMutationError::Invalid(format!(
3667 "node '{node_id}' is deployed; stop it before resetting cluster-fabric"
3668 )));
3669 }
3670 }
3671 ensure_provider_mcp_migration_ready(&app_data_dir)
3672 .map_err(ConfigSectionMutationError::Store)?;
3673 let facade = config_facade.as_ref().ok_or_else(|| {
3674 ConfigSectionMutationError::Invalid(
3675 "section reset requires the modular configuration facade".to_string(),
3676 )
3677 })?;
3678 let mut candidate = config.read().await.clone();
3679 apply_runtime_section(id, &Config::default(), &mut candidate);
3680 let transaction_dir = app_data_dir.clone();
3681 let commit_facade = facade.clone();
3682 let (mut candidate, revision, cluster_commit, section_commit) =
3683 tokio::task::spawn_blocking(move || {
3684 if id == SectionId::ClusterFabric {
3685 let commit =
3686 bamboo_config::persist_cluster_fabric_reset_at_revision_with_adoption(
3687 &transaction_dir,
3688 &mut candidate,
3689 expected_revision,
3690 commit_facade.as_ref(),
3691 |_, _| {},
3692 )?;
3693 let revision = commit.revision;
3694 Ok::<_, ConfigStoreError>((candidate, revision, Some(commit), None))
3695 } else {
3696 let commit =
3697 bamboo_config::persist_credential_backed_section_reset_at_revision_with_adoption(
3698 &transaction_dir,
3699 &mut candidate,
3700 id,
3701 expected_revision,
3702 commit_facade.as_ref(),
3703 )?;
3704 let revision = commit.revision;
3705 Ok((candidate, revision, None, Some(commit)))
3706 }
3707 })
3708 .await
3709 .map_err(|error| {
3710 ConfigSectionMutationError::Runtime(format!(
3711 "section reset transaction task failed: {error}"
3712 ))
3713 })?
3714 .map_err(ConfigSectionMutationError::Store)?;
3715
3716 let cluster_runtime = match cluster_commit {
3717 Some(commit) => {
3718 let bamboo_config::ClusterFabricTransactionCommit {
3719 revision: _,
3720 adoption,
3721 credential_adoption,
3722 committed_recovery,
3723 runtime,
3724 } = commit;
3725 let runtime = match runtime {
3726 Ok(bamboo_config::ClusterFabricRuntimeSnapshot {
3727 cluster_fabric,
3728 credential_statuses,
3729 credential_health,
3730 }) => {
3731 candidate.cluster_fabric = cluster_fabric;
3732 Ok((credential_statuses, credential_health))
3733 }
3734 Err(error) if revision == expected_revision => {
3735 return Err(ConfigSectionMutationError::Store(error));
3738 }
3739 Err(error) => {
3740 candidate.clear_cluster_runtime_credentials();
3745 Err(error)
3746 }
3747 };
3748 Some((adoption, credential_adoption, committed_recovery, runtime))
3749 }
3750 None => None,
3751 };
3752 let (section_events, exact_section) = match section_commit {
3753 Some(commit) => {
3754 let installed = install_credential_section_commit(commit, &mut candidate)
3755 .map_err(ConfigSectionMutationError::Store)?;
3756 (installed.events, installed.section)
3757 }
3758 None => (Vec::new(), None),
3759 };
3760 if id == SectionId::Env {
3761 candidate.publish_env_vars();
3762 }
3763 *config.write().await = candidate.clone();
3764 if id != SectionId::ClusterFabric {
3765 publish_exact_facade_events(&account_sink, §ion_events)
3766 .await
3767 .map_err(|error| ConfigSectionMutationError::Runtime(error.to_string()))?;
3768 }
3769 let commit = if id == SectionId::ClusterFabric {
3770 let (cluster_adoption, credential_adoption, committed_recovery, cluster_runtime) =
3771 cluster_runtime.expect("cluster reset captures an exact runtime");
3772 let event = match cluster_adoption {
3773 Some(Ok(event)) => Some(event),
3774 Some(Err(error)) => {
3775 return Err(ConfigSectionMutationError::Runtime(format!(
3776 "cluster reset committed at revision {revision} but process adoption failed: {error}"
3777 )));
3778 }
3779 None if revision == expected_revision => None,
3780 None => {
3781 return Err(ConfigSectionMutationError::Runtime(format!(
3782 "cluster reset committed at revision {revision} without a process adoption result"
3783 )));
3784 }
3785 };
3786 let section = facade
3787 .registry()
3788 .envelope_value(SectionId::ClusterFabric)
3789 .map_err(|error| {
3790 if revision == expected_revision {
3791 ConfigSectionMutationError::Store(error)
3792 } else {
3793 ConfigSectionMutationError::Runtime(format!(
3794 "cluster reset committed at revision {revision} but its exact envelope is unavailable: {error}"
3795 ))
3796 }
3797 })?;
3798 if section.revision != revision {
3799 return Err(ConfigSectionMutationError::Runtime(format!(
3800 "cluster reset committed at revision {revision} but facade retained revision {}",
3801 section.revision
3802 )));
3803 }
3804 if let Some(event) = event.as_ref() {
3805 publish_registry_event(&account_sink, event).await;
3806 }
3807 if let Err(error) = committed_recovery {
3808 return Err(ConfigSectionMutationError::Runtime(format!(
3809 "cluster reset committed at revision {revision} but transaction recovery failed: {error}"
3810 )));
3811 }
3812 if let Some(Err(error)) = credential_adoption {
3813 return Err(ConfigSectionMutationError::Runtime(format!(
3814 "cluster reset committed at revision {revision} but credential facade adoption failed: {error}"
3815 )));
3816 }
3817 let (credential_statuses, credential_health) =
3818 cluster_runtime.map_err(|error| {
3819 ConfigSectionMutationError::Runtime(format!(
3820 "cluster reset committed at revision {revision} but could not materialize its exact runtime credentials: {error}"
3821 ))
3822 })?;
3823 CredentialBackedResetCommit::Cluster(Box::new(
3824 bamboo_server_tools::FabricCommitSnapshot {
3825 config: candidate.clone(),
3826 section,
3827 credential_statuses,
3828 credential_health,
3829 },
3830 ))
3831 } else {
3832 let section = exact_section.ok_or_else(|| {
3833 ConfigSectionMutationError::Runtime(format!(
3834 "{} reset committed at revision {revision} without its exact envelope",
3835 id.descriptor().name
3836 ))
3837 })?;
3838 if section.revision != revision {
3839 return Err(ConfigSectionMutationError::Runtime(format!(
3840 "{} reset committed at revision {revision} but captured revision {}",
3841 id.descriptor().name,
3842 section.revision
3843 )));
3844 }
3845 CredentialBackedResetCommit::Section(section)
3846 };
3847
3848 if id == SectionId::Core {
3849 match bamboo_llm::ProviderRegistry::from_config(&candidate, app_data_dir.clone())
3850 .await
3851 {
3852 Ok(registry) => {
3853 if let Some(candidate_provider) = registry.get_default() {
3854 provider_registry.replace_with(registry);
3855 *provider.write().await = candidate_provider;
3856 }
3857 }
3858 Err(error) => {
3859 tracing::warn!(error = %error, "core reset committed but provider reload failed");
3860 }
3861 }
3862 mcp_manager.reconcile_from_config(&candidate.mcp).await;
3863 }
3864
3865 Ok(commit)
3866 });
3867 transaction.await.map_err(|error| {
3868 ConfigSectionMutationError::Runtime(format!(
3869 "section reset transaction task failed: {error}"
3870 ))
3871 })?
3872 }
3873}
3874
3875fn credential_reference_inventory(value: &Value) -> std::collections::BTreeMap<String, Value> {
3876 fn collect(value: &Value, path: &str, output: &mut std::collections::BTreeMap<String, Value>) {
3877 match value {
3878 Value::Object(object) => {
3879 for (key, value) in object {
3880 let child_path =
3881 format!("{path}/{}", key.replace('~', "~0").replace('/', "~1"));
3882 let normalized = key
3883 .chars()
3884 .filter(|ch| ch.is_ascii_alphanumeric())
3885 .flat_map(char::to_lowercase)
3886 .collect::<String>();
3887 if normalized == "credentialref"
3888 || normalized.ends_with("credentialref")
3889 || normalized.ends_with("credentialrefs")
3890 {
3891 output.insert(child_path, value.clone());
3892 } else {
3893 collect(value, &child_path, output);
3894 }
3895 }
3896 }
3897 Value::Array(values) => {
3898 for (index, value) in values.iter().enumerate() {
3899 collect(value, &format!("{path}/{index}"), output);
3900 }
3901 }
3902 _ => {}
3903 }
3904 }
3905
3906 let mut output = std::collections::BTreeMap::new();
3907 collect(value, "", &mut output);
3908 output
3909}
3910
3911fn provider_durable_document(
3912 providers: &ProviderConfigs,
3913) -> Result<ProviderConfigs, ConfigSectionMutationError> {
3914 let mut document = providers.clone();
3915 macro_rules! sanitize {
3916 ($field:ident) => {
3917 if let Some(provider) = document.$field.as_mut() {
3918 provider.api_key.clear();
3919 provider.api_key_encrypted = None;
3920 }
3921 };
3922 }
3923 sanitize!(openai);
3924 sanitize!(anthropic);
3925 sanitize!(gemini);
3926 if let Some(provider) = document.bodhi.as_mut() {
3927 provider.api_key.clear();
3928 provider.api_key_encrypted = None;
3929 }
3930 validate_provider_config(&document).map_err(ConfigSectionMutationError::Invalid)?;
3931 Ok(document)
3932}
3933
3934fn retain_provider_credentials(current: &ProviderConfigs, candidate: &mut ProviderConfigs) {
3935 candidate.extra = current.extra.clone();
3936 macro_rules! retain {
3937 ($field:ident) => {
3938 if let (Some(current), Some(candidate)) = (¤t.$field, &mut candidate.$field) {
3939 candidate.api_key = current.api_key.clone();
3940 candidate.api_key_encrypted = current.api_key_encrypted.clone();
3941 if candidate.credential_ref.is_none() {
3942 candidate.credential_ref = current.credential_ref.clone();
3943 }
3944 if candidate.credential_ref != current.credential_ref {
3945 candidate.api_key.clear();
3946 candidate.api_key_encrypted = None;
3947 }
3948 candidate.api_key_from_env = current.api_key_from_env;
3949 candidate.request_overrides = current.request_overrides.clone();
3950 candidate.extra = current.extra.clone();
3951 }
3952 };
3953 }
3954 retain!(openai);
3955 retain!(anthropic);
3956 retain!(gemini);
3957 if let (Some(current), Some(candidate)) = (¤t.bodhi, &mut candidate.bodhi) {
3958 candidate.api_key = current.api_key.clone();
3959 candidate.api_key_encrypted = current.api_key_encrypted.clone();
3960 if candidate.credential_ref.is_none() {
3961 candidate.credential_ref = current.credential_ref.clone();
3962 }
3963 if candidate.credential_ref != current.credential_ref {
3964 candidate.api_key.clear();
3965 candidate.api_key_encrypted = None;
3966 }
3967 candidate.extra = current.extra.clone();
3968 }
3969 if let (Some(current), Some(candidate)) = (¤t.copilot, &mut candidate.copilot) {
3970 candidate.request_overrides = current.request_overrides.clone();
3971 candidate.extra = current.extra.clone();
3972 }
3973}
3974
3975fn retain_mcp_credentials(
3976 current: &McpConfig,
3977 candidate: &mut McpConfig,
3978 touched: &BTreeSet<bamboo_config::CredentialRef>,
3979) {
3980 for candidate_server in &mut candidate.servers {
3981 let Some(current_server) = current
3982 .servers
3983 .iter()
3984 .find(|server| server.id == candidate_server.id)
3985 else {
3986 continue;
3987 };
3988 if let (TransportConfig::Stdio(current), TransportConfig::Stdio(candidate)) =
3989 (¤t_server.transport, &mut candidate_server.transport)
3990 {
3991 if candidate.env.is_empty()
3992 && candidate.env_encrypted.is_empty()
3993 && candidate.env_credential_refs.is_empty()
3994 {
3995 for (name, reference) in ¤t.env_credential_refs {
3996 if mcp_credential_ref_is_touched(Some(reference), touched) {
3997 continue;
3998 }
3999 candidate
4000 .env_credential_refs
4001 .insert(name.clone(), reference.clone());
4002 if let Some(value) = current.env.get(name) {
4003 candidate.env.insert(name.clone(), value.clone());
4004 }
4005 if let Some(value) = current.env_encrypted.get(name) {
4006 candidate.env_encrypted.insert(name.clone(), value.clone());
4007 }
4008 }
4009 } else {
4010 for (name, reference) in ¤t.env_credential_refs {
4011 if candidate.env_credential_refs.get(name) != Some(reference) {
4012 continue;
4013 }
4014 if candidate.env.get(name).is_none_or(|value| value.is_empty()) {
4015 if let Some(value) = current.env.get(name) {
4016 candidate.env.insert(name.clone(), value.clone());
4017 }
4018 }
4019 }
4020 }
4021 }
4022 match (¤t_server.transport, &mut candidate_server.transport) {
4023 (TransportConfig::Sse(current), TransportConfig::Sse(candidate)) => {
4024 retain_mcp_header_credentials(¤t.headers, &mut candidate.headers)
4025 }
4026 (
4027 TransportConfig::StreamableHttp(current),
4028 TransportConfig::StreamableHttp(candidate),
4029 ) => retain_mcp_header_credentials(¤t.headers, &mut candidate.headers),
4030 _ => {}
4031 }
4032 }
4033}
4034
4035fn materialize_mcp_touched_replacements(
4036 candidate: &mut McpConfig,
4037 touched: &BTreeSet<bamboo_config::CredentialRef>,
4038) -> Result<(), String> {
4039 let mut replacements = BTreeMap::<bamboo_config::CredentialRef, String>::new();
4040 for server in &candidate.servers {
4041 match &server.transport {
4042 TransportConfig::Stdio(stdio) => {
4043 for (name, raw_reference) in &stdio.env_credential_refs {
4044 let reference = bamboo_config::CredentialRef::parse(raw_reference.clone())
4045 .map_err(|_| "MCP credential reference is invalid".to_string())?;
4046 if let Some(value) = stdio
4047 .env
4048 .get(name)
4049 .filter(|value| touched.contains(&reference) && !value.is_empty())
4050 {
4051 insert_mcp_replacement(&mut replacements, reference, value)?;
4052 }
4053 }
4054 }
4055 TransportConfig::Sse(http) => {
4056 collect_mcp_header_replacements(&http.headers, touched, &mut replacements)?
4057 }
4058 TransportConfig::StreamableHttp(http) => {
4059 collect_mcp_header_replacements(&http.headers, touched, &mut replacements)?
4060 }
4061 }
4062 }
4063 if replacements.is_empty() {
4064 return Ok(());
4065 }
4066 for server in &mut candidate.servers {
4067 match &mut server.transport {
4068 TransportConfig::Stdio(stdio) => {
4069 for (name, raw_reference) in &stdio.env_credential_refs {
4070 let reference = bamboo_config::CredentialRef::parse(raw_reference.clone())
4071 .map_err(|_| "MCP credential reference is invalid".to_string())?;
4072 if let Some(value) = replacements.get(&reference) {
4073 stdio.env.insert(name.clone(), value.clone());
4074 }
4075 }
4076 }
4077 TransportConfig::Sse(http) => {
4078 apply_mcp_header_replacements(&mut http.headers, &replacements)?
4079 }
4080 TransportConfig::StreamableHttp(http) => {
4081 apply_mcp_header_replacements(&mut http.headers, &replacements)?
4082 }
4083 }
4084 }
4085 Ok(())
4086}
4087
4088fn collect_mcp_header_replacements(
4089 headers: &[bamboo_mcp::HeaderConfig],
4090 touched: &BTreeSet<bamboo_config::CredentialRef>,
4091 replacements: &mut BTreeMap<bamboo_config::CredentialRef, String>,
4092) -> Result<(), String> {
4093 for header in headers {
4094 let Some(raw_reference) = header.credential_ref.as_ref() else {
4095 continue;
4096 };
4097 let reference = bamboo_config::CredentialRef::parse(raw_reference.clone())
4098 .map_err(|_| "MCP credential reference is invalid".to_string())?;
4099 if touched.contains(&reference) && !header.value.is_empty() {
4100 insert_mcp_replacement(replacements, reference, &header.value)?;
4101 }
4102 }
4103 Ok(())
4104}
4105
4106fn insert_mcp_replacement(
4107 replacements: &mut BTreeMap<bamboo_config::CredentialRef, String>,
4108 reference: bamboo_config::CredentialRef,
4109 value: &str,
4110) -> Result<(), String> {
4111 match replacements.get(&reference) {
4112 Some(existing) if existing != value => {
4113 Err("MCP updates assign conflicting values to one credential reference".to_string())
4114 }
4115 Some(_) => Ok(()),
4116 None => {
4117 replacements.insert(reference, value.to_string());
4118 Ok(())
4119 }
4120 }
4121}
4122
4123fn apply_mcp_header_replacements(
4124 headers: &mut [bamboo_mcp::HeaderConfig],
4125 replacements: &BTreeMap<bamboo_config::CredentialRef, String>,
4126) -> Result<(), String> {
4127 for header in headers {
4128 let Some(raw_reference) = header.credential_ref.as_ref() else {
4129 continue;
4130 };
4131 let reference = bamboo_config::CredentialRef::parse(raw_reference.clone())
4132 .map_err(|_| "MCP credential reference is invalid".to_string())?;
4133 if let Some(value) = replacements.get(&reference) {
4134 header.value = value.clone();
4135 }
4136 }
4137 Ok(())
4138}
4139
4140fn mcp_credential_ref_is_touched(
4141 raw_reference: Option<&String>,
4142 touched: &BTreeSet<bamboo_config::CredentialRef>,
4143) -> bool {
4144 raw_reference
4145 .and_then(|raw| bamboo_config::CredentialRef::parse(raw.clone()).ok())
4146 .is_some_and(|reference| touched.contains(&reference))
4147}
4148
4149fn retain_mcp_header_credentials(
4150 current: &[bamboo_mcp::HeaderConfig],
4151 candidate: &mut [bamboo_mcp::HeaderConfig],
4152) {
4153 for candidate_header in candidate {
4154 let Some(current_header) = current
4155 .iter()
4156 .find(|header| header.name == candidate_header.name)
4157 else {
4158 continue;
4159 };
4160 if candidate_header.credential_ref == current_header.credential_ref
4161 && candidate_header.value.is_empty()
4162 {
4163 candidate_header.value = current_header.value.clone();
4164 candidate_header.value_encrypted = current_header.value_encrypted.clone();
4165 }
4166 }
4167}
4168
4169fn credential_ref_mcp_document(
4170 runtime: &McpConfig,
4171) -> Result<McpConfig, ConfigSectionMutationError> {
4172 let mut document = runtime.clone();
4173 for server in &mut document.servers {
4174 match &mut server.transport {
4175 TransportConfig::Stdio(config) => {
4176 config.env_encrypted.clear();
4177 config.env.retain(|name, value| {
4178 !(value.is_empty() || config.env_credential_refs.contains_key(name))
4179 });
4180 if !config.env.is_empty() {
4181 return Err(ConfigSectionMutationError::Invalid(
4182 "MCP secret requires a credential reference".to_string(),
4183 ));
4184 }
4185 }
4186 TransportConfig::Sse(config) => reference_headers(&mut config.headers)?,
4187 TransportConfig::StreamableHttp(config) => reference_headers(&mut config.headers)?,
4188 }
4189 }
4190 Ok(document)
4191}
4192
4193fn retain_mcp_credential_refs(document: &McpConfig, runtime: &mut McpConfig) {
4194 for runtime_server in &mut runtime.servers {
4195 let Some(document_server) = document
4196 .servers
4197 .iter()
4198 .find(|server| server.id == runtime_server.id)
4199 else {
4200 continue;
4201 };
4202 match (&document_server.transport, &mut runtime_server.transport) {
4203 (TransportConfig::Stdio(document), TransportConfig::Stdio(runtime)) => {
4204 runtime.env_encrypted.clear();
4205 runtime.env_credential_refs = document.env_credential_refs.clone();
4206 }
4207 (TransportConfig::Sse(document), TransportConfig::Sse(runtime)) => {
4208 copy_header_ciphertext(&document.headers, &mut runtime.headers);
4209 }
4210 (
4211 TransportConfig::StreamableHttp(document),
4212 TransportConfig::StreamableHttp(runtime),
4213 ) => copy_header_ciphertext(&document.headers, &mut runtime.headers),
4214 _ => {}
4215 }
4216 }
4217}
4218
4219fn copy_header_ciphertext(
4220 document: &[bamboo_mcp::HeaderConfig],
4221 runtime: &mut [bamboo_mcp::HeaderConfig],
4222) {
4223 for runtime_header in runtime {
4224 if let Some(document_header) = document
4225 .iter()
4226 .find(|header| header.name == runtime_header.name)
4227 {
4228 runtime_header.value_encrypted = None;
4229 runtime_header.credential_ref = document_header.credential_ref.clone();
4230 }
4231 }
4232}
4233
4234fn reference_headers(
4235 headers: &mut [bamboo_mcp::HeaderConfig],
4236) -> Result<(), ConfigSectionMutationError> {
4237 for header in headers {
4238 if !header.value.is_empty() && header.credential_ref.is_none() {
4239 return Err(ConfigSectionMutationError::Invalid(
4240 "MCP secret requires a credential reference".to_string(),
4241 ));
4242 }
4243 header.value.clear();
4244 header.value_encrypted = None;
4245 }
4246 Ok(())
4247}
4248
4249fn mcp_credential_refs(
4257 config: &McpConfig,
4258) -> Result<BTreeSet<bamboo_config::CredentialRef>, ConfigSectionMutationError> {
4259 let mut references = BTreeSet::new();
4260 for server in &config.servers {
4261 match &server.transport {
4262 TransportConfig::Stdio(stdio) => {
4263 for raw in stdio.env_credential_refs.values() {
4264 references.insert(bamboo_config::CredentialRef::parse(raw.clone()).map_err(
4265 |_| {
4266 ConfigSectionMutationError::Invalid(
4267 "MCP credential reference is invalid".to_string(),
4268 )
4269 },
4270 )?);
4271 }
4272 }
4273 TransportConfig::Sse(http) => collect_mcp_header_refs(&http.headers, &mut references)?,
4274 TransportConfig::StreamableHttp(http) => {
4275 collect_mcp_header_refs(&http.headers, &mut references)?
4276 }
4277 }
4278 }
4279 Ok(references)
4280}
4281
4282fn collect_mcp_header_refs(
4283 headers: &[bamboo_mcp::HeaderConfig],
4284 output: &mut BTreeSet<bamboo_config::CredentialRef>,
4285) -> Result<(), ConfigSectionMutationError> {
4286 for raw in headers
4287 .iter()
4288 .filter_map(|header| header.credential_ref.as_ref())
4289 {
4290 output.insert(
4291 bamboo_config::CredentialRef::parse(raw.clone()).map_err(|_| {
4292 ConfigSectionMutationError::Invalid(
4293 "MCP credential reference is invalid".to_string(),
4294 )
4295 })?,
4296 );
4297 }
4298 Ok(())
4299}
4300
4301fn normalize_legacy_mcp_credentials(
4302 current: &McpConfig,
4303 candidate: &mut McpConfig,
4304) -> Result<BTreeSet<bamboo_config::CredentialRef>, AppError> {
4305 let current_refs = mcp_credential_refs(current).map_err(map_mcp_section_error)?;
4306 let mut intents = BTreeSet::new();
4307
4308 for candidate_server in &mut candidate.servers {
4309 let current_server = current
4310 .servers
4311 .iter()
4312 .find(|server| server.id == candidate_server.id);
4313 match &mut candidate_server.transport {
4314 TransportConfig::Stdio(candidate_stdio) => {
4315 if !candidate_stdio.env_encrypted.is_empty() {
4316 return Err(AppError::BadRequest(
4317 "MCP ciphertext is server-managed and cannot be supplied".to_string(),
4318 ));
4319 }
4320 let current_stdio = current_server.and_then(|server| match &server.transport {
4321 TransportConfig::Stdio(stdio) => Some(stdio),
4322 _ => None,
4323 });
4324 for (name, incoming_reference) in &candidate_stdio.env_credential_refs {
4325 let current_reference = current_stdio
4326 .and_then(|stdio| stdio.env_credential_refs.get(name))
4327 .map(String::as_str);
4328 if current_reference != Some(incoming_reference.as_str()) {
4329 return Err(AppError::BadRequest(
4330 "MCP credential references are server-managed and cannot be supplied"
4331 .to_string(),
4332 ));
4333 }
4334 }
4335 let incoming = std::mem::take(&mut candidate_stdio.env);
4336 candidate_stdio.env_credential_refs.clear();
4337 for (name, value) in incoming {
4338 let current_reference = current_stdio
4339 .and_then(|stdio| stdio.env_credential_refs.get(&name))
4340 .map(|raw| {
4341 bamboo_config::CredentialRef::parse(raw.clone()).map_err(|_| {
4342 AppError::BadRequest(
4343 "MCP credential reference is invalid".to_string(),
4344 )
4345 })
4346 })
4347 .transpose()?;
4348 let current_value = current_stdio.and_then(|stdio| stdio.env.get(&name));
4349 let (value, reference, touched) =
4350 if bamboo_config::patch::is_masked_api_key(&value) {
4351 let reference = current_reference.ok_or_else(|| {
4352 AppError::BadRequest(
4353 "masked MCP credential has no existing value".to_string(),
4354 )
4355 })?;
4356 let value = current_value.cloned().ok_or_else(|| {
4357 AppError::BadRequest(
4358 "referenced MCP credential is unavailable".to_string(),
4359 )
4360 })?;
4361 (value, reference, false)
4362 } else if value.is_empty() {
4363 if let Some(reference) = current_reference {
4364 intents.insert(reference);
4365 }
4366 continue;
4367 } else {
4368 let reference = current_reference.map_or_else(
4369 || {
4370 bamboo_config::credential_ref(
4371 "mcp",
4372 &candidate_server.id,
4373 &format!("env_{name}"),
4374 )
4375 .map_err(map_exact_credential_store_error)
4376 },
4377 Ok,
4378 )?;
4379 let touched = current_value != Some(&value);
4380 (value, reference, touched)
4381 };
4382 if touched {
4383 intents.insert(reference.clone());
4384 }
4385 candidate_stdio.env.insert(name.clone(), value);
4386 candidate_stdio
4387 .env_credential_refs
4388 .insert(name, reference.as_str().to_string());
4389 }
4390 }
4391 TransportConfig::Sse(candidate_http) => normalize_legacy_mcp_headers(
4392 &candidate_server.id,
4393 current_server.and_then(|server| match &server.transport {
4394 TransportConfig::Sse(http) => Some(http.headers.as_slice()),
4395 _ => None,
4396 }),
4397 &mut candidate_http.headers,
4398 &mut intents,
4399 )?,
4400 TransportConfig::StreamableHttp(candidate_http) => normalize_legacy_mcp_headers(
4401 &candidate_server.id,
4402 current_server.and_then(|server| match &server.transport {
4403 TransportConfig::StreamableHttp(http) => Some(http.headers.as_slice()),
4404 _ => None,
4405 }),
4406 &mut candidate_http.headers,
4407 &mut intents,
4408 )?,
4409 }
4410 }
4411
4412 let candidate_refs = mcp_credential_refs(candidate).map_err(map_mcp_section_error)?;
4413 intents.extend(current_refs.symmetric_difference(&candidate_refs).cloned());
4414 Ok(intents)
4415}
4416
4417fn normalize_legacy_mcp_headers(
4418 server_id: &str,
4419 current: Option<&[bamboo_mcp::HeaderConfig]>,
4420 candidate: &mut [bamboo_mcp::HeaderConfig],
4421 intents: &mut BTreeSet<bamboo_config::CredentialRef>,
4422) -> Result<(), AppError> {
4423 for header in candidate {
4424 if header.value_encrypted.is_some() {
4425 return Err(AppError::BadRequest(
4426 "MCP ciphertext is server-managed and cannot be supplied".to_string(),
4427 ));
4428 }
4429 let current_header =
4430 current.and_then(|headers| headers.iter().find(|current| current.name == header.name));
4431 if header.credential_ref.as_deref().is_some_and(|incoming| {
4432 current_header.and_then(|current| current.credential_ref.as_deref()) != Some(incoming)
4433 }) {
4434 return Err(AppError::BadRequest(
4435 "MCP credential references are server-managed and cannot be supplied".to_string(),
4436 ));
4437 }
4438 let current_reference = current_header
4439 .and_then(|current| current.credential_ref.as_ref())
4440 .map(|raw| {
4441 bamboo_config::CredentialRef::parse(raw.clone()).map_err(|_| {
4442 AppError::BadRequest("MCP credential reference is invalid".to_string())
4443 })
4444 })
4445 .transpose()?;
4446 let current_value = current_header.map(|current| ¤t.value);
4447 if bamboo_config::patch::is_masked_api_key(&header.value) {
4448 let reference = current_reference.ok_or_else(|| {
4449 AppError::BadRequest("masked MCP credential has no existing value".to_string())
4450 })?;
4451 header.value = current_value.cloned().ok_or_else(|| {
4452 AppError::BadRequest("referenced MCP credential is unavailable".to_string())
4453 })?;
4454 header.credential_ref = Some(reference.as_str().to_string());
4455 } else if header.value.is_empty() {
4456 if let Some(reference) = current_reference {
4457 intents.insert(reference);
4458 }
4459 header.credential_ref = None;
4460 } else {
4461 let reference = current_reference.map_or_else(
4462 || {
4463 bamboo_config::credential_ref(
4464 "mcp",
4465 server_id,
4466 &format!("header_{}", header.name),
4467 )
4468 .map_err(map_exact_credential_store_error)
4469 },
4470 Ok,
4471 )?;
4472 if current_value != Some(&header.value) {
4473 intents.insert(reference.clone());
4474 }
4475 header.credential_ref = Some(reference.as_str().to_string());
4476 }
4477 header.value_encrypted = None;
4478 }
4479 Ok(())
4480}
4481
4482fn map_mcp_section_error(error: ConfigSectionMutationError) -> AppError {
4483 match error {
4484 ConfigSectionMutationError::Store(error) => map_exact_credential_store_error(error),
4485 ConfigSectionMutationError::Invalid(message)
4486 | ConfigSectionMutationError::Runtime(message) => AppError::BadRequest(message),
4487 }
4488}
4489
4490impl AppState {
4491 pub async fn reload_provider(&self) -> Result<(), bamboo_llm::LLMError> {
4523 let _io = self.config_io_lock.lock().await;
4528 let config = self.config.read().await.clone();
4529 let candidate_registry =
4530 bamboo_llm::ProviderRegistry::from_config(&config, self.app_data_dir.clone()).await?;
4531 let default_provider_name = candidate_registry.default_provider_name();
4532 tracing::info!(
4533 default_provider = %default_provider_name,
4534 legacy_provider = %config.provider,
4535 has_provider_instances = config.has_provider_instances(),
4536 "Reloading provider runtime from current config"
4537 );
4538
4539 let new_provider = candidate_registry.get_default().ok_or_else(|| {
4540 let message = if config.has_provider_instances() {
4541 format!(
4542 "Default provider instance '{}' is not available or failed to initialize",
4543 default_provider_name
4544 )
4545 } else {
4546 format!(
4547 "Provider '{}' is not available or failed to initialize",
4548 config.provider
4549 )
4550 };
4551 bamboo_llm::LLMError::Auth(message)
4552 })?;
4553
4554 #[cfg(test)]
4555 run_generic_before_provider_publish_test_hook(&self.app_data_dir);
4556 let mut provider = self.provider.write().await;
4557 self.provider_registry.replace_with(candidate_registry);
4558 *provider = new_provider;
4559
4560 tracing::info!(
4561 default_provider = %default_provider_name,
4562 "Provider reloaded successfully"
4563 );
4564 Ok(())
4565 }
4566
4567 pub async fn reload_config(&self) -> Config {
4597 let _io = self.config_io_lock.lock().await;
4602 let mut config = self.config.write().await;
4603 let mut new_config = self
4604 .config_facade
4605 .as_ref()
4606 .map(|facade| load_facade_effective_config(facade, &self.app_data_dir))
4607 .unwrap_or_else(|| {
4608 Config::from_data_dir_without_publish(Some(self.app_data_dir.clone()))
4609 });
4610 preserve_runtime_broker(&mut new_config, &config);
4611 new_config.publish_env_vars();
4612 *config = new_config.clone();
4613 new_config
4614 }
4615
4616 pub async fn reload_config_and_runtime(&self) -> Result<Config, AppError> {
4624 let io = self.config_io_lock.clone().lock_owned().await;
4625 let app_data_dir = self.app_data_dir.clone();
4626 let config_facade = self.config_facade.clone();
4627 let config = self.config.clone();
4628 let provider_registry = self.provider_registry.clone();
4629 let provider = self.provider.clone();
4630 let mcp_manager = self.mcp_manager.clone();
4631 let account_sink = self.account_sink.clone();
4632 let config_live_health = self.config_live_health.clone();
4633 let mcp_config_live_health = self.mcp_config_live_health.clone();
4634 let transaction = tokio::spawn(async move {
4635 let _io = io;
4636 let mut new_config = config_facade
4637 .as_ref()
4638 .map(|facade| load_facade_effective_config(facade, &app_data_dir))
4639 .unwrap_or_else(|| {
4640 Config::from_data_dir_without_publish(Some(app_data_dir.clone()))
4641 });
4642 {
4643 let previous = config.read().await;
4644 preserve_runtime_broker(&mut new_config, &previous);
4645 }
4646 if let Err(error) = bamboo_llm::validate_provider_config(&new_config) {
4647 tracing::warn!("reloaded provider config is invalid");
4648 let message =
4649 "provider configuration is invalid; retaining last-known-good generation"
4650 .to_string();
4651 if let Some(facade) = config_facade.as_ref() {
4652 if let Some(event) = facade
4653 .registry()
4654 .mark_runtime_degraded(SectionId::Providers, message.clone())
4655 {
4656 publish_registry_event(&account_sink, &event).await;
4657 }
4658 }
4659 publish_section_failure(
4660 &config_live_health,
4661 &account_sink,
4662 "providers",
4663 SectionStatus::Invalid,
4664 message,
4665 )
4666 .await;
4667 return Err(AppError::BadRequest(format!(
4668 "Invalid configuration: {error}"
4669 )));
4670 }
4671 new_config.publish_env_vars();
4672 *config.write().await = new_config.clone();
4673 Self::apply_config_effects_owned(
4674 new_config.clone(),
4675 ConfigUpdateEffects {
4676 reload_provider: bamboo_config::patch::ReloadMode::Strict,
4677 reconcile_mcp: bamboo_config::patch::ReloadMode::Strict,
4678 },
4679 ConfigRuntimeEffectContext {
4680 app_data_dir,
4681 config_facade,
4682 provider_registry,
4683 provider,
4684 mcp_manager,
4685 account_sink,
4686 config_live_health,
4687 mcp_config_live_health,
4688 },
4689 )
4690 .await?;
4691 Ok::<_, AppError>(new_config)
4692 });
4693 transaction.await.map_err(|error| {
4694 AppError::InternalError(anyhow::anyhow!(
4695 "config/runtime reload transaction task failed: {error}"
4696 ))
4697 })?
4698 }
4699
4700 pub async fn reset_legacy_config_and_runtime(&self) -> Result<Config, AppError> {
4713 if self.config_facade.is_some() {
4714 return Err(AppError::BadRequest(
4715 "full config reset spans multiple revisioned sections and is disabled without a recoverable manifest; reset sections individually through the typed section API"
4716 .to_string(),
4717 ));
4718 }
4719
4720 let io = self.config_io_lock.clone().lock_owned().await;
4721 let app_data_dir = self.app_data_dir.clone();
4722 let config = self.config.clone();
4723 let provider_registry = self.provider_registry.clone();
4724 let provider = self.provider.clone();
4725 let mcp_manager = self.mcp_manager.clone();
4726 let account_sink = self.account_sink.clone();
4727 let config_live_health = self.config_live_health.clone();
4728 let mcp_config_live_health = self.mcp_config_live_health.clone();
4729 let transaction = tokio::spawn(async move {
4730 let _io = io;
4731 let mut deletion_error = None;
4732 for path in [
4733 app_data_dir.join("config.json"),
4734 app_data_dir.join("model_limits.json"),
4735 app_data_dir.join("connect.json"),
4736 app_data_dir.join("connect.json.bak"),
4737 ] {
4738 let result = match tokio::fs::try_exists(&path).await {
4739 Ok(true) => tokio::fs::remove_file(&path).await,
4740 Ok(false) => Ok(()),
4741 Err(error) => Err(error),
4742 };
4743 if let Err(error) = result {
4744 tracing::warn!(
4745 file = %path.display(),
4746 "failed to delete one legacy config artifact during reset"
4747 );
4748 deletion_error.get_or_insert(error);
4749 }
4750 }
4751 #[cfg(test)]
4752 run_reset_after_delete_test_hook(&app_data_dir);
4753
4754 let mut new_config = Config::from_data_dir_without_publish(Some(app_data_dir.clone()));
4755 {
4756 let previous = config.read().await;
4757 preserve_runtime_broker(&mut new_config, &previous);
4758 }
4759 new_config.publish_env_vars();
4760 *config.write().await = new_config.clone();
4761 Self::apply_config_effects_owned(
4762 new_config.clone(),
4763 ConfigUpdateEffects {
4764 reload_provider: bamboo_config::patch::ReloadMode::BestEffort,
4765 reconcile_mcp: bamboo_config::patch::ReloadMode::BestEffort,
4766 },
4767 ConfigRuntimeEffectContext {
4768 app_data_dir,
4769 config_facade: None,
4770 provider_registry,
4771 provider,
4772 mcp_manager,
4773 account_sink,
4774 config_live_health,
4775 mcp_config_live_health,
4776 },
4777 )
4778 .await?;
4779 match deletion_error {
4780 Some(error) => Err(AppError::StorageError(error)),
4781 None => Ok(new_config),
4782 }
4783 });
4784 transaction.await.map_err(|error| {
4785 AppError::InternalError(anyhow::anyhow!(
4786 "legacy config reset transaction task failed: {error}"
4787 ))
4788 })?
4789 }
4790
4791 async fn persist_config_snapshot(
4792 data_dir: PathBuf,
4793 config_facade: Option<Arc<bamboo_config::ConfigFacade>>,
4794 config: Config,
4795 ) -> Result<Option<bamboo_config::FacadeConfigCommit>, AppError> {
4796 if let Some(facade) = config_facade {
4797 tokio::task::spawn_blocking(move || {
4798 let result = bamboo_config::persist_facade_effective_config_with_adoption(
4799 &data_dir,
4800 &config,
4801 facade.as_ref(),
4802 );
4803 #[cfg(test)]
4804 if result.is_ok() {
4805 run_generic_before_event_test_hook(&data_dir);
4806 }
4807 result
4808 })
4809 .await
4810 .map_err(|error| {
4811 AppError::InternalError(anyhow::anyhow!("Config save task failed: {error}"))
4812 })?
4813 .map(Some)
4814 .map_err(map_exact_credential_store_error)
4815 } else {
4816 tokio::task::spawn_blocking(move || {
4817 let result = config.save_to_dir(data_dir.clone());
4818 #[cfg(test)]
4819 if result.is_ok() {
4820 run_generic_before_event_test_hook(&data_dir);
4821 }
4822 result
4823 })
4824 .await
4825 .map_err(|error| {
4826 AppError::InternalError(anyhow::anyhow!("Config save task failed: {error}"))
4827 })?
4828 .map_err(|error| {
4829 AppError::InternalError(anyhow::anyhow!("Failed to save config: {error}"))
4830 })?;
4831 Ok(None)
4832 }
4833 }
4834
4835 pub async fn update_config<F>(
4842 &self,
4843 update: F,
4844 effects: ConfigUpdateEffects,
4845 ) -> Result<Config, AppError>
4846 where
4847 F: FnOnce(&mut Config) -> Result<(), AppError>,
4848 {
4849 self.update_config_with_forced_mcp_replacements(update, effects, HashSet::new())
4850 .await
4851 }
4852
4853 pub(crate) async fn update_config_with_forced_mcp_replacements<F>(
4854 &self,
4855 update: F,
4856 effects: ConfigUpdateEffects,
4857 forced_mcp_replacements: HashSet<String>,
4858 ) -> Result<Config, AppError>
4859 where
4860 F: FnOnce(&mut Config) -> Result<(), AppError>,
4861 {
4862 let io = self.config_io_lock.clone().lock_owned().await;
4866 let (mut snapshot, live_base, enforcement_newly_off) = {
4867 let cfg = self.config.read().await;
4868 reject_if_recovery_pending(&cfg)?;
4874 let was_off = cfg.plugin_trust.enforcement_is_off();
4875 let live_base = cfg.clone();
4876 let mut candidate = cfg.clone();
4877 restore_authoritative_cluster_fabric(self.config_facade.as_ref(), &mut candidate);
4878 update(&mut candidate)?;
4879 restore_authoritative_cluster_fabric(self.config_facade.as_ref(), &mut candidate);
4881 if self.config_facade.is_none() {
4882 candidate.assign_connect_platform_ids();
4883 candidate.refresh_encrypted_secrets().map_err(|e| {
4884 AppError::InternalError(anyhow::anyhow!(
4885 "Failed to refresh encrypted secrets: {e}"
4886 ))
4887 })?;
4888 }
4889 let newly_off = !was_off && candidate.plugin_trust.enforcement_is_off();
4890 (candidate, live_base, newly_off)
4891 };
4892 if enforcement_newly_off {
4893 warn_plugin_trust_enforcement_off();
4894 }
4895 let config = self.config.clone();
4896 let app_data_dir = self.app_data_dir.clone();
4897 let config_facade = self.config_facade.clone();
4898 let account_sink = self.account_sink.clone();
4899 let provider_registry = self.provider_registry.clone();
4900 let provider = self.provider.clone();
4901 let mcp_manager = self.mcp_manager.clone();
4902 let config_live_health = self.config_live_health.clone();
4903 let mcp_config_live_health = self.mcp_config_live_health.clone();
4904 let transaction = tokio::spawn(async move {
4908 let snapshot = {
4915 let _io = io;
4916 let commit = Self::persist_config_snapshot(
4917 app_data_dir.clone(),
4918 config_facade.clone(),
4919 snapshot.clone(),
4920 )
4921 .await?;
4922 let events = match commit {
4923 Some(commit) => {
4924 let mut published = live_base;
4925 let events =
4926 install_facade_config_commit(commit, &mut published).map_err(|e| {
4927 AppError::InternalError(anyhow::anyhow!(
4928 "failed to install committed configuration section: {e}"
4929 ))
4930 })?;
4931 snapshot = published;
4932 events
4933 }
4934 None => Vec::new(),
4935 };
4936 {
4937 let mut cfg = config.write().await;
4938 preserve_runtime_broker(&mut snapshot, &cfg);
4939 snapshot.publish_env_vars();
4940 *cfg = snapshot.clone();
4941 }
4942 publish_exact_facade_events(&account_sink, &events).await?;
4946 Self::apply_config_effects_owned_after_forcing(
4947 snapshot.clone(),
4948 effects,
4949 ConfigRuntimeEffectContext {
4950 app_data_dir,
4951 config_facade,
4952 provider_registry,
4953 provider,
4954 mcp_manager,
4955 account_sink,
4956 config_live_health,
4957 mcp_config_live_health,
4958 },
4959 forced_mcp_replacements,
4960 )
4961 .await?;
4962 snapshot
4963 };
4964 Ok::<_, AppError>(snapshot)
4965 });
4966 transaction.await.map_err(|error| {
4967 AppError::InternalError(anyhow::anyhow!(
4968 "config update transaction task failed: {error}"
4969 ))
4970 })?
4971 }
4972
4973 pub async fn update_config_with_provider_credentials<F>(
4976 &self,
4977 update: F,
4978 provider_intents: std::collections::BTreeSet<String>,
4979 provider_instance_intents: std::collections::BTreeSet<String>,
4980 effects: ConfigUpdateEffects,
4981 ) -> Result<Config, AppError>
4982 where
4983 F: FnOnce(&mut Config) -> Result<(), AppError>,
4984 {
4985 if provider_intents.is_empty() && provider_instance_intents.is_empty() {
4986 return self.update_config(update, effects).await;
4987 }
4988 let io = self.config_io_lock.clone().lock_owned().await;
4989 let config_facade = self.config_facade.clone();
4990 let (mut candidate, live_base, enforcement_newly_off) = {
4991 let cfg = self.config.read().await;
4992 reject_if_recovery_pending(&cfg)?;
4993 let was_off = cfg.plugin_trust.enforcement_is_off();
4994 let live_base = cfg.clone();
4995 let mut candidate = cfg.clone();
4996 restore_authoritative_cluster_fabric(config_facade.as_ref(), &mut candidate);
4997 update(&mut candidate)?;
4998 restore_authoritative_cluster_fabric(config_facade.as_ref(), &mut candidate);
5000 let mut non_provider_candidate = candidate.clone();
5005 apply_runtime_section(SectionId::Providers, &cfg, &mut non_provider_candidate);
5006 let mut comparison_base = cfg.clone();
5007 restore_authoritative_cluster_fabric(config_facade.as_ref(), &mut comparison_base);
5008 let mut changed =
5009 bamboo_config::changed_facade_sections(&comparison_base, &non_provider_candidate)
5010 .map_err(|_| {
5011 AppError::InternalError(anyhow::anyhow!(
5012 "failed to compare modular configuration sections"
5013 ))
5014 })?;
5015 if serde_json::to_value(cfg.subagents()).ok()
5016 == serde_json::to_value(candidate.subagents()).ok()
5017 {
5018 changed.retain(|section| *section != SectionId::Subagents);
5019 }
5020 if let Some(other) = changed
5021 .into_iter()
5022 .find(|section| *section != SectionId::Providers)
5023 {
5024 return Err(AppError::BadRequest(format!(
5025 "provider credential updates cannot be combined with {} changes; split the request",
5026 other.descriptor().name
5027 )));
5028 }
5029 if config_facade.is_none() {
5030 candidate.assign_connect_platform_ids();
5031 candidate.refresh_encrypted_secrets().map_err(|error| {
5032 AppError::InternalError(anyhow::anyhow!(
5033 "Failed to refresh encrypted secrets: {error}"
5034 ))
5035 })?;
5036 }
5037 let newly_off = !was_off && candidate.plugin_trust.enforcement_is_off();
5038 (candidate, live_base, newly_off)
5039 };
5040 let config = self.config.clone();
5041 let app_data_dir = self.app_data_dir.clone();
5042 let account_sink = self.account_sink.clone();
5043 let provider_registry = self.provider_registry.clone();
5044 let provider = self.provider.clone();
5045 let mcp_manager = self.mcp_manager.clone();
5046 let config_live_health = self.config_live_health.clone();
5047 let mcp_config_live_health = self.mcp_config_live_health.clone();
5048 let transaction = tokio::spawn(async move {
5049 let snapshot = {
5050 let _io = io;
5051 let data_dir = app_data_dir.clone();
5052 let commit_facade = config_facade.clone();
5053 let (candidate, commit) = tokio::task::spawn_blocking(move || {
5054 let result = if let Some(facade) = commit_facade {
5055 let commit =
5056 bamboo_config::persist_provider_instance_credential_transaction_with_adoption(
5057 &data_dir,
5058 &mut candidate,
5059 &provider_intents,
5060 &provider_instance_intents,
5061 facade.as_ref(),
5062 )?;
5063 Ok::<_, ConfigStoreError>((candidate, Some(commit)))
5064 } else {
5065 bamboo_config::persist_provider_instance_credential_transaction(
5066 &data_dir,
5067 &mut candidate,
5068 &provider_intents,
5069 &provider_instance_intents,
5070 )?;
5071 Ok((load_committed_effective_config(&data_dir)?, None))
5072 };
5073 #[cfg(test)]
5074 run_generic_before_event_test_hook(&data_dir);
5075 result
5076 })
5077 .await
5078 .map_err(|error| {
5079 AppError::InternalError(anyhow::anyhow!(
5080 "provider credential transaction task failed: {error}"
5081 ))
5082 })?
5083 .map_err(|error| match error {
5084 ConfigStoreError::Conflict { expected, actual } => {
5085 AppError::ConfigConflict { expected, actual }
5086 }
5087 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
5088 ConfigStoreError::CommitIndeterminate(message) => AppError::InternalError(
5089 anyhow::anyhow!("configuration commit outcome is indeterminate: {message}"),
5090 ),
5091 ConfigStoreError::Io(error) => AppError::StorageError(error),
5092 ConfigStoreError::Json(_) => {
5093 AppError::BadRequest("configuration document is invalid".to_string())
5094 }
5095 ConfigStoreError::Watch(error) => AppError::InternalError(anyhow::anyhow!(
5096 "configuration watch failed: {error}"
5097 )),
5098 })?;
5099 let (mut snapshot, events) = match commit {
5100 Some(commit) => {
5101 let mut published = live_base;
5102 let installed = install_credential_section_commit(commit, &mut published)
5103 .map_err(|error| {
5104 AppError::InternalError(anyhow::anyhow!(
5105 "provider process adoption failed: {error}"
5106 ))
5107 })?;
5108 (published, installed.events)
5109 }
5110 None => (candidate, Vec::new()),
5111 };
5112 {
5113 let mut cfg = config.write().await;
5114 preserve_runtime_broker(&mut snapshot, &cfg);
5115 snapshot.publish_env_vars();
5116 *cfg = snapshot.clone();
5117 }
5118 publish_exact_facade_events(&account_sink, &events).await?;
5119 if enforcement_newly_off {
5120 warn_plugin_trust_enforcement_off();
5121 }
5122 Self::apply_config_effects_owned(
5123 snapshot.clone(),
5124 effects,
5125 ConfigRuntimeEffectContext {
5126 app_data_dir,
5127 config_facade,
5128 provider_registry,
5129 provider,
5130 mcp_manager,
5131 account_sink,
5132 config_live_health,
5133 mcp_config_live_health,
5134 },
5135 )
5136 .await?;
5137 snapshot
5138 };
5139 Ok::<_, AppError>(snapshot)
5140 });
5141 transaction.await.map_err(|error| {
5142 AppError::InternalError(anyhow::anyhow!(
5143 "provider config transaction task failed: {error}"
5144 ))
5145 })?
5146 }
5147
5148 pub async fn update_env_var_credentials<F>(
5152 &self,
5153 expected_revision: u64,
5154 mut env_intents: std::collections::BTreeSet<String>,
5155 full_replace: bool,
5156 update: F,
5157 ) -> Result<
5158 (
5159 Config,
5160 u64,
5161 bamboo_config::CredentialSectionRuntimeMetadata,
5162 Option<bamboo_config::SectionEnvelope<Value>>,
5163 ),
5164 AppError,
5165 >
5166 where
5167 F: FnOnce(&mut Config) -> Result<(), AppError> + Send + 'static,
5168 {
5169 let config_io_lock = self.config_io_lock.clone();
5170 let config = self.config.clone();
5171 let app_data_dir = self.app_data_dir.clone();
5172 let account_sink = self.account_sink.clone();
5173 let config_facade = self.config_facade.clone();
5174 let transaction = tokio::spawn(async move {
5175 let _io = config_io_lock.lock().await;
5176 let live_base = {
5177 let current = config.read().await;
5178 reject_if_recovery_pending(¤t)?;
5179 current.clone()
5180 };
5181 let mut candidate = live_base.clone();
5182 if config_facade.is_some() {
5183 install_exact_credential_section_mutation_base(
5184 app_data_dir.clone(),
5185 SectionId::Env,
5186 expected_revision,
5187 &mut candidate,
5188 )
5189 .await?;
5190 }
5191 if full_replace {
5192 env_intents.extend(candidate.env_vars.iter().map(|entry| entry.name.clone()));
5193 }
5194 update(&mut candidate)?;
5195 if config_facade.is_none() {
5196 candidate.assign_connect_platform_ids();
5197 }
5198 let transaction_dir = app_data_dir.clone();
5199 let commit_facade = config_facade.clone();
5200 let (candidate, revision, commit) = tokio::task::spawn_blocking(move || {
5201 if let Some(facade) = commit_facade {
5202 let commit =
5203 bamboo_config::persist_env_var_credential_transaction_at_revision_with_adoption(
5204 &transaction_dir,
5205 &mut candidate,
5206 &env_intents,
5207 expected_revision,
5208 facade.as_ref(),
5209 )?;
5210 #[cfg(test)]
5211 run_credential_after_commit_before_live_test_hook(
5212 &transaction_dir,
5213 SectionId::Env,
5214 );
5215 let revision = commit.revision;
5216 Ok::<_, ConfigStoreError>((candidate, revision, Some(commit)))
5217 } else {
5218 let revision =
5219 bamboo_config::persist_env_var_credential_transaction_at_revision(
5220 &transaction_dir,
5221 &mut candidate,
5222 &env_intents,
5223 expected_revision,
5224 )?;
5225 Ok((
5226 load_committed_effective_config(&transaction_dir)?,
5227 revision,
5228 None,
5229 ))
5230 }
5231 })
5232 .await
5233 .map_err(|error| {
5234 AppError::InternalError(anyhow::anyhow!(
5235 "env credential transaction task failed: {error}"
5236 ))
5237 })?
5238 .map_err(|error| match error {
5239 ConfigStoreError::Conflict { expected, actual } => {
5240 AppError::ConfigConflict { expected, actual }
5241 }
5242 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
5243 ConfigStoreError::CommitIndeterminate(message) => AppError::InternalError(
5244 anyhow::anyhow!("configuration commit outcome is indeterminate: {message}"),
5245 ),
5246 ConfigStoreError::Io(error) => AppError::StorageError(error),
5247 ConfigStoreError::Json(_) => {
5248 AppError::BadRequest("configuration document is invalid".to_string())
5249 }
5250 ConfigStoreError::Watch(error) => {
5251 AppError::InternalError(anyhow::anyhow!("configuration watch failed: {error}"))
5252 }
5253 })?;
5254 let (published, installed) = match commit {
5255 Some(commit) => {
5256 let mut published = live_base;
5257 let installed = install_credential_section_commit(commit, &mut published)
5258 .map_err(|error| {
5259 AppError::InternalError(anyhow::anyhow!(
5260 "env process adoption failed: {error}"
5261 ))
5262 })?;
5263 (published, installed)
5264 }
5265 None => (
5266 candidate,
5267 InstalledCredentialSectionCommit {
5268 events: Vec::new(),
5269 metadata: read_credential_runtime_metadata(&app_data_dir).map_err(
5270 |error| {
5271 AppError::InternalError(anyhow::anyhow!(
5272 "env credential status unavailable after commit: {error}"
5273 ))
5274 },
5275 )?,
5276 section: None,
5277 },
5278 ),
5279 };
5280 published.publish_env_vars();
5281 *config.write().await = published.clone();
5282 publish_exact_facade_events(&account_sink, &installed.events).await?;
5283 let section = installed.section;
5284 Ok::<_, AppError>((published, revision, installed.metadata, section))
5285 });
5286 transaction.await.map_err(|error| {
5287 AppError::InternalError(anyhow::anyhow!(
5288 "env credential transaction task failed: {error}"
5289 ))
5290 })?
5291 }
5292
5293 pub async fn update_notification_credentials<F>(
5298 &self,
5299 expected_revision: u64,
5300 secret_intents: std::collections::BTreeSet<String>,
5301 reset_domain: bool,
5302 update: F,
5303 ) -> Result<
5304 (
5305 Config,
5306 u64,
5307 bamboo_config::CredentialSectionRuntimeMetadata,
5308 Option<bamboo_config::SectionEnvelope<Value>>,
5309 ),
5310 AppError,
5311 >
5312 where
5313 F: FnOnce(&mut Config) -> Result<(), AppError> + Send + 'static,
5314 {
5315 let config_io_lock = self.config_io_lock.clone();
5316 let config = self.config.clone();
5317 let app_data_dir = self.app_data_dir.clone();
5318 let account_sink = self.account_sink.clone();
5319 let config_facade = self.config_facade.clone();
5320 let transaction = tokio::spawn(async move {
5321 let _io = config_io_lock.lock().await;
5322 let live_base = {
5323 let current = config.read().await;
5324 reject_if_recovery_pending(¤t)?;
5325 current.clone()
5326 };
5327 let mut candidate = live_base.clone();
5328 if config_facade.is_some() {
5329 install_exact_credential_section_mutation_base(
5330 app_data_dir.clone(),
5331 SectionId::Notifications,
5332 expected_revision,
5333 &mut candidate,
5334 )
5335 .await?;
5336 }
5337 update(&mut candidate)?;
5338 let transaction_dir = app_data_dir.clone();
5339 let commit_facade = config_facade.clone();
5340 let (candidate, revision, commit) = tokio::task::spawn_blocking(move || {
5341 if let Some(facade) = commit_facade {
5342 let commit =
5343 bamboo_config::persist_notification_credential_transaction_at_revision_with_reset_and_adoption(
5344 &transaction_dir,
5345 &mut candidate,
5346 &secret_intents,
5347 reset_domain,
5348 expected_revision,
5349 facade.as_ref(),
5350 )?;
5351 let revision = commit.revision;
5352 Ok::<_, ConfigStoreError>((candidate, revision, Some(commit)))
5353 } else {
5354 let revision =
5355 bamboo_config::persist_notification_credential_transaction_at_revision_with_reset(
5356 &transaction_dir,
5357 &mut candidate,
5358 &secret_intents,
5359 reset_domain,
5360 expected_revision,
5361 )?;
5362 Ok((
5363 load_committed_effective_config(&transaction_dir)?,
5364 revision,
5365 None,
5366 ))
5367 }
5368 })
5369 .await
5370 .map_err(|error| {
5371 AppError::InternalError(anyhow::anyhow!(
5372 "notification credential transaction task failed: {error}"
5373 ))
5374 })?
5375 .map_err(|error| match error {
5376 ConfigStoreError::Conflict { expected, actual } => {
5377 AppError::ConfigConflict { expected, actual }
5378 }
5379 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
5380 ConfigStoreError::CommitIndeterminate(message) => {
5381 AppError::InternalError(anyhow::anyhow!(
5382 "configuration commit outcome is indeterminate: {message}"
5383 ))
5384 }
5385 ConfigStoreError::Io(error) => AppError::StorageError(error),
5386 ConfigStoreError::Json(_) => {
5387 AppError::BadRequest("configuration document is invalid".to_string())
5388 }
5389 ConfigStoreError::Watch(error) => {
5390 AppError::InternalError(anyhow::anyhow!("configuration watch failed: {error}"))
5391 }
5392 })?;
5393 let (published, installed) = match commit {
5394 Some(commit) => {
5395 let mut published = live_base;
5396 let installed = install_credential_section_commit(commit, &mut published)
5397 .map_err(|error| {
5398 AppError::InternalError(anyhow::anyhow!(
5399 "notification process adoption failed: {error}"
5400 ))
5401 })?;
5402 (published, installed)
5403 }
5404 None => (
5405 candidate,
5406 InstalledCredentialSectionCommit {
5407 events: Vec::new(),
5408 metadata: read_credential_runtime_metadata(&app_data_dir).map_err(
5409 |error| {
5410 AppError::InternalError(anyhow::anyhow!(
5411 "notification credential status unavailable after commit: {error}"
5412 ))
5413 },
5414 )?,
5415 section: None,
5416 },
5417 ),
5418 };
5419 *config.write().await = published.clone();
5420 publish_exact_facade_events(&account_sink, &installed.events).await?;
5421 let section = installed.section;
5422 Ok::<_, AppError>((published, revision, installed.metadata, section))
5423 });
5424 transaction.await.map_err(|error| {
5425 AppError::InternalError(anyhow::anyhow!(
5426 "notification credential transaction task failed: {error}"
5427 ))
5428 })?
5429 }
5430
5431 pub async fn update_connect_credentials<F>(
5436 &self,
5437 expected_revision: u64,
5438 secret_intents: bamboo_config::patch::ConnectSecretIntents,
5439 update: F,
5440 ) -> Result<
5441 (
5442 Config,
5443 u64,
5444 bamboo_config::CredentialSectionRuntimeMetadata,
5445 Option<bamboo_config::SectionEnvelope<Value>>,
5446 ),
5447 AppError,
5448 >
5449 where
5450 F: FnOnce(&mut Config) -> Result<(), AppError> + Send + 'static,
5451 {
5452 let config_io_lock = self.config_io_lock.clone();
5453 let config = self.config.clone();
5454 let app_data_dir = self.app_data_dir.clone();
5455 let account_sink = self.account_sink.clone();
5456 let config_facade = self.config_facade.clone();
5457 let transaction = tokio::spawn(async move {
5458 let _io = config_io_lock.lock().await;
5459 let live_base = {
5460 let current = config.read().await;
5461 reject_if_recovery_pending(¤t)?;
5462 current.clone()
5463 };
5464 let mut candidate = live_base.clone();
5465 if config_facade.is_some() {
5466 install_exact_credential_section_mutation_base(
5467 app_data_dir.clone(),
5468 SectionId::Connect,
5469 expected_revision,
5470 &mut candidate,
5471 )
5472 .await?;
5473 }
5474 update(&mut candidate)?;
5475 candidate.assign_connect_platform_ids();
5476 let transaction_dir = app_data_dir.clone();
5477 let commit_facade = config_facade.clone();
5478 let (candidate, revision, commit) = tokio::task::spawn_blocking(move || {
5479 if let Some(facade) = commit_facade {
5480 let commit =
5481 bamboo_config::persist_connect_credential_transaction_at_revision_with_adoption(
5482 &transaction_dir,
5483 &mut candidate,
5484 &secret_intents,
5485 expected_revision,
5486 facade.as_ref(),
5487 )?;
5488 let revision = commit.revision;
5489 Ok::<_, ConfigStoreError>((candidate, revision, Some(commit)))
5490 } else {
5491 let revision =
5492 bamboo_config::persist_connect_credential_transaction_at_revision(
5493 &transaction_dir,
5494 &mut candidate,
5495 &secret_intents,
5496 expected_revision,
5497 )?;
5498 Ok((
5499 load_committed_effective_config(&transaction_dir)?,
5500 revision,
5501 None,
5502 ))
5503 }
5504 })
5505 .await
5506 .map_err(|error| {
5507 AppError::InternalError(anyhow::anyhow!(
5508 "connect credential transaction task failed: {error}"
5509 ))
5510 })?
5511 .map_err(|error| match error {
5512 ConfigStoreError::Conflict { expected, actual } => {
5513 AppError::ConfigConflict { expected, actual }
5514 }
5515 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
5516 ConfigStoreError::CommitIndeterminate(message) => AppError::InternalError(
5517 anyhow::anyhow!("configuration commit outcome is indeterminate: {message}"),
5518 ),
5519 ConfigStoreError::Io(error) => AppError::StorageError(error),
5520 ConfigStoreError::Json(_) => {
5521 AppError::BadRequest("configuration document is invalid".to_string())
5522 }
5523 ConfigStoreError::Watch(error) => {
5524 AppError::InternalError(anyhow::anyhow!("configuration watch failed: {error}"))
5525 }
5526 })?;
5527 let (published, installed) = match commit {
5528 Some(commit) => {
5529 let mut published = live_base;
5530 let installed = install_credential_section_commit(commit, &mut published)
5531 .map_err(|error| {
5532 AppError::InternalError(anyhow::anyhow!(
5533 "connect process adoption failed: {error}"
5534 ))
5535 })?;
5536 (published, installed)
5537 }
5538 None => (
5539 candidate,
5540 InstalledCredentialSectionCommit {
5541 events: Vec::new(),
5542 metadata: read_credential_runtime_metadata(&app_data_dir).map_err(
5543 |error| {
5544 AppError::InternalError(anyhow::anyhow!(
5545 "connect credential status unavailable after commit: {error}"
5546 ))
5547 },
5548 )?,
5549 section: None,
5550 },
5551 ),
5552 };
5553 *config.write().await = published.clone();
5554 publish_exact_facade_events(&account_sink, &installed.events).await?;
5555 let section = installed.section;
5556 Ok::<_, AppError>((published, revision, installed.metadata, section))
5557 });
5558 transaction.await.map_err(|error| {
5559 AppError::InternalError(anyhow::anyhow!(
5560 "connect credential transaction task failed: {error}"
5561 ))
5562 })?
5563 }
5564
5565 pub async fn update_access_control_credentials<F>(
5568 &self,
5569 expected_revision: u64,
5570 password_intent: bool,
5571 device_intents: std::collections::BTreeSet<String>,
5572 update: F,
5573 ) -> Result<
5574 (
5575 Config,
5576 u64,
5577 bamboo_config::CredentialSectionRuntimeMetadata,
5578 Option<bamboo_config::SectionEnvelope<Value>>,
5579 ),
5580 AppError,
5581 >
5582 where
5583 F: FnOnce(&mut Config) -> Result<(), AppError> + Send + 'static,
5584 {
5585 let config_io_lock = self.config_io_lock.clone();
5586 let config = self.config.clone();
5587 let app_data_dir = self.app_data_dir.clone();
5588 let account_sink = self.account_sink.clone();
5589 let config_facade = self.config_facade.clone();
5590 let transaction = tokio::spawn(async move {
5591 let _io = config_io_lock.lock().await;
5592 let live_base = {
5593 let current = config.read().await;
5594 reject_if_recovery_pending(¤t)?;
5595 current.clone()
5596 };
5597 let mut candidate = live_base.clone();
5598 if config_facade.is_some() {
5599 install_exact_credential_section_mutation_base(
5600 app_data_dir.clone(),
5601 SectionId::AccessControl,
5602 expected_revision,
5603 &mut candidate,
5604 )
5605 .await?;
5606 }
5607 update(&mut candidate)?;
5608 let transaction_dir = app_data_dir.clone();
5609 let commit_facade = config_facade.clone();
5610 let (candidate, revision, commit) = tokio::task::spawn_blocking(move || {
5611 if let Some(facade) = commit_facade {
5612 let commit =
5613 bamboo_config::persist_access_control_credential_transaction_at_revision_with_adoption(
5614 &transaction_dir,
5615 &mut candidate,
5616 password_intent,
5617 &device_intents,
5618 expected_revision,
5619 facade.as_ref(),
5620 )?;
5621 let revision = commit.revision;
5622 Ok::<_, ConfigStoreError>((candidate, revision, Some(commit)))
5623 } else {
5624 let revision =
5625 bamboo_config::persist_access_control_credential_transaction_at_revision(
5626 &transaction_dir,
5627 &mut candidate,
5628 password_intent,
5629 &device_intents,
5630 expected_revision,
5631 )?;
5632 Ok((
5633 load_committed_effective_config(&transaction_dir)?,
5634 revision,
5635 None,
5636 ))
5637 }
5638 })
5639 .await
5640 .map_err(|error| {
5641 AppError::InternalError(anyhow::anyhow!(
5642 "access-control credential transaction task failed: {error}"
5643 ))
5644 })?
5645 .map_err(|error| match error {
5646 ConfigStoreError::Conflict { expected, actual } => {
5647 AppError::ConfigConflict { expected, actual }
5648 }
5649 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
5650 ConfigStoreError::CommitIndeterminate(message) => AppError::InternalError(
5651 anyhow::anyhow!("configuration commit outcome is indeterminate: {message}"),
5652 ),
5653 ConfigStoreError::Io(error) => AppError::StorageError(error),
5654 ConfigStoreError::Json(_) => {
5655 AppError::BadRequest("configuration document is invalid".to_string())
5656 }
5657 ConfigStoreError::Watch(error) => {
5658 AppError::InternalError(anyhow::anyhow!("configuration watch failed: {error}"))
5659 }
5660 })?;
5661 let (published, installed) = match commit {
5662 Some(commit) => {
5663 let mut published = live_base;
5664 let installed = install_credential_section_commit(commit, &mut published)
5665 .map_err(|error| {
5666 AppError::InternalError(anyhow::anyhow!(
5667 "access-control process adoption failed: {error}"
5668 ))
5669 })?;
5670 (published, installed)
5671 }
5672 None => (
5673 candidate,
5674 InstalledCredentialSectionCommit {
5675 events: Vec::new(),
5676 metadata: read_credential_runtime_metadata(&app_data_dir).map_err(
5677 |error| {
5678 AppError::InternalError(anyhow::anyhow!(
5679 "access-control credential status unavailable after commit: {error}"
5680 ))
5681 },
5682 )?,
5683 section: None,
5684 },
5685 ),
5686 };
5687 *config.write().await = published.clone();
5688 publish_exact_facade_events(&account_sink, &installed.events).await?;
5689 let section = installed.section;
5690 Ok::<_, AppError>((published, revision, installed.metadata, section))
5691 });
5692 transaction.await.map_err(|error| {
5693 AppError::InternalError(anyhow::anyhow!(
5694 "access-control credential transaction task failed: {error}"
5695 ))
5696 })?
5697 }
5698
5699 pub async fn update_cluster_fabric_credentials<F>(
5704 &self,
5705 expected_revision: u64,
5706 node_intents: std::collections::BTreeMap<
5707 String,
5708 bamboo_config::ClusterNodeCredentialIntents,
5709 >,
5710 update: F,
5711 ) -> Result<bamboo_server_tools::FabricCommitSnapshot, AppError>
5712 where
5713 F: FnOnce(&mut Config) -> Result<(), AppError> + Send + 'static,
5714 {
5715 self.update_cluster_fabric_credentials_guarded(
5716 expected_revision,
5717 node_intents,
5718 None,
5719 update,
5720 )
5721 .await
5722 }
5723
5724 pub(crate) async fn delete_cluster_node_credentials<F>(
5729 &self,
5730 expected_revision: u64,
5731 node_id: String,
5732 node_intents: std::collections::BTreeMap<
5733 String,
5734 bamboo_config::ClusterNodeCredentialIntents,
5735 >,
5736 update: F,
5737 ) -> Result<bamboo_server_tools::FabricCommitSnapshot, AppError>
5738 where
5739 F: FnOnce(&mut Config) -> Result<(), AppError> + Send + 'static,
5740 {
5741 self.update_cluster_fabric_credentials_guarded(
5742 expected_revision,
5743 node_intents,
5744 Some(node_id),
5745 update,
5746 )
5747 .await
5748 }
5749
5750 async fn update_cluster_fabric_credentials_guarded<F>(
5751 &self,
5752 expected_revision: u64,
5753 node_intents: std::collections::BTreeMap<
5754 String,
5755 bamboo_config::ClusterNodeCredentialIntents,
5756 >,
5757 required_stopped_node: Option<String>,
5758 update: F,
5759 ) -> Result<bamboo_server_tools::FabricCommitSnapshot, AppError>
5760 where
5761 F: FnOnce(&mut Config) -> Result<(), AppError> + Send + 'static,
5762 {
5763 let config_io_lock = self.config_io_lock.clone();
5764 let config = self.config.clone();
5765 let app_data_dir = self.app_data_dir.clone();
5766 let account_sink = self.account_sink.clone();
5767 let config_facade = self.config_facade.clone();
5768 let deployed_registry = self.fabric_deployer.registry();
5769 let transaction = tokio::spawn(async move {
5770 let _io = config_io_lock.lock().await;
5771 if let Some(node_id) = required_stopped_node.as_deref() {
5772 let deployed = deployed_registry.lock().await;
5773 if deployed.contains_key(&bamboo_server_tools::registry_keys::node_key(node_id)) {
5774 return Err(AppError::BadRequest(format!(
5775 "node '{node_id}' is deployed; stop it before deleting it"
5776 )));
5777 }
5778 }
5779 let facade = config_facade.as_ref().ok_or_else(|| {
5780 AppError::BadRequest(
5781 "cluster mutations require the modular configuration facade".to_string(),
5782 )
5783 })?;
5784 let mut candidate = {
5785 let current = config.read().await;
5786 reject_if_recovery_pending(¤t)?;
5787 current.clone()
5788 };
5789 let snapshot_dir = app_data_dir.clone();
5802 let exact = tokio::task::spawn_blocking(move || {
5803 bamboo_config::read_exact_cluster_fabric_snapshot(&snapshot_dir, None)
5804 })
5805 .await
5806 .map_err(|error| {
5807 AppError::InternalError(anyhow::anyhow!("cluster snapshot task failed: {error}"))
5808 })?
5809 .map_err(|error| match error {
5810 ConfigStoreError::Conflict { expected, actual } => {
5811 AppError::ConfigConflict { expected, actual }
5812 }
5813 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
5814 ConfigStoreError::CommitIndeterminate(message) => AppError::InternalError(
5815 anyhow::anyhow!("configuration commit outcome is indeterminate: {message}"),
5816 ),
5817 ConfigStoreError::Io(error) => AppError::StorageError(error),
5818 ConfigStoreError::Json(_) => {
5819 AppError::BadRequest("configuration document is invalid".to_string())
5820 }
5821 ConfigStoreError::Watch(error) => {
5822 AppError::InternalError(anyhow::anyhow!("configuration watch failed: {error}"))
5823 }
5824 })?;
5825 if exact.section.revision != expected_revision {
5826 return Err(AppError::ConfigConflict {
5827 expected: expected_revision,
5828 actual: exact.section.revision,
5829 });
5830 }
5831 if exact.section.status != SectionStatus::Healthy
5832 || exact.section.source_kind != SectionSourceKind::File
5833 || exact.credential_health.status == SectionStatus::Degraded
5834 {
5835 return Err(AppError::BadRequest(
5836 "revision-bound cluster mutations require healthy primary authorities"
5837 .to_string(),
5838 ));
5839 }
5840 candidate.cluster_fabric = exact.cluster_fabric;
5841 update(&mut candidate)?;
5842 let transaction_dir = app_data_dir.clone();
5843 let commit_facade = facade.clone();
5844 let (mut candidate, commit) = tokio::task::spawn_blocking(move || {
5845 let commit =
5846 bamboo_config::persist_cluster_fabric_credential_transaction_with_adoption(
5847 &transaction_dir,
5848 &mut candidate,
5849 &node_intents,
5850 expected_revision,
5851 commit_facade.as_ref(),
5852 |_, _| {
5853 #[cfg(test)]
5854 run_cluster_after_commit_before_adoption_test_hook(
5855 &transaction_dir,
5856 expected_revision,
5857 );
5858 },
5859 )?;
5860 Ok::<_, ConfigStoreError>((candidate, commit))
5861 })
5862 .await
5863 .map_err(|error| {
5864 AppError::InternalError(anyhow::anyhow!(
5865 "cluster credential transaction task failed: {error}"
5866 ))
5867 })?
5868 .map_err(|error| match error {
5869 ConfigStoreError::Conflict { expected, actual } => {
5870 AppError::ConfigConflict { expected, actual }
5871 }
5872 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
5873 ConfigStoreError::CommitIndeterminate(message) => AppError::InternalError(
5874 anyhow::anyhow!("configuration commit outcome is indeterminate: {message}"),
5875 ),
5876 ConfigStoreError::Io(error) => AppError::StorageError(error),
5877 ConfigStoreError::Json(_) => {
5878 AppError::BadRequest("configuration document is invalid".to_string())
5879 }
5880 ConfigStoreError::Watch(error) => {
5881 AppError::InternalError(anyhow::anyhow!("configuration watch failed: {error}"))
5882 }
5883 })?;
5884 let bamboo_config::ClusterFabricTransactionCommit {
5885 revision,
5886 adoption,
5887 credential_adoption,
5888 committed_recovery,
5889 runtime,
5890 } = commit;
5891 let runtime = match runtime {
5892 Ok(bamboo_config::ClusterFabricRuntimeSnapshot {
5893 cluster_fabric,
5894 credential_statuses,
5895 credential_health,
5896 }) => {
5897 candidate.cluster_fabric = cluster_fabric;
5898 Ok((credential_statuses, credential_health))
5899 }
5900 Err(error) if revision == expected_revision => {
5901 return Err(AppError::InternalError(anyhow::anyhow!(
5902 "cluster configuration at revision {revision} could not materialize its exact runtime credentials: {error}"
5903 )));
5904 }
5905 Err(error) => {
5906 candidate.clear_cluster_runtime_credentials();
5907 Err(error)
5908 }
5909 };
5910 *config.write().await = candidate.clone();
5911 let event = match adoption {
5912 Some(Ok(event)) => Some(event),
5913 Some(Err(error)) => {
5914 return Err(AppError::InternalError(anyhow::anyhow!(
5915 "cluster configuration committed at revision {} but process adoption failed: {error}",
5916 revision
5917 )));
5918 }
5919 None if revision == expected_revision => None,
5920 None => {
5921 return Err(AppError::InternalError(anyhow::anyhow!(
5922 "cluster configuration committed at revision {} without a process adoption result",
5923 revision
5924 )));
5925 }
5926 };
5927 let section = facade
5928 .registry()
5929 .envelope_value(SectionId::ClusterFabric)
5930 .map_err(|error| {
5931 AppError::InternalError(anyhow::anyhow!(
5932 "committed cluster section envelope is unavailable: {error}"
5933 ))
5934 })?;
5935 if section.revision != revision {
5936 return Err(AppError::InternalError(anyhow::anyhow!(
5937 "cluster configuration committed at revision {} but facade retained revision {}",
5938 revision,
5939 section.revision
5940 )));
5941 }
5942 if let Some(event) = event.as_ref() {
5943 publish_registry_event(&account_sink, event).await;
5944 }
5945 if let Err(error) = committed_recovery {
5946 return Err(AppError::InternalError(anyhow::anyhow!(
5947 "cluster configuration committed at revision {revision} but transaction recovery failed: {error}"
5948 )));
5949 }
5950 if let Some(Err(error)) = credential_adoption {
5951 return Err(AppError::InternalError(anyhow::anyhow!(
5952 "cluster configuration committed at revision {revision} but credential facade adoption failed: {error}"
5953 )));
5954 }
5955 let (credential_statuses, credential_health) = runtime.map_err(|error| {
5956 AppError::InternalError(anyhow::anyhow!(
5957 "cluster configuration committed at revision {revision} but could not materialize its exact runtime credentials: {error}"
5958 ))
5959 })?;
5960 Ok::<_, AppError>(bamboo_server_tools::FabricCommitSnapshot {
5961 config: candidate,
5962 section,
5963 credential_statuses,
5964 credential_health,
5965 })
5966 });
5967 transaction.await.map_err(|error| {
5968 AppError::InternalError(anyhow::anyhow!(
5969 "cluster credential transaction task failed: {error}"
5970 ))
5971 })?
5972 }
5973
5974 pub async fn update_proxy_auth_credential(
5978 &self,
5979 auth: Option<bamboo_config::ProxyAuth>,
5980 expected_revision: u64,
5981 effects: ConfigUpdateEffects,
5982 ) -> Result<
5983 (
5984 Config,
5985 u64,
5986 bamboo_config::CredentialStatus,
5987 bamboo_config::CredentialStoreHealth,
5988 Option<bamboo_config::SectionEnvelope<Value>>,
5989 ),
5990 AppError,
5991 > {
5992 self.update_core_with_proxy_credential(expected_revision, effects, move |candidate| {
5993 candidate.proxy_auth = auth;
5994 })
5995 .await
5996 }
5997
5998 async fn update_core_with_proxy_credential<F>(
5999 &self,
6000 expected_revision: u64,
6001 effects: ConfigUpdateEffects,
6002 update: F,
6003 ) -> Result<
6004 (
6005 Config,
6006 u64,
6007 bamboo_config::CredentialStatus,
6008 bamboo_config::CredentialStoreHealth,
6009 Option<bamboo_config::SectionEnvelope<Value>>,
6010 ),
6011 AppError,
6012 >
6013 where
6014 F: FnOnce(&mut Config) + Send + 'static,
6015 {
6016 let config_io_lock = self.config_io_lock.clone();
6017 let config = self.config.clone();
6018 let app_data_dir = self.app_data_dir.clone();
6019 let credential_store = self.credential_store.clone();
6020 let provider_registry = self.provider_registry.clone();
6021 let provider = self.provider.clone();
6022 let mcp_manager = self.mcp_manager.clone();
6023 let config_live_health = self.config_live_health.clone();
6024 let mcp_config_live_health = self.mcp_config_live_health.clone();
6025 let config_facade = self.config_facade.clone();
6026 let account_sink = self.account_sink.clone();
6027
6028 let transaction = tokio::spawn(async move {
6033 let _io = config_io_lock.lock().await;
6034 let live_base = {
6035 let cfg = config.read().await;
6036 reject_if_recovery_pending(&cfg)?;
6037 cfg.clone()
6038 };
6039 let mut candidate = live_base.clone();
6040 if config_facade.is_some() {
6041 install_exact_credential_section_mutation_base(
6042 app_data_dir.clone(),
6043 SectionId::Core,
6044 expected_revision,
6045 &mut candidate,
6046 )
6047 .await?;
6048 }
6049 update(&mut candidate);
6050 if config_facade.is_none() {
6051 candidate.assign_connect_platform_ids();
6052 candidate.refresh_encrypted_secrets().map_err(|error| {
6053 AppError::InternalError(anyhow::anyhow!(
6054 "Failed to refresh encrypted secrets: {error}"
6055 ))
6056 })?;
6057 }
6058 let transaction_dir = app_data_dir.clone();
6059 let status_reference =
6060 candidate
6061 .proxy_auth_credential_ref
6062 .clone()
6063 .unwrap_or_else(|| {
6064 bamboo_config::CredentialRef::parse("proxy.default.auth")
6065 .expect("canonical proxy credential reference is valid")
6066 });
6067 let commit_facade = config_facade.clone();
6068 let (candidate, revision, reference, commit) =
6069 tokio::task::spawn_blocking(move || {
6070 if let Some(facade) = commit_facade {
6071 let commit =
6072 bamboo_config::persist_proxy_auth_credential_transaction_at_revision_with_adoption(
6073 &transaction_dir,
6074 &mut candidate,
6075 expected_revision,
6076 facade.as_ref(),
6077 )?;
6078 let revision = commit.revision;
6079 Ok::<_, ConfigStoreError>((
6080 candidate,
6081 revision,
6082 status_reference,
6083 Some(commit),
6084 ))
6085 } else {
6086 let revision =
6087 bamboo_config::persist_proxy_auth_credential_transaction_at_revision(
6088 &transaction_dir,
6089 &mut candidate,
6090 expected_revision,
6091 )?;
6092 Ok((
6093 load_committed_effective_config(&transaction_dir)?,
6094 revision,
6095 status_reference,
6096 None,
6097 ))
6098 }
6099 })
6100 .await
6101 .map_err(|error| {
6102 AppError::InternalError(anyhow::anyhow!(
6103 "proxy credential transaction task failed: {error}"
6104 ))
6105 })?
6106 .map_err(|error| match error {
6107 ConfigStoreError::Conflict { expected, actual } => {
6108 AppError::ConfigConflict { expected, actual }
6109 }
6110 ConfigStoreError::Validation(message) => AppError::BadRequest(message),
6111 ConfigStoreError::CommitIndeterminate(message) => AppError::InternalError(
6112 anyhow::anyhow!("configuration commit outcome is indeterminate: {message}"),
6113 ),
6114 ConfigStoreError::Io(error) => AppError::StorageError(error),
6115 ConfigStoreError::Json(_) => {
6116 AppError::BadRequest("configuration document is invalid".to_string())
6117 }
6118 ConfigStoreError::Watch(error) => {
6119 AppError::InternalError(anyhow::anyhow!("configuration watch failed: {error}"))
6120 }
6121 })?;
6122 let (published, installed) = match commit {
6123 Some(commit) => {
6124 let mut published = live_base;
6125 let installed = install_credential_section_commit(commit, &mut published)
6126 .map_err(|error| {
6127 AppError::InternalError(anyhow::anyhow!(
6128 "proxy process adoption failed: {error}"
6129 ))
6130 })?;
6131 (published, Some(installed))
6132 }
6133 None => (candidate, None),
6134 };
6135 let section = installed
6136 .as_ref()
6137 .and_then(|installed| installed.section.clone());
6138
6139 published.publish_env_vars();
6143 *config.write().await = published.clone();
6144
6145 if let Some(installed) = installed.as_ref() {
6146 publish_exact_facade_events(&account_sink, &installed.events).await?;
6147 }
6148
6149 Self::apply_config_effects_owned(
6150 published.clone(),
6151 effects,
6152 ConfigRuntimeEffectContext {
6153 app_data_dir,
6154 config_facade,
6155 provider_registry,
6156 provider,
6157 mcp_manager,
6158 account_sink,
6159 config_live_health,
6160 mcp_config_live_health,
6161 },
6162 )
6163 .await?;
6164
6165 let (status, health) = if let Some(installed) = installed {
6166 (
6167 installed.metadata.status(&reference),
6168 installed.metadata.credential_health,
6169 )
6170 } else {
6171 credential_store
6172 .status_with_health(&reference)
6173 .map_err(|error| match error {
6174 ConfigStoreError::Conflict { expected, actual } => {
6175 AppError::ConfigConflict { expected, actual }
6176 }
6177 ConfigStoreError::Validation(_)
6178 | ConfigStoreError::CommitIndeterminate(_)
6179 | ConfigStoreError::Json(_) => AppError::InternalError(anyhow::anyhow!(
6180 "credential store validation failed"
6181 )),
6182 ConfigStoreError::Io(error) => AppError::StorageError(error),
6183 ConfigStoreError::Watch(error) => AppError::InternalError(anyhow::anyhow!(
6184 "configuration watch failed: {error}"
6185 )),
6186 })?
6187 };
6188 Ok::<_, AppError>((published, revision, status, health, section))
6189 });
6190 transaction.await.map_err(|error| {
6191 AppError::InternalError(anyhow::anyhow!(
6192 "proxy credential mutation task failed: {error}"
6193 ))
6194 })?
6195 }
6196
6197 pub async fn replace_config(
6199 &self,
6200 mut new_config: Config,
6201 effects: ConfigUpdateEffects,
6202 ) -> Result<Config, AppError> {
6203 if self.config_facade.is_none() {
6209 new_config.assign_connect_platform_ids();
6210 new_config.refresh_encrypted_secrets().map_err(|e| {
6214 AppError::InternalError(anyhow::anyhow!("Failed to refresh encrypted secrets: {e}"))
6215 })?;
6216 }
6217
6218 let io = self.config_io_lock.clone().lock_owned().await;
6219 restore_authoritative_cluster_fabric(self.config_facade.as_ref(), &mut new_config);
6220 let (was_off, live_base) = {
6221 let cfg = self.config.read().await;
6222 reject_if_recovery_pending(&cfg)?;
6225 (cfg.plugin_trust.enforcement_is_off(), cfg.clone())
6226 };
6227 let config = self.config.clone();
6228 let app_data_dir = self.app_data_dir.clone();
6229 let config_facade = self.config_facade.clone();
6230 let account_sink = self.account_sink.clone();
6231 let provider_registry = self.provider_registry.clone();
6232 let provider = self.provider.clone();
6233 let mcp_manager = self.mcp_manager.clone();
6234 let config_live_health = self.config_live_health.clone();
6235 let mcp_config_live_health = self.mcp_config_live_health.clone();
6236 let transaction = tokio::spawn(async move {
6237 let new_config = {
6242 let _io = io;
6243 let commit = Self::persist_config_snapshot(
6244 app_data_dir.clone(),
6245 config_facade.clone(),
6246 new_config.clone(),
6247 )
6248 .await?;
6249 let mut published = if commit.is_some() {
6250 live_base
6251 } else {
6252 new_config
6253 };
6254 let events = match commit {
6255 Some(commit) => {
6256 install_facade_config_commit(commit, &mut published).map_err(|error| {
6257 AppError::InternalError(anyhow::anyhow!(
6258 "failed to install committed configuration section: {error}"
6259 ))
6260 })?
6261 }
6262 None => Vec::new(),
6263 };
6264 let enforcement_newly_off = !was_off && published.plugin_trust.enforcement_is_off();
6265 {
6266 let mut current = config.write().await;
6267 preserve_runtime_broker(&mut published, ¤t);
6268 published.publish_env_vars();
6269 *current = published.clone();
6270 }
6271 if enforcement_newly_off {
6274 warn_plugin_trust_enforcement_off();
6275 }
6276 publish_exact_facade_events(&account_sink, &events).await?;
6277 Self::apply_config_effects_owned(
6278 published.clone(),
6279 effects,
6280 ConfigRuntimeEffectContext {
6281 app_data_dir,
6282 config_facade,
6283 provider_registry,
6284 provider,
6285 mcp_manager,
6286 account_sink,
6287 config_live_health,
6288 mcp_config_live_health,
6289 },
6290 )
6291 .await?;
6292 published
6293 };
6294 Ok::<_, AppError>(new_config)
6295 });
6296 transaction.await.map_err(|error| {
6297 AppError::InternalError(anyhow::anyhow!(
6298 "config replacement transaction task failed: {error}"
6299 ))
6300 })?
6301 }
6302
6303 async fn apply_config_effects_owned(
6304 new_config: Config,
6305 effects: ConfigUpdateEffects,
6306 context: ConfigRuntimeEffectContext,
6307 ) -> Result<(), AppError> {
6308 Self::apply_config_effects_owned_after_forcing(new_config, effects, context, HashSet::new())
6309 .await
6310 }
6311
6312 async fn apply_config_effects_owned_after_forcing(
6313 new_config: Config,
6314 effects: ConfigUpdateEffects,
6315 context: ConfigRuntimeEffectContext,
6316 forced_mcp_replacements: HashSet<String>,
6317 ) -> Result<(), AppError> {
6318 let ConfigRuntimeEffectContext {
6319 app_data_dir,
6320 config_facade,
6321 provider_registry,
6322 provider,
6323 mcp_manager,
6324 account_sink,
6325 config_live_health,
6326 mcp_config_live_health,
6327 } = context;
6328 let mut provider_failure = None;
6333 if !matches!(
6334 effects.reload_provider,
6335 bamboo_config::patch::ReloadMode::None
6336 ) {
6337 let candidate = async {
6338 let candidate_registry =
6339 bamboo_llm::ProviderRegistry::from_config(&new_config, app_data_dir.clone())
6340 .await?;
6341 let default_provider_name = candidate_registry.default_provider_name();
6342 let candidate_provider = candidate_registry.get_default().ok_or_else(|| {
6343 let message = if new_config.has_provider_instances() {
6344 format!(
6345 "Default provider instance '{}' is not available or failed to initialize",
6346 default_provider_name
6347 )
6348 } else {
6349 format!(
6350 "Provider '{}' is not available or failed to initialize",
6351 new_config.provider
6352 )
6353 };
6354 bamboo_llm::LLMError::Auth(message)
6355 })?;
6356 Ok::<_, bamboo_llm::LLMError>((
6357 candidate_registry,
6358 candidate_provider,
6359 default_provider_name,
6360 ))
6361 }
6362 .await;
6363
6364 match candidate {
6365 Ok((candidate_registry, candidate_provider, default_provider_name)) => {
6366 #[cfg(test)]
6367 run_generic_before_provider_publish_test_hook(&app_data_dir);
6368 {
6369 let mut live_provider = provider.write().await;
6372 provider_registry.replace_with(candidate_registry);
6373 *live_provider = candidate_provider;
6374 }
6375 if let Some(facade) = config_facade.as_ref() {
6376 let snapshot = facade.registry().providers.snapshot();
6377 set_live_health_revision(
6378 &config_live_health,
6379 snapshot.revision,
6380 Some((snapshot.source_path.clone(), snapshot.source_kind)),
6381 );
6382 } else {
6383 update_live_health(
6384 &config_live_health,
6385 SectionStatus::Healthy,
6386 None,
6387 true,
6388 Some((app_data_dir.join("config.json"), SectionSourceKind::File)),
6389 );
6390 }
6391 tracing::info!(
6392 default_provider = %default_provider_name,
6393 "Provider reloaded successfully"
6394 );
6395 }
6396 Err(_) => {
6397 tracing::warn!("committed provider generation could not start");
6398 let message =
6399 "provider runtime initialization failed; retaining last-known-good runtime"
6400 .to_string();
6401 if let Some(facade) = config_facade.as_ref() {
6402 if let Some(event) = facade
6403 .registry()
6404 .mark_runtime_degraded(SectionId::Providers, message.clone())
6405 {
6406 let snapshot = facade.registry().providers.snapshot();
6407 set_live_health_from_snapshot(&config_live_health, &snapshot);
6408 publish_registry_event(&account_sink, &event).await;
6409 }
6410 } else {
6411 publish_section_failure(
6412 &config_live_health,
6413 &account_sink,
6414 "providers",
6415 SectionStatus::Degraded,
6416 message.clone(),
6417 )
6418 .await;
6419 }
6420 if matches!(
6421 effects.reload_provider,
6422 bamboo_config::patch::ReloadMode::Strict
6423 ) {
6424 provider_failure = Some(AppError::InternalError(anyhow::anyhow!(message)));
6425 }
6426 }
6427 }
6428 }
6429
6430 let mut mcp_failure = None;
6431 if !matches!(
6432 effects.reconcile_mcp,
6433 bamboo_config::patch::ReloadMode::None
6434 ) {
6435 match mcp_manager
6436 .reconcile_from_config_transactional_after_forcing(
6437 &new_config.mcp,
6438 &forced_mcp_replacements,
6439 || async { Ok(()) },
6440 )
6441 .await
6442 {
6443 Ok(()) => {
6444 if let Some(facade) = config_facade.as_ref() {
6445 let snapshot = facade.registry().mcp.snapshot();
6446 set_live_health_revision(
6447 &mcp_config_live_health,
6448 snapshot.revision,
6449 Some((snapshot.source_path.clone(), snapshot.source_kind)),
6450 );
6451 } else {
6452 update_live_health(
6453 &mcp_config_live_health,
6454 SectionStatus::Healthy,
6455 None,
6456 true,
6457 Some((app_data_dir.join("config.json"), SectionSourceKind::File)),
6458 );
6459 }
6460 }
6461 Err(_) => {
6462 tracing::warn!("committed MCP generation could not start");
6463 let message =
6464 "MCP runtime initialization failed; retaining last-known-good runtime"
6465 .to_string();
6466 if let Some(facade) = config_facade.as_ref() {
6467 if let Some(event) = facade
6468 .registry()
6469 .mark_runtime_degraded(SectionId::Mcp, message.clone())
6470 {
6471 let snapshot = facade.registry().mcp.snapshot();
6472 set_live_health_from_snapshot(&mcp_config_live_health, &snapshot);
6473 publish_registry_event(&account_sink, &event).await;
6474 }
6475 } else {
6476 publish_section_failure(
6477 &mcp_config_live_health,
6478 &account_sink,
6479 "mcp",
6480 SectionStatus::Degraded,
6481 message.clone(),
6482 )
6483 .await;
6484 }
6485 if matches!(
6486 effects.reconcile_mcp,
6487 bamboo_config::patch::ReloadMode::Strict
6488 ) {
6489 mcp_failure = Some(AppError::InternalError(anyhow::anyhow!(message)));
6490 }
6491 }
6492 }
6493 }
6494
6495 provider_failure.or(mcp_failure).map_or(Ok(()), Err)
6496 }
6497
6498 pub async fn confirm_config_recovery(&self, accept: bool) -> Result<Config, AppError> {
6514 let _io = self.config_io_lock.lock().await;
6515
6516 if !accept {
6517 let cfg = self.config.read().await;
6518 return match cfg.recovery_status() {
6519 Some(_) => Ok(cfg.clone()),
6520 None => Err(AppError::BadRequest(
6521 "No pending config-corruption recovery to resolve".to_string(),
6522 )),
6523 };
6524 }
6525
6526 let mut candidate = {
6527 let cfg = self.config.read().await;
6528 match cfg.recovery_status() {
6529 Some(_) => cfg.clone(),
6530 None => {
6531 return Err(AppError::BadRequest(
6532 "No pending config-corruption recovery to resolve".to_string(),
6533 ))
6534 }
6535 }
6536 };
6537
6538 let data_dir = self.app_data_dir.clone();
6539 candidate = tokio::task::spawn_blocking(move || {
6540 candidate
6541 .confirm_recovery_and_save_to_dir(data_dir)
6542 .map(|_| candidate)
6543 })
6544 .await
6545 .map_err(|e| {
6546 AppError::InternalError(anyhow::anyhow!("Config recovery-confirm task failed: {e}"))
6547 })?
6548 .map_err(|e| {
6549 AppError::InternalError(anyhow::anyhow!("Failed to save recovered config: {e}"))
6550 })?;
6551
6552 {
6553 let mut cfg = self.config.write().await;
6554 *cfg = candidate.clone();
6555 cfg.publish_env_vars();
6556 }
6557
6558 Ok(candidate)
6559 }
6560}
6561
6562fn reject_if_recovery_pending(cfg: &Config) -> Result<(), AppError> {
6570 if let Some(status) = cfg.recovery_status() {
6571 if !status.confirmed {
6572 return Err(AppError::ConfigRecoveryPending(format!(
6573 "config.json was recovered from corruption ({:?}) and is awaiting \
6574 confirmation; confirm or reject the recovery (see /bamboo/config/recovery-status \
6575 and /bamboo/config/recovery/confirm) before changing settings",
6576 status.source
6577 )));
6578 }
6579 }
6580 Ok(())
6581}
6582
6583pub(crate) fn warn_plugin_trust_enforcement_off() {
6593 tracing::warn!(
6594 "plugin_trust.enforcement is OFF — plugin installs from ANY URL are accepted \
6595 without host/signature/checksum verification (config.json plugin_trust.enforcement)"
6596 );
6597}
6598
6599#[cfg(test)]
6600mod live_reload_tests {
6601 use super::*;
6602 use bamboo_agent_core::{Message, ToolSchema};
6603 use bamboo_llm::{LLMError, LLMStream};
6604 use bamboo_mcp::{McpServerConfig, ReconnectConfig, StdioConfig};
6605
6606 struct WorkingProvider;
6607
6608 fn stop_config_watcher(state: &mut AppState) {
6609 state.config_watcher.stop.store(true, Ordering::Relaxed);
6610 if let Some(task) = state.config_watcher.apply_task.take() {
6611 task.abort();
6612 }
6613 if let Some(task) = state.config_watcher.watcher_task.take() {
6614 task.join().unwrap();
6615 }
6616 }
6617
6618 fn restart_config_watcher(state: &mut AppState) {
6619 let (runtime, provider_health, mcp_health) = ConfigWatcherRuntime::start(
6620 state.app_data_dir.clone(),
6621 state.config.clone(),
6622 state.config_facade.clone(),
6623 state.config_io_lock.clone(),
6624 state.provider_registry.clone(),
6625 state.provider.clone(),
6626 state.mcp_manager.clone(),
6627 state.account_sink.clone(),
6628 );
6629 state.config_watcher = runtime;
6630 state.config_live_health = provider_health;
6631 state.mcp_config_live_health = mcp_health;
6632 }
6633
6634 async fn insert_registry_worker(state: &AppState, key: String, worker_id: &str) {
6635 #[cfg(unix)]
6636 let child = tokio::process::Command::new("/bin/sleep")
6637 .arg("30")
6638 .spawn()
6639 .unwrap();
6640 #[cfg(windows)]
6641 let child = tokio::process::Command::new("cmd")
6642 .args(["/C", "timeout", "/T", "30", "/NOBREAK"])
6643 .spawn()
6644 .unwrap();
6645 state.fabric_deployer.registry().lock().await.insert(
6646 key,
6647 bamboo_server_tools::Deployed {
6648 env: "test".to_string(),
6649 handle: bamboo_broker::DeployedAgent::from_parts(worker_id, child, None),
6650 },
6651 );
6652 }
6653
6654 fn disabled_mcp_config(id: &str) -> McpConfig {
6655 McpConfig {
6656 version: 1,
6657 servers: vec![McpServerConfig {
6658 id: id.to_string(),
6659 name: None,
6660 enabled: false,
6661 transport: TransportConfig::Stdio(StdioConfig {
6662 command: "unused-disabled-command".to_string(),
6663 args: vec![],
6664 cwd: None,
6665 env: std::collections::HashMap::new(),
6666 env_encrypted: std::collections::HashMap::new(),
6667 env_credential_refs: std::collections::HashMap::new(),
6668 startup_timeout_ms: 100,
6669 }),
6670 request_timeout_ms: 100,
6671 healthcheck_interval_ms: 100,
6672 reconnect: ReconnectConfig::default(),
6673 allowed_tools: vec![],
6674 denied_tools: vec![],
6675 }],
6676 }
6677 }
6678
6679 fn working_stdio_mcp_config(dir: &Path, id: &str, secret: Option<&str>) -> McpConfig {
6680 let script = dir.join(format!("{id}-mcp-fixture.py"));
6681 std::fs::write(
6682 &script,
6683 r#"import json
6684import sys
6685
6686for line in sys.stdin:
6687 request = json.loads(line)
6688 request_id = request.get("id")
6689 if request_id is None:
6690 continue
6691 if request.get("method") == "server/discover":
6692 print(json.dumps({
6693 "jsonrpc": "2.0",
6694 "id": request_id,
6695 "error": {"code": -32601, "message": "Method not found"},
6696 }), flush=True)
6697 continue
6698 if request.get("method") == "initialize":
6699 result = {
6700 "protocolVersion": "2024-11-05",
6701 "capabilities": {"tools": {"listChanged": False}},
6702 "serverInfo": {"name": "config-generation-fixture", "version": "1.0.0"},
6703 }
6704 elif request.get("method") == "tools/list":
6705 result = {"tools": []}
6706 else:
6707 result = {}
6708 print(json.dumps({"jsonrpc": "2.0", "id": request_id, "result": result}), flush=True)
6709"#,
6710 )
6711 .unwrap();
6712 let python = ["python3", "python"]
6713 .into_iter()
6714 .find(|command| {
6715 std::process::Command::new(command)
6716 .arg("--version")
6717 .output()
6718 .is_ok_and(|output| output.status.success())
6719 })
6720 .expect("a Python interpreter is required for the MCP ordering fixture");
6721 let mut env = std::collections::HashMap::new();
6722 if let Some(secret) = secret {
6723 env.insert("TOKEN".to_string(), secret.to_string());
6724 }
6725 McpConfig {
6726 version: 1,
6727 servers: vec![McpServerConfig {
6728 id: id.to_string(),
6729 name: None,
6730 enabled: true,
6731 transport: TransportConfig::Stdio(StdioConfig {
6732 command: python.to_string(),
6733 args: vec![script.to_string_lossy().into_owned()],
6734 cwd: None,
6735 env,
6736 env_encrypted: std::collections::HashMap::new(),
6737 env_credential_refs: std::collections::HashMap::new(),
6738 startup_timeout_ms: 2_000,
6739 }),
6740 request_timeout_ms: 2_000,
6741 healthcheck_interval_ms: 10_000,
6742 reconnect: ReconnectConfig {
6743 enabled: false,
6744 ..Default::default()
6745 },
6746 allowed_tools: vec![],
6747 denied_tools: vec![],
6748 }],
6749 }
6750 }
6751
6752 #[tokio::test]
6753 async fn config_update_preserves_runtime_broker() {
6754 let dir = tempfile::tempdir().unwrap();
6755 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
6756 let expected = state
6757 .config
6758 .read()
6759 .await
6760 .subagents()
6761 .broker
6762 .clone()
6763 .expect("AppState embeds a runtime broker");
6764
6765 let updated = state
6766 .update_config(
6767 |config| {
6768 config.subagents_mut().max_concurrent = Some(3);
6769 Ok(())
6770 },
6771 ConfigUpdateEffects::default(),
6772 )
6773 .await
6774 .unwrap();
6775
6776 assert_eq!(updated.subagents().broker.as_ref(), Some(&expected));
6777 assert_eq!(
6778 state.config.read().await.subagents().broker.as_ref(),
6779 Some(&expected)
6780 );
6781 }
6782
6783 #[test]
6784 fn preserve_runtime_broker_keeps_explicit_broker() {
6785 let previous_broker = bamboo_config::BrokerClientConfig {
6786 endpoint: "ws://127.0.0.1:41001".to_string(),
6787 token: "previous".to_string(),
6788 token_encrypted: None,
6789 credential_ref: None,
6790 configured: false,
6791 };
6792 let explicit_broker = bamboo_config::BrokerClientConfig {
6793 endpoint: "wss://broker.example.test".to_string(),
6794 token: "explicit".to_string(),
6795 token_encrypted: None,
6796 credential_ref: None,
6797 configured: true,
6798 };
6799 let mut previous = Config::default();
6800 previous.subagents_mut().broker = Some(previous_broker);
6801 let mut incoming = Config::default();
6802 incoming.subagents_mut().broker = Some(explicit_broker.clone());
6803
6804 preserve_runtime_broker(&mut incoming, &previous);
6805
6806 assert_eq!(incoming.subagents().broker.as_ref(), Some(&explicit_broker));
6807 }
6808
6809 fn mcp_document_bytes(revision: u64, config: &McpConfig) -> Vec<u8> {
6810 serde_json::to_vec_pretty(&serde_json::json!({
6811 "schema_version": 1,
6812 "revision": revision,
6813 "data": config,
6814 }))
6815 .unwrap()
6816 }
6817
6818 #[test]
6819 fn legacy_mcp_rejects_client_owned_stdio_and_header_credential_refs() {
6820 let mut stdio_current = disabled_mcp_config("stdio-server");
6821 let stdio_reference =
6822 bamboo_config::credential_ref("mcp", "stdio-server", "env_TOKEN").unwrap();
6823 let TransportConfig::Stdio(stdio) = &mut stdio_current.servers[0].transport else {
6824 unreachable!()
6825 };
6826 stdio
6827 .env
6828 .insert("TOKEN".to_string(), "existing-secret".to_string());
6829 stdio
6830 .env_credential_refs
6831 .insert("TOKEN".to_string(), stdio_reference.as_str().to_string());
6832 let mut stdio_candidate = stdio_current.clone();
6833 let TransportConfig::Stdio(stdio) = &mut stdio_candidate.servers[0].transport else {
6834 unreachable!()
6835 };
6836 stdio
6837 .env_credential_refs
6838 .insert("TOKEN".to_string(), "mcp.foreign.env_token".to_string());
6839 let error = normalize_legacy_mcp_credentials(&stdio_current, &mut stdio_candidate)
6840 .expect_err("an arbitrary stdio credential ref must be rejected");
6841 assert!(matches!(
6842 error,
6843 AppError::BadRequest(message)
6844 if message == "MCP credential references are server-managed and cannot be supplied"
6845 ));
6846
6847 let header_reference =
6848 bamboo_config::credential_ref("mcp", "http-server", "header_Authorization").unwrap();
6849 let http_current = McpConfig {
6850 version: 1,
6851 servers: vec![McpServerConfig {
6852 id: "http-server".to_string(),
6853 name: None,
6854 enabled: false,
6855 transport: TransportConfig::Sse(bamboo_mcp::SseConfig {
6856 url: "https://example.test/sse".to_string(),
6857 headers: vec![bamboo_mcp::HeaderConfig {
6858 name: "Authorization".to_string(),
6859 value: "existing-secret".to_string(),
6860 value_encrypted: None,
6861 credential_ref: Some(header_reference.as_str().to_string()),
6862 }],
6863 connect_timeout_ms: 100,
6864 }),
6865 request_timeout_ms: 100,
6866 healthcheck_interval_ms: 100,
6867 reconnect: ReconnectConfig::default(),
6868 allowed_tools: vec![],
6869 denied_tools: vec![],
6870 }],
6871 };
6872 let mut http_candidate = http_current.clone();
6873 let TransportConfig::Sse(http) = &mut http_candidate.servers[0].transport else {
6874 unreachable!()
6875 };
6876 http.headers[0].credential_ref = Some("mcp.foreign.header_authorization".to_string());
6877 let error = normalize_legacy_mcp_credentials(&http_current, &mut http_candidate)
6878 .expect_err("an arbitrary header credential ref must be rejected");
6879 assert!(matches!(
6880 error,
6881 AppError::BadRequest(message)
6882 if message == "MCP credential references are server-managed and cannot be supplied"
6883 ));
6884 }
6885
6886 #[test]
6887 fn touched_shared_mcp_refs_stage_replacements_and_preserve_surviving_clears() {
6888 let shared =
6889 bamboo_config::CredentialRef::parse("mcp.shared.env_token".to_string()).unwrap();
6890 let mut current = disabled_mcp_config("first");
6891 let mut second = current.servers[0].clone();
6892 second.id = "second".to_string();
6893 current.servers.push(second);
6894 for server in &mut current.servers {
6895 let TransportConfig::Stdio(stdio) = &mut server.transport else {
6896 unreachable!()
6897 };
6898 stdio
6899 .env
6900 .insert("TOKEN".to_string(), "old-shared-secret".to_string());
6901 stdio
6902 .env_credential_refs
6903 .insert("TOKEN".to_string(), shared.as_str().to_string());
6904 }
6905 let touched = BTreeSet::from([shared]);
6906
6907 let mut replace = current.clone();
6908 for server in &mut replace.servers {
6909 let TransportConfig::Stdio(stdio) = &mut server.transport else {
6910 unreachable!()
6911 };
6912 stdio.env.get_mut("TOKEN").unwrap().clear();
6913 }
6914 let TransportConfig::Stdio(first) = &mut replace.servers[0].transport else {
6915 unreachable!()
6916 };
6917 first
6918 .env
6919 .insert("TOKEN".to_string(), "new-shared-secret".to_string());
6920 materialize_mcp_touched_replacements(&mut replace, &touched).unwrap();
6921 retain_mcp_credentials(¤t, &mut replace, &touched);
6922 for server in &replace.servers {
6923 let TransportConfig::Stdio(stdio) = &server.transport else {
6924 unreachable!()
6925 };
6926 assert_eq!(stdio.env["TOKEN"], "new-shared-secret");
6927 }
6928
6929 let mut clear_one = current.clone();
6930 for server in &mut clear_one.servers {
6931 let TransportConfig::Stdio(stdio) = &mut server.transport else {
6932 unreachable!()
6933 };
6934 stdio.env.get_mut("TOKEN").unwrap().clear();
6935 }
6936 let TransportConfig::Stdio(first) = &mut clear_one.servers[0].transport else {
6937 unreachable!()
6938 };
6939 first.env.remove("TOKEN");
6940 first.env_credential_refs.remove("TOKEN");
6941 materialize_mcp_touched_replacements(&mut clear_one, &touched).unwrap();
6942 retain_mcp_credentials(¤t, &mut clear_one, &touched);
6943 let TransportConfig::Stdio(first) = &clear_one.servers[0].transport else {
6944 unreachable!()
6945 };
6946 assert!(!first.env.contains_key("TOKEN"));
6947 assert!(!first.env_credential_refs.contains_key("TOKEN"));
6948 let TransportConfig::Stdio(second) = &clear_one.servers[1].transport else {
6949 unreachable!()
6950 };
6951 assert_eq!(second.env["TOKEN"], "old-shared-secret");
6952 assert_eq!(second.env_credential_refs["TOKEN"], "mcp.shared.env_token");
6953
6954 let header_ref =
6955 bamboo_config::CredentialRef::parse("mcp.shared.header_token".to_string()).unwrap();
6956 let current_http = McpConfig {
6957 version: 1,
6958 servers: vec![McpServerConfig {
6959 id: "http".to_string(),
6960 name: None,
6961 enabled: false,
6962 transport: TransportConfig::Sse(bamboo_mcp::SseConfig {
6963 url: "https://example.test/sse".to_string(),
6964 headers: vec![bamboo_mcp::HeaderConfig {
6965 name: "Authorization".to_string(),
6966 value: "old-header-secret".to_string(),
6967 value_encrypted: None,
6968 credential_ref: Some(header_ref.as_str().to_string()),
6969 }],
6970 connect_timeout_ms: 100,
6971 }),
6972 request_timeout_ms: 100,
6973 healthcheck_interval_ms: 100,
6974 reconnect: ReconnectConfig::default(),
6975 allowed_tools: vec![],
6976 denied_tools: vec![],
6977 }],
6978 };
6979 let mut delete_all_headers = current_http.clone();
6980 let TransportConfig::Sse(candidate) = &mut delete_all_headers.servers[0].transport else {
6981 unreachable!()
6982 };
6983 candidate.headers.clear();
6984 let touched = BTreeSet::from([header_ref]);
6985 materialize_mcp_touched_replacements(&mut delete_all_headers, &touched).unwrap();
6986 retain_mcp_credentials(¤t_http, &mut delete_all_headers, &touched);
6987 let TransportConfig::Sse(candidate) = &delete_all_headers.servers[0].transport else {
6988 unreachable!()
6989 };
6990 assert!(candidate.headers.is_empty());
6991 }
6992
6993 fn install_unrecoverable_pending_provider_migration(dir: &Path) {
6994 let transaction_id = uuid::Uuid::new_v4().to_string();
6995 std::fs::write(
6996 dir.join("config.json"),
6997 br#"{"providers":{"openai":{"model":"root-lkg"}}}"#,
6998 )
6999 .unwrap();
7000 std::fs::write(
7001 dir.join("providers.json"),
7002 br#"{"schema_version":1,"revision":2,"data":{"openai":{"model":"partial-must-not-load","credential_ref":"provider.openai.api_key"}}}"#,
7003 )
7004 .unwrap();
7005 std::fs::write(
7006 dir.join("config-credential-migration.json"),
7007 serde_json::to_vec_pretty(&serde_json::json!({
7008 "version": 1,
7009 "transaction_id": transaction_id.clone(),
7010 "stage_dir": format!(".config-credential-stage-v1-{transaction_id}"),
7011 "state": "pending",
7012 "files": [
7013 {
7014 "name": "credentials.json",
7015 "staged_name": "credentials.json",
7016 "sha256": "0".repeat(64),
7017 "sensitive": true
7018 },
7019 {
7020 "name": "providers.json",
7021 "staged_name": "providers.json",
7022 "sha256": "1".repeat(64),
7023 "original_sha256": "2".repeat(64),
7024 "migration_generation": 2,
7025 "sensitive": false
7026 }
7027 ]
7028 }))
7029 .unwrap(),
7030 )
7031 .unwrap();
7032 }
7033
7034 async fn wait_for_mcp_health(
7035 state: &AppState,
7036 status: SectionStatus,
7037 minimum_revision: u64,
7038 ) -> ConfigLiveHealth {
7039 match tokio::time::timeout(Duration::from_secs(4), async {
7040 loop {
7041 let health = state
7042 .mcp_config_live_health
7043 .read()
7044 .unwrap_or_else(|poisoned| poisoned.into_inner())
7045 .clone();
7046 if health.status == status && health.revision >= minimum_revision {
7047 break health;
7048 }
7049 tokio::time::sleep(Duration::from_millis(20)).await;
7050 }
7051 })
7052 .await
7053 {
7054 Ok(health) => health,
7055 Err(_) => panic!(
7056 "MCP health transition timed out: {:?}",
7057 state
7058 .mcp_config_live_health
7059 .read()
7060 .unwrap_or_else(|poisoned| poisoned.into_inner())
7061 .clone()
7062 ),
7063 }
7064 }
7065
7066 async fn next_config_event(
7067 feed: &mut tokio::sync::broadcast::Receiver<Arc<bamboo_engine::events::ChangeEvent>>,
7068 expected_section: &str,
7069 ) -> AgentEvent {
7070 tokio::time::timeout(Duration::from_secs(3), async {
7071 loop {
7072 let envelope = feed.recv().await.expect("account feed remains open");
7073 match &envelope.event {
7074 AgentEvent::ConfigChanged { section, .. }
7075 | AgentEvent::ConfigInvalid { section, .. }
7076 | AgentEvent::ConfigRecovered { section, .. }
7077 if section == expected_section =>
7078 {
7079 break envelope.event.clone();
7080 }
7081 _ => {}
7082 }
7083 }
7084 })
7085 .await
7086 .expect("config event timed out")
7087 }
7088
7089 async fn next_mcp_config_event(
7090 feed: &mut tokio::sync::broadcast::Receiver<Arc<bamboo_engine::events::ChangeEvent>>,
7091 ) -> AgentEvent {
7092 next_config_event(feed, "mcp").await
7093 }
7094
7095 async fn wait_for_root_outbox_to_clear(data_dir: &Path) {
7096 tokio::time::timeout(Duration::from_secs(6), async {
7097 loop {
7098 if !bamboo_config::has_pending_legacy_root_publications(data_dir).unwrap() {
7099 break;
7100 }
7101 tokio::time::sleep(Duration::from_millis(20)).await;
7102 }
7103 })
7104 .await
7105 .expect("legacy root outbox did not clear");
7106 }
7107
7108 #[tokio::test]
7109 async fn compatibility_update_cannot_reintroduce_an_unrevisioned_cluster_mutation() {
7110 let _key = bamboo_config::encryption::set_test_encryption_key([0x70; 32]);
7111 let dir = tempfile::tempdir().unwrap();
7112 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
7113 state
7114 .update_cluster_fabric_credentials(
7115 0,
7116 std::collections::BTreeMap::from([(
7117 "owned-node".to_string(),
7118 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
7119 )]),
7120 |config| {
7121 config.cluster_fabric.nodes.push(bamboo_config::Node {
7122 id: "owned-node".to_string(),
7123 label: "revisioned-label".to_string(),
7124 placement: bamboo_config::NodePlacement::Local,
7125 trust_level: bamboo_config::TrustLevel::Trusted,
7126 deploy: bamboo_config::DeployProfile::default(),
7127 state: None,
7128 enabled: true,
7129 });
7130 Ok(())
7131 },
7132 )
7133 .await
7134 .unwrap();
7135 let cluster_path = dir.path().join("cluster-fabric.json");
7136 let cluster_before = std::fs::read(&cluster_path).unwrap();
7137
7138 let updated = state
7139 .update_config(
7140 |config| {
7141 config.server.port = 21_000;
7142 config.cluster_fabric.node_mut("owned-node").unwrap().label =
7143 "unrevisioned-label".to_string();
7144 Ok(())
7145 },
7146 ConfigUpdateEffects::default(),
7147 )
7148 .await
7149 .unwrap();
7150
7151 assert_eq!(updated.server.port, 21_000);
7152 assert_eq!(
7153 updated.cluster_fabric.node("owned-node").unwrap().label,
7154 "revisioned-label"
7155 );
7156 assert_eq!(
7157 state
7158 .config_facade
7159 .as_ref()
7160 .unwrap()
7161 .registry()
7162 .cluster_fabric
7163 .snapshot()
7164 .revision,
7165 1
7166 );
7167 assert_eq!(std::fs::read(cluster_path).unwrap(), cluster_before);
7168 }
7169
7170 #[tokio::test]
7171 async fn stopped_watcher_compatibility_writers_install_only_their_owned_section() {
7172 let _key = bamboo_config::encryption::set_test_encryption_key([0x72; 32]);
7173 let dir = tempfile::tempdir().unwrap();
7174 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
7175 state
7176 .update_cluster_fabric_credentials(
7177 0,
7178 BTreeMap::from([(
7179 "shared-node".to_string(),
7180 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
7181 )]),
7182 |config| {
7183 config.cluster_fabric.nodes.push(bamboo_config::Node {
7184 id: "shared-node".to_string(),
7185 label: "generation-one".to_string(),
7186 placement: bamboo_config::NodePlacement::Local,
7187 trust_level: bamboo_config::TrustLevel::Trusted,
7188 deploy: bamboo_config::DeployProfile::default(),
7189 state: None,
7190 enabled: true,
7191 });
7192 Ok(())
7193 },
7194 )
7195 .await
7196 .unwrap();
7197 stop_config_watcher(&mut state);
7198 let state = Arc::new(state);
7199
7200 let external = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
7201 let mut external_candidate = external.effective_config();
7202 external_candidate
7203 .cluster_fabric
7204 .node_mut("shared-node")
7205 .unwrap()
7206 .label = "external-generation-two".to_string();
7207 assert_eq!(
7208 bamboo_config::persist_cluster_fabric_credential_transaction_at_revision(
7209 dir.path(),
7210 &mut external_candidate,
7211 &BTreeMap::new(),
7212 1,
7213 )
7214 .unwrap(),
7215 2
7216 );
7217 let cluster_path = dir.path().join("cluster-fabric.json");
7218 let cluster_r2 = std::fs::read(&cluster_path).unwrap();
7219 assert_eq!(
7220 state
7221 .config_facade
7222 .as_ref()
7223 .unwrap()
7224 .registry()
7225 .cluster_fabric
7226 .snapshot()
7227 .revision,
7228 1
7229 );
7230 assert_eq!(
7231 state
7232 .config
7233 .read()
7234 .await
7235 .cluster_fabric
7236 .node("shared-node")
7237 .unwrap()
7238 .label,
7239 "generation-one"
7240 );
7241
7242 let baseline_seq = state.account_sink.latest_seq();
7243 let mut core_feed = state.account_sink.subscribe();
7244 let mut cluster_feed = state.account_sink.subscribe();
7245 let stale_runtime = state.config.read().await;
7246 let updating = {
7247 let state = state.clone();
7248 tokio::spawn(async move {
7249 state
7250 .update_config(
7251 |config| {
7252 config.server.port = 23_332;
7253 Ok(())
7254 },
7255 ConfigUpdateEffects::default(),
7256 )
7257 .await
7258 })
7259 };
7260 assert!(
7261 tokio::time::timeout(
7262 Duration::from_millis(100),
7263 next_config_event(&mut core_feed, "core"),
7264 )
7265 .await
7266 .is_err(),
7267 "core event became observable while the old AppState snapshot was held"
7268 );
7269 assert_ne!(stale_runtime.server.port, 23_332);
7270 drop(stale_runtime);
7271 let published = updating.await.unwrap().unwrap();
7272 assert!(matches!(
7273 next_config_event(&mut core_feed, "core").await,
7274 AgentEvent::ConfigChanged { section, .. } if section == "core"
7275 ));
7276 assert_eq!(state.config.read().await.server.port, 23_332);
7277 assert!(
7278 tokio::time::timeout(
7279 Duration::from_millis(300),
7280 next_config_event(&mut cluster_feed, "cluster-fabric"),
7281 )
7282 .await
7283 .is_err(),
7284 "an unrelated compatibility update published a cluster event"
7285 );
7286 assert_eq!(
7287 published.cluster_fabric.node("shared-node").unwrap().label,
7288 "generation-one"
7289 );
7290 assert_eq!(std::fs::read(&cluster_path).unwrap(), cluster_r2);
7291 assert_eq!(
7292 state
7293 .config_facade
7294 .as_ref()
7295 .unwrap()
7296 .registry()
7297 .cluster_fabric
7298 .snapshot()
7299 .revision,
7300 1,
7301 "an unrelated compatibility update must not catch up cluster"
7302 );
7303 assert_eq!(
7304 state
7305 .config
7306 .read()
7307 .await
7308 .cluster_fabric
7309 .node("shared-node")
7310 .unwrap()
7311 .label,
7312 "generation-one"
7313 );
7314 tokio::time::sleep(Duration::from_millis(100)).await;
7315 let events = bamboo_engine::events::journal::read_since(
7316 state.account_sink.events_dir(),
7317 baseline_seq,
7318 )
7319 .unwrap();
7320 assert_eq!(
7321 events
7322 .iter()
7323 .filter(|event| matches!(
7324 &event.event,
7325 AgentEvent::ConfigChanged { section, .. } if section == "core"
7326 ))
7327 .count(),
7328 1
7329 );
7330 assert!(!events.iter().any(|event| matches!(
7331 &event.event,
7332 AgentEvent::ConfigChanged { section, .. } if section == "cluster-fabric"
7333 )));
7334
7335 let external = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
7336 let mut external_candidate = external.effective_config();
7337 external_candidate
7338 .cluster_fabric
7339 .node_mut("shared-node")
7340 .unwrap()
7341 .label = "external-generation-three".to_string();
7342 assert_eq!(
7343 bamboo_config::persist_cluster_fabric_credential_transaction_at_revision(
7344 dir.path(),
7345 &mut external_candidate,
7346 &BTreeMap::new(),
7347 2,
7348 )
7349 .unwrap(),
7350 3
7351 );
7352 let cluster_r3 = std::fs::read(&cluster_path).unwrap();
7353 assert_eq!(
7354 state
7355 .config_facade
7356 .as_ref()
7357 .unwrap()
7358 .registry()
7359 .cluster_fabric
7360 .snapshot()
7361 .revision,
7362 1,
7363 "the stopped watcher must remain stale before replace_config"
7364 );
7365
7366 let mut replacement = state.config.read().await.clone();
7367 replacement.server.port = 23_333;
7368 let baseline_seq = state.account_sink.latest_seq();
7369 let mut core_feed = state.account_sink.subscribe();
7370 let mut cluster_feed = state.account_sink.subscribe();
7371 let stale_runtime = state.config.read().await;
7372 let replacing = {
7373 let state = state.clone();
7374 tokio::spawn(async move {
7375 state
7376 .replace_config(replacement, ConfigUpdateEffects::default())
7377 .await
7378 })
7379 };
7380 assert!(
7381 tokio::time::timeout(
7382 Duration::from_millis(100),
7383 next_config_event(&mut core_feed, "core"),
7384 )
7385 .await
7386 .is_err(),
7387 "replacement event became observable while the old AppState snapshot was held"
7388 );
7389 assert_ne!(stale_runtime.server.port, 23_333);
7390 drop(stale_runtime);
7391 let published = replacing.await.unwrap().unwrap();
7392 assert!(matches!(
7393 next_config_event(&mut core_feed, "core").await,
7394 AgentEvent::ConfigChanged { section, .. } if section == "core"
7395 ));
7396 assert_eq!(state.config.read().await.server.port, 23_333);
7397 assert!(
7398 tokio::time::timeout(
7399 Duration::from_millis(300),
7400 next_config_event(&mut cluster_feed, "cluster-fabric"),
7401 )
7402 .await
7403 .is_err(),
7404 "an unrelated compatibility replacement published a cluster event"
7405 );
7406 assert_eq!(published.server.port, 23_333);
7407 assert_eq!(
7408 published.cluster_fabric.node("shared-node").unwrap().label,
7409 "generation-one"
7410 );
7411 assert_eq!(std::fs::read(&cluster_path).unwrap(), cluster_r3);
7412 assert_eq!(
7413 state
7414 .config_facade
7415 .as_ref()
7416 .unwrap()
7417 .registry()
7418 .cluster_fabric
7419 .snapshot()
7420 .revision,
7421 1
7422 );
7423 assert_eq!(
7424 state
7425 .config
7426 .read()
7427 .await
7428 .cluster_fabric
7429 .node("shared-node")
7430 .unwrap()
7431 .label,
7432 "generation-one"
7433 );
7434 tokio::time::sleep(Duration::from_millis(100)).await;
7435 let events = bamboo_engine::events::journal::read_since(
7436 state.account_sink.events_dir(),
7437 baseline_seq,
7438 )
7439 .unwrap();
7440 assert_eq!(
7441 events
7442 .iter()
7443 .filter(|event| matches!(
7444 &event.event,
7445 AgentEvent::ConfigChanged { section, .. } if section == "core"
7446 ))
7447 .count(),
7448 1
7449 );
7450 assert!(!events.iter().any(|event| matches!(
7451 &event.event,
7452 AgentEvent::ConfigChanged { section, .. } if section == "cluster-fabric"
7453 )));
7454 }
7455
7456 #[tokio::test]
7457 async fn exact_notification_publication_installs_only_its_owned_runtime_section() {
7458 let dir = tempfile::tempdir().unwrap();
7459 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
7460 {
7461 let mut live = state.config.write().await;
7462 live.connect
7463 .platforms
7464 .push(bamboo_config::ConnectPlatformConfig {
7465 id: None,
7466 project_id: None,
7467 platform_type: "runtime-sentinel".to_string(),
7468 token: None,
7469 token_encrypted: None,
7470 token_credential_ref: None,
7471 token_configured: false,
7472 app_id: None,
7473 app_secret: None,
7474 app_secret_encrypted: None,
7475 app_secret_credential_ref: None,
7476 app_secret_configured: false,
7477 domain: None,
7478 allow_from: Vec::new(),
7479 admin_from: Vec::new(),
7480 });
7481 live.providers_mut().openai = Some(bamboo_config::OpenAIConfig {
7482 api_key: "runtime-provider-sentinel".to_string(),
7483 ..Default::default()
7484 });
7485 }
7486 let connect_before = std::fs::read(dir.path().join("connect.json")).unwrap();
7487 let (published, revision, _, section) = state
7488 .update_notification_credentials(0, BTreeSet::new(), false, |candidate| {
7489 candidate.notifications.ntfy.enabled = true;
7490 candidate.notifications.ntfy.topic = "owned-notification".to_string();
7491 candidate.assign_connect_platform_ids();
7495 candidate
7496 .providers_mut()
7497 .openai
7498 .as_mut()
7499 .unwrap()
7500 .api_key
7501 .clear();
7502 Ok(())
7503 })
7504 .await
7505 .unwrap();
7506
7507 assert_eq!(revision, 1);
7508 assert_eq!(section.unwrap().revision, 1);
7509 assert_eq!(published.notifications.ntfy.topic, "owned-notification");
7510 assert!(published.connect.platforms[0].id.is_none());
7511 assert_eq!(
7512 published.providers().openai.as_ref().unwrap().api_key,
7513 "runtime-provider-sentinel"
7514 );
7515 let live = state.config.read().await;
7516 assert!(live.connect.platforms[0].id.is_none());
7517 assert_eq!(
7518 live.providers().openai.as_ref().unwrap().api_key,
7519 "runtime-provider-sentinel"
7520 );
7521 drop(live);
7522 assert_eq!(
7523 std::fs::read(dir.path().join("connect.json")).unwrap(),
7524 connect_before
7525 );
7526 assert_eq!(
7527 bamboo_config::ConfigFacade::open(dir.path())
7528 .unwrap()
7529 .registry()
7530 .connect
7531 .snapshot()
7532 .revision,
7533 0
7534 );
7535 }
7536
7537 #[tokio::test]
7538 async fn generic_update_cannot_forge_exact_core_credential_binding() {
7539 let dir = tempfile::tempdir().unwrap();
7540 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
7541 let core_before = std::fs::read(dir.path().join("core.json")).unwrap();
7542 let error = state
7543 .update_config(
7544 |candidate| {
7545 candidate.proxy_auth_credential_ref =
7546 Some(bamboo_config::CredentialRef::parse("proxy.default.auth").unwrap());
7547 Ok(())
7548 },
7549 ConfigUpdateEffects::default(),
7550 )
7551 .await
7552 .unwrap_err();
7553 assert!(matches!(error, AppError::BadRequest(_)));
7554 assert!(error.to_string().contains("credential bindings"));
7555 assert_eq!(
7556 std::fs::read(dir.path().join("core.json")).unwrap(),
7557 core_before
7558 );
7559 assert!(state
7560 .config
7561 .read()
7562 .await
7563 .proxy_auth_credential_ref
7564 .is_none());
7565 assert_eq!(
7566 state
7567 .config_facade
7568 .as_ref()
7569 .unwrap()
7570 .registry()
7571 .core
7572 .snapshot()
7573 .revision,
7574 0
7575 );
7576 }
7577
7578 #[tokio::test]
7579 async fn env_credential_commit_installs_owned_runtime_before_exact_events() {
7580 let _key = bamboo_config::encryption::set_test_encryption_key([0x73; 32]);
7581 let dir = tempfile::tempdir().unwrap();
7582 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
7583 state
7584 .update_cluster_fabric_credentials(
7585 0,
7586 BTreeMap::from([(
7587 "shared-node".to_string(),
7588 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
7589 )]),
7590 |config| {
7591 config.cluster_fabric.nodes.push(bamboo_config::Node {
7592 id: "shared-node".to_string(),
7593 label: "generation-one".to_string(),
7594 placement: bamboo_config::NodePlacement::Local,
7595 trust_level: bamboo_config::TrustLevel::Trusted,
7596 deploy: bamboo_config::DeployProfile::default(),
7597 state: None,
7598 enabled: true,
7599 });
7600 Ok(())
7601 },
7602 )
7603 .await
7604 .unwrap();
7605 stop_config_watcher(&mut state);
7606 let state = Arc::new(state);
7607
7608 let external = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
7609 let mut external_candidate = external.effective_config();
7610 external_candidate
7611 .cluster_fabric
7612 .node_mut("shared-node")
7613 .unwrap()
7614 .label = "external-generation-two".to_string();
7615 bamboo_config::persist_cluster_fabric_credential_transaction_at_revision(
7616 dir.path(),
7617 &mut external_candidate,
7618 &BTreeMap::new(),
7619 1,
7620 )
7621 .unwrap();
7622 let cluster_path = dir.path().join("cluster-fabric.json");
7623 let cluster_r2 = std::fs::read(&cluster_path).unwrap();
7624 let expected_revision = state
7625 .config_facade
7626 .as_ref()
7627 .unwrap()
7628 .registry()
7629 .env
7630 .snapshot()
7631 .revision;
7632
7633 let (reached_tx, reached_rx) = std::sync::mpsc::channel();
7634 let (release_tx, release_rx) = std::sync::mpsc::channel();
7635 set_credential_after_commit_before_live_test_hook(dir.path(), SectionId::Env, move || {
7636 reached_tx.send(()).unwrap();
7637 release_rx.recv().unwrap();
7638 });
7639 let baseline_seq = state.account_sink.latest_seq();
7640 let mut credential_feed = state.account_sink.subscribe();
7641 let mut env_feed = state.account_sink.subscribe();
7642 let mut cluster_feed = state.account_sink.subscribe();
7643 let updating = {
7644 let state = state.clone();
7645 tokio::spawn(async move {
7646 state
7647 .update_env_var_credentials(
7648 expected_revision,
7649 BTreeSet::from(["TOKEN".to_string()]),
7650 false,
7651 |config| {
7652 config.env_vars.push(bamboo_config::EnvVarEntry {
7653 name: "TOKEN".to_string(),
7654 value: "exact-secret".to_string(),
7655 secret: true,
7656 value_encrypted: None,
7657 credential_ref: None,
7658 configured: true,
7659 description: None,
7660 });
7661 Ok(())
7662 },
7663 )
7664 .await
7665 })
7666 };
7667 tokio::task::spawn_blocking(move || reached_rx.recv().unwrap())
7668 .await
7669 .unwrap();
7670 let stale_runtime = state.config.read().await;
7671 release_tx.send(()).unwrap();
7672 for (feed, section) in [
7673 (&mut credential_feed, "credentials"),
7674 (&mut env_feed, "env"),
7675 (&mut cluster_feed, "cluster-fabric"),
7676 ] {
7677 assert!(
7678 tokio::time::timeout(Duration::from_millis(100), next_config_event(feed, section),)
7679 .await
7680 .is_err(),
7681 "{section} event became observable before the owned runtime install"
7682 );
7683 }
7684 assert!(
7685 stale_runtime
7686 .env_vars
7687 .iter()
7688 .all(|entry| entry.name != "TOKEN"),
7689 "the held runtime must still be the pre-commit env generation"
7690 );
7691 assert_eq!(
7692 stale_runtime
7693 .cluster_fabric
7694 .node("shared-node")
7695 .unwrap()
7696 .label,
7697 "generation-one"
7698 );
7699 drop(stale_runtime);
7700
7701 let (published, revision, _, _) = updating.await.unwrap().unwrap();
7702 assert!(revision > expected_revision);
7703 assert!(published
7704 .env_vars
7705 .iter()
7706 .any(|entry| entry.name == "TOKEN" && entry.value == "exact-secret"));
7707 assert_eq!(
7708 published.cluster_fabric.node("shared-node").unwrap().label,
7709 "generation-one"
7710 );
7711 assert!(matches!(
7712 next_config_event(&mut credential_feed, "credentials").await,
7713 AgentEvent::ConfigChanged { section, .. } if section == "credentials"
7714 ));
7715 assert!(matches!(
7716 next_config_event(&mut env_feed, "env").await,
7717 AgentEvent::ConfigChanged { section, .. } if section == "env"
7718 ));
7719 assert!(tokio::time::timeout(
7720 Duration::from_millis(300),
7721 next_config_event(&mut cluster_feed, "cluster-fabric"),
7722 )
7723 .await
7724 .is_err());
7725 assert_eq!(std::fs::read(cluster_path).unwrap(), cluster_r2);
7726 assert_eq!(
7727 state
7728 .config_facade
7729 .as_ref()
7730 .unwrap()
7731 .registry()
7732 .cluster_fabric
7733 .snapshot()
7734 .revision,
7735 1
7736 );
7737 tokio::time::sleep(Duration::from_millis(100)).await;
7738 let events = bamboo_engine::events::journal::read_since(
7739 state.account_sink.events_dir(),
7740 baseline_seq,
7741 )
7742 .unwrap();
7743 assert_eq!(
7744 events
7745 .iter()
7746 .filter(|event| matches!(
7747 &event.event,
7748 AgentEvent::ConfigChanged { section, .. } if section == "credentials"
7749 ))
7750 .count(),
7751 1
7752 );
7753 assert_eq!(
7754 events
7755 .iter()
7756 .filter(|event| matches!(
7757 &event.event,
7758 AgentEvent::ConfigChanged { section, .. } if section == "env"
7759 ))
7760 .count(),
7761 1
7762 );
7763 assert!(!events.iter().any(|event| matches!(
7764 &event.event,
7765 AgentEvent::ConfigChanged { section, .. } if section == "cluster-fabric"
7766 )));
7767 }
7768
7769 #[tokio::test]
7770 async fn env_mutation_returns_its_captured_envelope_after_a_later_section_commit() {
7771 let _key = bamboo_config::encryption::set_test_encryption_key([0x74; 32]);
7772 let dir = tempfile::tempdir().unwrap();
7773 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
7774 stop_config_watcher(&mut state);
7775 let state = Arc::new(state);
7776
7777 let (reached_tx, reached_rx) = std::sync::mpsc::channel();
7778 let (release_tx, release_rx) = std::sync::mpsc::channel();
7779 set_credential_after_commit_before_live_test_hook(dir.path(), SectionId::Env, move || {
7780 reached_tx.send(()).unwrap();
7781 release_rx.recv().unwrap();
7782 });
7783 let updating = {
7784 let state = state.clone();
7785 tokio::spawn(async move {
7786 state
7787 .update_env_var_credentials(
7788 0,
7789 BTreeSet::from(["TOKEN".to_string()]),
7790 false,
7791 |config| {
7792 config.env_vars.push(bamboo_config::EnvVarEntry {
7793 name: "TOKEN".to_string(),
7794 value: "first-secret".to_string(),
7795 secret: true,
7796 value_encrypted: None,
7797 credential_ref: None,
7798 configured: true,
7799 description: Some("first generation".to_string()),
7800 });
7801 Ok(())
7802 },
7803 )
7804 .await
7805 })
7806 };
7807 tokio::task::spawn_blocking(move || reached_rx.recv().unwrap())
7808 .await
7809 .unwrap();
7810
7811 let external_dir = dir.path().to_path_buf();
7812 let process_facade = state.config_facade.clone().unwrap();
7813 let later = tokio::task::spawn_blocking(move || {
7814 let external = bamboo_config::ConfigFacade::open(&external_dir).unwrap();
7815 let mut candidate = external.effective_config();
7816 candidate.env_vars[0].description = Some("later generation".to_string());
7817 bamboo_config::persist_env_var_credential_transaction_at_revision_with_adoption(
7818 &external_dir,
7819 &mut candidate,
7820 &BTreeSet::from(["TOKEN".to_string()]),
7821 1,
7822 process_facade.as_ref(),
7823 )
7824 .unwrap()
7825 })
7826 .await
7827 .unwrap();
7828 assert_eq!(later.revision, 2);
7829 assert_eq!(later.section.unwrap().revision, 2);
7830 release_tx.send(()).unwrap();
7831
7832 let (_, revision, _, section) = updating.await.unwrap().unwrap();
7833 let section = section.expect("modular mutation returns its exact section");
7834 assert_eq!(revision, 1);
7835 assert_eq!(section.revision, 1);
7836 assert_eq!(section.data[0]["description"], "first generation");
7837 assert_eq!(
7838 state
7839 .config_facade
7840 .as_ref()
7841 .unwrap()
7842 .registry()
7843 .env
7844 .snapshot()
7845 .revision,
7846 2,
7847 "the process facade advanced, but the response retained its own commit"
7848 );
7849 }
7850
7851 #[tokio::test]
7852 async fn cluster_commit_installs_runtime_before_one_authoritative_event() {
7853 let _key = bamboo_config::encryption::set_test_encryption_key([0x71; 32]);
7854 let dir = tempfile::tempdir().unwrap();
7855 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
7856 let revision = state
7857 .config_facade
7858 .as_ref()
7859 .unwrap()
7860 .registry()
7861 .cluster_fabric
7862 .snapshot()
7863 .revision;
7864 let baseline_seq = state.account_sink.latest_seq();
7865 let mut feed = state.account_sink.subscribe();
7866 let runtime = state.config.clone();
7867 let observer = tokio::spawn(async move {
7868 tokio::time::timeout(Duration::from_secs(3), async move {
7869 loop {
7870 let event = feed.recv().await.unwrap();
7871 match &event.event {
7872 AgentEvent::ConfigChanged { section, .. } if section == "credentials" => {
7873 panic!("cluster mutation published an internal credential event")
7874 }
7875 AgentEvent::ConfigChanged { section, revision }
7876 if section == "cluster-fabric" =>
7877 {
7878 assert!(
7879 runtime
7880 .read()
7881 .await
7882 .cluster_fabric
7883 .node("event-node")
7884 .is_some(),
7885 "event observer saw the old runtime snapshot"
7886 );
7887 return *revision;
7888 }
7889 _ => {}
7890 }
7891 }
7892 })
7893 .await
7894 .expect("cluster event timed out")
7895 });
7896
7897 let node = bamboo_config::Node {
7898 id: "event-node".to_string(),
7899 label: "event-node".to_string(),
7900 placement: bamboo_config::NodePlacement::Local,
7901 trust_level: bamboo_config::TrustLevel::Trusted,
7902 deploy: bamboo_config::DeployProfile::default(),
7903 state: None,
7904 enabled: true,
7905 };
7906 let committed = state
7907 .update_cluster_fabric_credentials(
7908 revision,
7909 BTreeMap::from([(
7910 "event-node".to_string(),
7911 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
7912 )]),
7913 move |config| {
7914 config.cluster_fabric.nodes.push(node);
7915 Ok(())
7916 },
7917 )
7918 .await
7919 .unwrap();
7920 let committed = committed.section.revision;
7921 assert_eq!(committed, revision + 1);
7922 assert_eq!(observer.await.unwrap(), committed);
7923
7924 tokio::time::sleep(Duration::from_millis(100)).await;
7925 let events = bamboo_engine::events::journal::read_since(
7926 state.account_sink.events_dir(),
7927 baseline_seq,
7928 )
7929 .unwrap();
7930 let cluster_events = events
7931 .iter()
7932 .filter(|event| {
7933 matches!(
7934 &event.event,
7935 AgentEvent::ConfigChanged { section, revision: event_revision }
7936 if section == "cluster-fabric" && *event_revision == committed
7937 )
7938 })
7939 .count();
7940 let credential_events = events
7941 .iter()
7942 .filter(|event| {
7943 matches!(
7944 &event.event,
7945 AgentEvent::ConfigChanged { section, .. } if section == "credentials"
7946 )
7947 })
7948 .count();
7949 assert_eq!(cluster_events, 1);
7950 assert_eq!(credential_events, 0);
7951 }
7952
7953 #[tokio::test]
7954 async fn stale_process_cluster_candidate_rebases_on_exact_durable_client_generation() {
7955 let dir = tempfile::tempdir().unwrap();
7956 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
7957 state
7958 .update_cluster_fabric_credentials(
7959 0,
7960 BTreeMap::from([(
7961 "shared-node".to_string(),
7962 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
7963 )]),
7964 |config| {
7965 config.cluster_fabric.nodes.push(bamboo_config::Node {
7966 id: "shared-node".to_string(),
7967 label: "generation-one".to_string(),
7968 placement: bamboo_config::NodePlacement::Local,
7969 trust_level: bamboo_config::TrustLevel::Trusted,
7970 deploy: bamboo_config::DeployProfile::default(),
7971 state: None,
7972 enabled: true,
7973 });
7974 Ok(())
7975 },
7976 )
7977 .await
7978 .unwrap();
7979 stop_config_watcher(&mut state);
7980
7981 let external = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
7982 let mut external_candidate = external.effective_config();
7983 external_candidate
7984 .cluster_fabric
7985 .clusters
7986 .push(bamboo_config::Cluster {
7987 name: "external-cluster".to_string(),
7988 description: Some("durable-r2-field".to_string()),
7989 node_ids: vec!["shared-node".to_string()],
7990 });
7991 assert_eq!(
7992 bamboo_config::persist_cluster_fabric_credential_transaction_at_revision(
7993 dir.path(),
7994 &mut external_candidate,
7995 &BTreeMap::new(),
7996 1,
7997 )
7998 .unwrap(),
7999 2
8000 );
8001 assert_eq!(
8002 state
8003 .config_facade
8004 .as_ref()
8005 .unwrap()
8006 .registry()
8007 .cluster_fabric
8008 .snapshot()
8009 .revision,
8010 1
8011 );
8012 assert!(
8013 state
8014 .config
8015 .read()
8016 .await
8017 .cluster_fabric
8018 .cluster("external-cluster")
8019 .is_none(),
8020 "the process runtime is intentionally stale at r1"
8021 );
8022
8023 let committed = state
8024 .update_cluster_fabric_credentials(2, BTreeMap::new(), |config| {
8025 config.cluster_fabric.node_mut("shared-node").unwrap().label =
8026 "client-r3-edit".to_string();
8027 Ok(())
8028 })
8029 .await
8030 .unwrap();
8031 assert_eq!(committed.section.revision, 3);
8032 assert_eq!(
8033 committed
8034 .config
8035 .cluster_fabric
8036 .node("shared-node")
8037 .unwrap()
8038 .label,
8039 "client-r3-edit"
8040 );
8041 assert_eq!(
8042 committed
8043 .config
8044 .cluster_fabric
8045 .cluster("external-cluster")
8046 .unwrap()
8047 .description
8048 .as_deref(),
8049 Some("durable-r2-field")
8050 );
8051 assert_eq!(
8052 state
8053 .config_facade
8054 .as_ref()
8055 .unwrap()
8056 .registry()
8057 .cluster_fabric
8058 .snapshot()
8059 .revision,
8060 3,
8061 "compound adoption must safely catch the stale r1 facade up to r3"
8062 );
8063
8064 let runtime_before_conflict = state.config.read().await.cluster_fabric.clone();
8065 let conflict = state
8066 .update_cluster_fabric_credentials(2, BTreeMap::new(), |config| {
8067 config.cluster_fabric.nodes.clear();
8068 config.cluster_fabric.clusters.clear();
8069 Ok(())
8070 })
8071 .await;
8072 assert!(matches!(
8073 conflict,
8074 Err(AppError::ConfigConflict {
8075 expected: 2,
8076 actual: 3
8077 })
8078 ));
8079 assert_eq!(
8080 state.config.read().await.cluster_fabric,
8081 runtime_before_conflict,
8082 "a durable CAS conflict must not overwrite the process runtime"
8083 );
8084 }
8085
8086 #[tokio::test]
8087 async fn stale_process_cluster_noop_catches_up_exact_durable_generation() {
8088 let dir = tempfile::tempdir().unwrap();
8089 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
8090 stop_config_watcher(&mut state);
8091
8092 let external = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
8093 let mut external_candidate = external.effective_config();
8094 external_candidate
8095 .cluster_fabric
8096 .nodes
8097 .push(bamboo_config::Node {
8098 id: "external-node".to_string(),
8099 label: "external-r1".to_string(),
8100 placement: bamboo_config::NodePlacement::Local,
8101 trust_level: bamboo_config::TrustLevel::Trusted,
8102 deploy: bamboo_config::DeployProfile::default(),
8103 state: None,
8104 enabled: true,
8105 });
8106 assert_eq!(
8107 bamboo_config::persist_cluster_fabric_credential_transaction_at_revision(
8108 dir.path(),
8109 &mut external_candidate,
8110 &BTreeMap::from([(
8111 "external-node".to_string(),
8112 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
8113 )]),
8114 0,
8115 )
8116 .unwrap(),
8117 1
8118 );
8119 assert_eq!(
8120 state
8121 .config_facade
8122 .as_ref()
8123 .unwrap()
8124 .registry()
8125 .cluster_fabric
8126 .snapshot()
8127 .revision,
8128 0
8129 );
8130 let baseline_seq = state.account_sink.latest_seq();
8131
8132 let committed = state
8133 .update_cluster_fabric_credentials(1, BTreeMap::new(), |_| Ok(()))
8134 .await
8135 .unwrap();
8136 assert_eq!(committed.section.revision, 1);
8137 assert_eq!(
8138 committed
8139 .config
8140 .cluster_fabric
8141 .node("external-node")
8142 .unwrap()
8143 .label,
8144 "external-r1"
8145 );
8146 assert_eq!(
8147 state
8148 .config_facade
8149 .as_ref()
8150 .unwrap()
8151 .registry()
8152 .cluster_fabric
8153 .snapshot()
8154 .revision,
8155 1
8156 );
8157 assert_eq!(
8158 state
8159 .config
8160 .read()
8161 .await
8162 .cluster_fabric
8163 .node("external-node")
8164 .unwrap()
8165 .label,
8166 "external-r1"
8167 );
8168
8169 tokio::time::sleep(Duration::from_millis(100)).await;
8170 let events = bamboo_engine::events::journal::read_since(
8171 state.account_sink.events_dir(),
8172 baseline_seq,
8173 )
8174 .unwrap();
8175 let revisions = events
8176 .iter()
8177 .filter_map(|event| match &event.event {
8178 AgentEvent::ConfigChanged { section, revision } if section == "cluster-fabric" => {
8179 Some(*revision)
8180 }
8181 _ => None,
8182 })
8183 .collect::<Vec<_>>();
8184 assert_eq!(revisions, vec![1]);
8185 }
8186
8187 #[tokio::test]
8188 async fn stale_process_cluster_reset_noop_catches_up_exact_durable_generation() {
8189 let dir = tempfile::tempdir().unwrap();
8190 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
8191 stop_config_watcher(&mut state);
8192
8193 let external = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
8194 let mut external_candidate = external.effective_config();
8195 external_candidate
8196 .cluster_fabric
8197 .nodes
8198 .push(bamboo_config::Node {
8199 id: "reset-node".to_string(),
8200 label: "reset-node".to_string(),
8201 placement: bamboo_config::NodePlacement::Local,
8202 trust_level: bamboo_config::TrustLevel::Trusted,
8203 deploy: bamboo_config::DeployProfile::default(),
8204 state: None,
8205 enabled: true,
8206 });
8207 assert_eq!(
8208 bamboo_config::persist_cluster_fabric_credential_transaction_at_revision(
8209 dir.path(),
8210 &mut external_candidate,
8211 &BTreeMap::from([(
8212 "reset-node".to_string(),
8213 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
8214 )]),
8215 0,
8216 )
8217 .unwrap(),
8218 1
8219 );
8220 let reset_facade = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
8221 let mut reset_candidate = reset_facade.effective_config();
8222 reset_candidate.cluster_fabric = bamboo_config::ClusterFabricConfig::default();
8223 let external_reset = bamboo_config::persist_cluster_fabric_reset_at_revision_with_adoption(
8224 dir.path(),
8225 &mut reset_candidate,
8226 1,
8227 &reset_facade,
8228 |_, _| {},
8229 )
8230 .unwrap();
8231 assert_eq!(external_reset.revision, 2);
8232 assert_eq!(
8233 state
8234 .config_facade
8235 .as_ref()
8236 .unwrap()
8237 .registry()
8238 .cluster_fabric
8239 .snapshot()
8240 .revision,
8241 0
8242 );
8243 let baseline_seq = state.account_sink.latest_seq();
8244
8245 let committed = state
8246 .reset_credential_backed_section(SectionId::ClusterFabric, 2)
8247 .await
8248 .unwrap();
8249 let CredentialBackedResetCommit::Cluster(committed) = committed else {
8250 panic!("cluster reset must return its exact snapshot")
8251 };
8252 assert_eq!(committed.section.revision, 2);
8253 assert!(committed.config.cluster_fabric.nodes.is_empty());
8254 assert_eq!(
8255 state
8256 .config_facade
8257 .as_ref()
8258 .unwrap()
8259 .registry()
8260 .cluster_fabric
8261 .snapshot()
8262 .revision,
8263 2
8264 );
8265 assert!(state.config.read().await.cluster_fabric.nodes.is_empty());
8266
8267 tokio::time::sleep(Duration::from_millis(100)).await;
8268 let events = bamboo_engine::events::journal::read_since(
8269 state.account_sink.events_dir(),
8270 baseline_seq,
8271 )
8272 .unwrap();
8273 let revisions = events
8274 .iter()
8275 .filter_map(|event| match &event.event {
8276 AgentEvent::ConfigChanged { section, revision } if section == "cluster-fabric" => {
8277 Some(*revision)
8278 }
8279 _ => None,
8280 })
8281 .collect::<Vec<_>>();
8282 assert_eq!(revisions, vec![2]);
8283 }
8284
8285 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8286 async fn generic_events_follow_serialized_local_commit_order() {
8287 let dir = tempfile::tempdir().unwrap();
8288 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
8289 stop_config_watcher(&mut state);
8290 let state = Arc::new(state);
8291 let baseline_seq = state.account_sink.latest_seq();
8292 let (reached_tx, reached_rx) = std::sync::mpsc::sync_channel(0);
8293 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
8294 set_generic_before_event_test_hook(dir.path(), move || {
8295 reached_tx.send(()).unwrap();
8296 release_rx.recv().unwrap();
8297 });
8298
8299 let first = {
8300 let state = state.clone();
8301 tokio::spawn(async move {
8302 state
8303 .update_config(
8304 |config| {
8305 config.server.port = 22_231;
8306 Ok(())
8307 },
8308 ConfigUpdateEffects::default(),
8309 )
8310 .await
8311 })
8312 };
8313 tokio::task::spawn_blocking(move || reached_rx.recv().unwrap())
8314 .await
8315 .unwrap();
8316 let second = {
8317 let state = state.clone();
8318 tokio::spawn(async move {
8319 state
8320 .update_config(
8321 |config| {
8322 config.server.port = 22_232;
8323 Ok(())
8324 },
8325 ConfigUpdateEffects::default(),
8326 )
8327 .await
8328 })
8329 };
8330 tokio::time::sleep(Duration::from_millis(100)).await;
8331 assert!(
8332 !second.is_finished(),
8333 "the later writer must remain behind the first writer's event"
8334 );
8335 let events = bamboo_engine::events::journal::read_since(
8336 state.account_sink.events_dir(),
8337 baseline_seq,
8338 )
8339 .unwrap();
8340 assert!(
8341 events.iter().all(|event| !matches!(
8342 &event.event,
8343 AgentEvent::ConfigChanged { section, .. } if section == "core"
8344 )),
8345 "neither local commit can publish while the first owns config_io_lock"
8346 );
8347
8348 release_tx.send(()).unwrap();
8349 assert_eq!(first.await.unwrap().unwrap().server.port, 22_231);
8350 assert_eq!(second.await.unwrap().unwrap().server.port, 22_232);
8351 tokio::time::timeout(Duration::from_secs(3), async {
8352 loop {
8353 let events = bamboo_engine::events::journal::read_since(
8354 state.account_sink.events_dir(),
8355 baseline_seq,
8356 )
8357 .unwrap();
8358 let revisions = events
8359 .iter()
8360 .filter_map(|event| match &event.event {
8361 AgentEvent::ConfigChanged { section, revision } if section == "core" => {
8362 Some(*revision)
8363 }
8364 _ => None,
8365 })
8366 .collect::<Vec<_>>();
8367 if revisions.len() == 2 {
8368 break revisions;
8369 }
8370 tokio::time::sleep(Duration::from_millis(20)).await;
8371 }
8372 })
8373 .await
8374 .map(|revisions| assert_eq!(revisions, vec![1, 2]))
8375 .expect("both serialized core events must reach the journal");
8376 }
8377
8378 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8379 async fn generic_runtime_effects_finish_before_later_config_writer() {
8380 let dir = tempfile::tempdir().unwrap();
8381 let script = dir.path().join("mcp-fixture.py");
8382 std::fs::write(
8383 &script,
8384 r#"import json
8385import sys
8386
8387for line in sys.stdin:
8388 request = json.loads(line)
8389 request_id = request.get("id")
8390 if request_id is None:
8391 continue
8392 if request.get("method") == "server/discover":
8393 print(json.dumps({
8394 "jsonrpc": "2.0",
8395 "id": request_id,
8396 "error": {"code": -32601, "message": "Method not found"},
8397 }), flush=True)
8398 continue
8399 if request.get("method") == "initialize":
8400 result = {
8401 "protocolVersion": "2024-11-05",
8402 "capabilities": {"tools": {"listChanged": False}},
8403 "serverInfo": {"name": "config-order-fixture", "version": "1.0.0"},
8404 }
8405 elif request.get("method") == "tools/list":
8406 result = {"tools": []}
8407 else:
8408 result = {}
8409 print(json.dumps({"jsonrpc": "2.0", "id": request_id, "result": result}), flush=True)
8410"#,
8411 )
8412 .unwrap();
8413 let python = ["python3", "python"]
8414 .into_iter()
8415 .find(|command| {
8416 std::process::Command::new(command)
8417 .arg("--version")
8418 .output()
8419 .is_ok_and(|output| output.status.success())
8420 })
8421 .expect("a Python interpreter is required for the MCP ordering fixture");
8422
8423 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
8424 stop_config_watcher(&mut state);
8425 let state = Arc::new(state);
8426 let held_provider = state.provider.write().await;
8427 let (provider_ready_tx, provider_ready_rx) = std::sync::mpsc::channel();
8428 set_generic_before_provider_publish_test_hook(dir.path(), move || {
8429 provider_ready_tx.send(()).unwrap();
8430 });
8431
8432 let first = {
8433 let state = state.clone();
8434 tokio::spawn(async move {
8435 state
8436 .update_config(
8437 |config| {
8438 config.provider = "copilot".to_string();
8439 Ok(())
8440 },
8441 ConfigUpdateEffects {
8442 reload_provider: bamboo_config::patch::ReloadMode::Strict,
8443 reconcile_mcp: bamboo_config::patch::ReloadMode::Strict,
8444 },
8445 )
8446 .await
8447 })
8448 };
8449 tokio::task::spawn_blocking(move || provider_ready_rx.recv().unwrap())
8450 .await
8451 .unwrap();
8452 assert!(
8453 state.config_io_lock.try_lock().is_err(),
8454 "the first writer must retain config_io_lock until its runtime effects finish"
8455 );
8456 assert!(
8457 !first.is_finished(),
8458 "the first writer must still be waiting to publish its provider"
8459 );
8460
8461 let later_mcp = McpConfig {
8462 version: 1,
8463 servers: vec![McpServerConfig {
8464 id: "later-winner".to_string(),
8465 name: None,
8466 enabled: true,
8467 transport: TransportConfig::Stdio(StdioConfig {
8468 command: python.to_string(),
8469 args: vec![script.to_string_lossy().into_owned()],
8470 cwd: None,
8471 env: std::collections::HashMap::new(),
8472 env_encrypted: std::collections::HashMap::new(),
8473 env_credential_refs: std::collections::HashMap::new(),
8474 startup_timeout_ms: 2_000,
8475 }),
8476 request_timeout_ms: 2_000,
8477 healthcheck_interval_ms: 10_000,
8478 reconnect: ReconnectConfig {
8479 enabled: false,
8480 ..Default::default()
8481 },
8482 allowed_tools: vec![],
8483 denied_tools: vec![],
8484 }],
8485 };
8486 let second = {
8487 let state = state.clone();
8488 tokio::spawn(async move {
8489 state
8490 .update_config(
8491 move |config| {
8492 config.mcp = later_mcp;
8493 Ok(())
8494 },
8495 ConfigUpdateEffects {
8496 reload_provider: bamboo_config::patch::ReloadMode::None,
8497 reconcile_mcp: bamboo_config::patch::ReloadMode::Strict,
8498 },
8499 )
8500 .await
8501 })
8502 };
8503
8504 drop(held_provider);
8505 first.await.unwrap().unwrap();
8506 let published = second.await.unwrap().unwrap();
8507 assert_eq!(published.mcp.servers[0].id, "later-winner");
8508 assert_eq!(state.config.read().await.mcp.servers[0].id, "later-winner");
8509 assert_eq!(
8510 state.mcp_manager.list_servers(),
8511 vec!["later-winner".to_string()],
8512 "the later durable config generation must remain the final runtime generation"
8513 );
8514 state.mcp_manager.shutdown_all().await;
8515 }
8516
8517 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8518 async fn direct_provider_reload_cannot_publish_after_later_config_generation() {
8519 let dir = tempfile::tempdir().unwrap();
8520 let mut initial = Config::default();
8521 initial.provider = "openai".to_string();
8522 initial.providers_mut().openai = Some(bamboo_config::OpenAIConfig {
8523 api_key: "first-generation-key".to_string(),
8524 base_url: Some("http://127.0.0.1:1/v1".to_string()),
8525 ..Default::default()
8526 });
8527 let mut state = AppState::new_with_provider(
8528 dir.path().to_path_buf(),
8529 initial,
8530 Arc::new(WorkingProvider),
8531 )
8532 .await
8533 .unwrap();
8534 stop_config_watcher(&mut state);
8535 let state = Arc::new(state);
8536 let quiesced = tokio::time::timeout(Duration::from_secs(5), state.config_io_lock.lock())
8537 .await
8538 .expect("startup config work must quiesce");
8539 drop(quiesced);
8540 let (reload_ready_tx, reload_ready_rx) = tokio::sync::oneshot::channel();
8541 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
8542 set_generic_before_provider_publish_test_hook(dir.path(), move || {
8543 let _ = reload_ready_tx.send(());
8544 release_rx.recv().unwrap();
8545 });
8546
8547 let reload = {
8548 let state = state.clone();
8549 tokio::spawn(async move { state.reload_provider().await })
8550 };
8551 tokio::time::timeout(Duration::from_secs(5), reload_ready_rx)
8552 .await
8553 .expect("direct reload reaches provider publication hook")
8554 .unwrap();
8555 assert!(state.config_io_lock.try_lock().is_err());
8556
8557 let mut later_instance: bamboo_config::ProviderInstanceConfig =
8558 serde_json::from_value(serde_json::json!({
8559 "provider_type": "openai",
8560 "base_url": "http://127.0.0.1:1/v1",
8561 "enabled": true
8562 }))
8563 .unwrap();
8564 later_instance.api_key = "later-generation-key".to_string();
8565 let (later_started_tx, later_started_rx) = tokio::sync::oneshot::channel();
8566 let later = {
8567 let state = state.clone();
8568 tokio::spawn(async move {
8569 let _ = later_started_tx.send(());
8570 state
8571 .update_config_with_provider_credentials(
8572 move |config| {
8573 config
8574 .provider_instances
8575 .insert("later-winner".to_string(), later_instance);
8576 config.default_provider_instance = Some("later-winner".to_string());
8577 Ok(())
8578 },
8579 BTreeSet::new(),
8580 BTreeSet::from(["later-winner".to_string()]),
8581 ConfigUpdateEffects {
8582 reload_provider: bamboo_config::patch::ReloadMode::Strict,
8583 reconcile_mcp: bamboo_config::patch::ReloadMode::None,
8584 },
8585 )
8586 .await
8587 })
8588 };
8589 later_started_rx.await.unwrap();
8590 assert!(!later.is_finished());
8591
8592 release_tx.send(()).unwrap();
8593 tokio::time::timeout(Duration::from_secs(10), async {
8594 reload.await.unwrap().unwrap();
8595 later.await.unwrap().unwrap();
8596 })
8597 .await
8598 .expect("serialized provider generations finish");
8599 assert_eq!(
8600 state
8601 .config
8602 .read()
8603 .await
8604 .default_provider_instance
8605 .as_deref(),
8606 Some("later-winner")
8607 );
8608 assert_eq!(
8609 state.provider_registry.default_provider_name(),
8610 "later-winner"
8611 );
8612 let registry_default = state.provider_registry.get_default().unwrap();
8613 let live_provider = state.provider.read().await.clone();
8614 assert!(
8615 Arc::ptr_eq(®istry_default, &live_provider),
8616 "registry default and reloadable provider handle must publish one generation"
8617 );
8618 }
8619
8620 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8621 async fn combined_reload_cannot_publish_captured_mcp_after_later_generation() {
8622 let dir = tempfile::tempdir().unwrap();
8623 let mut initial = Config::default();
8624 initial.provider = "openai".to_string();
8625 initial.providers_mut().openai = Some(bamboo_config::OpenAIConfig {
8626 api_key: "combined-reload-key".to_string(),
8627 base_url: Some("http://127.0.0.1:1/v1".to_string()),
8628 ..Default::default()
8629 });
8630 let mut state = AppState::new_with_provider(
8631 dir.path().to_path_buf(),
8632 initial,
8633 Arc::new(WorkingProvider),
8634 )
8635 .await
8636 .unwrap();
8637 stop_config_watcher(&mut state);
8638 state
8639 .update_config_with_provider_credentials(
8640 |_| Ok(()),
8641 BTreeSet::from(["openai".to_string()]),
8642 BTreeSet::new(),
8643 ConfigUpdateEffects::default(),
8644 )
8645 .await
8646 .unwrap();
8647 let first_mcp = working_stdio_mcp_config(dir.path(), "captured-first", None);
8648 state
8649 .update_config(
8650 move |config| {
8651 config.mcp = first_mcp;
8652 Ok(())
8653 },
8654 ConfigUpdateEffects::default(),
8655 )
8656 .await
8657 .unwrap();
8658 let state = Arc::new(state);
8659 let (reload_ready_tx, reload_ready_rx) = tokio::sync::oneshot::channel();
8660 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
8661 set_generic_before_provider_publish_test_hook(dir.path(), move || {
8662 let _ = reload_ready_tx.send(());
8663 release_rx.recv().unwrap();
8664 });
8665
8666 let reload = {
8667 let state = state.clone();
8668 tokio::spawn(async move { state.reload_config_and_runtime().await })
8669 };
8670 tokio::time::timeout(Duration::from_secs(5), reload_ready_rx)
8671 .await
8672 .expect("combined reload reaches provider publication hook")
8673 .unwrap();
8674 assert!(state.config_io_lock.try_lock().is_err());
8675
8676 let later_mcp = working_stdio_mcp_config(dir.path(), "later-winner", None);
8677 let (later_started_tx, later_started_rx) = tokio::sync::oneshot::channel();
8678 let later = {
8679 let state = state.clone();
8680 tokio::spawn(async move {
8681 let _ = later_started_tx.send(());
8682 state
8683 .update_config(
8684 move |config| {
8685 config.mcp = later_mcp;
8686 Ok(())
8687 },
8688 ConfigUpdateEffects {
8689 reload_provider: bamboo_config::patch::ReloadMode::None,
8690 reconcile_mcp: bamboo_config::patch::ReloadMode::Strict,
8691 },
8692 )
8693 .await
8694 })
8695 };
8696 later_started_rx.await.unwrap();
8697 assert!(!later.is_finished());
8698
8699 release_tx.send(()).unwrap();
8700 tokio::time::timeout(Duration::from_secs(10), async {
8701 reload.await.unwrap().unwrap();
8702 later.await.unwrap().unwrap();
8703 })
8704 .await
8705 .expect("serialized config/runtime generations finish");
8706 assert_eq!(state.config.read().await.mcp.servers[0].id, "later-winner");
8707 assert_eq!(
8708 state.mcp_manager.list_servers(),
8709 vec!["later-winner".to_string()]
8710 );
8711 state.mcp_manager.shutdown_all().await;
8712 }
8713
8714 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8715 async fn legacy_mcp_credentials_round_trip_without_plaintext_in_durable_or_events() {
8716 let dir = tempfile::tempdir().unwrap();
8717 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
8718 stop_config_watcher(&mut state);
8719 let state = Arc::new(state);
8720 let baseline_seq = state.account_sink.latest_seq();
8721 let secret = "legacy-mcp-roundtrip-secret";
8722 let mut candidate = disabled_mcp_config("credential-server");
8723 let TransportConfig::Stdio(stdio) = &mut candidate.servers[0].transport else {
8724 unreachable!()
8725 };
8726 stdio.env.insert("TOKEN".to_string(), secret.to_string());
8727
8728 state
8729 .update_legacy_mcp_config(BTreeSet::new(), move |mcp| {
8730 *mcp = candidate;
8731 Ok(())
8732 })
8733 .await
8734 .unwrap();
8735 let live = state.config.read().await.clone();
8736 let TransportConfig::Stdio(stdio) = &live.mcp.servers[0].transport else {
8737 unreachable!()
8738 };
8739 assert_eq!(stdio.env["TOKEN"], secret);
8740 let reference =
8741 bamboo_config::CredentialRef::parse(stdio.env_credential_refs["TOKEN"].clone())
8742 .unwrap();
8743 assert_eq!(
8744 state
8745 .credential_store
8746 .resolve(&reference)
8747 .unwrap()
8748 .unwrap()
8749 .expose(),
8750 secret
8751 );
8752 for path in [
8753 dir.path().join("mcp.json"),
8754 dir.path().join("credentials.json"),
8755 ] {
8756 let bytes = std::fs::read(path).unwrap();
8757 assert!(!String::from_utf8_lossy(&bytes).contains(secret));
8758 }
8759 let events = bamboo_engine::events::journal::read_since(
8760 state.account_sink.events_dir(),
8761 baseline_seq,
8762 )
8763 .unwrap();
8764 assert!(!format!("{events:?}").contains(secret));
8765
8766 state
8767 .update_legacy_mcp_config(BTreeSet::new(), |mcp| {
8768 let TransportConfig::Stdio(stdio) = &mut mcp.servers[0].transport else {
8769 unreachable!()
8770 };
8771 stdio
8772 .env
8773 .insert("TOKEN".to_string(), "****...****".to_string());
8774 Ok(())
8775 })
8776 .await
8777 .unwrap();
8778 let live = state.config.read().await.clone();
8779 let TransportConfig::Stdio(stdio) = &live.mcp.servers[0].transport else {
8780 unreachable!()
8781 };
8782 assert_eq!(stdio.env["TOKEN"], secret);
8783 assert_eq!(stdio.env_credential_refs["TOKEN"], reference.as_str());
8784 }
8785
8786 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8787 async fn legacy_mcp_cancelled_start_finishes_before_later_delete() {
8788 let dir = tempfile::tempdir().unwrap();
8789 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
8790 stop_config_watcher(&mut state);
8791 let state = Arc::new(state);
8792 let secret = "legacy-mcp-cancel-secret";
8793 let candidate = working_stdio_mcp_config(dir.path(), "cancelled-start", Some(secret));
8794 let reference =
8795 bamboo_config::credential_ref("mcp", "cancelled-start", "env_TOKEN").unwrap();
8796 let (commit_tx, commit_rx) = tokio::sync::oneshot::channel();
8797 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
8798 set_credential_after_commit_before_live_test_hook(dir.path(), SectionId::Mcp, move || {
8799 let _ = commit_tx.send(());
8800 release_rx.recv().unwrap();
8801 });
8802
8803 let operation = {
8804 let state = state.clone();
8805 tokio::spawn(async move {
8806 state
8807 .update_legacy_mcp_config(BTreeSet::new(), move |mcp| {
8808 *mcp = candidate;
8809 Ok(())
8810 })
8811 .await
8812 })
8813 };
8814 tokio::time::timeout(Duration::from_secs(5), commit_rx)
8815 .await
8816 .expect("legacy MCP write reaches durable-before-live hook")
8817 .unwrap();
8818 assert!(state.config_io_lock.try_lock().is_err());
8819 assert!(state.mcp_manager.list_servers().is_empty());
8820 assert!(
8821 !String::from_utf8_lossy(&std::fs::read(dir.path().join("mcp.json")).unwrap())
8822 .contains(secret)
8823 );
8824 operation.abort();
8825 assert!(operation.await.unwrap_err().is_cancelled());
8826
8827 let (delete_started_tx, delete_started_rx) = tokio::sync::oneshot::channel();
8828 let delete = {
8829 let state = state.clone();
8830 tokio::spawn(async move {
8831 let _ = delete_started_tx.send(());
8832 state
8833 .update_legacy_mcp_config(BTreeSet::new(), |mcp| {
8834 mcp.servers.clear();
8835 Ok(())
8836 })
8837 .await
8838 })
8839 };
8840 delete_started_rx.await.unwrap();
8841 assert!(!delete.is_finished());
8842 release_tx.send(()).unwrap();
8843 delete.await.unwrap().unwrap();
8844
8845 assert!(state.config.read().await.mcp.servers.is_empty());
8846 assert!(state.mcp_manager.list_servers().is_empty());
8847 assert!(state.mcp_manager.tool_index().all_aliases().is_empty());
8848 assert!(state
8849 .credential_store
8850 .resolve(&reference)
8851 .unwrap()
8852 .is_none());
8853 assert!(
8854 !String::from_utf8_lossy(&std::fs::read(dir.path().join("mcp.json")).unwrap())
8855 .contains(secret)
8856 );
8857 }
8858
8859 #[tokio::test]
8860 async fn legacy_mcp_rejects_secret_bearing_url_before_runtime_or_commit() {
8861 let dir = tempfile::tempdir().unwrap();
8862 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
8863 stop_config_watcher(&mut state);
8864 let before = std::fs::read(dir.path().join("mcp.json")).unwrap();
8865 let secret = "must-never-connect-or-log";
8866 let candidate = McpConfig {
8867 version: 1,
8868 servers: vec![McpServerConfig {
8869 id: "unsafe-url".to_string(),
8870 name: None,
8871 enabled: true,
8872 transport: TransportConfig::Sse(bamboo_mcp::SseConfig {
8873 url: format!("https://example.test/sse?token={secret}"),
8874 headers: vec![],
8875 connect_timeout_ms: 100,
8876 }),
8877 request_timeout_ms: 100,
8878 healthcheck_interval_ms: 100,
8879 reconnect: ReconnectConfig::default(),
8880 allowed_tools: vec![],
8881 denied_tools: vec![],
8882 }],
8883 };
8884 let error = state
8885 .update_legacy_mcp_config(BTreeSet::new(), move |mcp| {
8886 *mcp = candidate;
8887 Ok(())
8888 })
8889 .await
8890 .unwrap_err();
8891 assert!(matches!(error, AppError::BadRequest(_)));
8892 assert!(!error.to_string().contains(secret));
8893 assert_eq!(std::fs::read(dir.path().join("mcp.json")).unwrap(), before);
8894 assert!(state.config.read().await.mcp.servers.is_empty());
8895 assert!(state.mcp_manager.list_servers().is_empty());
8896 }
8897
8898 #[tokio::test]
8899 async fn legacy_mcp_start_failure_keeps_every_authority_on_the_previous_generation() {
8900 let dir = tempfile::tempdir().unwrap();
8901 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
8902 stop_config_watcher(&mut state);
8903 let disk_before = std::fs::read(dir.path().join("mcp.json")).unwrap();
8904 let config_before = state.config.read().await.clone();
8905 let facade_before = state
8906 .config_facade
8907 .as_ref()
8908 .unwrap()
8909 .registry()
8910 .mcp
8911 .snapshot();
8912 let health_before = state
8913 .mcp_config_live_health
8914 .read()
8915 .unwrap_or_else(|poisoned| poisoned.into_inner())
8916 .clone();
8917 let baseline_seq = state.account_sink.latest_seq();
8918 let mut failing = disabled_mcp_config("never-committed");
8919 failing.servers[0].enabled = true;
8920 let TransportConfig::Stdio(stdio) = &mut failing.servers[0].transport else {
8921 unreachable!()
8922 };
8923 stdio.command = "definitely-not-a-real-mcp-command-before-commit-736".to_string();
8924
8925 let error = state
8926 .update_legacy_mcp_config(BTreeSet::new(), move |mcp| {
8927 *mcp = failing;
8928 Ok(())
8929 })
8930 .await
8931 .expect_err("runtime staging must fail before the MCP durable boundary");
8932 assert!(matches!(error, AppError::InternalError(_)));
8933 assert_eq!(
8934 error.to_string(),
8935 "Internal server error: MCP runtime initialization failed before commit; retaining last-known-good generation"
8936 );
8937 assert_eq!(
8938 std::fs::read(dir.path().join("mcp.json")).unwrap(),
8939 disk_before
8940 );
8941 assert_eq!(
8942 serde_json::to_value(state.config.read().await.clone()).unwrap(),
8943 serde_json::to_value(config_before).unwrap()
8944 );
8945 assert!(state.mcp_manager.list_servers().is_empty());
8946 assert!(state.mcp_manager.tool_index().all_aliases().is_empty());
8947
8948 let facade_after = state
8949 .config_facade
8950 .as_ref()
8951 .unwrap()
8952 .registry()
8953 .mcp
8954 .snapshot();
8955 assert_eq!(facade_after.revision, facade_before.revision);
8956 assert_eq!(facade_after.loaded_at, facade_before.loaded_at);
8957 assert_eq!(facade_after.status, facade_before.status);
8958 let health_after = state
8959 .mcp_config_live_health
8960 .read()
8961 .unwrap_or_else(|poisoned| poisoned.into_inner())
8962 .clone();
8963 assert_eq!(health_after.revision, health_before.revision);
8964 assert_eq!(health_after.loaded_at, health_before.loaded_at);
8965 assert_eq!(health_after.status, health_before.status);
8966 assert_eq!(health_after.last_error, health_before.last_error);
8967 assert!(bamboo_engine::events::journal::read_since(
8968 state.account_sink.events_dir(),
8969 baseline_seq,
8970 )
8971 .unwrap()
8972 .into_iter()
8973 .all(|event| !matches!(
8974 event.event,
8975 AgentEvent::ConfigChanged { ref section, .. }
8976 | AgentEvent::ConfigInvalid { ref section, .. }
8977 | AgentEvent::ConfigRecovered { ref section, .. }
8978 if section == "mcp"
8979 )));
8980 }
8981
8982 #[tokio::test]
8983 async fn invalid_explicit_reload_retains_live_provider_generation_and_marks_health() {
8984 let dir = tempfile::tempdir().unwrap();
8985 let mut initial = Config::default();
8986 initial.server.port = 24_301;
8987 initial.save_to_dir(dir.path().to_path_buf()).unwrap();
8988 let injected: Arc<dyn LLMProvider> = Arc::new(WorkingProvider);
8989 let mut state = AppState::new_with_provider(
8990 dir.path().to_path_buf(),
8991 initial.clone(),
8992 injected.clone(),
8993 )
8994 .await
8995 .unwrap();
8996 stop_config_watcher(&mut state);
8997 let expected_provider = state.config.read().await.provider.clone();
8998
8999 let mut invalid = initial;
9000 invalid.provider = "unknown-invalid-provider".to_string();
9001 invalid.save_to_dir(dir.path().to_path_buf()).unwrap();
9002 let error = state.reload_config_and_runtime().await.unwrap_err();
9003 assert!(matches!(error, AppError::BadRequest(_)));
9004 assert_eq!(state.config.read().await.server.port, 24_301);
9005 assert_eq!(state.config.read().await.provider, expected_provider);
9006 let live_provider = state.provider.read().await.clone();
9007 assert!(Arc::ptr_eq(&live_provider, &injected));
9008 let health = state
9009 .config_live_health
9010 .read()
9011 .unwrap_or_else(|poisoned| poisoned.into_inner())
9012 .clone();
9013 assert_eq!(health.status, SectionStatus::Invalid);
9014 assert_eq!(
9015 health.last_error.as_deref(),
9016 Some("provider configuration is invalid; retaining last-known-good generation")
9017 );
9018 }
9019
9020 #[tokio::test]
9021 async fn committed_provider_start_failure_publishes_one_exact_revision() {
9022 let dir = tempfile::tempdir().unwrap();
9023 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9024 stop_config_watcher(&mut state);
9025 let baseline_seq = state.account_sink.latest_seq();
9026 let previous_provider = state.provider.read().await.clone();
9027 let previous_default = state.provider_registry.default_provider_name();
9028
9029 let published = state
9030 .update_config(
9031 |config| {
9032 config.provider = "unknown-runtime-provider".to_string();
9033 config.default_provider_instance = None;
9034 config.provider_instances.clear();
9035 Ok(())
9036 },
9037 ConfigUpdateEffects {
9038 reload_provider: bamboo_config::patch::ReloadMode::BestEffort,
9039 reconcile_mcp: bamboo_config::patch::ReloadMode::None,
9040 },
9041 )
9042 .await
9043 .expect("the durable provider generation commits with degraded runtime health");
9044 assert_eq!(published.provider, "unknown-runtime-provider");
9045 assert_eq!(
9046 state.config.read().await.provider,
9047 "unknown-runtime-provider"
9048 );
9049 assert_eq!(
9050 state.provider_registry.default_provider_name(),
9051 previous_default
9052 );
9053 assert!(Arc::ptr_eq(
9054 &state.provider.read().await.clone(),
9055 &previous_provider
9056 ));
9057
9058 let facade_snapshot = state
9059 .config_facade
9060 .as_ref()
9061 .unwrap()
9062 .registry()
9063 .providers
9064 .snapshot();
9065 let health = state
9066 .config_live_health
9067 .read()
9068 .unwrap_or_else(|poisoned| poisoned.into_inner())
9069 .clone();
9070 assert_eq!(facade_snapshot.revision, 1);
9071 assert_eq!(facade_snapshot.status, SectionStatus::Degraded);
9072 assert_eq!(health.revision, facade_snapshot.revision);
9073 assert_eq!(health.loaded_at, facade_snapshot.loaded_at);
9074 assert_eq!(health.source_path, facade_snapshot.source_path);
9075 assert_eq!(health.source_kind, facade_snapshot.source_kind);
9076 assert_eq!(health.status, facade_snapshot.status);
9077 assert_eq!(health.last_error, facade_snapshot.last_error);
9078
9079 let invalid_revisions = bamboo_engine::events::journal::read_since(
9080 state.account_sink.events_dir(),
9081 baseline_seq,
9082 )
9083 .unwrap()
9084 .into_iter()
9085 .filter_map(|event| match event.event {
9086 AgentEvent::ConfigInvalid { section, revision } if section == "providers" => {
9087 Some(revision)
9088 }
9089 _ => None,
9090 })
9091 .collect::<Vec<_>>();
9092 assert_eq!(invalid_revisions, vec![facade_snapshot.revision]);
9093 }
9094
9095 #[tokio::test]
9096 async fn committed_mcp_start_failure_publishes_one_exact_revision() {
9097 let dir = tempfile::tempdir().unwrap();
9098 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9099 stop_config_watcher(&mut state);
9100 let baseline_seq = state.account_sink.latest_seq();
9101 let mut failing = disabled_mcp_config("committed-but-unstartable");
9102 failing.servers[0].enabled = true;
9103 let TransportConfig::Stdio(stdio) = &mut failing.servers[0].transport else {
9104 unreachable!()
9105 };
9106 stdio.command = "definitely-not-a-real-mcp-command-736".to_string();
9107
9108 let published = state
9109 .update_config(
9110 move |config| {
9111 config.mcp = failing;
9112 Ok(())
9113 },
9114 ConfigUpdateEffects {
9115 reload_provider: bamboo_config::patch::ReloadMode::None,
9116 reconcile_mcp: bamboo_config::patch::ReloadMode::BestEffort,
9117 },
9118 )
9119 .await
9120 .expect("the durable MCP generation commits with degraded runtime health");
9121 assert_eq!(published.mcp.servers[0].id, "committed-but-unstartable");
9122 assert_eq!(
9123 state.config.read().await.mcp.servers[0].id,
9124 "committed-but-unstartable"
9125 );
9126 assert!(state.mcp_manager.list_servers().is_empty());
9127
9128 let facade_snapshot = state
9129 .config_facade
9130 .as_ref()
9131 .unwrap()
9132 .registry()
9133 .mcp
9134 .snapshot();
9135 let health = state
9136 .mcp_config_live_health
9137 .read()
9138 .unwrap_or_else(|poisoned| poisoned.into_inner())
9139 .clone();
9140 assert_eq!(facade_snapshot.revision, 1);
9141 assert_eq!(facade_snapshot.status, SectionStatus::Degraded);
9142 assert_eq!(health.revision, facade_snapshot.revision);
9143 assert_eq!(health.loaded_at, facade_snapshot.loaded_at);
9144 assert_eq!(health.source_path, facade_snapshot.source_path);
9145 assert_eq!(health.source_kind, facade_snapshot.source_kind);
9146 assert_eq!(health.status, facade_snapshot.status);
9147 assert_eq!(health.last_error, facade_snapshot.last_error);
9148
9149 let invalid_revisions = bamboo_engine::events::journal::read_since(
9150 state.account_sink.events_dir(),
9151 baseline_seq,
9152 )
9153 .unwrap()
9154 .into_iter()
9155 .filter_map(|event| match event.event {
9156 AgentEvent::ConfigInvalid { section, revision } if section == "mcp" => Some(revision),
9157 _ => None,
9158 })
9159 .collect::<Vec<_>>();
9160 assert_eq!(invalid_revisions, vec![facade_snapshot.revision]);
9161 }
9162
9163 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
9164 async fn cancelled_legacy_reset_finishes_deletion_and_runtime_publication() {
9165 let dir = tempfile::tempdir().unwrap();
9166 let mut initial = Config::default();
9167 initial.server.port = 24_302;
9168 initial.save_to_dir(dir.path().to_path_buf()).unwrap();
9169 std::fs::write(dir.path().join("config.json.bak"), b"recovery-marker").unwrap();
9170 std::fs::write(dir.path().join("model_limits.json"), b"{}").unwrap();
9171 std::fs::write(dir.path().join("connect.json"), b"{}").unwrap();
9172 std::fs::write(dir.path().join("connect.json.bak"), b"credential-backup").unwrap();
9173 let injected: Arc<dyn LLMProvider> = Arc::new(WorkingProvider);
9174 let mut state =
9175 AppState::new_with_provider(dir.path().to_path_buf(), initial, injected.clone())
9176 .await
9177 .unwrap();
9178 stop_config_watcher(&mut state);
9179 let state = Arc::new(state);
9180 let (deleted_tx, deleted_rx) = tokio::sync::oneshot::channel();
9181 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
9182 set_reset_after_delete_test_hook(dir.path(), move || {
9183 let _ = deleted_tx.send(());
9184 release_rx.recv().unwrap();
9185 });
9186
9187 let operation = {
9188 let state = state.clone();
9189 tokio::spawn(async move { state.reset_legacy_config_and_runtime().await })
9190 };
9191 tokio::time::timeout(Duration::from_secs(5), deleted_rx)
9192 .await
9193 .expect("reset reaches durable delete boundary")
9194 .unwrap();
9195 for path in [
9196 dir.path().join("config.json"),
9197 dir.path().join("model_limits.json"),
9198 dir.path().join("connect.json"),
9199 dir.path().join("connect.json.bak"),
9200 ] {
9201 assert!(!path.exists());
9202 }
9203 assert_eq!(
9204 std::fs::read(dir.path().join("config.json.bak")).unwrap(),
9205 b"recovery-marker"
9206 );
9207 assert_eq!(state.config.read().await.server.port, 24_302);
9208 operation.abort();
9209 assert!(operation.await.unwrap_err().is_cancelled());
9210
9211 release_tx.send(()).unwrap();
9212 let completed = tokio::time::timeout(Duration::from_secs(5), state.config_io_lock.lock())
9213 .await
9214 .expect("detached reset must finish live/runtime publication");
9215 drop(completed);
9216 assert_eq!(state.config.read().await.server.port, 9562);
9217 assert!(state.mcp_manager.list_servers().is_empty());
9218 let health = state
9219 .config_live_health
9220 .read()
9221 .unwrap_or_else(|poisoned| poisoned.into_inner())
9222 .clone();
9223 assert_eq!(
9224 health.status,
9225 SectionStatus::Degraded,
9226 "the committed default config has no usable Anthropic credential, so reset must report truthful runtime degradation"
9227 );
9228 assert_eq!(
9229 health.last_error.as_deref(),
9230 Some("provider runtime initialization failed; retaining last-known-good runtime")
9231 );
9232 assert!(Arc::ptr_eq(&state.provider.read().await.clone(), &injected));
9233 }
9234
9235 #[tokio::test]
9236 async fn legacy_reset_converges_runtime_after_partial_delete_failure() {
9237 let dir = tempfile::tempdir().unwrap();
9238 let mut initial = Config::default();
9239 initial.server.port = 24_303;
9240 initial.save_to_dir(dir.path().to_path_buf()).unwrap();
9241 std::fs::write(dir.path().join("model_limits.json"), b"{}").unwrap();
9242 let injected: Arc<dyn LLMProvider> = Arc::new(WorkingProvider);
9243 let mut state =
9244 AppState::new_with_provider(dir.path().to_path_buf(), initial, injected.clone())
9245 .await
9246 .unwrap();
9247 stop_config_watcher(&mut state);
9248
9249 std::fs::create_dir(dir.path().join("connect.json")).unwrap();
9253 std::fs::write(dir.path().join("connect.json.bak"), b"credential-backup").unwrap();
9254
9255 let error = state.reset_legacy_config_and_runtime().await.unwrap_err();
9256 assert!(matches!(error, AppError::StorageError(_)));
9257 assert!(!dir.path().join("config.json").exists());
9258 assert!(!dir.path().join("model_limits.json").exists());
9259 assert!(dir.path().join("connect.json").is_dir());
9260 assert!(!dir.path().join("connect.json.bak").exists());
9261 assert_eq!(state.config.read().await.server.port, 9562);
9262 assert!(state.mcp_manager.list_servers().is_empty());
9263 assert!(Arc::ptr_eq(&state.provider.read().await.clone(), &injected));
9264 }
9265
9266 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
9267 async fn generic_update_cancellation_after_commit_finishes_publication() {
9268 let dir = tempfile::tempdir().unwrap();
9269 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9270 stop_config_watcher(&mut state);
9271 let state = Arc::new(state);
9272 let mut feed = state.account_sink.subscribe();
9273 let (reached_tx, reached_rx) = std::sync::mpsc::sync_channel(0);
9274 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
9275 set_generic_before_event_test_hook(dir.path(), move || {
9276 reached_tx.send(()).unwrap();
9277 release_rx.recv().unwrap();
9278 });
9279
9280 let operation = {
9281 let state = state.clone();
9282 tokio::spawn(async move {
9283 state
9284 .update_config(
9285 |config| {
9286 config.server.port = 22_240;
9287 Ok(())
9288 },
9289 ConfigUpdateEffects::default(),
9290 )
9291 .await
9292 })
9293 };
9294 tokio::task::spawn_blocking(move || reached_rx.recv().unwrap())
9295 .await
9296 .unwrap();
9297 assert_eq!(
9298 state
9299 .config_facade
9300 .as_ref()
9301 .unwrap()
9302 .registry()
9303 .core
9304 .snapshot()
9305 .revision,
9306 1,
9307 "the abort boundary must follow durable commit and facade adoption"
9308 );
9309 operation.abort();
9310 assert!(operation.await.unwrap_err().is_cancelled());
9311 release_tx.send(()).unwrap();
9312 let converged = tokio::time::timeout(Duration::from_secs(5), state.config_io_lock.lock())
9313 .await
9314 .expect("detached generic update must finish live publication");
9315 drop(converged);
9316
9317 assert_eq!(state.config.read().await.server.port, 22_240);
9318 assert_eq!(
9319 bamboo_config::ConfigFacade::open(dir.path())
9320 .unwrap()
9321 .effective_config()
9322 .server
9323 .port,
9324 22_240
9325 );
9326 assert!(matches!(
9327 next_config_event(&mut feed, "core").await,
9328 AgentEvent::ConfigChanged {
9329 section,
9330 revision: 1
9331 } if section == "core"
9332 ));
9333 }
9334
9335 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
9336 async fn provider_update_cancellation_after_commit_finishes_publication() {
9337 let _key = bamboo_config::encryption::set_test_encryption_key([0x7d; 32]);
9338 let dir = tempfile::tempdir().unwrap();
9339 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9340 stop_config_watcher(&mut state);
9341 let state = Arc::new(state);
9342 let mut feed = state.account_sink.subscribe();
9343 let (reached_tx, reached_rx) = std::sync::mpsc::sync_channel(0);
9344 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
9345 set_generic_before_event_test_hook(dir.path(), move || {
9346 reached_tx.send(()).unwrap();
9347 release_rx.recv().unwrap();
9348 });
9349
9350 let operation = {
9351 let state = state.clone();
9352 tokio::spawn(async move {
9353 state
9354 .update_config_with_provider_credentials(
9355 |config| {
9356 config.provider = "openai".to_string();
9357 config.providers_mut().openai = Some(bamboo_config::OpenAIConfig {
9358 api_key: "cancellation-secret".to_string(),
9359 model: Some("cancellation-model".to_string()),
9360 ..Default::default()
9361 });
9362 Ok(())
9363 },
9364 BTreeSet::from(["openai".to_string()]),
9365 BTreeSet::new(),
9366 ConfigUpdateEffects::default(),
9367 )
9368 .await
9369 })
9370 };
9371 tokio::task::spawn_blocking(move || reached_rx.recv().unwrap())
9372 .await
9373 .unwrap();
9374 assert_eq!(
9375 state
9376 .config_facade
9377 .as_ref()
9378 .unwrap()
9379 .registry()
9380 .providers
9381 .snapshot()
9382 .revision,
9383 1,
9384 "the abort boundary must follow provider durable/facade adoption"
9385 );
9386 operation.abort();
9387 assert!(operation.await.unwrap_err().is_cancelled());
9388 release_tx.send(()).unwrap();
9389 let converged = tokio::time::timeout(Duration::from_secs(5), state.config_io_lock.lock())
9390 .await
9391 .expect("detached provider update must finish live publication");
9392 drop(converged);
9393
9394 assert_eq!(state.config.read().await.provider, "openai");
9395 let durable = bamboo_config::ConfigFacade::open(dir.path())
9396 .unwrap()
9397 .effective_config();
9398 assert_eq!(durable.provider, "openai");
9399 assert_eq!(
9400 durable
9401 .providers()
9402 .openai
9403 .as_ref()
9404 .unwrap()
9405 .model
9406 .as_deref(),
9407 Some("cancellation-model")
9408 );
9409 assert!(matches!(
9410 next_config_event(&mut feed, "providers").await,
9411 AgentEvent::ConfigChanged {
9412 section,
9413 revision: 1
9414 } if section == "providers"
9415 ));
9416 for file in ["providers.json", "credentials.json"] {
9417 assert!(
9418 !std::fs::read_to_string(dir.path().join(file))
9419 .unwrap()
9420 .contains("cancellation-secret"),
9421 "{file} must remain secret-free"
9422 );
9423 }
9424 }
9425
9426 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
9427 async fn replace_config_cancellation_after_commit_finishes_publication() {
9428 let dir = tempfile::tempdir().unwrap();
9429 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9430 stop_config_watcher(&mut state);
9431 let state = Arc::new(state);
9432 let mut replacement = state.config.read().await.clone();
9433 replacement.server.port = 22_241;
9434 let mut feed = state.account_sink.subscribe();
9435 let (reached_tx, reached_rx) = std::sync::mpsc::sync_channel(0);
9436 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
9437 set_generic_before_event_test_hook(dir.path(), move || {
9438 reached_tx.send(()).unwrap();
9439 release_rx.recv().unwrap();
9440 });
9441
9442 let operation = {
9443 let state = state.clone();
9444 tokio::spawn(async move {
9445 state
9446 .replace_config(replacement, ConfigUpdateEffects::default())
9447 .await
9448 })
9449 };
9450 tokio::task::spawn_blocking(move || reached_rx.recv().unwrap())
9451 .await
9452 .unwrap();
9453 assert_eq!(
9454 state
9455 .config_facade
9456 .as_ref()
9457 .unwrap()
9458 .registry()
9459 .core
9460 .snapshot()
9461 .revision,
9462 1,
9463 "the abort boundary must follow replacement durable/facade adoption"
9464 );
9465 operation.abort();
9466 assert!(operation.await.unwrap_err().is_cancelled());
9467 release_tx.send(()).unwrap();
9468 let converged = tokio::time::timeout(Duration::from_secs(5), state.config_io_lock.lock())
9469 .await
9470 .expect("detached replacement must finish live publication");
9471 drop(converged);
9472
9473 assert_eq!(state.config.read().await.server.port, 22_241);
9474 assert_eq!(
9475 bamboo_config::ConfigFacade::open(dir.path())
9476 .unwrap()
9477 .effective_config()
9478 .server
9479 .port,
9480 22_241
9481 );
9482 assert!(matches!(
9483 next_config_event(&mut feed, "core").await,
9484 AgentEvent::ConfigChanged {
9485 section,
9486 revision: 1
9487 } if section == "core"
9488 ));
9489 }
9490
9491 #[tokio::test]
9492 async fn deployed_node_delete_and_cluster_reset_reject_before_commit_and_remain_stoppable() {
9493 let dir = tempfile::tempdir().unwrap();
9494 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9495 state
9496 .update_cluster_fabric_credentials(
9497 0,
9498 BTreeMap::from([(
9499 "live-node".to_string(),
9500 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
9501 )]),
9502 |config| {
9503 config.cluster_fabric.nodes.push(bamboo_config::Node {
9504 id: "live-node".to_string(),
9505 label: "live-node".to_string(),
9506 placement: bamboo_config::NodePlacement::Local,
9507 trust_level: bamboo_config::TrustLevel::Trusted,
9508 deploy: bamboo_config::DeployProfile::default(),
9509 state: Some(bamboo_config::NodeState {
9510 status: bamboo_config::NodeStatus::Running,
9511 worker_id: Some("live-worker".to_string()),
9512 ..Default::default()
9513 }),
9514 enabled: true,
9515 });
9516 Ok(())
9517 },
9518 )
9519 .await
9520 .unwrap();
9521 insert_registry_worker(
9522 &state,
9523 bamboo_server_tools::registry_keys::node_key("live-node"),
9524 "live-worker",
9525 )
9526 .await;
9527 let transaction_marker = dir.path().join("config-credential-migration.json");
9528 let marker_before_guard = std::fs::read(&transaction_marker).ok();
9529 bamboo_config::set_cluster_exact_commit_test_fault(
9530 dir.path().to_path_buf(),
9531 bamboo_config::ClusterExactCommitTestFault::AfterManifestRecoveryFailure,
9532 );
9533
9534 let delete = state
9535 .delete_cluster_node_credentials(
9536 1,
9537 "live-node".to_string(),
9538 BTreeMap::from([(
9539 "live-node".to_string(),
9540 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
9541 )]),
9542 |config| {
9543 config
9544 .cluster_fabric
9545 .nodes
9546 .retain(|node| node.id != "live-node");
9547 Ok(())
9548 },
9549 )
9550 .await;
9551 assert!(matches!(delete, Err(AppError::BadRequest(_))));
9552 let reset = state
9553 .reset_credential_backed_section(SectionId::ClusterFabric, 1)
9554 .await;
9555 assert!(matches!(reset, Err(ConfigSectionMutationError::Invalid(_))));
9556 assert_eq!(
9557 std::fs::read(&transaction_marker).ok(),
9558 marker_before_guard,
9559 "registry guards must reject before opening a new durable transaction"
9560 );
9561 assert_eq!(
9562 state
9563 .config_facade
9564 .as_ref()
9565 .unwrap()
9566 .registry()
9567 .cluster_fabric
9568 .snapshot()
9569 .revision,
9570 1
9571 );
9572 assert!(state
9573 .config
9574 .read()
9575 .await
9576 .cluster_fabric
9577 .node("live-node")
9578 .is_some());
9579 assert!(state
9580 .fabric_deployer
9581 .registry()
9582 .lock()
9583 .await
9584 .contains_key(&bamboo_server_tools::registry_keys::node_key("live-node")));
9585
9586 bamboo_config::clear_cluster_exact_commit_test_fault(dir.path());
9587 let stopped = state
9588 .fabric_deployer
9589 .stop_at_revision("live-node", 1)
9590 .await
9591 .unwrap();
9592 assert_eq!(stopped.snapshot.section.revision, 2);
9593 assert!(!state
9594 .fabric_deployer
9595 .registry()
9596 .lock()
9597 .await
9598 .contains_key(&bamboo_server_tools::registry_keys::node_key("live-node")));
9599 let deleted = state
9600 .delete_cluster_node_credentials(
9601 2,
9602 "live-node".to_string(),
9603 BTreeMap::from([(
9604 "live-node".to_string(),
9605 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
9606 )]),
9607 |config| {
9608 config
9609 .cluster_fabric
9610 .nodes
9611 .retain(|node| node.id != "live-node");
9612 Ok(())
9613 },
9614 )
9615 .await
9616 .unwrap();
9617 assert_eq!(deleted.section.revision, 3);
9618 assert!(deleted.config.cluster_fabric.node("live-node").is_none());
9619 }
9620
9621 #[tokio::test]
9622 async fn unrelated_agent_registry_entry_does_not_block_cluster_reset() {
9623 let dir = tempfile::tempdir().unwrap();
9624 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9625 state
9626 .update_cluster_fabric_credentials(
9627 0,
9628 BTreeMap::from([(
9629 "reset-node".to_string(),
9630 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
9631 )]),
9632 |config| {
9633 config.cluster_fabric.nodes.push(bamboo_config::Node {
9634 id: "reset-node".to_string(),
9635 label: "reset-node".to_string(),
9636 placement: bamboo_config::NodePlacement::Local,
9637 trust_level: bamboo_config::TrustLevel::Trusted,
9638 deploy: bamboo_config::DeployProfile::default(),
9639 state: None,
9640 enabled: true,
9641 });
9642 Ok(())
9643 },
9644 )
9645 .await
9646 .unwrap();
9647 let agent_key = bamboo_server_tools::registry_keys::agent_key("unrelated-agent");
9648 insert_registry_worker(&state, agent_key.clone(), "unrelated-agent").await;
9649
9650 state
9651 .reset_credential_backed_section(SectionId::ClusterFabric, 1)
9652 .await
9653 .unwrap();
9654 assert_eq!(
9655 state
9656 .config_facade
9657 .as_ref()
9658 .unwrap()
9659 .registry()
9660 .cluster_fabric
9661 .snapshot()
9662 .revision,
9663 2
9664 );
9665 assert!(state.config.read().await.cluster_fabric.nodes.is_empty());
9666 let unrelated = state
9667 .fabric_deployer
9668 .registry()
9669 .lock()
9670 .await
9671 .remove(&agent_key)
9672 .expect("agent registry entry must survive cluster reset");
9673 unrelated.handle.shutdown().await;
9674 }
9675
9676 #[tokio::test]
9677 async fn operator_cluster_crud_recovers_before_finish_and_converges_once() {
9678 let _key = bamboo_config::encryption::set_test_encryption_key([0x72; 32]);
9679 let dir = tempfile::tempdir().unwrap();
9680 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9681 let baseline_seq = state.account_sink.latest_seq();
9682 bamboo_config::set_cluster_exact_commit_test_fault(
9683 dir.path().to_path_buf(),
9684 bamboo_config::ClusterExactCommitTestFault::BeforeFinish,
9685 );
9686
9687 let committed = state
9688 .update_cluster_fabric_credentials(
9689 0,
9690 BTreeMap::from([(
9691 "recovered-crud-node".to_string(),
9692 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
9693 )]),
9694 |config| {
9695 config.cluster_fabric.nodes.push(bamboo_config::Node {
9696 id: "recovered-crud-node".to_string(),
9697 label: "recovered-crud-node".to_string(),
9698 placement: bamboo_config::NodePlacement::Local,
9699 trust_level: bamboo_config::TrustLevel::Trusted,
9700 deploy: bamboo_config::DeployProfile::default(),
9701 state: None,
9702 enabled: true,
9703 });
9704 Ok(())
9705 },
9706 )
9707 .await
9708 .expect("operator CRUD must recover the committed transaction");
9709 assert_eq!(committed.section.revision, 1);
9710 assert_eq!(
9711 committed
9712 .config
9713 .cluster_fabric
9714 .node("recovered-crud-node")
9715 .unwrap()
9716 .label,
9717 "recovered-crud-node"
9718 );
9719 assert_eq!(
9720 state
9721 .config
9722 .read()
9723 .await
9724 .cluster_fabric
9725 .node("recovered-crud-node")
9726 .unwrap()
9727 .label,
9728 "recovered-crud-node"
9729 );
9730 assert_eq!(
9731 state
9732 .config_facade
9733 .as_ref()
9734 .unwrap()
9735 .registry()
9736 .cluster_fabric
9737 .snapshot()
9738 .revision,
9739 1
9740 );
9741 let reopened = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
9742 assert_eq!(reopened.registry().cluster_fabric.snapshot().revision, 1);
9743 assert_eq!(
9744 reopened
9745 .effective_config()
9746 .cluster_fabric
9747 .node("recovered-crud-node")
9748 .unwrap()
9749 .label,
9750 "recovered-crud-node"
9751 );
9752 bamboo_config::ensure_provider_mcp_migration_ready(dir.path()).unwrap();
9753
9754 tokio::time::sleep(Duration::from_millis(100)).await;
9755 let events = bamboo_engine::events::journal::read_since(
9756 state.account_sink.events_dir(),
9757 baseline_seq,
9758 )
9759 .unwrap();
9760 let cluster_events = events
9761 .iter()
9762 .filter(|event| {
9763 matches!(
9764 &event.event,
9765 AgentEvent::ConfigChanged { section, revision }
9766 if section == "cluster-fabric" && *revision == 1
9767 )
9768 })
9769 .count();
9770 assert_eq!(cluster_events, 1);
9771 assert!(!events.iter().any(|event| {
9772 matches!(
9773 &event.event,
9774 AgentEvent::ConfigChanged { section, .. } if section == "credentials"
9775 )
9776 }));
9777 }
9778
9779 #[tokio::test]
9780 async fn cluster_replace_and_keep_noop_retain_the_exact_hydrated_runtime() {
9781 let _key = bamboo_config::encryption::set_test_encryption_key([0x73; 32]);
9782 let dir = tempfile::tempdir().unwrap();
9783 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9784 let baseline_seq = state.account_sink.latest_seq();
9785 let password_ref = bamboo_config::cluster_password_credential_ref("secret-node").unwrap();
9786 let password_from = |config: &Config| match &config
9787 .cluster_fabric
9788 .node("secret-node")
9789 .expect("secret node exists")
9790 .placement
9791 {
9792 bamboo_config::NodePlacement::Ssh(target) => match &target.auth {
9793 bamboo_config::SshAuth::Password { password, .. } => password.clone(),
9794 _ => panic!("expected password authentication"),
9795 },
9796 _ => panic!("expected SSH placement"),
9797 };
9798
9799 let replaced = state
9800 .update_cluster_fabric_credentials(
9801 0,
9802 BTreeMap::from([(
9803 "secret-node".to_string(),
9804 bamboo_config::ClusterNodeCredentialIntents {
9805 password: bamboo_config::ClusterCredentialAction::Replace(
9806 "exact-password".to_string(),
9807 ),
9808 private_key: bamboo_config::ClusterCredentialAction::Clear,
9809 passphrase: bamboo_config::ClusterCredentialAction::Clear,
9810 },
9811 )]),
9812 |config| {
9813 config.cluster_fabric.nodes.push(bamboo_config::Node {
9814 id: "secret-node".to_string(),
9815 label: "secret-node".to_string(),
9816 placement: bamboo_config::NodePlacement::Ssh(bamboo_config::SshTarget {
9817 host: "secret.example.test".to_string(),
9818 port: 22,
9819 username: "operator".to_string(),
9820 auth: bamboo_config::SshAuth::Password {
9821 password: String::new(),
9822 password_encrypted: None,
9823 },
9824 host_key_fingerprint: None,
9825 }),
9826 trust_level: bamboo_config::TrustLevel::Trusted,
9827 deploy: bamboo_config::DeployProfile::default(),
9828 state: None,
9829 enabled: true,
9830 });
9831 Ok(())
9832 },
9833 )
9834 .await
9835 .unwrap();
9836 assert_eq!(replaced.section.revision, 1);
9837 assert_eq!(password_from(&replaced.config), "exact-password");
9838 assert_eq!(
9839 password_from(&*state.config.read().await),
9840 "exact-password",
9841 "live runtime must install the under-lock hydrated candidate"
9842 );
9843 assert_eq!(replaced.credential_health.revision, 1);
9844 assert_eq!(replaced.credential_statuses.len(), 1);
9845 assert_eq!(replaced.credential_statuses[0].credential_ref, password_ref);
9846 assert!(replaced.credential_statuses[0].configured);
9847
9848 tokio::time::sleep(Duration::from_millis(500)).await;
9849 let replace_events = bamboo_engine::events::journal::read_since(
9850 state.account_sink.events_dir(),
9851 baseline_seq,
9852 )
9853 .unwrap();
9854 let cluster_revisions = replace_events
9855 .iter()
9856 .filter_map(|event| match &event.event {
9857 AgentEvent::ConfigChanged { section, revision } if section == "cluster-fabric" => {
9858 Some(*revision)
9859 }
9860 _ => None,
9861 })
9862 .collect::<Vec<_>>();
9863 assert_eq!(cluster_revisions, vec![1]);
9864 assert!(!replace_events.iter().any(|event| {
9865 matches!(
9866 &event.event,
9867 AgentEvent::ConfigChanged { section, .. }
9868 | AgentEvent::ConfigInvalid { section, .. }
9869 | AgentEvent::ConfigRecovered { section, .. }
9870 if section == "credentials"
9871 )
9872 }));
9873
9874 let noop_baseline_seq = state.account_sink.latest_seq();
9875 let kept = state
9876 .update_cluster_fabric_credentials(
9877 1,
9878 BTreeMap::from([(
9879 "secret-node".to_string(),
9880 bamboo_config::ClusterNodeCredentialIntents {
9881 password: bamboo_config::ClusterCredentialAction::Keep,
9882 private_key: bamboo_config::ClusterCredentialAction::Clear,
9883 passphrase: bamboo_config::ClusterCredentialAction::Clear,
9884 },
9885 )]),
9886 |config| {
9887 let node = config
9888 .cluster_fabric
9889 .node_mut("secret-node")
9890 .expect("secret node exists");
9891 let bamboo_config::NodePlacement::Ssh(target) = &mut node.placement else {
9892 panic!("expected SSH placement")
9893 };
9894 let bamboo_config::SshAuth::Password {
9895 password,
9896 password_encrypted,
9897 } = &mut target.auth
9898 else {
9899 panic!("expected password authentication")
9900 };
9901 password.clear();
9902 *password_encrypted = None;
9903 Ok(())
9904 },
9905 )
9906 .await
9907 .unwrap();
9908 assert_eq!(kept.section.revision, 1);
9909 assert_eq!(kept.credential_health.revision, 1);
9910 assert_eq!(password_from(&kept.config), "exact-password");
9911 assert_eq!(
9912 password_from(&*state.config.read().await),
9913 "exact-password",
9914 "semantic no-op must retain the exact credential snapshot"
9915 );
9916
9917 tokio::time::sleep(Duration::from_millis(500)).await;
9918 let noop_events = bamboo_engine::events::journal::read_since(
9919 state.account_sink.events_dir(),
9920 noop_baseline_seq,
9921 )
9922 .unwrap();
9923 assert!(!noop_events.iter().any(|event| {
9924 matches!(
9925 &event.event,
9926 AgentEvent::ConfigChanged { section, .. }
9927 | AgentEvent::ConfigInvalid { section, .. }
9928 | AgentEvent::ConfigRecovered { section, .. }
9929 if section == "cluster-fabric" || section == "credentials"
9930 )
9931 }));
9932 }
9933
9934 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
9935 async fn later_external_credential_winner_remains_observable_after_exact_cluster_commit() {
9936 let _key = bamboo_config::encryption::set_test_encryption_key([0x74; 32]);
9937 let dir = tempfile::tempdir().unwrap();
9938 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
9939 let baseline_seq = state.account_sink.latest_seq();
9940 let password_ref =
9941 bamboo_config::cluster_password_credential_ref("credential-race-node").unwrap();
9942 let external_ref = password_ref.clone();
9943 let (external_done_tx, external_done_rx) = std::sync::mpsc::sync_channel(1);
9944 set_cluster_after_commit_before_adoption_test_hook(dir.path(), 0, move |data_dir| {
9945 let data_dir = data_dir.to_path_buf();
9946 let (started_tx, started_rx) = std::sync::mpsc::sync_channel(0);
9947 std::thread::spawn(move || {
9948 started_tx.send(()).unwrap();
9949 let result = bamboo_config::CredentialStore::open(&data_dir).replace(
9950 external_ref,
9951 "later-external-password",
9952 bamboo_config::CredentialSource::User,
9953 1,
9954 );
9955 external_done_tx.send(result).unwrap();
9956 });
9957 started_rx
9958 .recv_timeout(Duration::from_secs(5))
9959 .expect("external credential writer must launch under the commit lock");
9960 });
9961
9962 let committed = state
9963 .update_cluster_fabric_credentials(
9964 0,
9965 BTreeMap::from([(
9966 "credential-race-node".to_string(),
9967 bamboo_config::ClusterNodeCredentialIntents {
9968 password: bamboo_config::ClusterCredentialAction::Replace(
9969 "exact-commit-password".to_string(),
9970 ),
9971 private_key: bamboo_config::ClusterCredentialAction::Clear,
9972 passphrase: bamboo_config::ClusterCredentialAction::Clear,
9973 },
9974 )]),
9975 |config| {
9976 config.cluster_fabric.nodes.push(bamboo_config::Node {
9977 id: "credential-race-node".to_string(),
9978 label: "credential-race-node".to_string(),
9979 placement: bamboo_config::NodePlacement::Ssh(bamboo_config::SshTarget {
9980 host: "race.example.test".to_string(),
9981 port: 22,
9982 username: "operator".to_string(),
9983 auth: bamboo_config::SshAuth::Password {
9984 password: String::new(),
9985 password_encrypted: None,
9986 },
9987 host_key_fingerprint: None,
9988 }),
9989 trust_level: bamboo_config::TrustLevel::Trusted,
9990 deploy: bamboo_config::DeployProfile::default(),
9991 state: None,
9992 enabled: true,
9993 });
9994 Ok(())
9995 },
9996 )
9997 .await
9998 .unwrap();
9999 let committed_password = match &committed
10000 .config
10001 .cluster_fabric
10002 .node("credential-race-node")
10003 .unwrap()
10004 .placement
10005 {
10006 bamboo_config::NodePlacement::Ssh(target) => match &target.auth {
10007 bamboo_config::SshAuth::Password { password, .. } => password,
10008 _ => panic!("expected password authentication"),
10009 },
10010 _ => panic!("expected SSH placement"),
10011 };
10012 assert_eq!(committed.section.revision, 1);
10013 assert_eq!(committed.credential_health.revision, 1);
10014 assert_eq!(committed_password, "exact-commit-password");
10015
10016 let external_revision = tokio::task::spawn_blocking(move || {
10017 external_done_rx
10018 .recv_timeout(Duration::from_secs(10))
10019 .expect("external credential writer must complete")
10020 .unwrap()
10021 .0
10022 })
10023 .await
10024 .unwrap();
10025 assert_eq!(external_revision, 2);
10026
10027 tokio::time::timeout(Duration::from_secs(5), async {
10028 loop {
10029 let facade_revision = state
10030 .config_facade
10031 .as_ref()
10032 .unwrap()
10033 .registry()
10034 .credentials
10035 .snapshot()
10036 .revision;
10037 let events = bamboo_engine::events::journal::read_since(
10038 state.account_sink.events_dir(),
10039 baseline_seq,
10040 )
10041 .unwrap();
10042 let saw_external_event = events.iter().any(|event| {
10043 matches!(
10044 &event.event,
10045 AgentEvent::ConfigChanged { section, revision }
10046 if section == "credentials" && *revision == 2
10047 )
10048 });
10049 if facade_revision == 2 && saw_external_event {
10050 break;
10051 }
10052 tokio::time::sleep(Duration::from_millis(20)).await;
10053 }
10054 })
10055 .await
10056 .expect("watcher must expose the later credential revision");
10057 tokio::time::sleep(Duration::from_millis(250)).await;
10058
10059 let runtime_password = match &state
10060 .config
10061 .read()
10062 .await
10063 .cluster_fabric
10064 .node("credential-race-node")
10065 .unwrap()
10066 .placement
10067 {
10068 bamboo_config::NodePlacement::Ssh(target) => match &target.auth {
10069 bamboo_config::SshAuth::Password { password, .. } => password.clone(),
10070 _ => panic!("expected password authentication"),
10071 },
10072 _ => panic!("expected SSH placement"),
10073 };
10074 assert_eq!(
10075 runtime_password, "exact-commit-password",
10076 "a status-only credential event must not rewrite the exact cluster runtime"
10077 );
10078 let credential_dir = dir.path().to_path_buf();
10079 let durable_password = tokio::task::spawn_blocking(move || {
10080 bamboo_config::CredentialStore::open(credential_dir)
10081 .resolve(&password_ref)
10082 .unwrap()
10083 .unwrap()
10084 .expose()
10085 .to_string()
10086 })
10087 .await
10088 .unwrap();
10089 assert_eq!(durable_password, "later-external-password");
10090
10091 let events = bamboo_engine::events::journal::read_since(
10092 state.account_sink.events_dir(),
10093 baseline_seq,
10094 )
10095 .unwrap();
10096 let relevant = events
10097 .iter()
10098 .filter_map(|event| match &event.event {
10099 AgentEvent::ConfigChanged { section, revision }
10100 if section == "cluster-fabric" || section == "credentials" =>
10101 {
10102 Some((section.as_str(), *revision))
10103 }
10104 _ => None,
10105 })
10106 .collect::<Vec<_>>();
10107 assert_eq!(
10108 relevant,
10109 vec![("cluster-fabric", 1), ("credentials", 2)],
10110 "the exact cluster event must precede the genuine later credential winner"
10111 );
10112 }
10113
10114 #[tokio::test]
10115 async fn changed_cluster_commit_publishes_secret_free_runtime_before_materialization_error() {
10116 let _key = bamboo_config::encryption::set_test_encryption_key([0x75; 32]);
10117 let dir = tempfile::tempdir().unwrap();
10118 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
10119 let password_ref =
10120 bamboo_config::cluster_password_credential_ref("corrupt-secret-node").unwrap();
10121 state
10122 .update_cluster_fabric_credentials(
10123 0,
10124 BTreeMap::from([(
10125 "corrupt-secret-node".to_string(),
10126 bamboo_config::ClusterNodeCredentialIntents {
10127 password: bamboo_config::ClusterCredentialAction::Replace(
10128 "initial-password".to_string(),
10129 ),
10130 private_key: bamboo_config::ClusterCredentialAction::Clear,
10131 passphrase: bamboo_config::ClusterCredentialAction::Clear,
10132 },
10133 )]),
10134 |config| {
10135 config.cluster_fabric.nodes.push(bamboo_config::Node {
10136 id: "corrupt-secret-node".to_string(),
10137 label: "before-corruption".to_string(),
10138 placement: bamboo_config::NodePlacement::Ssh(bamboo_config::SshTarget {
10139 host: "corrupt.example.test".to_string(),
10140 port: 22,
10141 username: "operator".to_string(),
10142 auth: bamboo_config::SshAuth::Password {
10143 password: String::new(),
10144 password_encrypted: None,
10145 },
10146 host_key_fingerprint: None,
10147 }),
10148 trust_level: bamboo_config::TrustLevel::Trusted,
10149 deploy: bamboo_config::DeployProfile::default(),
10150 state: None,
10151 enabled: true,
10152 });
10153 Ok(())
10154 },
10155 )
10156 .await
10157 .unwrap();
10158
10159 let credentials_path = dir.path().join("credentials.json");
10160 let mut document: Value =
10161 serde_json::from_slice(&std::fs::read(&credentials_path).unwrap()).unwrap();
10162 document["data"]["entries"][password_ref.as_str()]["ciphertext"] =
10163 Value::String("corrupt-ciphertext".to_string());
10164 std::fs::write(
10165 &credentials_path,
10166 serde_json::to_vec_pretty(&document).unwrap(),
10167 )
10168 .unwrap();
10169 tokio::time::sleep(Duration::from_millis(300)).await;
10170
10171 let noop_baseline_seq = state.account_sink.latest_seq();
10172 let noop = state
10173 .update_cluster_fabric_credentials(1, BTreeMap::new(), |_| Ok(()))
10174 .await;
10175 match noop {
10176 Err(AppError::InternalError(_)) => {}
10177 Err(error) => panic!("no-op materialization error was misclassified: {error}"),
10178 Ok(_) => panic!("corrupt credential unexpectedly materialized"),
10179 }
10180 let runtime = state.config.read().await;
10181 let node = runtime.cluster_fabric.node("corrupt-secret-node").unwrap();
10182 let bamboo_config::NodePlacement::Ssh(target) = &node.placement else {
10183 panic!("expected SSH placement")
10184 };
10185 let bamboo_config::SshAuth::Password { password, .. } = &target.auth else {
10186 panic!("expected password authentication")
10187 };
10188 assert_eq!(
10189 password, "initial-password",
10190 "a true no-op materialization failure must preserve the old runtime"
10191 );
10192 drop(runtime);
10193 assert_eq!(
10194 state
10195 .config_facade
10196 .as_ref()
10197 .unwrap()
10198 .registry()
10199 .cluster_fabric
10200 .snapshot()
10201 .revision,
10202 1
10203 );
10204 let noop_events = bamboo_engine::events::journal::read_since(
10205 state.account_sink.events_dir(),
10206 noop_baseline_seq,
10207 )
10208 .unwrap();
10209 assert!(!noop_events.iter().any(|event| {
10210 matches!(
10211 &event.event,
10212 AgentEvent::ConfigChanged { section, .. } if section == "cluster-fabric"
10213 )
10214 }));
10215
10216 let baseline_seq = state.account_sink.latest_seq();
10217 let result = state
10218 .update_cluster_fabric_credentials(1, BTreeMap::new(), |config| {
10219 config
10220 .cluster_fabric
10221 .node_mut("corrupt-secret-node")
10222 .unwrap()
10223 .label = "committed-metadata".to_string();
10224 Ok(())
10225 })
10226 .await;
10227 match result {
10228 Err(AppError::InternalError(_)) => {}
10229 Err(error) => panic!("post-commit materialization error was misclassified: {error}"),
10230 Ok(_) => panic!("corrupt credential unexpectedly materialized"),
10231 }
10232
10233 let runtime = state.config.read().await;
10234 let node = runtime.cluster_fabric.node("corrupt-secret-node").unwrap();
10235 assert_eq!(node.label, "committed-metadata");
10236 let bamboo_config::NodePlacement::Ssh(target) = &node.placement else {
10237 panic!("expected SSH placement")
10238 };
10239 let bamboo_config::SshAuth::Password {
10240 password,
10241 password_encrypted,
10242 } = &target.auth
10243 else {
10244 panic!("expected password authentication")
10245 };
10246 assert!(password.is_empty());
10247 assert!(password_encrypted.is_none());
10248 drop(runtime);
10249
10250 let section = state
10251 .config_facade
10252 .as_ref()
10253 .unwrap()
10254 .registry()
10255 .cluster_fabric
10256 .snapshot();
10257 assert_eq!(section.revision, 2);
10258 assert_eq!(
10259 section.data.0.node("corrupt-secret-node").unwrap().label,
10260 "committed-metadata"
10261 );
10262 let events = bamboo_engine::events::journal::read_since(
10263 state.account_sink.events_dir(),
10264 baseline_seq,
10265 )
10266 .unwrap();
10267 let cluster_revisions = events
10268 .iter()
10269 .filter_map(|event| match &event.event {
10270 AgentEvent::ConfigChanged { section, revision } if section == "cluster-fabric" => {
10271 Some(*revision)
10272 }
10273 _ => None,
10274 })
10275 .collect::<Vec<_>>();
10276 assert_eq!(cluster_revisions, vec![2]);
10277 }
10278
10279 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
10280 async fn cluster_commit_adopts_exact_revision_before_later_external_winner() {
10281 let _key = bamboo_config::encryption::set_test_encryption_key([0x72; 32]);
10282 let dir = tempfile::tempdir().unwrap();
10283 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
10284 let baseline_seq = state.account_sink.latest_seq();
10285 let (external_done_tx, external_done_rx) = std::sync::mpsc::sync_channel(1);
10286 set_cluster_after_commit_before_adoption_test_hook(dir.path(), 0, move |data_dir| {
10287 let data_dir = data_dir.to_path_buf();
10288 let (started_tx, started_rx) = std::sync::mpsc::sync_channel(0);
10289 std::thread::spawn(move || {
10290 started_tx.send(()).unwrap();
10291 let external = bamboo_config::ConfigFacade::open(&data_dir).unwrap();
10292 let mut winner = external.effective_config();
10293 winner.cluster_fabric.node_mut("race-node").unwrap().label =
10294 "external-winner".to_string();
10295 let result =
10296 bamboo_config::persist_cluster_fabric_credential_transaction_at_revision(
10297 &data_dir,
10298 &mut winner,
10299 &BTreeMap::new(),
10300 1,
10301 );
10302 external_done_tx.send(result).unwrap();
10303 });
10304 started_rx
10305 .recv_timeout(Duration::from_secs(5))
10306 .expect("external writer must launch after the durable commit");
10307 });
10308
10309 let committed = state
10310 .update_cluster_fabric_credentials(
10311 0,
10312 BTreeMap::from([(
10313 "race-node".to_string(),
10314 bamboo_config::ClusterNodeCredentialIntents::clear_all(),
10315 )]),
10316 |config| {
10317 config.cluster_fabric.nodes.push(bamboo_config::Node {
10318 id: "race-node".to_string(),
10319 label: "api-commit".to_string(),
10320 placement: bamboo_config::NodePlacement::Local,
10321 trust_level: bamboo_config::TrustLevel::Trusted,
10322 deploy: bamboo_config::DeployProfile::default(),
10323 state: None,
10324 enabled: true,
10325 });
10326 Ok(())
10327 },
10328 )
10329 .await
10330 .unwrap();
10331 assert_eq!(committed.section.revision, 1);
10332 assert_eq!(
10333 committed
10334 .config
10335 .cluster_fabric
10336 .node("race-node")
10337 .unwrap()
10338 .label,
10339 "api-commit",
10340 "the response must remain bound to its exact committed candidate"
10341 );
10342 assert_eq!(
10343 tokio::task::spawn_blocking(move || {
10344 external_done_rx
10345 .recv_timeout(Duration::from_secs(10))
10346 .expect("later external winner must complete")
10347 .unwrap()
10348 })
10349 .await
10350 .unwrap(),
10351 2
10352 );
10353
10354 tokio::time::timeout(Duration::from_secs(5), async {
10355 loop {
10356 let facade_revision = state
10357 .config_facade
10358 .as_ref()
10359 .unwrap()
10360 .registry()
10361 .cluster_fabric
10362 .snapshot()
10363 .revision;
10364 let runtime_label = state
10365 .config
10366 .read()
10367 .await
10368 .cluster_fabric
10369 .node("race-node")
10370 .map(|node| node.label.clone());
10371 if facade_revision == 2 && runtime_label.as_deref() == Some("external-winner") {
10372 break;
10373 }
10374 tokio::time::sleep(Duration::from_millis(20)).await;
10375 }
10376 })
10377 .await
10378 .expect("watcher must apply the later external revision");
10379
10380 let events = bamboo_engine::events::journal::read_since(
10381 state.account_sink.events_dir(),
10382 baseline_seq,
10383 )
10384 .unwrap();
10385 let revisions = events
10386 .iter()
10387 .filter_map(|event| match &event.event {
10388 AgentEvent::ConfigChanged { section, revision } if section == "cluster-fabric" => {
10389 Some(*revision)
10390 }
10391 _ => None,
10392 })
10393 .collect::<Vec<_>>();
10394 assert_eq!(
10395 revisions,
10396 vec![1, 2],
10397 "the exact API event must precede the later watcher winner exactly once"
10398 );
10399 assert!(!events.iter().any(|event| {
10400 matches!(
10401 &event.event,
10402 AgentEvent::ConfigChanged { section, .. } if section == "credentials"
10403 )
10404 }));
10405 }
10406
10407 async fn wait_for_facade_health(
10408 state: &AppState,
10409 id: SectionId,
10410 status: SectionStatus,
10411 revision: u64,
10412 ) -> bamboo_config::SectionHealth {
10413 tokio::time::timeout(Duration::from_secs(4), async {
10414 loop {
10415 let health = state
10416 .config_facade
10417 .as_ref()
10418 .expect("production state owns a facade")
10419 .registry()
10420 .health()
10421 .unwrap()
10422 .into_iter()
10423 .find(|health| health.section == id)
10424 .unwrap();
10425 if health.status == status && health.revision == revision {
10426 break health;
10427 }
10428 tokio::time::sleep(Duration::from_millis(20)).await;
10429 }
10430 })
10431 .await
10432 .expect("facade health transition timed out")
10433 }
10434
10435 #[test]
10436 fn initial_provider_health_validates_primary_and_backup() {
10437 let dir = tempfile::tempdir().unwrap();
10438 let store = AtomicJsonStore::new(dir.path().join("providers.json"), 1);
10439 let missing = initial_provider_health(&store);
10440 assert_eq!(missing.status, SectionStatus::Missing);
10441 assert_eq!(missing.source_kind, SectionSourceKind::Default);
10442
10443 std::fs::write(dir.path().join("providers.json"), b"{broken").unwrap();
10444 let invalid = initial_provider_health(&store);
10445 assert_eq!(invalid.status, SectionStatus::Invalid);
10446 assert_eq!(invalid.source_kind, SectionSourceKind::File);
10447
10448 std::fs::write(dir.path().join("providers.json.bak"), b"{}").unwrap();
10449 let recovered = initial_provider_health(&store);
10450 assert_eq!(recovered.status, SectionStatus::Degraded);
10451 assert_eq!(recovered.source_kind, SectionSourceKind::Backup);
10452 assert!(recovered
10453 .last_error
10454 .as_deref()
10455 .unwrap()
10456 .contains("last-known-good backup"));
10457
10458 std::fs::write(dir.path().join("providers.json"), b"{}").unwrap();
10459 let healthy = initial_provider_health(&store);
10460 assert_eq!(healthy.status, SectionStatus::Healthy);
10461 assert_eq!(healthy.source_kind, SectionSourceKind::File);
10462 }
10463
10464 #[tokio::test]
10465 async fn unrecoverable_pending_manifest_never_publishes_partial_provider_state() {
10466 let _key = bamboo_config::encryption::set_test_encryption_key([0x6c; 32]);
10467 let dir = tempfile::tempdir().unwrap();
10468 install_unrecoverable_pending_provider_migration(dir.path());
10469
10470 let loaded = Config::from_data_dir_without_env(Some(dir.path().to_path_buf()));
10471 assert_eq!(
10472 loaded.providers().openai.as_ref().unwrap().model.as_deref(),
10473 Some("root-lkg")
10474 );
10475 let store = AtomicJsonStore::new(dir.path().join("providers.json"), 1);
10476 let health = initial_provider_health(&store);
10477 assert_eq!(health.status, SectionStatus::Degraded);
10478 assert!(health
10479 .last_error
10480 .as_deref()
10481 .unwrap()
10482 .contains("migration is pending"));
10483 let error = match load_and_prepare_provider_candidate(&store, 0, loaded).await {
10484 Ok(_) => panic!("pending migration must reject provider candidate"),
10485 Err(error) => error,
10486 };
10487 assert!(error.message.contains("retaining last-known-good runtime"));
10488 assert!(!error.message.contains("partial-must-not-load"));
10489
10490 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
10491 assert_eq!(
10492 state
10493 .config
10494 .read()
10495 .await
10496 .providers()
10497 .openai
10498 .as_ref()
10499 .unwrap()
10500 .model
10501 .as_deref(),
10502 Some("root-lkg")
10503 );
10504 assert_eq!(
10505 state
10506 .config_live_health
10507 .read()
10508 .unwrap_or_else(|poisoned| poisoned.into_inner())
10509 .status,
10510 SectionStatus::Degraded
10511 );
10512 }
10513
10514 #[async_trait::async_trait]
10515 impl LLMProvider for WorkingProvider {
10516 async fn chat_stream(
10517 &self,
10518 _messages: &[Message],
10519 _tools: &[ToolSchema],
10520 _max_output_tokens: Option<u32>,
10521 _model: &str,
10522 ) -> Result<LLMStream, LLMError> {
10523 Err(LLMError::Api("working-provider-marker".to_string()))
10524 }
10525 }
10526
10527 #[tokio::test]
10528 async fn cancelled_provider_put_cannot_commit_before_publication_guards() {
10529 let _key = bamboo_config::encryption::set_test_encryption_key([0x53; 32]);
10530 let dir = tempfile::tempdir().unwrap();
10531 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
10532 let secret = "provider-cancel-secret";
10533 let reference = bamboo_config::credential_ref("provider", "openai", "api_key").unwrap();
10534 bamboo_config::CredentialStore::open(dir.path())
10535 .replace(
10536 reference.clone(),
10537 secret,
10538 bamboo_config::CredentialSource::User,
10539 0,
10540 )
10541 .unwrap();
10542 {
10543 let mut config = state.config.write().await;
10544 config.provider = "openai".to_string();
10545 *config.providers_mut() = ProviderConfigs {
10546 openai: Some(bamboo_config::OpenAIConfig {
10547 api_key: secret.to_string(),
10548 credential_ref: Some(reference),
10549 ..Default::default()
10550 }),
10551 ..Default::default()
10552 };
10553 }
10554 let provider_lock = state.provider.clone();
10555 let held_provider = provider_lock.write().await;
10556 let providers_before = std::fs::read(dir.path().join("providers.json")).unwrap();
10557 let mut operation = Box::pin(state.put_provider_section(
10558 0,
10559 ProviderConfigs {
10560 openai: Some(bamboo_config::OpenAIConfig {
10561 model: Some("candidate-model".to_string()),
10562 ..Default::default()
10563 }),
10564 ..Default::default()
10565 },
10566 ));
10567
10568 assert!(
10569 tokio::time::timeout(Duration::from_millis(500), &mut operation)
10570 .await
10571 .is_err()
10572 );
10573 drop(operation);
10574 assert_eq!(
10575 std::fs::read(dir.path().join("providers.json")).unwrap(),
10576 providers_before,
10577 "cancellation while waiting for publication guards must precede durable commit"
10578 );
10579 drop(held_provider);
10580 }
10581
10582 #[test]
10583 fn cancelled_provider_settings_request_finishes_exact_commit_and_live_publication() {
10584 let _key = bamboo_config::encryption::set_test_encryption_key([0x71; 32]);
10585 let runtime = tokio::runtime::Builder::new_multi_thread()
10586 .worker_threads(2)
10587 .max_blocking_threads(1)
10588 .enable_all()
10589 .build()
10590 .unwrap();
10591 runtime.block_on(async {
10592 let dir = tempfile::tempdir().unwrap();
10593 let state = Arc::new(AppState::new(dir.path().to_path_buf()).await.unwrap());
10594 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
10595
10596 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
10597 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
10598 let blocker = tokio::task::spawn_blocking(move || {
10599 let _ = started_tx.send(());
10600 release_rx.recv().unwrap();
10601 });
10602 started_rx.await.unwrap();
10603
10604 let operation_state = state.clone();
10605 let operation = tokio::spawn(async move {
10606 operation_state
10607 .put_provider_settings(0, |_current, candidate| {
10608 candidate.provider = "openai".to_string();
10609 candidate.providers_mut().openai = Some(bamboo_config::OpenAIConfig {
10610 api_key: "provider-settings-cancel-secret".to_string(),
10611 model: Some("provider-settings-cancel-model".to_string()),
10612 ..Default::default()
10613 });
10614 Ok((BTreeSet::from(["openai".to_string()]), BTreeSet::new()))
10615 })
10616 .await
10617 });
10618
10619 tokio::time::timeout(Duration::from_secs(1), async {
10620 loop {
10621 if state.config_io_lock.try_lock().is_err() {
10622 break;
10623 }
10624 assert!(!operation.is_finished());
10625 tokio::task::yield_now().await;
10626 }
10627 })
10628 .await
10629 .expect("provider settings mutation acquires the config IO lock");
10630 operation.abort();
10631 let _ = operation.await;
10632 release_tx.send(()).unwrap();
10633 blocker.await.unwrap();
10634
10635 tokio::time::timeout(Duration::from_secs(5), async {
10636 loop {
10637 let committed = std::fs::read(dir.path().join("providers.json"))
10638 .ok()
10639 .and_then(|bytes| serde_json::from_slice::<Value>(&bytes).ok())
10640 .is_some_and(|value| {
10641 value["revision"] == 1
10642 && value["data"]["openai"]["model"]
10643 == "provider-settings-cancel-model"
10644 });
10645 if committed {
10646 break;
10647 }
10648 tokio::task::yield_now().await;
10649 }
10650 })
10651 .await
10652 .expect("owned provider settings transaction completes after cancellation");
10653
10654 let converged =
10655 tokio::time::timeout(Duration::from_secs(5), state.config_io_lock.lock())
10656 .await
10657 .expect("owned provider runtime publication completes after cancellation");
10658 drop(converged);
10659 let live = state.config.read().await;
10660 let openai = live.providers().openai.as_ref().unwrap();
10661 assert_eq!(
10662 openai.model.as_deref(),
10663 Some("provider-settings-cancel-model")
10664 );
10665 assert_eq!(openai.api_key, "provider-settings-cancel-secret");
10666 drop(live);
10667 let providers = std::fs::read_to_string(dir.path().join("providers.json")).unwrap();
10668 let credentials = std::fs::read_to_string(dir.path().join("credentials.json")).unwrap();
10669 assert!(!providers.contains("provider-settings-cancel-secret"));
10670 assert!(!credentials.contains("provider-settings-cancel-secret"));
10671 });
10672 }
10673
10674 #[test]
10675 fn cancelled_proxy_update_cannot_leave_durable_state_ahead_of_live_snapshot() {
10676 let runtime = tokio::runtime::Builder::new_multi_thread()
10677 .worker_threads(2)
10678 .max_blocking_threads(1)
10679 .enable_all()
10680 .build()
10681 .unwrap();
10682 runtime.block_on(async {
10683 let dir = tempfile::tempdir().unwrap();
10684 let state = Arc::new(AppState::new(dir.path().to_path_buf()).await.unwrap());
10685 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
10686
10687 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
10691 let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
10692 let blocker = tokio::task::spawn_blocking(move || {
10693 let _ = started_tx.send(());
10694 release_rx.recv().unwrap();
10695 });
10696 started_rx.await.unwrap();
10697
10698 let operation_state = state.clone();
10699 let operation = tokio::spawn(async move {
10700 operation_state
10701 .update_proxy_auth_credential(
10702 Some(bamboo_config::ProxyAuth {
10703 username: "cancel-user".to_string(),
10704 password: "cancel-secret".to_string(),
10705 }),
10706 0,
10707 ConfigUpdateEffects {
10708 reload_provider: bamboo_config::patch::ReloadMode::Strict,
10709 reconcile_mcp: bamboo_config::patch::ReloadMode::Strict,
10710 },
10711 )
10712 .await
10713 });
10714
10715 tokio::time::timeout(Duration::from_secs(1), async {
10720 loop {
10721 if state.config_io_lock.try_lock().is_err() {
10722 break;
10723 }
10724 assert!(!operation.is_finished());
10725 tokio::task::yield_now().await;
10726 }
10727 })
10728 .await
10729 .expect("proxy mutation acquires the config IO lock");
10730 operation.abort();
10731 let _ = operation.await;
10732 release_tx.send(()).unwrap();
10733 blocker.await.unwrap();
10734
10735 tokio::time::timeout(Duration::from_secs(5), async {
10736 loop {
10737 let credentials_ready = std::fs::read(dir.path().join("credentials.json"))
10738 .ok()
10739 .and_then(|bytes| serde_json::from_slice::<serde_json::Value>(&bytes).ok())
10740 .and_then(|value| value.get("revision").and_then(|value| value.as_u64()))
10741 == Some(1);
10742 let config_ready = std::fs::read(dir.path().join("core.json"))
10743 .ok()
10744 .and_then(|bytes| serde_json::from_slice::<serde_json::Value>(&bytes).ok())
10745 .and_then(|value| {
10746 value
10747 .get("data")
10748 .and_then(|value| value.get("proxy_auth_credential_ref"))
10749 .and_then(|value| value.as_str())
10750 .map(str::to_string)
10751 })
10752 .as_deref()
10753 == Some("proxy.default.auth");
10754 if credentials_ready && config_ready {
10755 break;
10756 }
10757 tokio::task::yield_now().await;
10758 }
10759 })
10760 .await
10761 .expect("owned durable transaction completes after caller cancellation");
10762
10763 let converged =
10767 tokio::time::timeout(Duration::from_secs(5), state.config_io_lock.lock())
10768 .await
10769 .expect("owned runtime convergence completes after cancellation");
10770 drop(converged);
10771
10772 let live = state.config.read().await;
10773 assert_eq!(
10774 live.proxy_auth_credential_ref
10775 .as_ref()
10776 .map(|reference| reference.as_str()),
10777 Some("proxy.default.auth")
10778 );
10779 let auth = live
10780 .proxy_auth
10781 .as_ref()
10782 .expect("durable proxy auth must be published despite cancellation");
10783 assert_eq!(auth.username, "cancel-user");
10784 assert_eq!(auth.password, "cancel-secret");
10785 drop(live);
10786
10787 let root = std::fs::read_to_string(dir.path().join("core.json")).unwrap();
10788 let credentials = std::fs::read_to_string(dir.path().join("credentials.json")).unwrap();
10789 assert!(!root.contains("cancel-secret"));
10790 assert!(!credentials.contains("cancel-secret"));
10791 });
10792 }
10793
10794 #[tokio::test]
10795 async fn missing_or_corrupt_referenced_credentials_reject_candidates_redacted() {
10796 let _key = bamboo_config::encryption::set_test_encryption_key([0x68; 32]);
10797 for corrupt_credentials in [false, true] {
10798 let dir = tempfile::tempdir().unwrap();
10799 let reference = bamboo_config::credential_ref("provider", "openai", "api_key").unwrap();
10800 let providers = ProviderConfigs {
10801 openai: Some(bamboo_config::OpenAIConfig {
10802 credential_ref: Some(reference),
10803 model: Some("candidate".to_string()),
10804 ..Default::default()
10805 }),
10806 ..Default::default()
10807 };
10808 let provider_store = AtomicJsonStore::new(dir.path().join("providers.json"), 1);
10809 provider_store
10810 .commit(0, providers, validate_provider_config)
10811 .unwrap();
10812 if corrupt_credentials {
10813 std::fs::write(dir.path().join("credentials.json"), b"{corrupt-secret").unwrap();
10814 }
10815 let error =
10816 match load_and_prepare_provider_candidate(&provider_store, 0, Config::default())
10817 .await
10818 {
10819 Ok(_) => panic!("unavailable credential must reject provider candidate"),
10820 Err(error) => error,
10821 };
10822 assert_eq!(error.message, "provider credential is unavailable");
10823 assert!(!error
10824 .message
10825 .contains(dir.path().to_string_lossy().as_ref()));
10826
10827 let mut mcp = disabled_mcp_config("credential-lkg");
10828 let TransportConfig::Stdio(stdio) = &mut mcp.servers[0].transport else {
10829 unreachable!()
10830 };
10831 stdio.env_credential_refs.insert(
10832 "TOKEN".to_string(),
10833 bamboo_config::credential_ref("mcp", "credential-lkg", "env_TOKEN")
10834 .unwrap()
10835 .as_str()
10836 .to_string(),
10837 );
10838 let mcp_store = AtomicJsonStore::new(dir.path().join("mcp.json"), 1);
10839 mcp_store.commit(0, mcp, validate_mcp_config).unwrap();
10840 let error = match load_and_validate_mcp_candidate(
10841 &mcp_store,
10842 0,
10843 Config::default(),
10844 false,
10845 )
10846 .await
10847 {
10848 Ok(_) => panic!("unavailable credential must reject MCP candidate"),
10849 Err(error) => error,
10850 };
10851 assert_eq!(error.message, "MCP credential is unavailable");
10852 assert!(!error.message.contains("TOKEN"));
10853 assert!(!error
10854 .message
10855 .contains(dir.path().to_string_lossy().as_ref()));
10856 }
10857 }
10858
10859 #[tokio::test]
10860 async fn typed_provider_put_switches_refs_and_rejects_missing_ref_without_mutation() {
10861 let _key = bamboo_config::encryption::set_test_encryption_key([0x6d; 32]);
10862 let dir = tempfile::tempdir().unwrap();
10863 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
10864 let ref_a = bamboo_config::credential_ref("provider", "openai-a", "api_key").unwrap();
10865 let ref_b = bamboo_config::credential_ref("provider", "openai-b", "api_key").unwrap();
10866 let missing = bamboo_config::credential_ref("provider", "missing", "api_key").unwrap();
10867 state
10868 .credential_store
10869 .replace(
10870 ref_a.clone(),
10871 "provider-secret-a",
10872 bamboo_config::CredentialSource::User,
10873 0,
10874 )
10875 .unwrap();
10876 state
10877 .credential_store
10878 .replace(
10879 ref_b.clone(),
10880 "provider-secret-b",
10881 bamboo_config::CredentialSource::User,
10882 1,
10883 )
10884 .unwrap();
10885 {
10886 let mut config = state.config.write().await;
10887 config.provider = "openai".to_string();
10888 *config.providers_mut() = ProviderConfigs {
10889 openai: Some(bamboo_config::OpenAIConfig {
10890 api_key: "provider-secret-a".to_string(),
10891 credential_ref: Some(ref_a),
10892 ..Default::default()
10893 }),
10894 ..Default::default()
10895 };
10896 }
10897
10898 let revision = state
10899 .put_provider_section(
10900 0,
10901 ProviderConfigs {
10902 openai: Some(bamboo_config::OpenAIConfig {
10903 credential_ref: Some(ref_b.clone()),
10904 model: Some("switched".to_string()),
10905 ..Default::default()
10906 }),
10907 ..Default::default()
10908 },
10909 )
10910 .await
10911 .unwrap();
10912 assert_eq!(revision, 1);
10913 let runtime = state.config.read().await;
10914 let openai = runtime.providers().openai.as_ref().unwrap();
10915 assert_eq!(openai.credential_ref.as_ref(), Some(&ref_b));
10916 assert_eq!(openai.api_key, "provider-secret-b");
10917 drop(runtime);
10918 let disk_before = std::fs::read(dir.path().join("providers.json")).unwrap();
10919
10920 assert!(state
10921 .put_provider_section(
10922 1,
10923 ProviderConfigs {
10924 openai: Some(bamboo_config::OpenAIConfig {
10925 credential_ref: Some(missing),
10926 model: Some("must-not-publish".to_string()),
10927 ..Default::default()
10928 }),
10929 ..Default::default()
10930 },
10931 )
10932 .await
10933 .is_err());
10934 assert_eq!(
10935 std::fs::read(dir.path().join("providers.json")).unwrap(),
10936 disk_before
10937 );
10938 let runtime = state.config.read().await;
10939 let openai = runtime.providers().openai.as_ref().unwrap();
10940 assert_eq!(openai.credential_ref.as_ref(), Some(&ref_b));
10941 assert_eq!(openai.api_key, "provider-secret-b");
10942 }
10943
10944 #[tokio::test]
10945 async fn typed_mcp_put_switches_stdio_and_header_refs_atomically() {
10946 let _key = bamboo_config::encryption::set_test_encryption_key([0x6e; 32]);
10947 let dir = tempfile::tempdir().unwrap();
10948 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
10949 let refs = [
10950 bamboo_config::credential_ref("mcp", "stdio-a", "env_TOKEN").unwrap(),
10951 bamboo_config::credential_ref("mcp", "header-a", "header_Authorization").unwrap(),
10952 bamboo_config::credential_ref("mcp", "stdio-b", "env_TOKEN").unwrap(),
10953 bamboo_config::credential_ref("mcp", "header-b", "header_Authorization").unwrap(),
10954 ];
10955 for (revision, (reference, value)) in refs
10956 .iter()
10957 .zip(["env-a", "header-a", "env-b", "header-b"])
10958 .enumerate()
10959 {
10960 state
10961 .credential_store
10962 .replace(
10963 reference.clone(),
10964 value,
10965 bamboo_config::CredentialSource::User,
10966 revision as u64,
10967 )
10968 .unwrap();
10969 }
10970 let make_config = |env_ref: &bamboo_config::CredentialRef,
10971 header_ref: &bamboo_config::CredentialRef| {
10972 McpConfig {
10973 version: 1,
10974 servers: vec![
10975 McpServerConfig {
10976 id: "switch-stdio".to_string(),
10977 name: None,
10978 enabled: false,
10979 transport: TransportConfig::Stdio(StdioConfig {
10980 command: "unused-disabled-command".to_string(),
10981 args: vec![],
10982 cwd: None,
10983 env: std::collections::HashMap::new(),
10984 env_encrypted: std::collections::HashMap::new(),
10985 env_credential_refs: std::collections::HashMap::from([(
10986 "TOKEN".to_string(),
10987 env_ref.as_str().to_string(),
10988 )]),
10989 startup_timeout_ms: 100,
10990 }),
10991 request_timeout_ms: 100,
10992 healthcheck_interval_ms: 100,
10993 reconnect: ReconnectConfig::default(),
10994 allowed_tools: vec![],
10995 denied_tools: vec![],
10996 },
10997 McpServerConfig {
10998 id: "switch-header".to_string(),
10999 name: None,
11000 enabled: false,
11001 transport: TransportConfig::Sse(bamboo_mcp::SseConfig {
11002 url: "https://example.test/sse".to_string(),
11003 headers: vec![bamboo_mcp::HeaderConfig {
11004 name: "Authorization".to_string(),
11005 value: String::new(),
11006 value_encrypted: None,
11007 credential_ref: Some(header_ref.as_str().to_string()),
11008 }],
11009 connect_timeout_ms: 100,
11010 }),
11011 request_timeout_ms: 100,
11012 healthcheck_interval_ms: 100,
11013 reconnect: ReconnectConfig::default(),
11014 allowed_tools: vec![],
11015 denied_tools: vec![],
11016 },
11017 ],
11018 }
11019 };
11020 let mut current = make_config(&refs[0], &refs[1]);
11021 if let TransportConfig::Stdio(stdio) = &mut current.servers[0].transport {
11022 stdio.env.insert("TOKEN".to_string(), "env-a".to_string());
11023 }
11024 if let TransportConfig::Sse(sse) = &mut current.servers[1].transport {
11025 sse.headers[0].value = "header-a".to_string();
11026 }
11027 state.config.write().await.mcp = current;
11028
11029 assert_eq!(
11030 state
11031 .put_mcp_section(0, make_config(&refs[2], &refs[3]))
11032 .await
11033 .unwrap(),
11034 1
11035 );
11036 let runtime = state.config.read().await;
11037 let TransportConfig::Stdio(stdio) = &runtime
11038 .mcp
11039 .servers
11040 .iter()
11041 .find(|server| server.id == "switch-stdio")
11042 .expect("stdio server")
11043 .transport
11044 else {
11045 panic!("stdio transport")
11046 };
11047 assert_eq!(stdio.env["TOKEN"], "env-b");
11048 let TransportConfig::Sse(sse) = &runtime
11049 .mcp
11050 .servers
11051 .iter()
11052 .find(|server| server.id == "switch-header")
11053 .expect("SSE server")
11054 .transport
11055 else {
11056 panic!("sse transport")
11057 };
11058 assert_eq!(sse.headers[0].value, "header-b");
11059 drop(runtime);
11060 let disk_before = std::fs::read(dir.path().join("mcp.json")).unwrap();
11061 let missing_env = bamboo_config::credential_ref("mcp", "missing", "env_TOKEN").unwrap();
11062 let missing_header =
11063 bamboo_config::credential_ref("mcp", "missing", "header_Authorization").unwrap();
11064 assert!(state
11065 .put_mcp_section(1, make_config(&missing_env, &missing_header))
11066 .await
11067 .is_err());
11068 assert_eq!(
11069 std::fs::read(dir.path().join("mcp.json")).unwrap(),
11070 disk_before
11071 );
11072 let runtime = state.config.read().await;
11073 let TransportConfig::Stdio(stdio) = &runtime
11074 .mcp
11075 .servers
11076 .iter()
11077 .find(|server| server.id == "switch-stdio")
11078 .expect("stdio server")
11079 .transport
11080 else {
11081 panic!("stdio transport")
11082 };
11083 assert_eq!(stdio.env["TOKEN"], "env-b");
11084 }
11085
11086 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
11087 async fn stale_initial_mcp_batch_does_not_reapply_after_typed_revision_advances() {
11088 let dir = tempfile::tempdir().unwrap();
11089 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11090 let make_server = |id: &str, transport: TransportConfig| McpServerConfig {
11091 id: id.to_string(),
11092 name: None,
11093 enabled: false,
11094 transport,
11095 request_timeout_ms: 100,
11096 healthcheck_interval_ms: 100,
11097 reconnect: ReconnectConfig::default(),
11098 allowed_tools: vec![],
11099 denied_tools: vec![],
11100 };
11101 let initial = McpConfig {
11102 version: 1,
11103 servers: vec![make_server(
11104 "initial",
11105 TransportConfig::Stdio(StdioConfig {
11106 command: "unused-initial-command".to_string(),
11107 args: vec![],
11108 cwd: None,
11109 env: std::collections::HashMap::new(),
11110 env_encrypted: std::collections::HashMap::new(),
11111 env_credential_refs: std::collections::HashMap::new(),
11112 startup_timeout_ms: 100,
11113 }),
11114 )],
11115 };
11116 assert_eq!(state.put_mcp_section(0, initial).await.unwrap(), 1);
11117 stop_config_watcher(&mut state);
11118
11119 let (reached_tx, reached_rx) = std::sync::mpsc::channel();
11120 let (release_tx, release_rx) = std::sync::mpsc::channel();
11121 let (done_tx, done_rx) = std::sync::mpsc::channel();
11122 set_initial_mcp_apply_test_hook(
11123 dir.path(),
11124 move || {
11125 reached_tx.send(()).unwrap();
11126 release_rx.recv().unwrap();
11127 },
11128 move || {
11129 done_tx.send(()).unwrap();
11130 },
11131 );
11132 restart_config_watcher(&mut state);
11133 tokio::task::spawn_blocking(move || reached_rx.recv().unwrap())
11134 .await
11135 .unwrap();
11136
11137 let latest = McpConfig {
11142 version: 1,
11143 servers: vec![
11144 make_server(
11145 "z-stdio",
11146 TransportConfig::Stdio(StdioConfig {
11147 command: "unused-latest-command".to_string(),
11148 args: vec![],
11149 cwd: None,
11150 env: std::collections::HashMap::new(),
11151 env_encrypted: std::collections::HashMap::new(),
11152 env_credential_refs: std::collections::HashMap::new(),
11153 startup_timeout_ms: 100,
11154 }),
11155 ),
11156 make_server(
11157 "a-sse",
11158 TransportConfig::Sse(bamboo_mcp::SseConfig {
11159 url: "https://example.test/sse".to_string(),
11160 headers: vec![],
11161 connect_timeout_ms: 100,
11162 }),
11163 ),
11164 ],
11165 };
11166 let baseline = state.account_sink.latest_seq();
11167 assert_eq!(state.put_mcp_section(1, latest).await.unwrap(), 2);
11168 release_tx.send(()).unwrap();
11169 tokio::task::spawn_blocking(move || done_rx.recv().unwrap())
11170 .await
11171 .unwrap();
11172
11173 let runtime = state.config.read().await;
11174 assert_eq!(
11175 runtime
11176 .mcp
11177 .servers
11178 .iter()
11179 .map(|server| server.id.as_str())
11180 .collect::<Vec<_>>(),
11181 vec!["z-stdio", "a-sse"],
11182 "the superseded startup generation must not reapply"
11183 );
11184 drop(runtime);
11185 let mcp_events =
11186 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), baseline)
11187 .unwrap()
11188 .into_iter()
11189 .filter(|change| {
11190 matches!(
11191 &change.event,
11192 AgentEvent::ConfigChanged { section, revision }
11193 | AgentEvent::ConfigRecovered { section, revision }
11194 if section == "mcp" && *revision == 2
11195 )
11196 })
11197 .count();
11198 assert_eq!(
11199 mcp_events, 1,
11200 "the startup batch must not emit a pseudo event"
11201 );
11202 }
11203
11204 #[tokio::test]
11205 async fn failed_candidate_keeps_existing_provider_registry_and_runtime() {
11206 let dir = tempfile::tempdir().unwrap();
11207 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11208 let working: Arc<dyn LLMProvider> = Arc::new(WorkingProvider);
11209 state
11210 .provider_registry
11211 .insert("working".to_string(), working.clone());
11212 state.provider_registry.set_default("working".to_string());
11213 *state.provider.write().await = working.clone();
11214 state.config.write().await.provider = "openai".to_string();
11215
11216 assert!(state.reload_provider().await.is_err());
11217 assert_eq!(state.provider_registry.default_provider_name(), "working");
11218 assert!(Arc::ptr_eq(
11219 &state.provider_registry.get_default().unwrap(),
11220 &working
11221 ));
11222 let live = state.provider.read().await;
11223 assert!(Arc::ptr_eq(&*live, &working));
11224 }
11225
11226 #[tokio::test]
11227 async fn provider_watcher_retains_lkg_on_invalid_and_recovers_after_repair() {
11228 let _key = bamboo_config::encryption::set_test_encryption_key([0x43; 32]);
11229 let dir = tempfile::tempdir().unwrap();
11230 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11231 let working: Arc<dyn LLMProvider> = Arc::new(WorkingProvider);
11232 state
11233 .provider_registry
11234 .insert("working".to_string(), working.clone());
11235 state.provider_registry.set_default("working".to_string());
11236 *state.provider.write().await = working.clone();
11237 state.config.write().await.provider = "openai".to_string();
11238 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
11239 let mut feed = state.account_sink.subscribe();
11240 let providers_path = dir.path().join("providers.json");
11241
11242 std::fs::write(&providers_path, b"{broken").unwrap();
11243 tokio::time::timeout(Duration::from_secs(3), async {
11244 loop {
11245 if state
11246 .config_live_health
11247 .read()
11248 .unwrap_or_else(|poisoned| poisoned.into_inner())
11249 .status
11250 == SectionStatus::Invalid
11251 {
11252 break;
11253 }
11254 tokio::time::sleep(Duration::from_millis(20)).await;
11255 }
11256 })
11257 .await
11258 .unwrap();
11259 assert_eq!(
11260 state
11261 .config_live_health
11262 .read()
11263 .unwrap_or_else(|poisoned| poisoned.into_inner())
11264 .revision,
11265 0,
11266 "invalid edits must not advance the LKG revision"
11267 );
11268 {
11269 let health = state
11270 .config_live_health
11271 .read()
11272 .unwrap_or_else(|poisoned| poisoned.into_inner());
11273 assert_eq!(health.status, SectionStatus::Invalid);
11274 assert_eq!(health.source_kind, SectionSourceKind::File);
11275 assert_eq!(health.source_path, providers_path);
11276 }
11277 assert!(Arc::ptr_eq(
11278 &state.provider_registry.get_default().unwrap(),
11279 &working
11280 ));
11281 let invalid = next_config_event(&mut feed, "providers").await;
11282 assert!(matches!(
11283 invalid,
11284 AgentEvent::ConfigInvalid { revision: 0, .. }
11285 ));
11286
11287 let reference = bamboo_config::credential_ref("provider", "openai", "api_key").unwrap();
11288 let credential_store = bamboo_config::CredentialStore::open(dir.path());
11289 let credential_revision = credential_store.revision().unwrap();
11290 credential_store
11291 .replace(
11292 reference.clone(),
11293 "watcher-test-key",
11294 bamboo_config::CredentialSource::User,
11295 credential_revision,
11296 )
11297 .unwrap();
11298 let providers = ProviderConfigs {
11299 openai: Some(bamboo_config::OpenAIConfig {
11300 credential_ref: Some(reference),
11301 ..Default::default()
11302 }),
11303 ..Default::default()
11304 };
11305 std::fs::write(
11306 &providers_path,
11307 serde_json::to_vec_pretty(&providers).unwrap(),
11308 )
11309 .unwrap();
11310 tokio::time::timeout(Duration::from_secs(3), async {
11311 loop {
11312 let health = state
11313 .config_live_health
11314 .read()
11315 .unwrap_or_else(|poisoned| poisoned.into_inner())
11316 .clone();
11317 if health.status == SectionStatus::Healthy && health.revision == 1 {
11318 break;
11319 }
11320 tokio::time::sleep(Duration::from_millis(20)).await;
11321 }
11322 })
11323 .await
11324 .unwrap();
11325 let recovered = next_config_event(&mut feed, "providers").await;
11326 assert!(matches!(
11327 recovered,
11328 AgentEvent::ConfigRecovered { revision: 1, .. }
11329 ));
11330 assert_eq!(state.provider_registry.default_provider_name(), "openai");
11331 }
11332
11333 #[tokio::test]
11334 async fn ordinary_section_watcher_updates_runtime_retains_lkg_and_recovers() {
11335 let _key = bamboo_config::encryption::set_test_encryption_key([0x44; 32]);
11336 let dir = tempfile::tempdir().unwrap();
11337 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11338 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
11339 let mut feed = state.account_sink.subscribe();
11340 let path = dir.path().join("core.json");
11341 let mut document: serde_json::Value =
11342 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
11343 document["revision"] = serde_json::json!(2);
11344 document["data"]["server"]["port"] = serde_json::json!(9876);
11345 std::fs::write(&path, serde_json::to_vec_pretty(&document).unwrap()).unwrap();
11346
11347 wait_for_facade_health(&state, SectionId::Core, SectionStatus::Healthy, 2).await;
11348 tokio::time::timeout(Duration::from_secs(3), async {
11349 loop {
11350 if state.config.read().await.server.port == 9876 {
11351 break;
11352 }
11353 tokio::time::sleep(Duration::from_millis(20)).await;
11354 }
11355 })
11356 .await
11357 .unwrap();
11358 assert!(matches!(
11359 next_config_event(&mut feed, "core").await,
11360 AgentEvent::ConfigChanged { revision: 2, .. }
11361 ));
11362
11363 std::fs::write(&path, b"{broken").unwrap();
11364 wait_for_facade_health(&state, SectionId::Core, SectionStatus::Invalid, 2).await;
11365 assert_eq!(state.config.read().await.server.port, 9876);
11366 assert!(matches!(
11367 next_config_event(&mut feed, "core").await,
11368 AgentEvent::ConfigInvalid { revision: 2, .. }
11369 ));
11370
11371 document["revision"] = serde_json::json!(3);
11372 document["data"]["server"]["port"] = serde_json::json!(9877);
11373 std::fs::write(&path, serde_json::to_vec_pretty(&document).unwrap()).unwrap();
11374 wait_for_facade_health(&state, SectionId::Core, SectionStatus::Healthy, 3).await;
11375 tokio::time::timeout(Duration::from_secs(3), async {
11376 loop {
11377 if state.config.read().await.server.port == 9877 {
11378 break;
11379 }
11380 tokio::time::sleep(Duration::from_millis(20)).await;
11381 }
11382 })
11383 .await
11384 .unwrap();
11385 assert!(matches!(
11386 next_config_event(&mut feed, "core").await,
11387 AgentEvent::ConfigRecovered { revision: 3, .. }
11388 ));
11389
11390 std::fs::remove_file(&path).unwrap();
11391 wait_for_facade_health(&state, SectionId::Core, SectionStatus::Missing, 3).await;
11392 assert_eq!(state.config.read().await.server.port, 9877);
11393 assert!(matches!(
11394 next_config_event(&mut feed, "core").await,
11395 AgentEvent::ConfigInvalid { revision: 3, .. }
11396 ));
11397
11398 document["revision"] = serde_json::json!(4);
11399 document["data"]["server"]["port"] = serde_json::json!(9878);
11400 let swap = dir.path().join("core.json.swap");
11401 std::fs::write(&swap, serde_json::to_vec_pretty(&document).unwrap()).unwrap();
11402 std::fs::rename(&swap, &path).unwrap();
11403 wait_for_facade_health(&state, SectionId::Core, SectionStatus::Healthy, 4).await;
11404 tokio::time::timeout(Duration::from_secs(3), async {
11405 loop {
11406 if state.config.read().await.server.port == 9878 {
11407 break;
11408 }
11409 tokio::time::sleep(Duration::from_millis(20)).await;
11410 }
11411 })
11412 .await
11413 .unwrap();
11414 assert!(matches!(
11415 next_config_event(&mut feed, "core").await,
11416 AgentEvent::ConfigRecovered { revision: 4, .. }
11417 ));
11418 }
11419
11420 #[tokio::test]
11421 async fn mcp_watcher_updates_lkg_rejects_invalid_and_recovers_after_atomic_replace() {
11422 let _key = bamboo_config::encryption::set_test_encryption_key([0x45; 32]);
11423 let dir = tempfile::tempdir().unwrap();
11424 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11425 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
11426 let mut feed = state.account_sink.subscribe();
11427 let path = dir.path().join("mcp.json");
11428
11429 std::fs::write(&path, mcp_document_bytes(2, &disabled_mcp_config("first"))).unwrap();
11430 let first = wait_for_mcp_health(&state, SectionStatus::Healthy, 2).await;
11431 assert_eq!(first.revision, 2);
11432 assert_eq!(state.config.read().await.mcp.servers[0].id, "first");
11433 assert!(matches!(
11434 next_mcp_config_event(&mut feed).await,
11435 AgentEvent::ConfigChanged { revision: 2, .. }
11436 ));
11437
11438 std::fs::write(&path, b"{broken").unwrap();
11439 let invalid = wait_for_mcp_health(&state, SectionStatus::Invalid, 2).await;
11440 assert_eq!(invalid.revision, 2, "invalid candidates cannot advance LKG");
11441 assert_eq!(state.config.read().await.mcp.servers[0].id, "first");
11442 assert!(matches!(
11443 next_mcp_config_event(&mut feed).await,
11444 AgentEvent::ConfigInvalid { revision: 2, .. }
11445 ));
11446
11447 let swap = dir.path().join("mcp.json.swap");
11452 std::fs::write(
11453 &swap,
11454 mcp_document_bytes(3, &disabled_mcp_config("intermediate")),
11455 )
11456 .unwrap();
11457 std::fs::rename(&swap, &path).unwrap();
11458 std::fs::write(
11459 &path,
11460 mcp_document_bytes(3, &disabled_mcp_config("recovered")),
11461 )
11462 .unwrap();
11463 let recovered = wait_for_mcp_health(&state, SectionStatus::Healthy, 3).await;
11464 assert_eq!(recovered.revision, 3, "rename burst should coalesce once");
11465 assert_eq!(state.config.read().await.mcp.servers[0].id, "recovered");
11466 assert!(matches!(
11467 next_mcp_config_event(&mut feed).await,
11468 AgentEvent::ConfigRecovered { revision: 3, .. }
11469 ));
11470
11471 std::fs::write(
11476 &path,
11477 mcp_document_bytes(3, &disabled_mcp_config("normalized")),
11478 )
11479 .unwrap();
11480 let normalized = wait_for_mcp_health(&state, SectionStatus::Healthy, 4).await;
11481 assert_eq!(normalized.revision, 4);
11482 assert_eq!(state.config.read().await.mcp.servers[0].id, "normalized");
11483 assert!(matches!(
11484 next_mcp_config_event(&mut feed).await,
11485 AgentEvent::ConfigChanged { revision: 4, .. }
11486 ));
11487 assert!(
11488 tokio::time::timeout(Duration::from_millis(500), feed.recv())
11489 .await
11490 .is_err()
11491 );
11492 let persisted: serde_json::Value =
11493 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
11494 assert_eq!(persisted["revision"], 4);
11495 }
11496
11497 #[tokio::test]
11498 async fn mcp_sidecar_present_at_startup_is_applied_through_runtime_transaction() {
11499 let _key = bamboo_config::encryption::set_test_encryption_key([0x47; 32]);
11500 let dir = tempfile::tempdir().unwrap();
11501 std::fs::write(
11502 dir.path().join("mcp.json"),
11503 mcp_document_bytes(1, &disabled_mcp_config("startup-sidecar")),
11504 )
11505 .unwrap();
11506
11507 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11508 let health = wait_for_mcp_health(&state, SectionStatus::Healthy, 1).await;
11509 assert_eq!(health.revision, 1);
11510 assert_eq!(health.source_kind, SectionSourceKind::File);
11511 assert_eq!(
11512 state.config.read().await.mcp.servers[0].id,
11513 "startup-sidecar"
11514 );
11515 }
11516
11517 #[tokio::test]
11518 async fn mcp_startup_uses_valid_backup_and_reports_degraded_invalid_health() {
11519 let _key = bamboo_config::encryption::set_test_encryption_key([0x48; 32]);
11520 let dir = tempfile::tempdir().unwrap();
11521 let path = dir.path().join("mcp.json");
11522 let store = AtomicJsonStore::new(&path, 1);
11523 store
11524 .commit(0, disabled_mcp_config("backup-lkg"), validate_mcp_config)
11525 .unwrap();
11526 store
11527 .commit(1, disabled_mcp_config("new-primary"), validate_mcp_config)
11528 .unwrap();
11529 std::fs::write(&path, b"{corrupt-primary").unwrap();
11530
11531 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11532 let health = wait_for_mcp_health(&state, SectionStatus::Degraded, 1).await;
11533 assert_eq!(health.revision, 1);
11534 assert_eq!(health.source_kind, SectionSourceKind::Backup);
11535 assert_eq!(health.source_path, path.with_extension("json.bak"));
11536 assert!(health
11537 .last_error
11538 .as_deref()
11539 .unwrap()
11540 .contains("last-known-good backup runtime"));
11541 assert_eq!(state.config.read().await.mcp.servers[0].id, "backup-lkg");
11542
11543 tokio::time::timeout(Duration::from_secs(3), async {
11544 loop {
11545 if state.account_sink.latest_seq() > 0 {
11546 break;
11547 }
11548 tokio::time::sleep(Duration::from_millis(20)).await;
11549 }
11550 })
11551 .await
11552 .unwrap();
11553 tokio::time::sleep(Duration::from_millis(500)).await;
11554 let events =
11555 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
11556 assert_eq!(
11557 events
11558 .iter()
11559 .filter(|event| matches!(
11560 &event.event,
11561 AgentEvent::ConfigInvalid { section, revision }
11562 if section == "mcp" && *revision == 1
11563 ))
11564 .count(),
11565 1
11566 );
11567 let stable_health = state
11568 .mcp_config_live_health
11569 .read()
11570 .unwrap_or_else(|poisoned| poisoned.into_inner())
11571 .clone();
11572 assert_eq!(stable_health.status, SectionStatus::Degraded);
11573 assert_eq!(stable_health.source_kind, SectionSourceKind::Backup);
11574 assert_eq!(stable_health.source_path, path.with_extension("json.bak"));
11575 }
11576
11577 #[tokio::test]
11578 async fn mcp_runtime_init_failure_marks_degraded_and_retains_lkg_config() {
11579 let _key = bamboo_config::encryption::set_test_encryption_key([0x46; 32]);
11580 let dir = tempfile::tempdir().unwrap();
11581 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11582 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
11583 let mut feed = state.account_sink.subscribe();
11584 let path = dir.path().join("mcp.json");
11585
11586 std::fs::write(
11587 &path,
11588 mcp_document_bytes(1, &disabled_mcp_config("last-known-good")),
11589 )
11590 .unwrap();
11591 wait_for_mcp_health(&state, SectionStatus::Healthy, 1).await;
11592 let _ = next_mcp_config_event(&mut feed).await;
11593
11594 let mut failing = disabled_mcp_config("candidate");
11595 failing.servers[0].enabled = true;
11596 if let TransportConfig::Stdio(stdio) = &mut failing.servers[0].transport {
11597 stdio.command = "definitely-not-a-real-mcp-command-597".to_string();
11598 }
11599 std::fs::write(&path, mcp_document_bytes(2, &failing)).unwrap();
11600
11601 let degraded = wait_for_mcp_health(&state, SectionStatus::Degraded, 1).await;
11602 assert_eq!(degraded.revision, 1);
11603 assert!(degraded
11604 .last_error
11605 .as_deref()
11606 .unwrap()
11607 .contains("last-known-good runtime"));
11608 assert_eq!(
11609 state.config.read().await.mcp.servers[0].id,
11610 "last-known-good"
11611 );
11612 assert!(state.mcp_manager.list_servers().is_empty());
11613 assert!(matches!(
11614 next_mcp_config_event(&mut feed).await,
11615 AgentEvent::ConfigInvalid { revision: 1, .. }
11616 ));
11617 }
11618
11619 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
11620 async fn stopped_root_commit_is_installed_before_one_confirmed_event_on_same_facade_restart() {
11621 let _key = bamboo_config::encryption::set_test_encryption_key([0x52; 32]);
11622 let dir = tempfile::tempdir().unwrap();
11623 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11624 stop_config_watcher(&mut state);
11625 std::fs::write(
11626 dir.path().join("config.json"),
11627 br#"{"server":{"port":25201}}"#,
11628 )
11629 .unwrap();
11630 let committed = state
11631 .config_facade
11632 .as_ref()
11633 .unwrap()
11634 .reconcile_reappeared_legacy_root()
11635 .unwrap()
11636 .unwrap();
11637 assert_eq!(committed.committed.len(), 1);
11638 assert_ne!(state.config.read().await.server.port, 25_201);
11639
11640 let mut feed = state.account_sink.subscribe();
11641 let config = state.config.clone();
11642 let read_guard = config.read().await;
11643 restart_config_watcher(&mut state);
11644 assert!(
11645 tokio::time::timeout(
11646 Duration::from_millis(350),
11647 next_config_event(&mut feed, "core")
11648 )
11649 .await
11650 .is_err(),
11651 "the account event must wait for the runtime write"
11652 );
11653 drop(read_guard);
11654
11655 assert!(matches!(
11656 next_config_event(&mut feed, "core").await,
11657 AgentEvent::ConfigChanged { revision: 1, .. }
11658 ));
11659 assert_eq!(state.config.read().await.server.port, 25_201);
11660 wait_for_root_outbox_to_clear(dir.path()).await;
11661 tokio::time::sleep(Duration::from_millis(250)).await;
11662 let events =
11663 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
11664 assert_eq!(
11665 events
11666 .iter()
11667 .filter(|event| matches!(
11668 &event.event,
11669 AgentEvent::ConfigChanged { section, revision }
11670 if section == "core" && *revision == 1
11671 ))
11672 .count(),
11673 1
11674 );
11675
11676 stop_config_watcher(&mut state);
11677 restart_config_watcher(&mut state);
11678 assert!(tokio::time::timeout(
11679 Duration::from_millis(500),
11680 next_config_event(&mut feed, "core")
11681 )
11682 .await
11683 .is_err());
11684 }
11685
11686 #[tokio::test]
11687 async fn pending_root_resolver_unavailable_keeps_lkg_silent_and_requests_retry() {
11688 let _key = bamboo_config::encryption::set_test_encryption_key([0x59; 32]);
11689 let dir = tempfile::tempdir().unwrap();
11690 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11691 stop_config_watcher(&mut state);
11692 let old_port = state.config.read().await.server.port;
11693 std::fs::write(
11694 dir.path().join("config.json"),
11695 br#"{"server":{"port":25901}}"#,
11696 )
11697 .unwrap();
11698 let committed = state
11699 .config_facade
11700 .as_ref()
11701 .unwrap()
11702 .reconcile_reappeared_legacy_root()
11703 .unwrap()
11704 .unwrap();
11705 let event = committed.committed[0].clone();
11706 let mut synthetic_events = BTreeMap::from([(SectionId::Core, event.clone())]);
11707 let mut pending_root_publications = BTreeMap::from([(SectionId::Core, event)]);
11708 let mut reported_root_runtime_failures = BTreeSet::new();
11709 let mut feed = state.account_sink.subscribe();
11710 std::fs::remove_file(dir.path().join("config-section-layout-completion.json")).unwrap();
11711
11712 let retry = reload_and_apply_ordinary_sections(
11713 dir.path(),
11714 &state.config,
11715 state.config_facade.as_ref().unwrap(),
11716 &state.account_sink,
11717 std::iter::once(SectionId::Core),
11718 OrdinarySectionReloadState {
11719 synthetic_events: &mut synthetic_events,
11720 pending_root_publications: &mut pending_root_publications,
11721 reported_root_runtime_failures: &mut reported_root_runtime_failures,
11722 },
11723 )
11724 .await;
11725
11726 assert!(retry);
11727 assert_eq!(state.config.read().await.server.port, old_port);
11728 assert!(
11729 tokio::time::timeout(Duration::from_millis(250), feed.recv())
11730 .await
11731 .is_err()
11732 );
11733 assert!(pending_root_publications.contains_key(&SectionId::Core));
11734 }
11735
11736 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
11737 async fn pending_root_then_new_root_survives_coalesced_mcp_noop_and_requeues() {
11738 let _key = bamboo_config::encryption::set_test_encryption_key([0x53; 32]);
11739 let dir = tempfile::tempdir().unwrap();
11740 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11741 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
11742 let mut feed = state.account_sink.subscribe();
11743 let io = state.config_io_lock.clone().lock_owned().await;
11744 std::fs::write(
11745 dir.path().join("config.json"),
11746 br#"{"server":{"port":25301}}"#,
11747 )
11748 .unwrap();
11749 let first = state
11750 .config_facade
11751 .as_ref()
11752 .unwrap()
11753 .reconcile_reappeared_legacy_root()
11754 .unwrap()
11755 .unwrap();
11756 assert_eq!(first.committed.len(), 1);
11757 std::fs::write(
11758 dir.path().join("config.json"),
11759 br#"{"server":{"port":25302}}"#,
11760 )
11761 .unwrap();
11762 let mcp_bytes = std::fs::read(dir.path().join("mcp.json")).unwrap();
11763 std::fs::write(dir.path().join("mcp.json"), mcp_bytes).unwrap();
11764 tokio::time::sleep(Duration::from_millis(300)).await;
11765 drop(io);
11766
11767 let first_event = next_config_event(&mut feed, "core").await;
11768 let second_event = next_config_event(&mut feed, "core").await;
11769 assert!(matches!(
11770 first_event,
11771 AgentEvent::ConfigChanged { revision: 1, .. }
11772 ));
11773 assert!(matches!(
11774 second_event,
11775 AgentEvent::ConfigChanged { revision: 2, .. }
11776 ));
11777 tokio::time::timeout(Duration::from_secs(5), async {
11778 loop {
11779 if state.config.read().await.server.port == 25_302 {
11780 break;
11781 }
11782 tokio::time::sleep(Duration::from_millis(20)).await;
11783 }
11784 })
11785 .await
11786 .unwrap();
11787 wait_for_root_outbox_to_clear(dir.path()).await;
11788 }
11789
11790 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
11791 async fn startup_handoff_always_catches_root_generation_written_before_watcher_registration() {
11792 let _key = bamboo_config::encryption::set_test_encryption_key([0x56; 32]);
11793 let dir = tempfile::tempdir().unwrap();
11794 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11795 stop_config_watcher(&mut state);
11796 std::fs::write(
11797 dir.path().join("config.json"),
11798 br#"{"server":{"port":25601}}"#,
11799 )
11800 .unwrap();
11801 let startup_facade =
11802 Arc::new(bamboo_config::ConfigFacade::open_or_migrate(dir.path()).unwrap());
11803 assert!(bamboo_config::has_pending_legacy_root_publications(dir.path()).unwrap());
11804 state.config_facade = Some(startup_facade);
11805 std::fs::write(
11806 dir.path().join("config.json"),
11807 br#"{"server":{"port":25602}}"#,
11808 )
11809 .unwrap();
11810 let mut feed = state.account_sink.subscribe();
11811
11812 restart_config_watcher(&mut state);
11813
11814 assert!(matches!(
11815 next_config_event(&mut feed, "core").await,
11816 AgentEvent::ConfigChanged { revision: 1, .. }
11817 ));
11818 assert!(matches!(
11819 next_config_event(&mut feed, "core").await,
11820 AgentEvent::ConfigChanged { revision: 2, .. }
11821 ));
11822 tokio::time::timeout(Duration::from_secs(5), async {
11823 loop {
11824 if state.config.read().await.server.port == 25_602 {
11825 break;
11826 }
11827 tokio::time::sleep(Duration::from_millis(20)).await;
11828 }
11829 })
11830 .await
11831 .unwrap();
11832 wait_for_root_outbox_to_clear(dir.path()).await;
11833 }
11834
11835 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
11836 async fn same_facade_restart_replays_one_rejection_and_one_lost_recovery() {
11837 let _key = bamboo_config::encryption::set_test_encryption_key([0x54; 32]);
11838 let dir = tempfile::tempdir().unwrap();
11839 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11840 stop_config_watcher(&mut state);
11841 std::fs::write(
11842 dir.path().join("config.json"),
11843 br#"{"server":{"vendor_api_key":"never-persist-this"}}"#,
11844 )
11845 .unwrap();
11846 let rejected = state
11847 .config_facade
11848 .as_ref()
11849 .unwrap()
11850 .reconcile_reappeared_legacy_root()
11851 .unwrap()
11852 .unwrap();
11853 assert_eq!(rejected.rejected.len(), 1);
11854 assert_eq!(
11855 state
11856 .config_facade
11857 .as_ref()
11858 .unwrap()
11859 .registry()
11860 .core
11861 .snapshot()
11862 .status,
11863 SectionStatus::Healthy
11864 );
11865 let mut feed = state.account_sink.subscribe();
11866 restart_config_watcher(&mut state);
11867 assert!(matches!(
11868 next_config_event(&mut feed, "core").await,
11869 AgentEvent::ConfigInvalid { revision: 0, .. }
11870 ));
11871 stop_config_watcher(&mut state);
11872
11873 std::fs::write(dir.path().join("config.json"), b"{}").unwrap();
11874 let recovered = state
11875 .config_facade
11876 .as_ref()
11877 .unwrap()
11878 .reconcile_reappeared_legacy_root()
11879 .unwrap()
11880 .unwrap();
11881 assert_eq!(recovered.recovered, vec![SectionId::Core]);
11882 assert_eq!(
11883 state
11884 .config_facade
11885 .as_ref()
11886 .unwrap()
11887 .registry()
11888 .core
11889 .snapshot()
11890 .status,
11891 SectionStatus::Degraded
11892 );
11893 restart_config_watcher(&mut state);
11894 assert!(matches!(
11895 next_config_event(&mut feed, "core").await,
11896 AgentEvent::ConfigRecovered { revision: 0, .. }
11897 ));
11898 assert_eq!(
11899 state
11900 .config_facade
11901 .as_ref()
11902 .unwrap()
11903 .registry()
11904 .core
11905 .snapshot()
11906 .status,
11907 SectionStatus::Healthy
11908 );
11909
11910 stop_config_watcher(&mut state);
11911 restart_config_watcher(&mut state);
11912 assert!(tokio::time::timeout(
11913 Duration::from_millis(500),
11914 next_config_event(&mut feed, "core")
11915 )
11916 .await
11917 .is_err());
11918 let events =
11919 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
11920 assert_eq!(
11921 events
11922 .iter()
11923 .filter(|event| matches!(
11924 &event.event,
11925 AgentEvent::ConfigInvalid { section, .. } if section == "core"
11926 ))
11927 .count(),
11928 1
11929 );
11930 assert_eq!(
11931 events
11932 .iter()
11933 .filter(|event| matches!(
11934 &event.event,
11935 AgentEvent::ConfigRecovered { section, .. } if section == "core"
11936 ))
11937 .count(),
11938 1
11939 );
11940 }
11941
11942 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
11943 async fn degraded_root_mcp_is_carried_while_new_root_core_commits_then_recovers() {
11944 let _key = bamboo_config::encryption::set_test_encryption_key([0x57; 32]);
11945 let dir = tempfile::tempdir().unwrap();
11946 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
11947 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
11948 let failing = working_stdio_mcp_config(dir.path(), "root-carry", None);
11949 let script = match &failing.servers[0].transport {
11950 TransportConfig::Stdio(stdio) => PathBuf::from(&stdio.args[0]),
11951 _ => unreachable!(),
11952 };
11953 std::fs::remove_file(&script).unwrap();
11954 let mut feed = state.account_sink.subscribe();
11955 std::fs::write(
11956 dir.path().join("config.json"),
11957 serde_json::to_vec(&serde_json::json!({"mcp": failing})).unwrap(),
11958 )
11959 .unwrap();
11960 assert!(matches!(
11961 next_mcp_config_event(&mut feed).await,
11962 AgentEvent::ConfigInvalid { revision: 1, .. }
11963 ));
11964
11965 std::fs::write(
11966 dir.path().join("config.json"),
11967 br#"{"server":{"port":25701}}"#,
11968 )
11969 .unwrap();
11970 assert!(matches!(
11971 next_config_event(&mut feed, "core").await,
11972 AgentEvent::ConfigChanged { revision: 1, .. }
11973 ));
11974 tokio::time::timeout(Duration::from_secs(5), async {
11975 loop {
11976 if state.config.read().await.server.port == 25_701 {
11977 break;
11978 }
11979 tokio::time::sleep(Duration::from_millis(20)).await;
11980 }
11981 })
11982 .await
11983 .unwrap();
11984 assert!(bamboo_config::has_pending_legacy_root_publications(dir.path()).unwrap());
11985
11986 let repaired = working_stdio_mcp_config(dir.path(), "root-carry", None);
11987 assert_eq!(repaired.servers[0].id, "root-carry");
11988 assert!(matches!(
11989 next_mcp_config_event(&mut feed).await,
11990 AgentEvent::ConfigRecovered { revision: 1, .. }
11991 ));
11992 wait_for_root_outbox_to_clear(dir.path()).await;
11993 let events =
11994 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
11995 assert_eq!(
11996 events
11997 .iter()
11998 .filter(|event| matches!(
11999 &event.event,
12000 AgentEvent::ConfigInvalid { section, revision }
12001 if section == "mcp" && *revision == 1
12002 ))
12003 .count(),
12004 1
12005 );
12006 assert_eq!(
12007 events
12008 .iter()
12009 .filter(|event| matches!(
12010 &event.event,
12011 AgentEvent::ConfigRecovered { section, revision }
12012 if section == "mcp" && *revision == 1
12013 ))
12014 .count(),
12015 1
12016 );
12017 }
12018
12019 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
12020 async fn rejected_new_mcp_keeps_old_degraded_publication_dormant_until_clean_root() {
12021 let _key = bamboo_config::encryption::set_test_encryption_key([0x5b; 32]);
12022 let dir = tempfile::tempdir().unwrap();
12023 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
12024 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
12025 let failing = working_stdio_mcp_config(dir.path(), "root-dormant", None);
12026 let script = match &failing.servers[0].transport {
12027 TransportConfig::Stdio(stdio) => PathBuf::from(&stdio.args[0]),
12028 _ => unreachable!(),
12029 };
12030 std::fs::remove_file(&script).unwrap();
12031 let mut feed = state.account_sink.subscribe();
12032 std::fs::write(
12033 dir.path().join("config.json"),
12034 serde_json::to_vec(&serde_json::json!({"mcp": failing})).unwrap(),
12035 )
12036 .unwrap();
12037 assert!(matches!(
12038 next_mcp_config_event(&mut feed).await,
12039 AgentEvent::ConfigInvalid { revision: 1, .. }
12040 ));
12041
12042 std::fs::write(
12043 dir.path().join("config.json"),
12044 serde_json::to_vec(&serde_json::json!({
12045 "server": {"port": 25801},
12046 "mcpServers": {
12047 "rejected-next": {
12048 "command": "unused-rejected-command",
12049 "disabled": true,
12050 "access_token_value": "must-not-cross"
12051 }
12052 }
12053 }))
12054 .unwrap(),
12055 )
12056 .unwrap();
12057 assert!(matches!(
12058 next_config_event(&mut feed, "core").await,
12059 AgentEvent::ConfigChanged { revision: 1, .. }
12060 ));
12061 assert_eq!(state.config.read().await.server.port, 25_801);
12062 tokio::time::sleep(Duration::from_millis(500)).await;
12063 let dormant_events =
12064 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
12065 assert!(!dormant_events.iter().any(|event| matches!(
12066 &event.event,
12067 AgentEvent::ConfigRecovered { section, revision }
12068 if section == "mcp" && *revision == 1
12069 )));
12070 assert!(bamboo_config::has_pending_legacy_root_publications(dir.path()).unwrap());
12071
12072 std::fs::write(dir.path().join("config.json"), b"{}").unwrap();
12073 tokio::time::timeout(Duration::from_secs(5), async {
12074 loop {
12075 if !bamboo_config::legacy_root_rejected_sections(dir.path())
12076 .unwrap()
12077 .contains(&SectionId::Mcp)
12078 {
12079 break;
12080 }
12081 tokio::time::sleep(Duration::from_millis(20)).await;
12082 }
12083 })
12084 .await
12085 .unwrap();
12086 let repaired = working_stdio_mcp_config(dir.path(), "root-dormant", None);
12087 assert_eq!(repaired.servers[0].id, "root-dormant");
12088 assert!(matches!(
12089 next_mcp_config_event(&mut feed).await,
12090 AgentEvent::ConfigRecovered { revision: 1, .. }
12091 ));
12092 wait_for_root_outbox_to_clear(dir.path()).await;
12093 }
12094
12095 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
12096 async fn changed_before_ack_then_runtime_failure_recovers_with_exact_kind() {
12097 let _key = bamboo_config::encryption::set_test_encryption_key([0x58; 32]);
12098 let dir = tempfile::tempdir().unwrap();
12099 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
12100 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
12101 stop_config_watcher(&mut state);
12102 let failing = working_stdio_mcp_config(dir.path(), "root-crash-window", None);
12103 let script = match &failing.servers[0].transport {
12104 TransportConfig::Stdio(stdio) => PathBuf::from(&stdio.args[0]),
12105 _ => unreachable!(),
12106 };
12107 std::fs::remove_file(&script).unwrap();
12108 std::fs::write(
12109 dir.path().join("config.json"),
12110 serde_json::to_vec(&serde_json::json!({"mcp": failing})).unwrap(),
12111 )
12112 .unwrap();
12113 let committed = state
12114 .config_facade
12115 .as_ref()
12116 .unwrap()
12117 .reconcile_reappeared_legacy_root()
12118 .unwrap()
12119 .unwrap();
12120 assert_eq!(
12121 committed.committed,
12122 vec![ConfigSectionEvent::Changed {
12123 section: "mcp".to_string(),
12124 revision: 1,
12125 }]
12126 );
12127 assert!(
12128 state
12129 .account_sink
12130 .record_confirmed(None, ®istry_agent_event(&committed.committed[0]))
12131 .await
12132 );
12133 assert!(bamboo_config::has_pending_legacy_root_publications(dir.path()).unwrap());
12134
12135 let events_dir = state.account_sink.events_dir().to_path_buf();
12136 state.account_sink = bamboo_engine::events::AccountEventSink::new(events_dir).unwrap();
12137 tokio::task::yield_now().await;
12138 let mut feed = state.account_sink.subscribe();
12139 restart_config_watcher(&mut state);
12140 assert!(matches!(
12141 next_mcp_config_event(&mut feed).await,
12142 AgentEvent::ConfigInvalid { revision: 1, .. }
12143 ));
12144 let repaired = working_stdio_mcp_config(dir.path(), "root-crash-window", None);
12145 assert_eq!(repaired.servers[0].id, "root-crash-window");
12146 assert!(matches!(
12147 next_mcp_config_event(&mut feed).await,
12148 AgentEvent::ConfigRecovered { revision: 1, .. }
12149 ));
12150 wait_for_root_outbox_to_clear(dir.path()).await;
12151
12152 let events =
12153 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
12154 let transitions = events
12155 .iter()
12156 .filter_map(|event| match &event.event {
12157 AgentEvent::ConfigChanged { section, revision } if section == "mcp" => {
12158 Some(("changed", *revision))
12159 }
12160 AgentEvent::ConfigInvalid { section, revision } if section == "mcp" => {
12161 Some(("invalid", *revision))
12162 }
12163 AgentEvent::ConfigRecovered { section, revision } if section == "mcp" => {
12164 Some(("recovered", *revision))
12165 }
12166 _ => None,
12167 })
12168 .collect::<Vec<_>>();
12169 assert_eq!(
12170 transitions,
12171 vec![("changed", 1), ("invalid", 1), ("recovered", 1)]
12172 );
12173 }
12174
12175 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
12176 async fn invalid_journaled_before_canonical_mark_restarts_as_recovered_only() {
12177 let _key = bamboo_config::encryption::set_test_encryption_key([0x5b; 32]);
12178 let dir = tempfile::tempdir().unwrap();
12179 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
12180 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
12181 stop_config_watcher(&mut state);
12182 let candidate = disabled_mcp_config("root-invalid-mark-crash");
12183 std::fs::write(
12184 dir.path().join("config.json"),
12185 serde_json::to_vec(&serde_json::json!({"mcp": candidate})).unwrap(),
12186 )
12187 .unwrap();
12188 let committed = state
12189 .config_facade
12190 .as_ref()
12191 .unwrap()
12192 .reconcile_reappeared_legacy_root()
12193 .unwrap()
12194 .unwrap();
12195 assert_eq!(
12196 committed.committed,
12197 vec![ConfigSectionEvent::Changed {
12198 section: "mcp".to_string(),
12199 revision: 1,
12200 }]
12201 );
12202 let invalid = ConfigSectionEvent::Invalid {
12203 section: "mcp".to_string(),
12204 revision: 1,
12205 };
12206 assert!(
12207 state
12208 .account_sink
12209 .record_confirmed(None, ®istry_agent_event(&invalid))
12210 .await
12211 );
12212 let envelope = state
12213 .config_facade
12214 .as_ref()
12215 .unwrap()
12216 .registry()
12217 .envelope_value(SectionId::Mcp)
12218 .unwrap();
12219 assert!(matches!(
12220 bamboo_config::legacy_root_publication_success_event(
12221 dir.path(),
12222 &committed.committed[0],
12223 &envelope.data,
12224 )
12225 .unwrap(),
12226 Some(ConfigSectionEvent::Changed { revision: 1, .. })
12227 ));
12228
12229 let events_dir = state.account_sink.events_dir().to_path_buf();
12230 state.account_sink = bamboo_engine::events::AccountEventSink::new(events_dir).unwrap();
12231 assert!(state
12232 .account_sink
12233 .latest_config_transition_is_invalid("mcp", 1));
12234 let mut feed = state.account_sink.subscribe();
12235 restart_config_watcher(&mut state);
12236 assert!(matches!(
12237 next_mcp_config_event(&mut feed).await,
12238 AgentEvent::ConfigRecovered { revision: 1, .. }
12239 ));
12240 wait_for_root_outbox_to_clear(dir.path()).await;
12241
12242 let events =
12243 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
12244 let transitions = events
12245 .iter()
12246 .filter_map(|event| match &event.event {
12247 AgentEvent::ConfigChanged { section, revision } if section == "mcp" => {
12248 Some(("changed", *revision))
12249 }
12250 AgentEvent::ConfigInvalid { section, revision } if section == "mcp" => {
12251 Some(("invalid", *revision))
12252 }
12253 AgentEvent::ConfigRecovered { section, revision } if section == "mcp" => {
12254 Some(("recovered", *revision))
12255 }
12256 _ => None,
12257 })
12258 .collect::<Vec<_>>();
12259 assert_eq!(transitions, vec![("invalid", 1), ("recovered", 1)]);
12260 }
12261
12262 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
12263 async fn startup_lagging_mcp_facade_installs_and_acknowledges_root_publication() {
12264 let _key = bamboo_config::encryption::set_test_encryption_key([0x5a; 32]);
12265 let dir = tempfile::tempdir().unwrap();
12266 let mut state = AppState::new(dir.path().to_path_buf()).await.unwrap();
12267 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
12268 stop_config_watcher(&mut state);
12269 let external = bamboo_config::ConfigFacade::open(dir.path()).unwrap();
12270 let candidate = disabled_mcp_config("startup-lag-root");
12271 std::fs::write(
12272 dir.path().join("config.json"),
12273 serde_json::to_vec(&serde_json::json!({"mcp": candidate})).unwrap(),
12274 )
12275 .unwrap();
12276 let committed = external
12277 .reconcile_reappeared_legacy_root()
12278 .unwrap()
12279 .unwrap();
12280 assert_eq!(
12281 committed.committed,
12282 vec![ConfigSectionEvent::Changed {
12283 section: "mcp".to_string(),
12284 revision: 1,
12285 }]
12286 );
12287 assert_eq!(
12288 state
12289 .config_facade
12290 .as_ref()
12291 .unwrap()
12292 .registry()
12293 .mcp
12294 .snapshot()
12295 .revision,
12296 0
12297 );
12298
12299 let mut feed = state.account_sink.subscribe();
12300 restart_config_watcher(&mut state);
12301 assert!(matches!(
12302 next_mcp_config_event(&mut feed).await,
12303 AgentEvent::ConfigChanged { revision: 1, .. }
12304 ));
12305 wait_for_root_outbox_to_clear(dir.path()).await;
12306 assert!(state
12307 .config
12308 .read()
12309 .await
12310 .mcp
12311 .servers
12312 .iter()
12313 .any(|server| server.id == "startup-lag-root"));
12314 }
12315
12316 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
12317 async fn persistent_root_mcp_runtime_failure_retries_without_invalid_event_storm() {
12318 let _key = bamboo_config::encryption::set_test_encryption_key([0x55; 32]);
12319 let dir = tempfile::tempdir().unwrap();
12320 let state = AppState::new(dir.path().to_path_buf()).await.unwrap();
12321 wait_for_mcp_health(&state, SectionStatus::Healthy, 0).await;
12322 let failing = working_stdio_mcp_config(dir.path(), "root-retry", None);
12323 let script = match &failing.servers[0].transport {
12324 TransportConfig::Stdio(stdio) => PathBuf::from(&stdio.args[0]),
12325 _ => unreachable!(),
12326 };
12327 std::fs::remove_file(&script).unwrap();
12328 let mut feed = state.account_sink.subscribe();
12329 std::fs::write(
12330 dir.path().join("config.json"),
12331 serde_json::to_vec(&serde_json::json!({"mcp": failing})).unwrap(),
12332 )
12333 .unwrap();
12334
12335 assert!(matches!(
12336 next_mcp_config_event(&mut feed).await,
12337 AgentEvent::ConfigInvalid { revision: 1, .. }
12338 ));
12339 tokio::time::sleep(Duration::from_secs(4)).await;
12340 let failed_events =
12341 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
12342 assert_eq!(
12343 failed_events
12344 .iter()
12345 .filter(|event| matches!(
12346 &event.event,
12347 AgentEvent::ConfigInvalid { section, .. } if section == "mcp"
12348 ))
12349 .count(),
12350 1
12351 );
12352 assert!(bamboo_config::has_pending_legacy_root_publications(dir.path()).unwrap());
12353
12354 let mut repaired = working_stdio_mcp_config(dir.path(), "root-retry-fixed", None);
12355 repaired.servers[0].id = "root-retry".to_string();
12356 std::fs::write(
12357 dir.path().join("config.json"),
12358 serde_json::to_vec(&serde_json::json!({"mcp": repaired})).unwrap(),
12359 )
12360 .unwrap();
12361 let repaired_root_event = next_mcp_config_event(&mut feed).await;
12362 let repaired_health = state
12363 .mcp_config_live_health
12364 .read()
12365 .unwrap_or_else(|poisoned| poisoned.into_inner())
12366 .clone();
12367 let repaired_snapshot = state
12368 .config_facade
12369 .as_ref()
12370 .unwrap()
12371 .registry()
12372 .mcp
12373 .snapshot();
12374 assert!(
12375 matches!(
12376 repaired_root_event,
12377 AgentEvent::ConfigChanged { revision: 2, .. }
12378 ),
12379 "unexpected repaired root event: {repaired_root_event:?}; health: {repaired_health:?}; typed: {:?}",
12380 repaired_snapshot.data
12381 );
12382 wait_for_root_outbox_to_clear(dir.path()).await;
12383 assert!(state.config.read().await.mcp.servers.iter().any(|server| {
12384 server.id == "root-retry"
12385 && matches!(
12386 &server.transport,
12387 TransportConfig::Stdio(stdio)
12388 if stdio.args.iter().any(|arg| arg.contains("root-retry-fixed"))
12389 )
12390 }));
12391 let events =
12392 bamboo_engine::events::journal::read_since(state.account_sink.events_dir(), 0).unwrap();
12393 assert_eq!(
12394 events
12395 .iter()
12396 .filter(|event| matches!(
12397 &event.event,
12398 AgentEvent::ConfigInvalid { section, .. } if section == "mcp"
12399 ))
12400 .count(),
12401 1
12402 );
12403 assert_eq!(
12404 events
12405 .iter()
12406 .filter(|event| matches!(
12407 &event.event,
12408 AgentEvent::ConfigChanged { section, revision }
12409 if section == "mcp" && *revision == 2
12410 ))
12411 .count(),
12412 1
12413 );
12414 assert_eq!(
12415 events
12416 .iter()
12417 .filter(|event| matches!(
12418 &event.event,
12419 AgentEvent::ConfigChanged { section, revision }
12420 if section == "mcp" && *revision == 1
12421 ))
12422 .count(),
12423 0
12424 );
12425 }
12426}