Skip to main content

clawft_kernel/
boot.rs

1//! Kernel boot sequence and state machine.
2//!
3//! The [`Kernel`] struct is the central coordinator. It wraps
4//! [`AppContext`] and manages the process table, service registry,
5//! IPC, and health subsystem through a boot/shutdown lifecycle.
6
7use std::sync::Arc;
8use std::time::Instant;
9
10use serde::{Deserialize, Serialize};
11use tracing::{error, info};
12#[cfg(feature = "exochain")]
13use tracing::warn;
14
15use clawft_core::bootstrap::AppContext;
16use clawft_core::bus::MessageBus;
17use clawft_platform::Platform;
18use clawft_types::config::Config;
19
20use crate::a2a::A2ARouter;
21use crate::capability::{AgentCapabilities, CapabilityChecker};
22use crate::cluster::{ClusterConfig, ClusterMembership};
23use crate::console::{BootEvent, BootLog, BootPhase, KernelEventLog};
24use crate::error::{KernelError, KernelResult};
25use crate::health::HealthSystem;
26use crate::ipc::KernelIpc;
27use crate::process::{ProcessEntry, ProcessState, ProcessTable, ResourceUsage};
28use crate::service::ServiceRegistry;
29use crate::supervisor::AgentSupervisor;
30use crate::topic::TopicRouter;
31use clawft_types::config::KernelConfig;
32
33/// Kernel lifecycle state.
34#[non_exhaustive]
35#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
36pub enum KernelState {
37    /// Kernel is in the process of booting.
38    Booting,
39    /// Kernel is running and accepting commands.
40    Running,
41    /// Kernel is shutting down.
42    ShuttingDown,
43    /// Kernel has been halted.
44    Halted,
45}
46
47impl std::fmt::Display for KernelState {
48    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
49        match self {
50            KernelState::Booting => write!(f, "booting"),
51            KernelState::Running => write!(f, "running"),
52            KernelState::ShuttingDown => write!(f, "shutting_down"),
53            KernelState::Halted => write!(f, "halted"),
54        }
55    }
56}
57
58// ── Feature-gated subsystem bundles ──────────────────────────────
59//
60// These group related feature-gated fields so the Kernel struct
61// has fewer conditional fields (3 instead of 18) and init code
62// is easier to read.
63
64/// Exochain subsystem: chain manager, resource tree, and governance gate.
65#[cfg(feature = "exochain")]
66pub struct ChainSubsystem {
67    pub(crate) chain_manager: Option<Arc<crate::chain::ChainManager>>,
68    pub(crate) tree_manager: Option<Arc<crate::tree_manager::TreeManager>>,
69    pub(crate) governance_gate: Option<Arc<dyn crate::gate::GateBackend>>,
70}
71
72/// ECC cognitive substrate: HNSW, causal graph, cognitive tick,
73/// cross-references, impulse queue, and boot-time calibration.
74#[cfg(feature = "ecc")]
75pub struct EccSubsystem {
76    pub(crate) hnsw: Option<Arc<crate::hnsw_service::HnswService>>,
77    pub(crate) causal: Option<Arc<crate::causal::CausalGraph>>,
78    pub(crate) tick: Option<Arc<crate::cognitive_tick::CognitiveTick>>,
79    pub(crate) crossrefs: Option<Arc<crate::crossref::CrossRefStore>>,
80    pub(crate) impulses: Option<Arc<crate::impulse::ImpulseQueue>>,
81    pub(crate) calibration: Option<crate::calibration::EccCalibration>,
82}
83
84/// OS-patterns observability: metrics, structured logging, timers,
85/// and the dead-letter queue.
86#[cfg(feature = "os-patterns")]
87pub struct ObservabilitySubsystem {
88    pub(crate) metrics_registry: Option<Arc<crate::metrics::MetricsRegistry>>,
89    pub(crate) log_service: Option<Arc<crate::log_service::LogService>>,
90    pub(crate) timer_service: Option<Arc<crate::timer::TimerService>>,
91    pub(crate) dead_letter_queue: Option<Arc<crate::dead_letter::DeadLetterQueue>>,
92}
93
94/// The WeftOS kernel.
95///
96/// Wraps `AppContext<P>` in a managed boot sequence with process
97/// tracking, service lifecycle, IPC, and health monitoring.
98///
99/// # Lifecycle
100///
101/// 1. Call [`Kernel::boot`] to initialize all subsystems.
102/// 2. The kernel transitions from `Booting` -> `Running`.
103/// 3. Call [`Kernel::shutdown`] to gracefully stop everything.
104/// 4. The kernel transitions from `Running` -> `ShuttingDown` -> `Halted`.
105pub struct Kernel<P: Platform> {
106    state: KernelState,
107    config: KernelConfig,
108    app_context: Option<AppContext<P>>,
109    bus: Arc<MessageBus>,
110    process_table: Arc<ProcessTable>,
111    service_registry: Arc<ServiceRegistry>,
112    ipc: Arc<KernelIpc>,
113    a2a_router: Arc<A2ARouter>,
114    cron_service: Arc<crate::cron::CronService>,
115    health: HealthSystem,
116    supervisor: AgentSupervisor<P>,
117    boot_log: BootLog,
118    event_log: Arc<KernelEventLog>,
119    boot_time: Instant,
120    cluster_membership: Arc<ClusterMembership>,
121    #[cfg(feature = "exochain")]
122    chain: ChainSubsystem,
123    #[cfg(feature = "ecc")]
124    ecc: EccSubsystem,
125    #[cfg(feature = "os-patterns")]
126    observability: ObservabilitySubsystem,
127}
128
129impl<P: Platform> Kernel<P> {
130    /// Boot the kernel from configuration and platform.
131    ///
132    /// This is the primary entry point. It:
133    /// 1. Creates subsystems (process table, service registry, IPC, health)
134    /// 2. Creates AppContext (reuses existing bootstrap)
135    /// 3. Registers the kernel process (PID 0)
136    /// 4. Starts all registered services
137    /// 5. Transitions to Running state
138    ///
139    /// # Errors
140    ///
141    /// Returns [`KernelError::Boot`] if any critical subsystem fails
142    /// to initialize.
143    pub async fn boot(
144        config: Config,
145        kernel_config: KernelConfig,
146        platform: Arc<P>,
147    ) -> KernelResult<Self> {
148        let boot_time = Instant::now();
149        let mut boot_log = BootLog::new();
150
151        info!("WeftOS kernel booting");
152        boot_log.push(BootEvent::info(BootPhase::Init, "WeftOS v0.1.0 booting..."));
153        boot_log.push(BootEvent::info(BootPhase::Init, "PID 0 (kernel)"));
154
155        // 1. Create subsystems
156        let process_table = Arc::new(ProcessTable::new(kernel_config.max_processes));
157        let service_registry = Arc::new(ServiceRegistry::new());
158
159        boot_log.push(BootEvent::info(
160            BootPhase::Config,
161            format!("Max processes: {}", kernel_config.max_processes),
162        ));
163        boot_log.push(BootEvent::info(
164            BootPhase::Config,
165            format!(
166                "Health check interval: {}s",
167                kernel_config.health_check_interval_secs
168            ),
169        ));
170
171        // 2. Create AppContext
172        let app_context = AppContext::new(config, platform)
173            .await
174            .map_err(|e| KernelError::Boot(format!("AppContext init failed: {e}")))?;
175
176        // 3. Create IPC from the AppContext's MessageBus
177        let bus = app_context.bus().clone();
178        let ipc = Arc::new(KernelIpc::new(bus.clone()));
179
180        // 4. Create health system
181        let health = HealthSystem::new(kernel_config.health_check_interval_secs);
182
183        // 5. Register kernel process (PID 0)
184        let kernel_entry = ProcessEntry {
185            pid: 0,
186            agent_id: "kernel".to_owned(),
187            state: ProcessState::Running,
188            capabilities: AgentCapabilities::default(),
189            resource_usage: ResourceUsage::default(),
190            cancel_token: tokio_util::sync::CancellationToken::new(),
191            parent_pid: None,
192        };
193        process_table
194            .insert_with_pid(kernel_entry)
195            .map_err(|e| KernelError::Boot(format!("failed to register kernel process: {e}")))?;
196
197        boot_log.push(BootEvent::info(
198            BootPhase::Services,
199            "Service registry ready",
200        ));
201
202        // 5a. Construct A2ARouter (per-PID inboxes, capability-checked routing)
203        let capability_checker = Arc::new(CapabilityChecker::new(process_table.clone()));
204        let topic_router = Arc::new(TopicRouter::new(process_table.clone()));
205        let a2a_router = Arc::new(A2ARouter::new(
206            process_table.clone(),
207            capability_checker,
208            topic_router,
209        ));
210
211        boot_log.push(BootEvent::info(BootPhase::Services, "A2A router ready"));
212
213        // 5b. Register cron service (K0 gate requirement)
214        let cron_svc = Arc::new(crate::cron::CronService::new());
215        if let Err(e) = service_registry.register(cron_svc.clone()) {
216            error!(error = %e, "failed to register cron service");
217        } else {
218            boot_log.push(BootEvent::info(BootPhase::Services, "Cron service registered"));
219        }
220
221        // 5c. Register container service (K4)
222        let container_manager = std::sync::Arc::new(
223            crate::container::ContainerManager::new(crate::container::ContainerConfig::default()),
224        );
225        let container_service = std::sync::Arc::new(
226            crate::container::ContainerService::new(container_manager.clone()),
227        );
228        if let Err(e) = service_registry.register(container_service) {
229            error!(error = %e, "failed to register container service");
230        } else {
231            boot_log.push(BootEvent::info(
232                BootPhase::Services,
233                "Container service registered",
234            ));
235        }
236
237        // 6. Create cluster membership (universal, always present)
238        let cluster_config = ClusterConfig {
239            node_id: uuid::Uuid::new_v4().to_string(),
240            node_name: kernel_config
241                .cluster
242                .as_ref()
243                .and_then(|c| c.node_name.clone())
244                .unwrap_or_else(|| "local".into()),
245            heartbeat_interval_secs: kernel_config
246                .cluster
247                .as_ref()
248                .map(|c| c.heartbeat_interval_secs)
249                .unwrap_or(5),
250            ..ClusterConfig::default()
251        };
252        let cluster_membership = Arc::new(ClusterMembership::new(cluster_config));
253
254        boot_log.push(BootEvent::info(
255            BootPhase::Network,
256            format!(
257                "Cluster membership ready (node {})",
258                cluster_membership.local_node_id()
259            ),
260        ));
261
262        // 7. Register cluster service (when feature-gated ruvector integration is enabled)
263        #[cfg(feature = "cluster")]
264        {
265            use crate::cluster::ClusterService;
266            use ruvector_cluster::StaticDiscovery;
267
268            let net_config = kernel_config
269                .cluster
270                .clone()
271                .unwrap_or_default();
272            let seed_addrs: Vec<std::net::SocketAddr> = net_config
273                .seed_nodes
274                .iter()
275                .filter_map(|s| s.parse().ok())
276                .collect();
277            let seed_nodes: Vec<ruvector_cluster::ClusterNode> = seed_addrs
278                .into_iter()
279                .map(|addr| ruvector_cluster::ClusterNode::new(addr.to_string(), addr))
280                .collect();
281            let discovery = Box::new(StaticDiscovery::new(seed_nodes));
282            let node_id = cluster_membership.local_node_id().to_owned();
283
284            match ClusterService::new(
285                net_config,
286                node_id,
287                discovery,
288                Arc::clone(&cluster_membership),
289            ) {
290                Ok(cluster_svc) => {
291                    let svc = Arc::new(cluster_svc);
292                    if let Err(e) = service_registry.register(svc) {
293                        error!(error = %e, "failed to register cluster service");
294                    } else {
295                        boot_log.push(BootEvent::info(
296                            BootPhase::Network,
297                            "Cluster service registered (ruvector)",
298                        ));
299                    }
300                }
301                Err(e) => {
302                    error!(error = %e, "failed to create cluster service");
303                    boot_log.push(BootEvent::info(
304                        BootPhase::Network,
305                        format!("Cluster service failed: {e}"),
306                    ));
307                }
308            }
309        }
310
311        // 8. Start services (none registered by default at boot, unless cluster feature added one)
312        service_registry
313            .start_all()
314            .await
315            .map_err(|e| KernelError::Boot(format!("service start failed: {e}")))?;
316
317        // 8b. Initialize local exochain (when exochain feature is enabled)
318        //     Restores from checkpoint file if available; otherwise fresh genesis.
319        #[cfg(feature = "exochain")]
320        let chain_manager = {
321            let chain_config = kernel_config.chain.clone().unwrap_or_default();
322            if chain_config.enabled {
323                // Load or generate Ed25519 signing key for chain integrity.
324                let signing_key = if let Some(ref ckpt_path) = chain_config.effective_checkpoint_path() {
325                    let key_path = std::path::PathBuf::from(ckpt_path).with_extension("key");
326                    match crate::chain::ChainManager::load_or_create_key(&key_path) {
327                        Ok(key) => {
328                            boot_log.push(BootEvent::info(
329                                BootPhase::Services,
330                                format!("Ed25519 signing key loaded: {}", key_path.display()),
331                            ));
332                            Some(key)
333                        }
334                        Err(e) => {
335                            warn!(error = %e, "failed to load/create signing key");
336                            boot_log.push(BootEvent::info(
337                                BootPhase::Services,
338                                format!("Signing key unavailable: {e} — chain will be unsigned"),
339                            ));
340                            None
341                        }
342                    }
343                } else {
344                    None
345                };
346
347                let cm = if let Some(ref ckpt_path) = chain_config.effective_checkpoint_path() {
348                    let json_path = std::path::PathBuf::from(ckpt_path);
349                    // Derive RVF path from JSON path: same directory, `.rvf` extension
350                    let rvf_path = json_path.with_extension("rvf");
351
352                    if rvf_path.exists() {
353                        // Prefer RVF format (cryptographic integrity verification)
354                        match crate::chain::ChainManager::load_from_rvf(&rvf_path, chain_config.checkpoint_interval) {
355                            Ok(restored) => {
356                                let seq = restored.sequence();
357                                boot_log.push(BootEvent::info(
358                                    BootPhase::Services,
359                                    format!(
360                                        "Chain restored from RVF (seq={}, chain_id={})",
361                                        seq, chain_config.chain_id,
362                                    ),
363                                ));
364                                Arc::new(restored)
365                            }
366                            Err(e) => {
367                                error!(error = %e, "failed to restore RVF chain, trying JSON fallback");
368                                // Fall back to JSON
369                                if json_path.exists() {
370                                    match crate::chain::ChainManager::load_from_file(&json_path, chain_config.checkpoint_interval) {
371                                        Ok(restored) => {
372                                            let seq = restored.sequence();
373                                            boot_log.push(BootEvent::info(
374                                                BootPhase::Services,
375                                                format!(
376                                                    "Chain restored from JSON fallback (seq={}, chain_id={})",
377                                                    seq, chain_config.chain_id,
378                                                ),
379                                            ));
380                                            Arc::new(restored)
381                                        }
382                                        Err(e2) => {
383                                            error!(error = %e2, "JSON fallback also failed, starting fresh");
384                                            boot_log.push(BootEvent::info(
385                                                BootPhase::Services,
386                                                format!("Chain restore failed (RVF: {e}, JSON: {e2}), starting fresh"),
387                                            ));
388                                            Arc::new(crate::chain::ChainManager::new(
389                                                chain_config.chain_id,
390                                                chain_config.checkpoint_interval,
391                                            ))
392                                        }
393                                    }
394                                } else {
395                                    boot_log.push(BootEvent::info(
396                                        BootPhase::Services,
397                                        format!("RVF restore failed: {e}, starting fresh"),
398                                    ));
399                                    Arc::new(crate::chain::ChainManager::new(
400                                        chain_config.chain_id,
401                                        chain_config.checkpoint_interval,
402                                    ))
403                                }
404                            }
405                        }
406                    } else if json_path.exists() {
407                        // Legacy JSON format
408                        match crate::chain::ChainManager::load_from_file(&json_path, chain_config.checkpoint_interval) {
409                            Ok(restored) => {
410                                let seq = restored.sequence();
411                                boot_log.push(BootEvent::info(
412                                    BootPhase::Services,
413                                    format!(
414                                        "Chain restored from JSON (seq={}, chain_id={}, will migrate to RVF)",
415                                        seq, chain_config.chain_id,
416                                    ),
417                                ));
418                                Arc::new(restored)
419                            }
420                            Err(e) => {
421                                error!(error = %e, "failed to restore chain, starting fresh");
422                                boot_log.push(BootEvent::info(
423                                    BootPhase::Services,
424                                    format!("Chain restore failed: {e}, starting fresh"),
425                                ));
426                                Arc::new(crate::chain::ChainManager::new(
427                                    chain_config.chain_id,
428                                    chain_config.checkpoint_interval,
429                                ))
430                            }
431                        }
432                    } else {
433                        Arc::new(crate::chain::ChainManager::new(
434                            chain_config.chain_id,
435                            chain_config.checkpoint_interval,
436                        ))
437                    }
438                } else {
439                    Arc::new(crate::chain::ChainManager::new(
440                        chain_config.chain_id,
441                        chain_config.checkpoint_interval,
442                    ))
443                };
444
445                // Attach Ed25519 signing key if available.
446                let mut cm = cm;
447                if let Some(key) = signing_key
448                    && let Some(inner) = Arc::get_mut(&mut cm)
449                {
450                    // Generate ML-DSA key from Ed25519 key bytes for dual signing.
451                    let ml_dsa_seed = key.to_bytes();
452                    let (ml_key, _ml_vk) = weftos_rvf_crypto::MlDsa65Key::generate(&ml_dsa_seed);
453                    inner.set_signing_key(key);
454                    inner.set_ml_dsa_key(ml_key);
455                    boot_log.push(BootEvent::info(
456                        BootPhase::Services,
457                        "Dual signing enabled (Ed25519 + ML-DSA-65)",
458                    ));
459                }
460
461                boot_log.push(BootEvent::info(
462                    BootPhase::Services,
463                    format!(
464                        "Local chain ready (chain_id={}, seq={}, signed={})",
465                        chain_config.chain_id,
466                        cm.sequence(),
467                        cm.has_signing_key(),
468                    ),
469                ));
470
471                // Log boot phases to chain
472                cm.append(
473                    "kernel",
474                    "boot.init",
475                    Some(serde_json::json!({"version": "0.1.0"})),
476                );
477                cm.append(
478                    "kernel",
479                    "boot.config",
480                    Some(serde_json::json!({
481                        "max_processes": kernel_config.max_processes,
482                        "health_interval": kernel_config.health_check_interval_secs,
483                    })),
484                );
485                cm.append(
486                    "kernel",
487                    "boot.services",
488                    Some(serde_json::json!({
489                        "count": service_registry.len(),
490                    })),
491                );
492
493                Some(cm)
494            } else {
495                boot_log.push(BootEvent::info(
496                    BootPhase::Services,
497                    "Local chain disabled",
498                ));
499                None
500            }
501        };
502
503        // 8c. Bootstrap resource tree via TreeManager (when exochain feature is enabled)
504        //     First attempt to restore from checkpoint; fall back to fresh bootstrap.
505        #[cfg(feature = "exochain")]
506        let tree_manager = {
507            let rt_config = kernel_config.resource_tree.clone().unwrap_or_default();
508            if rt_config.enabled {
509                if let Some(ref cm) = chain_manager {
510                    let tm = Arc::new(crate::tree_manager::TreeManager::new(Arc::clone(cm)));
511
512                    // Derive tree checkpoint path from chain checkpoint path
513                    let chain_cfg = kernel_config.chain.clone().unwrap_or_default();
514                    let tree_ckpt_path = chain_cfg
515                        .effective_checkpoint_path()
516                        .map(|p| std::path::PathBuf::from(p).with_extension("tree.json"));
517
518                    let mut restored_from_checkpoint = false;
519                    if let Some(ref tree_path) = tree_ckpt_path
520                        && tree_path.exists()
521                    {
522                        match tm.load_checkpoint(tree_path) {
523                            Ok(()) => {
524                                let stats = tm.stats();
525                                // Verify tree root hash against the chain's last recorded hash.
526                                let chain_tree_hash = cm.last_tree_root_hash();
527                                if let Some(ref expected) = chain_tree_hash {
528                                    if stats.root_hash == *expected {
529                                        boot_log.push(BootEvent::info(
530                                            BootPhase::ResourceTree,
531                                            format!(
532                                                "Resource tree restored from checkpoint ({} nodes, root={}..., hash verified)",
533                                                stats.node_count,
534                                                &stats.root_hash[..12],
535                                            ),
536                                        ));
537                                        restored_from_checkpoint = true;
538                                    } else {
539                                        warn!(
540                                            expected = %expected,
541                                            actual = %stats.root_hash,
542                                            "tree checkpoint root hash mismatch — falling back to fresh bootstrap"
543                                        );
544                                        boot_log.push(BootEvent::info(
545                                            BootPhase::ResourceTree,
546                                            format!(
547                                                "Tree checkpoint hash mismatch (expected={}..., got={}...), bootstrapping fresh",
548                                                &expected[..std::cmp::min(12, expected.len())],
549                                                &stats.root_hash[..12],
550                                            ),
551                                        ));
552                                        // Don't set restored_from_checkpoint — falls through to bootstrap.
553                                    }
554                                } else {
555                                    // No hash in chain to verify against — accept as-is.
556                                    boot_log.push(BootEvent::info(
557                                        BootPhase::ResourceTree,
558                                        format!(
559                                            "Resource tree restored from checkpoint ({} nodes, root={}...)",
560                                            stats.node_count,
561                                            &stats.root_hash[..12],
562                                        ),
563                                    ));
564                                    restored_from_checkpoint = true;
565                                }
566                            }
567                            Err(e) => {
568                                error!(error = %e, "failed to restore tree checkpoint, bootstrapping fresh");
569                            }
570                        }
571                    }
572
573                    if !restored_from_checkpoint {
574                        if let Err(e) = tm.bootstrap() {
575                            error!(error = %e, "failed to bootstrap resource tree");
576                            // Still allow boot to proceed without tree
577                        } else {
578                            let stats = tm.stats();
579                            boot_log.push(BootEvent::info(
580                                BootPhase::ResourceTree,
581                                format!(
582                                    "Resource tree bootstrapped ({} nodes, root={}...)",
583                                    stats.node_count,
584                                    &stats.root_hash[..12],
585                                ),
586                            ));
587                        }
588                    }
589
590                    // Register cron service in tree with manifest (5b wiring)
591                    if let Err(e) = tm.register_service_with_manifest("cron", "scheduler") {
592                        tracing::debug!(error = %e, "failed to register cron in tree (may already exist)");
593                    }
594
595                    // K4: Register container namespace in the resource tree
596                    {
597                        use exo_resource_tree::model::{ResourceId, ResourceKind};
598                        if let Err(e) = tm.insert(
599                            ResourceId::new("/kernel/containers"),
600                            ResourceKind::Namespace,
601                            ResourceId::new("/kernel"),
602                        ) {
603                            tracing::debug!(
604                                error = %e,
605                                "failed to register containers namespace (may already exist)"
606                            );
607                        }
608                    }
609
610                    // K3c: Register ECC namespaces in the resource tree
611                    #[cfg(feature = "ecc")]
612                    {
613                        use exo_resource_tree::model::{ResourceId, ResourceKind};
614                        let ecc_namespaces = [
615                            ("/kernel/services/ecc", "/kernel/services"),
616                            ("/kernel/services/ecc/hnsw", "/kernel/services/ecc"),
617                            ("/kernel/services/ecc/causal", "/kernel/services/ecc"),
618                            ("/kernel/services/ecc/tick", "/kernel/services/ecc"),
619                            ("/kernel/services/ecc/calibration", "/kernel/services/ecc"),
620                            ("/kernel/services/ecc/crossrefs", "/kernel/services/ecc"),
621                        ];
622                        for (path, parent) in &ecc_namespaces {
623                            if let Err(e) = tm.insert(
624                                ResourceId::new(*path),
625                                ResourceKind::Namespace,
626                                ResourceId::new(*parent),
627                            ) {
628                                tracing::debug!(
629                                    error = %e, path = *path,
630                                    "failed to register ECC namespace (may already exist)"
631                                );
632                            }
633                        }
634                        boot_log.push(BootEvent::info(
635                            BootPhase::Ecc,
636                            "ECC resource tree namespaces registered",
637                        ));
638                    }
639
640                    // K3: Register tool namespaces in the resource tree
641                    {
642                        use exo_resource_tree::model::{ResourceId, ResourceKind};
643                        let tool_namespaces = [
644                            ("/kernel/tools", "/kernel"),
645                            ("/kernel/tools/fs", "/kernel/tools"),
646                            ("/kernel/tools/agent", "/kernel/tools"),
647                            ("/kernel/tools/sys", "/kernel/tools"),
648                            ("/kernel/tools/ipc", "/kernel/tools"),
649                            #[cfg(feature = "ecc")]
650                            ("/kernel/tools/ecc", "/kernel/tools"),
651                        ];
652                        for (path, parent) in &tool_namespaces {
653                            if let Err(e) = tm.insert(
654                                ResourceId::new(*path),
655                                ResourceKind::Namespace,
656                                ResourceId::new(*parent),
657                            ) {
658                                tracing::debug!(
659                                    error = %e, path = *path,
660                                    "failed to register tool namespace (may already exist)"
661                                );
662                            }
663                        }
664                        // Register each built-in tool as a Tool node
665                        let catalog = crate::wasm_runner::builtin_tool_catalog();
666                        for spec in &catalog {
667                            let cat_path = match spec.category {
668                                crate::wasm_runner::ToolCategory::Filesystem => "/kernel/tools/fs",
669                                crate::wasm_runner::ToolCategory::Agent => "/kernel/tools/agent",
670                                crate::wasm_runner::ToolCategory::System => "/kernel/tools/sys",
671                                crate::wasm_runner::ToolCategory::Ecc => "/kernel/tools/ecc",
672                                crate::wasm_runner::ToolCategory::User => "/kernel/tools",
673                            };
674                            let tool_path = format!("{}/{}", cat_path, spec.name.replace('.', "/"));
675                            if let Err(e) = tm.insert(
676                                ResourceId::new(&tool_path),
677                                ResourceKind::Tool,
678                                ResourceId::new(cat_path),
679                            ) {
680                                tracing::debug!(
681                                    error = %e, tool = %spec.name,
682                                    "failed to register tool node (may already exist)"
683                                );
684                            }
685                        }
686                        boot_log.push(BootEvent::info(
687                            BootPhase::ResourceTree,
688                            format!("Registered {} built-in tools in resource tree", catalog.len()),
689                        ));
690                    }
691
692                    Some(tm)
693                } else {
694                    boot_log.push(BootEvent::info(
695                        BootPhase::ResourceTree,
696                        "Resource tree requires chain — skipped",
697                    ));
698                    None
699                }
700            } else {
701                boot_log.push(BootEvent::info(
702                    BootPhase::ResourceTree,
703                    "Resource tree disabled",
704                ));
705                None
706            }
707        };
708
709        // 8f. Initialize governance engine with chain-anchored genesis rules
710        //
711        // The governance gate is the constitutional layer of the kernel.
712        // At boot, genesis rules are written to the chain as immutable
713        // entries. Once anchored, they cannot be modified — only superseded
714        // by a `governance.root.supersede` event that references the
715        // original genesis sequence.
716        #[cfg(feature = "exochain")]
717        let governance_gate: Option<Arc<dyn crate::gate::GateBackend>> = {
718            if let Some(ref cm) = chain_manager {
719                use crate::governance::{GovernanceBranch, GovernanceRule, RuleSeverity};
720                use crate::gate::GovernanceGate;
721
722                // Default risk threshold (0.7 for production safety)
723                let risk_threshold = 0.7;
724                let human_approval = false;
725
726                let mut gate = GovernanceGate::new(risk_threshold, human_approval)
727                    .with_chain(Arc::clone(cm));
728
729                // ── Genesis governance rules (immutable chain entries) ────
730                // These are the constitutional rules that govern all agent
731                // behavior. Once written to the chain, they cannot be
732                // modified — only superseded by a governance.root.supersede
733                // event.
734
735                let genesis_rules = vec![
736                    // ── Core constitutional rules (GOV-001 .. GOV-007) ──────
737                    GovernanceRule {
738                        id: "GOV-001".into(),
739                        description: "High-risk operations require elevated review".into(),
740                        branch: GovernanceBranch::Judicial,
741                        severity: RuleSeverity::Blocking,
742                        active: true,
743                        reference_url: None,
744                        sop_category: None,
745                    },
746                    GovernanceRule {
747                        id: "GOV-002".into(),
748                        description: "Security-sensitive actions must not exceed security threshold".into(),
749                        branch: GovernanceBranch::Judicial,
750                        severity: RuleSeverity::Blocking,
751                        active: true,
752                        reference_url: None,
753                        sop_category: None,
754                    },
755                    GovernanceRule {
756                        id: "GOV-003".into(),
757                        description: "Privacy-impacting operations flagged for review".into(),
758                        branch: GovernanceBranch::Legislative,
759                        severity: RuleSeverity::Warning,
760                        active: true,
761                        reference_url: None,
762                        sop_category: None,
763                    },
764                    GovernanceRule {
765                        id: "GOV-004".into(),
766                        description: "Novel/unprecedented actions require advisory logging".into(),
767                        branch: GovernanceBranch::Executive,
768                        severity: RuleSeverity::Advisory,
769                        active: true,
770                        reference_url: None,
771                        sop_category: None,
772                    },
773                    GovernanceRule {
774                        id: "GOV-005".into(),
775                        description: "Filesystem write operations scored for risk".into(),
776                        branch: GovernanceBranch::Legislative,
777                        severity: RuleSeverity::Warning,
778                        active: true,
779                        reference_url: None,
780                        sop_category: None,
781                    },
782                    GovernanceRule {
783                        id: "GOV-006".into(),
784                        description: "Agent spawn operations require governance clearance".into(),
785                        branch: GovernanceBranch::Executive,
786                        severity: RuleSeverity::Blocking,
787                        active: true,
788                        reference_url: None,
789                        sop_category: None,
790                    },
791                    GovernanceRule {
792                        id: "GOV-007".into(),
793                        description: "IPC messages between agents logged for audit trail".into(),
794                        branch: GovernanceBranch::Judicial,
795                        severity: RuleSeverity::Advisory,
796                        active: true,
797                        reference_url: None,
798                        sop_category: None,
799                    },
800                    // ── AI-SDLC SOP rules: Legislative (6) ──────────────────
801                    GovernanceRule {
802                        id: "SOP-L001".into(),
803                        description: "AI-IRB approval required before high-impact deployments".into(),
804                        branch: GovernanceBranch::Legislative,
805                        severity: RuleSeverity::Blocking,
806                        active: true,
807                        reference_url: Some("https://github.com/AISDLC/AI-SDLC-SOPs/blob/main/sops/SOP-1300-01-AI_IRB_Approval.md".into()),
808                        sop_category: Some("governance".into()),
809                    },
810                    GovernanceRule {
811                        id: "SOP-L002".into(),
812                        description: "Version control and branching policies must be enforced".into(),
813                        branch: GovernanceBranch::Legislative,
814                        severity: RuleSeverity::Warning,
815                        active: true,
816                        reference_url: Some("https://github.com/AISDLC/AI-SDLC-SOPs/blob/main/sops/SOP-1003-01-AI_Version_Control.md".into()),
817                        sop_category: Some("governance".into()),
818                    },
819                    GovernanceRule {
820                        id: "SOP-L003".into(),
821                        description: "Requirements must include AI-IRB ethical review".into(),
822                        branch: GovernanceBranch::Legislative,
823                        severity: RuleSeverity::Warning,
824                        active: true,
825                        reference_url: Some("https://github.com/AISDLC/AI-SDLC-SOPs/blob/main/sops/SOP-1040-01-AI_Requirements.md".into()),
826                        sop_category: Some("engineering".into()),
827                    },
828                    GovernanceRule {
829                        id: "SOP-L004".into(),
830                        description: "Release planning must follow structured lifecycle gates".into(),
831                        branch: GovernanceBranch::Legislative,
832                        severity: RuleSeverity::Advisory,
833                        active: true,
834                        reference_url: Some("https://github.com/AISDLC/AI-SDLC-SOPs/blob/main/sops/SOP-1005-01-AI_Release_Planning.md".into()),
835                        sop_category: Some("lifecycle".into()),
836                    },
837                    GovernanceRule {
838                        id: "SOP-L005".into(),
839                        description: "Data protection and PII handling must comply with policy".into(),
840                        branch: GovernanceBranch::Legislative,
841                        severity: RuleSeverity::Blocking,
842                        active: true,
843                        reference_url: Some("https://github.com/AISDLC/AI-SDLC-SOPs/blob/main/sops/SOP-1303-01-AI_Data_Protection.md".into()),
844                        sop_category: Some("ethics".into()),
845                    },
846                    GovernanceRule {
847                        id: "SOP-L006".into(),
848                        description: "Risk register must be maintained and reviewed".into(),
849                        branch: GovernanceBranch::Legislative,
850                        severity: RuleSeverity::Warning,
851                        active: true,
852                        reference_url: Some("https://github.com/AISDLC/AI-SDLC-SOPs/blob/main/sops/SOP-1062-01-AI_Risk_Register.md".into()),
853                        sop_category: Some("governance".into()),
854                    },
855                    // ── AI-SDLC SOP rules: Executive (5) ────────────────────
856                    GovernanceRule {
857                        id: "SOP-E001".into(),
858                        description: "Secure coding standards must be followed".into(),
859                        branch: GovernanceBranch::Executive,
860                        severity: RuleSeverity::Warning,
861                        active: true,
862                        reference_url: Some("https://github.com/AISDLC/AI-SDLC-SOPs/blob/main/sops/SOP-1200-01-AI_Secure_Coding.md".into()),
863                        sop_category: Some("engineering".into()),
864                    },
865                    GovernanceRule {
866                        id: "SOP-E002".into(),
867                        description: "Deployment requires governance clearance checkpoint".into(),
868                        branch: GovernanceBranch::Executive,
869                        severity: RuleSeverity::Blocking,
870                        active: true,
871                        reference_url: Some("https://github.com/AISDLC/AI-SDLC-SOPs/blob/main/sops/SOP-1220-01-AI_Deployment_Clearance.md".into()),
872                        sop_category: Some("lifecycle".into()),
873                    },
874                    GovernanceRule {
875                        id: "SOP-E003".into(),
876                        description: "Incident response procedures must be documented and followed".into(),
877                        branch: GovernanceBranch::Executive,
878                        severity: RuleSeverity::Warning,
879                        active: true,
880                        reference_url: Some("https://github.com/AISDLC/AI-SDLC-SOPs/blob/main/sops/SOP-1008-01-AI_Incident_Response.md".into()),
881                        sop_category: Some("security".into()),
882                    },
883                    GovernanceRule {
884                        id: "SOP-E004".into(),
885                        description: "Decommissioning must follow structured teardown procedure".into(),
886                        branch: GovernanceBranch::Executive,
887                        severity: RuleSeverity::Advisory,
888                        active: true,
889                        reference_url: Some("https://github.com/AISDLC/AI-SDLC-SOPs/blob/main/sops/SOP-1011-01-AI_Decommissioning.md".into()),
890                        sop_category: Some("lifecycle".into()),
891                    },
892                    GovernanceRule {
893                        id: "SOP-E005".into(),
894                        description: "Third-party AI procurement requires screening".into(),
895                        branch: GovernanceBranch::Executive,
896                        severity: RuleSeverity::Advisory,
897                        active: true,
898                        reference_url: Some("https://github.com/AISDLC/AI-SDLC-SOPs/blob/main/sops/SOP-1004-01-AI_Procurement_Screening.md".into()),
899                        sop_category: Some("governance".into()),
900                    },
901                    // ── AI-SDLC SOP rules: Judicial (4) ─────────────────────
902                    GovernanceRule {
903                        id: "SOP-J001".into(),
904                        description: "Bias and fairness assessments required for model outputs".into(),
905                        branch: GovernanceBranch::Judicial,
906                        severity: RuleSeverity::Blocking,
907                        active: true,
908                        reference_url: Some("https://github.com/AISDLC/AI-SDLC-SOPs/blob/main/sops/SOP-1301-01-AI_Bias_Fairness.md".into()),
909                        sop_category: Some("ethics".into()),
910                    },
911                    GovernanceRule {
912                        id: "SOP-J002".into(),
913                        description: "Explainability documentation required for decision systems".into(),
914                        branch: GovernanceBranch::Judicial,
915                        severity: RuleSeverity::Warning,
916                        active: true,
917                        reference_url: Some("https://github.com/AISDLC/AI-SDLC-SOPs/blob/main/sops/SOP-1302-01-AI_Explainability.md".into()),
918                        sop_category: Some("ethics".into()),
919                    },
920                    GovernanceRule {
921                        id: "SOP-J003".into(),
922                        description: "Model drift detection and monitoring must be active".into(),
923                        branch: GovernanceBranch::Judicial,
924                        severity: RuleSeverity::Warning,
925                        active: true,
926                        reference_url: Some("https://github.com/AISDLC/AI-SDLC-SOPs/blob/main/sops/SOP-1009-01-AI_Drift_Detection.md".into()),
927                        sop_category: Some("lifecycle".into()),
928                    },
929                    GovernanceRule {
930                        id: "SOP-J004".into(),
931                        description: "Quality records must be maintained for audit compliance".into(),
932                        branch: GovernanceBranch::Judicial,
933                        severity: RuleSeverity::Advisory,
934                        active: true,
935                        reference_url: Some("https://github.com/AISDLC/AI-SDLC-SOPs/blob/main/sops/SOP-2002-01-AI_Quality_Records.md".into()),
936                        sop_category: Some("quality".into()),
937                    },
938                ];
939
940                // Anchor genesis rules to chain
941                let genesis_seq = cm.sequence();
942                let rules_json: Vec<serde_json::Value> = genesis_rules.iter().map(|r| {
943                    serde_json::json!({
944                        "id": r.id,
945                        "description": r.description,
946                        "branch": format!("{}", r.branch),
947                        "severity": format!("{}", r.severity),
948                        "reference_url": r.reference_url,
949                        "sop_category": r.sop_category,
950                    })
951                }).collect();
952
953                // The governance.genesis event is the ROOT — it establishes the
954                // constitutional rules. Its chain sequence is the governance root.
955                cm.append(
956                    "governance",
957                    "governance.genesis",
958                    Some(serde_json::json!({
959                        "version": "2.0.0",
960                        "risk_threshold": risk_threshold,
961                        "human_approval_required": human_approval,
962                        "rules": rules_json,
963                        "rule_count": genesis_rules.len(),
964                        "genesis_seq": genesis_seq,
965                    })),
966                );
967
968                // Anchor each rule individually for granular verification
969                for rule in &genesis_rules {
970                    cm.append(
971                        "governance",
972                        "governance.rule",
973                        Some(serde_json::json!({
974                            "rule_id": rule.id,
975                            "branch": format!("{}", rule.branch),
976                            "severity": format!("{}", rule.severity),
977                            "genesis_seq": genesis_seq,
978                        })),
979                    );
980                }
981
982                // Add rules to the gate
983                for rule in genesis_rules {
984                    gate = gate.add_rule(rule);
985                }
986
987                boot_log.push(BootEvent::info(
988                    BootPhase::Services,
989                    format!(
990                        "Governance genesis anchored (seq={}, {} rules, threshold={:.1})",
991                        genesis_seq,
992                        gate.engine().rule_count(),
993                        risk_threshold,
994                    ),
995                ));
996
997                Some(Arc::new(gate) as Arc<dyn crate::gate::GateBackend>)
998            } else {
999                boot_log.push(BootEvent::info(
1000                    BootPhase::Services,
1001                    "Governance: no chain available, using open governance",
1002                ));
1003                None
1004            }
1005        };
1006
1007        // Wire governance gate into A2A router for dual-layer enforcement.
1008        // The A2ARouter is already behind Arc, so we use set_gate() which
1009        // relies on OnceLock interior mutability.
1010        #[cfg(feature = "exochain")]
1011        if let Some(ref gate) = governance_gate {
1012            a2a_router.set_gate(Arc::clone(gate));
1013        }
1014
1015        // 8d. Log cluster and boot.ready chain events
1016        #[cfg(feature = "exochain")]
1017        if let Some(ref cm) = chain_manager {
1018            cm.append(
1019                "kernel",
1020                "boot.cluster",
1021                Some(serde_json::json!({
1022                    "node_id": cluster_membership.local_node_id(),
1023                })),
1024            );
1025
1026            let elapsed_ms = boot_time.elapsed().as_millis() as u64;
1027            let mut ready_payload = serde_json::json!({
1028                "elapsed_ms": elapsed_ms,
1029                "processes": process_table.len(),
1030                "services": service_registry.len(),
1031            });
1032
1033            if let Some(ref tm) = tree_manager {
1034                let root_hash = tm.stats().root_hash;
1035                ready_payload
1036                    .as_object_mut()
1037                    .unwrap()
1038                    .insert("tree_root_hash".to_string(), serde_json::json!(root_hash));
1039            }
1040
1041            cm.append("kernel", "boot.ready", Some(ready_payload));
1042
1043            // Emit boot manifest as a chain event (becomes an ExochainCheckpoint
1044            // segment in RVF persistence, capturing the complete boot state).
1045            let mut manifest = serde_json::json!({
1046                "version": env!("CARGO_PKG_VERSION"),
1047                "node_id": cluster_membership.local_node_id(),
1048                "process_count": process_table.len(),
1049                "service_count": service_registry.len(),
1050                "chain_sequence": cm.sequence(),
1051                "boot_elapsed_ms": boot_time.elapsed().as_millis() as u64,
1052            });
1053            if let Some(ref tm) = tree_manager {
1054                let stats = tm.stats();
1055                manifest.as_object_mut().unwrap().insert(
1056                    "tree_root_hash".to_string(),
1057                    serde_json::json!(stats.root_hash),
1058                );
1059                manifest.as_object_mut().unwrap().insert(
1060                    "tree_node_count".to_string(),
1061                    serde_json::json!(stats.node_count),
1062                );
1063            }
1064            cm.append("kernel", "boot.manifest", Some(manifest));
1065        }
1066
1067        // 8e. Initialize ECC cognitive substrate (when ecc feature is enabled)
1068        #[cfg(feature = "ecc")]
1069        let (ecc_hnsw, ecc_causal, ecc_tick, ecc_crossrefs, ecc_impulses, ecc_calibration) = {
1070            use crate::calibration::{EccCalibrationConfig, run_calibration};
1071            use crate::causal::CausalGraph;
1072            use crate::cognitive_tick::{CognitiveTick, CognitiveTickConfig};
1073            use crate::crossref::CrossRefStore;
1074            use crate::hnsw_service::{HnswService, HnswServiceConfig};
1075            use crate::impulse::ImpulseQueue;
1076
1077            boot_log.push(BootEvent::info(BootPhase::Ecc, "Initializing ECC cognitive substrate"));
1078
1079            let hnsw = Arc::new(HnswService::new(HnswServiceConfig::default()));
1080            let causal = Arc::new(CausalGraph::new());
1081            let crossrefs = Arc::new(CrossRefStore::new());
1082            let impulses = Arc::new(ImpulseQueue::new());
1083
1084            // Run boot-time calibration
1085            let cal_config = EccCalibrationConfig::default();
1086            let calibration = run_calibration(&hnsw, &causal, &cal_config);
1087
1088            boot_log.push(BootEvent::info(
1089                BootPhase::Ecc,
1090                format!(
1091                    "ECC calibration complete (p50={}us, p95={}us, tick={}ms, spectral={})",
1092                    calibration.compute_p50_us,
1093                    calibration.compute_p95_us,
1094                    calibration.tick_interval_ms,
1095                    calibration.spectral_capable,
1096                ),
1097            ));
1098
1099            // Create cognitive tick with calibrated interval
1100            let tick_config = CognitiveTickConfig {
1101                tick_interval_ms: calibration.tick_interval_ms,
1102                ..CognitiveTickConfig::default()
1103            };
1104            let tick = Arc::new(CognitiveTick::new(tick_config));
1105
1106            // Register ECC services
1107            if let Err(e) = service_registry.register(hnsw.clone()) {
1108                tracing::debug!(error = %e, "failed to register HNSW service");
1109            }
1110            if let Err(e) = service_registry.register(tick.clone()) {
1111                tracing::debug!(error = %e, "failed to register cognitive tick service");
1112            }
1113
1114            // Log calibration to chain
1115            #[cfg(feature = "exochain")]
1116            if let Some(ref cm) = chain_manager {
1117                cm.append(
1118                    "ecc",
1119                    "ecc.boot.calibration",
1120                    Some(serde_json::json!({
1121                        "compute_p50_us": calibration.compute_p50_us,
1122                        "compute_p95_us": calibration.compute_p95_us,
1123                        "tick_interval_ms": calibration.tick_interval_ms,
1124                        "spectral_capable": calibration.spectral_capable,
1125                    })),
1126                );
1127            }
1128
1129            boot_log.push(BootEvent::info(
1130                BootPhase::Ecc,
1131                format!(
1132                    "ECC ready (hnsw={}, causal={} nodes, tick={}ms)",
1133                    hnsw.len(),
1134                    causal.node_count(),
1135                    calibration.tick_interval_ms,
1136                ),
1137            ));
1138
1139            (
1140                Some(hnsw),
1141                Some(causal),
1142                Some(tick),
1143                Some(crossrefs),
1144                Some(impulses),
1145                Some(calibration),
1146            )
1147        };
1148
1149        // 8g. Initialize os-patterns observability modules
1150        #[cfg(feature = "os-patterns")]
1151        let (metrics_registry, log_svc, timer_svc, dead_letter_queue) = {
1152            use crate::dead_letter::DeadLetterQueue;
1153            use crate::log_service::LogService;
1154            use crate::metrics::MetricsRegistry;
1155            use crate::timer::TimerService;
1156
1157            boot_log.push(BootEvent::info(
1158                BootPhase::Services,
1159                "Initializing os-patterns observability modules",
1160            ));
1161
1162            // MetricsRegistry with built-in + boot-specific gauges
1163            let registry = Arc::new(MetricsRegistry::with_builtins());
1164            // Seed additional kernel gauges requested by W3
1165            registry.gauge_set("kernel.process.count", process_table.len() as i64);
1166            registry.gauge_set("kernel.agent.count", 0);
1167            registry.gauge_set("kernel.ipc.messages_sent", 0);
1168            registry.counter_add("kernel.ipc.messages_failed", 0);
1169            registry.gauge_set("kernel.chain.height", 0);
1170            registry.gauge_set("kernel.uptime_secs", 0);
1171
1172            boot_log.push(BootEvent::info(
1173                BootPhase::Services,
1174                "MetricsRegistry ready (built-in + kernel gauges)",
1175            ));
1176
1177            // LogService (ring-buffer structured logs)
1178            let log_svc = Arc::new(LogService::with_default_capacity());
1179            boot_log.push(BootEvent::info(BootPhase::Services, "LogService ready"));
1180
1181            // TimerService (sub-second precision timers)
1182            let timer_svc = Arc::new(TimerService::new());
1183            boot_log.push(BootEvent::info(BootPhase::Services, "TimerService ready"));
1184
1185            // DeadLetterQueue (undeliverable message sink)
1186            let dlq = Arc::new(DeadLetterQueue::with_default_capacity());
1187            boot_log.push(BootEvent::info(
1188                BootPhase::Services,
1189                "DeadLetterQueue ready",
1190            ));
1191
1192            // Wire DLQ into the A2A router
1193            a2a_router.set_dead_letter_queue(Arc::clone(&dlq));
1194
1195            (Some(registry), Some(log_svc), Some(timer_svc), Some(dlq))
1196        };
1197
1198        let elapsed = boot_time.elapsed();
1199        boot_log.push(BootEvent::info(
1200            BootPhase::Ready,
1201            format!(
1202                "Boot complete in {:.1}s ({} processes, {} services)",
1203                elapsed.as_secs_f64(),
1204                process_table.len(),
1205                service_registry.len(),
1206            ),
1207        ));
1208
1209        info!(
1210            elapsed_ms = elapsed.as_millis(),
1211            processes = process_table.len(),
1212            services = service_registry.len(),
1213            "kernel boot complete"
1214        );
1215
1216        // 9. Create agent supervisor
1217        let supervisor = AgentSupervisor::new(
1218            process_table.clone(),
1219            ipc.clone(),
1220            AgentCapabilities::default(),
1221        );
1222
1223        // 9b. Wire A2ARouter and cron into supervisor
1224        let supervisor = supervisor.with_a2a_router(a2a_router.clone(), cron_svc.clone());
1225
1226        // 9c. Wire exochain managers into supervisor
1227        #[cfg(feature = "exochain")]
1228        let supervisor = supervisor.with_exochain(
1229            tree_manager.clone(),
1230            chain_manager.clone(),
1231        );
1232
1233        // 10. Seed the event ring buffer with boot events
1234        let event_log = Arc::new(KernelEventLog::new());
1235        event_log.ingest_boot_log(&boot_log);
1236
1237        Ok(Self {
1238            state: KernelState::Running,
1239            config: kernel_config,
1240            app_context: Some(app_context),
1241            bus,
1242            process_table,
1243            service_registry,
1244            ipc,
1245            a2a_router,
1246            cron_service: cron_svc,
1247            health,
1248            supervisor,
1249            boot_log,
1250            event_log,
1251            boot_time,
1252            cluster_membership,
1253            #[cfg(feature = "exochain")]
1254            chain: ChainSubsystem {
1255                chain_manager,
1256                tree_manager,
1257                governance_gate,
1258            },
1259            #[cfg(feature = "ecc")]
1260            ecc: EccSubsystem {
1261                hnsw: ecc_hnsw,
1262                causal: ecc_causal,
1263                tick: ecc_tick,
1264                crossrefs: ecc_crossrefs,
1265                impulses: ecc_impulses,
1266                calibration: ecc_calibration,
1267            },
1268            #[cfg(feature = "os-patterns")]
1269            observability: ObservabilitySubsystem {
1270                metrics_registry,
1271                log_service: log_svc,
1272                timer_service: timer_svc,
1273                dead_letter_queue,
1274            },
1275        })
1276    }
1277
1278    /// Shut down the kernel gracefully.
1279    ///
1280    /// Stops all services, cancels all processes, and transitions
1281    /// to the `Halted` state.
1282    pub async fn shutdown(&mut self) -> KernelResult<()> {
1283        if self.state != KernelState::Running {
1284            return Err(KernelError::WrongState {
1285                expected: "Running".into(),
1286                actual: self.state.to_string(),
1287            });
1288        }
1289
1290        info!("kernel shutting down");
1291        self.state = KernelState::ShuttingDown;
1292        self.event_log.info("kernel", "shutdown initiated");
1293
1294        // Stop all services
1295        if let Err(e) = self.service_registry.stop_all().await {
1296            error!(error = %e, "error stopping services during shutdown");
1297        }
1298
1299        // Checkpoint tree+chain before shutting down
1300        #[cfg(feature = "exochain")]
1301        if let Some(ref tm) = self.chain.tree_manager
1302            && let Some(ref cm) = self.chain.chain_manager
1303        {
1304            let stats = tm.stats();
1305            cm.append(
1306                "kernel",
1307                "shutdown",
1308                Some(serde_json::json!({
1309                    "tree_root_hash": stats.root_hash,
1310                    "chain_seq": cm.sequence(),
1311                    "tree_nodes": stats.node_count,
1312                })),
1313            );
1314        }
1315
1316        // Abort all running agent tasks
1317        self.supervisor.abort_all();
1318
1319        // Cancel all processes
1320        for entry in self.process_table.list() {
1321            if entry.pid == 0 {
1322                continue; // Don't cancel the kernel process
1323            }
1324            entry.cancel_token.cancel();
1325
1326            // Log agent stop in tree/chain
1327            #[cfg(feature = "exochain")]
1328            if let Some(ref tm) = self.chain.tree_manager {
1329                let _ = tm.unregister_agent(&entry.agent_id, entry.pid, 0);
1330            }
1331
1332            let _ = self
1333                .process_table
1334                .update_state(entry.pid, ProcessState::Exited(0));
1335        }
1336
1337        // Persist chain to RVF checkpoint (primary), JSON as fallback
1338        #[cfg(feature = "exochain")]
1339        if let Some(ref cm) = self.chain.chain_manager {
1340            let chain_config = self.config.chain.clone().unwrap_or_default();
1341            if let Some(ref ckpt_path) = chain_config.effective_checkpoint_path() {
1342            let json_path = std::path::PathBuf::from(ckpt_path);
1343            let rvf_path = json_path.with_extension("rvf");
1344
1345            // Save RVF format (primary)
1346            match cm.save_to_rvf(&rvf_path) {
1347                Ok(()) => info!(path = %rvf_path.display(), "chain saved to RVF checkpoint"),
1348                Err(e) => {
1349                    error!(error = %e, "failed to save RVF checkpoint, falling back to JSON");
1350                    // Fallback: save JSON
1351                    match cm.save_to_file(&json_path) {
1352                        Ok(()) => info!(path = %json_path.display(), "chain saved to JSON checkpoint (fallback)"),
1353                        Err(e2) => error!(error = %e2, "failed to save JSON checkpoint fallback"),
1354                    }
1355                }
1356            }
1357
1358            // Save tree checkpoint alongside chain
1359            if let Some(ref tm) = self.chain.tree_manager {
1360                let tree_path = json_path.with_extension("tree.json");
1361                match tm.save_checkpoint(&tree_path) {
1362                    Ok(()) => {
1363                        info!(path = %tree_path.display(), "tree checkpoint saved");
1364                        cm.append(
1365                            "tree",
1366                            "tree.checkpoint",
1367                            Some(serde_json::json!({
1368                                "path": tree_path.display().to_string(),
1369                                "root_hash": tm.stats().root_hash,
1370                            })),
1371                        );
1372                    }
1373                    Err(e) => error!(error = %e, "failed to save tree checkpoint"),
1374                }
1375            }
1376            }
1377        }
1378
1379        self.state = KernelState::Halted;
1380        self.event_log.info("kernel", "halted");
1381        info!("kernel halted");
1382        Ok(())
1383    }
1384
1385    /// Get the current kernel state.
1386    pub fn state(&self) -> &KernelState {
1387        &self.state
1388    }
1389
1390    /// Get the kernel configuration.
1391    pub fn kernel_config(&self) -> &KernelConfig {
1392        &self.config
1393    }
1394
1395    /// Get the process table.
1396    pub fn process_table(&self) -> &Arc<ProcessTable> {
1397        &self.process_table
1398    }
1399
1400    /// Get the service registry.
1401    pub fn services(&self) -> &Arc<ServiceRegistry> {
1402        &self.service_registry
1403    }
1404
1405    /// Get the IPC subsystem.
1406    pub fn ipc(&self) -> &Arc<KernelIpc> {
1407        &self.ipc
1408    }
1409
1410    /// Get the message bus.
1411    pub fn bus(&self) -> &Arc<MessageBus> {
1412        &self.bus
1413    }
1414
1415    /// Get the A2A router.
1416    pub fn a2a_router(&self) -> &Arc<A2ARouter> {
1417        &self.a2a_router
1418    }
1419
1420    /// Get the cron service.
1421    pub fn cron_service(&self) -> &Arc<crate::cron::CronService> {
1422        &self.cron_service
1423    }
1424
1425    /// Get the health system.
1426    pub fn health(&self) -> &HealthSystem {
1427        &self.health
1428    }
1429
1430    /// Get the agent supervisor.
1431    pub fn supervisor(&self) -> &AgentSupervisor<P> {
1432        &self.supervisor
1433    }
1434
1435    /// Get the boot log.
1436    pub fn boot_log(&self) -> &BootLog {
1437        &self.boot_log
1438    }
1439
1440    /// Get the runtime event log (ring buffer).
1441    pub fn event_log(&self) -> &Arc<KernelEventLog> {
1442        &self.event_log
1443    }
1444
1445    /// Get kernel uptime.
1446    pub fn uptime(&self) -> std::time::Duration {
1447        self.boot_time.elapsed()
1448    }
1449
1450    /// Get the cluster membership tracker.
1451    pub fn cluster_membership(&self) -> &Arc<ClusterMembership> {
1452        &self.cluster_membership
1453    }
1454
1455    /// Get the local chain manager (when exochain feature is enabled).
1456    #[cfg(feature = "exochain")]
1457    pub fn chain_manager(&self) -> Option<&Arc<crate::chain::ChainManager>> {
1458        self.chain.chain_manager.as_ref()
1459    }
1460
1461    /// Get the tree manager (when exochain feature is enabled).
1462    #[cfg(feature = "exochain")]
1463    pub fn tree_manager(&self) -> Option<&Arc<crate::tree_manager::TreeManager>> {
1464        self.chain.tree_manager.as_ref()
1465    }
1466
1467    /// Get the governance gate backend (if configured).
1468    #[cfg(feature = "exochain")]
1469    pub fn governance_gate(&self) -> Option<&Arc<dyn crate::gate::GateBackend>> {
1470        self.chain.governance_gate.as_ref()
1471    }
1472
1473    /// Get the ECC HNSW service (if ecc feature enabled).
1474    #[cfg(feature = "ecc")]
1475    pub fn ecc_hnsw(&self) -> Option<&Arc<crate::hnsw_service::HnswService>> {
1476        self.ecc.hnsw.as_ref()
1477    }
1478
1479    /// Get the ECC causal graph (if ecc feature enabled).
1480    #[cfg(feature = "ecc")]
1481    pub fn ecc_causal(&self) -> Option<&Arc<crate::causal::CausalGraph>> {
1482        self.ecc.causal.as_ref()
1483    }
1484
1485    /// Get the ECC cognitive tick (if ecc feature enabled).
1486    #[cfg(feature = "ecc")]
1487    pub fn ecc_tick(&self) -> Option<&Arc<crate::cognitive_tick::CognitiveTick>> {
1488        self.ecc.tick.as_ref()
1489    }
1490
1491    /// Get the ECC calibration results (if ecc feature enabled).
1492    #[cfg(feature = "ecc")]
1493    pub fn ecc_calibration(&self) -> Option<&crate::calibration::EccCalibration> {
1494        self.ecc.calibration.as_ref()
1495    }
1496
1497    /// Get the ECC cross-reference store (if ecc feature enabled).
1498    #[cfg(feature = "ecc")]
1499    pub fn ecc_crossrefs(&self) -> Option<&Arc<crate::crossref::CrossRefStore>> {
1500        self.ecc.crossrefs.as_ref()
1501    }
1502
1503    /// Get the ECC impulse queue (if ecc feature enabled).
1504    #[cfg(feature = "ecc")]
1505    pub fn ecc_impulses(&self) -> Option<&Arc<crate::impulse::ImpulseQueue>> {
1506        self.ecc.impulses.as_ref()
1507    }
1508
1509    /// Get the os-patterns metrics registry (if os-patterns feature enabled).
1510    #[cfg(feature = "os-patterns")]
1511    pub fn metrics_registry(&self) -> Option<&Arc<crate::metrics::MetricsRegistry>> {
1512        self.observability.metrics_registry.as_ref()
1513    }
1514
1515    /// Get the os-patterns log service (if os-patterns feature enabled).
1516    #[cfg(feature = "os-patterns")]
1517    pub fn log_service(&self) -> Option<&Arc<crate::log_service::LogService>> {
1518        self.observability.log_service.as_ref()
1519    }
1520
1521    /// Get the os-patterns timer service (if os-patterns feature enabled).
1522    #[cfg(feature = "os-patterns")]
1523    pub fn timer_service(&self) -> Option<&Arc<crate::timer::TimerService>> {
1524        self.observability.timer_service.as_ref()
1525    }
1526
1527    /// Get the os-patterns dead letter queue (if os-patterns feature enabled).
1528    #[cfg(feature = "os-patterns")]
1529    pub fn dead_letter_queue(&self) -> Option<&Arc<crate::dead_letter::DeadLetterQueue>> {
1530        self.observability.dead_letter_queue.as_ref()
1531    }
1532
1533    /// Take ownership of the AppContext for agent loop consumption.
1534    ///
1535    /// This is a one-shot operation: after calling this, the kernel
1536    /// no longer holds the AppContext. Use this before calling
1537    /// `AppContext::into_agent_loop()`.
1538    pub fn take_app_context(&mut self) -> Option<AppContext<P>> {
1539        self.app_context.take()
1540    }
1541}
1542
1543#[cfg(test)]
1544mod tests {
1545    use super::*;
1546    use clawft_platform::NativePlatform;
1547    use clawft_types::config::{AgentDefaults, AgentsConfig};
1548
1549    fn test_config() -> Config {
1550        Config {
1551            agents: AgentsConfig {
1552                defaults: AgentDefaults {
1553                    workspace: "~/.clawft/workspace".into(),
1554                    model: "test/model".into(),
1555                    max_tokens: 1024,
1556                    temperature: 0.5,
1557                    max_tool_iterations: 5,
1558                    memory_window: 10,
1559                },
1560            },
1561            ..Config::default()
1562        }
1563    }
1564
1565    fn test_kernel_config() -> KernelConfig {
1566        KernelConfig {
1567            enabled: true,
1568            max_processes: 16,
1569            health_check_interval_secs: 5,
1570            cluster: None,
1571            chain: None,
1572            resource_tree: None,
1573        }
1574    }
1575
1576    #[tokio::test]
1577    async fn boot_and_shutdown() {
1578        let platform = Arc::new(NativePlatform::new());
1579        let mut kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
1580            .await
1581            .unwrap();
1582
1583        assert_eq!(*kernel.state(), KernelState::Running);
1584        // Uptime should be non-negative (boot_time is set before boot completes)
1585        let _uptime = kernel.uptime();
1586
1587        // Kernel process should be PID 0
1588        let kernel_proc = kernel.process_table().get(0).unwrap();
1589        assert_eq!(kernel_proc.agent_id, "kernel");
1590        assert_eq!(kernel_proc.state, ProcessState::Running);
1591
1592        kernel.shutdown().await.unwrap();
1593        assert_eq!(*kernel.state(), KernelState::Halted);
1594    }
1595
1596    #[tokio::test]
1597    async fn double_shutdown_fails() {
1598        let platform = Arc::new(NativePlatform::new());
1599        let mut kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
1600            .await
1601            .unwrap();
1602
1603        kernel.shutdown().await.unwrap();
1604        let result = kernel.shutdown().await;
1605        assert!(result.is_err());
1606    }
1607
1608    #[tokio::test]
1609    async fn boot_log_has_events() {
1610        let platform = Arc::new(NativePlatform::new());
1611        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
1612            .await
1613            .unwrap();
1614
1615        let log = kernel.boot_log();
1616        assert!(!log.is_empty());
1617
1618        let formatted = log.format_all();
1619        assert!(formatted.contains("WeftOS v0.1.0"));
1620        assert!(formatted.contains("Boot complete"));
1621    }
1622
1623    #[tokio::test]
1624    async fn process_table_accessible() {
1625        let platform = Arc::new(NativePlatform::new());
1626        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
1627            .await
1628            .unwrap();
1629
1630        let pt = kernel.process_table();
1631        assert_eq!(pt.len(), 1); // kernel process only
1632        assert_eq!(pt.max_processes(), 16);
1633    }
1634
1635    #[tokio::test]
1636    async fn services_accessible() {
1637        let platform = Arc::new(NativePlatform::new());
1638        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
1639            .await
1640            .unwrap();
1641
1642        // cron + containers + cluster registered at boot; + hnsw + cognitive_tick with ecc
1643        let count = kernel.services().len();
1644        #[cfg(all(feature = "ecc", feature = "cluster"))]
1645        assert_eq!(count, 5, "expected cron+containers+cluster+hnsw+cognitive_tick");
1646        #[cfg(all(feature = "ecc", not(feature = "cluster")))]
1647        assert_eq!(count, 4, "expected cron+containers+hnsw+cognitive_tick");
1648        #[cfg(all(not(feature = "ecc"), feature = "cluster"))]
1649        assert_eq!(count, 3, "expected cron+containers+cluster");
1650        #[cfg(all(not(feature = "ecc"), not(feature = "cluster")))]
1651        assert_eq!(count, 2, "expected cron+containers");
1652    }
1653
1654    #[tokio::test]
1655    async fn take_app_context() {
1656        let platform = Arc::new(NativePlatform::new());
1657        let mut kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
1658            .await
1659            .unwrap();
1660
1661        let ctx = kernel.take_app_context();
1662        assert!(ctx.is_some());
1663
1664        // Second take returns None
1665        let ctx2 = kernel.take_app_context();
1666        assert!(ctx2.is_none());
1667    }
1668
1669    #[tokio::test]
1670    async fn ipc_accessible() {
1671        let platform = Arc::new(NativePlatform::new());
1672        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
1673            .await
1674            .unwrap();
1675
1676        let ipc = kernel.ipc();
1677        assert!(Arc::ptr_eq(ipc.bus(), kernel.bus()));
1678    }
1679
1680    #[test]
1681    fn kernel_state_display() {
1682        assert_eq!(KernelState::Booting.to_string(), "booting");
1683        assert_eq!(KernelState::Running.to_string(), "running");
1684        assert_eq!(KernelState::ShuttingDown.to_string(), "shutting_down");
1685        assert_eq!(KernelState::Halted.to_string(), "halted");
1686    }
1687
1688    // ── Full-stack integration helpers ─────────────────────────────
1689
1690    /// Kernel config with exochain + resource tree enabled (no checkpoint
1691    /// path so everything stays in-memory).
1692    #[cfg(all(feature = "exochain", feature = "ecc", feature = "wasm-sandbox"))]
1693    fn test_kernel_config_full_stack() -> KernelConfig {
1694        use clawft_types::config::{ChainConfig, ResourceTreeConfig};
1695        KernelConfig {
1696            enabled: true,
1697            max_processes: 32,
1698            health_check_interval_secs: 5,
1699            cluster: None,
1700            chain: Some(ChainConfig {
1701                enabled: true,
1702                checkpoint_interval: 10_000,
1703                chain_id: 0,
1704                checkpoint_path: None,
1705            }),
1706            resource_tree: Some(ResourceTreeConfig {
1707                enabled: true,
1708                checkpoint_path: None,
1709            }),
1710        }
1711    }
1712
1713    // ── Integration: full-stack kernel ──────────────────────────────
1714
1715    /// K0-K5 integration: native agent + WASM tool + container service + ECC
1716    /// all connected in a single kernel with chain witnessing.
1717    #[tokio::test]
1718    #[cfg(all(feature = "exochain", feature = "ecc", feature = "wasm-sandbox"))]
1719    async fn integration_full_stack_kernel() {
1720        let platform = Arc::new(NativePlatform::new());
1721        let kernel = Kernel::boot(test_config(), test_kernel_config_full_stack(), platform)
1722            .await
1723            .unwrap();
1724
1725        // ── K0: Kernel is running ──────────────────────────────────
1726        assert_eq!(*kernel.state(), KernelState::Running);
1727
1728        // ── K0: Services registered at boot ────────────────────────
1729        // cron + container + hnsw + cognitive_tick = 4 with ecc
1730        let svc_count = kernel.services().len();
1731        assert!(svc_count >= 4, "expected >= 4 services, got {svc_count}");
1732
1733        // ── K1: Spawn a native agent ───────────────────────────────
1734        let spawn_result = kernel.supervisor().spawn(
1735            crate::supervisor::SpawnRequest {
1736                agent_id: "integration-agent-1".into(),
1737                capabilities: None,
1738                parent_pid: None,
1739                env: std::collections::HashMap::new(),
1740                backend: None, // defaults to Native
1741            },
1742        );
1743        assert!(spawn_result.is_ok(), "native agent spawn failed: {:?}", spawn_result.err());
1744        let agent_pid = spawn_result.unwrap().pid;
1745        assert!(agent_pid > 0, "agent should get PID > 0");
1746
1747        // ── K1: Agent appears in process table ─────────────────────
1748        let processes = kernel.process_table().list();
1749        assert!(
1750            processes.iter().any(|p| p.pid == agent_pid),
1751            "agent not in process table"
1752        );
1753
1754        // ── K2: A2A messaging between kernel and agent ─────────────
1755        let a2a = kernel.a2a_router();
1756        let _inbox = a2a.create_inbox(agent_pid);
1757
1758        // Send a message from kernel (PID 0) to agent
1759        let msg = crate::ipc::KernelMessage::new(
1760            0, // from kernel
1761            crate::ipc::MessageTarget::Process(agent_pid),
1762            crate::ipc::MessagePayload::Json(serde_json::json!({"cmd": "ping"})),
1763        );
1764        let send_result = a2a.send(msg).await;
1765        // This should succeed: PID 0 exists, agent inbox exists
1766        assert!(send_result.is_ok(), "A2A send failed: {:?}", send_result.err());
1767
1768        // ── K3: WASM tool execution ────────────────────────────────
1769        let wasm_config = crate::wasm_runner::WasmSandboxConfig::default();
1770        let wasm_runner = crate::wasm_runner::WasmToolRunner::new(wasm_config);
1771
1772        // Minimal WAT module that exports _start and immediately returns
1773        let noop_wat = r#"(module (func (export "_start")))"#;
1774        let result = wasm_runner
1775            .execute_bytes("integration-tool", noop_wat.as_bytes(), serde_json::json!({}))
1776            .await;
1777        assert!(result.is_ok(), "WASM execution failed: {:?}", result.err());
1778        let wasm_result = result.unwrap();
1779        assert_eq!(wasm_result.exit_code, 0, "WASM tool should exit cleanly");
1780        assert!(wasm_result.fuel_consumed > 0, "WASM should consume fuel");
1781
1782        // ── K3: WASM tool in registry ──────────────────────────────
1783        let wasm_runner_arc = Arc::new(
1784            crate::wasm_runner::WasmToolRunner::new(
1785                crate::wasm_runner::WasmSandboxConfig::default(),
1786            ),
1787        );
1788        let mut registry = crate::wasm_runner::ToolRegistry::new();
1789        registry
1790            .register_wasm_tool(
1791                "demo-tool",
1792                "A demo WASM tool for integration testing",
1793                noop_wat.as_bytes().to_vec(),
1794                wasm_runner_arc,
1795            )
1796            .unwrap();
1797        assert!(registry.get("demo-tool").is_some(), "WASM tool should be in registry");
1798        let tool_list = registry.list();
1799        assert!(
1800            tool_list.contains(&"demo-tool".to_string()),
1801            "WASM tool should appear in listing"
1802        );
1803
1804        // ── K4: Container service registered ───────────────────────
1805        let svc_list = kernel.services().list();
1806        assert!(
1807            svc_list.iter().any(|(name, _)| name == "containers"),
1808            "container service should be registered"
1809        );
1810
1811        // ── K3c: ECC cognitive substrate active ────────────────────
1812        let ecc_hnsw = kernel.ecc_hnsw().expect("HNSW service should be present");
1813        // After calibration the store is cleared, so it should be empty
1814        assert_eq!(ecc_hnsw.len(), 0, "HNSW should be empty after calibration cleanup");
1815
1816        let ecc_causal = kernel.ecc_causal().expect("causal graph should be present");
1817        assert_eq!(ecc_causal.node_count(), 0, "causal graph should be empty after cleanup");
1818
1819        let _ecc_tick = kernel.ecc_tick().expect("cognitive tick should be present");
1820
1821        let calibration = kernel.ecc_calibration().expect("calibration should exist");
1822        assert!(calibration.compute_p95_us > 0, "calibration should have run");
1823        assert!(calibration.tick_interval_ms > 0, "tick interval should be auto-calibrated");
1824
1825        // ── ECC: Insert a vector and search ────────────────────────
1826        ecc_hnsw.insert(
1827            "test-doc".into(),
1828            vec![1.0, 0.0, 0.0, 0.0],
1829            serde_json::json!({"text": "hello"}),
1830        );
1831        assert_eq!(ecc_hnsw.len(), 1);
1832        let results = ecc_hnsw.search(&[1.0, 0.0, 0.0, 0.0], 5);
1833        assert_eq!(results.len(), 1);
1834        assert_eq!(results[0].id, "test-doc");
1835
1836        // ── ECC: Causal graph operations ───────────────────────────
1837        let node1 = ecc_causal.add_node("concept-A".into(), serde_json::json!({}));
1838        let node2 = ecc_causal.add_node("concept-B".into(), serde_json::json!({}));
1839        ecc_causal.link(
1840            node1,
1841            node2,
1842            crate::causal::CausalEdgeType::Causes,
1843            1.0,
1844            0,
1845            0,
1846        );
1847        assert_eq!(ecc_causal.node_count(), 2);
1848        assert_eq!(ecc_causal.edge_count(), 1);
1849        let path = ecc_causal.find_path(node1, node2, 5);
1850        assert!(path.is_some(), "should find path between linked nodes");
1851
1852        // ── ExoChain: Chain has witnessed boot events ──────────────
1853        let chain = kernel.chain_manager().expect("chain should exist");
1854        assert!(
1855            chain.sequence() > 5,
1856            "chain should have boot events, got seq={}",
1857            chain.sequence()
1858        );
1859
1860        // Verify chain integrity
1861        let verify_result = chain.verify_integrity();
1862        assert!(verify_result.valid, "chain should be valid: {:?}", verify_result.errors);
1863        assert!(verify_result.errors.is_empty(), "no integrity errors");
1864
1865        // ── Governance: genesis rules anchored to chain ────────────
1866        let _governance = kernel.governance_gate().expect("governance gate should exist");
1867        let all_events = chain.tail(0);
1868        let genesis_events: Vec<_> = all_events.iter()
1869            .filter(|e| e.kind == "governance.genesis")
1870            .collect();
1871        assert!(!genesis_events.is_empty(), "governance genesis should be on chain");
1872
1873        // Verify the genesis payload contains the correct rule count
1874        // Find the kernel's own genesis event (version 2.0.0)
1875        let genesis_payload = genesis_events.iter()
1876            .filter_map(|e| e.payload.as_ref())
1877            .find(|p| p.get("version").and_then(|v| v.as_str()) == Some("2.0.0"))
1878            .expect("should find v2.0.0 governance genesis on chain");
1879        assert_eq!(
1880            genesis_payload["rule_count"].as_u64().unwrap(),
1881            22,
1882            "genesis should contain 22 rules"
1883        );
1884        assert_eq!(
1885            genesis_payload["version"].as_str().unwrap(),
1886            "2.0.0",
1887            "genesis version should be 2.0.0"
1888        );
1889
1890        // Each rule should be individually anchored (at least 22)
1891        let rule_events: Vec<_> = all_events.iter()
1892            .filter(|e| e.kind == "governance.rule")
1893            .collect();
1894        assert!(
1895            rule_events.len() >= 22,
1896            "at least 22 genesis rules should be individually anchored, got {}",
1897            rule_events.len(),
1898        );
1899
1900        // Verify all rule IDs are present (GOV-001..007 + SOP-L/E/J)
1901        let rule_ids: Vec<&str> = rule_events.iter()
1902            .filter_map(|e| e.payload.as_ref()?.get("rule_id")?.as_str())
1903            .collect();
1904        for expected_id in &[
1905            "GOV-001", "GOV-002", "GOV-003", "GOV-004", "GOV-005", "GOV-006", "GOV-007",
1906            "SOP-L001", "SOP-L002", "SOP-L003", "SOP-L004", "SOP-L005", "SOP-L006",
1907            "SOP-E001", "SOP-E002", "SOP-E003", "SOP-E004", "SOP-E005",
1908            "SOP-J001", "SOP-J002", "SOP-J003", "SOP-J004",
1909        ] {
1910            assert!(
1911                rule_ids.contains(expected_id),
1912                "{expected_id} should be anchored on chain"
1913            );
1914        }
1915
1916        // ── ExoChain: ECC calibration event logged ─────────────────
1917        let ecc_events: Vec<_> = all_events.iter().filter(|e| e.kind.starts_with("ecc.")).collect();
1918        assert!(
1919            !ecc_events.is_empty(),
1920            "ECC boot calibration should be logged to chain"
1921        );
1922
1923        // ── Resource Tree: namespaces exist ────────────────────────
1924        let tree = kernel.tree_manager().expect("tree should exist");
1925        let stats = tree.stats();
1926        assert!(
1927            stats.node_count > 20,
1928            "tree should have many nodes (got {})",
1929            stats.node_count
1930        );
1931
1932        // ── K2: ServiceApi + adapters compile ──────────────────────
1933        // (validated by service::tests; here we just confirm registry is queryable)
1934        assert!(kernel.services().get("cron").is_some(), "cron service should be accessible");
1935        assert!(
1936            kernel.services().get("containers").is_some(),
1937            "container service should be accessible"
1938        );
1939
1940        // ── Graceful shutdown ──────────────────────────────────────
1941        let mut kernel = kernel;
1942        kernel.shutdown().await.unwrap();
1943        assert_eq!(*kernel.state(), KernelState::Halted);
1944    }
1945
1946    // ── Integration: cross-backend tools ───────────────────────────
1947
1948    /// Demonstrates that native (built-in) tools and WASM tools coexist
1949    /// in a single ToolRegistry and can be dispatched through the same
1950    /// lookup interface.
1951    #[test]
1952    #[cfg(feature = "wasm-sandbox")]
1953    fn integration_cross_backend_tools() {
1954        use crate::wasm_runner::{
1955            BuiltinTool, BuiltinToolSpec, ToolCategory, ToolError, ToolRegistry,
1956            WasmSandboxConfig, WasmToolRunner,
1957        };
1958        use crate::governance::EffectVector;
1959
1960        // ── A native (Rust) tool ───────────────────────────────────
1961        struct EchoTool;
1962
1963        impl BuiltinTool for EchoTool {
1964            fn name(&self) -> &str {
1965                "native.echo"
1966            }
1967            fn spec(&self) -> &BuiltinToolSpec {
1968                // Leak a static spec for test simplicity
1969                static SPEC: std::sync::OnceLock<BuiltinToolSpec> = std::sync::OnceLock::new();
1970                SPEC.get_or_init(|| BuiltinToolSpec {
1971                    name: "native.echo".into(),
1972                    category: ToolCategory::System,
1973                    description: "Echoes input back".into(),
1974                    parameters: serde_json::json!({}),
1975                    gate_action: "tool.native.echo".into(),
1976                    effect: EffectVector::default(),
1977                    native: true,
1978                })
1979            }
1980            fn execute(&self, args: serde_json::Value) -> Result<serde_json::Value, ToolError> {
1981                Ok(serde_json::json!({"echo": args}))
1982            }
1983        }
1984
1985        // ── Build a mixed registry ─────────────────────────────────
1986        let mut registry = ToolRegistry::new();
1987
1988        // Register native tool
1989        registry.register(Arc::new(EchoTool));
1990
1991        // Register WASM tool
1992        let runner = Arc::new(WasmToolRunner::new(WasmSandboxConfig::default()));
1993        let noop_wat = r#"(module (func (export "_start")))"#;
1994        registry
1995            .register_wasm_tool(
1996                "wasm.noop",
1997                "A no-op WASM tool",
1998                noop_wat.as_bytes().to_vec(),
1999                runner,
2000            )
2001            .unwrap();
2002
2003        // ── Both tools accessible through one registry ─────────────
2004        assert_eq!(registry.list().len(), 2);
2005
2006        let native = registry.get("native.echo").expect("native tool should exist");
2007        assert!(native.spec().native, "native tool should be marked native");
2008
2009        let wasm = registry.get("wasm.noop").expect("wasm tool should exist");
2010        assert!(!wasm.spec().native, "wasm tool should NOT be marked native");
2011
2012        // ── Native tool executes synchronously ─────────────────────
2013        let result = native.execute(serde_json::json!({"hello": "world"})).unwrap();
2014        assert_eq!(result["echo"]["hello"], "world");
2015
2016        // ── Hierarchical registry ──────────────────────────────────
2017        // Child registry overlays parent; WASM tools from parent visible
2018        let parent = Arc::new(registry);
2019        let mut child = ToolRegistry::with_parent(parent);
2020
2021        // Child sees parent tools
2022        assert!(child.get("native.echo").is_some(), "child sees parent native tool");
2023        assert!(child.get("wasm.noop").is_some(), "child sees parent wasm tool");
2024
2025        // Child can shadow
2026        struct OverrideTool;
2027        impl BuiltinTool for OverrideTool {
2028            fn name(&self) -> &str {
2029                "native.echo"
2030            }
2031            fn spec(&self) -> &BuiltinToolSpec {
2032                static SPEC: std::sync::OnceLock<BuiltinToolSpec> = std::sync::OnceLock::new();
2033                SPEC.get_or_init(|| BuiltinToolSpec {
2034                    name: "native.echo".into(),
2035                    category: ToolCategory::System,
2036                    description: "Overridden echo".into(),
2037                    parameters: serde_json::json!({}),
2038                    gate_action: "tool.native.echo".into(),
2039                    effect: EffectVector::default(),
2040                    native: true,
2041                })
2042            }
2043            fn execute(&self, _args: serde_json::Value) -> Result<serde_json::Value, ToolError> {
2044                Ok(serde_json::json!({"overridden": true}))
2045            }
2046        }
2047
2048        child.register(Arc::new(OverrideTool));
2049        let result = child.get("native.echo").unwrap().execute(serde_json::json!({})).unwrap();
2050        assert_eq!(result["overridden"], true, "child should shadow parent tool");
2051
2052        // Parent WASM tool still reachable from child
2053        assert!(child.get("wasm.noop").is_some(), "WASM tool still reachable via parent");
2054    }
2055
2056    // ── Boot path coverage tests ─────────────────────────────────
2057
2058    #[tokio::test]
2059    async fn boot_reaches_running_state() {
2060        let platform = Arc::new(NativePlatform::new());
2061        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2062            .await
2063            .unwrap();
2064        assert_eq!(*kernel.state(), KernelState::Running);
2065    }
2066
2067    #[tokio::test]
2068    async fn boot_registers_cron_service() {
2069        let platform = Arc::new(NativePlatform::new());
2070        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2071            .await
2072            .unwrap();
2073        assert!(
2074            kernel.services().get("cron").is_some(),
2075            "cron service must be registered at boot"
2076        );
2077    }
2078
2079    #[tokio::test]
2080    async fn boot_registers_container_service() {
2081        let platform = Arc::new(NativePlatform::new());
2082        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2083            .await
2084            .unwrap();
2085        assert!(
2086            kernel.services().get("containers").is_some(),
2087            "container service must be registered at boot"
2088        );
2089    }
2090
2091    #[cfg(feature = "exochain")]
2092    fn test_kernel_config_exochain() -> KernelConfig {
2093        use clawft_types::config::{ChainConfig, ResourceTreeConfig};
2094        KernelConfig {
2095            enabled: true,
2096            max_processes: 16,
2097            health_check_interval_secs: 5,
2098            cluster: None,
2099            chain: Some(ChainConfig {
2100                enabled: true,
2101                checkpoint_interval: 10_000,
2102                chain_id: 0,
2103                checkpoint_path: None,
2104            }),
2105            resource_tree: Some(ResourceTreeConfig {
2106                enabled: true,
2107                checkpoint_path: None,
2108            }),
2109        }
2110    }
2111
2112    #[cfg(feature = "exochain")]
2113    #[tokio::test]
2114    async fn boot_exochain_creates_chain_manager() {
2115        let platform = Arc::new(NativePlatform::new());
2116        let kernel = Kernel::boot(test_config(), test_kernel_config_exochain(), platform)
2117            .await
2118            .unwrap();
2119        assert!(
2120            kernel.chain_manager().is_some(),
2121            "chain manager must be present with exochain feature and enabled chain config"
2122        );
2123    }
2124
2125    #[cfg(feature = "exochain")]
2126    #[tokio::test]
2127    async fn boot_exochain_creates_tree_manager() {
2128        let platform = Arc::new(NativePlatform::new());
2129        let kernel = Kernel::boot(test_config(), test_kernel_config_exochain(), platform)
2130            .await
2131            .unwrap();
2132        assert!(
2133            kernel.tree_manager().is_some(),
2134            "tree manager must be present with exochain feature and enabled config"
2135        );
2136    }
2137
2138    #[cfg(feature = "exochain")]
2139    #[tokio::test]
2140    async fn boot_exochain_chain_has_boot_events() {
2141        let platform = Arc::new(NativePlatform::new());
2142        let kernel = Kernel::boot(test_config(), test_kernel_config_exochain(), platform)
2143            .await
2144            .unwrap();
2145        let chain = kernel.chain_manager().unwrap();
2146        // Chain should have boot.init, boot.config, boot.services, boot.cluster, boot.ready, boot.manifest at minimum
2147        assert!(
2148            chain.sequence() >= 6,
2149            "chain should have at least 6 boot events, got {}",
2150            chain.sequence()
2151        );
2152    }
2153
2154    #[cfg(feature = "exochain")]
2155    #[tokio::test]
2156    async fn boot_exochain_governance_gate_present() {
2157        let platform = Arc::new(NativePlatform::new());
2158        let kernel = Kernel::boot(test_config(), test_kernel_config_exochain(), platform)
2159            .await
2160            .unwrap();
2161        assert!(
2162            kernel.governance_gate().is_some(),
2163            "governance gate should be present when chain is enabled"
2164        );
2165    }
2166
2167    #[cfg(feature = "ecc")]
2168    #[tokio::test]
2169    async fn boot_ecc_registers_hnsw_service() {
2170        let platform = Arc::new(NativePlatform::new());
2171        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2172            .await
2173            .unwrap();
2174        assert!(
2175            kernel.ecc_hnsw().is_some(),
2176            "HNSW service must be present with ecc feature"
2177        );
2178    }
2179
2180    #[cfg(feature = "ecc")]
2181    #[tokio::test]
2182    async fn boot_ecc_registers_cognitive_tick() {
2183        let platform = Arc::new(NativePlatform::new());
2184        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2185            .await
2186            .unwrap();
2187        assert!(
2188            kernel.ecc_tick().is_some(),
2189            "cognitive tick must be present with ecc feature"
2190        );
2191    }
2192
2193    #[cfg(feature = "ecc")]
2194    #[tokio::test]
2195    async fn boot_ecc_calibration_has_valid_results() {
2196        let platform = Arc::new(NativePlatform::new());
2197        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2198            .await
2199            .unwrap();
2200        let cal = kernel.ecc_calibration().unwrap();
2201        assert!(cal.tick_interval_ms > 0, "tick interval must be positive");
2202        assert!(cal.compute_p50_us > 0, "p50 latency must be measured");
2203        assert!(cal.compute_p95_us >= cal.compute_p50_us, "p95 >= p50");
2204    }
2205
2206    #[cfg(feature = "ecc")]
2207    #[tokio::test]
2208    async fn boot_ecc_causal_graph_accessible() {
2209        let platform = Arc::new(NativePlatform::new());
2210        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2211            .await
2212            .unwrap();
2213        assert!(kernel.ecc_causal().is_some(), "causal graph must be accessible");
2214        assert_eq!(kernel.ecc_causal().unwrap().node_count(), 0, "causal graph starts empty");
2215    }
2216
2217    #[cfg(feature = "ecc")]
2218    #[tokio::test]
2219    async fn boot_ecc_crossrefs_and_impulses_accessible() {
2220        let platform = Arc::new(NativePlatform::new());
2221        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2222            .await
2223            .unwrap();
2224        assert!(kernel.ecc_crossrefs().is_some(), "cross-ref store must be accessible");
2225        assert!(kernel.ecc_impulses().is_some(), "impulse queue must be accessible");
2226    }
2227
2228    #[tokio::test]
2229    async fn boot_cluster_membership_accessible() {
2230        let platform = Arc::new(NativePlatform::new());
2231        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2232            .await
2233            .unwrap();
2234        let cm = kernel.cluster_membership();
2235        assert!(!cm.local_node_id().is_empty(), "cluster membership should have a node ID");
2236    }
2237
2238    #[tokio::test]
2239    async fn shutdown_transitions_to_halted() {
2240        let platform = Arc::new(NativePlatform::new());
2241        let mut kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2242            .await
2243            .unwrap();
2244        assert_eq!(*kernel.state(), KernelState::Running);
2245        kernel.shutdown().await.unwrap();
2246        assert_eq!(*kernel.state(), KernelState::Halted);
2247    }
2248
2249    #[tokio::test]
2250    async fn shutdown_from_halted_fails() {
2251        let platform = Arc::new(NativePlatform::new());
2252        let mut kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2253            .await
2254            .unwrap();
2255        kernel.shutdown().await.unwrap();
2256        let err = kernel.shutdown().await.unwrap_err();
2257        match err {
2258            KernelError::WrongState { expected, actual } => {
2259                assert_eq!(expected, "Running");
2260                assert_eq!(actual, "halted");
2261            }
2262            other => panic!("expected WrongState, got: {other:?}"),
2263        }
2264    }
2265
2266    #[tokio::test]
2267    async fn process_table_has_kernel_pid_zero() {
2268        let platform = Arc::new(NativePlatform::new());
2269        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2270            .await
2271            .unwrap();
2272        let entry = kernel.process_table().get(0).expect("PID 0 should exist");
2273        assert_eq!(entry.agent_id, "kernel");
2274        assert_eq!(entry.state, ProcessState::Running);
2275        assert_eq!(entry.pid, 0);
2276    }
2277
2278    #[tokio::test]
2279    async fn a2a_router_accessible_after_boot() {
2280        let platform = Arc::new(NativePlatform::new());
2281        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2282            .await
2283            .unwrap();
2284        let a2a = kernel.a2a_router();
2285        // Should be able to create an inbox for a future PID
2286        // (A2ARouter is wired to the same process table)
2287        let _inbox = a2a.create_inbox(0); // kernel PID
2288    }
2289
2290    #[tokio::test]
2291    async fn boot_log_contains_expected_phases() {
2292        let platform = Arc::new(NativePlatform::new());
2293        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2294            .await
2295            .unwrap();
2296        let formatted = kernel.boot_log().format_all();
2297        assert!(formatted.contains("WeftOS v0.1.0"), "should contain version");
2298        assert!(formatted.contains("Service registry ready"), "should have service phase");
2299        assert!(formatted.contains("A2A router ready"), "should have A2A phase");
2300        assert!(formatted.contains("Boot complete"), "should have ready phase");
2301    }
2302
2303    #[tokio::test]
2304    async fn event_log_populated_after_boot() {
2305        let platform = Arc::new(NativePlatform::new());
2306        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2307            .await
2308            .unwrap();
2309        // The event_log is populated from boot_log during boot
2310        let events = kernel.event_log();
2311        // At minimum there should be some events from boot ingestion
2312        assert!(!events.is_empty(), "event log should have entries after boot");
2313    }
2314
2315    #[tokio::test]
2316    async fn health_system_accessible() {
2317        let platform = Arc::new(NativePlatform::new());
2318        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2319            .await
2320            .unwrap();
2321        let _health = kernel.health();
2322        // Health system should be constructable and accessible
2323    }
2324
2325    #[tokio::test]
2326    async fn uptime_is_positive_after_boot() {
2327        let platform = Arc::new(NativePlatform::new());
2328        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2329            .await
2330            .unwrap();
2331        let uptime = kernel.uptime();
2332        assert!(uptime.as_nanos() > 0, "uptime should be positive");
2333    }
2334
2335    #[tokio::test]
2336    async fn cron_service_accessible_and_empty() {
2337        let platform = Arc::new(NativePlatform::new());
2338        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2339            .await
2340            .unwrap();
2341        assert_eq!(kernel.cron_service().job_count(), 0, "no cron jobs at boot");
2342    }
2343
2344    #[tokio::test]
2345    async fn max_processes_matches_config() {
2346        let platform = Arc::new(NativePlatform::new());
2347        let mut kconfig = test_kernel_config();
2348        kconfig.max_processes = 42;
2349        let kernel = Kernel::boot(test_config(), kconfig, platform)
2350            .await
2351            .unwrap();
2352        assert_eq!(kernel.process_table().max_processes(), 42);
2353    }
2354
2355    // ── Sprint 09a: KernelState serde + display tests ────────────
2356
2357    #[test]
2358    fn kernel_state_serde_roundtrip_all_variants() {
2359        for state in [
2360            KernelState::Booting,
2361            KernelState::Running,
2362            KernelState::ShuttingDown,
2363            KernelState::Halted,
2364        ] {
2365            let json = serde_json::to_string(&state).unwrap();
2366            let restored: KernelState = serde_json::from_str(&json).unwrap();
2367            assert_eq!(restored, state);
2368        }
2369    }
2370
2371    #[test]
2372    fn kernel_state_equality() {
2373        assert_eq!(KernelState::Running, KernelState::Running);
2374        assert_ne!(KernelState::Running, KernelState::Halted);
2375    }
2376
2377    #[tokio::test]
2378    async fn kernel_state_is_running_after_boot() {
2379        let platform = Arc::new(NativePlatform::new());
2380        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2381            .await
2382            .unwrap();
2383        assert_eq!(kernel.state(), &KernelState::Running);
2384    }
2385
2386    #[tokio::test]
2387    async fn boot_log_has_entries() {
2388        let platform = Arc::new(NativePlatform::new());
2389        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2390            .await
2391            .unwrap();
2392        let log = kernel.boot_log();
2393        assert!(log.events().len() > 0, "boot log should have events");
2394    }
2395
2396    #[tokio::test]
2397    async fn bus_is_accessible_after_boot() {
2398        let platform = Arc::new(NativePlatform::new());
2399        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2400            .await
2401            .unwrap();
2402        // Bus should exist (used for message passing)
2403        let _bus = kernel.bus();
2404    }
2405
2406    #[tokio::test]
2407    async fn a2a_router_has_inboxes_after_boot() {
2408        let platform = Arc::new(NativePlatform::new());
2409        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2410            .await
2411            .unwrap();
2412        let _router = kernel.a2a_router();
2413        // Router should be accessible after boot
2414    }
2415
2416    #[tokio::test]
2417    async fn ipc_accessible_after_boot() {
2418        let platform = Arc::new(NativePlatform::new());
2419        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2420            .await
2421            .unwrap();
2422        let _ipc = kernel.ipc();
2423    }
2424
2425    // ── W5: Test hardening — additional coverage ─────────────────
2426
2427    #[tokio::test]
2428    async fn supervisor_accessible_after_boot() {
2429        let platform = Arc::new(NativePlatform::new());
2430        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2431            .await
2432            .unwrap();
2433        let _supervisor = kernel.supervisor();
2434    }
2435
2436    #[tokio::test]
2437    async fn kernel_config_accessor_returns_boot_config() {
2438        let platform = Arc::new(NativePlatform::new());
2439        let mut kconfig = test_kernel_config();
2440        kconfig.max_processes = 99;
2441        kconfig.health_check_interval_secs = 42;
2442        let kernel = Kernel::boot(test_config(), kconfig, platform)
2443            .await
2444            .unwrap();
2445        assert_eq!(kernel.kernel_config().max_processes, 99);
2446        assert_eq!(kernel.kernel_config().health_check_interval_secs, 42);
2447    }
2448
2449    #[tokio::test]
2450    async fn shutdown_exits_all_agent_processes() {
2451        let platform = Arc::new(NativePlatform::new());
2452        let mut kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2453            .await
2454            .unwrap();
2455
2456        // Spawn a couple of agents
2457        let spawn1 = kernel.supervisor().spawn(crate::supervisor::SpawnRequest {
2458            agent_id: "test-agent-1".into(),
2459            capabilities: None,
2460            parent_pid: None,
2461            env: std::collections::HashMap::new(),
2462            backend: None,
2463        });
2464        let spawn2 = kernel.supervisor().spawn(crate::supervisor::SpawnRequest {
2465            agent_id: "test-agent-2".into(),
2466            capabilities: None,
2467            parent_pid: None,
2468            env: std::collections::HashMap::new(),
2469            backend: None,
2470        });
2471        assert!(spawn1.is_ok());
2472        assert!(spawn2.is_ok());
2473        let pid1 = spawn1.unwrap().pid;
2474        let pid2 = spawn2.unwrap().pid;
2475
2476        // Verify agents exist in process table
2477        assert!(kernel.process_table().get(pid1).is_some());
2478        assert!(kernel.process_table().get(pid2).is_some());
2479
2480        kernel.shutdown().await.unwrap();
2481
2482        // After shutdown, agent processes should no longer be Running.
2483        // They may be Exited (if the shutdown loop reached them) or still
2484        // Starting (if spawn() never transitioned them to Running because
2485        // no agent loop was attached in this test). Either is acceptable;
2486        // the key assertion is that shutdown completed without error.
2487        if let Some(entry1) = kernel.process_table().get(pid1) {
2488            assert!(
2489                !matches!(entry1.state, ProcessState::Running),
2490                "agent 1 should not be Running after shutdown, got: {}",
2491                entry1.state
2492            );
2493        }
2494        if let Some(entry2) = kernel.process_table().get(pid2) {
2495            assert!(
2496                !matches!(entry2.state, ProcessState::Running),
2497                "agent 2 should not be Running after shutdown, got: {}",
2498                entry2.state
2499            );
2500        }
2501    }
2502
2503    #[tokio::test]
2504    async fn shutdown_stops_services() {
2505        let platform = Arc::new(NativePlatform::new());
2506        let mut kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2507            .await
2508            .unwrap();
2509
2510        // Services should be running before shutdown
2511        let svc_count_before = kernel.services().len();
2512        assert!(svc_count_before > 0, "should have services before shutdown");
2513
2514        kernel.shutdown().await.unwrap();
2515        // After shutdown, services stop_all() was called — the registry is
2516        // still there but state is Halted
2517        assert_eq!(*kernel.state(), KernelState::Halted);
2518    }
2519
2520    #[tokio::test]
2521    async fn boot_with_custom_max_processes() {
2522        let platform = Arc::new(NativePlatform::new());
2523        let mut kconfig = test_kernel_config();
2524        kconfig.max_processes = 128;
2525        let kernel = Kernel::boot(test_config(), kconfig, platform)
2526            .await
2527            .unwrap();
2528        assert_eq!(kernel.process_table().max_processes(), 128);
2529    }
2530
2531    #[tokio::test]
2532    async fn boot_creates_a2a_router_with_topic_router() {
2533        let platform = Arc::new(NativePlatform::new());
2534        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2535            .await
2536            .unwrap();
2537        // A2A router should have a topic router wired in
2538        let _topic_router = kernel.a2a_router().topic_router();
2539    }
2540
2541    #[tokio::test]
2542    async fn event_log_has_shutdown_events() {
2543        let platform = Arc::new(NativePlatform::new());
2544        let mut kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2545            .await
2546            .unwrap();
2547
2548        let events_before = kernel.event_log().len();
2549        kernel.shutdown().await.unwrap();
2550
2551        // Shutdown should have added events
2552        let events_after = kernel.event_log().len();
2553        assert!(
2554            events_after > events_before,
2555            "event log should grow during shutdown (before={events_before}, after={events_after})"
2556        );
2557    }
2558
2559    #[tokio::test]
2560    async fn boot_log_contains_max_processes() {
2561        let platform = Arc::new(NativePlatform::new());
2562        let mut kconfig = test_kernel_config();
2563        kconfig.max_processes = 64;
2564        let kernel = Kernel::boot(test_config(), kconfig, platform)
2565            .await
2566            .unwrap();
2567        let formatted = kernel.boot_log().format_all();
2568        assert!(
2569            formatted.contains("Max processes: 64"),
2570            "boot log should contain max_processes config value"
2571        );
2572    }
2573
2574    #[tokio::test]
2575    async fn boot_log_contains_health_interval() {
2576        let platform = Arc::new(NativePlatform::new());
2577        let mut kconfig = test_kernel_config();
2578        kconfig.health_check_interval_secs = 15;
2579        let kernel = Kernel::boot(test_config(), kconfig, platform)
2580            .await
2581            .unwrap();
2582        let formatted = kernel.boot_log().format_all();
2583        assert!(
2584            formatted.contains("Health check interval: 15s"),
2585            "boot log should contain health check interval"
2586        );
2587    }
2588
2589    #[tokio::test]
2590    async fn take_app_context_returns_some_then_none() {
2591        let platform = Arc::new(NativePlatform::new());
2592        let mut kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2593            .await
2594            .unwrap();
2595
2596        assert!(kernel.take_app_context().is_some(), "first take should succeed");
2597        assert!(kernel.take_app_context().is_none(), "second take should return None");
2598        assert!(kernel.take_app_context().is_none(), "third take should also return None");
2599    }
2600
2601    #[tokio::test]
2602    async fn kernel_pid_zero_has_default_capabilities() {
2603        let platform = Arc::new(NativePlatform::new());
2604        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2605            .await
2606            .unwrap();
2607        let entry = kernel.process_table().get(0).unwrap();
2608        assert_eq!(entry.agent_id, "kernel");
2609        assert!(entry.parent_pid.is_none(), "kernel should have no parent");
2610        // Default capabilities: can_ipc should be true, ipc_scope should be All
2611        assert!(entry.capabilities.can_ipc, "kernel should be able to IPC");
2612    }
2613
2614    #[tokio::test]
2615    async fn cluster_membership_has_nonempty_node_id() {
2616        let platform = Arc::new(NativePlatform::new());
2617        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2618            .await
2619            .unwrap();
2620        let node_id = kernel.cluster_membership().local_node_id();
2621        assert!(!node_id.is_empty(), "cluster node ID should not be empty");
2622        // UUID v4 format: 8-4-4-4-12 = 36 chars
2623        assert_eq!(node_id.len(), 36, "node ID should be UUID format");
2624    }
2625
2626    #[tokio::test]
2627    async fn boot_log_contains_cluster_info() {
2628        let platform = Arc::new(NativePlatform::new());
2629        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2630            .await
2631            .unwrap();
2632        let formatted = kernel.boot_log().format_all();
2633        assert!(
2634            formatted.contains("Cluster membership ready"),
2635            "boot log should mention cluster membership"
2636        );
2637    }
2638
2639    #[cfg(feature = "exochain")]
2640    #[tokio::test]
2641    async fn shutdown_with_exochain_completes() {
2642        let platform = Arc::new(NativePlatform::new());
2643        let mut kernel = Kernel::boot(test_config(), test_kernel_config_exochain(), platform)
2644            .await
2645            .unwrap();
2646
2647        assert!(kernel.chain_manager().is_some());
2648        kernel.shutdown().await.unwrap();
2649        assert_eq!(*kernel.state(), KernelState::Halted);
2650    }
2651
2652    #[cfg(feature = "exochain")]
2653    #[tokio::test]
2654    async fn exochain_chain_integrity_valid_after_boot() {
2655        let platform = Arc::new(NativePlatform::new());
2656        let kernel = Kernel::boot(test_config(), test_kernel_config_exochain(), platform)
2657            .await
2658            .unwrap();
2659        let chain = kernel.chain_manager().unwrap();
2660        let result = chain.verify_integrity();
2661        assert!(result.valid, "chain integrity should be valid after boot");
2662        assert!(result.errors.is_empty(), "no integrity errors expected");
2663    }
2664
2665    #[cfg(feature = "exochain")]
2666    #[tokio::test]
2667    async fn exochain_boot_manifest_event_present() {
2668        let platform = Arc::new(NativePlatform::new());
2669        let kernel = Kernel::boot(test_config(), test_kernel_config_exochain(), platform)
2670            .await
2671            .unwrap();
2672        let chain = kernel.chain_manager().unwrap();
2673        let all_events = chain.tail(0);
2674        let manifest_events: Vec<_> = all_events
2675            .iter()
2676            .filter(|e| e.kind == "boot.manifest")
2677            .collect();
2678        assert!(
2679            !manifest_events.is_empty(),
2680            "boot.manifest event should be on chain"
2681        );
2682    }
2683
2684    // ── os-patterns observability wiring tests (W3) ─────────────────
2685
2686    #[tokio::test]
2687    #[cfg(feature = "os-patterns")]
2688    async fn boot_creates_metrics_registry() {
2689        let platform = Arc::new(NativePlatform::new());
2690        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2691            .await
2692            .unwrap();
2693
2694        let registry = kernel.metrics_registry();
2695        assert!(registry.is_some(), "MetricsRegistry should be created at boot");
2696        // Verify a built-in gauge was seeded
2697        let r = registry.unwrap();
2698        let val = r.gauge_get("kernel.process.count");
2699        assert!(val >= 1, "kernel.process.count gauge should be seeded");
2700    }
2701
2702    #[tokio::test]
2703    #[cfg(feature = "os-patterns")]
2704    async fn boot_creates_log_service() {
2705        let platform = Arc::new(NativePlatform::new());
2706        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2707            .await
2708            .unwrap();
2709
2710        assert!(
2711            kernel.log_service().is_some(),
2712            "LogService should be created at boot"
2713        );
2714    }
2715
2716    #[tokio::test]
2717    #[cfg(feature = "os-patterns")]
2718    async fn boot_creates_timer_service() {
2719        let platform = Arc::new(NativePlatform::new());
2720        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2721            .await
2722            .unwrap();
2723
2724        assert!(
2725            kernel.timer_service().is_some(),
2726            "TimerService should be created at boot"
2727        );
2728    }
2729
2730    #[tokio::test]
2731    #[cfg(feature = "os-patterns")]
2732    async fn boot_creates_dead_letter_queue() {
2733        let platform = Arc::new(NativePlatform::new());
2734        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2735            .await
2736            .unwrap();
2737
2738        let dlq = kernel.dead_letter_queue();
2739        assert!(dlq.is_some(), "DeadLetterQueue should be created at boot");
2740        assert!(dlq.unwrap().is_empty(), "DLQ should start empty");
2741    }
2742
2743    #[tokio::test]
2744    #[cfg(feature = "os-patterns")]
2745    async fn kernel_metrics_accessor_returns_registry() {
2746        let platform = Arc::new(NativePlatform::new());
2747        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2748            .await
2749            .unwrap();
2750
2751        let registry = kernel.metrics_registry().unwrap();
2752        // Verify counter_inc works on built-in metric
2753        registry.counter_inc(crate::metrics::METRIC_MESSAGES_SENT);
2754        assert_eq!(registry.counter_get(crate::metrics::METRIC_MESSAGES_SENT), 1);
2755    }
2756
2757    #[tokio::test]
2758    #[cfg(feature = "os-patterns")]
2759    async fn dlq_accessible_via_kernel_accessor() {
2760        let platform = Arc::new(NativePlatform::new());
2761        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2762            .await
2763            .unwrap();
2764
2765        let dlq = kernel.dead_letter_queue().unwrap();
2766        assert_eq!(dlq.capacity(), crate::dead_letter::DEFAULT_DLQ_CAPACITY);
2767    }
2768
2769    #[tokio::test]
2770    #[cfg(feature = "os-patterns")]
2771    async fn failed_a2a_send_routes_to_dlq() {
2772        use crate::ipc::{KernelMessage, MessageTarget};
2773
2774        let platform = Arc::new(NativePlatform::new());
2775        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2776            .await
2777            .unwrap();
2778
2779        // Register a sender agent
2780        let sender_entry = ProcessEntry {
2781            pid: 0,
2782            agent_id: "dlq-sender".to_owned(),
2783            state: ProcessState::Running,
2784            capabilities: AgentCapabilities::default(),
2785            resource_usage: ResourceUsage::default(),
2786            cancel_token: tokio_util::sync::CancellationToken::new(),
2787            parent_pid: None,
2788        };
2789        let sender_pid = kernel.process_table().insert(sender_entry).unwrap();
2790        kernel.a2a_router().create_inbox(sender_pid);
2791
2792        // Send to a PID that has no inbox (PID 9999 doesn't exist)
2793        let msg = KernelMessage::text(
2794            sender_pid,
2795            MessageTarget::Process(9999),
2796            "dead letter test",
2797        );
2798        let result = kernel.a2a_router().send(msg).await;
2799        assert!(result.is_err(), "send to unknown PID should fail");
2800
2801        // Verify the message landed in the DLQ
2802        let dlq = kernel.dead_letter_queue().unwrap();
2803        assert_eq!(dlq.len(), 1, "DLQ should contain 1 dead letter");
2804    }
2805
2806    #[tokio::test]
2807    #[cfg(feature = "os-patterns")]
2808    async fn kernel_metrics_gauge_seeded_at_boot() {
2809        let platform = Arc::new(NativePlatform::new());
2810        let kernel = Kernel::boot(test_config(), test_kernel_config(), platform)
2811            .await
2812            .unwrap();
2813
2814        let registry = kernel.metrics_registry().unwrap();
2815        // kernel.process.count should reflect at least the kernel PID 0
2816        assert!(
2817            registry.gauge_get("kernel.process.count") >= 1,
2818            "kernel.process.count should be >= 1 at boot"
2819        );
2820        // Verify additional gauges exist (they were set to 0, so gauge_get returns 0 not None)
2821        assert_eq!(
2822            registry.gauge_get("kernel.uptime_secs"),
2823            0,
2824            "kernel.uptime_secs gauge should exist and be 0 at boot"
2825        );
2826        assert_eq!(
2827            registry.gauge_get("kernel.chain.height"),
2828            0,
2829            "kernel.chain.height gauge should exist and be 0 at boot"
2830        );
2831    }
2832}