vyre_runtime/megakernel/
mod.rs

1//! Persistent megakernel  -  the GPU becomes a VIR0 bytecode interpreter.
2//!
3//! One dispatch compiles the program; the kernel loops forever, pulling
4//! packed bytecode slots from a host-fed ring buffer and executing each.
5//! The host never re-dispatches  -  it only writes new slots and observes
6//! atomic counters in the control buffer.
7//!
8//! ## Layout
9//!
10//! - `protocol`  -  ring-buffer slot layout, control words, opcodes.
11//! - `handlers`  -  built-in opcode handlers + extension mechanism.
12//! - `builder`  -  IR `Program` construction (interpreted + JIT).
13//! - `execution`  -  compiled persistent-kernel handle and dispatch path.
14//! - `resident`  -  host mirrors for GPU-resident runtime buffers.
15//! - `readback`  -  strict output-buffer decoding after dispatch.
16//! - `recovery`  -  device-loss classification and pipeline rebuild.
17//!
18//! ## Coordination protocol
19//!
20//! 1. Read `control[SHUTDOWN]`; if non-zero, `Node::Return`.
21//! 2. Read this slot's `status`; skip idle slots without tenant metadata loads.
22//! 3. If PUBLISHED, read `tenant_id`; authorize via tenant-mask table.
23//! 4. CAS `ring_buffer[status]` from PUBLISHED → CLAIMED.
24//! 5. Dispatch on opcode through If-tree (or JIT fused body).
25//! 6. `atomic_add(control[DONE_COUNT], 1)`.
26//! 7. Store DONE into the status word.
27
28#[cfg(feature = "megakernel-batch")]
29pub mod advanced;
30pub mod builder;
31pub mod descriptor;
32pub mod execution;
33pub mod handlers;
34pub mod io;
35pub mod ir_util;
36pub mod planner;
37pub mod policy;
38pub mod protocol;
39mod protocol_api;
40pub mod readback;
41pub mod recovery;
42pub mod resident;
43pub mod ring;
44#[cfg(feature = "megakernel-batch")]
45pub mod rule_catalog;
46pub mod scaling;
47pub mod scheduler;
48pub mod speculation;
49mod staging_reserve;
50pub mod task;
51pub mod telemetry;
52pub mod workspace_adapter;
53pub mod workspace_layout;
54
55use vyre_driver::backend::BackendError;
56
57// Re-export protocol constants at the megakernel level for back-compat.
58pub use builder::{
59    build_program, build_program_jit, build_program_jit_slots, build_program_priority,
60    build_program_priority_slots, build_program_sharded, build_program_sharded_no_io,
61    build_program_sharded_once_slots, build_program_sharded_once_slots_control_report_shared,
62    build_program_sharded_once_slots_shared, build_program_sharded_slots,
63    build_program_sharded_slots_shared, build_program_sharded_with_io_polling,
64    build_program_sharded_with_workspace_adapter, build_program_with_self_loading_miss_handler,
65    persistent_body, persistent_body_jit, persistent_body_priority, persistent_body_priority_slots,
66};
67pub use descriptor::{
68    BatchDescriptor, BuiltinOpcode, PackedOpDescriptor, SlotDescriptor, SlotOpcode, WindowClass,
69    WindowDescriptor,
70};
71pub use execution::{
72    Megakernel, MegakernelDispatchOutput, MegakernelDispatchStats, MegakernelResidentHandles,
73};
74pub use handlers::OpcodeHandler;
75pub use io::{IoCompletion, IoRequest, MegakernelIoQueue, IO_SLOT_COUNT, IO_SLOT_WORDS};
76#[cfg(feature = "self-substrate-adapters")]
77pub use planner::{
78    build_bellman_tn_order_program, build_kfac_autotune_step_program,
79    build_persistent_fixpoint_program, build_scallop_lineage_with_scratch,
80    build_scallop_provenance_wide_program, build_sinkhorn_clustering_program,
81    build_sinkhorn_full_clustering_program,
82};
83pub use planner::{
84    build_scallop_lineage_with_program_and_scratch, default_worker_groups_from_limits,
85    dispatch_grid_for, padded_slot_count, plan_compact_fusion_into,
86    prune_redundant_work_items_into, prune_redundant_work_items_with_scratch_into,
87    select_fused_subset, select_fused_subset_compact, select_fused_subset_compact_into,
88    select_fused_subset_into, select_fused_subset_with_rate, select_optimal_fused_subset,
89    try_detect_cross_arm_redundancy, try_prune_redundant_work_items_into,
90    try_prune_redundant_work_items_with_scratch_into, worker_workgroup_size,
91    CompactFusionPlanningScratch, CrossArmRedundancy, FusionSelectionScratch, MegakernelCaps,
92    MegakernelConfig, MegakernelGridLimits, MegakernelGridPlan, MegakernelGridRequest,
93    MegakernelLaunchGeometry, MegakernelReport, MegakernelSizingPolicy, MegakernelTelemetry,
94    MegakernelWorkItem, MegakernelWorkloadHints, RedundantWorkItemPruneScratch,
95};
96pub use policy::{
97    diffuse_priority_across_siblings, diffuse_priority_across_siblings_into,
98    MegakernelDispatchTopology, MegakernelExecutionMode, MegakernelLaunchCacheStats,
99    MegakernelLaunchPolicy, MegakernelLaunchRecommendation, MegakernelLaunchRequest,
100    MegakernelQueuePressure, PriorityRequeueAccounting,
101};
102pub use protocol::{
103    control, control_byte_len, count_done_ring_slots, debug, debug_log_byte_len, encode_control,
104    encode_empty_debug_log, encode_empty_ring, opcode, read_debug_log, read_done_count, read_epoch,
105    read_metrics, read_observable, ring_byte_len, slot, try_count_done_ring_slots,
106    try_encode_control, try_encode_control_into, try_encode_empty_debug_log,
107    try_encode_empty_debug_log_into, try_encode_empty_ring, try_encode_empty_ring_into,
108    try_read_debug_log, try_read_done_count, try_read_epoch, try_read_metrics, try_read_observable,
109    DebugRecord, ProtocolError, ARG0_WORD, ARGS_PER_SLOT, CONTROL_MIN_WORDS, OPCODE_WORD,
110    PRIORITY_WORD, SLOT_WORDS, STATUS_WORD, TENANT_WORD,
111};
112pub use readback::{MegakernelReadback, MegakernelReadbackCounters};
113pub use recovery::{
114    backend_error_indicates_device_loss, MegakernelRecoveryDecision, MegakernelRecoveryPolicy,
115};
116pub use resident::{MegakernelResidentBuffers, MegakernelResidentDispatchScratch};
117#[cfg(feature = "megakernel-batch")]
118pub use rule_catalog::{BatchRuleProgram, BatchRuleRejection};
119pub use scheduler::{
120    default_priority_offsets, priority_partition_active_lane_count,
121    priority_partition_probe_budget, priority_partition_probe_count, priority_scan_body,
122    priority_scan_body_with_stride, try_default_priority_offsets, write_default_priority_offsets,
123};
124pub use speculation::{PairedSpeculationSample, PairedSpeculationUpdate, PairedSpeculationWindow};
125pub use task::{TaskPriority, TaskQueueSnapshot, TaskState, TaskWorkItem};
126pub use telemetry::{
127    ControlSnapshot, CountMinSketch, MegakernelRuntimeCounters, RingOccupancy, RingSlotSnapshot,
128    RingStatus, RingTelemetry, SketchTelemetry, WindowTelemetry,
129};
130pub use workspace_adapter::MegakernelWorkspaceAdapter;
131pub use workspace_layout::{
132    build_workspace_regions, first_workspace_region, next_record_workspace_region,
133    next_workspace_region, workspace_record_words, MegakernelWorkspaceLayoutError,
134    MegakernelWorkspaceRegion, MegakernelWorkspaceRegionSpec,
135};
136/// Backend-neutral megakernel dispatch contract.
137pub trait MegakernelDispatch {
138    /// Drain the requested megakernel dispatch.
139    fn dispatch_megakernel(
140        &self,
141        work_queue: &[MegakernelWorkItem],
142        config: &MegakernelConfig,
143    ) -> Result<MegakernelReport, BackendError>;
144}
vyre_runtime/megakernel/mod.rs

vyre_runtime/megakernel/
mod.rs
