use std::panic::AssertUnwindSafe;
use std::sync::Arc;
use std::sync::mpsc::{Receiver, SyncSender, TrySendError, sync_channel};
use std::time::Duration;
use crate::fkc::verify::{LedgerRecord, is_transcendental};
#[cfg(feature = "cuda")]
use crate::fkc::verify::{
Bound, CudaInvoker, HostTensor, KernelInvoker, ProbeInputs, VerificationLedger, VerifyError,
VerifyOutcome, region_contains_transcendental, verify_bit_stability, verify_precision_bound,
widen_bound_for_transcendental,
};
#[cfg(feature = "cuda")]
use crate::kernel::BindingEntry;
#[cfg(feature = "cuda")]
use crate::runtime_fused_kernels::adopt_runtime_fused;
#[cfg(feature = "cuda")]
use fuel_cuda_backend::CudaDevice;
pub struct CandidateKernel {
pub entry_point: String,
pub kernel: crate::kernel::KernelRef,
pub op_params: crate::kernel::OpParams,
pub decompose: Option<fuel_graph::jit::PatternNode>,
pub operands: Vec<baracuda_kernels_types::OperandDesc>,
pub dtypes: Vec<fuel_ir::DType>,
pub kernel_revision_hash: u64,
pub declared: crate::fused::PrecisionGuarantee,
pub backend: fuel_ir::probe::BackendId,
pub claimed_op: Option<fuel_graph::registry::FusedOpId>,
}
pub struct RejectionReport {
pub entry_point: String,
pub failed_claim: &'static str,
pub detail: String,
pub ledger_record: Option<LedgerRecord>,
}
pub struct FlagReport {
pub entry_point: String,
pub claim: &'static str,
pub detail: String,
pub diff_summary: Option<String>,
pub escalate: bool,
}
pub trait ProviderFeedback: Send + Sync {
fn on_rejected(&self, report: &RejectionReport);
fn on_adopted(&self, _entry_point: &str, _id: fuel_graph::registry::FusedOpId) {}
fn on_flagged(&self, _report: &FlagReport) {}
}
pub enum IngestOutcome {
Adopted(fuel_graph::registry::FusedOpId),
Rejected(RejectionReport),
Flagged(FlagReport),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
pub enum OpClass {
Exact,
Transcendental,
}
#[derive(Debug, Clone)]
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
pub struct DiffOutcome {
pub within: bool,
pub max_ulp: Option<u64>,
pub detail: String,
}
#[derive(Debug, Clone)]
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
pub struct RefOutcome {
pub pass: bool,
pub claim: &'static str,
pub detail: String,
}
#[derive(Debug, Clone)]
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
pub struct CorpusOutcome {
pub adopt: bool,
pub claim: &'static str,
pub detail: String,
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
pub fn classify_floor_verdict(
kiss: Option<&DiffOutcome>,
recipe: Option<&RefOutcome>,
corpus: Option<&CorpusOutcome>,
) -> VerifyVerdict {
if let Some(c) = corpus {
return if c.adopt {
VerifyVerdict::Pass
} else {
VerifyVerdict::Fail {
claim: c.claim,
detail: c.detail.clone(),
}
};
}
if let Some(r) = recipe {
return if r.pass {
VerifyVerdict::Pass
} else {
VerifyVerdict::Fail {
claim: r.claim,
detail: r.detail.clone(),
}
};
}
if let Some(k) = kiss {
let detail = format!(
"no authoritative reference; kiss-ref {} (escalate to corpus): {}",
if k.within { "agrees" } else { "disagrees" },
k.detail
);
return VerifyVerdict::Inconclusive {
claim: "max_ulp",
detail,
};
}
VerifyVerdict::Fail {
claim: "no_reference",
detail: "no reference available".to_string(),
}
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
pub fn corpus_verdict(
_op: fuel_graph::jit::OpTag,
_dtype: fuel_ir::DType,
_seed: u64,
) -> Option<CorpusOutcome> {
None
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
fn outcome_from_nonadopt_verdict(
verdict: VerifyVerdict,
records: Vec<LedgerRecord>,
entry_point: &str,
) -> IngestOutcome {
match verdict {
VerifyVerdict::Pass => IngestOutcome::Rejected(RejectionReport {
entry_point: entry_point.to_string(),
failed_claim: "internal",
detail: "Pass routed to non-adopt mapping".to_string(),
ledger_record: None,
}),
VerifyVerdict::Fail { claim, detail } => {
let ledger_record = records.into_iter().find(|r| r.claim == claim);
IngestOutcome::Rejected(RejectionReport {
entry_point: entry_point.to_string(),
failed_claim: claim,
detail,
ledger_record,
})
}
VerifyVerdict::Inconclusive { claim, detail } => {
let diff_summary = records
.iter()
.find(|r| r.claim == "kiss_ref_advisory")
.map(kiss_advisory_diff_summary);
IngestOutcome::Flagged(FlagReport {
entry_point: entry_point.to_string(),
claim,
detail,
diff_summary,
escalate: true,
})
}
}
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
fn kiss_advisory_diff_summary(r: &LedgerRecord) -> String {
let field = |k: &str| {
r.evidence
.get(k)
.map(|v| v.to_string())
.unwrap_or_else(|| "?".to_string())
};
format!(
"kiss_ref_advisory={} max_ulp={} mismatches={} op_count={}",
r.result,
field("max_ulp"),
field("mismatches"),
field("op_count"),
)
}
#[allow(dead_code)]
fn single_primitive_optag(
dec: Option<&fuel_graph::jit::PatternNode>,
) -> Option<fuel_graph::jit::OpTag> {
use fuel_graph::jit::PatternNode;
let PatternNode::Op { op, operands, .. } = dec? else {
return None;
};
for (i, operand) in operands.iter().enumerate() {
match operand {
PatternNode::Bind { index } if *index as usize == i => {}
_ => return None,
}
}
Some(*op)
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
fn bytes_to_f32(bytes: &[u8]) -> Vec<f32> {
let (chunks, _rest) = bytes.as_chunks::<4>();
chunks.iter().map(|c| f32::from_le_bytes(*c)).collect()
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
fn bytes_to_f64(bytes: &[u8]) -> Vec<f64> {
let (chunks, _rest) = bytes.as_chunks::<8>();
chunks.iter().map(|c| f64::from_le_bytes(*c)).collect()
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
fn bytes_to_f16(bytes: &[u8]) -> Vec<half::f16> {
let (chunks, _rest) = bytes.as_chunks::<2>();
chunks
.iter()
.map(|c| half::f16::from_le_bytes(*c))
.collect()
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
fn bytes_to_bf16(bytes: &[u8]) -> Vec<half::bf16> {
let (chunks, _rest) = bytes.as_chunks::<2>();
chunks
.iter()
.map(|c| half::bf16::from_le_bytes(*c))
.collect()
}
const ADVISORY_FALLBACK_TRANSCENDENTAL_ULP_CEILING: u64 = 4;
fn advisory_op_ulp_ceiling(op: fuel_graph::jit::OpTag) -> Option<u64> {
use fuel_graph::jit::OpTag as T;
use kiss_ops_vocab::Op as K;
if !is_transcendental(op) {
return None;
}
let kiss = match op {
T::Exp => Some(K::Exp),
T::Log => Some(K::Log),
T::Sin => Some(K::Sin),
T::Cos => Some(K::Cos),
T::Erf => Some(K::Erf),
T::Tanh => Some(K::Tanh),
T::Sigmoid => Some(K::Sigmoid),
T::Silu => Some(K::Silu),
T::GeluTanh => Some(K::GeluTanh),
T::Gelu => Some(K::Gelu),
T::Rsqrt => Some(K::Rsqrt),
_ => None,
};
let declared = kiss.and_then(|k| k.ulp_ceiling());
Some(
declared.map_or(ADVISORY_FALLBACK_TRANSCENDENTAL_ULP_CEILING, |c| {
c.ceil() as u64
}),
)
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
fn advisory_ulp_band(region: &fuel_graph::jit::PatternNode) -> Option<u64> {
use fuel_graph::jit::PatternNode;
fn walk(node: &PatternNode, n_ops: &mut u64, n_exact: &mut u64, trans_sum: &mut Option<u64>) {
match node {
PatternNode::Op { op, operands, .. } => {
*n_ops += 1;
match advisory_op_ulp_ceiling(*op) {
Some(c) => {
*trans_sum = Some(trans_sum.unwrap_or(0).saturating_add(c));
}
None => *n_exact += 1,
}
for o in operands {
walk(o, n_ops, n_exact, trans_sum);
}
}
PatternNode::SeeThrough { then } => walk(then, n_ops, n_exact, trans_sum),
PatternNode::Bind { .. } | PatternNode::Any => {}
}
}
let (mut n_ops, mut n_exact, mut trans_sum) = (0u64, 0u64, None);
walk(region, &mut n_ops, &mut n_exact, &mut trans_sum);
if n_ops == 0 {
return None;
}
match trans_sum {
None => {
if n_ops == 1 {
None
} else {
Some(n_ops - 1)
}
}
Some(sum) => Some(sum.saturating_add(n_exact.saturating_sub(1))),
}
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
fn advisory_region(
decompose: Option<&fuel_graph::jit::PatternNode>,
claimed: Option<fuel_graph::registry::FusedOpId>,
) -> Option<fuel_graph::jit::PatternNode> {
decompose
.cloned()
.or_else(|| claimed.and_then(fuel_graph::runtime_fused::runtime_region))
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
fn nonf32_escalate_eligible(out_dtype: fuel_ir::DType, region_kiss_supported: bool) -> bool {
region_kiss_supported
&& matches!(
out_dtype,
fuel_ir::DType::F64 | fuel_ir::DType::F16 | fuel_ir::DType::BF16
)
}
#[cfg(test)]
mod flag_not_verdict_tests {
use super::*;
#[test]
fn new_outcome_types_construct_and_match() {
let flag = FlagReport {
entry_point: "k".into(),
claim: "max_ulp",
detail: "kiss-ref discrepancy".into(),
diff_summary: Some("max_ulp=3".into()),
escalate: true,
};
assert!(matches!(IngestOutcome::Flagged(flag),
IngestOutcome::Flagged(ref r) if r.escalate && r.claim == "max_ulp"));
let v = VerifyVerdict::Inconclusive {
claim: "max_ulp",
detail: "x".into(),
};
assert!(matches!(v, VerifyVerdict::Inconclusive { claim, .. } if claim == "max_ulp"));
}
#[test]
fn classify_corpus_wins_when_present() {
let c = CorpusOutcome {
adopt: true,
claim: "max_ulp",
detail: "corpus".into(),
};
assert!(matches!(
classify_floor_verdict(None, None, Some(&c)),
VerifyVerdict::Pass
));
let cr = CorpusOutcome {
adopt: false,
claim: "max_ulp",
detail: "corpus".into(),
};
assert!(matches!(classify_floor_verdict(None, None, Some(&cr)),
VerifyVerdict::Fail { claim, .. } if claim == "max_ulp"));
}
#[test]
fn classify_recipe_is_interim_verdict_kiss_advisory() {
let kiss = DiffOutcome {
within: false,
max_ulp: Some(5),
detail: "disagree".into(),
};
let recipe = RefOutcome {
pass: true,
claim: "max_ulp",
detail: "recipe ok".into(),
};
assert!(matches!(
classify_floor_verdict(Some(&kiss), Some(&recipe), None),
VerifyVerdict::Pass
));
}
#[test]
fn classify_no_reference_but_kiss_is_inconclusive() {
let agree = DiffOutcome {
within: true,
max_ulp: Some(0),
detail: "agree".into(),
};
assert!(
matches!(
classify_floor_verdict(Some(&agree), None, None),
VerifyVerdict::Inconclusive { .. }
),
"kiss agreement != Adopt"
);
let off = DiffOutcome {
within: false,
max_ulp: Some(4),
detail: "disagree".into(),
};
assert!(
matches!(
classify_floor_verdict(Some(&off), None, None),
VerifyVerdict::Inconclusive { .. }
),
"kiss discrepancy != Reject"
);
}
#[test]
fn classify_all_none_fails() {
assert!(matches!(classify_floor_verdict(None, None, None),
VerifyVerdict::Fail { claim, .. } if claim == "no_reference"));
}
#[test]
fn corpus_verdict_is_dormant_returns_none() {
assert!(corpus_verdict(fuel_graph::jit::OpTag::Add, fuel_ir::DType::F32, 0).is_none());
}
#[test]
fn map_fail_to_rejected_and_inconclusive_to_flagged() {
let out = outcome_from_nonadopt_verdict(
VerifyVerdict::Fail {
claim: "max_ulp",
detail: "off".into(),
},
vec![],
"k",
);
assert!(matches!(out, IngestOutcome::Rejected(ref r) if r.failed_claim == "max_ulp"));
let out = outcome_from_nonadopt_verdict(
VerifyVerdict::Inconclusive {
claim: "max_ulp",
detail: "esc".into(),
},
vec![],
"k",
);
assert!(matches!(out, IngestOutcome::Flagged(ref r) if r.escalate && r.claim == "max_ulp"));
}
#[test]
fn map_inconclusive_carries_diff_summary_from_advisory_record() {
let advisory = LedgerRecord {
kernel_ref: "k".into(),
backend: "Cuda".into(),
dtypes: vec!["F64".into(), "F64".into()],
kernel_revision_hash: 0,
claim: "kiss_ref_advisory".into(),
result: "flag".into(),
verified_at: "epoch:0".into(),
protocol_version: 1,
evidence: serde_json::json!({
"dtype": "F64",
"op_count": 1,
"max_ulp": 3,
"mismatches": 2,
}),
};
let out = outcome_from_nonadopt_verdict(
VerifyVerdict::Inconclusive {
claim: "max_ulp",
detail: "esc".into(),
},
vec![advisory],
"k",
);
match out {
IngestOutcome::Flagged(r) => {
let summary = r
.diff_summary
.expect("diff_summary threaded from the advisory record");
assert!(
summary.contains("max_ulp=3"),
"names the max ULP: {summary}"
);
assert!(
summary.contains("mismatches=2"),
"and the mismatch count: {summary}"
);
assert!(
summary.contains("flag"),
"and the advisory result: {summary}"
);
}
_ => panic!("expected Flagged"),
}
let out = outcome_from_nonadopt_verdict(
VerifyVerdict::Inconclusive {
claim: "max_ulp",
detail: "esc".into(),
},
vec![],
"k",
);
match out {
IngestOutcome::Flagged(r) => assert!(r.diff_summary.is_none()),
_ => panic!("expected Flagged"),
}
}
#[test]
fn single_primitive_optag_extracts_and_declines() {
use fuel_graph::jit::{OpAttrs, OpTag, PatternNode};
let add = PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Bind { index: 0 },
PatternNode::Bind { index: 1 },
],
};
assert!(matches!(
single_primitive_optag(Some(&add)),
Some(OpTag::Add)
));
let reordered = PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Bind { index: 1 },
PatternNode::Bind { index: 0 },
],
};
assert!(single_primitive_optag(Some(&reordered)).is_none());
let nested = PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![add.clone(), PatternNode::Bind { index: 1 }],
};
assert!(single_primitive_optag(Some(&nested)).is_none());
assert!(single_primitive_optag(None).is_none());
}
#[test]
fn bytes_to_f32_roundtrips() {
let v = [1.0f32, -2.5, 3.25];
let bytes: Vec<u8> = v.iter().flat_map(|x| x.to_le_bytes()).collect();
assert_eq!(bytes_to_f32(&bytes), vec![1.0, -2.5, 3.25]);
}
#[test]
fn bytes_to_f64_f16_bf16_roundtrip() {
let v64 = [1.0f64, -2.5, 1e300];
let bytes: Vec<u8> = v64.iter().flat_map(|x| x.to_le_bytes()).collect();
assert_eq!(bytes_to_f64(&bytes), v64.to_vec());
let v16: Vec<half::f16> = [1.0f32, -2.5, 0.5]
.iter()
.map(|&x| half::f16::from_f32(x))
.collect();
let bytes: Vec<u8> = v16.iter().flat_map(|x| x.to_le_bytes()).collect();
assert_eq!(bytes_to_f16(&bytes), v16);
let vb: Vec<half::bf16> = [1.0f32, -2.5, 0.5]
.iter()
.map(|&x| half::bf16::from_f32(x))
.collect();
let bytes: Vec<u8> = vb.iter().flat_map(|x| x.to_le_bytes()).collect();
assert_eq!(bytes_to_bf16(&bytes), vb);
}
#[test]
fn advisory_ulp_band_selects_by_region_shape() {
use fuel_graph::jit::{OpAttrs, OpTag, PatternNode};
let bind = |i: u8| PatternNode::Bind { index: i };
let node = |op: OpTag, operands: Vec<PatternNode>| PatternNode::Op {
op,
operands,
attrs: OpAttrs::default(),
};
let add = node(OpTag::Add, vec![bind(0), bind(1)]);
assert_eq!(advisory_ulp_band(&add), None);
let relu_add = node(OpTag::Relu, vec![add.clone()]);
assert_eq!(advisory_ulp_band(&relu_add), Some(1));
let exp = node(OpTag::Exp, vec![bind(0)]);
assert_eq!(advisory_ulp_band(&exp), Some(4));
let exp_add = node(OpTag::Exp, vec![add.clone()]);
assert_eq!(advisory_ulp_band(&exp_add), Some(4));
let tanh_exp_add = node(
OpTag::Tanh,
vec![node(
OpTag::Add,
vec![node(OpTag::Exp, vec![bind(0)]), bind(1)],
)],
);
assert_eq!(advisory_ulp_band(&tanh_exp_add), Some(8));
let exp_tanh = node(OpTag::Exp, vec![node(OpTag::Tanh, vec![bind(0)])]);
assert_eq!(advisory_ulp_band(&exp_tanh), Some(8));
let sqrt_add = node(OpTag::Sqrt, vec![add.clone()]);
assert_eq!(advisory_ulp_band(&sqrt_add), Some(1));
assert_eq!(advisory_ulp_band(&bind(0)), None);
}
#[test]
fn advisory_ulp_band_matches_shared_cases() {
for (region, expected) in fuel_kernel_seam_types::advisory_band_reference_cases() {
assert_eq!(
advisory_ulp_band(®ion),
expected,
"live band drifted from the shared fixture for {region:?}"
);
}
}
#[test]
fn advisory_region_resolves_runtime_claimed_id_and_declines_static() {
use fuel_graph::jit::{OpAttrs, OpTag, PatternNode};
use fuel_graph::registry::{FusedOpId, FusedOps};
let bind = |i: u8| PatternNode::Bind { index: i };
let region = PatternNode::Op {
op: OpTag::Mul,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![bind(0), bind(1)],
},
bind(0),
],
};
let id = fuel_graph::runtime_fused::register_runtime_fused(
"test::advisory_region::mul_add",
region.clone(),
)
.expect("runtime registration");
assert_eq!(advisory_region(None, Some(id)), Some(region.clone()));
let dec = PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![bind(0), bind(1)],
};
assert_eq!(advisory_region(Some(&dec), Some(id)), Some(dec.clone()));
assert_eq!(advisory_region(None, Some(FusedOps::ROPE)), None);
let unregistered = FusedOpId(FusedOpId::RUNTIME_FUSED_BASE + 0x7FF0);
assert_eq!(advisory_region(None, Some(unregistered)), None);
assert_eq!(advisory_region(None, None), None);
}
#[test]
fn nonf32_escalate_eligible_gates_on_dtype_and_region_support() {
use fuel_ir::DType;
assert!(nonf32_escalate_eligible(DType::F64, true));
assert!(nonf32_escalate_eligible(DType::F16, true));
assert!(nonf32_escalate_eligible(DType::BF16, true));
assert!(!nonf32_escalate_eligible(DType::F64, false));
assert!(!nonf32_escalate_eligible(DType::F16, false));
assert!(!nonf32_escalate_eligible(DType::BF16, false));
assert!(!nonf32_escalate_eligible(DType::F32, true));
assert!(!nonf32_escalate_eligible(DType::I32, true));
assert!(!nonf32_escalate_eligible(DType::F8E4M3, true));
}
#[test]
#[ignore = "requires a live CUDA device"]
#[cfg(feature = "cuda")]
fn kiss_ref_advisory_records_for_add_f32() {
use baracuda_kernels_types::{ElementKind, OperandDesc};
use fuel_cuda_backend::CudaDevice;
use fuel_graph::jit::{OpAttrs, OpTag, PatternNode};
use fuel_ir::DType;
use fuel_ir::probe::BackendId;
let Ok(dev) = CudaDevice::new(0) else {
return fuel_test_support::hardware::skip(
fuel_test_support::hardware::Hardware::Cuda,
fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
);
};
let decompose = PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Bind { index: 0 },
PatternNode::Bind { index: 1 },
],
};
let od = OperandDesc::new(1, &[4], &[1], ElementKind::F32, 16);
let cand = CandidateKernel {
entry_point: "test::kiss_advisory::add_f32".to_string(),
kernel: crate::baracuda_dispatch::binary::add_f32,
op_params: crate::kernel::OpParams::None,
decompose: Some(decompose),
operands: vec![od, od],
dtypes: vec![DType::F32, DType::F32],
kernel_revision_hash: 0x1_9E57_ADD1,
declared: crate::fused::PrecisionGuarantee::REFERENCE,
backend: BackendId::Cuda,
claimed_op: None,
};
let (_verdict, records) = verify_candidate(&cand, &dev);
let advisory = records.iter().find(|r| r.claim == "kiss_ref_advisory");
assert!(
advisory.is_some(),
"kiss-ref advisory record must be present for a supported f32 Add"
);
assert_eq!(
advisory.unwrap().result,
"pass",
"CUDA add_f32 must match kiss-ref add exactly (0 ULP)"
);
}
#[test]
#[ignore = "requires a live CUDA device"]
#[cfg(feature = "cuda")]
fn multi_node_region_advisory_flags_add_kernel_for_relu_add() {
use baracuda_kernels_types::{ElementKind, OperandDesc};
use fuel_cuda_backend::CudaDevice;
use fuel_graph::jit::{OpAttrs, OpTag, PatternNode};
use fuel_ir::DType;
use fuel_ir::probe::BackendId;
let Ok(dev) = CudaDevice::new(0) else {
return fuel_test_support::hardware::skip(
fuel_test_support::hardware::Hardware::Cuda,
fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
);
};
let decompose = PatternNode::Op {
op: OpTag::Relu,
attrs: OpAttrs::default(),
operands: vec![PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Bind { index: 0 },
PatternNode::Bind { index: 1 },
],
}],
};
let od = OperandDesc::new(1, &[8], &[1], ElementKind::F32, 32);
let cand = CandidateKernel {
entry_point: "test::multi_node_advisory::relu_add".to_string(),
kernel: crate::baracuda_dispatch::binary::add_f32,
op_params: crate::kernel::OpParams::None,
decompose: Some(decompose),
operands: vec![od, od],
dtypes: vec![DType::F32, DType::F32],
kernel_revision_hash: 0x1_9E57_4EAD,
declared: crate::fused::PrecisionGuarantee::REFERENCE,
backend: BackendId::Cuda,
claimed_op: None,
};
let (_verdict, records) = verify_candidate(&cand, &dev);
let advisory = records
.iter()
.find(|r| r.claim == "kiss_ref_advisory")
.expect("a multi-node region must reach the generalized advisory cross-check");
assert_eq!(
advisory.evidence["op_count"],
serde_json::json!(2),
"advisory ran over the 2-op relu(add) region: {advisory:?}"
);
assert_eq!(
advisory.result, "flag",
"an add kernel disagrees with kiss-ref's relu(add) reference: {advisory:?}"
);
}
#[test]
#[ignore = "requires a live CUDA device"]
#[cfg(feature = "cuda")]
fn claimed_op_candidate_reaches_advisory_via_runtime_region() {
use baracuda_kernels_types::{ElementKind, OperandDesc};
use fuel_cuda_backend::CudaDevice;
use fuel_graph::jit::{OpAttrs, OpTag, PatternNode};
use fuel_ir::DType;
use fuel_ir::probe::BackendId;
let Ok(dev) = CudaDevice::new(0) else {
return fuel_test_support::hardware::skip(
fuel_test_support::hardware::Hardware::Cuda,
fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
);
};
let region = PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Bind { index: 0 },
PatternNode::Bind { index: 1 },
],
};
let claimed = fuel_graph::runtime_fused::register_runtime_fused(
"test::claimed_op_advisory::add",
region,
)
.expect("runtime registration");
let od = OperandDesc::new(1, &[4], &[1], ElementKind::F32, 16);
let cand = CandidateKernel {
entry_point: "test::claimed_op_advisory::add_f32".to_string(),
kernel: crate::baracuda_dispatch::binary::add_f32,
op_params: crate::kernel::OpParams::None,
decompose: None,
operands: vec![od, od],
dtypes: vec![DType::F32, DType::F32],
kernel_revision_hash: 0x1_9E57_C1A1,
declared: crate::fused::PrecisionGuarantee::REFERENCE,
backend: BackendId::Cuda,
claimed_op: Some(claimed),
};
let (_verdict, records) = verify_candidate(&cand, &dev);
let advisory = records
.iter()
.find(|r| r.claim == "kiss_ref_advisory")
.expect("a claimed-op candidate must reach advisory via its runtime region");
assert_eq!(
advisory.evidence["source"],
serde_json::json!("claimed_recipe"),
"the region came from the claimed op's registered recipe, not a decompose: {advisory:?}"
);
assert_eq!(
advisory.result, "pass",
"add_f32 matches kiss-ref add exactly (0 ULP): {advisory:?}"
);
}
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
fn region_arity(region: &fuel_graph::jit::PatternNode) -> usize {
region
.bind_indices()
.iter()
.max()
.map(|m| *m as usize + 1)
.unwrap_or(0)
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
fn push_placeholder_leaves(g: &mut fuel_graph::Graph, arity: usize) -> Vec<fuel_graph::NodeId> {
(0..arity)
.map(|_| {
g.push(fuel_graph::Node {
op: fuel_graph::Op::Const,
inputs: vec![],
shape: fuel_ir::Shape::from_dims(&[1]),
dtype: fuel_ir::DType::F32,
})
})
.collect()
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
fn base_map_hash_of_region(region: &fuel_graph::jit::PatternNode) -> Option<u64> {
let graph = std::sync::Arc::new(std::sync::RwLock::new(fuel_graph::Graph::new()));
let sink = {
let mut g = graph.write().ok()?;
let inputs = push_placeholder_leaves(&mut g, region_arity(region));
fuel_graph::runtime_fused::emit_region(&mut g, region, &inputs, &[])
};
let roots = fuel_graph::opt::lower_to_base_map(&graph, &[sink]);
let root = *roots.first()?;
let g = graph.read().ok()?;
Some(fuel_graph::opt::base_map_hash(&g, root))
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
fn base_map_hash_of_fused(
claimed_op: fuel_graph::registry::FusedOpId,
arity: usize,
) -> Option<u64> {
let graph = std::sync::Arc::new(std::sync::RwLock::new(fuel_graph::Graph::new()));
let fused = {
let mut g = graph.write().ok()?;
let inputs = push_placeholder_leaves(&mut g, arity);
g.push(fuel_graph::Node {
op: fuel_graph::Op::Fused(claimed_op, fused_params_for(claimed_op)),
inputs,
shape: fuel_ir::Shape::from_dims(&[1]),
dtype: fuel_ir::DType::F32,
})
};
let roots = fuel_graph::opt::lower_to_base_map(&graph, &[fused]);
let root = *roots.first()?;
let g = graph.read().ok()?;
Some(fuel_graph::opt::base_map_hash(&g, root))
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
fn recipe_identity_matches(
claimed_op: fuel_graph::registry::FusedOpId,
submitted: &fuel_graph::jit::PatternNode,
) -> bool {
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let Some(submitted_hash) = base_map_hash_of_region(submitted) else {
return false;
};
let registered_hash = match fuel_graph::runtime_fused::runtime_region(claimed_op) {
Some(region) => base_map_hash_of_region(®ion),
None => base_map_hash_of_fused(claimed_op, region_arity(submitted)),
};
registered_hash == Some(submitted_hash)
}))
.unwrap_or(false)
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
fn fused_params_for(id: fuel_graph::registry::FusedOpId) -> fuel_graph::registry::FusedOpParams {
let _ = id;
fuel_graph::registry::FusedOpParams::Rope
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
fn claimed_op_lacks_numeric_bound(
claimed_op: Option<fuel_graph::registry::FusedOpId>,
declared: &crate::fused::PrecisionGuarantee,
) -> bool {
claimed_op.is_some()
&& declared.max_ulp.is_none()
&& declared.max_relative.is_none()
&& declared.max_absolute.is_none()
}
#[cfg(feature = "cuda")]
fn expected_input_arity(id: fuel_graph::registry::FusedOpId) -> Option<usize> {
if id == fuel_graph::registry::FusedOps::ROPE {
Some(3)
} else {
None
}
}
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
#[derive(Debug)]
pub enum VerifyVerdict {
Pass,
Fail { claim: &'static str, detail: String },
Inconclusive { claim: &'static str, detail: String },
}
#[cfg(feature = "cuda")]
struct FixedOutput(HostTensor);
#[cfg(feature = "cuda")]
impl KernelInvoker for FixedOutput {
fn invoke(
&self,
_entry: &BindingEntry,
_inputs: &[HostTensor],
) -> std::result::Result<HostTensor, VerifyError> {
Ok(self.0.clone())
}
}
#[cfg(feature = "cuda")]
fn verified_at_now() -> String {
use std::time::{SystemTime, UNIX_EPOCH};
let secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
format!("epoch:{secs}")
}
#[cfg(feature = "cuda")]
fn first_numeric_claim(g: &crate::fused::PrecisionGuarantee) -> &'static str {
if g.max_ulp.is_some() {
"max_ulp"
} else if g.max_relative.is_some() {
"max_relative"
} else {
"max_absolute"
}
}
#[cfg(feature = "cuda")]
fn check_numeric_bound(
cand_out: &HostTensor,
reference: &HostTensor,
entry: &BindingEntry,
probe: &ProbeInputs,
bound: Bound,
) -> std::result::Result<(), String> {
let cand_fx = FixedOutput(cand_out.clone());
let ref_fx = FixedOutput(reference.clone());
match verify_precision_bound(&cand_fx, &ref_fx, entry, std::slice::from_ref(probe), bound) {
Ok(VerifyOutcome::Pass) => Ok(()),
Ok(VerifyOutcome::Fail { detail }) => Err(detail),
Ok(VerifyOutcome::NoReference) => {
Err("verify_precision_bound returned NoReference".to_string())
}
Err(e) => Err(format!("{e:?}")),
}
}
#[cfg(feature = "cuda")]
fn run_region_diff(
region: &fuel_graph::jit::PatternNode,
out_dtype: fuel_ir::DType,
cand_bytes: &[u8],
probe: &ProbeInputs,
tol: fuel_kiss_ref_backend::Tolerance,
) -> std::result::Result<fuel_kiss_ref_backend::DiffReport, fuel_kiss_ref_backend::KissRefError> {
use fuel_ir::DType;
macro_rules! dispatch {
($decode:path, $diff:path) => {{
let cand = $decode(cand_bytes);
let cols: Vec<_> = probe.iter().map(|t| $decode(&t.bytes)).collect();
let refs: Vec<_> = cols.iter().map(|v| v.as_slice()).collect();
$diff(region, &cand, &refs, tol)
}};
}
match out_dtype {
DType::F32 => dispatch!(bytes_to_f32, fuel_kiss_ref_backend::diff_region_f32),
DType::F64 => dispatch!(bytes_to_f64, fuel_kiss_ref_backend::diff_region_f64),
DType::F16 => dispatch!(bytes_to_f16, fuel_kiss_ref_backend::diff_region_f16),
DType::BF16 => dispatch!(bytes_to_bf16, fuel_kiss_ref_backend::diff_region_bf16),
other => Err(fuel_kiss_ref_backend::KissRefError::UnsupportedDtype(other)),
}
}
#[cfg(feature = "cuda")]
fn resolve_reference_failure(
kiss_outcome: Option<&DiffOutcome>,
fail_claim: &'static str,
fail_detail: String,
ledger: &mut VerificationLedger,
make_record: &dyn Fn(&str, &str, serde_json::Value) -> LedgerRecord,
) -> (VerifyVerdict, Vec<LedgerRecord>) {
match classify_floor_verdict(kiss_outcome, None, None) {
VerifyVerdict::Inconclusive { claim, detail } => {
ledger.upsert(make_record(
claim,
"inconclusive",
serde_json::json!({
"detail": detail.clone(),
"realize_error": fail_detail,
"note": "recipe reference unusable; kiss-ref advisory escalates \
(flag, never verdict — §6.6-0007)",
}),
));
(
VerifyVerdict::Inconclusive { claim, detail },
ledger.records().to_vec(),
)
}
_ => {
ledger.upsert(make_record(
fail_claim,
"fail",
serde_json::json!({ "detail": fail_detail.clone() }),
));
(
VerifyVerdict::Fail {
claim: fail_claim,
detail: fail_detail,
},
ledger.records().to_vec(),
)
}
}
}
#[cfg(feature = "cuda")]
pub fn verify_candidate(
cand: &CandidateKernel,
device: &CudaDevice,
) -> (VerifyVerdict, Vec<LedgerRecord>) {
match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
verify_candidate_impl(cand, device)
})) {
Ok(result) => result,
Err(_) => (
VerifyVerdict::Fail {
claim: "panic",
detail: "candidate verification panicked (kernel invoke or reference realize)"
.to_string(),
},
Vec::new(),
),
}
}
#[cfg(feature = "cuda")]
fn claimed_op_structural_gate(
cand: &CandidateKernel,
probe: &ProbeInputs,
ledger: &mut VerificationLedger,
make_record: &dyn Fn(&str, &str, serde_json::Value) -> LedgerRecord,
) -> Result<(), (VerifyVerdict, Vec<LedgerRecord>)> {
let Some(claimed) = cand.claimed_op else {
return Ok(());
};
if let Some(dec) = &cand.decompose {
if !recipe_identity_matches(claimed, dec) {
let detail = "submitted recipe's base map differs from Fuel's registered \
recipe for the claimed op — not the same op"
.to_string();
ledger.upsert(make_record(
"recipe_identity",
"fail",
serde_json::json!({ "detail": detail.clone() }),
));
return Err((
VerifyVerdict::Fail {
claim: "recipe_identity",
detail,
},
ledger.records().to_vec(),
));
}
}
let expected_inputs = expected_input_arity(claimed).unwrap_or(cand.operands.len());
if probe.len() != expected_inputs {
let detail = format!(
"probe arity {} does not match the claimed op's expected input count {}",
probe.len(),
expected_inputs
);
ledger.upsert(make_record(
"probe_arity",
"fail",
serde_json::json!({ "detail": detail.clone() }),
));
return Err((
VerifyVerdict::Fail {
claim: "probe_arity",
detail,
},
ledger.records().to_vec(),
));
}
Ok(())
}
#[cfg(feature = "cuda")]
fn verify_candidate_impl(
cand: &CandidateKernel,
device: &CudaDevice,
) -> (VerifyVerdict, Vec<LedgerRecord>) {
let mut ledger = VerificationLedger::default();
let make_record = |claim: &str, result: &str, evidence: serde_json::Value| -> LedgerRecord {
LedgerRecord {
kernel_ref: cand.entry_point.clone(),
backend: "Cuda".to_string(),
dtypes: cand.dtypes.iter().map(|d| format!("{d:?}")).collect(),
kernel_revision_hash: cand.kernel_revision_hash,
claim: claim.to_string(),
result: result.to_string(),
verified_at: verified_at_now(),
protocol_version: 1,
evidence,
}
};
if claimed_op_lacks_numeric_bound(cand.claimed_op, &cand.declared) {
let detail = "a claimed-op candidate must declare a numeric bound (max_ulp/max_relative/\
max_absolute) to verify against the registered recipe"
.to_string();
ledger.upsert(make_record(
"no_guarantee",
"fail",
serde_json::json!({ "detail": detail.clone() }),
));
return (
VerifyVerdict::Fail {
claim: "no_guarantee",
detail,
},
ledger.records().to_vec(),
);
}
let seed = 0x5EED_C0DE_1234_5678_u64 ^ cand.kernel_revision_hash;
let probe = match crate::jit_ingest_probe::probe_from_operands(&cand.operands, seed) {
Some(p) if !p.is_empty() => p,
Some(_) => {
return (
VerifyVerdict::Fail {
claim: "probe",
detail: "candidate declares no operands to probe".to_string(),
},
Vec::new(),
);
}
None => return (
VerifyVerdict::Fail {
claim: "probe",
detail:
"candidate carries an operand whose dtype cannot be encoded as a probe input"
.to_string(),
},
Vec::new(),
),
};
let out_dtype = probe[0].dtype;
let out_shape = probe[0].shape.clone();
let numeric_declared = cand.declared.max_ulp.is_some()
|| cand.declared.max_relative.is_some()
|| cand.declared.max_absolute.is_some();
let advisory = advisory_region(cand.decompose.as_ref(), cand.claimed_op);
if numeric_declared && out_dtype != fuel_ir::DType::F32 {
let region_kiss_supported = advisory
.as_ref()
.is_some_and(|r| fuel_kiss_ref_backend::region_supported(r, out_dtype));
if !nonf32_escalate_eligible(out_dtype, region_kiss_supported) {
let claim = first_numeric_claim(&cand.declared);
let detail =
"numeric bound verification is f32-only; non-f32 candidate cannot be numerically verified yet"
.to_string();
ledger.upsert(make_record(
claim,
"fail",
serde_json::json!({ "detail": detail.clone(), "out_dtype": format!("{out_dtype:?}") }),
));
return (
VerifyVerdict::Fail { claim, detail },
ledger.records().to_vec(),
);
}
}
let entry = BindingEntry {
kernel: cand.kernel,
caps: crate::kernel::KernelCaps::empty(),
precision: cand.declared,
cost: crate::kernel::unknown_cost,
kernel_source: "candidate",
is_generic: false,
kernel_revision_hash: cand.kernel_revision_hash,
cost_expr: None,
};
let inv = CudaInvoker::new(device.clone(), out_dtype, out_shape.clone())
.with_params(cand.op_params.clone());
let cand_out = match inv.invoke(&entry, &probe) {
Ok(o) => o,
Err(e) => {
return (
VerifyVerdict::Fail {
claim: "invoke",
detail: format!("candidate kernel invoke failed: {e:?}"),
},
Vec::new(),
);
}
};
#[allow(unused_variables)]
let kiss_outcome: Option<DiffOutcome> = advisory.as_ref().and_then(|region| {
let dtype_supported = matches!(
out_dtype,
fuel_ir::DType::F32 | fuel_ir::DType::F64 | fuel_ir::DType::F16 | fuel_ir::DType::BF16
);
if !dtype_supported || !fuel_kiss_ref_backend::region_supported(region, out_dtype) {
return None;
}
let band = advisory_ulp_band(region);
let tol = band.map_or(
fuel_kiss_ref_backend::Tolerance::Exact,
fuel_kiss_ref_backend::Tolerance::Ulp,
);
let report = run_region_diff(region, out_dtype, &cand_out.bytes, &probe, tol).ok()?;
let op_count = fuel_kiss_ref_backend::region_op_count(region);
let source = if cand.decompose.is_some() {
"decompose"
} else {
"claimed_recipe"
};
ledger.upsert(make_record(
"kiss_ref_advisory",
if report.conforms() { "pass" } else { "flag" },
serde_json::json!({
"dtype": format!("{out_dtype:?}"),
"op_count": op_count,
"max_ulp": report.max_ulp,
"mismatches": report.mismatches,
"advisory_band_ulp": band,
"source": source,
"note": "advisory only; kiss-ref flags, never verdicts (§6.6-0007)"
}),
));
Some(DiffOutcome {
within: report.conforms(),
max_ulp: Some(report.max_ulp),
detail: format!(
"kiss-ref region diff (op_count={op_count}, max_ulp={}, mismatches={})",
report.max_ulp, report.mismatches
),
})
});
if cand.declared.bit_stable_on_same_hardware {
match verify_bit_stability(&inv, &entry, std::slice::from_ref(&probe), 16) {
Ok(VerifyOutcome::Pass) => ledger.upsert(make_record(
"bit_stable_on_same_hardware",
"pass",
serde_json::json!({ "repeat_calls": 16 }),
)),
Ok(VerifyOutcome::Fail { detail }) => {
ledger.upsert(make_record(
"bit_stable_on_same_hardware",
"fail",
serde_json::json!({ "detail": detail.clone() }),
));
return (
VerifyVerdict::Fail {
claim: "bit_stable_on_same_hardware",
detail,
},
ledger.records().to_vec(),
);
}
Ok(VerifyOutcome::NoReference) => {}
Err(e) => {
let detail = format!("bit-stability invoke failed: {e:?}");
ledger.upsert(make_record(
"bit_stable_on_same_hardware",
"fail",
serde_json::json!({ "detail": detail.clone() }),
));
return (
VerifyVerdict::Fail {
claim: "bit_stable_on_same_hardware",
detail,
},
ledger.records().to_vec(),
);
}
}
}
if numeric_declared {
let corpus = advisory
.as_ref()
.and_then(|region| single_primitive_optag(Some(region)))
.and_then(|op| corpus_verdict(op, out_dtype, seed));
if let Some(c) = &corpus {
ledger.upsert(make_record(
c.claim,
if c.adopt { "pass" } else { "fail" },
serde_json::json!({ "detail": c.detail.clone(), "source": "corpus" }),
));
return (
classify_floor_verdict(None, None, Some(c)),
ledger.records().to_vec(),
);
}
if out_dtype != fuel_ir::DType::F32 {
if let Err(early) = claimed_op_structural_gate(cand, &probe, &mut ledger, &make_record)
{
return early;
}
let claim = first_numeric_claim(&cand.declared);
let detail = "numeric bound verification is f32-only; non-f32 candidate escalated \
to the kiss-ref advisory (flag, never verdict — §6.6-0007)"
.to_string();
return resolve_reference_failure(
kiss_outcome.as_ref(),
claim,
detail,
&mut ledger,
&make_record,
);
}
let reference = match cand.claimed_op {
Some(claimed) => {
if let Err(early) =
claimed_op_structural_gate(cand, &probe, &mut ledger, &make_record)
{
return early;
}
let params = fused_params_for(claimed);
match crate::jit_ingest_probe::reference_from_registered_recipe(
claimed,
¶ms,
&probe,
out_dtype,
out_shape.clone(),
device,
) {
Ok(r) => r,
Err(e) => {
let claim = first_numeric_claim(&cand.declared);
let detail =
format!("reference realize from registered recipe failed: {e:?}");
return resolve_reference_failure(
kiss_outcome.as_ref(),
claim,
detail,
&mut ledger,
&make_record,
);
}
}
}
None => match &cand.decompose {
Some(dec) => match crate::jit_ingest_probe::reference_output(
dec,
&probe,
out_dtype,
out_shape.clone(),
device,
) {
Ok(r) => r,
Err(e) => {
let claim = first_numeric_claim(&cand.declared);
let detail = format!("reference realize from decompose failed: {e:?}");
return resolve_reference_failure(
kiss_outcome.as_ref(),
claim,
detail,
&mut ledger,
&make_record,
);
}
},
None => {
let claim = first_numeric_claim(&cand.declared);
let detail =
"no decompose: cannot verify numeric claim against a reference".to_string();
return resolve_reference_failure(
kiss_outcome.as_ref(),
claim,
detail,
&mut ledger,
&make_record,
);
}
},
};
let transcendental = cand
.decompose
.as_ref()
.is_some_and(|r| region_contains_transcendental(r));
let widen = |b: Bound| {
if transcendental {
widen_bound_for_transcendental(b)
} else {
b
}
};
if let Some(b) = cand.declared.max_ulp {
match check_numeric_bound(
&cand_out,
&reference,
&entry,
&probe,
widen(Bound::MaxUlp(b)),
) {
Ok(()) => ledger.upsert(make_record(
"max_ulp",
"pass",
serde_json::json!({ "bound": b, "transcendental_band": transcendental }),
)),
Err(detail) => {
ledger.upsert(make_record(
"max_ulp",
"fail",
serde_json::json!({ "detail": detail.clone(), "bound": b, "transcendental_band": transcendental }),
));
return (
VerifyVerdict::Fail {
claim: "max_ulp",
detail,
},
ledger.records().to_vec(),
);
}
}
}
if let Some(b) = cand.declared.max_relative {
match check_numeric_bound(
&cand_out,
&reference,
&entry,
&probe,
widen(Bound::MaxRelative(b)),
) {
Ok(()) => ledger.upsert(make_record(
"max_relative",
"pass",
serde_json::json!({ "bound": b, "transcendental_band": transcendental }),
)),
Err(detail) => {
ledger.upsert(make_record(
"max_relative",
"fail",
serde_json::json!({ "detail": detail.clone(), "bound": b, "transcendental_band": transcendental }),
));
return (
VerifyVerdict::Fail {
claim: "max_relative",
detail,
},
ledger.records().to_vec(),
);
}
}
}
if let Some(b) = cand.declared.max_absolute {
match check_numeric_bound(
&cand_out,
&reference,
&entry,
&probe,
widen(Bound::MaxAbsolute(b)),
) {
Ok(()) => ledger.upsert(make_record(
"max_absolute",
"pass",
serde_json::json!({ "bound": b, "transcendental_band": transcendental }),
)),
Err(detail) => {
ledger.upsert(make_record(
"max_absolute",
"fail",
serde_json::json!({ "detail": detail.clone(), "bound": b, "transcendental_band": transcendental }),
));
return (
VerifyVerdict::Fail {
claim: "max_absolute",
detail,
},
ledger.records().to_vec(),
);
}
}
}
}
(VerifyVerdict::Pass, ledger.records().to_vec())
}
#[cfg(feature = "cuda")]
pub fn ingest_one(cand: &CandidateKernel, device: &CudaDevice) -> IngestOutcome {
let (verdict, records) = verify_candidate(cand, device);
match verdict {
VerifyVerdict::Pass => adopt_verified(cand),
other => outcome_from_nonadopt_verdict(other, records, &cand.entry_point),
}
}
#[cfg(feature = "cuda")]
fn adopt_verified(cand: &CandidateKernel) -> IngestOutcome {
let Some(region) = cand.decompose.clone() else {
return IngestOutcome::Rejected(RejectionReport {
entry_point: cand.entry_point.clone(),
failed_claim: "no_decompose",
detail: "Pass verdict but candidate has no decompose region to adopt".to_string(),
ledger_record: None,
});
};
let entry_point = cand.entry_point.clone();
let kernel = cand.kernel;
let dtypes = cand.dtypes.clone();
let backend = cand.backend;
match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
adopt_runtime_fused(entry_point, region, kernel, dtypes, backend)
})) {
Ok(Some(id)) => IngestOutcome::Adopted(id),
Ok(None) => IngestOutcome::Rejected(RejectionReport {
entry_point: cand.entry_point.clone(),
failed_claim: "adopt_failed",
detail: "adopt_runtime_fused returned None (region not registrable)".to_string(),
ledger_record: None,
}),
Err(_) => IngestOutcome::Rejected(RejectionReport {
entry_point: cand.entry_point.clone(),
failed_claim: "adopt_failed",
detail: "adopt_runtime_fused panicked during registration".to_string(),
ledger_record: None,
}),
}
}
#[derive(Debug, Clone, Copy)]
pub struct IngestionConfig {
pub queue_bound: usize,
pub max_concurrent: usize,
pub idle_load_threshold: u32,
}
impl Default for IngestionConfig {
fn default() -> Self {
Self {
queue_bound: 32,
max_concurrent: 1,
idle_load_threshold: 1,
}
}
}
#[derive(Debug)]
pub struct Backpressure;
fn queue_full_report(cand: &CandidateKernel) -> RejectionReport {
RejectionReport {
entry_point: cand.entry_point.clone(),
failed_claim: "queue_full",
detail: format!(
"ingestion queue is at capacity; candidate '{}' dropped under backpressure",
cand.entry_point
),
ledger_record: None,
}
}
type IngestItem = (CandidateKernel, Option<Arc<dyn ProviderFeedback>>);
pub struct IngestionService {
sender: Option<SyncSender<IngestItem>>,
worker: Option<std::thread::JoinHandle<()>>,
}
impl IngestionService {
#[cfg(feature = "cuda")]
pub fn start(device: CudaDevice, cfg: IngestionConfig) -> Self {
Self::start_with_verify(move |cand| ingest_one(cand, &device), cfg)
}
pub fn start_with_verify<F>(verify: F, cfg: IngestionConfig) -> Self
where
F: Fn(&CandidateKernel) -> IngestOutcome + Send + 'static,
{
let (sender, receiver) = sync_channel::<IngestItem>(cfg.queue_bound);
let idle_load_threshold = cfg.idle_load_threshold.max(1);
let worker = std::thread::spawn(move || worker_loop(receiver, verify, idle_load_threshold));
Self {
sender: Some(sender),
worker: Some(worker),
}
}
pub fn enqueue(
&self,
cand: CandidateKernel,
feedback: Option<Arc<dyn ProviderFeedback>>,
) -> Result<(), Backpressure> {
let Some(sender) = &self.sender else {
return Err(Backpressure);
};
match sender.try_send((cand, feedback)) {
Ok(()) => Ok(()),
Err(TrySendError::Full((cand, feedback))) => {
if let Some(fb) = &feedback {
fb.on_rejected(&queue_full_report(&cand));
}
Err(Backpressure)
}
Err(TrySendError::Disconnected(_)) => Err(Backpressure),
}
}
pub fn shutdown(mut self) {
self.sender.take();
if let Some(worker) = self.worker.take() {
let _ = worker.join();
}
}
}
fn worker_loop<F>(receiver: Receiver<IngestItem>, verify: F, idle_load_threshold: u32)
where
F: Fn(&CandidateKernel) -> IngestOutcome,
{
while let Ok((cand, feedback)) = receiver.recv() {
while crate::dispatch::inflight_count(fuel_ir::DeviceLocation::Cuda { gpu_id: 0 })
>= idle_load_threshold
{
std::thread::sleep(Duration::from_millis(5));
}
let outcome = std::panic::catch_unwind(AssertUnwindSafe(|| match verify(&cand) {
IngestOutcome::Adopted(id) => {
if let Some(fb) = &feedback {
fb.on_adopted(&cand.entry_point, id);
}
}
IngestOutcome::Rejected(report) => {
if let Some(fb) = &feedback {
fb.on_rejected(&report);
}
}
IngestOutcome::Flagged(report) => {
if let Some(fb) = &feedback {
fb.on_flagged(&report);
}
}
}));
if outcome.is_err() {
eprintln!(
"jit_ingest: verify or feedback callback panicked for candidate '{}'; skipping (worker continues)",
cand.entry_point
);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn candidate_kernel_carries_claimed_op() {
let c = CandidateKernel {
entry_point: "k".into(),
kernel: dummy_kernel,
op_params: crate::kernel::OpParams::None,
decompose: None,
operands: vec![],
dtypes: vec![],
kernel_revision_hash: 0,
declared: crate::fused::PrecisionGuarantee::REFERENCE,
backend: fuel_ir::probe::BackendId::Cuda,
claimed_op: Some(fuel_graph::registry::FusedOps::ROPE),
};
assert_eq!(c.claimed_op, Some(fuel_graph::registry::FusedOps::ROPE));
}
#[test]
fn provider_feedback_receives_the_report() {
use std::sync::Mutex;
struct Rec(Mutex<Vec<String>>);
impl ProviderFeedback for Rec {
fn on_rejected(&self, r: &RejectionReport) {
self.0.lock().unwrap().push(r.failed_claim.into());
}
}
let rec = Rec(Mutex::new(vec![]));
rec.on_rejected(&RejectionReport {
entry_point: "k".into(),
failed_claim: "max_ulp",
detail: "d".into(),
ledger_record: None,
});
assert_eq!(rec.0.lock().unwrap().as_slice(), &["max_ulp".to_string()]);
}
#[test]
fn claimed_op_without_a_numeric_bound_is_refused() {
use fuel_graph::registry::FusedOps;
let empty = crate::fused::PrecisionGuarantee {
bit_stable_on_same_hardware: true,
max_ulp: None,
max_relative: None,
max_absolute: None,
notes: "test-only: bit-stable claim but no numeric bound",
};
assert!(
claimed_op_lacks_numeric_bound(Some(FusedOps::ROPE), &empty),
"a claimed-op candidate declaring only bit-stability (no numeric bound) must be refused"
);
assert!(!claimed_op_lacks_numeric_bound(
Some(FusedOps::ROPE),
&crate::fused::PrecisionGuarantee {
max_ulp: Some(1),
..empty
}
));
assert!(!claimed_op_lacks_numeric_bound(
Some(FusedOps::ROPE),
&crate::fused::PrecisionGuarantee {
max_relative: Some(1e-3),
..empty
}
));
assert!(!claimed_op_lacks_numeric_bound(
Some(FusedOps::ROPE),
&crate::fused::PrecisionGuarantee {
max_absolute: Some(1e-3),
..empty
}
));
assert!(!claimed_op_lacks_numeric_bound(None, &empty));
}
#[test]
fn recipe_identity_rejects_a_mismatched_submitted_decompose() {
use fuel_graph::jit::{OpAttrs, OpTag, PatternNode};
use fuel_graph::runtime_fused::register_runtime_fused;
let add_region = PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Bind { index: 0 },
PatternNode::Bind { index: 1 },
],
};
let mul_region = PatternNode::Op {
op: OpTag::Mul,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Bind { index: 0 },
PatternNode::Bind { index: 1 },
],
};
let claimed_id = register_runtime_fused("test::recipe_identity::add", add_region.clone())
.expect("register add region");
assert!(
recipe_identity_matches(claimed_id, &add_region),
"the submitted decompose that IS the registered recipe must match"
);
assert!(
!recipe_identity_matches(claimed_id, &mul_region),
"a Mul-region decompose must NOT match an Add-region registered recipe"
);
}
#[test]
#[ignore = "requires a live CUDA device"]
#[cfg(feature = "cuda")]
fn verify_candidate_add_f32_passes_against_its_decompose() {
use baracuda_kernels_types::{ElementKind, OperandDesc};
use fuel_cuda_backend::CudaDevice;
use fuel_graph::jit::{OpAttrs, OpTag, PatternNode};
use fuel_ir::DType;
use fuel_ir::probe::BackendId;
let Ok(dev) = CudaDevice::new(0) else {
return fuel_test_support::hardware::skip(
fuel_test_support::hardware::Hardware::Cuda,
fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
);
};
let decompose = PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Bind { index: 0 },
PatternNode::Bind { index: 1 },
],
};
let od = OperandDesc::new(1, &[4], &[1], ElementKind::F32, 16);
let cand = CandidateKernel {
entry_point: "add_f32".to_string(),
kernel: crate::baracuda_dispatch::binary::add_f32,
op_params: crate::kernel::OpParams::None,
decompose: Some(decompose),
operands: vec![od, od],
dtypes: vec![DType::F32, DType::F32],
kernel_revision_hash: 0x00AD_DF32,
declared: crate::fused::PrecisionGuarantee::REFERENCE,
backend: BackendId::Cuda,
claimed_op: None,
};
let (verdict, records) = verify_candidate(&cand, &dev);
assert!(
matches!(verdict, VerifyVerdict::Pass),
"expected Pass, got {verdict:?}"
);
assert_eq!(
records.len(),
5,
"one pass record per declared claim + the kiss-ref advisory: {records:?}"
);
assert!(
records.iter().all(|r| r.result == "pass"),
"all pass: {records:?}"
);
assert!(records.iter().all(|r| r.backend == "Cuda"));
assert!(
records
.iter()
.all(|r| r.kernel_revision_hash == 0x00AD_DF32)
);
assert!(records.iter().any(|r| r.claim == "kiss_ref_advisory"));
assert!(
records
.iter()
.any(|r| r.claim == "bit_stable_on_same_hardware")
);
assert!(records.iter().any(|r| r.claim == "max_ulp"));
assert!(records.iter().any(|r| r.claim == "max_relative"));
assert!(records.iter().any(|r| r.claim == "max_absolute"));
}
#[test]
#[ignore = "requires a live CUDA device"]
#[cfg(feature = "cuda")]
fn verify_candidate_refuses_numeric_claim_for_non_f32() {
use baracuda_kernels_types::{ElementKind, OperandDesc};
use fuel_cuda_backend::CudaDevice;
use fuel_graph::jit::{OpAttrs, OpTag, PatternNode};
use fuel_ir::DType;
use fuel_ir::probe::BackendId;
let Ok(dev) = CudaDevice::new(0) else {
return fuel_test_support::hardware::skip(
fuel_test_support::hardware::Hardware::Cuda,
fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
);
};
let decompose = PatternNode::Op {
op: OpTag::MatMul,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Bind { index: 0 },
PatternNode::Bind { index: 1 },
],
};
let od = OperandDesc::new(1, &[4], &[1], ElementKind::Bf16, 16);
let cand = CandidateKernel {
entry_point: "matmul_bf16".to_string(),
kernel: crate::baracuda_dispatch::binary::add_f32,
op_params: crate::kernel::OpParams::None,
decompose: Some(decompose),
operands: vec![od, od],
dtypes: vec![DType::BF16, DType::BF16],
kernel_revision_hash: 0x00AD_DF16,
declared: crate::fused::PrecisionGuarantee {
bit_stable_on_same_hardware: false,
max_ulp: Some(1),
max_relative: None,
max_absolute: None,
notes: "test-only, f32-only guard regression (kiss-uncoverable region)",
},
backend: BackendId::Cuda,
claimed_op: None,
};
let (verdict, records) = verify_candidate(&cand, &dev);
match verdict {
VerifyVerdict::Fail { claim, detail } => {
assert_eq!(claim, "max_ulp", "first (only) declared numeric claim");
assert!(
detail.contains("f32-only"),
"expected an f32-only refusal detail, got: {detail}"
);
}
other => panic!("expected Fail (f32-only guard, kiss-uncoverable), got {other:?}"),
}
assert_eq!(
records.len(),
1,
"one fail record for the refused claim: {records:?}"
);
assert_eq!(records[0].claim, "max_ulp");
assert_eq!(records[0].result, "fail");
}
#[test]
#[ignore = "requires a live CUDA device"]
#[cfg(feature = "cuda")]
fn verify_candidate_add_f64_is_inconclusive_not_failed() {
use baracuda_kernels_types::{ElementKind, OperandDesc};
use fuel_cuda_backend::CudaDevice;
use fuel_graph::jit::{OpAttrs, OpTag, PatternNode};
use fuel_ir::DType;
use fuel_ir::probe::BackendId;
let Ok(dev) = CudaDevice::new(0) else {
return fuel_test_support::hardware::skip(
fuel_test_support::hardware::Hardware::Cuda,
fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
);
};
let decompose = PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Bind { index: 0 },
PatternNode::Bind { index: 1 },
],
};
let od = OperandDesc::new(1, &[4], &[1], ElementKind::F64, 32);
let cand = CandidateKernel {
entry_point: "test::t5_escalate::add_f64".to_string(),
kernel: crate::baracuda_dispatch::binary::add_f64,
op_params: crate::kernel::OpParams::None,
decompose: Some(decompose),
operands: vec![od, od],
dtypes: vec![DType::F64, DType::F64],
kernel_revision_hash: 0x00AD_DF64,
declared: crate::fused::PrecisionGuarantee::REFERENCE,
backend: BackendId::Cuda,
claimed_op: None,
};
let (verdict, records) = verify_candidate(&cand, &dev);
match verdict {
VerifyVerdict::Inconclusive { claim, detail } => {
assert_eq!(claim, "max_ulp", "escalate names the evidence claim");
assert!(
detail.contains("kiss-ref"),
"escalate detail references the kiss-ref advisory: {detail}"
);
}
other => panic!("expected Inconclusive (T5 escalate), got {other:?}"),
}
let advisory = records
.iter()
.find(|r| r.claim == "kiss_ref_advisory")
.expect("a kiss-coverable f64 Add must reach the advisory cross-check");
assert_eq!(advisory.evidence["dtype"], serde_json::json!("F64"));
assert_eq!(
advisory.result, "pass",
"add_f64 matches kiss-ref add exactly: {advisory:?}"
);
assert!(
records
.iter()
.any(|r| r.claim == "max_ulp" && r.result == "inconclusive"),
"the escalate earns an inconclusive max_ulp record: {records:?}"
);
}
#[test]
#[ignore = "requires a live CUDA device"]
#[cfg(feature = "cuda")]
fn ingest_rejects_mul_candidate_for_the_add_region() {
use baracuda_kernels_types::{ElementKind, OperandDesc};
use fuel_cuda_backend::CudaDevice;
use fuel_graph::jit::{OpAttrs, OpTag, PatternNode};
use fuel_ir::DType;
use fuel_ir::probe::BackendId;
let Ok(dev) = CudaDevice::new(0) else {
return fuel_test_support::hardware::skip(
fuel_test_support::hardware::Hardware::Cuda,
fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
);
};
let decompose = PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Bind { index: 0 },
PatternNode::Bind { index: 1 },
],
};
let od = OperandDesc::new(1, &[4], &[1], ElementKind::F32, 16);
let cand = CandidateKernel {
entry_point: "test::ingest::mul_for_add_region".to_string(),
kernel: crate::baracuda_dispatch::binary::mul_f32,
op_params: crate::kernel::OpParams::None,
decompose: Some(decompose),
operands: vec![od, od],
dtypes: vec![DType::F32, DType::F32],
kernel_revision_hash: 0x1_9E57_0001,
declared: crate::fused::PrecisionGuarantee::REFERENCE,
backend: BackendId::Cuda,
claimed_op: None,
};
match ingest_one(&cand, &dev) {
IngestOutcome::Rejected(r) => {
assert!(
r.failed_claim.contains("max") || r.failed_claim == "vs_decompose",
"expected a precision claim naming the mismatch, got: {:?}",
r.failed_claim
);
}
IngestOutcome::Adopted(_) => {
panic!(
"mul_f32 must NOT be adopted for an Add region — it computes a different function"
)
}
IngestOutcome::Flagged(r) => panic!("unexpected Flagged: {} / {}", r.claim, r.detail),
}
}
#[test]
#[ignore = "requires a live CUDA device"]
#[cfg(feature = "cuda")]
fn ingest_adopts_add_candidate_for_the_add_region() {
use baracuda_kernels_types::{ElementKind, OperandDesc};
use fuel_cuda_backend::CudaDevice;
use fuel_graph::jit::{OpAttrs, OpTag, PatternNode};
use fuel_ir::DType;
use fuel_ir::probe::BackendId;
let Ok(dev) = CudaDevice::new(0) else {
return fuel_test_support::hardware::skip(
fuel_test_support::hardware::Hardware::Cuda,
fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
);
};
let decompose = PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Bind { index: 0 },
PatternNode::Bind { index: 1 },
],
};
let od = OperandDesc::new(1, &[4], &[1], ElementKind::F32, 16);
let cand = CandidateKernel {
entry_point: "test::ingest::add_for_add_region::run_1nge_5702".to_string(),
kernel: crate::baracuda_dispatch::binary::add_f32,
op_params: crate::kernel::OpParams::None,
decompose: Some(decompose),
operands: vec![od, od],
dtypes: vec![DType::F32, DType::F32],
kernel_revision_hash: 0x1_9E57_0002,
declared: crate::fused::PrecisionGuarantee::REFERENCE,
backend: BackendId::Cuda,
claimed_op: None,
};
match ingest_one(&cand, &dev) {
IngestOutcome::Adopted(id) => {
let table = crate::dispatch::global_bindings();
assert!(
crate::runtime_fused_kernels::fused_kernel_available_in(
&table,
id,
BackendId::Cuda
),
"the adopted candidate's kernel must be visible to the capability gate on Cuda",
);
}
IngestOutcome::Rejected(r) => panic!(
"add_f32 for an Add region must be Adopted, got Rejected: {} / {}",
r.failed_claim, r.detail
),
IngestOutcome::Flagged(r) => panic!("unexpected Flagged: {} / {}", r.claim, r.detail),
}
}
#[cfg(feature = "cuda")]
fn interleaved_rope_apply_candidate_kernel(
inputs: &[Arc<std::sync::RwLock<fuel_memory::Storage>>],
outputs: &mut [Arc<std::sync::RwLock<fuel_memory::Storage>>],
_layouts: &[fuel_ir::Layout],
params: &crate::kernel::OpParams,
) -> fuel_ir::Result<()> {
use crate::dispatch::{cuda_input, cuda_output, read_storage, write_storage};
use crate::kernel::OpParams;
if inputs.len() != 3 || outputs.len() != 1 {
return Err(fuel_ir::Error::Msg(format!(
"interleaved_rope_apply_candidate_kernel: expected 3 inputs (x, cos, sin) + 1 \
output, got {} + {}",
inputs.len(),
outputs.len()
)));
}
let (outer_count, seq, head_dim) = match params {
OpParams::Rope {
outer_count,
seq,
head_dim,
} => (*outer_count, *seq, *head_dim),
other => {
return Err(fuel_ir::Error::Msg(format!(
"interleaved_rope_apply_candidate_kernel: expected OpParams::Rope, got {other:?}"
)));
}
};
let x_guard = read_storage(&inputs[0])?;
let cos_guard = read_storage(&inputs[1])?;
let sin_guard = read_storage(&inputs[2])?;
let mut out_guard = write_storage(&outputs[0])?;
let x_cuda = cuda_input(&x_guard)?;
let cos_cuda = cuda_input(&cos_guard)?;
let sin_cuda = cuda_input(&sin_guard)?;
let out_cuda = cuda_output(&mut out_guard)?;
fuel_cuda_backend::baracuda::attention::rope_apply_fused_f32_into(
x_cuda,
cos_cuda,
sin_cuda,
outer_count,
seq,
head_dim,
out_cuda,
)
}
#[test]
#[ignore = "requires a live CUDA device"]
#[cfg(feature = "cuda")]
fn rope_oracle_rejects_interleaved_rope_claiming_rope() {
use baracuda_kernels_types::{ElementKind, OperandDesc};
use fuel_cuda_backend::CudaDevice;
use fuel_graph::registry::FusedOps;
use fuel_ir::DType;
use fuel_ir::probe::BackendId;
let Ok(dev) = CudaDevice::new(0) else {
return fuel_test_support::hardware::skip(
fuel_test_support::hardware::Hardware::Cuda,
fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
);
};
let (seq, head_dim) = (4usize, 8usize); let x_od = OperandDesc::new(
3,
&[1, seq as i64, head_dim as i64],
&[(seq * head_dim) as i64, head_dim as i64, 1],
ElementKind::F32,
16,
);
let trig_od = OperandDesc::new(
2,
&[seq as i64, head_dim as i64],
&[head_dim as i64, 1],
ElementKind::F32,
16,
);
let cand = CandidateKernel {
entry_point: "test::rope_oracle::interleaved_rope_apply_claims_rope".to_string(),
kernel: interleaved_rope_apply_candidate_kernel,
op_params: crate::kernel::OpParams::Rope {
outer_count: 1,
seq,
head_dim,
},
decompose: None,
operands: vec![x_od, trig_od, trig_od],
dtypes: vec![DType::F32, DType::F32, DType::F32],
kernel_revision_hash: 0x1_9E57_0003,
declared: crate::fused::PrecisionGuarantee::REFERENCE,
backend: BackendId::Cuda,
claimed_op: Some(FusedOps::ROPE),
};
match ingest_one(&cand, &dev) {
IngestOutcome::Rejected(r) => {
assert!(
r.failed_claim.contains("max") || r.failed_claim == "recipe_identity",
"expected a precision/identity rejection (interleaved != rotate-half), \
got: {} / {}",
r.failed_claim,
r.detail
);
assert!(
!r.detail.contains("reference realize"),
"rejection must be a genuine numeric mismatch vs the realized rotate-half \
reference, not a reference-realize failure: {}",
r.detail
);
}
IngestOutcome::Adopted(_) => panic!(
"interleaved rope claiming ROPE must NOT be adopted — it computes a different \
function than Fuel's rotate-half rope"
),
IngestOutcome::Flagged(r) => panic!("unexpected Flagged: {} / {}", r.claim, r.detail),
}
}
#[test]
#[ignore = "requires a live CUDA device"]
#[cfg(feature = "cuda")]
fn rope_oracle_adopts_matching_elementwise_claimant() {
use baracuda_kernels_types::{ElementKind, OperandDesc};
use fuel_cuda_backend::CudaDevice;
use fuel_graph::jit::{OpAttrs, OpTag, PatternNode};
use fuel_graph::runtime_fused::register_runtime_fused;
use fuel_ir::DType;
use fuel_ir::probe::BackendId;
let Ok(dev) = CudaDevice::new(0) else {
return fuel_test_support::hardware::skip(
fuel_test_support::hardware::Hardware::Cuda,
fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
);
};
let add_region = PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Bind { index: 0 },
PatternNode::Bind { index: 1 },
],
};
let claimed_id = register_runtime_fused(
"test::rope_oracle::adopt::claimed_add::run_r0pe_0004",
add_region.clone(),
)
.expect("register the runtime Add op the candidate claims");
let od = OperandDesc::new(1, &[4], &[1], ElementKind::F32, 16);
let cand = CandidateKernel {
entry_point: "test::rope_oracle::adopt::add_f32::run_r0pe_0004".to_string(),
kernel: crate::baracuda_dispatch::binary::add_f32,
op_params: crate::kernel::OpParams::None,
decompose: Some(add_region),
operands: vec![od, od],
dtypes: vec![DType::F32, DType::F32],
kernel_revision_hash: 0x1_9E57_0004,
declared: crate::fused::PrecisionGuarantee::REFERENCE,
backend: BackendId::Cuda,
claimed_op: Some(claimed_id),
};
match ingest_one(&cand, &dev) {
IngestOutcome::Adopted(id) => {
let table = crate::dispatch::global_bindings();
assert!(
crate::runtime_fused_kernels::fused_kernel_available_in(
&table,
id,
BackendId::Cuda
),
"the adopted claimant's kernel must be visible to the capability gate on Cuda"
);
}
IngestOutcome::Rejected(r) => panic!(
"a matching add_f32 claimant (recipe-identity + 0-ULP numeric) must be Adopted, \
got Rejected: {} / {}",
r.failed_claim, r.detail
),
IngestOutcome::Flagged(r) => panic!("unexpected Flagged: {} / {}", r.claim, r.detail),
}
}
#[test]
#[cfg(feature = "cuda")]
fn adopt_verified_rejects_a_candidate_without_a_decompose() {
use baracuda_kernels_types::{ElementKind, OperandDesc};
use fuel_ir::DType;
use fuel_ir::probe::BackendId;
let od = OperandDesc::new(1, &[4], &[1], ElementKind::F32, 16);
let cand = CandidateKernel {
entry_point: "test::adopt_verified::no_decompose".to_string(),
kernel: crate::baracuda_dispatch::binary::add_f32,
op_params: crate::kernel::OpParams::None,
decompose: None,
operands: vec![od, od],
dtypes: vec![DType::F32, DType::F32],
kernel_revision_hash: 0x1_9E57_9001,
declared: crate::fused::PrecisionGuarantee::REFERENCE,
backend: BackendId::Cuda,
claimed_op: None,
};
match adopt_verified(&cand) {
IngestOutcome::Rejected(r) => {
assert_eq!(r.failed_claim, "no_decompose");
}
IngestOutcome::Adopted(_) => {
panic!("a candidate with no decompose must never be adopted")
}
IngestOutcome::Flagged(r) => panic!("unexpected Flagged: {} / {}", r.claim, r.detail),
}
}
#[test]
#[cfg(feature = "cuda")]
fn adopt_verified_rejects_when_the_region_is_not_registrable() {
use baracuda_kernels_types::{ElementKind, OperandDesc};
use fuel_graph::jit::{OpAttrs, OpTag, PatternNode};
use fuel_ir::DType;
use fuel_ir::probe::BackendId;
let region = PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Bind { index: 0 },
PatternNode::Bind { index: 2 },
],
};
let od = OperandDesc::new(1, &[4], &[1], ElementKind::F32, 16);
let cand = CandidateKernel {
entry_point: "test::adopt_verified::non_contiguous_binds::run_9002".to_string(),
kernel: crate::baracuda_dispatch::binary::add_f32,
op_params: crate::kernel::OpParams::None,
decompose: Some(region),
operands: vec![od, od],
dtypes: vec![DType::F32, DType::F32],
kernel_revision_hash: 0x1_9E57_9002,
declared: crate::fused::PrecisionGuarantee::REFERENCE,
backend: BackendId::Cuda,
claimed_op: None,
};
match adopt_verified(&cand) {
IngestOutcome::Rejected(r) => {
assert_eq!(r.failed_claim, "adopt_failed");
}
IngestOutcome::Adopted(_) => {
panic!("a non-contiguous-bind region must never be adopted")
}
IngestOutcome::Flagged(r) => panic!("unexpected Flagged: {} / {}", r.claim, r.detail),
}
}
fn dummy_kernel(
_inputs: &[Arc<std::sync::RwLock<fuel_memory::Storage>>],
_outputs: &mut [Arc<std::sync::RwLock<fuel_memory::Storage>>],
_layouts: &[fuel_ir::Layout],
_params: &crate::kernel::OpParams,
) -> fuel_ir::Result<()> {
Ok(())
}
fn test_candidate(entry_point: &str) -> CandidateKernel {
use baracuda_kernels_types::{ElementKind, OperandDesc};
use fuel_ir::DType;
use fuel_ir::probe::BackendId;
let od = OperandDesc::new(1, &[4], &[1], ElementKind::F32, 16);
CandidateKernel {
entry_point: entry_point.to_string(),
kernel: dummy_kernel,
op_params: crate::kernel::OpParams::None,
decompose: None,
operands: vec![od, od],
dtypes: vec![DType::F32, DType::F32],
kernel_revision_hash: 0xDEAD_BEEF,
declared: crate::fused::PrecisionGuarantee::REFERENCE,
backend: BackendId::Cuda,
claimed_op: None,
}
}
#[derive(Default)]
struct RecordingFeedback {
rejected: std::sync::Mutex<Vec<String>>,
adopted: std::sync::Mutex<Vec<(String, fuel_graph::registry::FusedOpId)>>,
flagged: std::sync::Mutex<Vec<(String, bool, Option<String>)>>,
notify: std::sync::Mutex<Option<std::sync::mpsc::Sender<()>>>,
}
impl RecordingFeedback {
fn with_notify(tx: std::sync::mpsc::Sender<()>) -> Self {
Self {
notify: std::sync::Mutex::new(Some(tx)),
..Default::default()
}
}
fn fire_notify(&self) {
if let Some(tx) = self.notify.lock().unwrap().take() {
let _ = tx.send(());
}
}
}
impl ProviderFeedback for RecordingFeedback {
fn on_rejected(&self, report: &RejectionReport) {
self.rejected
.lock()
.unwrap()
.push(report.failed_claim.to_string());
self.fire_notify();
}
fn on_adopted(&self, entry_point: &str, id: fuel_graph::registry::FusedOpId) {
self.adopted
.lock()
.unwrap()
.push((entry_point.to_string(), id));
self.fire_notify();
}
fn on_flagged(&self, report: &FlagReport) {
self.flagged.lock().unwrap().push((
report.claim.to_string(),
report.escalate,
report.diff_summary.clone(),
));
self.fire_notify();
}
}
#[test]
fn enqueue_backpressures_and_notifies_when_full() {
let (started_tx, started_rx) = std::sync::mpsc::channel::<()>();
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
let cfg = IngestionConfig {
queue_bound: 1,
max_concurrent: 1,
idle_load_threshold: 1,
};
let svc = IngestionService::start_with_verify(
move |_cand| {
started_tx.send(()).ok();
release_rx.recv().ok();
IngestOutcome::Rejected(RejectionReport {
entry_point: "mock".to_string(),
failed_claim: "mock_verify_result",
detail: "mock verify (Task 7 backpressure test)".to_string(),
ledger_record: None,
})
},
cfg,
);
svc.enqueue(test_candidate("c1"), None)
.expect("first enqueue is accepted");
started_rx
.recv()
.expect("worker started processing the first candidate");
svc.enqueue(test_candidate("c2"), None)
.expect("second enqueue fills the 1-slot buffer");
let fb = Arc::new(RecordingFeedback::default());
let result = svc.enqueue(test_candidate("c3"), Some(fb.clone()));
assert!(
matches!(result, Err(Backpressure)),
"queue is full; expected Backpressure"
);
assert_eq!(
fb.rejected.lock().unwrap().as_slice(),
&["queue_full".to_string()],
"a full queue must synchronously notify the provided feedback with queue_full"
);
release_tx.send(()).ok();
release_tx.send(()).ok();
svc.shutdown();
}
#[test]
fn worker_fires_on_adopted_for_adopted_outcome() {
let cfg = IngestionConfig::default();
let adopted_id = fuel_graph::registry::FusedOpId(0x8001);
let svc = IngestionService::start_with_verify(
move |_cand| IngestOutcome::Adopted(adopted_id),
cfg,
);
let (tx, rx) = std::sync::mpsc::channel::<()>();
let fb = Arc::new(RecordingFeedback::with_notify(tx));
svc.enqueue(test_candidate("adopted-one"), Some(fb.clone()))
.expect("enqueue accepted");
rx.recv_timeout(std::time::Duration::from_secs(10))
.expect("worker should report the adopted outcome");
assert_eq!(
fb.adopted.lock().unwrap().as_slice(),
&[("adopted-one".to_string(), adopted_id)],
"on_adopted must fire with the entry_point and the adopted id"
);
assert!(fb.rejected.lock().unwrap().is_empty());
svc.shutdown();
}
#[test]
fn worker_does_not_stall_when_idle_threshold_is_zero() {
let cfg = IngestionConfig {
idle_load_threshold: 0,
..Default::default()
};
let adopted_id = fuel_graph::registry::FusedOpId(0x8003);
let svc = IngestionService::start_with_verify(
move |_cand| IngestOutcome::Adopted(adopted_id),
cfg,
);
let (tx, rx) = std::sync::mpsc::channel::<()>();
let fb = Arc::new(RecordingFeedback::with_notify(tx));
svc.enqueue(test_candidate("zero-threshold"), Some(fb.clone()))
.expect("enqueue accepted");
rx.recv_timeout(std::time::Duration::from_secs(10))
.expect("worker must not stall forever with idle_load_threshold == 0");
assert_eq!(
fb.adopted.lock().unwrap().as_slice(),
&[("zero-threshold".to_string(), adopted_id)],
"on_adopted must fire once the worker processes the item"
);
svc.shutdown();
}
#[test]
fn worker_survives_a_panicking_verify() {
use std::sync::atomic::{AtomicUsize, Ordering};
let call_count = Arc::new(AtomicUsize::new(0));
let call_count_worker = call_count.clone();
let cfg = IngestionConfig::default();
let svc = IngestionService::start_with_verify(
move |_cand| {
let n = call_count_worker.fetch_add(1, Ordering::SeqCst);
if n == 0 {
panic!("mock verify panics on purpose (Task 7 panic-survival test)");
}
IngestOutcome::Rejected(RejectionReport {
entry_point: "after-panic".to_string(),
failed_claim: "mock_after_panic",
detail: "mock verify, second call, after the first panicked".to_string(),
ledger_record: None,
})
},
cfg,
);
svc.enqueue(test_candidate("panics"), None)
.expect("enqueue accepted");
let (tx, rx) = std::sync::mpsc::channel::<()>();
let fb = Arc::new(RecordingFeedback::with_notify(tx));
svc.enqueue(test_candidate("after-panic"), Some(fb.clone()))
.expect("enqueue accepted");
rx.recv_timeout(std::time::Duration::from_secs(10))
.expect("worker must survive the panic and process the next item");
assert_eq!(
fb.rejected.lock().unwrap().as_slice(),
&["mock_after_panic".to_string()],
"the post-panic item must be processed normally"
);
assert_eq!(
call_count.load(Ordering::SeqCst),
2,
"both candidates reached verify"
);
svc.shutdown();
}
#[test]
fn worker_survives_a_panicking_feedback_callback() {
use std::sync::atomic::{AtomicUsize, Ordering};
struct PanickingFeedback;
impl ProviderFeedback for PanickingFeedback {
fn on_rejected(&self, _report: &RejectionReport) {
panic!("mock on_rejected panics on purpose (Task 7 callback-panic-survival test)");
}
fn on_adopted(&self, _entry_point: &str, _id: fuel_graph::registry::FusedOpId) {
panic!("mock on_adopted panics on purpose (Task 7 callback-panic-survival test)");
}
}
let call_count = Arc::new(AtomicUsize::new(0));
let call_count_worker = call_count.clone();
let adopted_id = fuel_graph::registry::FusedOpId(0x8002);
let cfg = IngestionConfig::default();
let svc = IngestionService::start_with_verify(
move |_cand| {
call_count_worker.fetch_add(1, Ordering::SeqCst);
IngestOutcome::Adopted(adopted_id)
},
cfg,
);
let panicking_fb = Arc::new(PanickingFeedback);
svc.enqueue(test_candidate("panics-in-callback"), Some(panicking_fb))
.expect("enqueue accepted");
let (tx, rx) = std::sync::mpsc::channel::<()>();
let fb = Arc::new(RecordingFeedback::with_notify(tx));
svc.enqueue(test_candidate("after-callback-panic"), Some(fb.clone()))
.expect("enqueue accepted");
rx.recv_timeout(std::time::Duration::from_secs(10))
.expect("worker must survive the on_adopted panic and process the next item");
assert_eq!(
fb.adopted.lock().unwrap().as_slice(),
&[("after-callback-panic".to_string(), adopted_id)],
"the post-panic item must be processed normally"
);
assert_eq!(
call_count.load(Ordering::SeqCst),
2,
"both candidates reached verify"
);
svc.shutdown();
}
#[test]
fn worker_routes_flagged_to_on_flagged() {
let cfg = IngestionConfig::default();
let svc = IngestionService::start_with_verify(
move |_cand| {
IngestOutcome::Flagged(FlagReport {
entry_point: "flagged-one".to_string(),
claim: "max_ulp",
detail: "kiss-ref advisory escalate (mock, Task 7 seam)".to_string(),
diff_summary: Some("result=flag max_ulp=3 mismatches=2".to_string()),
escalate: true,
})
},
cfg,
);
let (tx, rx) = std::sync::mpsc::channel::<()>();
let fb = Arc::new(RecordingFeedback::with_notify(tx));
svc.enqueue(test_candidate("flagged-one"), Some(fb.clone()))
.expect("enqueue accepted");
rx.recv_timeout(std::time::Duration::from_secs(10))
.expect("worker should route the flagged outcome to on_flagged");
let flagged = fb.flagged.lock().unwrap();
assert_eq!(
flagged.len(),
1,
"exactly one on_flagged callback: {flagged:?}"
);
let (claim, escalate, summary) = &flagged[0];
assert_eq!(claim, "max_ulp", "the flag names the evidence claim");
assert!(*escalate, "the flag escalates to mint a corpus vector");
assert_eq!(
summary.as_deref(),
Some("result=flag max_ulp=3 mismatches=2"),
"the threaded kiss-ref diff summary reaches the provider"
);
assert!(
fb.rejected.lock().unwrap().is_empty(),
"a Flagged outcome must not route to on_rejected"
);
assert!(
fb.adopted.lock().unwrap().is_empty(),
"a Flagged outcome must not route to on_adopted"
);
svc.shutdown();
}
#[cfg(feature = "cuda")]
#[derive(Default)]
struct E2eFeedback {
adopted: std::sync::Mutex<Vec<(String, fuel_graph::registry::FusedOpId)>>,
rejected: std::sync::Mutex<Vec<RejectionReportSnapshot>>,
flagged: std::sync::Mutex<Vec<FlagReportSnapshot>>,
notify: std::sync::Mutex<Option<std::sync::mpsc::Sender<()>>>,
}
#[cfg(feature = "cuda")]
#[derive(Debug)]
struct RejectionReportSnapshot {
failed_claim: &'static str,
detail: String,
}
#[cfg(feature = "cuda")]
#[derive(Debug)]
struct FlagReportSnapshot {
claim: &'static str,
detail: String,
diff_summary: Option<String>,
escalate: bool,
}
#[cfg(feature = "cuda")]
impl E2eFeedback {
fn with_notify(tx: std::sync::mpsc::Sender<()>) -> Self {
Self {
notify: std::sync::Mutex::new(Some(tx)),
..Default::default()
}
}
fn fire_notify(&self) {
if let Some(tx) = self.notify.lock().unwrap().take() {
let _ = tx.send(());
}
}
}
#[cfg(feature = "cuda")]
impl ProviderFeedback for E2eFeedback {
fn on_rejected(&self, report: &RejectionReport) {
self.rejected.lock().unwrap().push(RejectionReportSnapshot {
failed_claim: report.failed_claim,
detail: report.detail.clone(),
});
self.fire_notify();
}
fn on_adopted(&self, entry_point: &str, id: fuel_graph::registry::FusedOpId) {
self.adopted
.lock()
.unwrap()
.push((entry_point.to_string(), id));
self.fire_notify();
}
fn on_flagged(&self, report: &FlagReport) {
self.flagged.lock().unwrap().push(FlagReportSnapshot {
claim: report.claim,
detail: report.detail.clone(),
diff_summary: report.diff_summary.clone(),
escalate: report.escalate,
});
self.fire_notify();
}
}
#[cfg(feature = "cuda")]
fn e2e_add_candidate() -> CandidateKernel {
use baracuda_kernels_types::{ElementKind, OperandDesc};
use fuel_graph::jit::{OpAttrs, OpTag, PatternNode};
use fuel_ir::DType;
use fuel_ir::probe::BackendId;
let decompose = PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Bind { index: 0 },
PatternNode::Bind { index: 1 },
],
};
let od = OperandDesc::new(1, &[4], &[1], ElementKind::F32, 16);
CandidateKernel {
entry_point: "test::e2e::add_for_add_region::run_e2e_8801".to_string(),
kernel: crate::baracuda_dispatch::binary::add_f32,
op_params: crate::kernel::OpParams::None,
decompose: Some(decompose),
operands: vec![od, od],
dtypes: vec![DType::F32, DType::F32],
kernel_revision_hash: 0x1_9E57_8801,
declared: crate::fused::PrecisionGuarantee::REFERENCE,
backend: BackendId::Cuda,
claimed_op: None,
}
}
#[cfg(feature = "cuda")]
fn e2e_mul_candidate() -> CandidateKernel {
use baracuda_kernels_types::{ElementKind, OperandDesc};
use fuel_graph::jit::{OpAttrs, OpTag, PatternNode};
use fuel_ir::DType;
use fuel_ir::probe::BackendId;
let decompose = PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Bind { index: 0 },
PatternNode::Bind { index: 1 },
],
};
let od = OperandDesc::new(1, &[4], &[1], ElementKind::F32, 16);
CandidateKernel {
entry_point: "test::e2e::mul_for_add_region::run_e2e_8802".to_string(),
kernel: crate::baracuda_dispatch::binary::mul_f32,
op_params: crate::kernel::OpParams::None,
decompose: Some(decompose),
operands: vec![od, od],
dtypes: vec![DType::F32, DType::F32],
kernel_revision_hash: 0x1_9E57_8802,
declared: crate::fused::PrecisionGuarantee::REFERENCE,
backend: BackendId::Cuda,
claimed_op: None,
}
}
#[cfg(feature = "cuda")]
fn e2e_add_f64_candidate() -> CandidateKernel {
use baracuda_kernels_types::{ElementKind, OperandDesc};
use fuel_graph::jit::{OpAttrs, OpTag, PatternNode};
use fuel_ir::DType;
use fuel_ir::probe::BackendId;
let decompose = PatternNode::Op {
op: OpTag::Add,
attrs: OpAttrs::default(),
operands: vec![
PatternNode::Bind { index: 0 },
PatternNode::Bind { index: 1 },
],
};
let od = OperandDesc::new(1, &[4], &[1], ElementKind::F64, 32);
CandidateKernel {
entry_point: "test::e2e::add_f64_escalate::run_e2e_8803".to_string(),
kernel: crate::baracuda_dispatch::binary::add_f64,
op_params: crate::kernel::OpParams::None,
decompose: Some(decompose),
operands: vec![od, od],
dtypes: vec![DType::F64, DType::F64],
kernel_revision_hash: 0x1_9E57_8803,
declared: crate::fused::PrecisionGuarantee::REFERENCE,
backend: BackendId::Cuda,
claimed_op: None,
}
}
#[test]
#[ignore = "requires a live CUDA device"]
#[cfg(feature = "cuda")]
fn ingestion_service_adopts_a_matching_add_candidate_e2e() {
use fuel_ir::probe::BackendId;
let Ok(dev) = CudaDevice::new(0) else {
return fuel_test_support::hardware::skip(
fuel_test_support::hardware::Hardware::Cuda,
fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
);
};
let svc = IngestionService::start(dev, IngestionConfig::default());
let (tx, rx) = std::sync::mpsc::channel::<()>();
let fb = Arc::new(E2eFeedback::with_notify(tx));
svc.enqueue(e2e_add_candidate(), Some(fb.clone()))
.expect("enqueue accepted (queue starts empty)");
rx.recv_timeout(Duration::from_secs(30))
.expect("worker should report the adopted outcome within 30s");
let adopted = fb.adopted.lock().unwrap();
assert_eq!(
adopted.len(),
1,
"exactly one on_adopted callback: {adopted:?}"
);
let (entry_point, id) = &adopted[0];
assert_eq!(entry_point, "test::e2e::add_for_add_region::run_e2e_8801");
assert!(
fb.rejected.lock().unwrap().is_empty(),
"the matching add_f32 candidate must not be rejected"
);
let table = crate::dispatch::global_bindings();
assert!(
crate::runtime_fused_kernels::fused_kernel_available_in(&table, *id, BackendId::Cuda),
"the adopted candidate's kernel must be visible to the capability gate on Cuda"
);
drop(table);
svc.shutdown();
}
#[test]
#[ignore = "requires a live CUDA device"]
#[cfg(feature = "cuda")]
fn ingestion_service_rejects_a_mismatched_mul_candidate_e2e() {
let Ok(dev) = CudaDevice::new(0) else {
return fuel_test_support::hardware::skip(
fuel_test_support::hardware::Hardware::Cuda,
fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
);
};
let svc = IngestionService::start(dev, IngestionConfig::default());
let (tx, rx) = std::sync::mpsc::channel::<()>();
let fb = Arc::new(E2eFeedback::with_notify(tx));
svc.enqueue(e2e_mul_candidate(), Some(fb.clone()))
.expect("enqueue accepted (queue starts empty)");
rx.recv_timeout(Duration::from_secs(30))
.expect("worker should report the rejected outcome within 30s");
let rejected = fb.rejected.lock().unwrap();
assert_eq!(
rejected.len(),
1,
"exactly one on_rejected callback: {rejected:?}"
);
assert!(
rejected[0].failed_claim.contains("max"),
"expected a precision claim naming the mismatch, got: {} / {}",
rejected[0].failed_claim,
rejected[0].detail
);
assert!(
fb.adopted.lock().unwrap().is_empty(),
"the mismatched mul_f32 candidate must not be adopted"
);
svc.shutdown();
}
#[test]
#[ignore = "requires a live CUDA device"]
#[cfg(feature = "cuda")]
fn ingestion_service_flags_an_f64_add_candidate_e2e() {
let Ok(dev) = CudaDevice::new(0) else {
return fuel_test_support::hardware::skip(
fuel_test_support::hardware::Hardware::Cuda,
fuel_test_support::hardware::Missing::device("CudaDevice::new(0) failed"),
);
};
let svc = IngestionService::start(dev, IngestionConfig::default());
let (tx, rx) = std::sync::mpsc::channel::<()>();
let fb = Arc::new(E2eFeedback::with_notify(tx));
svc.enqueue(e2e_add_f64_candidate(), Some(fb.clone()))
.expect("enqueue accepted (queue starts empty)");
rx.recv_timeout(Duration::from_secs(30))
.expect("worker should report the flagged outcome within 30s");
let flagged = fb.flagged.lock().unwrap();
assert_eq!(
flagged.len(),
1,
"exactly one on_flagged callback: {flagged:?}"
);
assert_eq!(
flagged[0].claim, "max_ulp",
"the escalate names the evidence claim"
);
assert!(
flagged[0].escalate,
"the flag escalates to mint a corpus vector"
);
assert!(
flagged[0].diff_summary.is_some(),
"the escalate threads the kiss-ref advisory summary (D8): {:?}",
flagged[0]
);
assert!(
flagged[0].detail.contains("kiss-ref"),
"the escalate detail references the kiss-ref advisory: {}",
flagged[0].detail
);
assert!(
fb.adopted.lock().unwrap().is_empty(),
"a non-f32 escalate must not be adopted (Fuel renders no verdict)"
);
assert!(
fb.rejected.lock().unwrap().is_empty(),
"a kiss-coverable escalate must not be rejected — §6.6-0007: flag, never verdict"
);
svc.shutdown();
}
}