1use std::collections::{BTreeSet, HashMap, HashSet};
9#[cfg(feature = "runtime")]
10use std::collections::BTreeMap;
11use std::path::Path;
12#[cfg(feature = "runtime")]
13use std::path::PathBuf;
14
15use anyhow::Result;
16#[cfg(feature = "runtime")]
17use anyhow::Context;
18use quote::ToTokens;
19use syn::visit::{self, Visit};
20
21use crate::cargo_context::{CargoContext, CargoOptions};
22use crate::dataflow::{self, GradState, NodeKind, NumericImpact};
23use crate::extract::Extractor;
24use crate::ir::{Acquisition, Certainty, CheckpointMatch};
25use crate::load::{self, Crate, ImplFn, StructDef};
26use crate::model_ir::{
27 ArchitectureEdge, Artifact, ArtifactKind, AssemblySite, BuilderNamespace, BuilderRole,
28 BuilderSourceKind, CargoSummary, Component, Confidence, DeviceFact, Evidence, EvidenceKind,
29 Finding, FindingSeverity, Function, FunctionParameter, LayoutFact, ModelIr, Module, Operation,
30 OptimizerMembership, Parameter, ParameterRole, PipelineStage, ShapeFact,
31 StableId, StageDispatchKind, StageKind, TensorContract, TensorRole, Visibility,
32};
33#[cfg(feature = "runtime")]
34use crate::model_ir::{EdgeTimingSummary, RuntimeSummary};
35use crate::op_semantics::{self, GradFlow};
36#[cfg(feature = "runtime")]
37use crate::runtime::{self, ExpectedIdentity, GradientState, RuntimeTrace};
38
39#[derive(Debug, Clone, Default)]
40pub struct ScanOptions {
41 pub cargo: CargoOptions,
42 #[cfg(feature = "runtime")]
43 pub runtime_trace: Option<PathBuf>,
44 pub component_root: Option<String>,
46 pub dataflow: bool,
48 pub heuristic_architecture: bool,
51}
52
53pub fn analyze(path: impl AsRef<Path>, options: &ScanOptions) -> Result<ModelIr> {
55 let path = path.as_ref();
56 let cargo_result = CargoContext::discover(path, &options.cargo);
57 let mut krate = match cargo_result.as_ref() {
58 Ok(context) => {
59 let roots = context.selected_source_roots(options.cargo.package_target.as_deref())?;
60 load::load_from_roots(path, &roots)?
61 }
62 Err(_) => load::load(path)?,
63 };
64 if let Ok(context) = cargo_result.as_ref() {
65 krate.set_dependency_aliases(context.dependency_aliases.clone());
66 }
67 if krate.all_structs().next().is_none() {
68 anyhow::bail!("no Rust structs found under {}", path.display());
69 }
70
71 let analysis_id = match cargo_result.as_ref() {
72 Ok(context) => StableId::new(
73 "analysis",
74 [cargo_build_id(
75 context,
76 options.cargo.package_target.as_deref(),
77 )],
78 ),
79 Err(_) => StableId::new("analysis", [canonical_label(path)]),
80 };
81 let mut model = ModelIr::empty(analysis_id);
82
83 let cargo = match cargo_result {
84 Ok(context) => {
85 model.cargo = Some(cargo_summary(
86 &context,
87 options.cargo.package_target.as_deref(),
88 ));
89 Some(context)
90 }
91 Err(error) => {
92 push_finding(
93 &mut model,
94 "cargo-context",
95 FindingSeverity::Warning,
96 Confidence::Proven,
97 format!("Cargo context unavailable: {error:#}"),
98 None,
99 Vec::new(),
100 );
101 None
102 }
103 };
104 for diagnostic in &krate.diagnostics {
105 push_finding(
106 &mut model,
107 "source-load",
108 FindingSeverity::Warning,
109 Confidence::Unknown,
110 format!("incomplete source analysis: {}", diagnostic.message),
111 Some(diagnostic.path.clone()),
112 Vec::new(),
113 );
114 }
115
116 let function_lookup = build_functions(&krate, cargo.as_ref(), &mut model);
117 link_calls(&krate, &function_lookup, &mut model);
118 discover_components(
119 &krate,
120 cargo.as_ref(),
121 options.component_root.as_deref(),
122 options.heuristic_architecture,
123 &mut model,
124 );
125 discover_composition_edges(&krate, &mut model);
126 discover_assembly_sites(&krate, &mut model);
127 add_contracts(&krate, &mut model);
128 if options.heuristic_architecture {
129 discover_architecture_edges(&krate, &mut model);
130 discover_subprocess_pipeline(&krate, &function_lookup, &mut model);
131 discover_pipeline_and_artifacts(&krate, &function_lookup, &mut model);
132 discover_optimizers(&krate, &mut model);
133 apply_optimizer_roles(&mut model);
134 } else {
135 push_finding(
136 &mut model,
137 "compiler-semantic-evidence",
138 FindingSeverity::Information,
139 Confidence::Unknown,
140 "architecture, pipeline, artifact, and optimizer relationships are unavailable until \
141 compiler-resolved value-flow evidence is implemented; use --heuristic-architecture \
142 for exploratory name- and call-order leads"
143 .to_string(),
144 None,
145 Vec::new(),
146 );
147 }
148
149 if options.dataflow {
150 add_dataflow(&krate, &mut model);
151 }
152
153 #[cfg(feature = "runtime")]
154 if let Some(path) = options.runtime_trace.as_deref() {
155 let text = std::fs::read_to_string(path)
156 .with_context(|| format!("reading runtime trace {}", path.display()))?;
157 let trace = runtime::parse(&text)
158 .with_context(|| format!("parsing runtime trace {}", path.display()))?;
159 merge_runtime(&mut model, &trace);
160 }
161
162 if let Some(cargo) = cargo {
163 flag_candle_semantics_version(&mut model, &cargo);
164 }
165 model.normalize();
166 Ok(model)
167}
168
169fn canonical_label(path: &Path) -> String {
170 path.canonicalize()
171 .unwrap_or_else(|_| path.to_path_buf())
172 .to_string_lossy()
173 .into_owned()
174}
175
176fn selected_target(context: &CargoContext, requested_target: Option<&str>) -> Option<String> {
177 requested_target.map(str::to_string).or_else(|| {
178 context
179 .targets
180 .iter()
181 .find(|target| target.kind.iter().any(|kind| kind == "lib"))
182 .or_else(|| {
183 context
184 .targets
185 .iter()
186 .find(|target| target.kind.iter().any(|kind| kind == "bin"))
187 })
188 .map(|target| target.name.clone())
189 })
190}
191
192fn cargo_build_id(context: &CargoContext, requested_target: Option<&str>) -> String {
193 let target = selected_target(context, requested_target).unwrap_or_else(|| "unknown".into());
194 let mut identity = String::from("candle-graph/build/1\0");
195 for part in [
196 context.package_name.as_str(),
197 context.package_version.as_str(),
198 target.as_str(),
199 ] {
200 identity.push_str(part);
201 identity.push('\0');
202 }
203 for feature in &context.active_features {
204 identity.push_str("feature=");
205 identity.push_str(feature);
206 identity.push('\0');
207 }
208 for cfg in &context.cfgs {
209 identity.push_str("cfg=");
210 identity.push_str(cfg);
211 identity.push('\0');
212 }
213 for (package, version) in &context.candle_versions {
214 identity.push_str("candle=");
215 identity.push_str(package);
216 identity.push('@');
217 identity.push_str(version);
218 identity.push('\0');
219 }
220
221 let hash = identity.bytes().fold(0xcbf29ce484222325_u64, |hash, byte| {
223 (hash ^ u64::from(byte)).wrapping_mul(0x100000001b3)
224 });
225 format!("candle-graph/build/1:{hash:016x}")
226}
227
228fn cargo_summary(context: &CargoContext, requested_target: Option<&str>) -> CargoSummary {
229 CargoSummary {
230 build_id: cargo_build_id(context, requested_target),
231 workspace_root: context.workspace_root.to_string_lossy().into_owned(),
232 manifest_path: context.manifest_path.to_string_lossy().into_owned(),
233 package_name: context.package_name.clone(),
234 package_version: context.package_version.clone(),
235 selected_target: selected_target(context, requested_target),
236 active_features: context.active_features.clone(),
237 active_cfg: context.cfgs.clone(),
238 candle_packages: context.candle_versions.clone(),
239 }
240}
241
242fn build_functions(
243 krate: &Crate,
244 cargo: Option<&CargoContext>,
245 model: &mut ModelIr,
246) -> HashMap<String, StableId> {
247 let mut lookup = HashMap::new();
248 for func in krate.all_functions().chain(krate.all_methods()) {
249 let cfg_active = cargo.and_then(|context| {
250 crate::cargo_context::cfg_predicates_active(&func.cfg_predicates, &context.cfgs)
251 });
252 if cfg_active == Some(false) {
253 continue;
254 }
255 let id = function_id(func);
256 lookup.insert(func.qualified_name.clone(), id.clone());
257 let tensor_signature = func.param_types.iter().any(|ty| is_tensor_type(ty))
258 || is_tensor_type(&func.return_type);
259 let is_loss = has_explicit_candle_loss_call(krate, func);
260 let is_entrypoint =
263 is_candle_module_entry(func) || is_loss || tensor_signature && func.visibility == "pub";
264 model.functions.push(Function {
265 id,
266 name: func.fn_name.clone(),
267 qualified_name: func.qualified_name.clone(),
268 owner_type: (!func.qualified_type_name.is_empty())
269 .then(|| func.qualified_type_name.clone()),
270 visibility: visibility(&func.visibility),
271 parameters: func
272 .params
273 .iter()
274 .zip(&func.param_types)
275 .map(|(name, type_name)| FunctionParameter {
276 name: name.clone(),
277 type_name: type_name.clone(),
278 })
279 .collect(),
280 return_type: (func.return_type != "()").then(|| func.return_type.clone()),
281 cfg_predicates: func.cfg_predicates.clone(),
282 cfg_active,
283 source: krate.file_label(func.span),
284 calls: Vec::new(),
285 tensor_inputs: Vec::new(),
286 tensor_outputs: Vec::new(),
287 is_entrypoint,
288 is_loss,
289 execution_phases: Vec::new(),
290 });
291 }
292 lookup
293}
294
295fn is_candle_module_entry(func: &ImplFn) -> bool {
296 let Some(trait_name) = func.trait_name.as_deref() else {
297 return false;
298 };
299 let trait_leaf = trait_name.rsplit("::").next().unwrap_or(trait_name);
300 matches!(
301 (trait_leaf, func.fn_name.as_str()),
302 ("Module", "forward")
303 | ("ModuleT", "forward_t")
304 | ("ModuleWithArgs", "forward")
305 | ("ModuleTWithArgs", "forward_t")
306 )
307}
308
309fn has_explicit_candle_loss_call(krate: &Crate, func: &ImplFn) -> bool {
310 let mut collector = CallCollector::default();
311 collector.visit_block(&func.block);
312 collector.calls.iter().any(|call| {
313 let segments = call.split("::").map(str::to_string).collect::<Vec<_>>();
314 let resolved = krate
315 .resolve_import_path(&func.module_path, &segments)
316 .join("::");
317 resolved
318 .strip_prefix("candle_nn::loss::")
319 .is_some_and(|leaf| {
320 matches!(
321 leaf,
322 "nll" | "cross_entropy" | "mse" | "binary_cross_entropy_with_logit" | "huber"
323 )
324 })
325 })
326}
327
328fn link_calls(krate: &Crate, lookup: &HashMap<String, StableId>, model: &mut ModelIr) {
329 let mut by_bare: HashMap<String, Vec<StableId>> = HashMap::new();
330 for function in &model.functions {
331 by_bare
332 .entry(function.name.clone())
333 .or_default()
334 .push(function.id.clone());
335 }
336 let index: HashMap<StableId, usize> = model
337 .functions
338 .iter()
339 .enumerate()
340 .map(|(index, function)| (function.id.clone(), index))
341 .collect();
342
343 for func in krate.all_functions().chain(krate.all_methods()) {
344 let mut collector = CallCollector::default();
345 collector.visit_block(&func.block);
346 let caller = function_id(func);
347 let Some(&caller_index) = index.get(&caller) else {
348 continue;
349 };
350 let mut calls = Vec::new();
351 for call in collector.calls {
352 if let Some(id) = resolve_call(&call, func, lookup, &by_bare) {
353 calls.push(id);
354 }
355 }
356 calls.sort();
357 calls.dedup();
358 model.functions[caller_index].calls = calls;
359 }
360}
361
362fn resolve_call(
363 call: &str,
364 caller: &ImplFn,
365 exact: &HashMap<String, StableId>,
366 bare: &HashMap<String, Vec<StableId>>,
367) -> Option<StableId> {
368 let clean = call.trim_start_matches("crate::");
369 for candidate in [
370 clean.to_string(),
371 qualify(&caller.module_path, clean),
372 clean
373 .strip_prefix("self::")
374 .map(|rest| qualify(&caller.module_path, rest))
375 .unwrap_or_default(),
376 ] {
377 if let Some(id) = exact.get(&candidate) {
378 return Some(id.clone());
379 }
380 }
381 if clean.contains("::") {
382 let suffix = format!("::{clean}");
383 let mut matches = exact
384 .iter()
385 .filter(|(name, _)| name.ends_with(&suffix))
386 .map(|(_, id)| id);
387 let first = matches.next().cloned();
388 if first.is_some() && matches.next().is_none() {
389 return first;
390 }
391 }
392 let leaf = clean.rsplit("::").next().unwrap_or(clean);
393 match bare.get(leaf).map(Vec::as_slice) {
394 Some([id]) => Some(id.clone()),
395 _ => None,
396 }
397}
398
399fn discover_architecture_edges(krate: &Crate, model: &mut ModelIr) {
400 let component_by_type: HashMap<String, StableId> = model
401 .components
402 .iter()
403 .flat_map(|component| {
404 [
405 (component.name.clone(), component.id.clone()),
406 (component.qualified_name.clone(), component.id.clone()),
407 ]
408 })
409 .collect();
410
411 let mut seen = HashSet::new();
412 for function in krate.all_functions().chain(krate.all_methods()) {
413 if !is_production_source(krate, function) {
414 continue;
415 }
416 let owner_fields = krate
417 .struct_candidates(&function.qualified_type_name)
418 .into_iter()
419 .next()
420 .map(|owner| {
421 owner
422 .fields
423 .iter()
424 .filter_map(|field| {
425 component_by_type
426 .get(&field.ty.base)
427 .cloned()
428 .map(|component| (field.name.clone(), component))
429 })
430 .collect()
431 })
432 .unwrap_or_default();
433 let mut collector = ComponentFlowCollector {
434 component_by_type: &component_by_type,
435 owner_fields,
436 locals: HashMap::new(),
437 sequence: Vec::new(),
438 };
439 let signature_sequence: Vec<StableId> = function
440 .param_types
441 .iter()
442 .filter_map(|type_name| {
443 type_base_from_text(type_name)
444 .and_then(|base| component_by_type.get(&base).cloned())
445 })
446 .fold(Vec::new(), |mut sequence, component| {
447 if sequence.last() != Some(&component) {
448 sequence.push(component);
449 }
450 sequence
451 });
452 collector.visit_block(&function.block);
453 let from_signature = signature_sequence.len() > collector.sequence.len();
454 let sequence = if from_signature {
455 signature_sequence
456 } else {
457 collector.sequence
458 };
459 for pair in sequence.windows(2) {
460 if pair[0] == pair[1] {
461 continue;
462 }
463 let key = (pair[0].clone(), pair[1].clone(), function_id(function));
464 if !seen.insert(key.clone()) {
465 continue;
466 }
467 model.architecture_edges.push(ArchitectureEdge {
468 id: StableId::new(
469 "architecture-edge",
470 [key.0 .0.as_str(), key.1 .0.as_str(), key.2 .0.as_str()],
471 ),
472 from: key.0,
473 to: key.1,
474 via_function: key.2,
475 source: krate.file_label(function.span),
476 evidence: vec![heuristic_source_evidence(
477 krate.file_label(function.span),
478 if from_signature {
479 "component-typed parameters establish this interface order"
480 } else {
481 "typed component receiver calls occur in this source order"
482 },
483 )],
484 });
485 }
486 }
487}
488
489fn infer_builder_role(name: &str) -> (BuilderRole, Confidence) {
490 let lower = name.to_ascii_lowercase();
491 if lower.contains("train") || lower == "adapter_vb" {
492 return (BuilderRole::Trainable, Confidence::Heuristic);
493 }
494 if lower.contains("base")
495 || lower.contains("frozen")
496 || lower.contains("mmap")
497 || lower.contains("pretrained")
498 {
499 return (BuilderRole::Frozen, Confidence::Heuristic);
500 }
501 if lower.contains("state") || lower.contains("running") {
502 return (BuilderRole::State, Confidence::Heuristic);
503 }
504 (BuilderRole::Unknown, Confidence::Unknown)
505}
506
507fn discover_composition_edges(krate: &Crate, model: &mut ModelIr) {
508 let component_by_type: HashMap<String, StableId> = model
509 .components
510 .iter()
511 .flat_map(|component| {
512 [
513 (component.name.clone(), component.id.clone()),
514 (component.qualified_name.clone(), component.id.clone()),
515 ]
516 })
517 .collect();
518
519 let mut seen = HashSet::new();
520 for component in model.components.clone() {
521 let Some(def) = krate
522 .struct_candidates(&component.name)
523 .into_iter()
524 .find(|def| def.qualified_name == component.qualified_name)
525 else {
526 continue;
527 };
528 for field in &def.fields {
529 let child = unique_qualified_struct(krate, &field.ty.base)
530 .and_then(|qualified| component_by_type.get(&qualified).cloned())
531 .or_else(|| component_by_type.get(&field.ty.base).cloned());
532 let Some(child) = child else {
533 continue;
534 };
535 if child == component.id {
536 continue;
537 }
538 let key = (component.id.clone(), child.clone(), field.name.clone());
539 if !seen.insert(key.clone()) {
540 continue;
541 }
542 model.architecture_edges.push(ArchitectureEdge {
543 id: StableId::new(
544 "composition-edge",
545 [
546 component.id.0.as_str(),
547 child.0.as_str(),
548 field.name.as_str(),
549 ],
550 ),
551 from: component.id.clone(),
552 to: child,
553 via_function: component.constructor.clone(),
554 source: krate.file_label(field.span),
555 evidence: vec![Evidence {
556 kind: EvidenceKind::Source,
557 confidence: Confidence::Heuristic,
558 source: Some(krate.file_label(field.span)),
559 detail: format!(
560 "struct field `{}` embeds component type `{}`",
561 field.name, field.ty.base
562 ),
563 }],
564 });
565 }
566 }
567}
568
569fn discover_assembly_sites(krate: &Crate, model: &mut ModelIr) {
570 let constructor_functions: HashMap<StableId, &Function> = model
571 .functions
572 .iter()
573 .map(|candidate| (candidate.id.clone(), candidate))
574 .collect();
575 let specs: Vec<ConstructorSpec> = model
576 .components
577 .iter()
578 .filter_map(|component| {
579 let constructor = constructor_functions.get(&component.constructor)?;
580 let builders = constructor
581 .parameters
582 .iter()
583 .enumerate()
584 .filter(|(_, parameter)| parameter.type_name.contains("VarBuilder"))
585 .zip(component.builders.iter())
586 .map(|((index, _), builder)| (index, builder.name.clone()))
587 .collect();
588 Some(ConstructorSpec {
589 component: component.id.clone(),
590 owner: component.name.clone(),
591 builders,
592 })
593 })
594 .collect();
595 if specs.is_empty() {
596 return;
597 }
598
599 for func in krate.all_functions().chain(krate.all_methods()) {
600 if !is_production_source(krate, func) {
601 continue;
602 }
603 let function_id = function_id(func);
604 let mut collector = AssemblyCollector {
605 specs: &specs,
606 function_id: function_id.clone(),
607 function_name: func.qualified_name.clone(),
608 source: krate.file_label(func.span),
609 varmap_checkpoints: HashMap::new(),
610 vb_bindings: HashMap::new(),
611 sites: Vec::new(),
612 };
613 collector.visit_block(&func.block);
614 for site in &mut collector.sites {
615 if site.checkpoint_load.is_none() {
616 site.checkpoint_load = site
617 .varmap
618 .as_ref()
619 .and_then(|varmap| collector.varmap_checkpoints.get(varmap).cloned());
620 }
621 }
622 model.assembly_sites.extend(collector.sites);
623 }
624}
625
626struct AssemblyCollector<'a> {
627 specs: &'a [ConstructorSpec],
628 function_id: StableId,
629 function_name: String,
630 source: String,
631 varmap_checkpoints: HashMap<String, String>,
632 vb_bindings: HashMap<String, BuilderExpressionFacts>,
633 sites: Vec<AssemblySite>,
634}
635
636impl AssemblyCollector<'_> {
637 fn resolve_builder_facts(&self, expression: &syn::Expr) -> BuilderExpressionFacts {
638 match expression {
639 syn::Expr::Path(path) if path.path.segments.len() == 1 => self
640 .vb_bindings
641 .get(&path.path.segments[0].ident.to_string())
642 .cloned()
643 .unwrap_or_else(empty_builder_facts),
644 _ => builder_expression_facts(expression, &self.vb_bindings),
645 }
646 }
647}
648
649impl<'ast> Visit<'ast> for AssemblyCollector<'_> {
650 fn visit_local(&mut self, node: &'ast syn::Local) {
651 if let (Some(name), Some(init)) = (pat_ident(&node.pat), node.init.as_ref()) {
652 let facts = builder_expression_facts(&init.expr, &self.vb_bindings);
653 if facts.source_kind != BuilderSourceKind::Unknown
654 || !facts.prefix_chain.is_empty()
655 || facts.varmap.is_some()
656 {
657 self.vb_bindings.insert(name, facts);
658 }
659 }
660 visit::visit_local(self, node);
661 }
662
663 fn visit_expr_call(&mut self, node: &'ast syn::ExprCall) {
664 if let syn::Expr::Path(path) = &*node.func {
665 let leaf = path
666 .path
667 .segments
668 .last()
669 .map(|segment| segment.ident.to_string())
670 .unwrap_or_default();
671 if leaf == "load_varmap_checked" || leaf.ends_with("load_varmap_checked") {
672 if let (Some(varmap), Some(checkpoint)) = (
673 node.args.first().and_then(load_varmap_target),
674 node.args
675 .get(1)
676 .map(|arg| arg.to_token_stream().to_string()),
677 ) {
678 self.varmap_checkpoints.insert(varmap, checkpoint);
679 }
680 }
681 let segments: Vec<_> = path
682 .path
683 .segments
684 .iter()
685 .map(|segment| segment.ident.to_string())
686 .collect();
687 if let Some(owner) = segments.get(segments.len().saturating_sub(2)) {
688 for spec in self.specs.iter().filter(|spec| &spec.owner == owner) {
689 for (index, builder_root) in &spec.builders {
690 let Some(argument) = node.args.iter().nth(*index) else {
691 continue;
692 };
693 let facts = self.resolve_builder_facts(argument);
694 let varmap = facts.varmap.clone();
695 let checkpoint = varmap
696 .as_ref()
697 .and_then(|name| self.varmap_checkpoints.get(name).cloned());
698 self.sites.push(AssemblySite {
699 id: StableId::new(
700 "assembly-site",
701 [
702 self.function_id.0.as_str(),
703 spec.component.0.as_str(),
704 builder_root.as_str(),
705 &facts.prefix_chain.join("/"),
706 ],
707 ),
708 function: self.function_id.clone(),
709 function_name: self.function_name.clone(),
710 component: spec.component.clone(),
711 component_name: spec.owner.clone(),
712 builder_root: builder_root.clone(),
713 prefix_chain: facts.prefix_chain,
714 varmap,
715 source_kind: facts.source_kind,
716 role: facts.role,
717 checkpoint_load: checkpoint,
718 source: self.source.clone(),
719 evidence: vec![Evidence {
720 kind: EvidenceKind::Source,
721 confidence: Confidence::Heuristic,
722 source: Some(self.source.clone()),
723 detail: format!(
724 "`{owner}::new` wired through `{builder_root}` in `{}`",
725 self.function_name
726 ),
727 }],
728 });
729 }
730 }
731 }
732 }
733 visit::visit_expr_call(self, node);
734 }
735}
736
737#[derive(Clone)]
738struct BuilderExpressionFacts {
739 prefix_chain: Vec<String>,
740 varmap: Option<String>,
741 source_kind: BuilderSourceKind,
742 role: BuilderRole,
743}
744
745fn empty_builder_facts() -> BuilderExpressionFacts {
746 BuilderExpressionFacts {
747 prefix_chain: Vec::new(),
748 varmap: None,
749 source_kind: BuilderSourceKind::Unknown,
750 role: BuilderRole::Unknown,
751 }
752}
753
754fn builder_expression_facts(
755 expression: &syn::Expr,
756 vb_bindings: &HashMap<String, BuilderExpressionFacts>,
757) -> BuilderExpressionFacts {
758 match expression {
759 syn::Expr::Path(path) if path.path.segments.len() == 1 => vb_bindings
760 .get(&path.path.segments[0].ident.to_string())
761 .cloned()
762 .unwrap_or_else(empty_builder_facts),
763 syn::Expr::MethodCall(call) if call.method == "pp" => {
764 let mut facts = builder_expression_facts(&call.receiver, vb_bindings);
765 if let Some(syn::Expr::Lit(lit)) = call.args.first() {
766 if let syn::Lit::Str(text) = &lit.lit {
767 facts.prefix_chain.insert(0, text.value());
768 }
769 }
770 facts
771 }
772 syn::Expr::Call(call) => {
773 let leaf = call_leaf_name(&call.func);
774 match leaf.as_deref() {
775 Some("from_varmap") => BuilderExpressionFacts {
776 prefix_chain: Vec::new(),
777 varmap: call.args.first().and_then(expr_identifier).or_else(|| {
778 call.args.first().and_then(|arg| match arg {
779 syn::Expr::Reference(reference) => expr_identifier(&reference.expr),
780 _ => None,
781 })
782 }),
783 source_kind: BuilderSourceKind::VarMap,
784 role: BuilderRole::Trainable,
785 },
786 Some("from_mmaped_safetensors") | Some("from_buffered_safetensors") => {
787 BuilderExpressionFacts {
788 prefix_chain: Vec::new(),
789 varmap: None,
790 source_kind: if leaf.as_deref() == Some("from_mmaped_safetensors") {
791 BuilderSourceKind::MmapSafetensors
792 } else {
793 BuilderSourceKind::BufferedSafetensors
794 },
795 role: BuilderRole::Frozen,
796 }
797 }
798 Some("from_tensors") => BuilderExpressionFacts {
799 prefix_chain: Vec::new(),
800 varmap: None,
801 source_kind: BuilderSourceKind::FromTensors,
802 role: BuilderRole::Frozen,
803 },
804 _ => BuilderExpressionFacts {
805 prefix_chain: Vec::new(),
806 varmap: varmap_sources(expression, &HashMap::new())
807 .into_iter()
808 .next(),
809 source_kind: BuilderSourceKind::Unknown,
810 role: BuilderRole::Unknown,
811 },
812 }
813 }
814 syn::Expr::Reference(reference) => builder_expression_facts(&reference.expr, vb_bindings),
815 syn::Expr::Try(value) => builder_expression_facts(&value.expr, vb_bindings),
816 syn::Expr::Await(value) => builder_expression_facts(&value.base, vb_bindings),
817 syn::Expr::Unsafe(value) => value
818 .block
819 .stmts
820 .iter()
821 .find_map(|statement| match statement {
822 syn::Stmt::Expr(expression, _) => {
823 Some(builder_expression_facts(expression, vb_bindings))
824 }
825 _ => None,
826 })
827 .unwrap_or_else(empty_builder_facts),
828 syn::Expr::Block(block) => block
829 .block
830 .stmts
831 .iter()
832 .find_map(|statement| match statement {
833 syn::Stmt::Expr(expression, _) => {
834 Some(builder_expression_facts(expression, vb_bindings))
835 }
836 _ => None,
837 })
838 .unwrap_or_else(empty_builder_facts),
839 syn::Expr::Paren(paren) => builder_expression_facts(&paren.expr, vb_bindings),
840 syn::Expr::Group(group) => builder_expression_facts(&group.expr, vb_bindings),
841 _ => empty_builder_facts(),
842 }
843}
844
845fn call_leaf_name(expression: &syn::Expr) -> Option<String> {
846 match expression {
847 syn::Expr::Path(path) => path
848 .path
849 .segments
850 .last()
851 .map(|segment| segment.ident.to_string()),
852 _ => None,
853 }
854}
855
856fn load_varmap_target(expression: &syn::Expr) -> Option<String> {
857 match expression {
858 syn::Expr::Reference(reference) => expr_identifier(&reference.expr),
859 _ => expr_identifier(expression),
860 }
861}
862
863#[derive(Clone)]
864struct SubprocessInvocation {
865 wrapper_function: StableId,
866 subprocess_key: String,
867 cli_flags: Vec<String>,
868 launcher: String,
869 source: String,
870}
871
872fn discover_subprocess_pipeline(
873 krate: &Crate,
874 functions: &HashMap<String, StableId>,
875 model: &mut ModelIr,
876) {
877 let launchers = subprocess_launcher_functions(krate);
878 if launchers.is_empty() {
879 return;
880 }
881
882 let function_by_id: HashMap<StableId, Function> = model
883 .functions
884 .iter()
885 .map(|function| (function.id.clone(), function.clone()))
886 .collect();
887 let by_bare: HashMap<String, Vec<StableId>> =
888 model
889 .functions
890 .iter()
891 .fold(HashMap::new(), |mut grouped, function| {
892 grouped
893 .entry(function.name.clone())
894 .or_default()
895 .push(function.id.clone());
896 grouped
897 });
898
899 let mut invocations = Vec::new();
900 for func in krate.all_functions().chain(krate.all_methods()) {
901 if !is_production_source(krate, func) {
902 continue;
903 }
904 let wrapper_id = function_id(func);
905 let mut collector = SubprocessInvocationCollector {
906 launchers: &launchers,
907 wrapper_id,
908 source: krate.file_label(func.span),
909 invocations: Vec::new(),
910 };
911 collector.visit_block(&func.block);
912 invocations.extend(collector.invocations);
913 }
914
915 let orchestrator = discover_orchestrator_order(krate, functions, model, &by_bare);
916 if orchestrator.is_empty() && invocations.is_empty() {
917 return;
918 }
919
920 let mut seen_functions = model
921 .stages
922 .iter()
923 .map(|stage| stage.function.clone())
924 .collect::<HashSet<_>>();
925 let order_base = model.stages.len();
926
927 for (order, (function_id, orchestrator_name)) in orchestrator.into_iter().enumerate() {
928 if seen_functions.contains(&function_id) {
929 continue;
930 }
931 let Some(function) = function_by_id.get(&function_id) else {
932 continue;
933 };
934 let invocation = invocations
935 .iter()
936 .find(|item| item.wrapper_function == function_id);
937 push_subprocess_stage(
938 model,
939 function,
940 invocation,
941 order_base + order,
942 Some(orchestrator_name),
943 );
944 seen_functions.insert(function_id);
945 }
946
947 for invocation in invocations {
948 if seen_functions.contains(&invocation.wrapper_function) {
949 if let Some(stage) = model
950 .stages
951 .iter_mut()
952 .find(|stage| stage.function == invocation.wrapper_function)
953 {
954 stage.dispatch = StageDispatchKind::Subprocess;
955 if stage.subprocess_key.is_none() {
956 stage.subprocess_key = Some(invocation.subprocess_key.clone());
957 }
958 stage.name = stage
959 .subprocess_key
960 .clone()
961 .unwrap_or_else(|| stage.name.clone());
962 stage.cli_flags.extend(invocation.cli_flags.iter().cloned());
963 stage.cli_flags.sort();
964 stage.cli_flags.dedup();
965 stage.launcher = Some(invocation.launcher.clone());
966 stage.evidence.push(Evidence {
967 kind: EvidenceKind::Source,
968 confidence: Confidence::Heuristic,
969 source: Some(invocation.source.clone()),
970 detail: format!(
971 "subprocess relaunch via `{}` with stage key `{}`",
972 invocation.launcher, invocation.subprocess_key
973 ),
974 });
975 }
976 continue;
977 }
978 let Some(function) = function_by_id.get(&invocation.wrapper_function) else {
979 continue;
980 };
981 push_subprocess_stage(
982 model,
983 function,
984 Some(&invocation),
985 order_base + seen_functions.len(),
986 None,
987 );
988 seen_functions.insert(invocation.wrapper_function.clone());
989 }
990}
991
992fn push_subprocess_stage(
993 model: &mut ModelIr,
994 function: &Function,
995 invocation: Option<&SubprocessInvocation>,
996 order: usize,
997 orchestrator: Option<String>,
998) {
999 let name = invocation
1000 .map(|item| item.subprocess_key.clone())
1001 .unwrap_or_else(|| stage_display_name(function));
1002 let id = StableId::new("subprocess-stage", [&function.qualified_name, &name]);
1003 let mut evidence = vec![Evidence {
1004 kind: EvidenceKind::Source,
1005 confidence: Confidence::Heuristic,
1006 source: Some(function.source.clone()),
1007 detail: if let Some(orchestrator) = orchestrator.as_deref() {
1008 format!(
1009 "orchestrator `{orchestrator}` calls `{}` in source order",
1010 function.name
1011 )
1012 } else {
1013 format!(
1014 "wrapper `{}` relaunches the current executable",
1015 function.name
1016 )
1017 },
1018 }];
1019 if let Some(item) = invocation {
1020 evidence.push(Evidence {
1021 kind: EvidenceKind::Source,
1022 confidence: Confidence::Heuristic,
1023 source: Some(item.source.clone()),
1024 detail: format!(
1025 "subprocess relaunch via `{}` with stage key `{}`",
1026 item.launcher, item.subprocess_key
1027 ),
1028 });
1029 }
1030 model.stages.push(PipelineStage {
1031 id,
1032 name,
1033 kind: stage_kind(&function.name),
1034 function: function.id.clone(),
1035 order: Some(order),
1036 components: reachable_components(function, model),
1037 consumes: Vec::new(),
1038 produces: Vec::new(),
1039 depends_on: Vec::new(),
1040 source: function.source.clone(),
1041 evidence,
1042 dispatch: if invocation.is_some() {
1043 StageDispatchKind::Subprocess
1044 } else {
1045 StageDispatchKind::Inline
1046 },
1047 subprocess_key: invocation.map(|item| item.subprocess_key.clone()),
1048 cli_flags: invocation
1049 .map(|item| item.cli_flags.clone())
1050 .unwrap_or_default(),
1051 launcher: invocation.map(|item| item.launcher.clone()),
1052 orchestrator,
1053 });
1054}
1055
1056fn discover_orchestrator_order(
1057 krate: &Crate,
1058 functions: &HashMap<String, StableId>,
1059 model: &ModelIr,
1060 by_bare: &HashMap<String, Vec<StableId>>,
1061) -> Vec<(StableId, String)> {
1062 let mut best = Vec::new();
1063 for pipeline in krate
1064 .all_functions()
1065 .filter(|function| function.fn_name == "run_pipeline")
1066 {
1067 let mut collector = OrderedCallCollector::default();
1068 collector.visit_block(&pipeline.block);
1069 let orchestrator_name = pipeline.qualified_name.clone();
1070 let mut ordered = Vec::new();
1071 for call in collector.calls {
1072 let Some(callee) = resolve_call(&call, pipeline, functions, by_bare) else {
1073 continue;
1074 };
1075 let Some(function) = model
1076 .functions
1077 .iter()
1078 .find(|function| function.id == callee)
1079 else {
1080 continue;
1081 };
1082 if is_orchestrator_stage_call(&function.name) {
1083 ordered.push((callee, orchestrator_name.clone()));
1084 }
1085 }
1086 if ordered.len() > best.len() {
1087 best = ordered;
1088 }
1089 }
1090 best
1091}
1092
1093fn subprocess_launcher_functions(krate: &Crate) -> HashSet<String> {
1094 let mut launchers = HashSet::new();
1095 launchers.insert("run_stage_command".to_string());
1096 for func in krate.all_functions().chain(krate.all_methods()) {
1097 let text = func.block.to_token_stream().to_string();
1098 if text.contains("current_exe") && text.contains("Command") {
1099 launchers.insert(func.fn_name.clone());
1100 }
1101 }
1102 launchers
1103}
1104
1105struct SubprocessInvocationCollector<'a> {
1106 launchers: &'a HashSet<String>,
1107 wrapper_id: StableId,
1108 source: String,
1109 invocations: Vec<SubprocessInvocation>,
1110}
1111
1112impl<'ast> Visit<'ast> for SubprocessInvocationCollector<'_> {
1113 fn visit_expr_call(&mut self, node: &'ast syn::ExprCall) {
1114 if let Some(leaf) = call_leaf_name(&node.func) {
1115 if self.launchers.contains(&leaf) {
1116 if let Some(subprocess_key) = node.args.first().and_then(string_literal_expr) {
1117 let cli_flags = node
1118 .args
1119 .get(1)
1120 .map(extract_cli_flags_from_expr)
1121 .unwrap_or_default();
1122 self.invocations.push(SubprocessInvocation {
1123 wrapper_function: self.wrapper_id.clone(),
1124 subprocess_key,
1125 cli_flags,
1126 launcher: leaf.clone(),
1127 source: self.source.clone(),
1128 });
1129 }
1130 } else if leaf == "run_training_stage_with_oom_recovery" {
1131 if let Some(subprocess_key) = node.args.get(2).and_then(string_literal_expr) {
1132 let mut cli_flags = node
1133 .args
1134 .get(7)
1135 .map(extract_cli_flags_from_expr)
1136 .unwrap_or_default();
1137 if cli_flags.is_empty() {
1138 cli_flags = node
1139 .args
1140 .iter()
1141 .flat_map(extract_cli_flags_from_expr)
1142 .collect::<BTreeSet<_>>()
1143 .into_iter()
1144 .collect();
1145 }
1146 self.invocations.push(SubprocessInvocation {
1147 wrapper_function: self.wrapper_id.clone(),
1148 subprocess_key,
1149 cli_flags,
1150 launcher: "run_stage_command".to_string(),
1151 source: self.source.clone(),
1152 });
1153 }
1154 }
1155 }
1156 visit::visit_expr_call(self, node);
1157 }
1158}
1159
1160#[derive(Default)]
1161struct OrderedCallCollector {
1162 calls: Vec<String>,
1163}
1164
1165impl<'ast> Visit<'ast> for OrderedCallCollector {
1166 fn visit_expr_call(&mut self, node: &'ast syn::ExprCall) {
1167 if let syn::Expr::Path(path) = &*node.func {
1168 self.calls
1169 .push(path.path.to_token_stream().to_string().replace(' ', ""));
1170 }
1171 visit::visit_expr_call(self, node);
1172 }
1173
1174 fn visit_expr_method_call(&mut self, node: &'ast syn::ExprMethodCall) {
1175 self.calls.push(node.method.to_string());
1176 visit::visit_expr_method_call(self, node);
1177 }
1178}
1179
1180fn is_orchestrator_stage_call(name: &str) -> bool {
1181 is_pipeline_stage_call(name) || name.starts_with("preflight_")
1182}
1183
1184fn string_literal_expr(expression: &syn::Expr) -> Option<String> {
1185 match expression {
1186 syn::Expr::Lit(lit) => match &lit.lit {
1187 syn::Lit::Str(text) => Some(text.value()),
1188 _ => None,
1189 },
1190 syn::Expr::Reference(reference) => string_literal_expr(&reference.expr),
1191 syn::Expr::Group(group) => string_literal_expr(&group.expr),
1192 syn::Expr::Paren(paren) => string_literal_expr(&paren.expr),
1193 _ => None,
1194 }
1195}
1196
1197fn extract_cli_flags_from_expr(expression: &syn::Expr) -> Vec<String> {
1198 match expression {
1199 syn::Expr::Closure(closure) => extract_cli_flags_from_expr(&closure.body),
1200 syn::Expr::Block(block) => {
1201 let mut flags = block
1202 .block
1203 .stmts
1204 .iter()
1205 .flat_map(|statement| match statement {
1206 syn::Stmt::Expr(expr, _) => extract_cli_flags_from_expr(expr),
1207 syn::Stmt::Macro(macro_stmt) => {
1208 flags_from_tokens(macro_stmt.mac.tokens.clone())
1209 }
1210 _ => Vec::new(),
1211 })
1212 .collect::<Vec<_>>();
1213 flags.sort();
1214 flags.dedup();
1215 flags
1216 }
1217 syn::Expr::Macro(expr_macro) => flags_from_tokens(expr_macro.mac.tokens.clone()),
1218 _ => extract_cli_flags(expression),
1219 }
1220}
1221
1222fn flags_from_tokens(tokens: proc_macro2::TokenStream) -> Vec<String> {
1223 if let Ok(expression) = syn::parse2::<syn::Expr>(tokens.clone()) {
1224 let flags = extract_cli_flags(&expression);
1225 if !flags.is_empty() {
1226 return flags;
1227 }
1228 }
1229 let mut flags = tokens
1230 .to_string()
1231 .split(|character: char| {
1232 !(character.is_ascii_alphanumeric() || character == '-' || character == '_')
1233 })
1234 .map(|token| token.trim_matches('"'))
1235 .filter(|token| token.starts_with("--") && token.len() > 2)
1236 .map(str::to_string)
1237 .collect::<Vec<_>>();
1238 flags.sort();
1239 flags.dedup();
1240 flags
1241}
1242
1243fn extract_cli_flags(expression: &syn::Expr) -> Vec<String> {
1244 let mut flags = string_literals(expression)
1245 .into_iter()
1246 .filter(|value| value.starts_with("--") && value.len() > 2)
1247 .collect::<Vec<_>>();
1248 flags.sort();
1249 flags.dedup();
1250 flags
1251}
1252
1253struct ComponentFlowCollector<'a> {
1254 component_by_type: &'a HashMap<String, StableId>,
1255 owner_fields: HashMap<String, StableId>,
1256 locals: HashMap<String, StableId>,
1257 sequence: Vec<StableId>,
1258}
1259
1260impl<'ast> Visit<'ast> for ComponentFlowCollector<'_> {
1261 fn visit_local(&mut self, node: &'ast syn::Local) {
1262 if let (Some(name), Some(init)) = (pat_ident(&node.pat), node.init.as_ref()) {
1263 if let Some(component) = constructor_component(&init.expr, self.component_by_type)
1264 .or_else(|| receiver_component(&init.expr, &self.locals, &self.owner_fields))
1265 {
1266 self.locals.insert(name, component);
1267 }
1268 }
1269 visit::visit_local(self, node);
1270 }
1271
1272 fn visit_expr_method_call(&mut self, node: &'ast syn::ExprMethodCall) {
1273 if is_model_entry_name(&node.method.to_string()) {
1274 if let Some(component) =
1275 receiver_component(&node.receiver, &self.locals, &self.owner_fields)
1276 {
1277 if self.sequence.last() != Some(&component) {
1278 self.sequence.push(component);
1279 }
1280 }
1281 }
1282 visit::visit_expr_method_call(self, node);
1283 }
1284}
1285
1286fn pat_ident(pattern: &syn::Pat) -> Option<String> {
1287 match pattern {
1288 syn::Pat::Ident(ident) => Some(ident.ident.to_string()),
1289 syn::Pat::Type(typed) => pat_ident(&typed.pat),
1290 _ => None,
1291 }
1292}
1293
1294fn constructor_component(
1295 expression: &syn::Expr,
1296 components: &HashMap<String, StableId>,
1297) -> Option<StableId> {
1298 match expression {
1299 syn::Expr::Call(call) => {
1300 let syn::Expr::Path(path) = &*call.func else {
1301 return None;
1302 };
1303 let segments: Vec<_> = path
1304 .path
1305 .segments
1306 .iter()
1307 .map(|segment| segment.ident.to_string())
1308 .collect();
1309 let owner = segments
1310 .get(segments.len().checked_sub(2)?)
1311 .map(String::as_str)?;
1312 components.get(owner).cloned()
1313 }
1314 syn::Expr::Try(value) => constructor_component(&value.expr, components),
1315 syn::Expr::Await(value) => constructor_component(&value.base, components),
1316 syn::Expr::Paren(value) => constructor_component(&value.expr, components),
1317 syn::Expr::Group(value) => constructor_component(&value.expr, components),
1318 _ => None,
1319 }
1320}
1321
1322fn receiver_component(
1323 expression: &syn::Expr,
1324 locals: &HashMap<String, StableId>,
1325 owner_fields: &HashMap<String, StableId>,
1326) -> Option<StableId> {
1327 match expression {
1328 syn::Expr::Path(path) if path.path.segments.len() == 1 => locals
1329 .get(&path.path.segments[0].ident.to_string())
1330 .cloned(),
1331 syn::Expr::Field(field)
1332 if matches!(
1333 &*field.base,
1334 syn::Expr::Path(path)
1335 if path.path.segments.len() == 1 && path.path.segments[0].ident == "self"
1336 ) =>
1337 {
1338 let syn::Member::Named(name) = &field.member else {
1339 return None;
1340 };
1341 owner_fields.get(&name.to_string()).cloned()
1342 }
1343 syn::Expr::Reference(reference) => {
1344 receiver_component(&reference.expr, locals, owner_fields)
1345 }
1346 syn::Expr::Try(value) => receiver_component(&value.expr, locals, owner_fields),
1347 syn::Expr::Await(value) => receiver_component(&value.base, locals, owner_fields),
1348 syn::Expr::MethodCall(call) => receiver_component(&call.receiver, locals, owner_fields),
1349 syn::Expr::Paren(paren) => receiver_component(&paren.expr, locals, owner_fields),
1350 syn::Expr::Group(group) => receiver_component(&group.expr, locals, owner_fields),
1351 _ => None,
1352 }
1353}
1354
1355fn type_base_from_text(text: &str) -> Option<String> {
1356 syn::parse_str::<syn::Type>(text)
1357 .ok()
1358 .and_then(|type_name| load::type_base_name(&type_name))
1359}
1360
1361fn discover_components(
1362 krate: &Crate,
1363 cargo: Option<&CargoContext>,
1364 selected_root: Option<&str>,
1365 heuristic_architecture: bool,
1366 model: &mut ModelIr,
1367) {
1368 let exported: HashSet<String> = krate
1369 .public_reexports
1370 .iter()
1371 .map(|item| item.name.clone())
1372 .collect();
1373 let mut candidates = Vec::new();
1374 for def in krate.all_structs() {
1375 if let Some(selected) = selected_root {
1376 if def.name != selected && def.qualified_name != selected {
1377 continue;
1378 }
1379 }
1380 if cargo.and_then(|context| {
1381 crate::cargo_context::cfg_predicates_active(&def.cfg_predicates, &context.cfgs)
1382 }) == Some(false)
1383 {
1384 continue;
1385 }
1386 let methods = krate.method_candidates(&def.qualified_name, "new");
1387 let mut constructors = if methods.is_empty() {
1388 krate.method_candidates(&def.qualified_name, "load")
1389 } else {
1390 methods
1391 };
1392 if constructors.is_empty() {
1393 constructors = krate
1394 .all_methods()
1395 .filter(|method| {
1396 method.qualified_type_name == def.qualified_name
1397 && method.trait_name.is_none()
1398 && !method.vb_params.is_empty()
1399 && return_mentions_type(&method.return_type, &def.name)
1400 })
1401 .collect();
1402 constructors.sort_by(|a, b| a.qualified_name.cmp(&b.qualified_name));
1403 }
1404 let Some(ctor) = constructors.into_iter().find(|method| {
1405 !method.vb_params.is_empty()
1406 && cargo.and_then(|context| {
1407 crate::cargo_context::cfg_predicates_active(
1408 &method.cfg_predicates,
1409 &context.cfgs,
1410 )
1411 }) != Some(false)
1412 }) else {
1413 continue;
1414 };
1415 let is_api_boundary =
1416 exported.contains(&def.name) || def.module_path.is_empty() && def.visibility == "pub";
1417 let is_varbuilder_component =
1420 def.visibility == "pub" && !ctor.vb_params.is_empty();
1421 if selected_root.is_some() || is_api_boundary || is_varbuilder_component {
1422 candidates.push((def, ctor));
1423 }
1424 }
1425 candidates.sort_by(|(a, _), (b, _)| a.qualified_name.cmp(&b.qualified_name));
1426 candidates.dedup_by(|(a, _), (b, _)| a.qualified_name == b.qualified_name);
1427
1428 for (def, ctor) in candidates {
1429 if selected_root.is_some() {
1430 for function in &mut model.functions {
1431 if function.owner_type.as_deref() == Some(def.qualified_name.as_str())
1432 && (function
1433 .parameters
1434 .iter()
1435 .any(|parameter| is_tensor_type(¶meter.type_name))
1436 || function.return_type.as_deref().is_some_and(is_tensor_type))
1437 {
1438 function.is_entrypoint = true;
1439 }
1440 }
1441 }
1442 let candle_version = cargo.and_then(|context| {
1443 op_semantics::matched_candle_version(
1444 context
1445 .candle_versions
1446 .get("candle-core")
1447 .map(String::as_str),
1448 context.candle_versions.get("candle-nn").map(String::as_str),
1449 )
1450 });
1451 add_component(
1452 krate,
1453 model,
1454 def,
1455 ctor,
1456 candle_version,
1457 heuristic_architecture,
1458 );
1459 }
1460}
1461
1462fn return_mentions_type(return_type: &str, type_name: &str) -> bool {
1463 return_type
1464 .split(|character: char| !character.is_alphanumeric() && character != '_')
1465 .any(|part| part == "Self" || part == type_name)
1466}
1467
1468fn add_component(
1469 krate: &Crate,
1470 model: &mut ModelIr,
1471 def: &StructDef,
1472 ctor: &ImplFn,
1473 candle_version: Option<&str>,
1474 heuristic_architecture: bool,
1475) {
1476 let component_id = StableId::new("component", [&def.qualified_name]);
1477 let constructor_id = function_id(ctor);
1478 let mut component = Component {
1479 id: component_id.clone(),
1480 name: def.name.clone(),
1481 qualified_name: def.qualified_name.clone(),
1482 source: krate.file_label(def.span),
1483 constructor: constructor_id,
1484 builders: ctor
1485 .vb_params
1486 .iter()
1487 .filter_map(|index| ctor.params.get(*index))
1488 .map(|name| {
1489 let (role, confidence) = if heuristic_architecture {
1490 infer_builder_role(name)
1491 } else {
1492 (BuilderRole::Unknown, Confidence::Unknown)
1493 };
1494 let mut evidence = vec![source_evidence(
1495 krate.file_label(ctor.span),
1496 format!("constructor parameter `{name}` has VarBuilder type"),
1497 )];
1498 if role != BuilderRole::Unknown {
1499 evidence.push(Evidence {
1500 kind: EvidenceKind::Inferred,
1501 confidence,
1502 source: Some(krate.file_label(ctor.span)),
1503 detail: format!("builder name `{name}` suggests role `{role:?}`"),
1504 });
1505 }
1506 BuilderNamespace {
1507 name: name.clone(),
1508 role,
1509 evidence,
1510 }
1511 })
1512 .collect(),
1513 modules: Vec::new(),
1514 parameters: Vec::new(),
1515 entrypoints: model
1516 .functions
1517 .iter()
1518 .filter(|function| {
1519 function
1520 .owner_type
1521 .as_deref()
1522 .is_some_and(|owner| owner == def.qualified_name)
1523 && function.is_entrypoint
1524 })
1525 .map(|function| function.id.clone())
1526 .collect(),
1527 evidence: vec![source_evidence(
1528 krate.file_label(def.span),
1529 "public model API type with a VarBuilder constructor",
1530 )],
1531 };
1532
1533 match Extractor::for_candle_version(krate, candle_version)
1534 .run(&def.qualified_name, Some(&ctor.fn_name))
1535 {
1536 Ok(structure) => {
1537 let mut module_ids = HashMap::new();
1538 for instance in &structure.instances {
1539 let module_def = structure.def(instance.def);
1540 let id = StableId::new(
1541 "module",
1542 [
1543 component_id.0.as_str(),
1544 instance.root.as_str(),
1545 instance.prefix.to_string().as_str(),
1546 module_def.name.as_str(),
1547 instance.id.0.to_string().as_str(),
1548 ],
1549 );
1550 module_ids.insert(instance.id, id.clone());
1551 component.modules.push(id.clone());
1552 model.modules.push(Module {
1553 id,
1554 component: component_id.clone(),
1555 parent: instance
1556 .parent
1557 .and_then(|parent| module_ids.get(&parent).cloned()),
1558 type_name: module_def.name.clone(),
1559 qualified_type: unique_qualified_struct(krate, &module_def.name),
1560 field: instance.via_field.clone(),
1561 builder_root: instance.root.clone(),
1562 prefix: instance.prefix.to_string(),
1563 repeat: instance
1564 .repeat
1565 .as_ref()
1566 .map(|repeat| format!("{} in 0..{}", repeat.var, repeat.bound)),
1567 source: krate.file_label(instance.origin),
1568 confidence: certainty_confidence(&instance.certainty),
1569 });
1570 }
1571 for param in &structure.params {
1572 let site = structure.site(param.site);
1573 let Some(module) = module_ids.get(¶m.owner).cloned() else {
1574 continue;
1575 };
1576 let key = param.key.to_string();
1577 let id = StableId::new(
1578 "parameter",
1579 [component_id.0.as_str(), param.root.as_str(), key.as_str()],
1580 );
1581 component.parameters.push(id.clone());
1582 let (checkpoint_shape, checkpoint_dtype) = match ¶m.checkpoint {
1583 CheckpointMatch::Found { shape, dtype, .. } => {
1584 (Some(shape.clone()), Some(dtype.clone()))
1585 }
1586 _ => (None, None),
1587 };
1588 model.parameters.push(Parameter {
1589 id,
1590 component: component_id.clone(),
1591 module,
1592 key,
1593 builder_root: param.root.clone(),
1594 role: match site.kind {
1595 crate::known::ParamKind::RunningMean
1596 | crate::known::ParamKind::RunningVar => ParameterRole::RunningState,
1597 _ => ParameterRole::Unknown,
1598 },
1599 kind: acquisition_label(&site.acquisition),
1600 symbolic_shape: site.shape.clone(),
1601 checkpoint_shape,
1602 checkpoint_dtype,
1603 source: krate.file_label(site.span),
1604 uses: Vec::new(),
1605 optimizer_memberships: Vec::new(),
1606 evidence: vec![Evidence {
1607 kind: EvidenceKind::Source,
1608 confidence: certainty_confidence(¶m.certainty),
1609 source: Some(krate.file_label(site.span)),
1610 detail: format!(
1611 "parameter registered through {}",
1612 acquisition_label(&site.acquisition)
1613 ),
1614 }],
1615 });
1616 }
1617 for diagnostic in structure.diagnostics {
1618 push_finding(
1619 model,
1620 "structure-unresolved",
1621 FindingSeverity::Warning,
1622 Confidence::Proven,
1623 diagnostic.message,
1624 Some(krate.file_label(diagnostic.span)),
1625 vec![component_id.clone()],
1626 );
1627 }
1628 }
1629 Err(error) => push_finding(
1630 model,
1631 "component-expansion",
1632 FindingSeverity::Warning,
1633 Confidence::Proven,
1634 format!("could not expand {}: {error:#}", def.qualified_name),
1635 Some(krate.file_label(ctor.span)),
1636 vec![component_id.clone()],
1637 ),
1638 }
1639 model.components.push(component);
1640}
1641
1642fn add_contracts(krate: &Crate, model: &mut ModelIr) {
1643 let analysis = crate::contracts::analyze(krate);
1644 let owners: HashMap<String, StableId> = model
1645 .functions
1646 .iter()
1647 .map(|function| (function.qualified_name.clone(), function.id.clone()))
1648 .collect();
1649 let function_indices: HashMap<StableId, usize> = model
1650 .functions
1651 .iter()
1652 .enumerate()
1653 .map(|(index, function)| (function.id.clone(), index))
1654 .collect();
1655
1656 for contracts in analysis.functions {
1657 let Some(owner) = owners.get(&contracts.qualified_name).cloned() else {
1658 continue;
1659 };
1660 for mut tensor in contracts.tensors {
1661 tensor.owner_function = owner.clone();
1662 tensor.id = StableId::new("tensor", [owner.0.as_str(), tensor.name.as_str()]);
1663 if tensor.dtype.eq_ignore_ascii_case("unknown") {
1664 tensor.dtype = "Unknown".to_string();
1665 }
1666 let id = tensor.id.clone();
1667 let role = tensor.role.clone();
1668 if let Some(&index) = function_indices.get(&owner) {
1669 if matches!(role, TensorRole::Input) {
1670 model.functions[index].tensor_inputs.push(id.clone());
1671 }
1672 if matches!(role, TensorRole::Output | TensorRole::Loss) {
1673 model.functions[index].tensor_outputs.push(id.clone());
1674 }
1675 }
1676 model.tensors.push(tensor);
1677 }
1678 }
1679}
1680
1681fn discover_pipeline_and_artifacts(
1682 krate: &Crate,
1683 functions: &HashMap<String, StableId>,
1684 model: &mut ModelIr,
1685) {
1686 let function_by_id: HashMap<StableId, &Function> = model
1687 .functions
1688 .iter()
1689 .map(|function| (function.id.clone(), function))
1690 .collect();
1691 let by_bare: HashMap<String, Vec<StableId>> =
1692 model
1693 .functions
1694 .iter()
1695 .fold(HashMap::new(), |mut grouped, function| {
1696 grouped
1697 .entry(function.name.clone())
1698 .or_default()
1699 .push(function.id.clone());
1700 grouped
1701 });
1702 let mut ordered_stage_functions: Vec<(StableId, Option<String>)> = Vec::new();
1703 for pipeline in krate
1704 .all_functions()
1705 .filter(|function| function.fn_name == "run_pipeline")
1706 {
1707 let mut collector = CallCollector::default();
1708 collector.visit_block(&pipeline.block);
1709 for call in collector.calls {
1710 let Some(callee) = resolve_call(&call, pipeline, functions, &by_bare) else {
1711 continue;
1712 };
1713 let Some(target) = function_by_id.get(&callee) else {
1714 continue;
1715 };
1716 if !is_pipeline_stage_call(&target.name) {
1717 continue;
1718 }
1719 let variants = stage_variants(krate, &callee);
1720 if variants.is_empty() {
1721 ordered_stage_functions.push((callee, None));
1722 } else {
1723 ordered_stage_functions.extend(
1724 variants
1725 .into_iter()
1726 .map(|variant| (callee.clone(), Some(variant))),
1727 );
1728 }
1729 }
1730 }
1731 if ordered_stage_functions.is_empty() {
1732 ordered_stage_functions.extend(
1733 model
1734 .functions
1735 .iter()
1736 .filter(|function| is_stage_entry_name(&function.name))
1737 .map(|function| (function.id.clone(), None)),
1738 );
1739 }
1740 ordered_stage_functions.dedup();
1741
1742 let mut stage_by_function = HashMap::new();
1743 for (order, (function_id, variant)) in ordered_stage_functions.into_iter().enumerate() {
1744 let Some(function) = function_by_id.get(&function_id) else {
1745 continue;
1746 };
1747 let name = variant.unwrap_or_else(|| stage_display_name(function));
1748 let id = StableId::new("stage", [&function.qualified_name, &name]);
1749 stage_by_function
1750 .entry(function_id.clone())
1751 .or_insert_with(|| id.clone());
1752 let related_components = reachable_components(function, model);
1753 model.stages.push(PipelineStage {
1754 id,
1755 name,
1756 kind: stage_kind(&function.name),
1757 function: function_id,
1758 order: Some(order),
1759 components: related_components,
1760 consumes: Vec::new(),
1761 produces: Vec::new(),
1762 depends_on: Vec::new(),
1763 source: function.source.clone(),
1764 evidence: vec![heuristic_source_evidence(
1765 function.source.clone(),
1766 "stage entrypoint discovered from pipeline call order",
1767 )],
1768 dispatch: StageDispatchKind::Unknown,
1769 subprocess_key: None,
1770 cli_flags: Vec::new(),
1771 launcher: None,
1772 orchestrator: None,
1773 });
1774 }
1775
1776 let mut facts = Vec::new();
1777 for func in krate.all_functions().chain(krate.all_methods()) {
1778 if !is_production_source(krate, func) {
1779 continue;
1780 }
1781 let mut visitor = ArtifactVisitor::default();
1782 visitor.visit_block(&func.block);
1783 let owner = function_id(func);
1784 for observed in visitor.observed {
1785 facts.push((owner.clone(), krate.file_label(func.span), observed));
1786 }
1787 }
1788 facts.sort_by(|a, b| a.2.path_expr.cmp(&b.2.path_expr));
1789
1790 let mut artifact_index: HashMap<String, usize> = HashMap::new();
1791 for (function_id, source, observed) in facts {
1792 let identity = artifact_identity(&observed.path_expr);
1793 let index = match artifact_index.get(&identity).copied() {
1794 Some(index) => index,
1795 None => {
1796 let id = StableId::new("artifact", [&identity]);
1797 let index = model.artifacts.len();
1798 model.artifacts.push(Artifact {
1799 id,
1800 name: observed.label.clone(),
1801 kind: artifact_kind(&observed.label),
1802 path_expr: observed.path_expr.clone(),
1803 produced_by: Vec::new(),
1804 consumed_by: Vec::new(),
1805 source: source.clone(),
1806 evidence: vec![heuristic_source_evidence(
1807 source.clone(),
1808 format!("path passed to `{}`", observed.operation),
1809 )],
1810 });
1811 artifact_index.insert(identity, index);
1812 index
1813 }
1814 };
1815 let stage = stage_for_function(&function_id, &stage_by_function, model);
1816 if let Some(stage_id) = stage {
1817 let artifact = &mut model.artifacts[index];
1818 if observed.produced {
1819 artifact.produced_by.push(stage_id.clone());
1820 } else {
1821 artifact.consumed_by.push(stage_id.clone());
1822 }
1823 }
1824 }
1825
1826 for artifact in &mut model.artifacts {
1827 artifact.produced_by.sort();
1828 artifact.produced_by.dedup();
1829 artifact.consumed_by.sort();
1830 artifact.consumed_by.dedup();
1831 }
1832 infer_artifact_stage_links(model);
1833 let stage_orders: HashMap<StableId, usize> = model
1834 .stages
1835 .iter()
1836 .map(|stage| (stage.id.clone(), stage.order.unwrap_or(usize::MAX)))
1837 .collect();
1838 for stage in &mut model.stages {
1839 let stage_order = stage.order.unwrap_or(usize::MAX);
1840 for artifact in &model.artifacts {
1841 if artifact.produced_by.contains(&stage.id) {
1842 stage.produces.push(artifact.id.clone());
1843 }
1844 if artifact.consumed_by.contains(&stage.id) {
1845 stage.consumes.push(artifact.id.clone());
1846 stage.depends_on.extend(
1847 artifact
1848 .produced_by
1849 .iter()
1850 .filter(|producer| {
1851 **producer != stage.id
1852 && stage_orders
1853 .get(*producer)
1854 .is_some_and(|producer_order| *producer_order < stage_order)
1855 })
1856 .cloned(),
1857 );
1858 }
1859 }
1860 stage.consumes.sort();
1861 stage.consumes.dedup();
1862 stage.produces.sort();
1863 stage.produces.dedup();
1864 stage.depends_on.sort();
1865 stage.depends_on.dedup();
1866 }
1867
1868 let _ = functions;
1869}
1870
1871fn is_production_source(krate: &Crate, function: &ImplFn) -> bool {
1872 !function
1873 .module_path
1874 .split("::")
1875 .any(|segment| segment == "tests" || segment == "test")
1876 && krate.files.get(function.span.file).is_some_and(|file| {
1877 let rel = file.rel.replace('\\', "/");
1878 rel.starts_with("src/") || !rel.contains('/')
1879 })
1880}
1881
1882#[derive(Clone)]
1883struct ConstructorSpec {
1884 component: StableId,
1885 owner: String,
1886 builders: Vec<(usize, String)>,
1887}
1888
1889fn component_varmap_bindings(
1890 _krate: &Crate,
1891 function: &ImplFn,
1892 model: &ModelIr,
1893) -> HashMap<String, Vec<(StableId, String)>> {
1894 let constructor_functions: HashMap<StableId, &Function> = model
1895 .functions
1896 .iter()
1897 .map(|candidate| (candidate.id.clone(), candidate))
1898 .collect();
1899 let specs: Vec<ConstructorSpec> = model
1900 .components
1901 .iter()
1902 .filter_map(|component| {
1903 let constructor = constructor_functions.get(&component.constructor)?;
1904 let builders = constructor
1905 .parameters
1906 .iter()
1907 .enumerate()
1908 .filter(|(_, parameter)| parameter.type_name.contains("VarBuilder"))
1909 .zip(component.builders.iter())
1910 .map(|((index, _), builder)| (index, builder.name.clone()))
1911 .collect();
1912 Some(ConstructorSpec {
1913 component: component.id.clone(),
1914 owner: component.name.clone(),
1915 builders,
1916 })
1917 })
1918 .collect();
1919
1920 let mut builder_collector = BuilderSourceCollector::default();
1921 builder_collector.visit_block(&function.block);
1922 let mut collector = ConstructorBindingCollector {
1923 specs: &specs,
1924 builder_sources: &builder_collector.sources,
1925 bindings: HashMap::new(),
1926 };
1927 collector.visit_block(&function.block);
1928 for bindings in collector.bindings.values_mut() {
1929 bindings.sort();
1930 bindings.dedup();
1931 }
1932 collector.bindings
1933}
1934
1935#[derive(Default)]
1936struct BuilderSourceCollector {
1937 sources: HashMap<String, BTreeSet<String>>,
1938}
1939
1940impl<'ast> Visit<'ast> for BuilderSourceCollector {
1941 fn visit_local(&mut self, node: &'ast syn::Local) {
1942 if let (Some(name), Some(init)) = (pat_ident(&node.pat), node.init.as_ref()) {
1943 let varmaps = varmap_sources(&init.expr, &self.sources);
1944 if !varmaps.is_empty() {
1945 self.sources.entry(name).or_default().extend(varmaps);
1946 }
1947 }
1948 visit::visit_local(self, node);
1949 }
1950}
1951
1952struct ConstructorBindingCollector<'a> {
1953 specs: &'a [ConstructorSpec],
1954 builder_sources: &'a HashMap<String, BTreeSet<String>>,
1955 bindings: HashMap<String, Vec<(StableId, String)>>,
1956}
1957
1958impl<'ast> Visit<'ast> for ConstructorBindingCollector<'_> {
1959 fn visit_expr_call(&mut self, node: &'ast syn::ExprCall) {
1960 if let syn::Expr::Path(path) = &*node.func {
1961 let segments: Vec<_> = path
1962 .path
1963 .segments
1964 .iter()
1965 .map(|segment| segment.ident.to_string())
1966 .collect();
1967 if let Some(owner) = segments.get(segments.len().saturating_sub(2)) {
1968 for spec in self.specs.iter().filter(|spec| &spec.owner == owner) {
1969 for (index, root) in &spec.builders {
1970 let Some(argument) = node.args.iter().nth(*index) else {
1971 continue;
1972 };
1973 for varmap in varmap_sources(argument, self.builder_sources) {
1974 self.bindings
1975 .entry(varmap)
1976 .or_default()
1977 .push((spec.component.clone(), root.clone()));
1978 }
1979 }
1980 }
1981 }
1982 }
1983 visit::visit_expr_call(self, node);
1984 }
1985}
1986
1987fn varmap_sources(
1988 expression: &syn::Expr,
1989 builder_sources: &HashMap<String, BTreeSet<String>>,
1990) -> BTreeSet<String> {
1991 match expression {
1992 syn::Expr::Path(path) if path.path.segments.len() == 1 => {
1993 let name = path.path.segments[0].ident.to_string();
1994 builder_sources.get(&name).cloned().unwrap_or_default()
1995 }
1996 syn::Expr::Call(call) => {
1997 let is_from_varmap = matches!(
1998 &*call.func,
1999 syn::Expr::Path(path)
2000 if path.path.segments.last().is_some_and(|segment| segment.ident == "from_varmap")
2001 );
2002 if is_from_varmap {
2003 call.args
2004 .first()
2005 .and_then(expr_identifier)
2006 .into_iter()
2007 .collect()
2008 } else {
2009 BTreeSet::new()
2010 }
2011 }
2012 syn::Expr::MethodCall(call) => varmap_sources(&call.receiver, builder_sources),
2013 syn::Expr::Reference(reference) => varmap_sources(&reference.expr, builder_sources),
2014 syn::Expr::Try(value) => varmap_sources(&value.expr, builder_sources),
2015 syn::Expr::Paren(paren) => varmap_sources(&paren.expr, builder_sources),
2016 syn::Expr::Group(group) => varmap_sources(&group.expr, builder_sources),
2017 _ => BTreeSet::new(),
2018 }
2019}
2020
2021fn discover_optimizers(krate: &Crate, model: &mut ModelIr) {
2022 let stage_functions: Vec<(StableId, StableId, String)> = model
2023 .stages
2024 .iter()
2025 .map(|stage| {
2026 (
2027 stage.id.clone(),
2028 stage.function.clone(),
2029 stage.source.clone(),
2030 )
2031 })
2032 .collect();
2033 for func in krate.all_functions().chain(krate.all_methods()) {
2034 if !is_production_source(krate, func) {
2035 continue;
2036 }
2037 let text = func.block.to_token_stream().to_string();
2038 if !text.contains("all_vars") && !text.contains("named_train_vars") {
2039 continue;
2040 }
2041 let mut visitor = OptimizerVisitor::default();
2042 visitor.visit_block(&func.block);
2043 visitor.excludes.sort();
2044 visitor.excludes.dedup();
2045 visitor.includes.sort();
2046 visitor.includes.dedup();
2047 if visitor.optimizer.is_none() {
2048 continue;
2049 }
2050 let component_bindings = component_varmap_bindings(krate, func, model);
2051 let function_id = function_id(func);
2052 let exact_stage = stage_functions
2053 .iter()
2054 .find(|(_, stage_function, _)| stage_function == &function_id)
2055 .map(|(stage, _, _)| stage.clone())
2056 .or_else(|| {
2057 stage_functions
2058 .iter()
2059 .find(|(_, _, source)| {
2060 source.split(':').next()
2061 == Some(
2062 krate
2063 .file_label(func.span)
2064 .split(':')
2065 .next()
2066 .unwrap_or_default(),
2067 )
2068 })
2069 .map(|(stage, _, _)| stage.clone())
2070 });
2071 let mut stages: Vec<StableId> = exact_stage.into_iter().collect();
2072 if stages.is_empty() {
2073 stages.push(StableId::new("stage", [&func.qualified_name]));
2074 }
2075
2076 for varmap in visitor.varmaps {
2077 let bindings = component_bindings.get(&varmap).cloned().unwrap_or_default();
2078 let components: Vec<StableId> = bindings
2079 .iter()
2080 .map(|(component, _)| component.clone())
2081 .collect();
2082 let components = expand_nested_components(model, components);
2083 let roots: Vec<String> = bindings.iter().map(|(_, root)| root.clone()).collect();
2084 for stage in &stages {
2085 let id = StableId::new(
2086 "optimizer-membership",
2087 [
2088 stage.0.as_str(),
2089 varmap.as_str(),
2090 func.qualified_name.as_str(),
2091 ],
2092 );
2093 model.optimizers.push(OptimizerMembership {
2094 id,
2095 stage: stage.clone(),
2096 optimizer: visitor
2097 .optimizer
2098 .clone()
2099 .unwrap_or_else(|| "optimizer".to_string()),
2100 varmap: varmap.clone(),
2101 components: components.clone(),
2102 builder_roots: roots.clone(),
2103 include_patterns: visitor.includes.clone(),
2104 exclude_patterns: visitor.excludes.clone(),
2105 conditional: (stages.len() > 1)
2106 .then(|| "pipeline stage/configuration dependent".to_string()),
2107 source: krate.file_label(func.span),
2108 evidence: vec![heuristic_source_evidence(
2109 krate.file_label(func.span),
2110 "optimizer consumes variables returned by named_train_vars/all_vars",
2111 )],
2112 });
2113 }
2114 }
2115 }
2116
2117 let components_by_stage = model.optimizers.iter().fold(
2118 HashMap::<StableId, Vec<StableId>>::new(),
2119 |mut map, optimizer| {
2120 map.entry(optimizer.stage.clone())
2121 .or_default()
2122 .extend(optimizer.components.iter().cloned());
2123 map
2124 },
2125 );
2126 for stage in &mut model.stages {
2127 if let Some(components) = components_by_stage.get(&stage.id) {
2128 stage.components.extend(components.iter().cloned());
2129 stage.components.sort();
2130 stage.components.dedup();
2131 }
2132 }
2133}
2134
2135fn expand_nested_components(model: &ModelIr, mut components: Vec<StableId>) -> Vec<StableId> {
2136 let qualified_components: HashMap<&str, &StableId> = model
2137 .components
2138 .iter()
2139 .map(|component| (component.qualified_name.as_str(), &component.id))
2140 .collect();
2141 loop {
2142 let mut added = Vec::new();
2143 for module in model
2144 .modules
2145 .iter()
2146 .filter(|module| components.contains(&module.component))
2147 {
2148 let Some(qualified_type) = module.qualified_type.as_deref() else {
2149 continue;
2150 };
2151 if let Some(component) = qualified_components.get(qualified_type) {
2152 if !components.contains(component) {
2153 added.push((*component).clone());
2154 }
2155 }
2156 }
2157 if added.is_empty() {
2158 break;
2159 }
2160 components.extend(added);
2161 components.sort();
2162 components.dedup();
2163 }
2164 components
2165}
2166
2167fn apply_optimizer_roles(model: &mut ModelIr) {
2168 let component_names: HashMap<StableId, String> = model
2169 .components
2170 .iter()
2171 .map(|component| (component.id.clone(), component.name.to_ascii_lowercase()))
2172 .collect();
2173 for parameter in &mut model.parameters {
2174 let mut matched = Vec::new();
2175 let mut excluded = false;
2176 let mut conditionally_excluded_component = false;
2177 for optimizer in &model.optimizers {
2178 let component_matches = optimizer.components.is_empty()
2179 || optimizer.components.contains(¶meter.component);
2180 let root_matches = optimizer.builder_roots.contains(¶meter.builder_root)
2181 || optimizer.builder_roots.is_empty()
2182 && similar_stem(&optimizer.varmap, ¶meter.builder_root);
2183 if !component_matches || !root_matches {
2184 continue;
2185 }
2186 if optimizer
2187 .exclude_patterns
2188 .iter()
2189 .any(|pattern| parameter.key.contains(pattern))
2190 {
2191 excluded = true;
2192 continue;
2193 }
2194 if optimizer.exclude_patterns.iter().any(|pattern| {
2195 let prefix = pattern
2196 .trim_matches(|character: char| !character.is_alphanumeric())
2197 .to_ascii_lowercase();
2198 !prefix.is_empty()
2199 && component_names
2200 .get(¶meter.component)
2201 .is_some_and(|component| component.contains(&prefix))
2202 }) {
2203 conditionally_excluded_component = true;
2204 }
2205 if optimizer.include_patterns.is_empty()
2206 || optimizer
2207 .include_patterns
2208 .iter()
2209 .any(|pattern| parameter.key.contains(pattern))
2210 {
2211 matched.push(optimizer.id.clone());
2212 }
2213 }
2214 parameter.optimizer_memberships = matched;
2215 parameter.role = if is_running_state(¶meter.key) {
2216 ParameterRole::RunningState
2217 } else if !parameter.optimizer_memberships.is_empty() && conditionally_excluded_component {
2218 ParameterRole::Conditional
2219 } else if !parameter.optimizer_memberships.is_empty() {
2220 ParameterRole::Optimized
2221 } else if excluded {
2222 ParameterRole::Excluded
2223 } else if model.optimizers.iter().any(|optimizer| {
2224 optimizer.components.contains(¶meter.component)
2225 && !optimizer.builder_roots.contains(¶meter.builder_root)
2226 }) {
2227 ParameterRole::Frozen
2228 } else {
2229 ParameterRole::Unknown
2230 };
2231 parameter.evidence.push(Evidence {
2232 kind: EvidenceKind::Source,
2233 confidence: if parameter.optimizer_memberships.is_empty() {
2234 Confidence::Conditional
2235 } else {
2236 Confidence::Proven
2237 },
2238 source: Some(parameter.source.clone()),
2239 detail: format!(
2240 "role {:?} derived from optimizer membership, not constructor naming",
2241 parameter.role
2242 ),
2243 });
2244 }
2245
2246 for component in &mut model.components {
2247 for builder in &mut component.builders {
2248 let roles: Vec<_> = model
2249 .parameters
2250 .iter()
2251 .filter(|parameter| {
2252 parameter.component == component.id && parameter.builder_root == builder.name
2253 })
2254 .map(|parameter| ¶meter.role)
2255 .collect();
2256 builder.role = if roles
2257 .iter()
2258 .any(|role| matches!(role, ParameterRole::Optimized))
2259 {
2260 BuilderRole::Trainable
2261 } else if roles.iter().any(|role| {
2262 matches!(
2263 role,
2264 ParameterRole::Frozen | ParameterRole::Excluded | ParameterRole::RunningState
2265 )
2266 }) {
2267 BuilderRole::Frozen
2268 } else {
2269 BuilderRole::Unknown
2270 };
2271 }
2272 }
2273}
2274
2275fn entrypoint_analysis_phases(
2276 name: &str,
2277 qualified_name: &str,
2278 is_loss: bool,
2279) -> Vec<crate::phase::ExecutionPhase> {
2280 #[cfg(feature = "runtime")]
2281 {
2282 crate::phase::entrypoint_phases(name, qualified_name, is_loss)
2283 }
2284 #[cfg(not(feature = "runtime"))]
2285 {
2286 let _ = (name, qualified_name, is_loss);
2287 vec![crate::phase::ExecutionPhase::Train]
2288 }
2289}
2290
2291fn add_dataflow(krate: &Crate, model: &mut ModelIr) {
2292 let candle_nn_version = model.cargo.as_ref().and_then(|cargo| {
2293 op_semantics::matched_candle_version(
2294 cargo.candle_packages.get("candle-core").map(String::as_str),
2295 cargo.candle_packages.get("candle-nn").map(String::as_str),
2296 )
2297 .map(str::to_string)
2298 });
2299 let selected: Vec<(StableId, String, String, bool)> = model
2300 .functions
2301 .iter()
2302 .filter(|function| function.is_entrypoint && function.cfg_active != Some(false))
2303 .map(|function| {
2304 (
2305 function.id.clone(),
2306 function.name.clone(),
2307 function.qualified_name.clone(),
2308 function.is_loss,
2309 )
2310 })
2311 .collect();
2312
2313 let function_indices: HashMap<StableId, usize> = model
2314 .functions
2315 .iter()
2316 .enumerate()
2317 .map(|(index, function)| (function.id.clone(), index))
2318 .collect();
2319 for (function_id, name, entry, is_loss) in selected {
2320 let phases = entrypoint_analysis_phases(&name, &entry, is_loss);
2321 if let Some(&function_index) = function_indices.get(&function_id) {
2322 model.functions[function_index].execution_phases = phases.clone();
2323 }
2324 for phase in phases {
2325 let graph = match dataflow::analyze_with_phase(
2326 krate,
2327 &entry,
2328 candle_nn_version.as_deref(),
2329 phase,
2330 ) {
2331 Ok(graph) => graph,
2332 Err(error) => {
2333 push_finding(
2334 model,
2335 "dataflow-entrypoint",
2336 FindingSeverity::Information,
2337 Confidence::Proven,
2338 format!("could not analyze `{entry}`: {error:#}"),
2339 None,
2340 vec![function_id.clone()],
2341 );
2342 continue;
2343 }
2344 };
2345 let mut tensor_ids = vec![None; graph.nodes.len()];
2346 for node_id in &graph.tensor_nodes {
2347 let node = graph.node(*node_id);
2348 let id = StableId::new(
2349 "tensor",
2350 [
2351 phase.as_str(),
2352 function_id.0.as_str(),
2353 node.id.0.to_string().as_str(),
2354 ],
2355 );
2356 let name = node_name(&node.kind);
2357 let role = match &node.kind {
2358 NodeKind::Param { .. } => TensorRole::Input,
2359 NodeKind::Return => TensorRole::Output,
2360 _ if graph.loss_nodes.contains(&node.id) || is_loss => TensorRole::Loss,
2361 _ => TensorRole::Activation,
2362 };
2363 model.tensors.push(TensorContract {
2364 id: id.clone(),
2365 name,
2366 role,
2367 owner_function: function_id.clone(),
2368 parameter: None,
2369 shape: ShapeFact {
2370 rank: shape_rank(node.shape.as_deref()),
2371 dimensions: Vec::new(),
2372 source_expr: node.shape.clone(),
2373 },
2374 dtype: node.dtype.to_string(),
2375 device: DeviceFact::Unknown,
2376 layout: layout_for_node(&node.kind),
2377 requires_grad: match node.grad {
2378 GradState::Trainable | GradState::Differentiable => Some(true),
2379 GradState::Frozen | GradState::Severed => Some(false),
2380 _ => None,
2381 },
2382 execution_phase: Some(phase),
2383 evidence: vec![source_evidence(
2384 krate.file_label(node.span),
2385 format!(
2386 "expression-level static dataflow; source type {}",
2387 node.type_name.as_deref().unwrap_or("Tensor")
2388 ),
2389 )],
2390 });
2391 tensor_ids[node.id.0] = Some(id);
2392 }
2393 if let Some(&function_index) = function_indices.get(&function_id) {
2394 if phase == crate::phase::ExecutionPhase::Train {
2395 model.functions[function_index].tensor_inputs = graph
2396 .nodes
2397 .iter()
2398 .filter(|node| matches!(node.kind, NodeKind::Param { .. }))
2399 .filter_map(|node| tensor_ids[node.id.0].clone())
2400 .collect();
2401 model.functions[function_index].tensor_outputs = graph
2402 .entry_return
2403 .and_then(|node| tensor_ids[node.0].clone())
2404 .map(|tensor| vec![tensor])
2405 .unwrap_or_default();
2406 }
2407 }
2408 for node in &graph.nodes {
2409 let NodeKind::Call { callee } = &node.kind else {
2410 continue;
2411 };
2412 let effect = op_semantics::lookup_resolved(callee, candle_nn_version.as_deref());
2413 if matches!(effect.grad, GradFlow::Unknown)
2414 && matches!(effect.dtype, crate::op_semantics::DtypeRule::Unknown)
2415 {
2416 continue;
2417 }
2418 let Some(output) = tensor_ids[node.id.0].clone() else {
2419 continue;
2420 };
2421 let inputs = graph
2422 .edges
2423 .iter()
2424 .filter(|edge| edge.to == node.id)
2425 .filter_map(|edge| tensor_ids[edge.from.0].clone())
2426 .collect();
2427 let operation_id = StableId::new(
2428 "operation",
2429 [
2430 phase.as_str(),
2431 function_id.0.as_str(),
2432 node.id.0.to_string().as_str(),
2433 ],
2434 );
2435 model.operations.push(Operation {
2436 id: operation_id.clone(),
2437 function: function_id.clone(),
2438 name: effect.name.clone(),
2439 qualified_name: callee.contains("::").then(|| callee.clone()),
2440 inputs,
2441 output,
2442 source: krate.file_label(node.span),
2443 dtype_rule: format!("{:?}", effect.dtype),
2444 gradient_rule: format!("{:?}", effect.grad),
2445 device_rule: if effect.name == "to_device" {
2446 "explicit".to_string()
2447 } else {
2448 "preserve".to_string()
2449 },
2450 shape_rule: shape_rule(&effect.name).to_string(),
2451 domain_rule: effect.domain_rule_label(),
2452 execution_phase: Some(phase),
2453 avg_duration_ns: None,
2454 timing_samples: None,
2455 evidence: vec![Evidence {
2456 kind: EvidenceKind::Source,
2457 confidence: Confidence::Proven,
2458 source: Some(krate.file_label(node.span)),
2459 detail: effect.note.unwrap_or("Candle operation rule").to_string(),
2460 }],
2461 });
2462 link_implicit_parameter_reads(model, &graph, node.id, &function_id, &operation_id);
2463 }
2464 let dead_params = graph.dead_params();
2465 for conflict in &graph.dtype_conflicts {
2466 push_finding(
2467 model,
2468 "dtype-conflict",
2469 FindingSeverity::Error,
2470 Confidence::Proven,
2471 conflict.message.clone(),
2472 Some(krate.file_label(conflict.span)),
2473 vec![function_id.clone()],
2474 );
2475 }
2476 for risk in &graph.dtype_risks {
2477 push_finding(
2478 model,
2479 "dtype-risk",
2480 FindingSeverity::Warning,
2481 Confidence::Conditional,
2482 risk.message.clone(),
2483 Some(krate.file_label(risk.span)),
2484 vec![function_id.clone()],
2485 );
2486 }
2487 let inference_entrypoint = phase == crate::phase::ExecutionPhase::Infer;
2488 for violation in &graph.numeric_domain_violations {
2489 let (severity, confidence) =
2490 numeric_finding_severity(violation.proven, violation.impact, inference_entrypoint);
2491 push_finding(
2492 model,
2493 "numeric-domain-violation",
2494 severity,
2495 confidence,
2496 violation.message.clone(),
2497 Some(krate.file_label(violation.span)),
2498 vec![function_id.clone()],
2499 );
2500 }
2501 for nan_shape in &graph.zero_times_infinity {
2502 let (severity, confidence) =
2503 numeric_finding_severity(true, nan_shape.impact, inference_entrypoint);
2504 push_finding(
2505 model,
2506 "zero-times-infinity",
2507 severity,
2508 confidence,
2509 nan_shape.message.clone(),
2510 Some(krate.file_label(nan_shape.span)),
2511 vec![function_id.clone()],
2512 );
2513 if nan_shape.library_cite.is_some()
2514 && matches!(
2515 nan_shape.impact,
2516 NumericImpact::TrainingLossNaN | NumericImpact::GradientPoison
2517 )
2518 {
2519 push_finding(
2520 model,
2521 "unstable-library-loss",
2522 FindingSeverity::Error,
2523 Confidence::Proven,
2524 nan_shape.message.clone(),
2525 Some(krate.file_label(nan_shape.span)),
2526 vec![function_id.clone()],
2527 );
2528 }
2529 }
2530 for dead in dead_params {
2531 let mut related = vec![function_id.clone()];
2532 if let Some(tensor) = tensor_ids[dead.0].clone() {
2533 related.push(tensor);
2534 }
2535 push_finding(
2536 model,
2537 "dead-gradient-path",
2538 FindingSeverity::Warning,
2539 Confidence::Conditional,
2540 format!(
2541 "trainable expression `{}` has no differentiable path to a loss",
2542 node_name(&graph.node(dead).kind)
2543 ),
2544 Some(krate.file_label(graph.node(dead).span)),
2545 related,
2546 );
2547 }
2548 } }
2550}
2551
2552fn link_implicit_parameter_reads(
2553 model: &mut ModelIr,
2554 graph: &dataflow::ExprGraph,
2555 operation: dataflow::NodeId,
2556 function_id: &StableId,
2557 operation_id: &StableId,
2558) {
2559 let Some(module_node) = graph
2560 .edges
2561 .iter()
2562 .find(|edge| edge.to == operation && edge.label.as_deref() == Some("module"))
2563 .map(|edge| graph.node(edge.from))
2564 else {
2565 return;
2566 };
2567 let NodeKind::Local { name } = &module_node.kind else {
2568 return;
2569 };
2570 let field = name.strip_prefix('.').unwrap_or(name);
2571 let Some(type_name) = module_node
2572 .type_name
2573 .as_deref()
2574 .and_then(|name| name.rsplit("::").next())
2575 else {
2576 return;
2577 };
2578 let modules = model
2579 .modules
2580 .iter()
2581 .filter(|module| {
2582 module.field.as_deref() == Some(field)
2583 && module.type_name.rsplit("::").next() == Some(type_name)
2584 })
2585 .map(|module| module.id.clone())
2586 .collect::<HashSet<_>>();
2587 let owner_type = model
2588 .functions
2589 .iter()
2590 .find(|function| &function.id == function_id)
2591 .and_then(|function| function.owner_type.as_deref());
2592 let mut components = model
2593 .components
2594 .iter()
2595 .filter(|component| Some(component.qualified_name.as_str()) == owner_type)
2596 .map(|component| component.id.clone())
2597 .collect::<HashSet<_>>();
2598 if let Some(owner_type) = owner_type {
2599 let owner_leaf = owner_type.rsplit("::").next().unwrap_or(owner_type);
2600 components.extend(
2601 model
2602 .modules
2603 .iter()
2604 .filter(|module| {
2605 module.type_name.rsplit("::").next() == Some(owner_leaf)
2606 || module
2607 .qualified_type
2608 .as_deref()
2609 .and_then(|name| name.rsplit("::").next())
2610 == Some(owner_leaf)
2611 })
2612 .map(|module| module.component.clone()),
2613 );
2614 }
2615 for parameter in &mut model.parameters {
2616 let field_prefix_match = components.contains(¶meter.component)
2617 && parameter.key.split('.').any(|segment| segment == field);
2618 if (modules.contains(¶meter.module) || field_prefix_match)
2619 && !parameter.uses.contains(operation_id)
2620 {
2621 parameter.uses.push(operation_id.clone());
2622 }
2623 }
2624}
2625
2626#[cfg(feature = "runtime")]
2627fn aggregate_edge_timings(trace: &RuntimeTrace) -> Vec<EdgeTimingSummary> {
2628 let mut edge_totals: BTreeMap<(String, String), (u64, u64)> = BTreeMap::new();
2629 for edge in &trace.edge_timings {
2630 let key = (edge.from_static_id.clone(), edge.to_static_id.clone());
2631 let entry = edge_totals.entry(key).or_insert((0, 0));
2632 entry.0 = entry.0.saturating_add(edge.duration_ns);
2633 entry.1 += 1;
2634 }
2635 edge_totals
2636 .into_iter()
2637 .map(|((from, to), (total, samples))| EdgeTimingSummary {
2638 from: StableId(from),
2639 to: StableId(to),
2640 avg_duration_ns: total.checked_div(samples).unwrap_or(0),
2641 samples,
2642 })
2643 .collect()
2644}
2645
2646#[cfg(feature = "runtime")]
2647fn merge_runtime(model: &mut ModelIr, trace: &RuntimeTrace) {
2648 let expected = ExpectedIdentity {
2649 analysis_id: Some(model.analysis_id.0.clone()),
2650 build_id: model.cargo.as_ref().map(|cargo| cargo.build_id.clone()),
2651 };
2652 let audit = trace.audit_with_identity(Some(&expected));
2653 let identity_matches = audit.identity_mismatches.is_empty();
2654
2655 if identity_matches {
2656 let static_ids = trace
2657 .tensors
2658 .iter()
2659 .filter_map(|observation| observation.static_id.as_deref())
2660 .collect::<BTreeSet<_>>();
2661 let mut unknown_static_ids = 0usize;
2662 for static_id in static_ids {
2663 let Some(observation) = trace.agreed_tensor(static_id) else {
2664 continue;
2665 };
2666 let Some(tensor) = model
2667 .tensors
2668 .iter_mut()
2669 .find(|tensor| tensor.id.0 == static_id)
2670 else {
2671 unknown_static_ids += 1;
2672 continue;
2673 };
2674 tensor.shape.rank = Some(observation.shape.len());
2675 tensor.shape.dimensions = observation
2676 .shape
2677 .iter()
2678 .map(|dimension| crate::model_ir::Dimension {
2679 name: None,
2680 expr: dimension.to_string(),
2681 })
2682 .collect();
2683 tensor.dtype = observation.dtype.clone();
2684 tensor.device = parse_device(&observation.device);
2685 tensor.layout = if observation.contiguous {
2686 LayoutFact::Contiguous
2687 } else {
2688 LayoutFact::NonContiguous
2689 };
2690 tensor.requires_grad = Some(observation.requires_grad);
2691 tensor.evidence.push(Evidence {
2692 kind: EvidenceKind::Runtime,
2693 confidence: Confidence::Proven,
2694 source: observation.source.clone(),
2695 detail: format!("agreed runtime observation {}", observation.event_id),
2696 });
2697 }
2698 if unknown_static_ids > 0 {
2699 push_finding(
2700 model,
2701 "runtime-unmapped",
2702 FindingSeverity::Information,
2703 Confidence::Unknown,
2704 format!(
2705 "{unknown_static_ids} runtime tensor identities did not exist in this analysis"
2706 ),
2707 None,
2708 Vec::new(),
2709 );
2710 }
2711
2712 for parameter in &mut model.parameters {
2713 if let Some(gradient) = trace.gradient(¶meter.builder_root, ¶meter.key) {
2714 parameter.evidence.push(Evidence {
2715 kind: EvidenceKind::Runtime,
2716 confidence: Confidence::Proven,
2717 source: None,
2718 detail: format!("gradient {:?}, norm {:?}", gradient.state, gradient.norm),
2719 });
2720 }
2721 }
2722 for observation in &trace.operations {
2723 let Some(static_id) = observation.static_id.as_deref() else {
2724 continue;
2725 };
2726 if let Some(operation) = model
2727 .operations
2728 .iter_mut()
2729 .find(|operation| operation.id.0 == static_id)
2730 {
2731 let mut detail = format!(
2732 "runtime operation {} observed as {}",
2733 observation.event_id, observation.op
2734 );
2735 if let Some(duration_ns) = observation.duration_ns {
2736 detail.push_str(&format!(" duration_ns={duration_ns}"));
2737 }
2738 operation.evidence.push(Evidence {
2739 kind: EvidenceKind::Runtime,
2740 confidence: Confidence::Proven,
2741 source: observation.source.clone(),
2742 detail,
2743 });
2744 }
2745 }
2746 let mut op_duration_totals: BTreeMap<String, (u64, u64)> = BTreeMap::new();
2747 for observation in &trace.operations {
2748 let Some(static_id) = observation.static_id.as_deref() else {
2749 continue;
2750 };
2751 let Some(duration_ns) = observation.duration_ns else {
2752 continue;
2753 };
2754 let entry = op_duration_totals
2755 .entry(static_id.to_string())
2756 .or_insert((0, 0));
2757 entry.0 = entry.0.saturating_add(duration_ns);
2758 entry.1 += 1;
2759 }
2760 for (static_id, (total, samples)) in op_duration_totals {
2761 if samples == 0 {
2762 continue;
2763 }
2764 if let Some(operation) = model
2765 .operations
2766 .iter_mut()
2767 .find(|operation| operation.id.0 == static_id)
2768 {
2769 operation.avg_duration_ns = Some(total / samples);
2770 operation.timing_samples = Some(samples);
2771 }
2772 }
2773 for identity in audit
2774 .missing_gradients
2775 .iter()
2776 .chain(&audit.zero_gradients)
2777 .chain(&audit.non_finite_gradients)
2778 {
2779 let Some(gradient) = trace.gradient(&identity.root, &identity.key) else {
2780 continue;
2781 };
2782 let severity = if gradient.state == GradientState::NonFinite {
2783 FindingSeverity::Error
2784 } else {
2785 FindingSeverity::Warning
2786 };
2787 push_finding(
2788 model,
2789 "runtime-gradient",
2790 severity,
2791 Confidence::Proven,
2792 format!(
2793 "{}:{} runtime gradient is {:?}",
2794 gradient.root, gradient.key, gradient.state
2795 ),
2796 None,
2797 model
2798 .parameters
2799 .iter()
2800 .filter(|parameter| {
2801 parameter.builder_root == gradient.root && parameter.key == gradient.key
2802 })
2803 .map(|parameter| parameter.id.clone())
2804 .collect(),
2805 );
2806 }
2807 }
2808 for conflict in &audit.tensor_conflicts {
2809 push_finding(
2810 model,
2811 "runtime-tensor-conflict",
2812 FindingSeverity::Error,
2813 Confidence::Unknown,
2814 format!(
2815 "{} has conflicting runtime {:?}: {}",
2816 conflict.static_id,
2817 conflict.kind,
2818 conflict.values.join(", ")
2819 ),
2820 None,
2821 vec![StableId(conflict.static_id.clone())],
2822 );
2823 }
2824 for conflict in &audit.gradient_conflicts {
2825 push_finding(
2826 model,
2827 "runtime-gradient-conflict",
2828 FindingSeverity::Error,
2829 Confidence::Unknown,
2830 format!(
2831 "{}:{} has contradictory gradient observations: {}",
2832 conflict.identity.root,
2833 conflict.identity.key,
2834 conflict.states.join(", ")
2835 ),
2836 None,
2837 model
2838 .parameters
2839 .iter()
2840 .filter(|parameter| {
2841 parameter.builder_root == conflict.identity.root
2842 && parameter.key == conflict.identity.key
2843 })
2844 .map(|parameter| parameter.id.clone())
2845 .collect(),
2846 );
2847 }
2848 for mismatch in &audit.identity_mismatches {
2849 push_finding(
2850 model,
2851 "runtime-identity",
2852 FindingSeverity::Error,
2853 Confidence::Unknown,
2854 format!(
2855 "runtime {} mismatch: expected {}, observed {}; trace was not merged",
2856 mismatch.field, mismatch.expected, mismatch.observed
2857 ),
2858 None,
2859 Vec::new(),
2860 );
2861 }
2862 model.coverage.runtime_observations = trace.tensors.len()
2863 + trace.operations.len()
2864 + trace.gradients.len()
2865 + trace.values.len()
2866 + trace.edge_timings.len();
2867 let edge_timing_summaries = aggregate_edge_timings(trace);
2868 let avg_operation_duration_ns = trace
2869 .operations
2870 .iter()
2871 .filter_map(|op| op.duration_ns)
2872 .reduce(|a, b| a.saturating_add(b))
2873 .and_then(|total| {
2874 let count = trace.operations.iter().filter(|op| op.duration_ns.is_some()).count();
2875 (count > 0).then_some(total / count as u64)
2876 });
2877 model.runtime = Some(RuntimeSummary {
2878 trace_schema: trace.schema.clone(),
2879 entrypoint: Some(trace.run.entrypoint.clone()),
2880 profile: Some(trace.run.profile.clone()),
2881 tensor_observations: trace.tensors.len(),
2882 operation_observations: trace.operations.len(),
2883 gradient_observations: trace.gradients.len(),
2884 missing_gradients: audit.missing_gradients.len(),
2885 zero_gradients: audit.zero_gradients.len(),
2886 non_finite_gradients: audit.non_finite_gradients.len(),
2887 tensor_conflicts: audit.tensor_conflicts.len(),
2888 gradient_conflicts: audit.gradient_conflicts.len(),
2889 identity_mismatches: audit.identity_mismatches.len(),
2890 first_non_finite_step: audit.first_non_finite_step,
2891 saturating_activations: audit.saturating_activations.len(),
2892 value_observations: trace.values.len(),
2893 execution_phase: trace
2894 .run
2895 .phase
2896 .as_deref()
2897 .and_then(crate::phase::ExecutionPhase::parse),
2898 avg_operation_duration_ns,
2899 edge_timings: edge_timing_summaries,
2900 });
2901}
2902
2903fn flag_candle_semantics_version(model: &mut ModelIr, cargo: &CargoContext) {
2904 for (package, version) in &cargo.candle_versions {
2905 if package == "candle-core" || package == "candle-nn" {
2906 let supported = op_semantics::is_audited_candle_version(version);
2907 if !supported {
2908 push_finding(
2909 model,
2910 "candle-semantics-version",
2911 FindingSeverity::Warning,
2912 Confidence::Proven,
2913 format!(
2914 "{package} {version} differs from the audited operation catalogs \
2915 {}; \
2916 unknown or changed operations remain explicit",
2917 op_semantics::AUDITED_CANDLE_VERSION
2918 ),
2919 Some(cargo.manifest_path.to_string_lossy().into_owned()),
2920 Vec::new(),
2921 );
2922 }
2923 }
2924 }
2925 let core = cargo.candle_versions.get("candle-core");
2926 let nn = cargo.candle_versions.get("candle-nn");
2927 if let (Some(core), Some(nn)) = (core, nn) {
2928 if core != nn {
2929 push_finding(
2930 model,
2931 "candle-semantics-version",
2932 FindingSeverity::Warning,
2933 Confidence::Proven,
2934 format!(
2935 "candle-core {core} and candle-nn {nn} do not match; version-sensitive \
2936 constructor and autograd rules remain Unknown"
2937 ),
2938 Some(cargo.manifest_path.to_string_lossy().into_owned()),
2939 Vec::new(),
2940 );
2941 }
2942 }
2943}
2944
2945#[derive(Default)]
2946struct CallCollector {
2947 calls: Vec<String>,
2948}
2949
2950impl<'ast> Visit<'ast> for CallCollector {
2951 fn visit_expr_call(&mut self, node: &'ast syn::ExprCall) {
2952 if let syn::Expr::Path(path) = &*node.func {
2953 self.calls
2954 .push(path.path.to_token_stream().to_string().replace(' ', ""));
2955 }
2956 visit::visit_expr_call(self, node);
2957 }
2958
2959 fn visit_expr_method_call(&mut self, node: &'ast syn::ExprMethodCall) {
2960 self.calls.push(node.method.to_string());
2961 visit::visit_expr_method_call(self, node);
2962 }
2963}
2964
2965#[derive(Debug)]
2966struct ArtifactObservation {
2967 operation: String,
2968 path_expr: String,
2969 label: String,
2970 produced: bool,
2971}
2972
2973#[derive(Default)]
2974struct ArtifactVisitor {
2975 observed: Vec<ArtifactObservation>,
2976}
2977
2978impl<'ast> Visit<'ast> for ArtifactVisitor {
2979 fn visit_local(&mut self, node: &'ast syn::Local) {
2980 if let Some(init) = &node.init {
2981 let path_expr = format!(
2982 "{} = {}",
2983 node.pat.to_token_stream(),
2984 init.expr.to_token_stream()
2985 );
2986 let strings = string_literals(&init.expr);
2987 if let Some(label) = strings
2988 .iter()
2989 .rev()
2990 .find(|value| is_artifact_literal(value))
2991 .cloned()
2992 .filter(|_| path_expr.len() <= 256)
2993 {
2994 self.observed.push(ArtifactObservation {
2995 operation: "path_binding".to_string(),
2996 path_expr,
2997 label,
2998 produced: false,
2999 });
3000 }
3001 }
3002 visit::visit_local(self, node);
3003 }
3004
3005 fn visit_expr_struct(&mut self, node: &'ast syn::ExprStruct) {
3006 for field in &node.fields {
3007 let strings = string_literals(&field.expr);
3008 if let Some(label) = strings
3009 .iter()
3010 .rev()
3011 .find(|value| is_artifact_literal(value))
3012 .cloned()
3013 {
3014 self.observed.push(ArtifactObservation {
3015 operation: "path_field".to_string(),
3016 path_expr: format!(
3017 "{} = {}",
3018 field.member.to_token_stream(),
3019 field.expr.to_token_stream()
3020 ),
3021 label,
3022 produced: false,
3023 });
3024 }
3025 }
3026 visit::visit_expr_struct(self, node);
3027 }
3028
3029 fn visit_expr_call(&mut self, node: &'ast syn::ExprCall) {
3030 let op = match &*node.func {
3031 syn::Expr::Path(path) => path.path.to_token_stream().to_string().replace(' ', ""),
3032 _ => String::new(),
3033 };
3034 if is_artifact_operation(&op) {
3035 for argument in &node.args {
3036 self.observe(&op, argument);
3037 }
3038 }
3039 visit::visit_expr_call(self, node);
3040 }
3041
3042 fn visit_expr_method_call(&mut self, node: &'ast syn::ExprMethodCall) {
3043 let op = node.method.to_string();
3044 if is_artifact_operation(&op) {
3045 self.observe(&op, &node.receiver);
3046 for argument in &node.args {
3047 self.observe(&op, argument);
3048 }
3049 }
3050 visit::visit_expr_method_call(self, node);
3051 }
3052}
3053
3054impl ArtifactVisitor {
3055 fn observe(&mut self, operation: &str, expr: &syn::Expr) {
3056 let text = expr.to_token_stream().to_string();
3057 let strings = string_literals(expr);
3058 let Some(label) = strings
3059 .iter()
3060 .rev()
3061 .find(|value| is_artifact_literal(value))
3062 .cloned()
3063 else {
3064 return;
3065 };
3066 self.observed.push(ArtifactObservation {
3067 operation: operation.to_string(),
3068 path_expr: text,
3069 label,
3070 produced: is_producer_operation(operation),
3071 });
3072 }
3073}
3074
3075#[derive(Default)]
3076struct StringCollector {
3077 values: Vec<String>,
3078}
3079
3080impl<'ast> Visit<'ast> for StringCollector {
3081 fn visit_lit_str(&mut self, node: &'ast syn::LitStr) {
3082 self.values.push(node.value());
3083 }
3084}
3085
3086fn string_literals(expr: &syn::Expr) -> Vec<String> {
3087 let mut collector = StringCollector::default();
3088 collector.visit_expr(expr);
3089 collector.values
3090}
3091
3092#[derive(Default)]
3093struct OptimizerVisitor {
3094 varmaps: BTreeSet<String>,
3095 includes: Vec<String>,
3096 excludes: Vec<String>,
3097 optimizer: Option<String>,
3098}
3099
3100impl<'ast> Visit<'ast> for OptimizerVisitor {
3101 fn visit_expr_method_call(&mut self, node: &'ast syn::ExprMethodCall) {
3102 let method = node.method.to_string();
3103 if method == "all_vars" {
3104 self.varmaps
3105 .insert(node.receiver.to_token_stream().to_string().replace(' ', ""));
3106 }
3107 if method == "retain" {
3108 let mut patterns = RetainPatternVisitor::default();
3109 for argument in &node.args {
3110 patterns.visit_expr(argument);
3111 }
3112 self.includes.extend(patterns.includes);
3113 self.excludes.extend(patterns.excludes);
3114 }
3115 visit::visit_expr_method_call(self, node);
3116 }
3117
3118 fn visit_expr_call(&mut self, node: &'ast syn::ExprCall) {
3119 if let syn::Expr::Path(path) = &*node.func {
3120 let name = path.path.to_token_stream().to_string().replace(' ', "");
3121 if name.ends_with("named_train_vars") {
3122 if let Some(varmap) = node.args.first().and_then(expr_identifier) {
3123 self.varmaps.insert(varmap);
3124 }
3125 }
3126 if name.contains("Adam") || name.contains("Optimizer") {
3127 self.optimizer = Some(name);
3128 }
3129 }
3130 visit::visit_expr_call(self, node);
3131 }
3132}
3133
3134#[derive(Default)]
3135struct RetainPatternVisitor {
3136 negated: bool,
3137 includes: Vec<String>,
3138 excludes: Vec<String>,
3139}
3140
3141impl<'ast> Visit<'ast> for RetainPatternVisitor {
3142 fn visit_expr_method_call(&mut self, node: &'ast syn::ExprMethodCall) {
3143 if matches!(
3144 node.method.to_string().as_str(),
3145 "contains" | "starts_with" | "ends_with"
3146 ) {
3147 if let Some(syn::Expr::Lit(syn::ExprLit {
3148 lit: syn::Lit::Str(value),
3149 ..
3150 })) = node.args.first()
3151 {
3152 if self.negated {
3153 self.excludes.push(value.value());
3154 } else {
3155 self.includes.push(value.value());
3156 }
3157 }
3158 }
3159 visit::visit_expr_method_call(self, node);
3160 }
3161
3162 fn visit_expr_unary(&mut self, node: &'ast syn::ExprUnary) {
3163 if matches!(node.op, syn::UnOp::Not(_)) {
3164 self.negated = !self.negated;
3165 self.visit_expr(&node.expr);
3166 self.negated = !self.negated;
3167 } else {
3168 visit::visit_expr_unary(self, node);
3169 }
3170 }
3171}
3172
3173fn expr_identifier(expression: &syn::Expr) -> Option<String> {
3174 match expression {
3175 syn::Expr::Path(path) if path.path.segments.len() == 1 => {
3176 Some(path.path.segments[0].ident.to_string())
3177 }
3178 syn::Expr::Reference(reference) => expr_identifier(&reference.expr),
3179 syn::Expr::Paren(paren) => expr_identifier(&paren.expr),
3180 syn::Expr::Group(group) => expr_identifier(&group.expr),
3181 syn::Expr::Try(value) => expr_identifier(&value.expr),
3182 _ => None,
3183 }
3184}
3185
3186fn function_id(func: &ImplFn) -> StableId {
3187 StableId::new(
3188 "function",
3189 [
3190 func.qualified_name.as_str(),
3191 func.cfg_predicates.join(" && ").as_str(),
3192 ],
3193 )
3194}
3195
3196fn unique_qualified_struct(krate: &Crate, name: &str) -> Option<String> {
3197 match krate.struct_candidates(name).as_slice() {
3198 [def] => Some(def.qualified_name.clone()),
3199 _ => None,
3200 }
3201}
3202
3203fn source_evidence(source: String, detail: impl Into<String>) -> Evidence {
3204 Evidence {
3205 kind: EvidenceKind::Source,
3206 confidence: Confidence::Proven,
3207 source: Some(source),
3208 detail: detail.into(),
3209 }
3210}
3211
3212fn heuristic_source_evidence(source: String, detail: impl Into<String>) -> Evidence {
3213 Evidence {
3214 kind: EvidenceKind::Source,
3215 confidence: Confidence::Heuristic,
3216 source: Some(source),
3217 detail: detail.into(),
3218 }
3219}
3220
3221fn numeric_finding_severity(
3222 proven: bool,
3223 impact: NumericImpact,
3224 inference_entrypoint: bool,
3225) -> (FindingSeverity, Confidence) {
3226 if !proven {
3227 return (FindingSeverity::Warning, Confidence::Unknown);
3228 }
3229 match impact {
3230 NumericImpact::TrainingLossNaN | NumericImpact::GradientPoison => {
3231 (FindingSeverity::Error, Confidence::Proven)
3232 }
3233 NumericImpact::InferenceOutputRisk if inference_entrypoint => {
3234 (FindingSeverity::Error, Confidence::Proven)
3235 }
3236 NumericImpact::InferenceOutputRisk | NumericImpact::LocalOnly => {
3237 (FindingSeverity::Warning, Confidence::Proven)
3238 }
3239 }
3240}
3241
3242fn push_finding(
3243 model: &mut ModelIr,
3244 rule: &str,
3245 severity: FindingSeverity,
3246 confidence: Confidence,
3247 message: String,
3248 source: Option<String>,
3249 related: Vec<StableId>,
3250) {
3251 model.findings.push(Finding {
3252 id: StableId::new(
3253 "finding",
3254 [rule, source.as_deref().unwrap_or(""), message.as_str()],
3255 ),
3256 rule: rule.to_string(),
3257 severity,
3258 confidence,
3259 message,
3260 source,
3261 related,
3262 evidence: Vec::new(),
3263 });
3264}
3265
3266fn visibility(text: &str) -> Visibility {
3267 match text {
3268 "" => Visibility::Private,
3269 "pub" => Visibility::Public,
3270 "pub(crate)" => Visibility::Crate,
3271 value if value.starts_with("pub(") => Visibility::Restricted,
3272 _ => Visibility::Unknown,
3273 }
3274}
3275
3276fn certainty_confidence(certainty: &Certainty) -> Confidence {
3277 match certainty {
3278 Certainty::Certain => Confidence::Proven,
3279 Certainty::Conditional(_) => Confidence::Conditional,
3280 Certainty::Unknown(_) => Confidence::Unknown,
3281 }
3282}
3283
3284fn is_tensor_type(ty: &str) -> bool {
3285 ty.split(|character: char| !character.is_alphanumeric() && character != '_')
3286 .any(|segment| segment == "Tensor")
3287}
3288
3289fn is_model_entry_name(name: &str) -> bool {
3290 matches!(
3291 name,
3292 "forward"
3293 | "forward_diff"
3294 | "forward_features"
3295 | "forward_t"
3296 | "encode"
3297 | "predict"
3298 | "compress"
3299 | "conditioning"
3300 | "condition"
3301 | "generate"
3302 | "decode"
3303 | "loss"
3304 ) || name.ends_with("_forward")
3305 || name.ends_with("_loss")
3306}
3307
3308fn is_stage_entry_name(name: &str) -> bool {
3309 name == "prepare"
3310 || name.starts_with("try_run_")
3311 || name.starts_with("run_")
3312 && (name.contains("train") || name.contains("eval") || name.contains("prepare"))
3313}
3314
3315fn is_pipeline_stage_call(name: &str) -> bool {
3316 name == "prepare_data"
3317 || name == "final_eval"
3318 || name.starts_with("train_")
3319 || name.starts_with("evaluate_")
3320 || name.starts_with("export_")
3321}
3322
3323fn stage_variants(krate: &Crate, target: &StableId) -> Vec<String> {
3324 let _ = (krate, target);
3325 Vec::new()
3326}
3327
3328fn stage_kind(name: &str) -> StageKind {
3329 if name.contains("eval") {
3330 StageKind::Evaluate
3331 } else if name.contains("prepare") || name.contains("data") {
3332 StageKind::Prepare
3333 } else if name.contains("train")
3334 || name.contains("bridge")
3335 || name == "latent"
3336 || name == "world"
3337 {
3338 StageKind::Train
3339 } else if name.contains("export") {
3340 StageKind::Export
3341 } else if name.starts_with("preflight_") {
3342 StageKind::Probe
3343 } else {
3344 StageKind::Unknown
3345 }
3346}
3347
3348fn stage_display_name(function: &Function) -> String {
3349 let module = function
3350 .qualified_name
3351 .rsplit_once("::")
3352 .map(|(module, _)| module.rsplit("::").next().unwrap_or(module))
3353 .unwrap_or_default();
3354 if module.is_empty() || function.name.contains(module) {
3355 function.name.clone()
3356 } else {
3357 format!("{module}:{}", function.name)
3358 }
3359}
3360
3361fn reachable_components(function: &Function, model: &ModelIr) -> Vec<StableId> {
3362 let mut components = Vec::new();
3363 let mut queue = vec![function.id.clone()];
3364 let mut seen = HashSet::new();
3365 while let Some(id) = queue.pop() {
3366 if !seen.insert(id.clone()) {
3367 continue;
3368 }
3369 if let Some(func) = model.functions.iter().find(|func| func.id == id) {
3370 if let Some(owner) = func.owner_type.as_deref() {
3371 components.extend(
3372 model
3373 .components
3374 .iter()
3375 .filter(|component| component.qualified_name == owner)
3376 .map(|component| component.id.clone()),
3377 );
3378 }
3379 for edge in model
3380 .architecture_edges
3381 .iter()
3382 .filter(|edge| edge.via_function == id)
3383 {
3384 components.push(edge.from.clone());
3385 components.push(edge.to.clone());
3386 }
3387 queue.extend(func.calls.iter().cloned());
3388 }
3389 }
3390 components.sort();
3391 components.dedup();
3392 components
3393}
3394
3395fn stage_for_function(
3396 function: &StableId,
3397 stages: &HashMap<StableId, StableId>,
3398 model: &ModelIr,
3399) -> Option<StableId> {
3400 if let Some(stage) = stages.get(function) {
3401 return Some(stage.clone());
3402 }
3403 model.stages.iter().find_map(|stage| {
3404 reachable_function(&stage.function, function, model, &mut HashSet::new())
3405 .then(|| stage.id.clone())
3406 })
3407}
3408
3409fn reachable_function(
3410 current: &StableId,
3411 target: &StableId,
3412 model: &ModelIr,
3413 seen: &mut HashSet<StableId>,
3414) -> bool {
3415 if current == target {
3416 return true;
3417 }
3418 if !seen.insert(current.clone()) {
3419 return false;
3420 }
3421 model
3422 .functions
3423 .iter()
3424 .find(|function| &function.id == current)
3425 .is_some_and(|function| {
3426 function
3427 .calls
3428 .iter()
3429 .any(|callee| reachable_function(callee, target, model, seen))
3430 })
3431}
3432
3433fn qualify(module: &str, path: &str) -> String {
3434 if module.is_empty() {
3435 path.to_string()
3436 } else if path.is_empty() {
3437 module.to_string()
3438 } else {
3439 format!("{module}::{path}")
3440 }
3441}
3442
3443fn acquisition_label(acquisition: &Acquisition) -> String {
3444 match acquisition {
3445 Acquisition::Constructor { func, .. } => func.clone(),
3446 Acquisition::RawGet { method } => method.clone(),
3447 }
3448}
3449
3450fn similar_stem(left: &str, right: &str) -> bool {
3451 fn stem(value: &str) -> String {
3452 value
3453 .replace("varmap", "")
3454 .replace("var_map", "")
3455 .replace("vb", "")
3456 .replace(['_', '&'], "")
3457 }
3458 let left = stem(left);
3459 let right = stem(right);
3460 !left.is_empty() && !right.is_empty() && (left.contains(&right) || right.contains(&left))
3461}
3462
3463fn is_running_state(key: &str) -> bool {
3464 key.contains("running_mean")
3465 || key.contains("running_var")
3466 || key.contains("num_batches_tracked")
3467}
3468
3469fn is_artifact_operation(name: &str) -> bool {
3470 let leaf = name.rsplit("::").next().unwrap_or(name);
3471 matches!(
3472 leaf,
3473 "load"
3474 | "save"
3475 | "read"
3476 | "write"
3477 | "open"
3478 | "mmap"
3479 | "from_mmaped_safetensors"
3480 | "from_buffered_safetensors"
3481 | "serialize_to_file"
3482 | "load_buffer"
3483 )
3484}
3485
3486fn is_producer_operation(name: &str) -> bool {
3487 let leaf = name.rsplit("::").next().unwrap_or(name);
3488 matches!(leaf, "save" | "write" | "serialize_to_file")
3489}
3490
3491fn is_artifact_literal(value: &str) -> bool {
3492 let lower = value.to_ascii_lowercase();
3493 [
3494 ".safetensors",
3495 ".safetensor",
3496 ".json",
3497 ".jsonl",
3498 ".bin",
3499 ".pt",
3500 ".pth",
3501 ".gguf",
3502 ".parquet",
3503 ".arrow",
3504 ]
3505 .iter()
3506 .any(|suffix| lower.ends_with(suffix))
3507}
3508
3509fn artifact_identity(path_expr: &str) -> String {
3510 path_expr
3511 .split_once('=')
3512 .map(|(binding, _)| binding.trim().replace(' ', ""))
3513 .filter(|binding| {
3514 !binding.is_empty()
3515 && binding
3516 .chars()
3517 .all(|character| character.is_alphanumeric() || character == '_')
3518 })
3519 .unwrap_or_else(|| path_expr.to_string())
3520}
3521
3522fn infer_artifact_stage_links(model: &mut ModelIr) {
3523 let _ = model;
3524}
3525
3526fn artifact_kind(label: &str) -> ArtifactKind {
3527 let lower = label.to_ascii_lowercase();
3528 if lower.ends_with(".safetensors")
3529 || lower.ends_with(".bin")
3530 || lower.ends_with(".pt")
3531 || lower.ends_with(".pth")
3532 || lower.ends_with(".gguf")
3533 {
3534 ArtifactKind::Checkpoint
3535 } else if lower.contains("eval") || lower.contains("report") || lower.contains("metric") {
3536 ArtifactKind::EvaluationReport
3537 } else if lower.ends_with(".parquet") || lower.ends_with(".arrow") {
3538 ArtifactKind::Dataset
3539 } else if lower.ends_with(".json") {
3540 ArtifactKind::Configuration
3541 } else {
3542 ArtifactKind::Unknown
3543 }
3544}
3545
3546fn node_name(kind: &NodeKind) -> String {
3547 match kind {
3548 NodeKind::Param { name } | NodeKind::Local { name } => name.clone(),
3549 NodeKind::Call { callee } => format!("result_of_{callee}"),
3550 NodeKind::Literal { text } => text.clone(),
3551 NodeKind::Phi => "branch_join".to_string(),
3552 NodeKind::Return => "return".to_string(),
3553 NodeKind::Unknown { reason } => format!("unknown:{reason}"),
3554 }
3555}
3556
3557fn shape_rank(shape: Option<&str>) -> Option<usize> {
3558 let shape = shape?;
3559 let trimmed = shape.trim();
3560 if (trimmed.starts_with('(') && trimmed.ends_with(')'))
3561 || (trimmed.starts_with('[') && trimmed.ends_with(']'))
3562 {
3563 Some(
3564 trimmed[1..trimmed.len() - 1]
3565 .split(',')
3566 .filter(|part| !part.trim().is_empty())
3567 .count(),
3568 )
3569 } else {
3570 None
3571 }
3572}
3573
3574fn layout_for_node(kind: &NodeKind) -> LayoutFact {
3575 match kind {
3576 NodeKind::Call { callee } if callee.ends_with("contiguous") => LayoutFact::Contiguous,
3577 NodeKind::Call { callee }
3578 if ["transpose", "permute", "narrow", "t"]
3579 .iter()
3580 .any(|op| callee.ends_with(op)) =>
3581 {
3582 LayoutFact::Strided
3583 }
3584 _ => LayoutFact::Unknown,
3585 }
3586}
3587
3588fn shape_rule(op: &str) -> &'static str {
3589 match op {
3590 "reshape" | "broadcast_as" | "expand" => "explicit_argument",
3591 "flatten_all" | "flatten_to" | "flatten_from" => "flatten",
3592 "squeeze" => "remove_dimension",
3593 "unsqueeze" => "insert_dimension",
3594 "transpose" | "permute" | "t" => "permute_dimensions",
3595 "narrow" => "slice_dimension",
3596 "matmul" | "broadcast_matmul" => "matrix_product",
3597 _ => "preserve_or_operation_defined",
3598 }
3599}
3600
3601#[cfg(feature = "runtime")]
3602fn parse_device(device: &str) -> DeviceFact {
3603 let lower = device.to_ascii_lowercase();
3604 if lower == "cpu" {
3605 DeviceFact::Cpu
3606 } else if lower.starts_with("cuda") {
3607 let ordinal = lower
3608 .split([':', '(', ')'])
3609 .find_map(|part| part.parse::<u32>().ok());
3610 DeviceFact::Cuda { ordinal }
3611 } else if lower.starts_with("metal") {
3612 DeviceFact::Metal
3613 } else {
3614 DeviceFact::Unknown
3615 }
3616}