1use std::sync::Arc;
2
3use tenferro_ops::dim_expr::DimExpr;
4use tenferro_ops::ext_op::{
5 ExtensionAlias, ExtensionAliasDeclaration, ExtensionEffectAccess, ExtensionEffectDeclaration,
6 ExtensionOp,
7};
8use tenferro_ops::shape_extent::ShapeExtent;
9use tenferro_tensor::Tensor;
10
11use crate::checkpoint::RetainedValue;
12
13use super::bindings::PendingBinding;
14use super::identity::SemanticIdentity;
15use super::metadata::SemanticProvenance;
16use super::op::{SemanticOp, SemanticOperation};
17use super::value::ProgramBuilderNonce;
18use super::{
19 Alias, BindingKey, CoreSemanticOp, Effect, EffectAccess, EffectResource, FrozenProgram,
20 ImportedProgramValues, ProgramBindingError, ProgramBindings, ProgramBuildError,
21 ProgramFinishError, ProgramImport, ProgramInputSpec, ProgramShapeRelation,
22 ProgramStructuralError, ProgramValue, ProgramValueMetadata, SemanticPlacementConstraint,
23 SemanticProgram, ShapeGuard,
24};
25
26pub struct SemanticProgramBuilder {
28 owner: ProgramBuilderNonce,
29 inputs: Vec<ProgramValue>,
30 input_specs: Vec<ProgramInputSpec>,
31 values: Vec<ProgramValueMetadata>,
32 operations: Vec<SemanticOperation>,
33 bindings: Vec<PendingBinding>,
34}
35
36impl Default for SemanticProgramBuilder {
37 fn default() -> Self {
38 Self::new()
39 }
40}
41
42impl SemanticProgramBuilder {
43 pub fn new() -> Self {
45 Self {
46 owner: ProgramBuilderNonce::fresh(),
47 inputs: Vec::new(),
48 input_specs: Vec::new(),
49 values: Vec::new(),
50 operations: Vec::new(),
51 bindings: Vec::new(),
52 }
53 }
54
55 pub fn bind_input(
64 &mut self,
65 input: ProgramValue,
66 tensor: Tensor,
67 ) -> Result<BindingKey, ProgramBuildError> {
68 self.validate_value(input)?;
69 if !self.inputs.contains(&input) {
70 return Err(ProgramBuildError::BindingTargetNotInput);
71 }
72 if self.bindings.iter().any(|binding| binding.input == input) {
73 return Err(ProgramBuildError::DuplicateBinding);
74 }
75 self.bind_input_retained(input, Arc::new(RetainedValue::from_tensor(tensor)))
76 }
77
78 #[doc(hidden)]
90 pub fn bind_input_retained(
91 &mut self,
92 input: ProgramValue,
93 tensor: Arc<RetainedValue>,
94 ) -> Result<BindingKey, ProgramBuildError> {
95 self.validate_value(input)?;
96 if !self.inputs.contains(&input) {
97 return Err(ProgramBuildError::BindingTargetNotInput);
98 }
99 if self.bindings.iter().any(|binding| binding.input == input) {
100 return Err(ProgramBuildError::DuplicateBinding);
101 }
102 let key = BindingKey::new(input.slot, self.owner);
103 self.bindings.push(PendingBinding { key, input, tensor });
104 Ok(key)
105 }
106
107 pub fn input(&mut self, spec: ProgramInputSpec) -> Result<ProgramValue, ProgramBuildError> {
114 require_scalar_identity(spec.metadata(), "a program input")?;
115 let slot = self.next_value_slot()?;
116 let value = ProgramValue::new(slot, self.owner);
117 self.values.push(spec.metadata().clone());
118 self.inputs.push(value);
119 self.input_specs.push(spec);
120 Ok(value)
121 }
122
123 pub fn validate_value(&self, value: ProgramValue) -> Result<(), ProgramBuildError> {
130 if value.owner != self.owner || value.slot as usize >= self.values.len() {
131 return Err(ProgramBuildError::ForeignValue);
132 }
133 Ok(())
134 }
135
136 pub fn value_metadata(
142 &self,
143 value: ProgramValue,
144 ) -> Result<&ProgramValueMetadata, ProgramBuildError> {
145 self.validate_value(value)?;
146 Ok(&self.values[value.slot as usize])
147 }
148
149 pub fn operation_count(&self) -> usize {
151 self.operations.len()
152 }
153
154 #[cfg(test)]
155 pub(crate) fn add_shape_guards_to_output(
156 &mut self,
157 output: ProgramValue,
158 guards: impl IntoIterator<Item = ShapeGuard>,
159 ) -> Result<(), ProgramBuildError> {
160 self.validate_value(output)?;
161 let operation = self
162 .operations
163 .iter_mut()
164 .find(|operation| operation.outputs.contains(&output))
165 .ok_or(ProgramBuildError::GuardTargetNotOperationOutput)?;
166 let mut combined = operation.shape_guards.to_vec();
167 combined.extend(guards);
168 operation.shape_guards = combined.into_boxed_slice();
169 Ok(())
170 }
171
172 pub fn import(
186 &mut self,
187 request: ProgramImport<'_>,
188 ) -> Result<ImportedProgramValues, ProgramBuildError> {
189 let transaction = ImportTransaction::prepare(self, request)?;
190 let roots = transaction.roots.clone();
191 self.inputs.extend(transaction.inputs);
192 self.input_specs.extend(transaction.input_specs);
193 self.values.extend(transaction.values);
194 self.operations.extend(transaction.operations);
195 self.bindings.extend(transaction.bindings);
196 Ok(ImportedProgramValues::new(roots))
197 }
198
199 pub(crate) fn snapshot_pruned(
212 &self,
213 outputs: &[ProgramValue],
214 ) -> Result<(FrozenProgram, Vec<ProgramValue>), ProgramFinishError> {
215 self.snapshot_pruned_with_aliases_and_specs(outputs, &[], &[])
216 }
217
218 pub(crate) fn snapshot_pruned_with_input_specs(
229 &self,
230 outputs: &[ProgramValue],
231 input_specs: &[(usize, ProgramInputSpec)],
232 ) -> Result<(FrozenProgram, Vec<ProgramValue>), ProgramFinishError> {
233 self.snapshot_pruned_with_aliases_and_specs(outputs, &[], input_specs)
234 }
235
236 pub(crate) fn snapshot_pruned_with_aliases(
247 &self,
248 outputs: &[ProgramValue],
249 aliases: &[crate::trace::CheckpointAlias],
250 ) -> Result<(FrozenProgram, Vec<ProgramValue>), ProgramFinishError> {
251 self.snapshot_pruned_with_aliases_and_specs(outputs, aliases, &[])
252 }
253
254 fn snapshot_pruned_with_aliases_and_specs(
255 &self,
256 outputs: &[ProgramValue],
257 aliases: &[crate::trace::CheckpointAlias],
258 input_specs: &[(usize, ProgramInputSpec)],
259 ) -> Result<(FrozenProgram, Vec<ProgramValue>), ProgramFinishError> {
260 let value_count = self.values.len();
261 let mut live = vec![false; value_count];
262 for output in outputs {
263 if output.owner != self.owner || output.slot as usize >= value_count {
264 return Err(ProgramFinishError::ForeignOutput);
265 }
266 live[output.slot as usize] = true;
267 }
268 let mut live_operation = vec![false; self.operations.len()];
272 for (index, operation) in self.operations.iter().enumerate() {
273 if operation.effects.is_empty() {
274 continue;
275 }
276 live_operation[index] = true;
277 for output in &operation.outputs {
278 live[output.slot as usize] = true;
279 }
280 }
281 for alias in aliases {
282 let slot = alias.input.slot as usize;
283 if live[slot] {
284 live[alias.target.slot as usize] = true;
285 }
286 }
287 let alias_of = |value: ProgramValue| {
288 aliases
289 .iter()
290 .find(|alias| alias.input == value)
291 .map(|alias| (alias.target, alias.bound_shape.clone()))
292 };
293 for _ in 0..=aliases.len() {
294 let mut changed = false;
295 for (index, operation) in self.operations.iter().enumerate().rev() {
296 if !live_operation[index]
297 && !operation
298 .outputs
299 .iter()
300 .any(|output| live[output.slot as usize])
301 {
302 continue;
303 }
304 live_operation[index] = true;
305 for input in &operation.inputs {
306 if !live[input.slot as usize] {
307 live[input.slot as usize] = true;
308 changed = true;
309 }
310 }
311 }
312 for alias in aliases {
313 if live[alias.input.slot as usize] && !live[alias.target.slot as usize] {
314 live[alias.target.slot as usize] = true;
315 changed = true;
316 }
317 }
318 if !changed {
319 break;
320 }
321 }
322 loop {
325 let mut changed = false;
326 for operation in self.operations.iter().rev() {
327 let SemanticOp::Extension(extension) = &operation.op else {
328 continue;
329 };
330 if !operation.effects.is_empty() {
331 continue;
332 }
333 let live_outputs: Vec<bool> = operation
334 .outputs
335 .iter()
336 .map(|output| live[output.slot as usize])
337 .collect();
338 if !live_outputs.iter().any(|live| *live) || live_outputs.iter().all(|live| *live) {
339 continue;
340 }
341 if extension.prune_outputs(&live_outputs).is_some() {
342 continue;
343 }
344 for output in &operation.outputs {
345 live[output.slot as usize] = true;
346 }
347 changed = true;
348 }
349 if !changed {
350 break;
351 }
352 }
353
354 let mut creation_order: Vec<(usize, Option<ProgramInputSpec>)> =
357 Vec::with_capacity(self.inputs.len());
358 for (source_index, declared) in input_specs {
359 if self.input_specs.get(*source_index).is_some() {
360 creation_order.push((*source_index, Some(declared.clone())));
361 }
362 }
363 for index in 0..self.inputs.len() {
364 if input_specs.iter().any(|(declared, _)| *declared == index) {
365 continue;
366 }
367 creation_order.push((index, None));
368 }
369
370 let mut input_index = std::collections::HashMap::new();
374 for (source_index, _) in &creation_order {
375 let Some(input) = self.inputs.get(*source_index) else {
376 continue;
377 };
378 if !live[input.slot as usize] || alias_of(*input).is_some() {
379 continue;
380 }
381 input_index.insert(*source_index, input_index.len());
382 }
383 fn remap_expr(
384 expr: &DimExpr,
385 input_index: &std::collections::HashMap<usize, usize>,
386 ) -> Option<DimExpr> {
387 Some(match expr {
388 DimExpr::Const(value) => DimExpr::Const(*value),
389 DimExpr::InputDim { input_idx, axis } => DimExpr::InputDim {
390 input_idx: *input_index.get(input_idx)?,
391 axis: *axis,
392 },
393 DimExpr::Add(lhs, rhs) => DimExpr::Add(
394 Box::new(remap_expr(lhs, input_index)?),
395 Box::new(remap_expr(rhs, input_index)?),
396 ),
397 DimExpr::Sub(lhs, rhs) => DimExpr::Sub(
398 Box::new(remap_expr(lhs, input_index)?),
399 Box::new(remap_expr(rhs, input_index)?),
400 ),
401 DimExpr::Mul(lhs, rhs) => DimExpr::Mul(
402 Box::new(remap_expr(lhs, input_index)?),
403 Box::new(remap_expr(rhs, input_index)?),
404 ),
405 DimExpr::FloorDiv(lhs, rhs) => DimExpr::FloorDiv(
406 Box::new(remap_expr(lhs, input_index)?),
407 Box::new(remap_expr(rhs, input_index)?),
408 ),
409 DimExpr::Min(lhs, rhs) => DimExpr::Min(
410 Box::new(remap_expr(lhs, input_index)?),
411 Box::new(remap_expr(rhs, input_index)?),
412 ),
413 DimExpr::Max(lhs, rhs) => DimExpr::Max(
414 Box::new(remap_expr(lhs, input_index)?),
415 Box::new(remap_expr(rhs, input_index)?),
416 ),
417 })
418 }
419
420 let mut rebuild = Self::new();
421 let mut kept_inputs = Vec::new();
422 let mut mapped: Vec<Option<ProgramValue>> = vec![None; value_count];
423 for (source_index, declared) in creation_order {
424 let (Some(input), Some(spec)) = (
425 self.inputs.get(source_index),
426 self.input_specs.get(source_index),
427 ) else {
428 continue;
429 };
430 if !live[input.slot as usize] || alias_of(*input).is_some() {
431 continue;
432 }
433 kept_inputs.push(*input);
434 let built = match declared {
435 Some(declared) => declared,
436 None => {
437 let metadata = spec.metadata();
438 let extents: Vec<ShapeExtent<DimExpr>> = metadata
439 .shape()
440 .iter()
441 .map(
442 |extent| -> Result<ShapeExtent<DimExpr>, ProgramFinishError> {
443 let invalid = || ProgramFinishError::SnapshotRebuild {
444 source: ProgramBuildError::InvalidImport {
445 source: ProgramStructuralError::InvalidValueReference,
446 },
447 };
448 Ok(match extent {
449 ShapeExtent::Exact(expr) => ShapeExtent::Exact(
450 remap_expr(expr, &input_index).ok_or_else(invalid)?,
451 ),
452 ShapeExtent::UpperBound(expr) => ShapeExtent::UpperBound(
453 remap_expr(expr, &input_index).ok_or_else(invalid)?,
454 ),
455 ShapeExtent::Unknown => ShapeExtent::Unknown,
456 })
457 },
458 )
459 .collect::<Result<_, _>>()?;
460 let mut built = ProgramInputSpec::from_metadata(
461 ProgramValueMetadata::from_extents(metadata.dtype(), extents),
462 );
463 if let Some(identity) = metadata.scalar_identity() {
464 built = built.with_scalar_identity(identity);
465 }
466 built
467 }
468 };
469 let rebuilt = rebuild.input(built).map_err(snapshot_rebuild)?;
470 if let Some(binding) = self.bindings.iter().find(|binding| binding.input == *input) {
471 rebuild
472 .bind_input_retained(rebuilt, Arc::clone(&binding.tensor))
473 .map_err(snapshot_rebuild)?;
474 }
475 mapped[input.slot as usize] = Some(rebuilt);
476 }
477
478 let materialize_alias = |rebuild: &mut Self,
479 mapped: &mut Vec<Option<ProgramValue>>,
480 input: ProgramValue|
481 -> Result<(), ProgramFinishError> {
482 if mapped[input.slot as usize].is_some() {
483 return Ok(());
484 }
485 let Some((target, shape)) = alias_of(input) else {
486 return Err(ProgramFinishError::SnapshotRebuild {
487 source: ProgramBuildError::InvalidImport {
488 source: ProgramStructuralError::InvalidValueReference,
489 },
490 });
491 };
492 let Some(mapped_target) = mapped[target.slot as usize] else {
493 return Err(ProgramFinishError::SnapshotRebuild {
494 source: ProgramBuildError::InvalidImport {
495 source: ProgramStructuralError::InvalidSsaOrder,
496 },
497 });
498 };
499 let rank = shape.len();
500 let slice = CoreSemanticOp::Slice(tenferro_tensor::SliceConfig {
501 starts: vec![0; rank],
502 limits: shape,
503 strides: vec![1; rank],
504 });
505 let sliced = rebuild
506 .add_op(slice, &[mapped_target])
507 .map_err(snapshot_rebuild)?[0];
508 mapped[input.slot as usize] = Some(sliced);
509 Ok(())
510 };
511
512 for (index, operation) in self.operations.iter().enumerate() {
513 if !live_operation[index] {
514 continue;
515 }
516 let live_outputs: Vec<bool> = operation
517 .outputs
518 .iter()
519 .map(|output| live[output.slot as usize])
520 .collect();
521 for input in &operation.inputs {
522 if alias_of(*input).is_some() {
523 materialize_alias(&mut rebuild, &mut mapped, *input)?;
524 }
525 }
526 let inputs = operation
527 .inputs
528 .iter()
529 .map(|input| {
530 mapped[input.slot as usize].ok_or(ProgramFinishError::SnapshotRebuild {
531 source: ProgramBuildError::InvalidImport {
532 source: ProgramStructuralError::InvalidSsaOrder,
533 },
534 })
535 })
536 .collect::<Result<Vec<_>, _>>()?;
537 let rebuilt_outputs = match &operation.op {
538 SemanticOp::Core(core) => rebuild.add_op(core.clone(), &inputs),
539 SemanticOp::Extension(extension) => rebuild.add_extension(
540 extension
541 .prune_outputs(&live_outputs)
542 .unwrap_or_else(|| extension.clone_arc()),
543 &inputs,
544 ),
545 }
546 .map_err(snapshot_rebuild)?;
547 discharge_static_guards(&mut rebuild);
548 let kept: Vec<ProgramValue> = operation
549 .outputs
550 .iter()
551 .zip(live_outputs.iter())
552 .filter(|(_, live)| **live)
553 .map(|(output, _)| *output)
554 .collect();
555 if rebuilt_outputs.len() != kept.len() {
556 return Err(ProgramFinishError::SnapshotRebuild {
557 source: ProgramBuildError::OutputMetadataCount {
558 expected: kept.len(),
559 actual: rebuilt_outputs.len(),
560 },
561 });
562 }
563 for (source, rebuilt) in kept.iter().zip(rebuilt_outputs.iter()) {
564 mapped[source.slot as usize] = Some(*rebuilt);
565 }
566 }
567
568 for output in outputs {
569 if alias_of(*output).is_some() {
570 materialize_alias(&mut rebuild, &mut mapped, *output)?;
571 }
572 }
573 let mapped_outputs = outputs
574 .iter()
575 .map(|output| {
576 mapped[output.slot as usize].ok_or(ProgramFinishError::SnapshotRebuild {
577 source: ProgramBuildError::InvalidImport {
578 source: ProgramStructuralError::InvalidValueReference,
579 },
580 })
581 })
582 .collect::<Result<Vec<_>, _>>()?;
583 Ok((rebuild.finish(&mapped_outputs)?, kept_inputs))
584 }
585
586 pub fn finish(self, outputs: &[ProgramValue]) -> Result<FrozenProgram, ProgramFinishError> {
595 if outputs
596 .iter()
597 .any(|output| output.owner != self.owner || output.slot as usize >= self.values.len())
598 {
599 return Err(ProgramFinishError::ForeignOutput);
600 }
601
602 validate_structure(
603 self.owner,
604 &self.inputs,
605 self.values.len(),
606 &self.operations,
607 )?;
608 validate_bindings(&self.inputs, &self.input_specs, &self.bindings)?;
609
610 let inputs = self.inputs.into_boxed_slice();
611 let outputs: Box<[ProgramValue]> = outputs.into();
612 let values = self.values.into_boxed_slice();
613 let operations = self.operations.into_boxed_slice();
614 let shape_guards: Box<[ShapeGuard]> = operations
615 .iter()
616 .flat_map(|operation| operation.shape_guards.iter().cloned())
617 .collect();
618 let identity =
619 SemanticIdentity::build(&inputs, &outputs, &values, &operations, &shape_guards);
620 let bindings = ProgramBindings::freeze(self.owner, self.bindings);
621 let program = SemanticProgram {
622 owner: self.owner,
623 inputs,
624 outputs,
625 values,
626 operations,
627 shape_guards,
628 identity,
629 };
630 Ok(FrozenProgram {
631 program: Arc::new(program),
632 bindings,
633 })
634 }
635
636 pub fn translate_input_shape_payload(
666 &self,
667 payload: &[DimExpr],
668 operation_inputs: &mut Vec<ProgramValue>,
669 ) -> Vec<DimExpr> {
670 payload
671 .iter()
672 .map(|extent| self.translate_input_shape_extent(extent, operation_inputs))
673 .collect()
674 }
675
676 fn translate_input_shape_extent(
677 &self,
678 extent: &DimExpr,
679 operation_inputs: &mut Vec<ProgramValue>,
680 ) -> DimExpr {
681 match extent {
682 DimExpr::Const(value) => DimExpr::Const(*value),
683 DimExpr::InputDim { input_idx, axis } => {
684 let Some(input) = self.inputs.get(*input_idx) else {
685 return extent.clone();
686 };
687 let positioned = match operation_inputs.iter().position(|value| value == input) {
688 Some(position) => position,
689 None => {
690 operation_inputs.push(*input);
691 operation_inputs.len() - 1
692 }
693 };
694 DimExpr::InputDim {
695 input_idx: positioned,
696 axis: *axis,
697 }
698 }
699 DimExpr::Add(lhs, rhs) => DimExpr::Add(
700 Box::new(self.translate_input_shape_extent(lhs, operation_inputs)),
701 Box::new(self.translate_input_shape_extent(rhs, operation_inputs)),
702 ),
703 DimExpr::Sub(lhs, rhs) => DimExpr::Sub(
704 Box::new(self.translate_input_shape_extent(lhs, operation_inputs)),
705 Box::new(self.translate_input_shape_extent(rhs, operation_inputs)),
706 ),
707 DimExpr::Mul(lhs, rhs) => DimExpr::Mul(
708 Box::new(self.translate_input_shape_extent(lhs, operation_inputs)),
709 Box::new(self.translate_input_shape_extent(rhs, operation_inputs)),
710 ),
711 DimExpr::FloorDiv(lhs, rhs) => DimExpr::FloorDiv(
712 Box::new(self.translate_input_shape_extent(lhs, operation_inputs)),
713 Box::new(self.translate_input_shape_extent(rhs, operation_inputs)),
714 ),
715 DimExpr::Min(lhs, rhs) => DimExpr::Min(
716 Box::new(self.translate_input_shape_extent(lhs, operation_inputs)),
717 Box::new(self.translate_input_shape_extent(rhs, operation_inputs)),
718 ),
719 DimExpr::Max(lhs, rhs) => DimExpr::Max(
720 Box::new(self.translate_input_shape_extent(lhs, operation_inputs)),
721 Box::new(self.translate_input_shape_extent(rhs, operation_inputs)),
722 ),
723 }
724 }
725
726 #[doc(hidden)]
728 pub fn inputs(&self) -> &[ProgramValue] {
729 &self.inputs
730 }
731
732 #[doc(hidden)]
738 pub fn resolved_extents(
739 &self,
740 value: ProgramValue,
741 ) -> Result<Vec<ShapeExtent<DimExpr>>, ProgramBuildError> {
742 self.value_metadata(value)?
743 .shape()
744 .iter()
745 .enumerate()
746 .map(|(axis, _)| self.resolved_extent(value, axis))
747 .collect()
748 }
749
750 #[doc(hidden)]
784 pub fn resolved_extent(
785 &self,
786 value: ProgramValue,
787 axis: usize,
788 ) -> Result<ShapeExtent<DimExpr>, ProgramBuildError> {
789 let metadata = self.value_metadata(value)?;
790 let extent =
791 metadata
792 .shape()
793 .get(axis)
794 .ok_or_else(|| ProgramBuildError::InvalidImport {
795 source: ProgramStructuralError::InvalidValueReference,
796 })?;
797 let resolve = |expr: &DimExpr| -> DimExpr {
800 if expr.max_input_idx().is_none() || self.bindings.is_empty() {
802 return expr.clone();
803 }
804 let mut resolved = expr.clone();
807 let mut positions = Vec::new();
808 collect_input_positions(expr, &mut positions);
809 positions.sort_unstable();
810 positions.dedup();
811 for position in positions {
812 let Some(input) = self.inputs.get(position) else {
813 continue;
814 };
815 let Some(bound) = self
816 .bindings
817 .iter()
818 .find(|binding| binding.input == *input)
819 .map(|binding| binding.tensor.shape())
820 else {
821 continue;
822 };
823 resolved = substitute_input_extents(&resolved, position, bound);
824 }
825 resolved
826 };
827 Ok(match extent {
828 ShapeExtent::Exact(expr) => ShapeExtent::Exact(resolve(expr)),
829 ShapeExtent::UpperBound(expr) => ShapeExtent::UpperBound(resolve(expr)),
830 ShapeExtent::Unknown => ShapeExtent::Unknown,
831 })
832 }
833
834 #[cfg(test)]
835 pub(crate) fn operation_views_for_test(
836 &self,
837 ) -> impl ExactSizeIterator<Item = super::SemanticOperationView<'_>> + '_ {
838 self.operations
839 .iter()
840 .map(super::SemanticOperationView::new)
841 }
842
843 pub fn add_op(
850 &mut self,
851 op: CoreSemanticOp,
852 inputs: &[ProgramValue],
853 ) -> Result<Box<[ProgramValue]>, ProgramBuildError> {
854 self.validate_inputs(inputs)?;
855 validate_arity(op.input_count(), inputs.len())?;
856 reject_core_external_dtype(&op)?;
857 let output_count = op.output_count();
858 let metadata = self.infer_core_metadata(&op, inputs)?;
859 validate_output_count(output_count, metadata.len())?;
860 let aliases = (0..output_count).map(Alias::fresh).collect();
861 self.append_operation(
862 SemanticOp::Core(op),
863 inputs,
864 metadata,
865 Vec::new(),
866 aliases,
867 Vec::new(),
868 )
869 }
870
871 pub fn add_extension(
937 &mut self,
938 op: Arc<dyn ExtensionOp>,
939 inputs: &[ProgramValue],
940 ) -> Result<Box<[ProgramValue]>, ProgramBuildError> {
941 self.validate_inputs(inputs)?;
942 validate_arity(op.input_count(), inputs.len())?;
943 let effects = extension_effects(op.as_ref())?;
944 let aliases = extension_aliases(op.as_ref())?;
945 validate_aliases(&aliases, inputs.len(), op.output_count())?;
946 let (metadata, guards) = self.infer_extension_metadata(op.as_ref(), inputs)?;
947 validate_output_count(op.output_count(), metadata.len())?;
948 self.append_operation(
949 SemanticOp::Extension(op),
950 inputs,
951 metadata,
952 effects,
953 aliases,
954 guards,
955 )
956 }
957
958 fn next_value_slot(&self) -> Result<u32, ProgramBuildError> {
959 u32::try_from(self.values.len()).map_err(|_| ProgramBuildError::TooManyValues)
960 }
961
962 fn validate_inputs(&self, inputs: &[ProgramValue]) -> Result<(), ProgramBuildError> {
963 inputs
964 .iter()
965 .try_for_each(|&value| self.validate_value(value))
966 }
967
968 fn input_metadata(
969 &self,
970 inputs: &[ProgramValue],
971 ) -> Result<Vec<&ProgramValueMetadata>, ProgramBuildError> {
972 inputs
973 .iter()
974 .map(|&value| self.value_metadata(value))
975 .collect()
976 }
977
978 fn infer_core_metadata(
979 &self,
980 op: &CoreSemanticOp,
981 inputs: &[ProgramValue],
982 ) -> Result<Vec<ProgramValueMetadata>, ProgramBuildError> {
983 let input_metadata = self.input_metadata(inputs)?;
984 let precision = crate::shape_infer::extent_precision(
985 input_metadata.iter().flat_map(|metadata| metadata.shape()),
986 );
987 let input_dtypes: Vec<_> = input_metadata
988 .iter()
989 .map(|metadata| metadata.dtype())
990 .collect();
991 let input_shapes = inference_shapes(&input_metadata);
992 let input_shape_refs: Vec<_> = input_shapes.iter().map(Vec::as_slice).collect();
993 let standard = tenferro_ops::std_tensor_op::StdTensorOp::from(op);
994 let dtype = crate::shape_infer::infer_output_dtype(&standard, &input_dtypes)
995 .map_err(metadata_error)?;
996 if core_output_uses_local_shape_coordinates(op) {
997 let local_input_shapes: Vec<_> = input_metadata
998 .iter()
999 .enumerate()
1000 .map(|(input_idx, metadata)| {
1001 DimExpr::input_shape(input_idx, metadata.shape().len())
1002 })
1003 .collect();
1004 let local_input_shape_refs: Vec<_> =
1005 local_input_shapes.iter().map(Vec::as_slice).collect();
1006 let output_extents =
1007 crate::shape_infer::infer_output_extents(&standard, &local_input_shape_refs)
1008 .map_err(metadata_error)?;
1009 output_extents
1010 .into_iter()
1011 .map(|shape| {
1012 resolve_inferred_extents(shape, precision, &input_shape_refs)
1013 .map(|shape| ProgramValueMetadata::from_extents(dtype, shape))
1014 })
1015 .collect()
1016 } else {
1017 let output_extents =
1018 crate::shape_infer::infer_output_extents(&standard, &input_shape_refs)
1019 .map_err(metadata_error)?;
1020 Ok(output_extents
1021 .into_iter()
1022 .map(|shape| {
1023 ProgramValueMetadata::from_extents(
1024 dtype,
1025 crate::shape_infer::withhold_unproved_exactness(shape, precision)
1026 .collect::<Vec<_>>(),
1027 )
1028 })
1029 .collect())
1030 }
1031 }
1032
1033 fn infer_extension_metadata(
1034 &self,
1035 op: &dyn ExtensionOp,
1036 inputs: &[ProgramValue],
1037 ) -> Result<(Vec<ProgramValueMetadata>, Vec<ShapeGuard>), ProgramBuildError> {
1038 let input_metadata = self.input_metadata(inputs)?;
1039 let precision = crate::shape_infer::extent_precision(
1040 input_metadata.iter().flat_map(|metadata| metadata.shape()),
1041 );
1042 let input_dtypes: Vec<_> = input_metadata
1043 .iter()
1044 .map(|metadata| metadata.dtype())
1045 .collect();
1046 let input_shapes = inference_shapes(&input_metadata);
1047 let input_shape_refs: Vec<_> = input_shapes.iter().map(Vec::as_slice).collect();
1048 let inferred = crate::shape_infer::infer_extension_output_meta_with_constraints(
1049 op,
1050 &input_dtypes,
1051 &input_shape_refs,
1052 )
1053 .map_err(metadata_error)?;
1054 let metadata = inferred
1055 .output_metas
1056 .into_iter()
1057 .map(|(dtype, shape)| {
1058 ProgramValueMetadata::from_extents(
1059 dtype,
1060 crate::shape_infer::withhold_unproved_exactness(
1061 shape.into_iter().map(ShapeExtent::Exact),
1062 precision,
1063 )
1064 .collect::<Vec<_>>(),
1065 )
1066 })
1067 .collect();
1068 let guards = inferred
1069 .constraints
1070 .into_iter()
1071 .map(|constraint| {
1072 let relation = match constraint.relation {
1073 tenferro_ops::ShapeRelation::Equal => ProgramShapeRelation::Equal,
1074 };
1075 ShapeGuard::new(relation, constraint.lhs, constraint.rhs)
1076 })
1077 .collect();
1078 Ok((metadata, guards))
1079 }
1080
1081 fn append_operation(
1082 &mut self,
1083 op: SemanticOp,
1084 inputs: &[ProgramValue],
1085 metadata: Vec<ProgramValueMetadata>,
1086 effects: Vec<Effect>,
1087 aliases: Vec<Alias>,
1088 shape_guards: Vec<ShapeGuard>,
1089 ) -> Result<Box<[ProgramValue]>, ProgramBuildError> {
1090 let identity = match &op {
1094 SemanticOp::Core(_) => None,
1095 SemanticOp::Extension(extension) => extension.scalar_identity(),
1096 };
1097 let mut metadata = metadata;
1098 for value in &mut metadata {
1099 if matches!(value.dtype(), tenferro_tensor::DType::External(_))
1100 && let Some(identity) = identity
1101 {
1102 *value = value.clone().with_scalar_identity(identity);
1103 }
1104 require_scalar_identity(value, "an operation output")?;
1105 }
1106 let provenance = match &op {
1107 SemanticOp::Core(_) => SemanticProvenance::builder(None),
1108 SemanticOp::Extension(extension) => {
1109 SemanticProvenance::builder(Some(extension.family_id()))
1110 }
1111 };
1112 let start = self.values.len();
1113 let end = start
1114 .checked_add(metadata.len())
1115 .ok_or(ProgramBuildError::TooManyValues)?;
1116 if end > u32::MAX as usize {
1117 return Err(ProgramBuildError::TooManyValues);
1118 }
1119 let outputs: Box<[_]> = (start..end)
1120 .map(|slot| ProgramValue::new(slot as u32, self.owner))
1121 .collect();
1122 self.values.extend(metadata);
1123 self.operations.push(SemanticOperation {
1124 op,
1125 inputs: inputs.into(),
1126 outputs: outputs.clone(),
1127 effects: effects.into(),
1128 aliases: aliases.into(),
1129 shape_guards: shape_guards.into(),
1130 placement: SemanticPlacementConstraint::any(),
1131 provenance,
1132 });
1133 Ok(outputs)
1134 }
1135}
1136
1137fn discharge_static_guards(builder: &mut SemanticProgramBuilder) {
1143 let Some(operation) = builder.operations.last_mut() else {
1144 return;
1145 };
1146 let kept: Vec<ShapeGuard> = operation
1147 .shape_guards
1148 .iter()
1149 .filter(|guard| {
1150 let lhs = static_dim_value(guard.lhs());
1151 let rhs = static_dim_value(guard.rhs());
1152 match (lhs, rhs) {
1153 (Some(lhs), Some(rhs)) => lhs != rhs,
1154 _ => true,
1155 }
1156 })
1157 .cloned()
1158 .collect();
1159 operation.shape_guards = kept.into_boxed_slice();
1160}
1161
1162fn static_dim_value(expr: &DimExpr) -> Option<usize> {
1163 Some(match expr {
1164 DimExpr::Const(value) => *value,
1165 DimExpr::InputDim { .. } => return None,
1166 DimExpr::Add(lhs, rhs) => static_dim_value(lhs)?.checked_add(static_dim_value(rhs)?)?,
1167 DimExpr::Sub(lhs, rhs) => static_dim_value(lhs)?.checked_sub(static_dim_value(rhs)?)?,
1168 DimExpr::Mul(lhs, rhs) => static_dim_value(lhs)?.checked_mul(static_dim_value(rhs)?)?,
1169 DimExpr::FloorDiv(lhs, rhs) => {
1170 static_dim_value(lhs)?.checked_div(static_dim_value(rhs)?)?
1171 }
1172 DimExpr::Min(lhs, rhs) => static_dim_value(lhs)?.min(static_dim_value(rhs)?),
1173 DimExpr::Max(lhs, rhs) => static_dim_value(lhs)?.max(static_dim_value(rhs)?),
1174 })
1175}
1176
1177fn snapshot_rebuild(source: ProgramBuildError) -> ProgramFinishError {
1178 ProgramFinishError::SnapshotRebuild { source }
1179}
1180
1181fn core_output_uses_local_shape_coordinates(op: &CoreSemanticOp) -> bool {
1182 matches!(
1183 op,
1184 CoreSemanticOp::Reshape { .. }
1185 | CoreSemanticOp::BroadcastInDim { .. }
1186 | CoreSemanticOp::GatherDynamicSliceSizes { .. }
1187 )
1188}
1189
1190pub(super) fn reject_core_external_dtype(op: &CoreSemanticOp) -> Result<(), ProgramBuildError> {
1196 let external = match op {
1197 CoreSemanticOp::Convert { from, to } => [Some(*from), Some(*to)]
1198 .into_iter()
1199 .flatten()
1200 .find(|dtype| matches!(dtype, tenferro_tensor::DType::External(_))),
1201 CoreSemanticOp::Constant { dtype, .. } => {
1202 matches!(dtype, tenferro_tensor::DType::External(_)).then_some(*dtype)
1203 }
1204 _ => None,
1205 };
1206 match external {
1207 Some(dtype) => Err(ProgramBuildError::ExternalScalarWithoutIdentity {
1208 dtype,
1209 site: "a core operation",
1210 }),
1211 None => Ok(()),
1212 }
1213}
1214
1215pub(super) fn require_scalar_identity(
1222 metadata: &ProgramValueMetadata,
1223 site: &'static str,
1224) -> Result<(), ProgramBuildError> {
1225 match (metadata.dtype(), metadata.scalar_identity()) {
1226 (tenferro_tensor::DType::External(_), None) => {
1227 Err(ProgramBuildError::ExternalScalarWithoutIdentity {
1228 dtype: metadata.dtype(),
1229 site,
1230 })
1231 }
1232 _ => Ok(()),
1233 }
1234}
1235
1236fn keep_scalar_identity(
1242 identity: &Option<&'static str>,
1243 metadata: ProgramValueMetadata,
1244) -> ProgramValueMetadata {
1245 match identity {
1246 Some(identity) => metadata.with_scalar_identity(identity),
1247 None => metadata,
1248 }
1249}
1250
1251fn validate_arity(expected: usize, actual: usize) -> Result<(), ProgramBuildError> {
1252 if expected == actual {
1253 Ok(())
1254 } else {
1255 Err(ProgramBuildError::Arity { expected, actual })
1256 }
1257}
1258
1259fn validate_output_count(expected: usize, actual: usize) -> Result<(), ProgramBuildError> {
1260 if expected == actual {
1261 Ok(())
1262 } else {
1263 Err(ProgramBuildError::OutputMetadataCount { expected, actual })
1264 }
1265}
1266
1267impl FrozenProgram {
1268 #[doc(hidden)]
1274 pub fn input_metadata_with_bound_shapes(&self) -> Box<[ProgramValueMetadata]> {
1275 let bound_input_shapes: Vec<Option<Vec<DimExpr>>> = self
1276 .program
1277 .inputs
1278 .iter()
1279 .map(|input| {
1280 self.bindings.tensor_for_input(*input).map(|tensor| {
1281 tensor
1282 .shape()
1283 .iter()
1284 .map(|&size| DimExpr::Const(size))
1285 .collect()
1286 })
1287 })
1288 .collect();
1289
1290 self.program
1291 .inputs
1292 .iter()
1293 .map(|input| {
1294 let metadata = self.program.values[input.slot as usize].clone();
1295 ProgramValueMetadata::from_extents(
1296 metadata.dtype(),
1297 metadata.shape().iter().map(|extent| match extent {
1298 ShapeExtent::Exact(expr) => ShapeExtent::Exact(
1299 resolve_dim_expr_from_input_shapes(expr, &bound_input_shapes),
1300 ),
1301 ShapeExtent::UpperBound(expr) => ShapeExtent::UpperBound(
1302 resolve_dim_expr_from_input_shapes(expr, &bound_input_shapes),
1303 ),
1304 ShapeExtent::Unknown => ShapeExtent::Unknown,
1305 }),
1306 )
1307 })
1308 .collect()
1309 }
1310
1311 #[doc(hidden)]
1320 pub fn with_input_prefix_bindings_from(
1321 &self,
1322 source: &FrozenProgram,
1323 ) -> Result<FrozenProgram, ProgramFinishError> {
1324 if self.program.inputs.len() < source.program.inputs.len() {
1325 return Err(ProgramFinishError::StructuralValidation {
1326 source: ProgramStructuralError::InvalidValueReference,
1327 });
1328 }
1329
1330 let mut bindings = Vec::new();
1331 for (source_input, destination_input) in
1332 source.program.inputs.iter().zip(self.program.inputs.iter())
1333 {
1334 if let Some(tensor) = source.bindings.tensor_for_input(*source_input) {
1335 bindings.push(PendingBinding {
1336 key: BindingKey::new(destination_input.slot, self.program.owner),
1337 input: *destination_input,
1338 tensor,
1339 });
1340 }
1341 }
1342
1343 let input_specs: Vec<_> = self
1344 .program
1345 .inputs
1346 .iter()
1347 .map(|input| {
1348 ProgramInputSpec::from_metadata(self.program.values[input.slot as usize].clone())
1349 })
1350 .collect();
1351 validate_bindings(&self.program.inputs, &input_specs, &bindings)?;
1352
1353 Ok(FrozenProgram {
1354 program: Arc::clone(&self.program),
1355 bindings: ProgramBindings::freeze(self.program.owner, bindings),
1356 })
1357 }
1358}
1359
1360struct ImportTransaction {
1361 inputs: Vec<ProgramValue>,
1362 input_specs: Vec<ProgramInputSpec>,
1363 values: Vec<ProgramValueMetadata>,
1364 operations: Vec<SemanticOperation>,
1365 bindings: Vec<PendingBinding>,
1366 roots: Box<[ProgramValue]>,
1367}
1368
1369impl ImportTransaction {
1370 fn prepare(
1371 destination: &SemanticProgramBuilder,
1372 request: ProgramImport<'_>,
1373 ) -> Result<Self, ProgramBuildError> {
1374 let source = request.program;
1375 if let Some(bindings) = request.bindings
1376 && !bindings.belongs_to(source.owner)
1377 {
1378 return Err(ProgramBuildError::ForeignBindings);
1379 }
1380 for (mapped_source, mapped_destination) in request.input_map {
1381 if mapped_source.owner != source.owner
1382 || mapped_source.slot as usize >= source.values.len()
1383 {
1384 return Err(ProgramBuildError::ForeignValue);
1385 }
1386 destination.validate_value(*mapped_destination)?;
1387 }
1388 if request
1389 .roots
1390 .iter()
1391 .any(|root| root.owner != source.owner || root.slot as usize >= source.values.len())
1392 {
1393 return Err(ProgramBuildError::ForeignImportRoot);
1394 }
1395
1396 let mut producer = vec![None; source.values.len()];
1397 for (operation_index, operation) in source.operations.iter().enumerate() {
1398 for output in &operation.outputs {
1399 producer[output.slot as usize] = Some(operation_index);
1400 }
1401 }
1402
1403 let mut needed_values = vec![false; source.values.len()];
1404 let mut needed_operations = vec![false; source.operations.len()];
1405 let mut pending: Vec<_> = request
1406 .roots
1407 .iter()
1408 .map(|root| root.slot as usize)
1409 .collect();
1410 if let Some(bindings) = request.bindings {
1411 pending.extend(bindings.bound_inputs().map(|input| input.slot as usize));
1412 }
1413 for (operation_index, operation) in source.operations.iter().enumerate() {
1414 if !operation.effects.is_empty() {
1415 needed_operations[operation_index] = true;
1416 for output in &operation.outputs {
1417 needed_values[output.slot as usize] = true;
1418 }
1419 pending.extend(operation.inputs.iter().map(|input| input.slot as usize));
1420 }
1421 }
1422 while let Some(slot) = pending.pop() {
1423 if needed_values[slot] {
1424 continue;
1425 }
1426 needed_values[slot] = true;
1427 if let Some(operation_index) = producer[slot]
1428 && !needed_operations[operation_index]
1429 {
1430 needed_operations[operation_index] = true;
1431 let operation = &source.operations[operation_index];
1432 for output in &operation.outputs {
1433 needed_values[output.slot as usize] = true;
1434 }
1435 pending.extend(operation.inputs.iter().map(|input| input.slot as usize));
1436 }
1437 }
1438
1439 loop {
1443 let mut positions = Vec::new();
1444 for input in &source.inputs {
1445 if !needed_values[input.slot as usize] {
1446 continue;
1447 }
1448 for extent in source.values[input.slot as usize].shape() {
1449 if let ShapeExtent::Exact(expr) | ShapeExtent::UpperBound(expr) = extent {
1450 collect_input_positions(expr, &mut positions);
1451 }
1452 }
1453 }
1454 for (operation, needed) in source.operations.iter().zip(&needed_operations) {
1455 if !needed {
1456 continue;
1457 }
1458 for guard in operation.shape_guards.iter() {
1459 collect_input_positions(guard.lhs(), &mut positions);
1460 collect_input_positions(guard.rhs(), &mut positions);
1461 }
1462 }
1463 let mut changed = false;
1464 for position in positions {
1465 let Some(input) = source.inputs.get(position) else {
1466 continue;
1467 };
1468 if !needed_values[input.slot as usize] {
1469 needed_values[input.slot as usize] = true;
1470 changed = true;
1471 }
1472 }
1473 if !changed {
1474 break;
1475 }
1476 }
1477
1478 let imported_input_count = source
1479 .inputs
1480 .iter()
1481 .filter(|input| needed_values[input.slot as usize])
1482 .count();
1483 let imported_output_count: usize = source
1484 .operations
1485 .iter()
1486 .zip(&needed_operations)
1487 .filter(|(_, needed)| **needed)
1488 .map(|(operation, _)| operation.outputs.len())
1489 .sum();
1490 let imported_value_count = imported_input_count
1491 .checked_add(imported_output_count)
1492 .ok_or(ProgramBuildError::TooManyValues)?;
1493 let final_value_count = destination
1494 .values
1495 .len()
1496 .checked_add(imported_value_count)
1497 .ok_or(ProgramBuildError::TooManyValues)?;
1498 if final_value_count > u32::MAX as usize {
1499 return Err(ProgramBuildError::TooManyValues);
1500 }
1501
1502 let mut transaction = Self {
1503 inputs: Vec::with_capacity(imported_input_count),
1504 input_specs: Vec::with_capacity(imported_input_count),
1505 values: Vec::with_capacity(imported_value_count),
1506 operations: Vec::with_capacity(
1507 needed_operations.iter().filter(|needed| **needed).count(),
1508 ),
1509 bindings: Vec::new(),
1510 roots: Box::new([]),
1511 };
1512 let mapped_input: std::collections::HashMap<ProgramValue, ProgramValue> =
1517 request.input_map.iter().copied().collect();
1518
1519 let mut remap = vec![None; source.values.len()];
1520 let mut source_input_index = vec![None; source.inputs.len()];
1524
1525 let mut reserved = Vec::new();
1529 for (position, &input) in source.inputs.iter().enumerate() {
1530 if !needed_values[input.slot as usize] {
1531 continue;
1532 }
1533 if let Some(&existing) = mapped_input.get(&input) {
1534 remap[input.slot as usize] = Some(existing);
1535 continue;
1536 }
1537 let slot = destination
1538 .values
1539 .len()
1540 .checked_add(transaction.values.len())
1541 .and_then(|slot| slot.checked_add(reserved.len()))
1542 .ok_or(ProgramBuildError::TooManyValues)?;
1543 let slot = u32::try_from(slot).map_err(|_| ProgramBuildError::TooManyValues)?;
1544 let imported = ProgramValue::new(slot, destination.owner);
1545 source_input_index[position] = Some(destination.inputs.len() + reserved.len());
1546 let tensor = request
1547 .bindings
1548 .and_then(|bindings| bindings.tensor_for_input(input));
1549 reserved.push((imported, &source.values[input.slot as usize], tensor));
1550 remap[input.slot as usize] = Some(imported);
1551 }
1552
1553 for (imported, source_metadata, tensor) in reserved {
1554 let identity = source_metadata.scalar_identity();
1555 let metadata = ProgramValueMetadata::from_extents(
1556 source_metadata.dtype(),
1557 source_metadata
1558 .shape()
1559 .iter()
1560 .cloned()
1561 .map(|extent| {
1562 remap_imported_extent(
1563 extent,
1564 source,
1565 &source_input_index,
1566 &mapped_input,
1567 destination,
1568 )
1569 })
1570 .collect::<Vec<_>>(),
1571 );
1572 let metadata = keep_scalar_identity(&identity, metadata);
1573 transaction.inputs.push(imported);
1574 transaction
1575 .input_specs
1576 .push(ProgramInputSpec::from_metadata(metadata.clone()));
1577 transaction.values.push(metadata);
1578 if let Some(tensor) = tensor {
1579 transaction.bindings.push(PendingBinding {
1580 key: BindingKey::new(imported.slot, destination.owner),
1581 input: imported,
1582 tensor,
1583 });
1584 }
1585 }
1586
1587 for (operation, needed) in source.operations.iter().zip(needed_operations) {
1588 if !needed {
1589 continue;
1590 }
1591 let inputs: Box<[_]> = operation
1592 .inputs
1593 .iter()
1594 .map(|input| {
1595 remap[input.slot as usize].ok_or(ProgramBuildError::InvalidImport {
1596 source: ProgramStructuralError::InvalidSsaOrder,
1597 })
1598 })
1599 .collect::<Result<_, _>>()?;
1600 let mut outputs = Vec::with_capacity(operation.outputs.len());
1601 for output in &operation.outputs {
1602 let imported =
1603 transaction.next_value(destination.values.len(), destination.owner)?;
1604 let meta = source.values[output.slot as usize].clone();
1605 let identity = meta.scalar_identity();
1606 let resolved = ProgramValueMetadata::from_extents(
1607 meta.dtype(),
1608 meta.shape()
1609 .iter()
1610 .cloned()
1611 .map(|extent| {
1612 remap_imported_extent(
1613 extent,
1614 source,
1615 &source_input_index,
1616 &mapped_input,
1617 destination,
1618 )
1619 })
1620 .collect::<Vec<_>>(),
1621 );
1622 transaction
1623 .values
1624 .push(keep_scalar_identity(&identity, resolved));
1625 remap[output.slot as usize] = Some(imported);
1626 outputs.push(imported);
1627 }
1628 let op = match &operation.op {
1629 SemanticOp::Core(op) => SemanticOp::Core(op.clone()),
1630 SemanticOp::Extension(op) => SemanticOp::Extension(op.clone_arc()),
1631 };
1632 let guards = operation
1633 .shape_guards
1634 .iter()
1635 .filter_map(|guard| {
1636 let resolve = |expr: &DimExpr| -> Option<DimExpr> {
1637 remap_imported_dim_expr(
1638 expr,
1639 source,
1640 &source_input_index,
1641 &mapped_input,
1642 destination,
1643 )
1644 };
1645 Some(ShapeGuard::new(
1646 guard.relation(),
1647 resolve(guard.lhs())?,
1648 resolve(guard.rhs())?,
1649 ))
1650 })
1651 .collect::<Vec<_>>();
1652 transaction.operations.push(SemanticOperation {
1653 op,
1654 inputs,
1655 outputs: outputs.into(),
1656 effects: operation.effects.clone(),
1657 aliases: operation.aliases.clone(),
1658 shape_guards: guards.into_boxed_slice(),
1659 placement: operation.placement,
1660 provenance: operation.provenance.clone(),
1661 });
1662 }
1663
1664 transaction.roots = request
1665 .roots
1666 .iter()
1667 .map(|root| {
1668 remap[root.slot as usize].ok_or(ProgramBuildError::InvalidImport {
1669 source: ProgramStructuralError::InvalidValueReference,
1670 })
1671 })
1672 .collect::<Result<_, _>>()?;
1673 Ok(transaction)
1674 }
1675
1676 fn next_value(
1677 &self,
1678 destination_value_count: usize,
1679 owner: ProgramBuilderNonce,
1680 ) -> Result<ProgramValue, ProgramBuildError> {
1681 let slot = destination_value_count
1682 .checked_add(self.values.len())
1683 .ok_or(ProgramBuildError::TooManyValues)?;
1684 let slot = u32::try_from(slot).map_err(|_| ProgramBuildError::TooManyValues)?;
1685 Ok(ProgramValue::new(slot, owner))
1686 }
1687}
1688
1689fn substitute_input_extents(expr: &DimExpr, position: usize, bound: &[usize]) -> DimExpr {
1695 match expr {
1696 DimExpr::Const(value) => DimExpr::Const(*value),
1697 DimExpr::InputDim { input_idx, axis } if *input_idx == position => bound
1698 .get(*axis)
1699 .map_or_else(|| expr.clone(), |size| DimExpr::Const(*size)),
1700 DimExpr::InputDim { .. } => expr.clone(),
1701 DimExpr::Add(lhs, rhs) => DimExpr::Add(
1702 Box::new(substitute_input_extents(lhs, position, bound)),
1703 Box::new(substitute_input_extents(rhs, position, bound)),
1704 ),
1705 DimExpr::Sub(lhs, rhs) => DimExpr::Sub(
1706 Box::new(substitute_input_extents(lhs, position, bound)),
1707 Box::new(substitute_input_extents(rhs, position, bound)),
1708 ),
1709 DimExpr::Mul(lhs, rhs) => DimExpr::Mul(
1710 Box::new(substitute_input_extents(lhs, position, bound)),
1711 Box::new(substitute_input_extents(rhs, position, bound)),
1712 ),
1713 DimExpr::FloorDiv(lhs, rhs) => DimExpr::FloorDiv(
1714 Box::new(substitute_input_extents(lhs, position, bound)),
1715 Box::new(substitute_input_extents(rhs, position, bound)),
1716 ),
1717 DimExpr::Min(lhs, rhs) => DimExpr::Min(
1718 Box::new(substitute_input_extents(lhs, position, bound)),
1719 Box::new(substitute_input_extents(rhs, position, bound)),
1720 ),
1721 DimExpr::Max(lhs, rhs) => DimExpr::Max(
1722 Box::new(substitute_input_extents(lhs, position, bound)),
1723 Box::new(substitute_input_extents(rhs, position, bound)),
1724 ),
1725 }
1726}
1727
1728fn collect_input_positions(expr: &DimExpr, positions: &mut Vec<usize>) {
1730 match expr {
1731 DimExpr::Const(_) => {}
1732 DimExpr::InputDim { input_idx, .. } => positions.push(*input_idx),
1733 DimExpr::Add(lhs, rhs)
1734 | DimExpr::Sub(lhs, rhs)
1735 | DimExpr::Mul(lhs, rhs)
1736 | DimExpr::FloorDiv(lhs, rhs)
1737 | DimExpr::Min(lhs, rhs)
1738 | DimExpr::Max(lhs, rhs) => {
1739 collect_input_positions(lhs, positions);
1740 collect_input_positions(rhs, positions);
1741 }
1742 }
1743}
1744
1745fn remap_imported_extent(
1750 extent: ShapeExtent<DimExpr>,
1751 source: &SemanticProgram,
1752 source_input_index: &[Option<usize>],
1753 mapped_input: &std::collections::HashMap<ProgramValue, ProgramValue>,
1754 destination: &SemanticProgramBuilder,
1755) -> ShapeExtent<DimExpr> {
1756 match extent {
1757 ShapeExtent::Exact(expr) => {
1758 remap_imported_dim_expr(&expr, source, source_input_index, mapped_input, destination)
1759 .map(ShapeExtent::Exact)
1760 .unwrap_or(ShapeExtent::Unknown)
1761 }
1762 ShapeExtent::UpperBound(expr) => {
1763 remap_imported_dim_expr(&expr, source, source_input_index, mapped_input, destination)
1764 .map(ShapeExtent::UpperBound)
1765 .unwrap_or(ShapeExtent::Unknown)
1766 }
1767 ShapeExtent::Unknown => ShapeExtent::Unknown,
1768 }
1769}
1770
1771fn remap_imported_dim_expr(
1778 expr: &DimExpr,
1779 source: &SemanticProgram,
1780 source_input_index: &[Option<usize>],
1781 mapped_input: &std::collections::HashMap<ProgramValue, ProgramValue>,
1782 destination: &SemanticProgramBuilder,
1783) -> Option<DimExpr> {
1784 match expr {
1785 DimExpr::Const(value) => Some(DimExpr::Const(*value)),
1786 DimExpr::InputDim { input_idx, axis } => {
1787 if let Some(index) = source_input_index.get(*input_idx).copied().flatten() {
1788 return Some(DimExpr::InputDim {
1789 input_idx: index,
1790 axis: *axis,
1791 });
1792 }
1793 let source_input = source.inputs.get(*input_idx)?;
1794 let destination_value = mapped_input.get(source_input)?;
1795 match destination
1796 .values
1797 .get(destination_value.slot as usize)?
1798 .shape()
1799 .get(*axis)?
1800 {
1801 ShapeExtent::Exact(expr) => Some(expr.clone()),
1802 _ => None,
1803 }
1804 }
1805 DimExpr::Add(lhs, rhs) => Some(DimExpr::Add(
1806 Box::new(remap_imported_dim_expr(
1807 lhs,
1808 source,
1809 source_input_index,
1810 mapped_input,
1811 destination,
1812 )?),
1813 Box::new(remap_imported_dim_expr(
1814 rhs,
1815 source,
1816 source_input_index,
1817 mapped_input,
1818 destination,
1819 )?),
1820 )),
1821 DimExpr::Sub(lhs, rhs) => Some(DimExpr::Sub(
1822 Box::new(remap_imported_dim_expr(
1823 lhs,
1824 source,
1825 source_input_index,
1826 mapped_input,
1827 destination,
1828 )?),
1829 Box::new(remap_imported_dim_expr(
1830 rhs,
1831 source,
1832 source_input_index,
1833 mapped_input,
1834 destination,
1835 )?),
1836 )),
1837 DimExpr::Mul(lhs, rhs) => Some(DimExpr::Mul(
1838 Box::new(remap_imported_dim_expr(
1839 lhs,
1840 source,
1841 source_input_index,
1842 mapped_input,
1843 destination,
1844 )?),
1845 Box::new(remap_imported_dim_expr(
1846 rhs,
1847 source,
1848 source_input_index,
1849 mapped_input,
1850 destination,
1851 )?),
1852 )),
1853 DimExpr::FloorDiv(lhs, rhs) => Some(DimExpr::FloorDiv(
1854 Box::new(remap_imported_dim_expr(
1855 lhs,
1856 source,
1857 source_input_index,
1858 mapped_input,
1859 destination,
1860 )?),
1861 Box::new(remap_imported_dim_expr(
1862 rhs,
1863 source,
1864 source_input_index,
1865 mapped_input,
1866 destination,
1867 )?),
1868 )),
1869 DimExpr::Min(lhs, rhs) => Some(DimExpr::Min(
1870 Box::new(remap_imported_dim_expr(
1871 lhs,
1872 source,
1873 source_input_index,
1874 mapped_input,
1875 destination,
1876 )?),
1877 Box::new(remap_imported_dim_expr(
1878 rhs,
1879 source,
1880 source_input_index,
1881 mapped_input,
1882 destination,
1883 )?),
1884 )),
1885 DimExpr::Max(lhs, rhs) => Some(DimExpr::Max(
1886 Box::new(remap_imported_dim_expr(
1887 lhs,
1888 source,
1889 source_input_index,
1890 mapped_input,
1891 destination,
1892 )?),
1893 Box::new(remap_imported_dim_expr(
1894 rhs,
1895 source,
1896 source_input_index,
1897 mapped_input,
1898 destination,
1899 )?),
1900 )),
1901 }
1902}
1903
1904fn resolve_inferred_extents(
1905 extents: impl IntoIterator<Item = ShapeExtent<DimExpr>>,
1906 precision: crate::shape_infer::ExtentPrecision,
1907 input_shapes: &[&[DimExpr]],
1908) -> Result<Vec<ShapeExtent<DimExpr>>, ProgramBuildError> {
1909 extents
1910 .into_iter()
1911 .map(|extent| {
1912 if matches!(precision, crate::shape_infer::ExtentPrecision::Unknown) {
1913 return Ok(ShapeExtent::Unknown);
1914 }
1915 let resolved = match extent {
1916 ShapeExtent::Exact(expression) => ShapeExtent::Exact(
1917 crate::shape_infer::resolve_dim_expr_from_shapes(&expression, input_shapes)
1918 .map_err(metadata_error)?,
1919 ),
1920 ShapeExtent::UpperBound(expression) => ShapeExtent::UpperBound(
1921 crate::shape_infer::resolve_dim_expr_from_shapes(&expression, input_shapes)
1922 .map_err(metadata_error)?,
1923 ),
1924 ShapeExtent::Unknown => ShapeExtent::Unknown,
1925 };
1926 Ok(match precision {
1927 crate::shape_infer::ExtentPrecision::Exact => resolved,
1928 crate::shape_infer::ExtentPrecision::Bounded => match resolved {
1929 ShapeExtent::Exact(expression) | ShapeExtent::UpperBound(expression) => {
1930 ShapeExtent::UpperBound(expression)
1931 }
1932 ShapeExtent::Unknown => ShapeExtent::Unknown,
1933 },
1934 crate::shape_infer::ExtentPrecision::Unknown => unreachable!("handled above"),
1935 })
1936 })
1937 .collect()
1938}
1939
1940fn inference_shapes(metadata: &[&ProgramValueMetadata]) -> Vec<Vec<DimExpr>> {
1941 metadata
1942 .iter()
1943 .enumerate()
1944 .map(|(input_idx, metadata)| {
1945 metadata
1946 .shape()
1947 .iter()
1948 .enumerate()
1949 .map(|(axis, extent)| match extent {
1950 ShapeExtent::Exact(expression) | ShapeExtent::UpperBound(expression) => {
1951 expression.clone()
1952 }
1953 ShapeExtent::Unknown => DimExpr::InputDim { input_idx, axis },
1954 })
1955 .collect()
1956 })
1957 .collect()
1958}
1959
1960fn metadata_error(source: crate::Error) -> ProgramBuildError {
1961 ProgramBuildError::Metadata {
1962 source: Box::new(source),
1963 }
1964}
1965
1966fn extension_effects(op: &dyn ExtensionOp) -> Result<Vec<Effect>, ProgramBuildError> {
1967 let family = op.family_id();
1968 let effects = match op.semantic_effects() {
1969 ExtensionEffectDeclaration::Undeclared => {
1970 return Err(ProgramBuildError::UndeclaredExtensionEffects { family })
1971 }
1972 ExtensionEffectDeclaration::Declared(effects) => effects,
1973 };
1974 effects
1975 .iter()
1976 .map(|effect| {
1977 let resource = EffectResource::new(effect.family, effect.key)
1978 .map_err(|source| ProgramBuildError::InvalidEffectResource { family, source })?;
1979 let access = match effect.access {
1980 ExtensionEffectAccess::Read => EffectAccess::Read,
1981 ExtensionEffectAccess::Write => EffectAccess::Write,
1982 };
1983 Ok(Effect::new(resource, access))
1984 })
1985 .collect()
1986}
1987
1988fn extension_aliases(op: &dyn ExtensionOp) -> Result<Vec<Alias>, ProgramBuildError> {
1989 let family = op.family_id();
1990 match op.semantic_aliases() {
1991 ExtensionAliasDeclaration::Undeclared => {
1992 Err(ProgramBuildError::UndeclaredExtensionAliases { family })
1993 }
1994 ExtensionAliasDeclaration::AllFresh => {
1995 Ok((0..op.output_count()).map(Alias::fresh).collect())
1996 }
1997 ExtensionAliasDeclaration::Declared(aliases) => aliases
1998 .iter()
1999 .map(|alias| match *alias {
2000 ExtensionAlias::Fresh { output } => Ok(Alias::fresh(output)),
2001 ExtensionAlias::ViewOf { output, input } => Ok(Alias::view_of(output, input)),
2002 ExtensionAlias::MustAlias { output, input } => Ok(Alias::must_alias(output, input)),
2003 ExtensionAlias::ExternalAlias {
2004 output,
2005 family: resource_family,
2006 key,
2007 } => EffectResource::new(resource_family, key)
2008 .map(|resource| Alias::external(output, resource))
2009 .map_err(|source| ProgramBuildError::InvalidEffectResource { family, source }),
2010 })
2011 .collect(),
2012 }
2013}
2014
2015fn validate_aliases(
2016 aliases: &[Alias],
2017 input_count: usize,
2018 output_count: usize,
2019) -> Result<(), ProgramBuildError> {
2020 let mut seen = vec![false; output_count];
2021 for &alias in aliases {
2022 let output = alias.output();
2023 let input = alias.input();
2024 if output >= output_count || input.is_some_and(|input| input >= input_count) {
2025 return Err(ProgramBuildError::AliasOutOfBounds {
2026 output,
2027 output_count,
2028 input,
2029 input_count,
2030 });
2031 }
2032 if seen[output] {
2033 return Err(ProgramBuildError::AliasCoverage {
2034 expected: output_count,
2035 actual: seen.iter().filter(|&&present| present).count(),
2036 });
2037 }
2038 seen[output] = true;
2039 }
2040 let actual = seen.iter().filter(|&&present| present).count();
2041 if actual != output_count {
2042 return Err(ProgramBuildError::AliasCoverage {
2043 expected: output_count,
2044 actual,
2045 });
2046 }
2047 Ok(())
2048}
2049
2050fn validate_structure(
2051 owner: ProgramBuilderNonce,
2052 inputs: &[ProgramValue],
2053 value_count: usize,
2054 operations: &[SemanticOperation],
2055) -> Result<(), ProgramFinishError> {
2056 let mut covered = vec![false; value_count];
2057 for input in inputs {
2058 if input.owner != owner
2059 || input.slot as usize >= value_count
2060 || covered[input.slot as usize]
2061 {
2062 return Err(ProgramFinishError::StructuralValidation {
2063 source: ProgramStructuralError::InvalidValueReference,
2064 });
2065 }
2066 covered[input.slot as usize] = true;
2067 }
2068 let mut previous_output = None;
2069 for operation in operations {
2070 let Some(first_output) = operation.outputs.first() else {
2071 if operation.inputs.iter().any(|value| {
2072 value.owner != owner
2073 || value.slot as usize >= value_count
2074 || !covered[value.slot as usize]
2075 }) {
2076 return Err(ProgramFinishError::StructuralValidation {
2077 source: ProgramStructuralError::InvalidValueReference,
2078 });
2079 }
2080 continue;
2081 };
2082 let output_start = first_output.slot as usize;
2083 let valid_input = operation.inputs.iter().all(|value| {
2084 value.owner == owner
2085 && (value.slot as usize) < output_start
2086 && (value.slot as usize) < value_count
2087 && covered[value.slot as usize]
2088 });
2089 let valid_output = operation.outputs.iter().enumerate().all(|(offset, value)| {
2090 value.owner == owner
2091 && value.slot as usize == output_start + offset
2092 && (value.slot as usize) < value_count
2093 && !covered[value.slot as usize]
2094 });
2095 let ordered = previous_output.is_none_or(|previous| output_start > previous);
2096 if !valid_input || !valid_output || !ordered {
2097 let source = if operation
2098 .inputs
2099 .iter()
2100 .chain(operation.outputs.iter())
2101 .any(|value| value.owner != owner || value.slot as usize >= value_count)
2102 {
2103 ProgramStructuralError::InvalidValueReference
2104 } else {
2105 ProgramStructuralError::InvalidSsaOrder
2106 };
2107 return Err(ProgramFinishError::StructuralValidation { source });
2108 }
2109 for output in &operation.outputs {
2110 covered[output.slot as usize] = true;
2111 }
2112 previous_output = operation.outputs.last().map(|value| value.slot as usize);
2113 }
2114 if covered.iter().any(|covered| !covered) {
2115 return Err(ProgramFinishError::StructuralValidation {
2116 source: ProgramStructuralError::InvalidSsaOrder,
2117 });
2118 }
2119 Ok(())
2120}
2121
2122fn validate_bindings(
2123 inputs: &[ProgramValue],
2124 input_specs: &[ProgramInputSpec],
2125 bindings: &[PendingBinding],
2126) -> Result<(), ProgramFinishError> {
2127 for binding in bindings {
2128 let input_index = inputs
2129 .iter()
2130 .position(|input| *input == binding.input)
2131 .ok_or(ProgramFinishError::BindingFinalization {
2132 source: ProgramBindingError::InvalidTarget,
2133 })?;
2134 let spec = &input_specs[input_index];
2135 let metadata = spec.metadata();
2136 let actual_dtype = binding.tensor.dtype();
2137 if actual_dtype != metadata.dtype() {
2138 return Err(ProgramFinishError::BindingFinalization {
2139 source: ProgramBindingError::DTypeMismatch {
2140 expected: metadata.dtype(),
2141 actual: actual_dtype,
2142 },
2143 });
2144 }
2145 let actual_shape = binding.tensor.shape();
2146 if actual_shape.len() != metadata.shape().len() {
2147 return Err(ProgramFinishError::BindingFinalization {
2148 source: ProgramBindingError::RankMismatch {
2149 expected: metadata.shape().len(),
2150 actual: actual_shape.len(),
2151 },
2152 });
2153 }
2154 for (axis, (extent, &actual)) in
2155 metadata.shape().iter().zip(actual_shape.iter()).enumerate()
2156 {
2157 match extent {
2158 ShapeExtent::Exact(DimExpr::Const(expected)) if *expected != actual => {
2159 return Err(ProgramFinishError::BindingFinalization {
2160 source: ProgramBindingError::ExactExtentMismatch {
2161 axis,
2162 expected: *expected,
2163 actual,
2164 },
2165 });
2166 }
2167 ShapeExtent::UpperBound(DimExpr::Const(bound)) if actual > *bound => {
2168 return Err(ProgramFinishError::BindingFinalization {
2169 source: ProgramBindingError::UpperBoundExceeded {
2170 axis,
2171 bound: *bound,
2172 actual,
2173 },
2174 });
2175 }
2176 _ => {}
2177 }
2178 }
2179 }
2180 Ok(())
2181}
2182
2183fn resolve_dim_expr_from_input_shapes(
2189 expr: &DimExpr,
2190 bound_input_shapes: &[Option<Vec<DimExpr>>],
2191) -> DimExpr {
2192 match expr {
2193 DimExpr::Const(_) => expr.clone(),
2194 DimExpr::InputDim { input_idx, axis } => {
2195 if let Some(Some(shape)) = bound_input_shapes.get(*input_idx)
2196 && let Some(dim) = shape.get(*axis)
2197 {
2198 return dim.clone();
2199 }
2200 expr.clone()
2201 }
2202 DimExpr::Add(a, b) => DimExpr::add(
2203 resolve_dim_expr_from_input_shapes(a, bound_input_shapes),
2204 resolve_dim_expr_from_input_shapes(b, bound_input_shapes),
2205 ),
2206 DimExpr::Sub(a, b) => DimExpr::sub(
2207 resolve_dim_expr_from_input_shapes(a, bound_input_shapes),
2208 resolve_dim_expr_from_input_shapes(b, bound_input_shapes),
2209 ),
2210 DimExpr::Mul(a, b) => DimExpr::mul(
2211 resolve_dim_expr_from_input_shapes(a, bound_input_shapes),
2212 resolve_dim_expr_from_input_shapes(b, bound_input_shapes),
2213 ),
2214 DimExpr::FloorDiv(a, b) => DimExpr::floor_div(
2215 resolve_dim_expr_from_input_shapes(a, bound_input_shapes),
2216 resolve_dim_expr_from_input_shapes(b, bound_input_shapes),
2217 ),
2218 DimExpr::Min(a, b) => DimExpr::min(
2219 resolve_dim_expr_from_input_shapes(a, bound_input_shapes),
2220 resolve_dim_expr_from_input_shapes(b, bound_input_shapes),
2221 ),
2222 DimExpr::Max(a, b) => DimExpr::max(
2223 resolve_dim_expr_from_input_shapes(a, bound_input_shapes),
2224 resolve_dim_expr_from_input_shapes(b, bound_input_shapes),
2225 ),
2226 }
2227}