Skip to main content

xlsynth_codegen/
block.rs

1// SPDX-License-Identifier: Apache-2.0
2
3use std::collections::{BTreeMap, BTreeSet};
4
5use xlsynth::vast_helpers::{RegisterDefinition, RegisterScope, Reset};
6use xlsynth_pir::ir::{
7    Binop, Block, BlockPort, InstantiationKind, MemberType, NodePayload, NodeRef, Package,
8    PackageMember, Register, Type, Unop,
9};
10use xlsynth_pir::ir_utils::{get_topological, operands};
11use xlsynth_pir::ir_verify::verify_package;
12use xlsynth_pir::{IrFormatPreference, IrValue};
13use xlsynth_vast::{
14    Expr, IndexableExpr, LiteralFormat, LogicRef, VastDataType, VastFile, VastFileType, VastModule,
15};
16
17use crate::stages::{ChildTimings, Stage, reconstruct_stages};
18use crate::{BlockCodegenError, BlockCodegenOptions, BlockCodegenOutput, Layout};
19
20/// Holds a packed SystemVerilog value and its expression-inlining information.
21#[derive(Clone, Copy, Debug)]
22pub(crate) struct Value {
23    pub(crate) expr: Expr,
24    pub(crate) indexable: Option<IndexableExpr>,
25    pub(crate) depth: usize,
26    pub(crate) bit_width: Option<usize>,
27    pub(crate) static_origin: Option<(IndexableExpr, usize)>,
28    pub(crate) array_rank: usize,
29}
30
31/// Collects registers whose writes use the same emitted reset expression.
32struct ResettableRegisterGroup {
33    signal: Expr,
34    registers: Vec<RegisterDefinition>,
35}
36
37impl Value {
38    /// Represents a declared, directly indexable SystemVerilog signal.
39    pub(crate) fn signal(signal: LogicRef) -> Self {
40        let indexable = signal.to_indexable_expr();
41        Self {
42            expr: signal.to_expr(),
43            indexable: Some(indexable),
44            depth: 0,
45            bit_width: None,
46            static_origin: Some((indexable, 0)),
47            array_rank: 0,
48        }
49    }
50
51    /// Represents an inline expression of the requested nesting depth.
52    pub(crate) fn expression(expr: Expr, depth: usize) -> Self {
53        Self {
54            expr,
55            indexable: None,
56            depth,
57            bit_width: None,
58            static_origin: None,
59            array_rank: 0,
60        }
61    }
62
63    /// Preserves the original packed signal for nested, portable part-selects.
64    pub(crate) fn static_slice(expr: Expr, source: Self, start: usize, width: usize) -> Self {
65        Self {
66            expr,
67            indexable: None,
68            depth: source.depth + 1,
69            bit_width: Some(width),
70            static_origin: source
71                .static_origin
72                .map(|(origin, offset)| (origin, offset + start)),
73            array_rank: 0,
74        }
75    }
76
77    /// Records a known packed width for scalar-index and slice legalization.
78    pub(crate) fn with_width(mut self, bit_width: usize) -> Self {
79        self.bit_width = Some(bit_width);
80        self
81    }
82
83    /// Records the dimensions of a declared packed value, keeping bit slices
84    /// distinct from array-element selections.
85    pub(crate) fn with_type(mut self, mut ty: &Type) -> Self {
86        self.bit_width = Some(ty.bit_count());
87        self.array_rank = 0;
88        while let Type::Array(array) = ty {
89            self.array_rank += 1;
90            ty = &array.element_type;
91        }
92        if self.array_rank != 0 {
93            self.static_origin = None;
94        }
95        self
96    }
97}
98
99/// Builds one module while preserving the source block's interfaces and state.
100pub(crate) struct BlockEmitter<'a, 'file> {
101    pub(crate) package: &'a Package,
102    blocks: &'a BTreeMap<&'a str, &'a Block>,
103    pub(crate) options: &'a BlockCodegenOptions,
104    pub(crate) block: &'a Block,
105    pub(crate) file: &'file mut VastFile,
106    pub(crate) module: VastModule,
107    pub(crate) values: Vec<Option<Value>>,
108    pub(crate) ports: BTreeMap<String, LogicRef>,
109    pub(crate) inout_nets: BTreeMap<&'a str, Value>,
110    pub(crate) register_refs: BTreeMap<String, LogicRef>,
111    pub(crate) users: Vec<Vec<NodeRef>>,
112    pub(crate) names: BTreeSet<String>,
113    pub(crate) genvar_names: BTreeMap<usize, String>,
114    pub(crate) priority_helpers: BTreeMap<crate::priority::PriorityHelper, String>,
115    pub(crate) arithmetic_helpers: BTreeMap<crate::arithmetic::ArithmeticHelper, String>,
116    pub(crate) slice_helpers: BTreeMap<crate::slicing::SliceHelper, String>,
117    pub(crate) current_stage: Option<usize>,
118    /// Connections indexed by instance, then by child port.
119    pub(crate) instance_connections: BTreeMap<String, BTreeMap<String, Value>>,
120}
121
122/// Emits a package after validating its selected block top.
123pub(crate) fn emit_package(
124    package: &Package,
125    options: &BlockCodegenOptions,
126    mut top_register_names: Option<&mut BTreeMap<String, String>>,
127) -> Result<BlockCodegenOutput, BlockCodegenError> {
128    let top = select_top(package, options.top.as_deref())?;
129    let ordered = blocks_in_emission_order(package, top)?;
130    for block in &ordered {
131        validate_supported_nodes(block, options.layout)?;
132    }
133    verify_package(package).map_err(|error| BlockCodegenError::InvalidBlock(error.to_string()))?;
134    let blocks = ordered.iter().map(|&b| (b.name.as_str(), b)).collect();
135    for &block in &ordered {
136        validate_single_register_writes(block)?;
137    }
138    crate::hierarchy::verify_hierarchy(&ordered)?;
139    if let Some(module_name) = options.module_name.as_deref()
140        && ordered
141            .iter()
142            .any(|block| block.name != top && block.name == module_name)
143    {
144        return Err(BlockCodegenError::InvalidBlock(format!(
145            "top module name `{module_name}` conflicts with an instantiated child block; \
146             choose a distinct module name"
147        )));
148    }
149
150    let mut file = VastFile::new(VastFileType::SystemVerilog);
151    let mut child_timings = ChildTimings::new();
152    for block in ordered {
153        let module_name = if block.name == top {
154            options.module_name.as_deref().unwrap_or(&block.name)
155        } else {
156            &block.name
157        };
158        validate_external_identifier(module_name, "module")?;
159        let module = file.add_module(module_name);
160        let mut emitter = BlockEmitter::new(package, &blocks, options, block, &mut file, module);
161        emitter.emit(&mut child_timings, block.name != top)?;
162        if block.name == top
163            && let Some(names) = top_register_names.take()
164        {
165            names.extend(
166                emitter
167                    .register_refs
168                    .into_iter()
169                    .map(|(name, signal)| (name, emitter.file.logic_ref_name(signal))),
170            );
171        }
172    }
173
174    Ok(BlockCodegenOutput {
175        system_verilog: file.emit(),
176    })
177}
178
179/// Rejects multiple enabled writes, which PIR permits but codegen cannot emit.
180fn validate_single_register_writes(block: &Block) -> Result<(), BlockCodegenError> {
181    let mut write_counts = BTreeMap::<&str, usize>::new();
182    for node in &block.nodes {
183        if let NodePayload::RegisterWrite { register, .. } = &node.payload {
184            *write_counts.entry(register.as_str()).or_default() += 1;
185        }
186    }
187
188    for register in &block.registers {
189        // Package verification already established at least one write.
190        let writes = write_counts[register.name.as_str()];
191        if writes != 1 {
192            return Err(BlockCodegenError::InvalidBlock(format!(
193                "register `{}` in block `{}` requires exactly one write, found {writes}",
194                register.name, block.name
195            )));
196        }
197    }
198
199    Ok(())
200}
201
202/// Checks which operations the requested layout can emit.
203fn validate_supported_nodes(block: &Block, layout: Layout) -> Result<(), BlockCodegenError> {
204    if layout == Layout::Pipeline
205        && let Some(net) = block.inout_nets.first()
206    {
207        return Err(BlockCodegenError::NotPipeline(format!(
208            "block `{}` cannot use layout=pipeline with inout net `{}`; use layout=none",
209            block.name, net.name
210        )));
211    }
212    for node in &block.nodes {
213        let (operation, lowering) = match &node.payload {
214            NodePayload::Invoke { to_apply, .. } => {
215                return Err(unsupported_invoke(block, to_apply));
216            }
217            NodePayload::CountedFor { .. } => ("counted_for", "unroll loops"),
218            NodePayload::Map { .. } => ("map", "expand maps"),
219            _ => continue,
220        };
221        return Err(BlockCodegenError::Unsupported(format!(
222            "{operation} is not supported in `{}`; {lowering} before block2sv code generation",
223            block.name
224        )));
225    }
226    Ok(())
227}
228
229/// Identifies a function call that the caller must inline before emission.
230pub(crate) fn unsupported_invoke(block: &Block, callee: &str) -> BlockCodegenError {
231    BlockCodegenError::Unsupported(format!(
232        "block `{}` contains an invoke of `{callee}`. Inline function calls before SystemVerilog emission.",
233        block.name
234    ))
235}
236
237/// Inlines function calls in the selected top and its transitive child blocks.
238pub fn inline_invokes_for_codegen(
239    package: Package,
240    explicit_top: Option<&str>,
241) -> Result<Package, BlockCodegenError> {
242    let top = select_top(&package, explicit_top)?;
243    let names = blocks_in_emission_order(&package, top)?
244        .into_iter()
245        .map(|block| block.name.clone())
246        .collect::<Vec<_>>();
247    xlsynth_pir::block_invoke_inline::inline_block_invokes_in_blocks(package, &names)
248        .map_err(BlockCodegenError::InvalidBlock)
249}
250
251/// Selects the explicitly requested, marked, or uniquely available block.
252fn select_top<'a>(
253    package: &'a Package,
254    explicit_top: Option<&str>,
255) -> Result<&'a str, BlockCodegenError> {
256    let blocks = package
257        .members
258        .iter()
259        .filter_map(|member| match member {
260            PackageMember::Block(block) => Some(block.name.as_str()),
261            PackageMember::Function(_) => None,
262        })
263        .collect::<Vec<_>>();
264
265    if let Some(requested) = explicit_top {
266        return blocks
267            .into_iter()
268            .find(|name| *name == requested)
269            .ok_or_else(|| {
270                BlockCodegenError::TopSelection(format!(
271                    "requested top block `{requested}` does not exist"
272                ))
273            });
274    }
275
276    if let Some((name, MemberType::Block)) = &package.top {
277        return blocks
278            .into_iter()
279            .find(|candidate| candidate == &name.as_str())
280            .ok_or_else(|| {
281                BlockCodegenError::TopSelection(format!(
282                    "package top block `{name}` does not exist"
283                ))
284            });
285    }
286
287    match blocks.as_slice() {
288        [only] => Ok(only),
289        [] => Err(BlockCodegenError::TopSelection(
290            "package contains no block suitable for SystemVerilog generation".to_owned(),
291        )),
292        _ => Err(BlockCodegenError::TopSelection(format!(
293            "package contains multiple blocks ({}) without a block top; specify --top",
294            blocks.join(", ")
295        ))),
296    }
297}
298
299/// Appends each transitive child once, before the block which instantiates it.
300fn blocks_in_emission_order<'a>(
301    package: &'a Package,
302    top: &'a str,
303) -> Result<Vec<&'a Block>, BlockCodegenError> {
304    let mut ordered = Vec::new();
305    collect_dependencies(
306        package,
307        top,
308        &mut ordered,
309        &mut BTreeSet::new(),
310        &mut BTreeSet::new(),
311    )?;
312    Ok(ordered)
313}
314
315/// Visits instantiated blocks in declaration order for deterministic emission.
316fn collect_dependencies<'a>(
317    package: &'a Package,
318    name: &'a str,
319    ordered: &mut Vec<&'a Block>,
320    visited: &mut BTreeSet<&'a str>,
321    visiting: &mut BTreeSet<&'a str>,
322) -> Result<(), BlockCodegenError> {
323    if visited.contains(name) {
324        return Ok(());
325    }
326    if !visiting.insert(name) {
327        return Err(BlockCodegenError::InvalidBlock(format!(
328            "block instantiation cycle includes `{name}`"
329        )));
330    }
331    let block = package
332        .members
333        .iter()
334        .find_map(|member| match member {
335            PackageMember::Block(block) if block.name == name => Some(block),
336            _ => None,
337        })
338        .ok_or_else(|| {
339            BlockCodegenError::InvalidBlock(format!(
340                "instantiated block `{name}` does not exist in the package"
341            ))
342        })?;
343
344    for instance in &block.instantiations {
345        if instance.kind == InstantiationKind::Block {
346            collect_dependencies(package, &instance.block, ordered, visited, visiting)?;
347        }
348    }
349    visiting.remove(name);
350    visited.insert(name);
351    ordered.push(block);
352    Ok(())
353}
354
355impl<'a, 'file> BlockEmitter<'a, 'file> {
356    /// Creates an emitter before constructing the module's public ports.
357    fn new(
358        package: &'a Package,
359        blocks: &'a BTreeMap<&'a str, &'a Block>,
360        options: &'a BlockCodegenOptions,
361        block: &'a Block,
362        file: &'file mut VastFile,
363        module: VastModule,
364    ) -> Self {
365        let mut users = vec![Vec::new(); block.nodes.len()];
366        for (index, node) in block.nodes.iter().enumerate() {
367            for operand in operands(&node.payload) {
368                users[operand.index].push(NodeRef { index });
369            }
370        }
371        Self {
372            package,
373            blocks,
374            options,
375            block,
376            file,
377            module,
378            values: vec![None; block.nodes.len()],
379            ports: BTreeMap::new(),
380            inout_nets: BTreeMap::new(),
381            register_refs: BTreeMap::new(),
382            users,
383            names: BTreeSet::new(),
384            genvar_names: BTreeMap::new(),
385            priority_helpers: BTreeMap::new(),
386            arithmetic_helpers: BTreeMap::new(),
387            slice_helpers: BTreeMap::new(),
388            current_stage: None,
389            instance_connections: BTreeMap::new(),
390        }
391    }
392
393    /// Emits ports, logic, state, output assignments, and child instances.
394    fn emit(
395        &mut self,
396        child_timings: &mut ChildTimings<'a>,
397        summarize_for_parent: bool,
398    ) -> Result<(), BlockCodegenError> {
399        self.reserve_fixed_names()?;
400        self.emit_ports()?;
401        self.emit_arithmetic_helpers()?;
402        self.emit_slice_helpers()?;
403        self.emit_priority_helpers(self.block)?;
404        match self.options.layout {
405            Layout::None => self.emit_flat()?,
406            Layout::Pipeline => {
407                let stages = reconstruct_stages(self.block, child_timings, summarize_for_parent)?;
408                self.emit_pipeline(&stages)?;
409            }
410        }
411        self.emit_outputs()?;
412        self.emit_instantiations()?;
413        Ok(())
414    }
415
416    /// Rejects conflicting fixed identifiers and protects them from generated
417    /// declarations throughout this module, regardless of emission order.
418    fn reserve_fixed_names(&mut self) -> Result<(), BlockCodegenError> {
419        let mut fixed_names = BTreeMap::new();
420        let mut reserve = |name: &str, kind: &'static str| {
421            validate_external_identifier(name, kind)?;
422            if let Some(previous) = fixed_names.insert(name.to_owned(), kind) {
423                return Err(BlockCodegenError::InvalidBlock(format!(
424                    "SystemVerilog name collision in block `{}`: `{name}` is used by both {previous} and {kind}",
425                    self.block.name
426                )));
427            }
428            Ok(())
429        };
430        if let Some(clock) = self.block.clock_port_name() {
431            reserve(clock, "port")?;
432        }
433        for port in self
434            .block
435            .input_ports()
436            .chain(self.block.output_ports())
437            .chain(self.block.inout_ports())
438        {
439            if self.block.port_type(port).bit_count() != 0 {
440                reserve(self.block.port_name(port), "port")?;
441            }
442        }
443        for instance in &self.block.instantiations {
444            reserve(&instance.name, "instance")?;
445        }
446        for node in &self.block.nodes {
447            match &node.payload {
448                NodePayload::Assert { label, .. }
449                    if self.options.emit_asserts && !label.is_empty() =>
450                {
451                    reserve(label, "assertion label")?;
452                }
453                NodePayload::Cover { label, .. } => reserve(label, "coverage label")?,
454                _ => {
455                    // Other nodes have generated names or introduce no fixed
456                    // declaration in the module scope.
457                }
458            }
459        }
460        self.names.extend(fixed_names.into_keys());
461        Ok(())
462    }
463
464    /// Emits representable ports in the source block's original header order.
465    fn emit_ports(&mut self) -> Result<(), BlockCodegenError> {
466        let mut aliases = BTreeMap::<&str, Vec<NodeRef>>::new();
467        for port in &self.block.ports {
468            let (name, ir_type, sv_type) = match port {
469                BlockPort::Clock(name) => (name.as_str(), &Type::Bits(1), None),
470                BlockPort::Input(node) | BlockPort::Output(node) | BlockPort::Inout(node) => {
471                    let sv_type = match &self.block.get_node(*node).payload {
472                        NodePayload::InputPort { sv_type, .. }
473                        | NodePayload::OutputPort { sv_type, .. } => sv_type.as_deref(),
474                        NodePayload::ExtInoutPort { net, sv_type, .. } => {
475                            aliases.entry(net).or_default().push(*node);
476                            sv_type.as_deref()
477                        }
478                        _ => unreachable!("verified block data port"),
479                    };
480                    (
481                        self.block.port_name(*node),
482                        self.block.port_type(*node),
483                        sv_type,
484                    )
485                }
486            };
487            if self.ports.contains_key(name) {
488                continue;
489            }
490            let width = ir_type.bit_count();
491            if width == 0 {
492                continue;
493            }
494            validate_external_identifier(name, "port")?;
495            let data_type = self.declared_data_type(ir_type, sv_type);
496            let signal = match port {
497                BlockPort::Inout(_) => self.file.add_inout(self.module, name, &data_type),
498                BlockPort::Output(_) => self.file.add_logic_output(self.module, name, &data_type),
499                BlockPort::Input(_) | BlockPort::Clock(_) => {
500                    self.file.add_logic_input(self.module, name, &data_type)
501                }
502            };
503            self.ports.insert(name.to_owned(), signal);
504        }
505
506        self.emit_inout_nets(&aliases)?;
507        for (index, node) in self.block.nodes.iter().enumerate() {
508            match &node.payload {
509                NodePayload::InputPort { name, sv_type } => {
510                    if let Some(&signal) = self.ports.get(name) {
511                        let value = Value::signal(signal);
512                        self.values[index] =
513                            Some(if self.options.emit_sv_types && sv_type.is_some() {
514                                value.with_width(node.ty.bit_count())
515                            } else {
516                                value.with_type(&node.ty)
517                            });
518                    }
519                }
520                NodePayload::ExtInoutRead { net } => {
521                    self.values[index] = Some(self.inout_nets[net.as_str()]);
522                }
523                _ => {
524                    // Other values are computed in topological order.
525                }
526            }
527        }
528        Ok(())
529    }
530
531    /// Allocates each electrical net and joins its public aliases in header
532    /// order.
533    fn emit_inout_nets(
534        &mut self,
535        aliases: &BTreeMap<&str, Vec<NodeRef>>,
536    ) -> Result<(), BlockCodegenError> {
537        for net in &self.block.inout_nets {
538            let value = if let Some((&first, others)) = aliases
539                .get(net.name.as_str())
540                .and_then(|ports| ports.split_first())
541            {
542                let signal = self.ports[self.block.port_name(first)];
543                for &other in others {
544                    let name = self.block.port_name(other);
545                    let tie = self.unique_name(&format!("{name}_tie"));
546                    self.file.add_tran(
547                        self.module,
548                        &tie,
549                        net.ty.bit_count(),
550                        signal,
551                        self.ports[name],
552                    )?;
553                }
554                let value = Value::signal(signal);
555                if self.options.emit_sv_types
556                    && matches!(
557                        &self.block.get_node(first).payload,
558                        NodePayload::ExtInoutPort {
559                            sv_type: Some(_),
560                            ..
561                        }
562                    )
563                {
564                    value.with_width(net.ty.bit_count())
565                } else {
566                    value.with_type(&net.ty)
567                }
568            } else {
569                let name = self.unique_name(&net.name);
570                let ty = self.value_type(&net.ty);
571                Value::signal(self.file.add_wire(self.module, &name, &ty)).with_type(&net.ty)
572            };
573            self.inout_nets.insert(&net.name, value);
574        }
575        Ok(())
576    }
577
578    /// Chooses a declared SystemVerilog type, including optional external
579    /// types.
580    fn declared_data_type(&mut self, ty: &Type, sv_type: Option<&str>) -> VastDataType {
581        if self.options.emit_sv_types
582            && let Some(sv_type) = sv_type
583        {
584            if let Some((package, ty)) = sv_type.rsplit_once("::") {
585                self.file.make_extern_package_type(package, ty)
586            } else {
587                self.file.make_extern_type(sv_type)
588            }
589        } else {
590            self.value_type(ty)
591        }
592    }
593
594    /// Emits existing register state and ordinary dependency-ordered logic.
595    fn emit_flat(&mut self) -> Result<(), BlockCodegenError> {
596        let start = self.file.module_member_count(self.module);
597        self.emit_input_array_views()?;
598        for register in &self.block.registers {
599            self.declare_register(register)?;
600        }
601        let mut writes = Vec::new();
602        for node_ref in get_topological(self.block) {
603            match &self.block.get_node(node_ref).payload {
604                NodePayload::Nil
605                | NodePayload::InputPort { .. }
606                | NodePayload::ExtInoutRead { .. }
607                | NodePayload::ExtInoutPort { .. }
608                | NodePayload::OutputPort { .. } => {
609                    // Reserved nodes and parameters are represented by module
610                    // ports.
611                }
612                NodePayload::RegisterRead { register } => {
613                    if let Some(signal) = self.register_refs.get(register).copied() {
614                        self.values[node_ref.index] =
615                            Some(Value::signal(signal).with_type(self.block.get_node_ty(node_ref)));
616                    }
617                }
618                NodePayload::RegisterWrite { .. } => writes.push(node_ref),
619                _ => self.emit_node(node_ref)?,
620            }
621        }
622        self.emit_register_groups(&writes)?;
623        self.file.hoist_module_declarations(self.module, start);
624        Ok(())
625    }
626
627    /// Emits register-delimited comments without altering sequential behavior.
628    fn emit_pipeline(&mut self, stages: &[Stage]) -> Result<(), BlockCodegenError> {
629        let mut emitted_visible_stage = false;
630        for (stage_index, stage) in stages.iter().enumerate() {
631            self.current_stage = Some(stage_index);
632            let visible =
633                !stage.combinational_nodes.is_empty() || !stage.register_writes.is_empty();
634            if visible {
635                if emitted_visible_stage {
636                    let blank = self.file.make_blank_line();
637                    self.file.add_member_blank_line(self.module, blank);
638                }
639                let comment = self
640                    .file
641                    .make_comment(&format!("===== Pipe stage {stage_index}:"));
642                self.file.add_member_comment(self.module, comment);
643                emitted_visible_stage = true;
644            }
645            self.emit_stage(stage, stage_index)?;
646        }
647        self.current_stage = None;
648        Ok(())
649    }
650
651    /// Emits one reconstructed stage and the registers written at its end.
652    fn emit_stage(&mut self, stage: &Stage, stage_index: usize) -> Result<(), BlockCodegenError> {
653        for &node_ref in &stage.register_reads {
654            if let NodePayload::RegisterRead { register } = &self.block.get_node(node_ref).payload {
655                if let Some(signal) = self.register_refs.get(register).copied() {
656                    self.values[node_ref.index] =
657                        Some(Value::signal(signal).with_type(self.block.get_node_ty(node_ref)));
658                }
659            }
660        }
661        let start = self.file.module_member_count(self.module);
662        if stage_index == 0 {
663            self.emit_input_array_views()?;
664        }
665        for &node_ref in &stage.combinational_nodes {
666            self.emit_node(node_ref)?;
667        }
668        self.file.hoist_module_declarations(self.module, start);
669        if !stage.register_writes.is_empty() {
670            let blank = self.file.make_blank_line();
671            self.file.add_member_blank_line(self.module, blank);
672            let comment = self
673                .file
674                .make_comment(&format!("Registers for pipe stage {stage_index}:"));
675            self.file.add_member_comment(self.module, comment);
676            for &node_ref in &stage.register_writes {
677                if let NodePayload::RegisterWrite { register, .. } =
678                    &self.block.get_node(node_ref).payload
679                {
680                    let definition = self
681                        .block
682                        .registers
683                        .iter()
684                        .find(|definition| definition.name == *register)
685                        .ok_or_else(|| {
686                            BlockCodegenError::InvalidBlock(format!(
687                                "register `{register}` has no declaration in block `{}`",
688                                self.block.name
689                            ))
690                        })?;
691                    self.declare_register(definition)?;
692                }
693            }
694            self.emit_register_groups(&stage.register_writes)?;
695        }
696        Ok(())
697    }
698
699    /// Declares one representable packed register and remembers its signal.
700    fn declare_register(&mut self, register: &Register) -> Result<(), BlockCodegenError> {
701        if register.ty.bit_count() == 0 || self.register_refs.contains_key(&register.name) {
702            return Ok(());
703        }
704        let data_type = self.value_type(&register.ty);
705        let name = self.unique_name(&register.name);
706        let signal = self.file.add_logic(self.module, &name, &data_type)?;
707        self.register_refs.insert(register.name.clone(), signal);
708        Ok(())
709    }
710
711    /// Groups resettable and non-resettable registers by their existing clock.
712    fn emit_register_groups(&mut self, writes: &[NodeRef]) -> Result<(), BlockCodegenError> {
713        let mut resettable = Vec::new();
714        let mut ordinary = Vec::new();
715        for &node_ref in writes {
716            let NodePayload::RegisterWrite {
717                arg,
718                register,
719                load_enable,
720                reset,
721            } = &self.block.get_node(node_ref).payload
722            else {
723                continue;
724            };
725            let Some(&signal) = self.register_refs.get(register) else {
726                continue;
727            };
728            let definition = self
729                .block
730                .registers
731                .iter()
732                .find(|definition| definition.name == *register)
733                .expect("verified register write has a declaration");
734            let next = self.required_value(*arg)?.expr;
735            let enable = load_enable
736                .map(|node| self.required_value(node).map(|value| value.expr))
737                .transpose()?;
738            let reset_signal = reset
739                .map(|node| self.required_value(node).map(|value| value.expr))
740                .transpose()?;
741            let reset_value = if reset_signal.is_some() {
742                definition
743                    .reset_value
744                    .as_ref()
745                    .map(|value| self.literal(value, &definition.ty))
746                    .transpose()?
747                    .flatten()
748            } else {
749                None
750            };
751            let register = RegisterDefinition {
752                reg: signal.to_expr(),
753                next,
754                reset_value,
755                enable,
756            };
757            if let Some(signal) = reset_signal {
758                resettable.push((signal, register));
759            } else {
760                ordinary.push(register);
761            }
762        }
763        if ordinary.is_empty() && resettable.is_empty() {
764            return Ok(());
765        }
766        let clock_name = self.block.clock_port_name().ok_or_else(|| {
767            BlockCodegenError::InvalidBlock(format!(
768                "block `{}` has registers but no clock port",
769                self.block.name
770            ))
771        })?;
772        let clock = self
773            .ports
774            .get(clock_name)
775            .ok_or_else(|| {
776                BlockCodegenError::InvalidBlock(format!(
777                    "block `{}` has no declared clock port `{clock_name}`",
778                    self.block.name
779                ))
780            })?
781            .to_expr();
782
783        if !ordinary.is_empty() {
784            xlsynth::vast_helpers::add_registers(
785                &clock,
786                None,
787                &ordinary,
788                RegisterScope::Module(self.module),
789                self.file,
790                self.options.register_codegen_options.as_ref(),
791            )?;
792        }
793        if !resettable.is_empty() {
794            let reset_metadata = self.block.reset.as_ref().ok_or_else(|| {
795                BlockCodegenError::InvalidBlock(format!(
796                    "block `{}` has resettable registers but no reset metadata",
797                    self.block.name
798                ))
799            })?;
800            let mut groups = Vec::<ResettableRegisterGroup>::new();
801            for (signal, register) in resettable {
802                if let Some(group) = groups.iter_mut().find(|group| group.signal == signal) {
803                    group.registers.push(register);
804                } else {
805                    groups.push(ResettableRegisterGroup {
806                        signal,
807                        registers: vec![register],
808                    });
809                }
810            }
811            for group in groups {
812                let reset = Reset {
813                    signal: group.signal,
814                    active_low: reset_metadata.active_low,
815                };
816                if reset_metadata.asynchronous {
817                    if let Some(options) = &self.options.register_codegen_options {
818                        let requires_enable = group
819                            .registers
820                            .iter()
821                            .any(|register| register.enable.is_some());
822                        if options.reg_with_reset_template.is_some()
823                            || (requires_enable
824                                && options.reg_with_reset_with_en_template.is_some())
825                        {
826                            return Err(BlockCodegenError::Unsupported(format!(
827                                concat!(
828                                    "block `{}` has asynchronous reset; custom reset-register ",
829                                    "templates cannot guarantee the required reset edge semantics"
830                                ),
831                                self.block.name
832                            )));
833                        }
834                    }
835                    self.emit_asynchronous_registers(&clock, reset, &group.registers)?;
836                } else {
837                    xlsynth::vast_helpers::add_registers(
838                        &clock,
839                        Some(reset),
840                        &group.registers,
841                        RegisterScope::Module(self.module),
842                        self.file,
843                        self.options.register_codegen_options.as_ref(),
844                    )?;
845                }
846            }
847        }
848        Ok(())
849    }
850
851    /// Emits an asynchronous reset edge with reset priority over load enables.
852    fn emit_asynchronous_registers(
853        &mut self,
854        clock: &Expr,
855        reset: Reset,
856        registers: &[RegisterDefinition],
857    ) -> Result<(), BlockCodegenError> {
858        let clock_edge = self.file.make_pos_edge(clock);
859        let reset_edge = if reset.active_low {
860            self.file.make_neg_edge(&reset.signal)
861        } else {
862            self.file.make_pos_edge(&reset.signal)
863        };
864        let always = self
865            .file
866            .add_always_ff(self.module, &[&clock_edge, &reset_edge])?;
867        let body = self.file.statement_block(always);
868        let condition = if reset.active_low {
869            self.file.make_logical_not(&reset.signal)
870        } else {
871            reset.signal
872        };
873        let conditional = self.file.block_add_cond(body, &condition);
874        let reset_body = self.file.conditional_then_block(conditional);
875        let update_body = self.file.conditional_add_else(conditional);
876        for register in registers {
877            let reset_value = register
878                .reset_value
879                .expect("resettable register group requires reset values");
880            self.file
881                .block_add_nonblocking_assignment(reset_body, &register.reg, &reset_value);
882            let next = if let Some(enable) = register.enable {
883                self.file
884                    .make_ternary(&enable, &register.next, &register.reg)
885            } else {
886                register.next
887            };
888            self.file
889                .block_add_nonblocking_assignment(update_body, &register.reg, &next);
890        }
891        Ok(())
892    }
893
894    /// Connects representable output ports after their driving logic exists.
895    fn emit_outputs(&mut self) -> Result<(), BlockCodegenError> {
896        for port in self.block.output_ports() {
897            let name = self.block.port_name(port);
898            let node_ref = self.block.output_value(port);
899            let Some(signal) = self.ports.get(name).copied() else {
900                continue;
901            };
902            let value = self.required_value(node_ref)?;
903            let assignment = self
904                .file
905                .make_continuous_assignment(&signal.to_expr(), &value.expr);
906            self.file
907                .add_member_continuous_assignment(self.module, assignment);
908        }
909        Ok(())
910    }
911
912    /// Emits deterministic child connections after all connection nodes exist.
913    fn emit_instantiations(&mut self) -> Result<(), BlockCodegenError> {
914        if self.block.instantiations.is_empty() {
915            return Ok(());
916        }
917        if self.options.layout == Layout::Pipeline {
918            let blank = self.file.make_blank_line();
919            self.file.add_member_blank_line(self.module, blank);
920            let comment = self.file.make_comment("===== Instantiations");
921            self.file.add_member_comment(self.module, comment);
922        }
923
924        for instance in &self.block.instantiations {
925            validate_external_identifier(&instance.name, "instance")?;
926            let mut connections = self
927                .instance_connections
928                .remove(&instance.name)
929                .unwrap_or_default()
930                .into_iter()
931                .map(|(port, value)| (port, value.expr))
932                .collect::<Vec<_>>();
933
934            if instance.kind == InstantiationKind::Block {
935                let child = self.blocks[instance.block.as_str()];
936                if let Some(clock_name) = child.clock_port_name() {
937                    if !connections.iter().any(|(name, _)| name == clock_name) {
938                        if let Some(parent_clock) = self
939                            .block
940                            .clock_port_name()
941                            .and_then(|name| self.ports.get(name))
942                        {
943                            connections.push((clock_name.to_string(), parent_clock.to_expr()));
944                        } else {
945                            return Err(BlockCodegenError::InvalidBlock(format!(
946                                "instance `{}` of block `{}` requires clock port \
947                                 `{clock_name}`, but parent block `{}` has no clock",
948                                instance.name, instance.block, self.block.name
949                            )));
950                        }
951                    }
952                }
953                let order = child
954                    .ports
955                    .iter()
956                    .map(|port| match port {
957                        BlockPort::Clock(name) => name.as_str(),
958                        BlockPort::Input(node)
959                        | BlockPort::Output(node)
960                        | BlockPort::Inout(node) => child.port_name(*node),
961                    })
962                    .collect::<Vec<_>>();
963                connections.sort_by_key(|(name, _)| {
964                    (
965                        order
966                            .iter()
967                            .position(|port| *port == name)
968                            .unwrap_or(usize::MAX),
969                        name.clone(),
970                    )
971                });
972                let names = connections
973                    .iter()
974                    .map(|(name, _)| name.as_str())
975                    .collect::<Vec<_>>();
976                let expressions = connections
977                    .iter()
978                    .map(|(_, expr)| Some(expr))
979                    .collect::<Vec<_>>();
980                let handle = self.file.make_instantiation(
981                    &instance.block,
982                    &instance.name,
983                    &[],
984                    &[],
985                    &names,
986                    &expressions,
987                );
988                self.file.add_member_instantiation(self.module, handle);
989            } else {
990                self.emit_external_instantiation(instance, &connections)?;
991            }
992        }
993        Ok(())
994    }
995
996    /// Substitutes external-function placeholders into its public FFI template.
997    fn emit_external_instantiation(
998        &mut self,
999        instance: &xlsynth_pir::ir::Instantiation,
1000        connections: &[(String, Expr)],
1001    ) -> Result<(), BlockCodegenError> {
1002        let foreign = self.package.get_fn(&instance.block).ok_or_else(|| {
1003            BlockCodegenError::InvalidBlock(format!(
1004                "external instance `{}` references missing function `{}`",
1005                instance.name, instance.block
1006            ))
1007        })?;
1008        let template = foreign
1009            .ffi_proto()
1010            .and_then(|ffi| ffi.code_template.as_deref())
1011            .ok_or_else(|| {
1012                BlockCodegenError::Unsupported(format!(
1013                    "external function `{}` has no ffi_proto code_template",
1014                    foreign.name
1015                ))
1016            })?;
1017        let template = std::str::from_utf8(template).map_err(|error| {
1018            BlockCodegenError::InvalidBlock(format!(
1019                "external function `{}` has a non-UTF-8 ffi_proto code_template at byte {}",
1020                foreign.name,
1021                error.valid_up_to()
1022            ))
1023        })?;
1024        let bindings = connections
1025            .iter()
1026            .map(|(name, value)| (name.as_str(), self.file.emit_expression(value)))
1027            .collect::<BTreeMap<_, _>>();
1028        let template = substitute_external_template(template, &instance.name, &bindings)?;
1029        let statement = self.file.make_inline_verilog_statement(template.trim());
1030        self.file
1031            .add_member_inline_statement(self.module, statement);
1032        Ok(())
1033    }
1034
1035    /// Builds the packed scalar or vector type used for one IR value.
1036    pub(crate) fn bits_type(&mut self, width: usize) -> VastDataType {
1037        self.file.make_bit_vector_type(width as i64, false)
1038    }
1039
1040    /// Preserves outer array dimensions while packing tuple leaves into bits.
1041    pub(crate) fn value_type(&mut self, mut ty: &Type) -> VastDataType {
1042        let mut dimensions = Vec::new();
1043        while let Type::Array(array) = ty {
1044            dimensions.push(array.element_count as i64);
1045            ty = &array.element_type;
1046        }
1047        let element = self.bits_type(ty.bit_count());
1048        if dimensions.is_empty() {
1049            element
1050        } else {
1051            self.file.make_packed_array_type(element, &dimensions)
1052        }
1053    }
1054
1055    /// Gives custom-typed array input ports shaped views without changing
1056    /// their explicitly requested SystemVerilog types.
1057    fn emit_input_array_views(&mut self) -> Result<(), BlockCodegenError> {
1058        for (index, node) in self.block.nodes.iter().enumerate() {
1059            if matches!(
1060                node.payload,
1061                NodePayload::InputPort { .. } | NodePayload::ExtInoutRead { .. }
1062            ) && matches!(node.ty, Type::Array(_))
1063                && let Some(value) = self.values[index]
1064                && value.array_rank == 0
1065            {
1066                let view = self.assign_node(NodeRef { index }, value.expr)?;
1067                self.values[index] = Some(view);
1068            }
1069        }
1070        Ok(())
1071    }
1072
1073    /// Returns the represented expression for a node with nonzero bit width.
1074    pub(crate) fn required_value(&self, node: NodeRef) -> Result<Value, BlockCodegenError> {
1075        self.values[node.index].ok_or_else(|| {
1076            BlockCodegenError::InvalidBlock(format!(
1077                "node `{}` in block `{}` has no representable SystemVerilog value",
1078                self.block.get_node(node).name.clone().unwrap_or_else(|| {
1079                    format!(
1080                        "{}.{}",
1081                        self.block.get_node(node).payload.get_operator(),
1082                        self.block.get_node(node).text_id
1083                    )
1084                }),
1085                self.block.name
1086            ))
1087        })
1088    }
1089
1090    /// Converts an arbitrary-width IR literal into its packed representation.
1091    pub(crate) fn literal(
1092        &mut self,
1093        value: &IrValue,
1094        ty: &Type,
1095    ) -> Result<Option<Expr>, BlockCodegenError> {
1096        if ty.bit_count() == 0 {
1097            return Ok(None);
1098        }
1099        match ty {
1100            Type::Bits(_) => {
1101                let formatted = value.to_string_fmt(IrFormatPreference::Hex);
1102                Ok(Some(
1103                    self.file.make_literal(&formatted, &LiteralFormat::Hex)?,
1104                ))
1105            }
1106            Type::Tuple(types) => {
1107                let values = value.as_elements()?;
1108                let mut expressions = Vec::new();
1109                for (value, ty) in values.iter().zip(types) {
1110                    if let Some(expression) = self.literal(value, ty)? {
1111                        expressions.push(expression);
1112                    }
1113                }
1114                Ok(self.concat_or_only(&expressions))
1115            }
1116            Type::Array(array) => {
1117                let values = value.as_elements()?;
1118                let mut expressions = Vec::new();
1119                for value in values.iter().rev() {
1120                    if let Some(expression) = self.literal(value, &array.element_type)? {
1121                        expressions.push(expression);
1122                    }
1123                }
1124                Ok(self.concat_or_only(&expressions))
1125            }
1126            Type::Token => Ok(None),
1127        }
1128    }
1129
1130    /// Concatenates expressions while avoiding an unnecessary one-element
1131    /// brace.
1132    pub(crate) fn concat_or_only(&mut self, expressions: &[Expr]) -> Option<Expr> {
1133        match expressions {
1134            [] => None,
1135            [only] => Some(*only),
1136            many => {
1137                let refs = many.iter().collect::<Vec<_>>();
1138                Some(self.file.make_concat(&refs))
1139            }
1140        }
1141    }
1142
1143    /// Allocates a deterministic module-local identifier without collisions.
1144    pub(crate) fn unique_name(&mut self, requested: &str) -> String {
1145        let mut base = sanitize_identifier(requested);
1146        if self.names.insert(base.clone()) {
1147            return base;
1148        }
1149        let mut index = 1usize;
1150        loop {
1151            let candidate = format!("{base}__{index}");
1152            if self.names.insert(candidate.clone()) {
1153                return candidate;
1154            }
1155            index += 1;
1156            if index == usize::MAX {
1157                base.push('_');
1158                index = 1;
1159            }
1160        }
1161    }
1162
1163    /// Declares a stable, indexable signal for one materialized IR node.
1164    pub(crate) fn declare_node(
1165        &mut self,
1166        node_ref: NodeRef,
1167    ) -> Result<LogicRef, BlockCodegenError> {
1168        let name = self.node_name(node_ref);
1169        let unique = self.unique_name(&name);
1170        let ty = self.value_type(self.block.get_node_ty(node_ref));
1171        Ok(self.file.add_logic(self.module, &unique, &ty)?)
1172    }
1173
1174    /// Builds the requested node name in the current source-layout scope.
1175    pub(crate) fn node_name(&self, node_ref: NodeRef) -> String {
1176        let node = self.block.get_node(node_ref);
1177        let original_name = node
1178            .name
1179            .clone()
1180            .unwrap_or_else(|| format!("{}_{}", node.payload.get_operator(), node.text_id));
1181        if let Some(stage) = self.current_stage {
1182            let base = strip_pipeline_prefix(&original_name);
1183            format!("p{stage}_{base}_comb")
1184        } else {
1185            original_name
1186        }
1187    }
1188
1189    /// Emits a stable signal assignment and returns its indexable value.
1190    pub(crate) fn assign_node(
1191        &mut self,
1192        node_ref: NodeRef,
1193        expression: Expr,
1194    ) -> Result<Value, BlockCodegenError> {
1195        let signal = self.declare_node(node_ref)?;
1196        let assignment = self
1197            .file
1198            .make_continuous_assignment(&signal.to_expr(), &expression);
1199        self.file
1200            .add_member_continuous_assignment(self.module, assignment);
1201        Ok(Value::signal(signal).with_type(self.block.get_node_ty(node_ref)))
1202    }
1203
1204    /// Returns whether this node should be assigned a dedicated named signal.
1205    pub(crate) fn should_assign(&self, node_ref: NodeRef, depth: usize) -> bool {
1206        self.must_assign(node_ref)
1207            || self.block.get_node(node_ref).name.is_some()
1208            || self.users[node_ref.index].len() > 1
1209            || depth > self.options.max_inline_depth
1210    }
1211
1212    /// Preserves fixed-width and indexability boundaries required by lowering.
1213    fn must_assign(&self, node_ref: NodeRef) -> bool {
1214        let node = self.block.get_node(node_ref);
1215        matches!(node.ty, Type::Array(_))
1216            || matches!(
1217                node.payload,
1218                NodePayload::ArrayIndex { .. }
1219                    | NodePayload::ExtCarryOut { .. }
1220                    | NodePayload::ExtNaryAdd { .. }
1221            )
1222            || matches!(
1223                node.payload,
1224                NodePayload::Binop(
1225                    Binop::Add
1226                        | Binop::Sub
1227                        | Binop::Smul
1228                        | Binop::Umul
1229                        | Binop::Smulp
1230                        | Binop::Umulp
1231                        | Binop::Sdiv
1232                        | Binop::Udiv,
1233                    _,
1234                    _
1235                )
1236            )
1237            || self.options.separate_lines
1238            || self.users[node_ref.index]
1239                .iter()
1240                .any(|user| self.operand_must_be_named_reference(*user, node_ref))
1241    }
1242
1243    /// Identifies operand roles whose lowering requires a declared signal.
1244    fn operand_must_be_named_reference(&self, user: NodeRef, operand: NodeRef) -> bool {
1245        match &self.block.get_node(user).payload {
1246            NodePayload::RegisterWrite { .. } => true,
1247            NodePayload::BitSlice { arg, .. } => *arg == operand,
1248            NodePayload::TupleIndex { tuple, .. } => *tuple == operand,
1249            NodePayload::SignExt { arg, .. } => {
1250                *arg == operand && self.block.get_node_ty(operand).bit_count() > 1
1251            }
1252            NodePayload::ExtNaryAdd { terms, .. } => {
1253                terms.iter().any(|term| term.operand == operand)
1254            }
1255            NodePayload::OneHot { arg, .. }
1256            | NodePayload::Encode { arg }
1257            | NodePayload::ExtPrioEncode { arg, .. }
1258            | NodePayload::ExtClz { arg, .. }
1259            | NodePayload::ExtNormalizeLeft { arg, .. }
1260            | NodePayload::Unop(Unop::Reverse, arg) => {
1261                *arg == operand && self.block.get_node_ty(operand).bit_count() > 1
1262            }
1263            NodePayload::Sel {
1264                selector, cases, ..
1265            }
1266            | NodePayload::PrioritySel {
1267                selector, cases, ..
1268            }
1269            | NodePayload::OneHotSel { selector, cases } => *selector == operand && cases.len() > 1,
1270            NodePayload::ArraySlice {
1271                array,
1272                start,
1273                width,
1274            } => *width > 1 && (*array == operand || *start == operand),
1275            NodePayload::ArrayUpdate { indices, .. } => {
1276                // Genvar comparisons must not widen an inline index expression.
1277                indices.contains(&operand)
1278            }
1279            _ => false,
1280        }
1281    }
1282}
1283
1284/// Substitutes named FFI expressions while unescaping doubled literal braces.
1285fn substitute_external_template(
1286    template: &str,
1287    instance_name: &str,
1288    bindings: &BTreeMap<&str, String>,
1289) -> Result<String, BlockCodegenError> {
1290    let bytes = template.as_bytes();
1291    let mut output = String::with_capacity(template.len());
1292    let mut index = 0usize;
1293    while index < bytes.len() {
1294        match bytes[index] {
1295            b'{' if bytes.get(index + 1) == Some(&b'{') => {
1296                output.push('{');
1297                index += 2;
1298            }
1299            b'}' if bytes.get(index + 1) == Some(&b'}') => {
1300                output.push('}');
1301                index += 2;
1302            }
1303            b'{' => {
1304                let start = index + 1;
1305                let mut nesting = 1usize;
1306                index = start;
1307                while index < bytes.len() && nesting != 0 {
1308                    match bytes[index] {
1309                        b'{' => nesting += 1,
1310                        b'}' => nesting -= 1,
1311                        _ => {
1312                            // Ordinary expression characters do not change
1313                            // brace nesting.
1314                        }
1315                    }
1316                    if nesting != 0 {
1317                        index += 1;
1318                    }
1319                }
1320                if index >= bytes.len() {
1321                    return Err(BlockCodegenError::InvalidBlock(format!(
1322                        "external instance `{instance_name}` has an unterminated \
1323                         ffi_proto placeholder"
1324                    )));
1325                }
1326                let name = &template[start..index];
1327                if name == "fn" {
1328                    output.push_str(instance_name);
1329                } else if let Some(value) = bindings.get(name) {
1330                    output.push_str(value);
1331                } else {
1332                    return Err(BlockCodegenError::InvalidBlock(format!(
1333                        "external instance `{instance_name}` leaves ffi_proto \
1334                         placeholder `{{{name}}}` unresolved"
1335                    )));
1336                }
1337                index += 1;
1338            }
1339            b'}' => {
1340                return Err(BlockCodegenError::InvalidBlock(format!(
1341                    "external instance `{instance_name}` has an unmatched \
1342                     closing brace in its ffi_proto template"
1343                )));
1344            }
1345            _ => {
1346                let character = template[index..]
1347                    .chars()
1348                    .next()
1349                    .expect("byte index is positioned at a UTF-8 character");
1350                output.push(character);
1351                index += character.len_utf8();
1352            }
1353        }
1354    }
1355    Ok(output)
1356}
1357
1358/// Rejects public interface names that SystemVerilog cannot represent safely.
1359pub(crate) fn validate_external_identifier(
1360    name: &str,
1361    kind: &str,
1362) -> Result<(), BlockCodegenError> {
1363    if sanitize_identifier(name) != name {
1364        return Err(BlockCodegenError::InvalidBlock(format!(
1365            "invalid SystemVerilog {kind} identifier `{name}`: identifier is reserved, \
1366             malformed, or requires escaping"
1367        )));
1368    }
1369    Ok(())
1370}
1371
1372/// Produces a valid, deterministic SystemVerilog identifier from an IR name.
1373fn sanitize_identifier(name: &str) -> String {
1374    let mut output = String::with_capacity(name.len() + 1);
1375    for (index, character) in name.chars().enumerate() {
1376        if character.is_ascii_alphabetic()
1377            || character == '_'
1378            || (index != 0 && (character.is_ascii_digit() || character == '$'))
1379        {
1380            output.push(character);
1381        } else {
1382            if index == 0 && character.is_ascii_digit() {
1383                output.push('_');
1384                output.push(character);
1385            } else {
1386                output.push('_');
1387            }
1388        }
1389    }
1390    if output.is_empty() {
1391        output.push('_');
1392    }
1393    if xlsynth_vast::is_system_verilog_keyword(&output) {
1394        output.push('_');
1395    }
1396    output
1397}
1398
1399/// Removes an existing `p<digits>_` prefix before assigning a new stage.
1400fn strip_pipeline_prefix(name: &str) -> &str {
1401    let Some(rest) = name.strip_prefix('p') else {
1402        return name;
1403    };
1404    let digit_count = rest.chars().take_while(char::is_ascii_digit).count();
1405    if digit_count > 0 && rest[digit_count..].starts_with('_') {
1406        &rest[digit_count + 1..]
1407    } else {
1408        name
1409    }
1410}