facet_format/deserializer/

entry.rs

use std::borrow::Cow;

use facet_core::{Def, OpaqueDeserialize, ScalarType, Shape, StructKind, Type, UserType};
use facet_reflect::{DeserStrategy, Partial, ReflectErrorKind, Span};

use crate::{
    ContainerKind, DeserializeError, DeserializeErrorKind, FieldEvidence, FieldLocationHint,
    FormatDeserializer, ParseEventKind, ScalarTypeHint, ScalarValue, SpanGuard, ValueMeta,
};

/// Specifies where metadata should come from during deserialization.
#[derive(Debug, Clone, Default)]
#[non_exhaustive]
pub enum MetaSource<'a> {
    /// Use explicit metadata from an outer context (borrowed).
    ///
    /// Use cases:
    /// - **Consumed a VariantTag**: We consumed `@tag` before a value and need to pass
    ///   the tag name (and doc if present) to the inner value so metadata containers
    ///   can capture it.
    /// - **Recursive through wrappers**: Going through proxies, transparent converts,
    ///   pointers, `begin_inner` - same logical value, pass through same metadata.
    /// - **Merged metadata**: When we've built up metadata from multiple sources
    ///   (e.g., tag span + value span combined) and need to pass the merged result.
    Explicit(&'a ValueMeta<'a>),

    /// Use explicit metadata that was constructed locally (owned).
    ///
    /// Use cases:
    /// - **Struct field with attached metadata**: The field key had doc comments or
    ///   other metadata that should apply to the field value.
    Owned(ValueMeta<'a>),

    /// Get fresh metadata from the events being parsed.
    ///
    /// Use this when deserializing a new value that has no pre-consumed context:
    /// list items, map keys/values, struct fields without special metadata, etc.
    #[default]
    FromEvents,
}

impl<'a> From<&'a ValueMeta<'a>> for MetaSource<'a> {
    fn from(meta: &'a ValueMeta<'a>) -> Self {
        MetaSource::Explicit(meta)
    }
}

impl<'a> From<ValueMeta<'a>> for MetaSource<'a> {
    fn from(meta: ValueMeta<'a>) -> Self {
        MetaSource::Owned(meta)
    }
}

impl<'parser, 'input, const BORROW: bool> FormatDeserializer<'parser, 'input, BORROW> {
    /// Main deserialization entry point - deserialize into a Partial.
    ///
    /// Uses the precomputed `DeserStrategy` from TypePlan for fast dispatch.
    /// The strategy is computed once at Partial allocation time, eliminating
    /// repeated runtime inspection of Shape/Def/vtable during deserialization.
    ///
    /// The `meta` parameter specifies where metadata should come from:
    /// - `MetaSource::Explicit(meta)` - use provided metadata from outer context
    /// - `MetaSource::FromEvents` - read fresh metadata from the events being parsed
    #[inline(never)]
    pub fn deserialize_into(
        &mut self,
        wip: Partial<'input, BORROW>,
        meta: MetaSource<'input>,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        self.deserialize_into_inner(wip, meta)
    }

    #[inline(never)]
    pub(crate) fn deserialize_into_inner(
        &mut self,
        wip: Partial<'input, BORROW>,
        meta: MetaSource<'input>,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        let _guard = SpanGuard::new(self.last_span);
        let shape = wip.shape();
        trace!(
            shape_name = %shape,
            "deserialize_into: starting"
        );

        // === SPECIAL CASES (cannot be precomputed) ===

        // Check for raw capture type (e.g., RawJson) - parser-specific
        if self.parser.raw_capture_shape() == Some(shape) {
            let Some(raw) = self.capture_raw()? else {
                return Err(DeserializeErrorKind::RawCaptureNotSupported { shape }
                    .with_span(self.last_span));
            };
            let raw_cow = if BORROW {
                Cow::Borrowed(raw)
            } else {
                Cow::Owned(raw.to_owned())
            };
            return Ok(wip
                .begin_nth_field(0)?
                .with(|w| self.set_string_value(w, raw_cow))?
                .end()?);
        }

        // Check for builder_shape (immutable collections like Bytes -> BytesMut)
        // This MUST be checked at runtime because begin_inner() transitions to the
        // builder shape but keeps the same TypePlan node. If we used a precomputed
        // strategy, we'd get infinite recursion (BytesMut would still have Builder strategy).
        if shape.builder_shape.is_some() {
            return Ok(wip
                .begin_inner()?
                .with(|w| self.deserialize_into(w, meta))?
                .end()?);
        }

        // === STRATEGY-BASED DISPATCH ===
        // All other cases use precomputed DeserStrategy for O(1) dispatch.
        // Use the precomputed DeserStrategy for O(1) dispatch

        let strategy = wip.deser_strategy();
        trace!(?strategy, "deserialize_into: using precomputed strategy");

        match strategy {
            Some(DeserStrategy::ContainerProxy) => self.deserialize_container_proxy(wip, meta),

            Some(DeserStrategy::FieldProxy) => self.deserialize_field_proxy(wip, meta),

            Some(DeserStrategy::Pointer { .. }) => {
                trace!("deserialize_into: dispatching to deserialize_pointer");
                self.deserialize_pointer(wip, meta)
            }

            Some(DeserStrategy::TransparentConvert { .. }) => {
                trace!("deserialize_into: dispatching via begin_inner (transparent convert)");
                self.deserialize_transparent_convert(wip, meta)
            }

            Some(DeserStrategy::Scalar {
                scalar_type,
                is_from_str,
            }) => {
                let scalar_type = *scalar_type; // Copy before moving wip
                let is_from_str = *is_from_str;
                trace!("deserialize_into: dispatching to deserialize_scalar");
                self.deserialize_scalar(wip, scalar_type, is_from_str)
            }

            Some(DeserStrategy::Struct) => {
                trace!("deserialize_into: dispatching to deserialize_struct");
                self.deserialize_struct(wip)
            }

            Some(DeserStrategy::Tuple {
                field_count,
                is_single_field_transparent,
            }) => {
                let field_count = *field_count;
                let is_single_field_transparent = *is_single_field_transparent;
                trace!("deserialize_into: dispatching to deserialize_tuple");
                self.deserialize_tuple(wip, field_count, is_single_field_transparent)
            }

            Some(DeserStrategy::Enum) => {
                trace!("deserialize_into: dispatching to deserialize_enum");
                self.deserialize_enum(wip)
            }

            Some(DeserStrategy::Option { .. }) => {
                trace!("deserialize_into: dispatching to deserialize_option");
                self.deserialize_option(wip)
            }

            Some(DeserStrategy::Result { .. }) => {
                trace!("deserialize_into: dispatching to deserialize_result_as_enum");
                self.deserialize_result_as_enum(wip)
            }

            Some(DeserStrategy::List { is_byte_vec, .. }) => {
                let is_byte_vec = *is_byte_vec;
                trace!("deserialize_into: dispatching to deserialize_list");
                self.deserialize_list(wip, is_byte_vec)
            }

            Some(DeserStrategy::Map { .. }) => {
                trace!("deserialize_into: dispatching to deserialize_map");
                self.deserialize_map(wip)
            }

            Some(DeserStrategy::Set { .. }) => {
                trace!("deserialize_into: dispatching to deserialize_set");
                self.deserialize_set(wip)
            }

            Some(DeserStrategy::Array { .. }) => {
                trace!("deserialize_into: dispatching to deserialize_array");
                self.deserialize_array(wip)
            }

            Some(DeserStrategy::DynamicValue) => {
                trace!("deserialize_into: dispatching to deserialize_dynamic_value");
                self.deserialize_dynamic_value(wip)
            }

            Some(DeserStrategy::MetadataContainer) => {
                trace!("deserialize_into: dispatching to deserialize_metadata_container");
                self.deserialize_metadata_container(wip, meta)
            }

            Some(DeserStrategy::BackRef { .. }) => {
                // BackRef is automatically resolved by deser_strategy() - this branch
                // should never be reached. If it is, something is wrong with TypePlan.
                unreachable!("deser_strategy() should resolve BackRef to target strategy")
            }

            Some(DeserStrategy::Opaque) => self.deserialize_opaque(wip),

            Some(DeserStrategy::OpaquePointer) => self.unsupported_opaque_pointer(shape),

            // A deserialization strategy variant added since this match was written.
            Some(_) => self.unsupported_strategy(shape),

            None => self.missing_strategy(shape),
        }
    }

    #[inline(never)]
    fn deserialize_container_proxy(
        &mut self,
        wip: Partial<'input, BORROW>,
        meta: MetaSource<'input>,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        let format_ns = self.parser.format_namespace();
        let (wip, _) = wip.begin_custom_deserialization_from_shape_with_format(format_ns)?;
        Ok(wip.with(|w| self.deserialize_into(w, meta))?.end()?)
    }

    #[inline(never)]
    fn deserialize_field_proxy(
        &mut self,
        wip: Partial<'input, BORROW>,
        meta: MetaSource<'input>,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        let format_ns = self.parser.format_namespace();
        let wip = wip.begin_custom_deserialization_with_format(format_ns)?;
        Ok(wip.with(|w| self.deserialize_into(w, meta))?.end()?)
    }

    #[inline(never)]
    fn deserialize_transparent_convert(
        &mut self,
        wip: Partial<'input, BORROW>,
        meta: MetaSource<'input>,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        Ok(wip
            .begin_inner()?
            .with(|w| self.deserialize_into(w, meta))?
            .end()?)
    }

    #[inline(never)]
    fn deserialize_opaque(
        &mut self,
        wip: Partial<'input, BORROW>,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        let shape = wip.shape();
        let Some(adapter) = shape.opaque_adapter else {
            return Err(DeserializeErrorKind::Unsupported {
                message: format!(
                    "cannot deserialize opaque type {} - add a proxy or opaque adapter",
                    shape
                )
                .into(),
            }
            .with_span(self.last_span));
        };

        let trailing_opaque = wip
            .nearest_field()
            .is_some_and(|f| f.has_builtin_attr("trailing"));

        if self.is_non_self_describing() {
            let handled = if trailing_opaque {
                self.parser.hint_remaining_byte_sequence()
            } else {
                self.parser.hint_byte_sequence()
            };
            if !handled {
                self.parser.hint_scalar_type(ScalarTypeHint::Bytes);
            }
        }

        let expected = if trailing_opaque {
            "remaining bytes for trailing opaque adapter"
        } else {
            "bytes for opaque adapter"
        };
        let event = self.expect_event(expected)?;
        let input = match event.kind {
            ParseEventKind::Scalar(ScalarValue::Bytes(bytes)) => {
                if BORROW {
                    match bytes {
                        Cow::Borrowed(b) => OpaqueDeserialize::Borrowed(b),
                        Cow::Owned(v) => OpaqueDeserialize::Owned(v),
                    }
                } else {
                    OpaqueDeserialize::Owned(bytes.into_owned())
                }
            }
            _ => {
                return Err(self.mk_err(
                    &wip,
                    DeserializeErrorKind::UnexpectedToken {
                        expected,
                        got: event.kind_name().into(),
                    },
                ));
            }
        };

        let adapter = *adapter;
        #[allow(unsafe_code)]
        let wip = unsafe {
            wip.set_from_function(move |target| match (adapter.deserialize)(input, target) {
                Ok(_) => Ok(()),
                Err(message) => Err(ReflectErrorKind::OperationFailedOwned {
                    shape,
                    operation: format!("opaque adapter deserialize failed: {message}"),
                }),
            })?
        };
        Ok(wip)
    }

    #[inline(never)]
    fn unsupported_opaque_pointer(
        &self,
        shape: &'static Shape,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        Err(DeserializeErrorKind::Unsupported {
            message: format!(
                "cannot deserialize opaque type {} - add a proxy to make it deserializable",
                shape
            )
            .into(),
        }
        .with_span(self.last_span))
    }

    #[inline(never)]
    fn unsupported_strategy(
        &self,
        shape: &'static Shape,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        Err(DeserializeErrorKind::Unsupported {
            message: format!("unsupported deserialization strategy for {:?}", shape.def).into(),
        }
        .with_span(self.last_span))
    }

    #[inline(never)]
    fn missing_strategy(
        &self,
        shape: &'static Shape,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        Err(DeserializeErrorKind::Unsupported {
            message: format!(
                "missing deserialization strategy for shape: {:?} (TypePlan bug)",
                shape.def
            )
            .into(),
        }
        .with_span(self.last_span))
    }

    /// Deserialize a metadata container (like `Spanned<T>`, `Documented<T>`).
    ///
    /// These require special handling - the value field gets the data,
    /// metadata fields are populated from the passed `meta`.
    ///
    /// VariantTag events (like `@tag"hello"` in Styx) are already consumed by
    /// `deserialize_into` and passed down via `meta`.
    fn deserialize_metadata_container(
        &mut self,
        mut wip: Partial<'input, BORROW>,
        meta: MetaSource<'input>,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        // Check if this metadata container has a "tag" metadata field.
        // Only consume VariantTag events if the container can store them.
        // Otherwise, the VariantTag belongs to the inner value (e.g., an enum).
        let has_tag_field = if let Type::User(UserType::Struct(st)) = &wip.shape().ty {
            st.fields.iter().any(|f| f.metadata_kind() == Some("tag"))
        } else {
            false
        };

        // Check for VariantTag at the start - this handles tagged values like `@tag"hello"`.
        // We consume it here and merge it into meta, but ONLY if this container has a tag field.
        let event = self.expect_peek("value for metadata container")?;
        let (meta_owned, tag_span) =
            if has_tag_field && let ParseEventKind::VariantTag(tag) = &event.kind {
                let tag_span = event.span;
                let tag = tag.map(Cow::Borrowed);
                let _ = self.expect_event("variant tag")?; // consume it

                // Merge tag with any existing meta (preserving doc comments)
                let mut builder = ValueMeta::builder().span(tag_span);
                let existing_meta = match &meta {
                    MetaSource::Explicit(m) => Some(*m),
                    MetaSource::Owned(m) => Some(m),
                    MetaSource::FromEvents => None,
                };
                if let Some(existing) = existing_meta
                    && let Some(doc) = existing.doc()
                {
                    builder = builder.doc(doc.to_vec());
                }
                if let Some(tag) = tag {
                    builder = builder.tag(tag);
                }
                (Some(builder.build()), Some(tag_span))
            } else {
                (None, None)
            };

        // Resolve meta: use constructed meta from VariantTag, or explicit meta, or empty
        static EMPTY_META: ValueMeta<'static> = ValueMeta::empty();
        let meta: &ValueMeta<'_> = match (&meta_owned, &meta) {
            (Some(owned), _) => owned,
            (None, MetaSource::Explicit(explicit)) => explicit,
            (None, MetaSource::Owned(owned)) => owned,
            (None, MetaSource::FromEvents) => &EMPTY_META,
        };

        let shape = wip.shape();
        trace!(%shape, "deserialize_into: metadata container detected");

        // Deserialize the value field and track its span
        let mut value_span = Span::default();
        if let Type::User(UserType::Struct(st)) = &shape.ty {
            for field in st.fields {
                if field.metadata_kind().is_none() {
                    // This is the value field - recurse into it (fresh metadata from events)
                    wip = wip
                        .begin_field(field.effective_name())?
                        .with(|w| self.deserialize_into(w, MetaSource::FromEvents))?
                        .end()?;
                    value_span = self.last_span;
                    break;
                }
            }
        }

        // Compute the full span: if we have a tag span, extend from tag start to value end.
        // Otherwise, just use the value's span.
        let full_span = if let Some(tag_span) = tag_span {
            Span {
                offset: tag_span.offset,
                len: (value_span.offset + value_span.len).saturating_sub(tag_span.offset),
            }
        } else {
            value_span
        };

        // Populate metadata fields
        if let Type::User(UserType::Struct(st)) = &shape.ty {
            for field in st.fields {
                if let Some(kind) = field.metadata_kind() {
                    wip = wip.begin_field(field.effective_name())?;
                    wip = self.populate_metadata_field_with_span(wip, kind, meta, full_span)?;
                    wip = wip.end()?;
                }
            }
        }
        Ok(wip)
    }

    /// Populate a single metadata field on a metadata container.
    fn populate_metadata_field(
        &mut self,
        wip: Partial<'input, BORROW>,
        kind: &str,
        meta: &ValueMeta<'input>,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        self.populate_metadata_field_with_span(wip, kind, meta, self.last_span)
    }

    /// Populate a single metadata field on a metadata container with an explicit span.
    fn populate_metadata_field_with_span(
        &mut self,
        mut wip: Partial<'input, BORROW>,
        kind: &str,
        meta: &ValueMeta<'input>,
        span: Span,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        match kind {
            "span" => {
                // Check if the field is Option<Span> or just Span
                let is_option = matches!(wip.shape().def, Def::Option(_));
                if is_option {
                    wip = wip.begin_some()?;
                }
                wip = wip
                    .begin_field("offset")?
                    .set(span.offset)?
                    .end()?
                    .begin_field("len")?
                    .set(span.len)?
                    .end()?;
                if is_option {
                    wip = wip.end()?;
                }
            }
            "doc" => {
                if let Some(doc_lines) = meta.doc() {
                    // Set as Some(Vec<String>)
                    wip = wip.begin_some()?.init_list()?;
                    for line in doc_lines {
                        wip = wip
                            .begin_list_item()?
                            .with(|w| self.set_string_value(w, line.clone()))?
                            .end()?;
                    }
                    wip = wip.end()?;
                } else {
                    wip = wip.set_default()?;
                }
            }
            "tag" => {
                if let Some(tag_name) = meta.tag() {
                    wip = wip
                        .begin_some()?
                        .with(|w| self.set_string_value(w, tag_name.clone()))?
                        .end()?;
                } else {
                    wip = wip.set_default()?;
                }
            }
            _ => {
                // Unknown metadata kind - set to default
                wip = wip.set_default()?;
            }
        }
        Ok(wip)
    }

    /// Deserialize using an explicit source shape for parser hints.
    ///
    /// This walks `hint_shape` for control flow and parser hints, but builds
    /// into the `wip` Partial (which should be a DynamicValue like `Value`).
    pub fn deserialize_into_with_shape(
        &mut self,
        wip: Partial<'input, BORROW>,
        hint_shape: &'static Shape,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        self.deserialize_value_recursive(wip, hint_shape)
    }

    /// Internal recursive deserialization using hint_shape for dispatch.
    pub(crate) fn deserialize_value_recursive(
        &mut self,
        mut wip: Partial<'input, BORROW>,
        hint_shape: &'static Shape,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        // Handle Option
        if let Def::Option(opt_def) = &hint_shape.def {
            if self.is_non_self_describing() {
                self.parser.hint_option();
            }
            let event = self.expect_peek("value for option")?;
            // Treat both Null and Unit as None
            // Unit is used by Styx for tags without payload (e.g., @string vs @string{...})
            if matches!(
                event.kind,
                ParseEventKind::Scalar(ScalarValue::Null | ScalarValue::Unit)
            ) {
                let _ = self.expect_event("null or unit")?;
                wip = wip.set_default()?;
            } else if matches!(event.kind, ParseEventKind::OptionSome) {
                let _ = self.expect_event("option some")?;
                wip = wip
                    .begin_some()?
                    .with(|w| self.deserialize_value_recursive(w, opt_def.t))?
                    .end()?;
            } else {
                wip = self.deserialize_value_recursive(wip, opt_def.t)?;
            }
            return Ok(wip);
        }

        // Handle smart pointers - unwrap to inner type
        if let Def::Pointer(ptr_def) = &hint_shape.def
            && let Some(pointee) = ptr_def.pointee()
        {
            return self.deserialize_value_recursive(wip, pointee);
        }

        // Handle transparent wrappers (but not collections)
        if let Some(inner) = hint_shape.inner
            && !matches!(
                &hint_shape.def,
                Def::List(_) | Def::Map(_) | Def::Set(_) | Def::Array(_)
            )
        {
            return self.deserialize_value_recursive(wip, inner);
        }

        // Dispatch based on hint shape type
        match &hint_shape.ty {
            Type::User(UserType::Struct(struct_def)) => {
                if matches!(struct_def.kind, StructKind::Tuple | StructKind::TupleStruct) {
                    self.deserialize_tuple_dynamic(wip, struct_def.fields)
                } else {
                    self.deserialize_struct_dynamic(wip, struct_def.fields)
                }
            }
            Type::User(UserType::Enum(enum_def)) => self.deserialize_enum_dynamic(wip, enum_def),
            _ => match &hint_shape.def {
                Def::Scalar => self.deserialize_scalar_dynamic(wip, hint_shape),
                Def::List(list_def) => self.deserialize_list_dynamic(wip, list_def.t),
                Def::Array(array_def) => {
                    self.deserialize_array_dynamic(wip, array_def.t, array_def.n)
                }
                Def::Map(map_def) => self.deserialize_map_dynamic(wip, map_def.k, map_def.v),
                Def::Set(set_def) => self.deserialize_list_dynamic(wip, set_def.t),
                _ => Err(DeserializeErrorKind::Unsupported {
                    message: format!(
                        "unsupported hint shape for dynamic deserialization: {:?}",
                        hint_shape.def
                    )
                    .into(),
                }
                .with_span(self.last_span)),
            },
        }
    }

    pub(crate) fn deserialize_option(
        &mut self,
        mut wip: Partial<'input, BORROW>,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        let _guard = SpanGuard::new(self.last_span);

        // Hint to non-self-describing parsers that an Option is expected
        if self.is_non_self_describing() {
            self.parser.hint_option();
        }

        let event = self.expect_peek("value for option")?;

        // Treat both Null and Unit as None
        // Unit is used by Styx for tags without payload (e.g., @string vs @string{...})
        if matches!(
            event.kind,
            ParseEventKind::Scalar(ScalarValue::Null | ScalarValue::Unit)
        ) {
            // Consume the null/unit
            let _ = self.expect_event("null or unit")?;
            // Set to None (default)
            wip = wip.set_default()?;
        } else if matches!(event.kind, ParseEventKind::OptionSome) {
            let _ = self.expect_event("option some")?;
            wip = wip
                .begin_some()?
                .with(|w| self.deserialize_into_inner(w, MetaSource::FromEvents))?
                .end()?;
        } else {
            // Some(value)
            wip = wip
                .begin_some()?
                .with(|w| self.deserialize_into_inner(w, MetaSource::FromEvents))?
                .end()?;
        }
        Ok(wip)
    }

    pub(crate) fn deserialize_struct(
        &mut self,
        wip: Partial<'input, BORROW>,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        let struct_plan = wip.struct_plan().unwrap();
        if struct_plan.has_flatten {
            self.deserialize_struct_with_flatten(wip)
        } else {
            self.deserialize_struct_simple(wip)
        }
    }

    pub(crate) fn deserialize_tuple(
        &mut self,
        mut wip: Partial<'input, BORROW>,
        field_count: usize,
        is_single_field_transparent: bool,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        let _guard = SpanGuard::new(self.last_span);

        // Special case: transparent newtypes (marked with #[facet(transparent)] or
        // #[repr(transparent)]) can accept values directly without a sequence wrapper.
        // This enables patterns like:
        //   #[facet(transparent)]
        //   struct Wrapper(i32);
        //   toml: "value = 42"  ->  Wrapper(42)
        // Plain tuple structs without the transparent attribute use array syntax.
        //
        // IMPORTANT: This check must come BEFORE hint_struct_fields() because transparent
        // newtypes don't consume struct events - they deserialize the inner value directly.
        // If we hint struct fields first, non-self-describing parsers will expect to emit
        // StructStart, causing "unexpected token: got struct start" errors.
        if is_single_field_transparent {
            // Unwrap into field 0 and deserialize directly
            return Ok(wip
                .begin_nth_field(0)?
                .with(|w| self.deserialize_into(w, MetaSource::FromEvents))?
                .end()?);
        }

        // Hint to non-self-describing parsers how many fields to expect
        // Tuples are like positional structs, so we use hint_struct_fields
        if self.is_non_self_describing() {
            self.parser.hint_struct_fields(field_count);
        }

        // Special case: unit type () can accept Scalar(Unit) or Scalar(Null) directly
        // This enables patterns like styx bare identifiers: { id, name } -> IndexMap<String, ()>
        // and JSON null values for unit types (e.g., ConfigValue::Null(Spanned<()>))
        if field_count == 0 {
            let peeked = self.expect_peek("value")?;
            if matches!(
                peeked.kind,
                ParseEventKind::Scalar(ScalarValue::Unit | ScalarValue::Null)
            ) {
                self.expect_event("value")?; // consume the unit/null scalar
                return Ok(wip);
            }
        }

        let event = self.expect_event("value")?;

        // Accept either SequenceStart (JSON arrays) or StructStart (for
        // non-self-describing formats like postcard where tuples are positional structs)
        let struct_mode = match event.kind {
            ParseEventKind::SequenceStart(_) => false,
            // For non-self-describing formats, StructStart(Object) is valid for tuples
            // because hint_struct_fields was called and tuples are positional structs
            ParseEventKind::StructStart(_) if !self.parser.is_self_describing() => true,
            // For self-describing formats like TOML/JSON, objects with numeric keys
            // (e.g., { "0" = true, "1" = 1 }) are valid tuple representations
            ParseEventKind::StructStart(ContainerKind::Object) => true,
            ParseEventKind::StructStart(kind) => {
                return Err(DeserializeError {
                    span: Some(self.last_span),
                    path: Some(wip.path()),
                    kind: DeserializeErrorKind::UnexpectedToken {
                        expected: "array",
                        got: kind.name().into(),
                    },
                });
            }
            _ => {
                return Err(DeserializeError {
                    span: Some(self.last_span),
                    path: Some(wip.path()),
                    kind: DeserializeErrorKind::UnexpectedToken {
                        expected: "sequence start for tuple",
                        got: event.kind_name().into(),
                    },
                });
            }
        };

        let mut index = 0usize;
        loop {
            let event = self.expect_peek("value")?;

            // Check for end of container
            if matches!(
                event.kind,
                ParseEventKind::SequenceEnd | ParseEventKind::StructEnd
            ) {
                self.expect_event("value")?;
                break;
            }

            // In struct mode, skip FieldKey events
            if struct_mode && matches!(event.kind, ParseEventKind::FieldKey(_)) {
                self.expect_event("value")?;
                continue;
            }

            // Select field by index
            wip = wip
                .begin_nth_field(index)?
                .with(|w| self.deserialize_into(w, MetaSource::FromEvents))?
                .end()?;
            index += 1;
        }

        Ok(wip)
    }

    /// Helper to collect field evidence using save/restore.
    ///
    /// This saves the deserializer state (parser position AND event buffer),
    /// reads through the current struct to collect field names and their scalar
    /// values, then restores the state.
    pub(crate) fn collect_evidence(
        &mut self,
    ) -> Result<Vec<FieldEvidence<'input>>, DeserializeError> {
        let save_point = self.save();

        let mut evidence = Vec::new();
        let mut depth = 0i32;
        let mut pending_field_name: Option<Cow<'input, str>> = None;

        // Read through the structure
        while let Ok(event) = self.expect_event("evidence") {
            match event.kind {
                ParseEventKind::StructStart(_) => {
                    depth += 1;
                    // If we were expecting a value, record field with no scalar
                    if depth > 1
                        && let Some(name) = pending_field_name.take()
                    {
                        evidence.push(FieldEvidence {
                            name,
                            location: FieldLocationHint::KeyValue,
                            value_type: None,
                            scalar_value: None,
                        });
                    }
                }
                ParseEventKind::StructEnd => {
                    depth -= 1;
                    if depth == 0 {
                        break;
                    }
                }
                ParseEventKind::SequenceStart(_) => {
                    depth += 1;
                    // If we were expecting a value, record field with no scalar
                    if let Some(name) = pending_field_name.take() {
                        evidence.push(FieldEvidence {
                            name,
                            location: FieldLocationHint::KeyValue,
                            value_type: None,
                            scalar_value: None,
                        });
                    }
                }
                ParseEventKind::SequenceEnd => {
                    depth -= 1;
                }
                ParseEventKind::FieldKey(key) => {
                    // If there's a pending field, record it without a value
                    if let Some(name) = pending_field_name.take() {
                        evidence.push(FieldEvidence {
                            name,
                            location: FieldLocationHint::KeyValue,
                            value_type: None,
                            scalar_value: None,
                        });
                    }
                    if depth == 1 {
                        // Top-level field - save name, wait for value
                        pending_field_name = key.name().cloned();
                    }
                }
                ParseEventKind::Scalar(scalar) => {
                    if let Some(name) = pending_field_name.take() {
                        // Record field with its scalar value
                        evidence.push(FieldEvidence {
                            name,
                            location: FieldLocationHint::KeyValue,
                            value_type: None,
                            scalar_value: Some(scalar),
                        });
                    }
                }
                ParseEventKind::OrderedField
                | ParseEventKind::OptionSome
                | ParseEventKind::VariantTag(_) => {}
            }
        }

        // Handle any remaining pending field
        if let Some(name) = pending_field_name.take() {
            evidence.push(FieldEvidence {
                name,
                location: FieldLocationHint::KeyValue,
                value_type: None,
                scalar_value: None,
            });
        }

        self.restore(save_point);
        Ok(evidence)
    }

    pub(crate) fn deserialize_list(
        &mut self,
        mut wip: Partial<'input, BORROW>,
        is_byte_vec: bool,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        trace!("deserialize_list: starting");

        // Try the optimized byte sequence path for Vec<u8>
        // (is_byte_vec is precomputed in TypePlan)
        if is_byte_vec && self.parser.hint_byte_sequence() {
            // Parser supports bulk byte reading - expect Scalar(Bytes(...))
            let event = self.expect_event("bytes")?;
            trace!(?event, "deserialize_list: got bytes event");

            return match event.kind {
                ParseEventKind::Scalar(ScalarValue::Bytes(bytes)) => {
                    self.set_bytes_value(wip, bytes)
                }
                _ => Err(DeserializeError {
                    span: Some(self.last_span),
                    path: Some(wip.path()),
                    kind: DeserializeErrorKind::UnexpectedToken {
                        expected: "bytes",
                        got: event.kind_name().into(),
                    },
                }),
            };
        }

        // Fallback: element-by-element deserialization. Most self-describing
        // parsers ignore this, but formats with ambiguous container syntax
        // (for example Lua's `{}`) can use it to disambiguate empty sequences.
        self.parser.hint_sequence();

        let event = self.expect_event("value")?;
        trace!(?event, "deserialize_list: got container start event");

        // Expect SequenceStart for lists
        match event.kind {
            ParseEventKind::SequenceStart(_) => {
                trace!("deserialize_list: got sequence start");
            }
            ParseEventKind::StructStart(kind) => {
                return Err(DeserializeError {
                    span: Some(self.last_span),
                    path: Some(wip.path()),
                    kind: DeserializeErrorKind::UnexpectedToken {
                        expected: "array",
                        got: kind.name().into(),
                    },
                });
            }
            _ => {
                return Err(DeserializeError {
                    span: Some(self.last_span),
                    path: Some(wip.path()),
                    kind: DeserializeErrorKind::UnexpectedToken {
                        expected: "sequence start",
                        got: event.kind_name().into(),
                    },
                });
            }
        };

        // Count buffered items to pre-reserve capacity
        let capacity_hint = self.count_buffered_sequence_items();
        trace!("deserialize_list: capacity hint = {capacity_hint}");

        // Initialize the list with capacity hint
        wip = wip.init_list_with_capacity(capacity_hint)?;
        trace!("deserialize_list: initialized list, starting loop");

        loop {
            let event = self.expect_peek("value")?;
            trace!(?event, "deserialize_list: loop iteration");

            // Check for end of sequence
            if matches!(event.kind, ParseEventKind::SequenceEnd) {
                self.expect_event("value")?;
                trace!("deserialize_list: reached end of sequence");
                break;
            }

            trace!("deserialize_list: deserializing list item");
            wip = wip
                .begin_list_item()?
                .with(|w| self.deserialize_into(w, MetaSource::FromEvents))?
                .end()?;
        }

        trace!("deserialize_list: completed");
        Ok(wip)
    }

    pub(crate) fn deserialize_array(
        &mut self,
        mut wip: Partial<'input, BORROW>,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        let _guard = SpanGuard::new(self.last_span);
        // Get the fixed array length from the type definition
        let array_len = match &wip.shape().def {
            Def::Array(array_def) => array_def.n,
            _ => {
                return Err(DeserializeErrorKind::UnexpectedToken {
                    expected: "array",
                    got: format!("{:?}", wip.shape().def).into(),
                }
                .with_span(self.last_span));
            }
        };

        // Hint that a fixed-size array is expected. Most self-describing parsers
        // ignore this, but formats with ambiguous container syntax can use it to
        // disambiguate empty arrays.
        self.parser.hint_array(array_len);

        let event = self.expect_event("value")?;

        // Expect SequenceStart for arrays
        match event.kind {
            ParseEventKind::SequenceStart(_) => {}
            ParseEventKind::StructStart(kind) => {
                return Err(DeserializeError {
                    span: Some(self.last_span),
                    path: Some(wip.path()),
                    kind: DeserializeErrorKind::UnexpectedToken {
                        expected: "array",
                        got: kind.name().into(),
                    },
                });
            }
            _ => {
                return Err(DeserializeError {
                    span: Some(self.last_span),
                    path: Some(wip.path()),
                    kind: DeserializeErrorKind::UnexpectedToken {
                        expected: "sequence start for array",
                        got: event.kind_name().into(),
                    },
                });
            }
        };

        // Transition to Array tracker state. This is important for empty arrays
        // like [u8; 0] which have no elements to initialize but still need
        // their tracker state set correctly for require_full_initialization to pass.
        wip = wip.init_array()?;

        let mut index = 0usize;
        loop {
            let event = self.expect_peek("value")?;

            // Check for end of sequence
            if matches!(event.kind, ParseEventKind::SequenceEnd) {
                self.expect_event("value")?;
                break;
            }

            wip = wip
                .begin_nth_field(index)?
                .with(|w| self.deserialize_into(w, MetaSource::FromEvents))?
                .end()?;
            index += 1;
        }

        Ok(wip)
    }

    pub(crate) fn deserialize_set(
        &mut self,
        mut wip: Partial<'input, BORROW>,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        let _guard = SpanGuard::new(self.last_span);

        // Hint that a set is represented by a sequence. Most self-describing
        // parsers ignore this, but formats with ambiguous container syntax can
        // use it to disambiguate empty sets.
        self.parser.hint_sequence();

        let event = self.expect_event("value")?;

        // Expect SequenceStart for sets
        match event.kind {
            ParseEventKind::SequenceStart(_) => {}
            ParseEventKind::StructStart(kind) => {
                return Err(DeserializeError {
                    span: Some(self.last_span),
                    path: Some(wip.path()),
                    kind: DeserializeErrorKind::UnexpectedToken {
                        expected: "set",
                        got: kind.name().into(),
                    },
                });
            }
            _ => {
                return Err(DeserializeError {
                    span: Some(self.last_span),
                    path: Some(wip.path()),
                    kind: DeserializeErrorKind::UnexpectedToken {
                        expected: "sequence start for set",
                        got: event.kind_name().into(),
                    },
                });
            }
        };

        // Initialize the set
        wip = wip.init_set()?;

        loop {
            let event = self.expect_peek("value")?;

            // Check for end of sequence
            if matches!(event.kind, ParseEventKind::SequenceEnd) {
                self.expect_event("value")?;
                break;
            }

            wip = wip
                .begin_set_item()?
                .with(|w| self.deserialize_into(w, MetaSource::FromEvents))?
                .end()?;
        }

        Ok(wip)
    }

    pub(crate) fn deserialize_map(
        &mut self,
        mut wip: Partial<'input, BORROW>,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        let _guard = SpanGuard::new(self.last_span);

        // For non-self-describing formats, hint that a map is expected
        if self.is_non_self_describing() {
            self.parser.hint_map();
        }

        let event = self.expect_event("value")?;

        // Initialize the map
        wip = wip.init_map()?;

        // Handle both self-describing (StructStart) and non-self-describing (SequenceStart) formats
        match event.kind {
            ParseEventKind::StructStart(_) => {
                // Self-describing format (e.g., JSON): maps are represented as objects
                loop {
                    let event = self.expect_event("value")?;
                    match event.kind {
                        ParseEventKind::StructEnd => break,
                        ParseEventKind::FieldKey(key) => {
                            // Begin key
                            wip = wip
                                .begin_key()?
                                .with(|w| {
                                    self.deserialize_map_key(w, key.name().cloned(), key.meta())
                                })?
                                .end()?;

                            // Begin value
                            wip = wip
                                .begin_value()?
                                .with(|w| self.deserialize_into(w, MetaSource::FromEvents))?
                                .end()?;
                        }
                        _ => {
                            return Err(DeserializeError {
                                span: Some(self.last_span),
                                path: Some(wip.path()),
                                kind: DeserializeErrorKind::UnexpectedToken {
                                    expected: "field key or struct end for map",
                                    got: event.kind_name().into(),
                                },
                            });
                        }
                    }
                }
            }
            ParseEventKind::SequenceStart(_) => {
                // Non-self-describing format (e.g., postcard): maps are sequences of key-value pairs
                loop {
                    let event = self.expect_peek("value")?;
                    match event.kind {
                        ParseEventKind::SequenceEnd => {
                            self.expect_event("value")?;
                            break;
                        }
                        ParseEventKind::OrderedField => {
                            self.expect_event("value")?;

                            // Deserialize key
                            wip = wip
                                .begin_key()?
                                .with(|w| self.deserialize_into(w, MetaSource::FromEvents))?
                                .end()?;

                            // Deserialize value
                            wip = wip
                                .begin_value()?
                                .with(|w| self.deserialize_into(w, MetaSource::FromEvents))?
                                .end()?;
                        }
                        _ => {
                            return Err(DeserializeError {
                                span: Some(self.last_span),
                                path: Some(wip.path()),
                                kind: DeserializeErrorKind::UnexpectedToken {
                                    expected: "ordered field or sequence end for map",
                                    got: event.kind_name().into(),
                                },
                            });
                        }
                    }
                }
            }
            _ => {
                return Err(DeserializeError {
                    span: Some(self.last_span),
                    path: Some(wip.path()),
                    kind: DeserializeErrorKind::UnexpectedToken {
                        expected: "struct start or sequence start for map",
                        got: event.kind_name().into(),
                    },
                });
            }
        }

        Ok(wip)
    }

    pub(crate) fn deserialize_scalar(
        &mut self,
        mut wip: Partial<'input, BORROW>,
        scalar_type: Option<ScalarType>,
        is_from_str: bool,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        // Only hint for non-self-describing formats (e.g., postcard)
        // Self-describing formats like JSON already know the types
        if self.is_non_self_describing() {
            let shape = wip.shape();

            // First, try hint_opaque_scalar for types that may have format-specific
            // binary representations (e.g., UUID as 16 raw bytes in postcard)
            let opaque_handled = if scalar_type.is_some() {
                // Standard primitives are never opaque
                false
            } else {
                // For all other scalar types, ask the parser if it handles them specially
                // TODO: Consider using shape.id instead of type_identifier for faster matching
                self.parser.hint_opaque_scalar(shape.type_identifier, shape)
            };

            // If the parser didn't handle the opaque type, fall back to standard hints
            if !opaque_handled {
                // Use precomputed is_from_str instead of runtime vtable check
                let hint = scalar_type_to_hint(scalar_type).or(if is_from_str {
                    Some(ScalarTypeHint::String)
                } else {
                    None
                });
                if let Some(hint) = hint {
                    self.parser.hint_scalar_type(hint);
                }
            }
        }

        let event = self.expect_event("value")?;

        match event.kind {
            ParseEventKind::Scalar(scalar) => {
                wip = self.set_scalar(wip, scalar)?;
                Ok(wip)
            }
            ParseEventKind::StructStart(_container_kind) => {
                // When deserializing into a scalar, extract the _arg value.
                let mut found_scalar: Option<ScalarValue<'input>> = None;

                loop {
                    let inner_event = self.expect_event("field or struct end")?;
                    match inner_event.kind {
                        ParseEventKind::StructEnd => break,
                        ParseEventKind::FieldKey(key) => {
                            // Look for _arg field (single argument)
                            if key.name().map(|c| c.as_ref()) == Some("_arg") {
                                let value_event = self.expect_event("argument value")?;
                                if let ParseEventKind::Scalar(scalar) = value_event.kind {
                                    found_scalar = Some(scalar);
                                } else {
                                    // Skip non-scalar argument
                                    self.skip_value()?;
                                }
                            } else {
                                // Skip other fields (_node_name, _arguments, properties, etc.)
                                self.skip_value()?;
                            }
                        }
                        _ => {
                            // Skip unexpected events
                        }
                    }
                }

                if let Some(scalar) = found_scalar {
                    wip = self.set_scalar(wip, scalar)?;
                    Ok(wip)
                } else {
                    Err(DeserializeError {
                        span: Some(self.last_span),
                        path: Some(wip.path()),
                        kind: DeserializeErrorKind::UnexpectedToken {
                            expected: "scalar value or node with argument",
                            got: "node without argument".into(),
                        },
                    })
                }
            }
            _ => Err(DeserializeError {
                span: Some(self.last_span),
                path: Some(wip.path()),
                kind: DeserializeErrorKind::UnexpectedToken {
                    expected: "scalar value",
                    got: event.kind_name().into(),
                },
            }),
        }
    }

    /// Deserialize a map key from a string or tag.
    ///
    /// Format parsers typically emit string keys, but the target map might have non-string key types
    /// (e.g., integers, enums). This function parses the string key into the appropriate type:
    /// - String types: set directly
    /// - Enum unit variants: use select_variant_named
    /// - Integer types: parse the string as a number
    /// - Transparent newtypes: descend into the inner type
    /// - Option types: None key becomes None, Some(key) recurses into inner type
    /// - Metadata containers (like `Documented<T>`): populate doc/tag metadata and recurse into value
    ///
    /// The `meta.tag` is for formats like Styx where keys can be type patterns (e.g., `@string`).
    /// When present, it indicates the key was a tag rather than a bare identifier.
    pub(crate) fn deserialize_map_key(
        &mut self,
        mut wip: Partial<'input, BORROW>,
        key: Option<Cow<'input, str>>,
        meta: Option<&ValueMeta<'input>>,
    ) -> Result<Partial<'input, BORROW>, DeserializeError> {
        let _guard = SpanGuard::new(self.last_span);
        let shape = wip.shape();

        trace!(shape_name = %shape, shape_def = ?shape.def, ?key, ?meta, "deserialize_map_key");

        let format_ns = self.parser.format_namespace();
        let (next_wip, began_proxy) =
            wip.begin_custom_deserialization_from_shape_with_format(format_ns)?;
        if began_proxy {
            return Ok(next_wip
                .with(|w| self.deserialize_map_key(w, key, meta))?
                .end()?);
        }
        wip = next_wip;

        // Handle metadata containers (like `Documented<T>` or `ObjectKey`): populate metadata and recurse into value
        if shape.is_metadata_container() {
            trace!("deserialize_map_key: metadata container detected");
            let empty_meta = ValueMeta::default();
            let meta = meta.unwrap_or(&empty_meta);

            // Find field info from the shape's struct type
            if let Type::User(UserType::Struct(st)) = &shape.ty {
                for field in st.fields {
                    match field.metadata_kind() {
                        Some(kind) => {
                            wip = wip.begin_field(field.effective_name())?;
                            wip = self.populate_metadata_field(wip, kind, meta)?;
                            wip = wip.end()?;
                        }
                        None => {
                            // This is the value field - recurse with the key and tag.
                            // Doc is already consumed by this container, but tag may be needed
                            // by a nested metadata container (e.g., Documented<ObjectKey>).
                            let inner_meta =
                                ValueMeta::builder().maybe_tag(meta.tag().cloned()).build();
                            wip = wip
                                .begin_field(field.effective_name())?
                                .with(|w| {
                                    self.deserialize_map_key(w, key.clone(), Some(&inner_meta))
                                })?
                                .end()?;
                        }
                    }
                }
            }

            return Ok(wip);
        }

        // Handle Option<T> key types: None key -> None variant, Some(key) -> Some(inner)
        if let Def::Option(_) = &shape.def {
            match key {
                None => {
                    // Unit key -> None variant (use set_default to mark as initialized)
                    wip = wip.set_default()?;
                    return Ok(wip);
                }
                Some(inner_key) => {
                    // Named key -> Some(inner)
                    return Ok(wip
                        .begin_some()?
                        .with(|w| self.deserialize_map_key(w, Some(inner_key), None))?
                        .end()?);
                }
            }
        }

        // From here on, we need an actual key name.
        // For tagged keys (e.g., @schema in Styx), use the tag (with @ prefix) as the key.
        let key = key
            .or_else(|| {
                meta.and_then(|m| m.tag())
                    .filter(|t| !t.is_empty())
                    .map(|t| Cow::Owned(format!("@{}", t)))
            })
            .ok_or_else(|| DeserializeError {
                span: Some(self.last_span),
                path: Some(wip.path()),
                kind: DeserializeErrorKind::UnexpectedToken {
                    expected: "named key",
                    got: "unit key".into(),
                },
            })?;

        // For transparent types (like UserId(String)), we need to use begin_inner
        // to set the inner value. But NOT for pointer types like &str or Cow<str>
        // which are handled directly.
        let is_pointer = matches!(shape.def, Def::Pointer(_));
        if shape.inner.is_some() && !is_pointer {
            return Ok(wip
                .begin_inner()?
                .with(|w| self.deserialize_map_key(w, Some(key), None))?
                .end()?);
        }

        // Handle terminal cases (enum, numeric, string) via non-generic inner function
        use crate::deserializer::setters::{
            MapKeyTerminalResult, deserialize_map_key_terminal_inner,
        };
        match deserialize_map_key_terminal_inner(wip, key, self.last_span) {
            Ok(wip) => Ok(wip),
            Err(MapKeyTerminalResult::NeedsSetString { wip, s }) => self.set_string_value(wip, s),
            Err(MapKeyTerminalResult::Error(e)) => Err(e),
        }
    }
}

/// Convert a ScalarType to a ScalarTypeHint for non-self-describing parsers.
///
/// Returns None for types that don't have a direct hint mapping (Unit, CowStr,
/// network addresses, ConstTypeId).
#[inline]
fn scalar_type_to_hint(scalar_type: Option<ScalarType>) -> Option<ScalarTypeHint> {
    match scalar_type? {
        ScalarType::Bool => Some(ScalarTypeHint::Bool),
        ScalarType::U8 => Some(ScalarTypeHint::U8),
        ScalarType::U16 => Some(ScalarTypeHint::U16),
        ScalarType::U32 => Some(ScalarTypeHint::U32),
        ScalarType::U64 => Some(ScalarTypeHint::U64),
        ScalarType::U128 => Some(ScalarTypeHint::U128),
        ScalarType::USize => Some(ScalarTypeHint::Usize),
        ScalarType::I8 => Some(ScalarTypeHint::I8),
        ScalarType::I16 => Some(ScalarTypeHint::I16),
        ScalarType::I32 => Some(ScalarTypeHint::I32),
        ScalarType::I64 => Some(ScalarTypeHint::I64),
        ScalarType::I128 => Some(ScalarTypeHint::I128),
        ScalarType::ISize => Some(ScalarTypeHint::Isize),
        ScalarType::F32 => Some(ScalarTypeHint::F32),
        ScalarType::F64 => Some(ScalarTypeHint::F64),
        ScalarType::Char => Some(ScalarTypeHint::Char),
        ScalarType::Str | ScalarType::String => Some(ScalarTypeHint::String),
        // Types that need special handling or FromStr fallback
        _ => None,
    }
}