wasmtime-cli 46.0.0

use super::{async_functions::CountPending, gc_store, ref_types_module};
use std::num::NonZeroU32;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst};
use std::task::{Context, Poll};
use wasmtime::*;

struct SetFlagOnDrop(Arc<AtomicBool>);

impl Drop for SetFlagOnDrop {
    fn drop(&mut self) {
        self.0.store(true, SeqCst);
    }
}

#[test]
#[cfg_attr(miri, ignore)]
fn smoke_test_gc_no_epochs() -> Result<()> {
    smoke_test_gc_impl(false)
}

#[test]
#[cfg_attr(miri, ignore)]
fn smoke_test_gc_yes_epochs() -> Result<()> {
    smoke_test_gc_impl(true)
}

fn smoke_test_gc_impl(use_epochs: bool) -> Result<()> {
    let (mut store, module) = ref_types_module(
        use_epochs,
        r#"
            (module
                (import "" "" (func $do_gc))
                (func $recursive (export "func") (param i32 externref) (result externref)
                    local.get 0
                    i32.eqz
                    if (result externref)
                        call $do_gc
                        local.get 1
                    else
                        local.get 0
                        i32.const 1
                        i32.sub
                        local.get 1
                        call $recursive
                    end
                )
            )
        "#,
    )?;

    let do_gc = Func::wrap(&mut store, |mut caller: Caller<'_, _>| {
        // Do a GC with `externref`s on the stack in Wasm frames.
        caller.gc(None)
    });
    let instance = Instance::new(&mut store, &module, &[do_gc.into()])?;
    let func = instance.get_func(&mut store, "func").unwrap();

    let inner_dropped = Arc::new(AtomicBool::new(false));

    {
        let mut scope = RootScope::new(&mut store);

        let r = ExternRef::new(&mut scope, SetFlagOnDrop(inner_dropped.clone()))?;
        {
            let args = [Val::I32(5), Val::ExternRef(Some(r))];
            func.call(&mut scope, &args, &mut [Val::I32(0)])?;
        }

        // Doing a GC should see that there aren't any `externref`s on the stack in
        // Wasm frames anymore.
        scope.as_context_mut().gc(None)?;

        // But the scope should still be rooting `r`.
        assert!(!inner_dropped.load(SeqCst));
    }

    // Exiting the scope and unrooting `r` should have dropped the inner
    // `SetFlagOnDrop` value.
    store.gc(None)?;
    assert!(inner_dropped.load(SeqCst));

    Ok(())
}

struct CountDrops(Arc<AtomicUsize>);

impl Drop for CountDrops {
    fn drop(&mut self) {
        self.0.fetch_add(1, SeqCst);
    }
}

#[test]
#[cfg_attr(miri, ignore)]
fn wasm_dropping_refs() -> Result<()> {
    let (mut store, module) = ref_types_module(
        false,
        r#"
            (module
                (func (export "drop_ref") (param externref)
                    nop
                )
            )
        "#,
    )?;

    let instance = Instance::new(&mut store, &module, &[])?;
    let drop_ref = instance.get_func(&mut store, "drop_ref").unwrap();

    let num_refs_dropped = Arc::new(AtomicUsize::new(0));

    for _ in 0..4096 {
        let mut scope = RootScope::new(&mut store);
        let r = ExternRef::new(&mut scope, CountDrops(num_refs_dropped.clone()))?;
        let args = [Val::ExternRef(Some(r))];
        drop_ref.call(&mut scope, &args, &mut [])?;
    }

    // After doing a GC, all the refs should have been dropped.
    store.gc(None)?;
    assert_eq!(num_refs_dropped.load(SeqCst), 4096);

    return Ok(());
}

#[test]
#[cfg_attr(miri, ignore)]
fn many_live_refs() -> Result<()> {
    let mut wat = r#"
        (module
            ;; Make new `externref`s.
            (import "" "make_ref" (func $make_ref (result externref)))

            ;; Observe an `externref` so it is kept live.
            (import "" "observe_ref" (func $observe_ref (param externref)))

            (func (export "many_live_refs")
    "#
    .to_string();

    // This is more than the initial `VMExternRefActivationsTable` capacity, so
    // it will need to allocate additional bump chunks.
    const NUM_LIVE_REFS: usize = 1024;

    // Push `externref`s onto the stack.
    for _ in 0..NUM_LIVE_REFS {
        wat.push_str("(call $make_ref)\n");
    }

    // Pop `externref`s from the stack. Because we pass each of them to a
    // function call here, they are all live references for the duration of
    // their lifetimes.
    for _ in 0..NUM_LIVE_REFS {
        wat.push_str("(call $observe_ref)\n");
    }

    wat.push_str(
        "
            ) ;; func
        ) ;; module
        ",
    );

    let (mut store, module) = ref_types_module(false, &wat)?;

    let live_refs = Arc::new(AtomicUsize::new(0));

    let make_ref = Func::wrap(&mut store, {
        let live_refs = live_refs.clone();
        move |mut caller: Caller<'_, _>| {
            Ok(Some(ExternRef::new(
                &mut caller,
                CountLiveRefs::new(live_refs.clone()),
            )?))
        }
    });

    let observe_ref = Func::wrap(
        &mut store,
        |caller: Caller<'_, _>, r: Option<Rooted<ExternRef>>| {
            let r = r
                .unwrap()
                .data(&caller)
                .unwrap()
                .unwrap()
                .downcast_ref::<CountLiveRefs>()
                .unwrap();
            assert!(r.live_refs.load(SeqCst) > 0);
        },
    );

    let instance = Instance::new(&mut store, &module, &[make_ref.into(), observe_ref.into()])?;
    let many_live_refs = instance.get_func(&mut store, "many_live_refs").unwrap();

    many_live_refs.call(&mut store, &[], &mut [])?;

    store.as_context_mut().gc(None)?;
    assert_eq!(live_refs.load(SeqCst), 0);

    return Ok(());

    struct CountLiveRefs {
        live_refs: Arc<AtomicUsize>,
    }

    impl CountLiveRefs {
        fn new(live_refs: Arc<AtomicUsize>) -> Self {
            live_refs.fetch_add(1, SeqCst);
            Self { live_refs }
        }
    }

    impl Drop for CountLiveRefs {
        fn drop(&mut self) {
            self.live_refs.fetch_sub(1, SeqCst);
        }
    }
}

#[test]
#[cfg_attr(miri, ignore)]
fn drop_externref_via_table_set() -> Result<()> {
    let (mut store, module) = ref_types_module(
        false,
        r#"
            (module
                (table $t 1 externref)

                (func (export "table-set") (param externref)
                  (table.set $t (i32.const 0) (local.get 0))
                )
            )
        "#,
    )?;

    let instance = Instance::new(&mut store, &module, &[])?;
    let table_set = instance.get_func(&mut store, "table-set").unwrap();

    let foo_is_dropped = Arc::new(AtomicBool::new(false));
    let bar_is_dropped = Arc::new(AtomicBool::new(false));

    {
        let mut scope = RootScope::new(&mut store);

        let foo = ExternRef::new(&mut scope, SetFlagOnDrop(foo_is_dropped.clone()))?;
        let bar = ExternRef::new(&mut scope, SetFlagOnDrop(bar_is_dropped.clone()))?;

        {
            let args = vec![Val::ExternRef(Some(foo))];
            table_set.call(&mut scope, &args, &mut [])?;
        }

        scope.as_context_mut().gc(None)?;
        assert!(!foo_is_dropped.load(SeqCst));
        assert!(!bar_is_dropped.load(SeqCst));

        {
            let args = vec![Val::ExternRef(Some(bar))];
            table_set.call(&mut scope, &args, &mut [])?;
        }
    }

    store.gc(None)?;
    assert!(foo_is_dropped.load(SeqCst));
    assert!(!bar_is_dropped.load(SeqCst));

    table_set.call(&mut store, &[Val::ExternRef(None)], &mut [])?;
    store.gc(None)?;
    assert!(foo_is_dropped.load(SeqCst));
    assert!(bar_is_dropped.load(SeqCst));

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn global_drops_externref() -> Result<()> {
    let _ = env_logger::try_init();
    test_engine(&Engine::default())?;

    let mut config = Config::new();
    config.allocation_strategy(crate::small_pool_config());
    test_engine(&Engine::new(&config)?)?;

    return Ok(());

    fn test_engine(engine: &Engine) -> Result<()> {
        let mut store = Store::new(&engine, ());
        let flag = Arc::new(AtomicBool::new(false));
        let externref = ExternRef::new(&mut store, SetFlagOnDrop(flag.clone()))?;
        Global::new(
            &mut store,
            GlobalType::new(ValType::EXTERNREF, Mutability::Const),
            externref.into(),
        )?;
        drop(store);
        assert!(flag.load(SeqCst));

        let mut store = Store::new(&engine, ());
        let module = Module::new(
            &engine,
            r#"
                (module
                    (global (mut externref) (ref.null extern))

                    (func (export "run") (param externref)
                        local.get 0
                        global.set 0
                    )
                )
            "#,
        )?;
        let instance = Instance::new(&mut store, &module, &[])?;
        let run = instance.get_typed_func::<Option<Rooted<ExternRef>>, ()>(&mut store, "run")?;
        let flag = Arc::new(AtomicBool::new(false));
        let externref = ExternRef::new(&mut store, SetFlagOnDrop(flag.clone()))?;
        run.call(&mut store, Some(externref))?;
        drop(store);
        assert!(flag.load(SeqCst));
        Ok(())
    }
}

#[test]
#[cfg_attr(miri, ignore)]
fn table_drops_externref() -> Result<()> {
    let _ = env_logger::try_init();
    test_engine(&Engine::default())?;

    let mut config = Config::new();
    config.allocation_strategy(crate::small_pool_config());
    test_engine(&Engine::new(&config)?)?;

    return Ok(());

    fn test_engine(engine: &Engine) -> Result<()> {
        let mut store = Store::new(&engine, ());
        let flag = Arc::new(AtomicBool::new(false));
        let externref = ExternRef::new(&mut store, SetFlagOnDrop(flag.clone()))?;
        Table::new(
            &mut store,
            TableType::new(RefType::EXTERNREF, 1, None),
            externref.into(),
        )?;
        drop(store);
        assert!(flag.load(SeqCst));

        let mut store = Store::new(&engine, ());
        let module = Module::new(
            &engine,
            r#"
            (module
                (table 1 externref)

                (func (export "run") (param externref)
                    i32.const 0
                    local.get 0
                    table.set 0
                )
            )
        "#,
        )?;
        let instance = Instance::new(&mut store, &module, &[])?;
        let run = instance.get_typed_func::<Option<Rooted<ExternRef>>, ()>(&mut store, "run")?;
        let flag = Arc::new(AtomicBool::new(false));
        let externref = ExternRef::new(&mut store, SetFlagOnDrop(flag.clone()))?;
        run.call(&mut store, Some(externref))?;
        drop(store);
        assert!(flag.load(SeqCst));
        Ok(())
    }
}

#[test]
#[cfg_attr(miri, ignore)]
fn global_init_no_leak() -> Result<()> {
    let (mut store, module) = ref_types_module(
        false,
        r#"
            (module
                (import "" "" (global externref))
                (global externref (global.get 0))
            )
        "#,
    )?;

    let flag = Arc::new(AtomicBool::new(false));
    let externref = ExternRef::new(&mut store, SetFlagOnDrop(flag.clone()))?;
    let global = Global::new(
        &mut store,
        GlobalType::new(ValType::EXTERNREF, Mutability::Const),
        externref.into(),
    )?;
    Instance::new(&mut store, &module, &[global.into()])?;
    drop(store);
    assert!(flag.load(SeqCst));

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn no_gc_middle_of_args() -> Result<()> {
    let (mut store, module) = ref_types_module(
        false,
        r#"
            (module
                (import "" "return_some" (func $return (result externref externref externref)))
                (import "" "take_some" (func $take (param externref externref externref)))
                (func (export "run")
                    (local i32)
                    i32.const 1000
                    local.set 0
                    loop
                        call $return
                        call $take
                        local.get 0
                        i32.const -1
                        i32.add
                        local.tee 0
                        br_if 0
                    end
                )
            )
        "#,
    )?;

    let mut linker = Linker::new(store.engine());
    linker.func_wrap("", "return_some", |mut caller: Caller<'_, _>| {
        let a = Some(ExternRef::new(&mut caller, String::from("a"))?);
        let b = Some(ExternRef::new(&mut caller, String::from("b"))?);
        let c = Some(ExternRef::new(&mut caller, String::from("c"))?);
        Ok((a, b, c))
    })?;
    linker.func_wrap(
        "",
        "take_some",
        |caller: Caller<'_, _>,
         a: Option<Rooted<ExternRef>>,
         b: Option<Rooted<ExternRef>>,
         c: Option<Rooted<ExternRef>>| {
            let a = a.unwrap();
            let b = b.unwrap();
            let c = c.unwrap();
            assert_eq!(
                a.data(&caller)
                    .expect("rooted")
                    .expect("host data")
                    .downcast_ref::<String>()
                    .expect("is string"),
                "a"
            );
            assert_eq!(
                b.data(&caller)
                    .expect("rooted")
                    .expect("host data")
                    .downcast_ref::<String>()
                    .expect("is string"),
                "b"
            );
            assert_eq!(
                c.data(&caller)
                    .expect("rooted")
                    .expect("host data")
                    .downcast_ref::<String>()
                    .expect("is string"),
                "c"
            );
        },
    )?;

    let instance = linker.instantiate(&mut store, &module)?;
    let func = instance.get_typed_func::<(), ()>(&mut store, "run")?;
    func.call(&mut store, ())?;

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn gc_and_tail_calls_and_stack_arguments() -> Result<()> {
    // Test that GC refs in tail-calls' stack arguments get properly accounted
    // for in stack maps.
    //
    // What we do _not_ want to happen is for tail callers to be responsible for
    // including stack arguments in their stack maps (and therefore whether or
    // not they get marked at runtime). If that was the case, then we could have
    // the following scenario:
    //
    // * `f` calls `g` without any stack arguments,
    // * `g` tail calls `h` with GC ref stack arguments,
    // * and then `h` triggers a GC.
    //
    // Because `g`, who is responsible for including the GC refs in its stack
    // map in this hypothetical scenario, is no longer on the stack, we never
    // see its stack map, and therefore never mark the GC refs, and then we
    // collect them too early, and then we can get user-after-free bugs. Not
    // good! Note also that `f`, which is the frame that `h` will return to,
    // _cannot_ be responsible for including these stack arguments in its stack
    // map, because it has no idea what frame will be returning to it, and it
    // could be any number of different functions using that frame for long (and
    // indirect!) tail-call chains.
    //
    // In Cranelift we avoid this scenario because stack arguments are eagerly
    // loaded into virtual registers, and then when we insert a GC safe point,
    // we spill these virtual registers to the callee stack frame, and the stack
    // map includes entries for these stack slots.
    //
    // Nonetheless, this test exercises the above scenario just in case we do
    // something in the future like lazily load stack arguments into virtual
    // registers, to make sure that everything shows up in stack maps like they
    // are supposed to.

    let (mut store, module) = ref_types_module(
        false,
        r#"
            (module
                (import "" "make_some" (func $make (result externref externref externref)))
                (import "" "take_some" (func $take (param externref externref externref)))
                (import "" "gc" (func $gc))

                (func $stack_args (param externref externref externref externref externref externref externref externref externref externref externref externref externref externref externref externref externref externref externref externref externref externref externref externref externref externref externref externref externref externref)
                  call $gc
                  ;; Make sure all these GC refs are live, so that they need to
                  ;; be put into the stack map.
                  local.get 0
                  local.get 1
                  local.get 2
                  call $take
                  local.get 3
                  local.get 4
                  local.get 5
                  call $take
                  local.get 6
                  local.get 7
                  local.get 8
                  call $take
                  local.get 9
                  local.get 10
                  local.get 11
                  call $take
                  local.get 12
                  local.get 13
                  local.get 14
                  call $take
                  local.get 15
                  local.get 16
                  local.get 17
                  call $take
                  local.get 18
                  local.get 19
                  local.get 20
                  call $take
                  local.get 21
                  local.get 22
                  local.get 23
                  call $take
                  local.get 24
                  local.get 25
                  local.get 26
                  call $take
                  local.get 27
                  local.get 28
                  local.get 29
                  call $take
                )

                (func $no_stack_args
                  call $make
                  call $make
                  call $make
                  call $make
                  call $make
                  call $make
                  call $make
                  call $make
                  call $make
                  call $make
                  return_call $stack_args
                )

                (func (export "run")
                    (local i32)
                    i32.const 1000
                    local.set 0
                    loop
                        call $no_stack_args
                        local.get 0
                        i32.const -1
                        i32.add
                        local.tee 0
                        br_if 0
                    end
                )
            )
        "#,
    )?;

    let mut linker = Linker::new(store.engine());
    linker.func_wrap("", "make_some", |mut caller: Caller<'_, _>| {
        Ok((
            Some(ExternRef::new(&mut caller, "a".to_string())?),
            Some(ExternRef::new(&mut caller, "b".to_string())?),
            Some(ExternRef::new(&mut caller, "c".to_string())?),
        ))
    })?;
    linker.func_wrap(
        "",
        "take_some",
        |caller: Caller<'_, _>,
         a: Option<Rooted<ExternRef>>,
         b: Option<Rooted<ExternRef>>,
         c: Option<Rooted<ExternRef>>| {
            let a = a.unwrap();
            let b = b.unwrap();
            let c = c.unwrap();
            assert_eq!(
                a.data(&caller)
                    .unwrap()
                    .unwrap()
                    .downcast_ref::<String>()
                    .unwrap(),
                "a"
            );
            assert_eq!(
                b.data(&caller)
                    .unwrap()
                    .unwrap()
                    .downcast_ref::<String>()
                    .unwrap(),
                "b"
            );
            assert_eq!(
                c.data(&caller)
                    .unwrap()
                    .unwrap()
                    .downcast_ref::<String>()
                    .unwrap(),
                "c"
            );
        },
    )?;
    linker.func_wrap("", "gc", |mut caller: Caller<()>| caller.gc(None))?;

    let instance = linker.instantiate(&mut store, &module)?;
    let func = instance.get_typed_func::<(), ()>(&mut store, "run")?;
    func.call(&mut store, ())?;

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn no_leak_with_global_get_elem_segment() -> wasmtime::Result<()> {
    let dropped = Arc::new(AtomicBool::new(false));

    let engine = Engine::default();
    let mut store = Store::new(&engine, ());
    let module = Module::new(
        &engine,
        r#"
            (module
                (import "" "" (global $init externref))
                (start $f)
                (table $t 1 externref)
                (elem $e externref (global.get $init))

                (func $f
                    i32.const 0
                    i32.const 0
                    i32.const 1
                    table.init $t $e

                    i32.const 0
                    i32.const 0
                    i32.const 1
                    table.init $t $e
                )
            )
        "#,
    )?;

    let externref = ExternRef::new(&mut store, SetFlagOnDrop(dropped.clone()))?;
    let global = Global::new(
        &mut store,
        GlobalType::new(ValType::EXTERNREF, Mutability::Const),
        externref.into(),
    )?;

    Instance::new(&mut store, &module, &[global.into()])?;

    drop(store);

    assert!(dropped.load(SeqCst));
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn table_init_with_externref_global_get() -> wasmtime::Result<()> {
    let dropped = Arc::new(AtomicBool::new(false));

    let mut config = Config::new();
    config.wasm_function_references(true);
    let engine = Engine::new(&config)?;
    let mut store = Store::new(&engine, ());
    let module = Module::new(
        &engine,
        r#"
            (module
                (import "" "" (global $init externref))
                (table $t 1 externref (global.get $init))
            )
        "#,
    )?;

    let externref = ExternRef::new(&mut store, SetFlagOnDrop(dropped.clone()))?;
    let global = Global::new(
        &mut store,
        GlobalType::new(ValType::EXTERNREF, Mutability::Const),
        externref.into(),
    )?;

    Instance::new(&mut store, &module, &[global.into()])?;

    drop(store);

    assert!(dropped.load(SeqCst));
    Ok(())
}

#[test]
fn rooted_gets_collected_after_scope_exit() -> Result<()> {
    let mut store = Store::<()>::default();
    let flag = Arc::new(AtomicBool::new(false));

    {
        let mut scope = RootScope::new(&mut store);
        let _externref = ExternRef::new(&mut scope, SetFlagOnDrop(flag.clone()))?;

        scope.as_context_mut().gc(None)?;
        assert!(!flag.load(SeqCst), "not dropped when still rooted");
    }

    store.as_context_mut().gc(None)?;
    assert!(flag.load(SeqCst), "dropped after being unrooted");

    Ok(())
}

#[test]
fn owned_rooted_gets_collected_after_unrooting() -> Result<()> {
    let mut store = Store::<()>::default();
    let flag = Arc::new(AtomicBool::new(false));

    let externref = {
        let mut scope = RootScope::new(&mut store);
        ExternRef::new(&mut scope, SetFlagOnDrop(flag.clone()))?.to_owned_rooted(&mut scope)?
    };

    store.gc(None)?;
    assert!(!flag.load(SeqCst), "not dropped when still rooted");

    drop(externref);
    store.gc(None)?;
    assert!(flag.load(SeqCst), "dropped after being unrooted");

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn round_trip_gc_ref_through_typed_wasm_func() -> Result<()> {
    let mut store = Store::<()>::default();
    let module = Module::new(
        store.engine(),
        r#"
            (module
                (import "" "" (func $gc))
                (func (export "f") (param externref) (result externref)
                    call $gc
                    local.get 0
                )
            )
        "#,
    )?;
    let gc = Func::wrap(&mut store, |mut caller: Caller<'_, _>| caller.gc(None));
    let instance = Instance::new(&mut store, &module, &[gc.into()])?;
    let f = instance
        .get_typed_func::<Option<Rooted<ExternRef>>, Option<Rooted<ExternRef>>>(&mut store, "f")?;
    let x1 = ExternRef::new(&mut store, 1234)?;
    let x2 = f.call(&mut store, Some(x1))?.unwrap();
    assert!(Rooted::ref_eq(&store, &x1, &x2)?);
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn round_trip_gc_ref_through_func_wrap() -> Result<()> {
    let mut store = Store::<()>::default();
    let f = Func::wrap(
        &mut store,
        |mut caller: Caller<'_, _>, x: Rooted<ExternRef>| {
            caller.gc(None)?;
            Ok(x)
        },
    );
    let f = f.typed::<Rooted<ExternRef>, Rooted<ExternRef>>(&store)?;
    let x1 = ExternRef::new(&mut store, 1234)?;
    let x2 = f.call(&mut store, x1)?;
    assert!(Rooted::ref_eq(&store, &x1, &x2)?);
    Ok(())
}

#[test]
fn to_raw_from_raw_doesnt_leak() -> Result<()> {
    let mut store = Store::<()>::default();
    let flag = Arc::new(AtomicBool::new(false));

    {
        let mut scope = RootScope::new(&mut store);
        let x = ExternRef::new(&mut scope, SetFlagOnDrop(flag.clone()))?;
        let raw = x.to_raw(&mut scope)?;
        let _x = ExternRef::from_raw(&mut scope, raw);
    }

    store.gc(None)?;
    assert!(flag.load(SeqCst));
    Ok(())
}

#[test]
fn table_fill_doesnt_leak() -> Result<()> {
    let _ = env_logger::try_init();

    let mut store = Store::<()>::default();
    let flag = Arc::new(AtomicBool::new(false));

    {
        let mut scope = RootScope::new(&mut store);
        let x = ExternRef::new(&mut scope, SetFlagOnDrop(flag.clone()))?;
        let table = Table::new(
            &mut scope,
            TableType::new(RefType::EXTERNREF, 10, Some(10)),
            x.into(),
        )?;
        table.fill(&mut scope, 0, Ref::Extern(None), 10)?;
    }

    store.gc(None)?;
    assert!(flag.load(SeqCst));
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn table_copy_doesnt_leak() -> Result<()> {
    let _ = env_logger::try_init();

    let mut store = Store::<()>::default();
    let flag = Arc::new(AtomicBool::new(false));

    {
        let mut scope = RootScope::new(&mut store);
        let table = Table::new(
            &mut scope,
            TableType::new(RefType::EXTERNREF, 10, Some(10)),
            Ref::Extern(None),
        )?;

        let x = ExternRef::new(&mut scope, SetFlagOnDrop(flag.clone()))?;
        table.fill(&mut scope, 2, x.into(), 3)?;

        Table::copy(&mut scope, &table, 0, &table, 5, 5)?;
    }

    store.gc(None)?;
    assert!(flag.load(SeqCst));
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn table_set_doesnt_leak() -> Result<()> {
    let _ = env_logger::try_init();

    let mut store = Store::<()>::default();
    let flag = Arc::new(AtomicBool::new(false));

    {
        let mut scope = RootScope::new(&mut store);
        let table = Table::new(
            &mut scope,
            TableType::new(RefType::EXTERNREF, 10, Some(10)),
            Ref::Extern(None),
        )?;

        let x = ExternRef::new(&mut scope, SetFlagOnDrop(flag.clone()))?;
        table.set(&mut scope, 2, x.into())?;
        table.set(&mut scope, 2, x.into())?;
        table.set(&mut scope, 2, Ref::Extern(None))?;
    }

    store.gc(None)?;
    assert!(flag.load(SeqCst));
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn table_grow_doesnt_leak() -> Result<()> {
    let _ = env_logger::try_init();

    let mut store = Store::<()>::default();
    let flag = Arc::new(AtomicBool::new(false));

    {
        let mut scope = RootScope::new(&mut store);
        let table = Table::new(
            &mut scope,
            TableType::new(RefType::EXTERNREF, 10, Some(10)),
            Ref::Extern(None),
        )?;

        let x = ExternRef::new(&mut scope, SetFlagOnDrop(flag.clone()))?;
        table.grow(&mut scope, 0, x.into())?;
    }

    store.gc(None)?;
    assert!(flag.load(SeqCst));
    Ok(())
}

// This is a test that the argument to `table.init` is properly handled w.r.t.
// write barriers and such. This doesn't use `SetFlagOnDrop` because that would
// require initializing a table with an element initialized from a global and
// the global keeps the externref alive regardless. Instead this has a small GC
// heap and we continuously make more garbage than is in the heap and expect
// this to work as when it triggers a GC everything prior should get cleaned up.
#[test]
#[cfg_attr(miri, ignore)]
fn table_init_doesnt_leak() -> Result<()> {
    const SIZE: u64 = 64 << 10;

    let _ = env_logger::try_init();

    let mut config = Config::new();
    config.wasm_gc(true);
    config.wasm_function_references(true);
    config.memory_may_move(false);
    config.memory_reservation(SIZE);
    config.memory_reservation_for_growth(0);
    let engine = Engine::new(&config)?;
    let mut store = Store::new(&engine, ());

    let module = Module::new(
        store.engine(),
        r#"
            (module
                (table 1 arrayref)

                (type $a (array i31ref))

                (func (export "run")
                    i32.const 0
                    i32.const 0
                    i32.const 1
                    table.init $e)
                (elem $e arrayref (array.new $a (ref.i31 (i32.const 0)) (i32.const 200)))
            )
        "#,
    )?;

    let instance = Instance::new(&mut store, &module, &[])?;
    let func = instance.get_typed_func::<(), ()>(&mut store, "run")?;
    for _ in 0..200 {
        func.call(&mut store, ())?;
    }

    Ok(())
}

#[test]
fn ref_matches() -> Result<()> {
    let mut store = Store::<()>::default();
    let engine = store.engine().clone();

    let func_ty = FuncType::new(&engine, None, None);
    let func_ref_ty = RefType::new(true, HeapType::ConcreteFunc(func_ty.clone()));
    let f = Func::new(&mut store, func_ty, |_, _, _| Ok(()));

    let pre = StructRefPre::new(&mut store, StructType::new(&engine, [])?);
    let s = StructRef::new(&mut store, &pre, &[])?.to_anyref();

    let pre = ArrayRefPre::new(
        &mut store,
        ArrayType::new(&engine, FieldType::new(Mutability::Const, StorageType::I8)),
    );
    let a = ArrayRef::new(&mut store, &pre, &Val::I32(0), 0)?.to_anyref();

    let i31 = AnyRef::from_i31(&mut store, I31::wrapping_i32(1234));

    let e = ExternRef::new(&mut store, "hello")?;

    for (val, ty, expected) in [
        // nulls to nullexternref
        (Ref::Extern(None), RefType::NULLEXTERNREF, true),
        (Ref::Any(None), RefType::NULLEXTERNREF, false),
        (Ref::Func(None), RefType::NULLEXTERNREF, false),
        // nulls to externref
        (Ref::Extern(None), RefType::EXTERNREF, true),
        (Ref::Any(None), RefType::EXTERNREF, false),
        (Ref::Func(None), RefType::EXTERNREF, false),
        // nulls to nullref
        (Ref::Extern(None), RefType::NULLREF, false),
        (Ref::Any(None), RefType::NULLREF, true),
        (Ref::Func(None), RefType::NULLREF, false),
        // nulls to structref
        (Ref::Extern(None), RefType::STRUCTREF, false),
        (Ref::Any(None), RefType::STRUCTREF, true),
        (Ref::Func(None), RefType::STRUCTREF, false),
        // nulls to arrayref
        (Ref::Extern(None), RefType::ARRAYREF, false),
        (Ref::Any(None), RefType::ARRAYREF, true),
        (Ref::Func(None), RefType::ARRAYREF, false),
        // nulls to i31ref
        (Ref::Extern(None), RefType::I31REF, false),
        (Ref::Any(None), RefType::I31REF, true),
        (Ref::Func(None), RefType::I31REF, false),
        // nulls to eqref
        (Ref::Extern(None), RefType::EQREF, false),
        (Ref::Any(None), RefType::EQREF, true),
        (Ref::Func(None), RefType::EQREF, false),
        // nulls to anyref
        (Ref::Extern(None), RefType::ANYREF, false),
        (Ref::Any(None), RefType::ANYREF, true),
        (Ref::Func(None), RefType::ANYREF, false),
        // non-null structref
        (Ref::Any(Some(s)), RefType::NULLFUNCREF, false),
        (Ref::Any(Some(s)), func_ref_ty.clone(), false),
        (Ref::Any(Some(s)), RefType::FUNCREF, false),
        (Ref::Any(Some(s)), RefType::NULLEXTERNREF, false),
        (Ref::Any(Some(s)), RefType::EXTERNREF, false),
        (Ref::Any(Some(s)), RefType::NULLREF, false),
        (Ref::Any(Some(s)), RefType::STRUCTREF, true),
        (Ref::Any(Some(s)), RefType::ARRAYREF, false),
        (Ref::Any(Some(s)), RefType::I31REF, false),
        (Ref::Any(Some(s)), RefType::EQREF, true),
        (Ref::Any(Some(s)), RefType::ANYREF, true),
        // non-null arrayref
        (Ref::Any(Some(a)), RefType::NULLFUNCREF, false),
        (Ref::Any(Some(a)), func_ref_ty.clone(), false),
        (Ref::Any(Some(a)), RefType::FUNCREF, false),
        (Ref::Any(Some(a)), RefType::NULLEXTERNREF, false),
        (Ref::Any(Some(a)), RefType::EXTERNREF, false),
        (Ref::Any(Some(a)), RefType::NULLREF, false),
        (Ref::Any(Some(a)), RefType::STRUCTREF, false),
        (Ref::Any(Some(a)), RefType::ARRAYREF, true),
        (Ref::Any(Some(a)), RefType::I31REF, false),
        (Ref::Any(Some(a)), RefType::EQREF, true),
        (Ref::Any(Some(a)), RefType::ANYREF, true),
        // non-null i31ref
        (Ref::Any(Some(i31)), RefType::NULLFUNCREF, false),
        (Ref::Any(Some(i31)), func_ref_ty.clone(), false),
        (Ref::Any(Some(i31)), RefType::FUNCREF, false),
        (Ref::Any(Some(i31)), RefType::NULLEXTERNREF, false),
        (Ref::Any(Some(i31)), RefType::EXTERNREF, false),
        (Ref::Any(Some(i31)), RefType::NULLREF, false),
        (Ref::Any(Some(i31)), RefType::STRUCTREF, false),
        (Ref::Any(Some(i31)), RefType::ARRAYREF, false),
        (Ref::Any(Some(i31)), RefType::I31REF, true),
        (Ref::Any(Some(i31)), RefType::EQREF, true),
        (Ref::Any(Some(i31)), RefType::ANYREF, true),
        // non-null funcref
        (Ref::Func(Some(f)), RefType::NULLFUNCREF, false),
        (Ref::Func(Some(f)), func_ref_ty.clone(), true),
        (Ref::Func(Some(f)), RefType::FUNCREF, true),
        (Ref::Func(Some(f)), RefType::NULLEXTERNREF, false),
        (Ref::Func(Some(f)), RefType::EXTERNREF, false),
        (Ref::Func(Some(f)), RefType::NULLREF, false),
        (Ref::Func(Some(f)), RefType::STRUCTREF, false),
        (Ref::Func(Some(f)), RefType::ARRAYREF, false),
        (Ref::Func(Some(f)), RefType::I31REF, false),
        (Ref::Func(Some(f)), RefType::EQREF, false),
        (Ref::Func(Some(f)), RefType::ANYREF, false),
        // non-null externref
        (Ref::Extern(Some(e)), RefType::NULLFUNCREF, false),
        (Ref::Extern(Some(e)), func_ref_ty.clone(), false),
        (Ref::Extern(Some(e)), RefType::FUNCREF, false),
        (Ref::Extern(Some(e)), RefType::NULLEXTERNREF, false),
        (Ref::Extern(Some(e)), RefType::EXTERNREF, true),
        (Ref::Extern(Some(e)), RefType::NULLREF, false),
        (Ref::Extern(Some(e)), RefType::STRUCTREF, false),
        (Ref::Extern(Some(e)), RefType::ARRAYREF, false),
        (Ref::Extern(Some(e)), RefType::I31REF, false),
        (Ref::Extern(Some(e)), RefType::EQREF, false),
        (Ref::Extern(Some(e)), RefType::ANYREF, false),
    ] {
        let actual = val.matches_ty(&mut store, &ty)?;
        assert_eq!(
            actual, expected,
            "{val:?} matches {ty:?}? expected {expected}, got {actual}"
        );
    }

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn issue_9669() -> Result<()> {
    let _ = env_logger::try_init();

    let mut config = Config::new();
    config.wasm_function_references(true);
    config.wasm_gc(true);
    config.collector(Collector::DeferredReferenceCounting);

    let engine = Engine::new(&config)?;

    let module = Module::new(
        &engine,
        r#"
            (module
                (type $empty (struct))
                (type $thing (struct
                    (field $field1 (ref $empty))
                    (field $field2 (ref $empty))
                ))

                (func (export "run")
                    (local $object (ref $thing))

                    struct.new $empty
                    struct.new $empty
                    struct.new $thing

                    local.tee $object
                    struct.get $thing $field1
                    drop

                    local.get $object
                    struct.get $thing $field2
                    drop
                )
            )
        "#,
    )?;

    let mut store = Store::new(&engine, ());
    let instance = Instance::new(&mut store, &module, &[])?;

    let func = instance.get_typed_func::<(), ()>(&mut store, "run")?;
    func.call(&mut store, ())?;

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn drc_transitive_drop_cons_list() -> Result<()> {
    let _ = env_logger::try_init();

    let mut config = Config::new();
    config.wasm_function_references(true);
    config.wasm_gc(true);
    config.collector(Collector::DeferredReferenceCounting);

    let engine = Engine::new(&config)?;

    // Define a module that defines a recursive type (a `cons` list of
    // `externref`s) and exports a global of that type so that we can access it
    // from the host, because we don't yet have host APIs for defining recursive
    // types or rec groups.
    let module = Module::new(
        &engine,
        r#"
            (module
                (type $cons (struct (field externref) (field (ref null $cons))))
                (global (export "g") (ref null $cons) (ref.null $cons))
            )
        "#,
    )?;

    let export = module.exports().nth(0).unwrap().ty();
    let global = export.unwrap_global();
    let ref_ty = global.content().unwrap_ref();
    let struct_ty = ref_ty.heap_type().unwrap_concrete_struct();

    let mut store = Store::new(&engine, ());

    let pre = StructRefPre::new(&mut store, struct_ty.clone());
    let num_refs_dropped = Arc::new(AtomicUsize::new(0));

    let len = if cfg!(miri) { 2 } else { 100 };
    {
        let mut store = RootScope::new(&mut store);

        let mut cdr = None;
        for _ in 0..len {
            let externref = ExternRef::new(&mut store, CountDrops(num_refs_dropped.clone()))?;
            let cons = StructRef::new(&mut store, &pre, &[externref.into(), cdr.into()])?;
            cdr = Some(cons);
        }

        // Still holding the cons list alive at this point.
        assert_eq!(num_refs_dropped.load(SeqCst), 0);
    }

    // Not holding the cons list alive anymore; should transitively drop
    // everything we created.
    store.gc(None)?;
    assert_eq!(num_refs_dropped.load(SeqCst), len);

    Ok(())
}

#[test]
fn drc_transitive_drop_nested_arrays_tree() -> Result<()> {
    let _ = env_logger::try_init();

    let mut config = Config::new();
    config.wasm_function_references(true);
    config.wasm_gc(true);
    config.collector(Collector::DeferredReferenceCounting);

    let engine = Engine::new(&config)?;

    let array_ty = ArrayType::new(
        &engine,
        FieldType::new(
            Mutability::Var,
            StorageType::ValType(ValType::Ref(RefType::ANYREF)),
        ),
    );

    let mut store = Store::new(&engine, ());
    let pre = ArrayRefPre::new(&mut store, array_ty);
    let num_refs_dropped = Arc::new(AtomicUsize::new(0));
    let mut expected = 0;

    fn recursively_build_tree(
        mut store: &mut RootScope<&mut Store<()>>,
        pre: &ArrayRefPre,
        num_refs_dropped: &Arc<AtomicUsize>,
        expected: &mut usize,
        depth: u32,
    ) -> Result<Rooted<AnyRef>> {
        let max = if cfg!(miri) { 1 } else { 3 };
        if depth >= max {
            *expected += 1;
            let e = ExternRef::new(&mut store, CountDrops(num_refs_dropped.clone()))?;
            AnyRef::convert_extern(&mut store, e)
        } else {
            let left = recursively_build_tree(store, pre, num_refs_dropped, expected, depth + 1)?;
            let right = recursively_build_tree(store, pre, num_refs_dropped, expected, depth + 1)?;
            let arr = ArrayRef::new_fixed(store, pre, &[left.into(), right.into()])?;
            Ok(arr.to_anyref())
        }
    }

    {
        let mut store = RootScope::new(&mut store);
        let _tree = recursively_build_tree(&mut store, &pre, &num_refs_dropped, &mut expected, 0)?;

        // Still holding the tree alive at this point.
        assert_eq!(num_refs_dropped.load(SeqCst), 0);
    }

    // Not holding the tree alive anymore; should transitively drop everything
    // we created.
    store.gc(None)?;
    assert_eq!(num_refs_dropped.load(SeqCst), expected);

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn drc_traces_the_correct_number_of_gc_refs_in_arrays() -> Result<()> {
    let _ = env_logger::try_init();

    let mut config = Config::new();
    config.wasm_function_references(true);
    config.wasm_gc(true);
    config.collector(Collector::DeferredReferenceCounting);

    let engine = Engine::new(&config)?;
    let mut store = Store::new(&engine, ());

    // The DRC collector was mistakenly reporting that arrays of GC refs had
    // `size_of(elems)` outgoing edges, rather than `len(elems)` edges. None of
    // our existing tests happened to trigger this bug because although we were
    // tricking the collector into tracing unallocated GC heap memory, it was
    // all zeroed out and was treated as null GC references. We can avoid that
    // in this regression test by first painting the heap with a large poison
    // value before we start allocating arrays, so that if the GC tries tracing
    // bogus heap memory, it finds very large GC ref heap indices and ultimately
    // tries to follow them outside the bounds of the GC heap, which (before
    // this bug was fixed) would lead to a panic.

    let array_i8_ty = ArrayType::new(&engine, FieldType::new(Mutability::Var, StorageType::I8));
    let array_i8_pre = ArrayRefPre::new(&mut store, array_i8_ty);

    {
        let mut store = RootScope::new(&mut store);

        // Spray a poison pattern across the heap.
        let len = 1_000_000;
        let _poison = ArrayRef::new(&mut store, &array_i8_pre, &Val::I32(-1), len);
    }

    // Make sure the poison array is collected.
    store.gc(None)?;

    // Allocate and then collect an array of GC refs from Wasm. This should not
    // trick the collector into tracing any poison and panicking.
    let module = Module::new(
        &engine,
        r#"
            (module
                (type $ty (array (mut anyref)))
                (start $f)
                (func $f
                    (drop (array.new $ty (ref.null any) (i32.const 1_000)))
                )
            )
        "#,
    )?;
    let _instance = Instance::new(&mut store, &module, &[])?;
    store.gc(None)?;

    Ok(())
}

// Regression test for a stack-map bug in `translate_per_element_copy` (used by
// `array.copy` on arrays whose elements are GC references).
//
// The per-element loop derived raw `src_elem_addr` / `dst_elem_addr` pointers
// once and used only those raw pointers inside the loop body. The source and
// destination array gc-refs themselves were dead in CLIF inside the loop, so
// they were absent from the stack maps at safe points there. Once the DRC
// collector started firing `force_gc` from inside the read barrier (when the
// over-approximated-stack-roots list grew past 1024 entries), a GC could run
// mid-copy with neither array marked from any frame's stack map. Sweep could
// then drop the arrays from OASR, freeing them and cascading dec-refs through
// their elements, after which the rest of the copy read from freed memory.
//
// The fix passes the source and destination array gc-refs as block params
// through the forward and backward iteration blocks so they stay live across
// the per-element loop.
#[test]
#[cfg_attr(miri, ignore)]
fn drc_array_copy_keeps_arrays_alive_across_in_loop_gc() -> Result<()> {
    let _ = env_logger::try_init();

    let mut config = Config::new();
    config.wasm_function_references(true);
    config.wasm_gc(true);
    config.collector(Collector::DeferredReferenceCounting);
    // Keep the GC heap small so allocations cycle back through the free
    // list quickly. This dramatically increases the chance that any slot
    // freed mid-copy will be reused before the test's verify pass reads
    // from it, making the bug observable.
    config.gc_heap_reservation(128 * 1024);
    config.gc_heap_reservation_for_growth(0);

    let engine = Engine::new(&config)?;
    let mut store = Store::new(&engine, ());

    let module = Module::new(
        &engine,
        r#"
            (module
                (import "wasmtime" "gc" (func $gc))
                (type $box (struct (field i32)))
                (type $arr (array (mut (ref null $box))))

                ;; Put the `array.copy` in a helper function so the source
                ;; and destination gc-refs are visibly dead in CLIF inside
                ;; the per-element loop -- they are function parameters
                ;; that are only used at the top of the function to compute
                ;; the initial address.
                (func $do_copy
                    (param $src (ref null $arr))
                    (param $dst (ref null $arr))
                    (param $len i32)
                    (array.copy $arr $arr
                        (local.get $dst) (i32.const 0)
                        (local.get $src) (i32.const 0)
                        (local.get $len)))

                (func (export "test") (result i32)
                    (local $src (ref null $arr))
                    (local $dst (ref null $arr))
                    (local $i i32)
                    (local $sum i32)

                    ;; N=2048 > 1024 so the per-element read barrier
                    ;; definitely fires `force_gc` during the copy.
                    (local.set $src (array.new $arr (ref.null $box) (i32.const 2048)))
                    (local.set $i (i32.const 0))
                    (loop $fill
                        (array.set $arr (local.get $src) (local.get $i)
                            (struct.new $box (local.get $i)))
                        (local.set $i (i32.add (local.get $i) (i32.const 1)))
                        (br_if $fill (i32.lt_s (local.get $i) (i32.const 2048))))

                    (local.set $dst (array.new $arr (ref.null $box) (i32.const 2048)))

                    (call $gc)

                    (call $do_copy (local.get $src) (local.get $dst) (i32.const 2048))

                    ;; Drop the source local and force a GC so any
                    ;; cascade-freed elements that should still be alive
                    ;; via $dst would surface as either a trap (cast
                    ;; failure / out-of-bounds) or wrong $dst contents.
                    (local.set $src (ref.null $arr))
                    (call $gc)

                    ;; Heavily allocate to reuse any freed slots. Each
                    ;; iteration allocates a $box with sentinel value -1;
                    ;; if any element of $dst was a stale reference to a
                    ;; cascade-freed $box, its slot gets reused here and
                    ;; reading it back will see -1 rather than its
                    ;; original index.
                    (local.set $i (i32.const 0))
                    (loop $churn
                        (drop (struct.new $box (i32.const -1)))
                        (local.set $i (i32.add (local.get $i) (i32.const 1)))
                        (br_if $churn (i32.lt_s (local.get $i) (i32.const 4096))))

                    (call $gc)

                    (local.set $i (i32.const 0))
                    (local.set $sum (i32.const 0))
                    (loop $verify
                        (local.set $sum
                            (i32.add (local.get $sum)
                                (struct.get $box 0
                                    (array.get $arr (local.get $dst) (local.get $i)))))
                        (local.set $i (i32.add (local.get $i) (i32.const 1)))
                        (br_if $verify (i32.lt_s (local.get $i) (i32.const 2048))))

                    (local.get $sum)))
        "#,
    )?;

    let mut linker = Linker::new(&engine);
    linker.func_wrap("wasmtime", "gc", |mut caller: Caller<_>| -> Result<()> {
        caller.gc(None)?;
        Ok(())
    })?;
    let instance = linker.instantiate(&mut store, &module)?;
    let test = instance.get_typed_func::<(), i32>(&mut store, "test")?;
    let result = test.call(&mut store, ())?;

    // Sum of 0..2048 = 2048 * 2047 / 2.
    assert_eq!(result, 2_096_128);

    Ok(())
}

// Test that we can completely fill the GC heap until we get an OOM. This
// exercises growing the GC heap and that we configure compilation tunables and
// runtime memories backing GC heaps correctly.
#[test]
#[cfg_attr(any(miri, not(target_pointer_width = "64")), ignore)]
fn gc_heap_oom() -> Result<()> {
    if std::env::var("WASMTIME_TEST_NO_HOG_MEMORY").is_ok() {
        return Ok(());
    }

    let _ = env_logger::try_init();

    for heap_size in [
        // Very small heap.
        1 << 16,
        // Bigger heap: 4 GiB
        1 << 32,
    ] {
        for pooling in [true, false] {
            let mut config = Config::new();
            config.wasm_function_references(true);
            config.wasm_gc(true);
            config.collector(Collector::Null);
            config.memory_reservation(heap_size);
            config.memory_reservation_for_growth(0);
            config.memory_guard_size(0);
            config.memory_may_move(false);
            config.gc_heap_reservation(heap_size);
            config.gc_heap_reservation_for_growth(0);
            config.gc_heap_guard_size(0);
            config.gc_heap_may_move(false);

            if pooling {
                let mut pooling = crate::small_pool_config();
                pooling.max_memory_size(heap_size.try_into().unwrap());
                config.allocation_strategy(InstanceAllocationStrategy::Pooling(pooling));
            }

            let engine = Engine::new(&config)?;

            let module = Module::new(
                &engine,
                r#"
                (module
                    (type $s (struct))
                    (global $g (export "g") (mut i32) (i32.const 0))
                    (func (export "run")
                      loop
                        struct.new $s

                        global.get $g
                        i32.const 1
                        i32.add
                        global.set $g

                        br 0
                      end
                    )
                )
            "#,
            )?;

            let mut store = Store::new(&engine, ());
            let instance = Instance::new(&mut store, &module, &[])?;

            let run = instance.get_typed_func::<(), ()>(&mut store, "run")?;
            let err = run.call(&mut store, ()).expect_err("should oom");
            assert!(err.is::<Trap>(), "should get trap, got: {err:?}");
            let trap = err.downcast::<Trap>().unwrap();
            assert_eq!(trap, Trap::AllocationTooLarge);

            let g = instance.get_global(&mut store, "g").unwrap();
            const SIZE_OF_NULL_GC_HEADER: u64 = 8;
            const FUDGE: u64 = 2;
            let actual = g.get(&mut store).unwrap_i32() as u64;
            let expected = heap_size / SIZE_OF_NULL_GC_HEADER;
            assert!(
                actual.abs_diff(expected) <= FUDGE,
                "actual approx= expected failed: \
                 actual = {actual}, expected = {expected}, FUDGE={FUDGE}"
            );
        }
    }
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn issue_10772() -> Result<()> {
    let mut store = crate::gc_store()?;
    let engine = store.engine().clone();

    let module = Module::new(
        &engine,
        r#"
            (module
                (type $empty (struct))
                (type $tuple-concrete (struct (field (ref $empty))))
                (type $tuple-abstract (struct (field (ref struct))))
                (func (export "abstract") (param $t (ref $tuple-abstract))
                    (drop (ref.cast (ref $tuple-concrete) (local.get $t)))
                )
            )
        "#,
    )?;

    let linker = Linker::new(&engine);

    let instance = linker.instantiate(&mut store, &module)?;
    let abstract_ = instance.get_func(&mut store, "abstract").unwrap();
    let empty_pre = StructRefPre::new(&mut store, StructType::new(&engine, [])?);
    let empty_struct = StructRef::new(&mut store, &empty_pre, &[])?;
    let tuple_pre = StructRefPre::new(
        &mut store,
        StructType::new(
            &engine,
            [FieldType::new(
                Mutability::Const,
                StorageType::ValType(ValType::Ref(RefType::new(false, HeapType::Struct))),
            )],
        )?,
    );
    let tuple_struct = StructRef::new(&mut store, &tuple_pre, &[empty_struct.into()])?;
    let tuple_any = Val::from(tuple_struct);

    match abstract_.call(store, &[tuple_any], &mut []) {
        Ok(()) => panic!("should have trapped on cast failure"),
        Err(e) => {
            let trap = e.downcast::<Trap>().expect("should fail with a trap");
            assert_eq!(trap, Trap::CastFailure);
        }
    }

    Ok(())
}

#[test]
fn drc_gc_inbetween_host_calls() -> Result<()> {
    let _ = env_logger::try_init();

    let mut config = Config::new();
    config.wasm_function_references(true);
    config.wasm_gc(true);
    config.collector(Collector::DeferredReferenceCounting);

    let engine = Engine::new(&config)?;

    let mut store = Store::new(&engine, ());
    let func = Func::wrap(&mut store, |_: Option<Rooted<ExternRef>>| {});

    let mut invoke_func = || {
        let inner_dropped = Arc::new(AtomicBool::new(false));
        {
            let mut scope = RootScope::new(&mut store);
            let r = ExternRef::new(&mut scope, SetFlagOnDrop(inner_dropped.clone()))?;
            func.call(&mut scope, &[r.into()], &mut [])?;
        }

        assert!(!inner_dropped.load(SeqCst));
        store.gc(None)?;
        assert!(inner_dropped.load(SeqCst));
        wasmtime::error::Ok(())
    };

    invoke_func()?;
    invoke_func()?;

    Ok(())
}

#[test]
fn owned_rooted() -> Result<()> {
    let _ = env_logger::try_init();

    let mut config = Config::new();
    config.wasm_function_references(true);
    config.wasm_gc(true);
    config.collector(Collector::DeferredReferenceCounting);

    let engine = Engine::new(&config)?;

    let mut store = Store::new(&engine, ());
    let inner_dropped = Arc::new(AtomicBool::new(false));
    let r = {
        let mut scope = RootScope::new(&mut store);
        let r = ExternRef::new(&mut scope, SetFlagOnDrop(inner_dropped.clone()))?;
        r.to_owned_rooted(&mut scope)?
    };
    assert!(!inner_dropped.load(SeqCst));
    store.gc(None)?;
    assert!(!inner_dropped.load(SeqCst));
    let r2 = r.clone();
    store.gc(None)?;
    assert!(!inner_dropped.load(SeqCst));
    drop(r);
    store.gc(None)?;
    assert!(!inner_dropped.load(SeqCst));
    drop(r2);
    store.gc(None)?;
    assert!(inner_dropped.load(SeqCst));

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn owned_rooted_lots_of_root_creation() -> Result<()> {
    let mut config = Config::new();
    config.wasm_function_references(true);
    config.wasm_gc(true);
    config.collector(Collector::DeferredReferenceCounting);

    let engine = Engine::new(&config)?;

    let mut store = Store::new(&engine, ());
    let inner_dropped = Arc::new(AtomicBool::new(false));
    let r = {
        let mut scope = RootScope::new(&mut store);
        let r = ExternRef::new(&mut scope, SetFlagOnDrop(inner_dropped.clone()))?;
        r.to_owned_rooted(&mut scope)?
    };
    assert!(!inner_dropped.load(SeqCst));
    store.gc(None)?;

    for _ in 0..100_000 {
        let mut scope = RootScope::new(&mut store);
        // Go through a LIFO root then back to an owned root to create
        // a distinct root.
        let r2 = r.to_rooted(&mut scope);
        let r3 = r2.to_owned_rooted(&mut scope);
        drop(r3);
    }

    store.gc(None)?;
    assert!(!inner_dropped.load(SeqCst));

    drop(r);
    store.gc(None)?;
    assert!(inner_dropped.load(SeqCst));

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn instantiate_global_init_oom() -> Result<()> {
    let mut config = Config::new();
    config.wasm_gc(true);
    config.wasm_function_references(true);
    config.memory_may_move(false);
    config.memory_reservation(64 << 10);
    config.memory_reservation_for_growth(0);
    config.gc_heap_may_move(false);
    config.gc_heap_reservation(64 << 10);
    config.gc_heap_reservation_for_growth(0);
    let engine = Engine::new(&config)?;
    let mut store = Store::new(&engine, ());

    let module = Module::new(
        store.engine(),
        r#"
            (module
                (table 100 arrayref)
                (type $a (array i31ref))
                (global (ref $a) (array.new_default $a (i32.const 10000000)))
            )
        "#,
    )?;

    Instance::new(&mut store, &module, &[])
        .unwrap_err()
        .downcast::<GcHeapOutOfMemory<()>>()?;

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn elem_const_eval_oom() -> Result<()> {
    let mut config = Config::new();
    config.wasm_gc(true);
    config.wasm_function_references(true);
    config.memory_may_move(false);
    config.memory_reservation(64 << 10);
    config.memory_reservation_for_growth(0);
    config.gc_heap_may_move(false);
    config.gc_heap_reservation(64 << 10);
    config.gc_heap_reservation_for_growth(0);
    let engine = Engine::new(&config)?;
    let mut store = Store::new(&engine, ());

    let module = Module::new(
        store.engine(),
        r#"
            (module
                (type $a (array i31ref))
                (elem arrayref
                    (array.new_default $a (i32.const 100))
                    (array.new_default $a (i32.const 10000))
                    (array.new_default $a (i32.const 10000))
                    (array.new_default $a (i32.const 10000))
                    (array.new_default $a (i32.const 1000000))
                )
            )
        "#,
    )?;

    Instance::new(&mut store, &module, &[])
        .unwrap_err()
        .downcast::<GcHeapOutOfMemory<()>>()?;

    Ok(())
}

// The result of a `select` instruction with a GC reference type should be
// declared as needing a stack map.
#[test]
#[cfg_attr(miri, ignore)]
fn select_gc_ref_stack_map() -> Result<()> {
    let _ = env_logger::try_init();

    let mut config = Config::new();
    config.wasm_function_references(true);
    config.wasm_gc(true);
    // Use pooling allocator with a tiny GC heap so that GC is triggered
    // frequently and freed memory is reused quickly.
    let mut pool = crate::small_pool_config();
    pool.max_memory_size(1 << 16); // 64 KiB
    config.allocation_strategy(pool);

    let engine = Engine::new(&config)?;
    let module = Module::new(
        &engine,
        r#"
            (module
                (type $pair (struct (field (mut i32))))
                (type $arr (array (mut i8)))

                (import "" "" (func $force_gc))

                (func (export "test") (param $cond i32) (result i32)
                    ;; The select result stays on the Wasm operand stack (never
                    ;; stored in a local variable). If the new SSA value created
                    ;; by select is not declared for stack maps, the GC will
                    ;; free it.
                    (select (result (ref null $pair))
                        (struct.new $pair (i32.const 111))
                        (ref.null $pair)
                        (local.get $cond)
                    )

                    ;; This call is a safepoint. The select result is live
                    ;; across it. The called function triggers GC which will
                    ;; free the struct if it is incorrectly omitted from the
                    ;; safepoint's stack map, and the new allocations would
                    ;; overwrite the freed memory.
                    (call $force_gc)

                    (drop (struct.new $pair (i32.const 222)))

                    ;; Use the select result. If it was incorrectly freed, then
                    ;; this will have the wrong value.
                    (struct.get $pair 0)
                )
            )
        "#,
    )?;

    let mut store = Store::new(&engine, ());
    let force_gc = Func::wrap(&mut store, |mut caller: Caller<'_, _>| {
        caller.gc(None)?;
        Ok(())
    });
    let instance = Instance::new(&mut store, &module, &[force_gc.into()])?;
    let test = instance.get_typed_func::<(i32,), i32>(&mut store, "test")?;

    let result = test.call(&mut store, (1,))?;
    assert_eq!(result, 111);

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn dropped_passive_elems_do_not_leak() -> Result<()> {
    let _ = env_logger::try_init();

    let mut config = Config::new();
    config.wasm_function_references(true);
    config.wasm_gc(true);
    config.collector(Collector::DeferredReferenceCounting);

    let engine = Engine::new(&config)?;

    let module = Module::new(
        &engine,
        r#"
            (module
                (type $a (array (mut externref)))
                (type $aa (array (ref $a)))

                (elem $e (ref $a) (array.new_default $a (i32.const 1)))

                (func (export "run") (param $x externref)
                    (local $l (ref null $aa))

                    ;; Create an array from the passive element segment.
                    (local.set $l (array.new_elem $aa $e (i32.const 0) (i32.const 1)))

                    ;; Put our externref into the passive element segment's array:
                    ;; `l[0][0] = x`.
                    (array.set $a
                               (array.get $aa (local.get $l) (i32.const 0))
                               (i32.const 0)
                               (local.get $x))

                    ;; Drop the passive element segment. This should make `x`
                    ;; unreachable upon return, since we aren't keeping our
                    ;; array live nor the passive element segment.
                    (elem.drop $e)
                )
            )
        "#,
    )?;

    let mut store = Store::new(&engine, ());
    let dropped = Arc::new(AtomicBool::new(false));

    {
        let mut store = RootScope::new(&mut store);
        let e = ExternRef::new(&mut store, SetFlagOnDrop(dropped.clone()))?;

        let instance = Instance::new(&mut store, &module, &[])?;
        let run = instance.get_typed_func::<Rooted<ExternRef>, ()>(&mut store, "run")?;
        run.call(&mut store, e)?;
    }

    store.gc(None)?;
    assert!(dropped.load(SeqCst));

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn gc_heap_does_not_grow_unboundedly() -> Result<()> {
    let _ = env_logger::try_init();

    let mut config = Config::new();
    config.wasm_function_references(true);
    config.wasm_gc(true);
    config.collector(Collector::DeferredReferenceCounting);

    let engine = Engine::new(&config)?;

    let module = Module::new(
        &engine,
        r#"
            (module
                (type $small (struct (field i32)))
                (import "" "check" (func $check))

                (func (export "run") (param i32)
                    (local $i i32)
                    (local $tmp (ref null $small))
                    (loop $loop
                        (local.set $tmp (struct.new $small (i32.const 42)))

                        ;; Call the host to check heap size.
                        (call $check)

                        ;; Loop counter.
                        (local.set $i (i32.add (local.get $i) (i32.const 1)))
                        (br_if $loop (i32.lt_u (local.get $i) (local.get 0)))
                    )
                )
            )
        "#,
    )?;

    let mut store = Store::new(&engine, ());

    let check = Func::wrap(&mut store, |caller: Caller<'_, _>| {
        let heap_size = caller.gc_heap_capacity();
        assert!(
            heap_size <= 65536,
            "GC heap grew too large: {heap_size} bytes (limit: 64KiB)"
        );
    });

    let instance = Instance::new(&mut store, &module, &[check.into()])?;
    let run = instance.get_typed_func::<(i32,), ()>(&mut store, "run")?;
    run.call(&mut store, (100_000,))?;

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn pooling_gc_different_configs_rejected() -> Result<()> {
    let _ = env_logger::try_init();

    let mut config = Config::new();
    config.wasm_function_references(true);
    config.wasm_gc(true);
    config.collector(Collector::Null);
    config.allocation_strategy(crate::small_pool_config());

    // Set GC heap reservation to a different value than memory reservation.
    config.gc_heap_reservation(0);

    assert!(Engine::new(&config).is_err());

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn issue_13141_gc_heap_may_not_move() -> Result<()> {
    let mut config = Config::new();
    config.wasm_gc(true);
    config.gc_heap_may_move(false);

    let engine = Engine::new(&config)?;

    let module = Module::new(
        &engine,
        r#"
            (module
                (type $a (array (mut i8)))

                (global $g (mut (ref $a)) (array.new_default $a (i32.const 12)))

                (func (export "array_get_nth") (param $p i32) (result i32)
                    (array.get_u $a (global.get $g) (local.get $p))
                )
            )
        "#,
    )?;

    let mut store = Store::new(&engine, ());
    let instance = Instance::new(&mut store, &module, &[])?;
    let array_get_nth = instance.get_typed_func::<i32, i32>(&mut store, "array_get_nth")?;
    let result = array_get_nth.call(&mut store, 0)?;
    assert_eq!(result, 0);
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn memory_guard_pages_but_no_gc_heap_guard_pages() -> Result<()> {
    if std::mem::size_of::<usize>() < std::mem::size_of::<u64>()
        || std::env::var("WASMTIME_TEST_NO_HOG_MEMORY").is_ok()
    {
        return Ok(());
    }

    let _ = env_logger::try_init();

    let mut config = Config::new();
    config.wasm_function_references(true);
    config.wasm_gc(true);
    config.collector(Collector::Null);

    // Memories get large guard pages (bounds checks elided for i32), but the
    // GC heap does not (bounds checks required).
    config.memory_reservation(1 << 32);
    config.memory_guard_size(1 << 32);
    config.gc_heap_reservation(0);
    config.gc_heap_guard_size(0);
    config.gc_heap_reservation_for_growth(1 << 20);

    let engine = Engine::new(&config)?;

    let module = Module::new(
        &engine,
        r#"
        (module
            (type $ty (struct (field (mut f32))))
            (func (export "roundtrip") (param (ref null $ty)) (result f32)
                (struct.get $ty 0 (local.get 0))
            )
            (func (export "alloc") (result (ref $ty))
                (struct.new $ty (f32.const 3.14))
            )
        )
        "#,
    )?;

    let mut store = Store::new(&engine, ());
    let instance = Instance::new(&mut store, &module, &[])?;

    let alloc = instance.get_typed_func::<(), Rooted<StructRef>>(&mut store, "alloc")?;
    let roundtrip =
        instance.get_typed_func::<(Option<Rooted<StructRef>>,), f32>(&mut store, "roundtrip")?;

    let s = alloc.call(&mut store, ())?;
    let val = roundtrip.call(&mut store, (Some(s),))?;
    assert_eq!(val, 3.14_f32);

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn gc_heap_guard_pages_but_no_memory_guard_pages() -> Result<()> {
    if std::mem::size_of::<usize>() < std::mem::size_of::<u64>()
        || std::env::var("WASMTIME_TEST_NO_HOG_MEMORY").is_ok()
    {
        return Ok(());
    }

    let _ = env_logger::try_init();

    let mut config = Config::new();
    config.wasm_function_references(true);
    config.wasm_gc(true);
    config.collector(Collector::Null);

    // GC heap gets large guard pages (bounds checks elided), but memories do
    // not (bounds checks required for memories).
    config.gc_heap_reservation(1 << 32);
    config.gc_heap_guard_size(1 << 32);
    config.memory_reservation(0);
    config.memory_guard_size(0);
    config.memory_reservation_for_growth(1 << 20);

    let engine = Engine::new(&config)?;

    let module = Module::new(
        &engine,
        r#"
        (module
            (type $ty (struct (field (mut f32))))
            (memory (export "memory") 1)
            (func (export "roundtrip") (param (ref null $ty)) (result f32)
                (struct.get $ty 0 (local.get 0))
            )
            (func (export "alloc") (result (ref $ty))
                (struct.new $ty (f32.const 2.72))
            )
            (func (export "load") (param i32) (result i32)
                (i32.load (local.get 0))
            )
        )
        "#,
    )?;

    let mut store = Store::new(&engine, ());
    let instance = Instance::new(&mut store, &module, &[])?;

    let alloc = instance.get_typed_func::<(), Rooted<StructRef>>(&mut store, "alloc")?;
    let roundtrip =
        instance.get_typed_func::<(Option<Rooted<StructRef>>,), f32>(&mut store, "roundtrip")?;
    let load = instance.get_typed_func::<(i32,), i32>(&mut store, "load")?;

    let s = alloc.call(&mut store, ())?;
    let val = roundtrip.call(&mut store, (Some(s),))?;
    assert_eq!(val, 2.72_f32);

    // Also exercise the memory access to make sure it works with bounds checks.
    let result = load.call(&mut store, (0,))?;
    assert_eq!(result, 0);

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn issue_13037_drc_leak_passing_objects_already_in_over_approx_stack_roots_list() -> Result<()> {
    let _ = env_logger::try_init();

    let mut config = Config::new();
    config.collector(Collector::DeferredReferenceCounting);
    let engine = Engine::new(&config)?;
    let mut store = Store::new(&engine, ());

    let module = Module::new(
        &engine,
        r#"(module (func (export "nop") (param externref)))"#,
    )?;

    let instance = Instance::new(&mut store, &module, &[])?;
    let nop = instance.get_typed_func::<Option<Rooted<ExternRef>>, ()>(&mut store, "nop")?;

    let dropped = Arc::new(AtomicBool::new(false));
    {
        let mut scope = RootScope::new(&mut store);
        let ext = ExternRef::new(&mut scope, SetFlagOnDrop(dropped.clone()))?;

        nop.call(&mut scope, Some(ext))?;
        nop.call(&mut scope, Some(ext))?;
    }

    store.gc(None)?;

    assert!(dropped.load(Ordering::SeqCst));
    Ok(())
}

fn copying_store_with_gc_zeal(counter: u32) -> Result<(Store<()>, Engine)> {
    let _ = env_logger::try_init();
    let mut config = Config::new();
    config.wasm_gc(true);
    config.wasm_function_references(true);
    config.collector(Collector::Copying);
    let _ = config.gc_zeal_alloc_counter(NonZeroU32::new(counter));
    let engine = Engine::new(&config)?;
    let store = Store::new(&engine, ());
    Ok((store, engine))
}

#[test]
#[cfg_attr(miri, ignore)]
fn copying_collector_externref_survives_gc() -> Result<()> {
    let (mut store, engine) = copying_store_with_gc_zeal(1)?;
    let module = Module::new(
        &engine,
        r#"
        (module
            (import "" "gc" (func $gc))
            (func (export "roundtrip") (param externref) (result externref)
                (call $gc)
                (local.get 0)
            )
        )
        "#,
    )?;
    let gc_func = Func::wrap(&mut store, |mut caller: Caller<'_, ()>| -> Result<()> {
        caller.gc(None)
    });
    let instance = Instance::new(&mut store, &module, &[gc_func.into()])?;
    let roundtrip = instance
        .get_typed_func::<Option<Rooted<ExternRef>>, Option<Rooted<ExternRef>>>(
            &mut store,
            "roundtrip",
        )?;

    {
        let val = ExternRef::new(&mut store, 42u32)?;
        let result = roundtrip.call(&mut store, Some(val))?;
        let result = result.unwrap();
        let data = result
            .data(&store)?
            .expect("should have data")
            .downcast_ref::<u32>()
            .copied()
            .unwrap();
        assert_eq!(data, 42u32);
    }
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn issue_13173_gc_heap_uses_gc_tunables_no_signals() -> Result<()> {
    let _ = env_logger::try_init();

    let mut config = Config::new();
    config.wasm_gc(true);
    config.wasm_function_references(true);
    config.collector(Collector::DeferredReferenceCounting);
    // Set `memory_reservation=0` while leaving `gc_heap_reservation` at its
    // default (4GB on 64-bit). Before the fix, the GC heap would incorrectly
    // use `memory_reservation=0`, giving it 0 capacity and causing segfaults
    // as the heap base pointer changed on every growth.
    config.memory_reservation(0);
    config.signals_based_traps(false);

    let engine = Engine::new(&config)?;

    let module = Module::new(
        &engine,
        r#"
        (module
            (type $arr (array (mut i32)))
            (func (export "run") (result i32)
                (local $i i32)
                (local $a (ref $arr))
                (local.set $a (array.new_default $arr (i32.const 64)))
                (block $break
                    (loop $loop
                        (br_if $break (i32.ge_u (local.get $i) (i32.const 1000)))
                        (array.set $arr
                            (local.get $a)
                            (i32.const 0)
                            (i32.add
                                (array.get $arr (local.get $a) (i32.const 0))
                                (i32.const 1)))
                        (local.set $i (i32.add (local.get $i) (i32.const 1)))
                        (br $loop)
                    )
                )
                (array.get $arr (local.get $a) (i32.const 0))
            )
        )
        "#,
    )?;

    let mut store = Store::new(&engine, ());
    let instance = Instance::new(&mut store, &module, &[])?;
    let run = instance.get_typed_func::<(), i32>(&mut store, "run")?;
    let result = run.call(&mut store, ())?;
    assert_eq!(result, 1000);

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn copying_collector_many_externrefs() -> Result<()> {
    let (mut store, engine) = copying_store_with_gc_zeal(2)?;
    let module = Module::new(
        &engine,
        r#"
        (module
            (import "" "gc" (func $gc))
            (import "" "make" (func $make (param i32) (result externref)))
            (import "" "check" (func $check (param externref i32)))

            (func (export "test")
                (local $a externref)
                (local $b externref)
                (local $c externref)

                (local.set $a (call $make (i32.const 100)))
                (local.set $b (call $make (i32.const 200)))
                (local.set $c (call $make (i32.const 300)))

                (call $gc)

                (call $check (local.get $a) (i32.const 100))
                (call $check (local.get $b) (i32.const 200))
                (call $check (local.get $c) (i32.const 300))
            )
        )
        "#,
    )?;

    let gc_func = Func::wrap(&mut store, |mut caller: Caller<'_, ()>| -> Result<()> {
        caller.gc(None)
    });
    let make = Func::wrap(
        &mut store,
        |mut caller: Caller<'_, ()>, val: i32| -> Result<Option<Rooted<ExternRef>>> {
            Ok(Some(ExternRef::new(&mut caller, val)?))
        },
    );
    let check = Func::wrap(
        &mut store,
        |caller: Caller<'_, ()>, ext: Option<Rooted<ExternRef>>, expected: i32| -> Result<()> {
            let ext = ext.unwrap();
            let val = *ext
                .data(&caller)?
                .expect("data")
                .downcast_ref::<i32>()
                .unwrap();
            assert_eq!(val, expected, "externref value mismatch after GC");
            Ok(())
        },
    );

    let instance = Instance::new(
        &mut store,
        &module,
        &[gc_func.into(), make.into(), check.into()],
    )?;
    let test = instance.get_typed_func::<(), ()>(&mut store, "test")?;
    test.call(&mut store, ())?;
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn issue_13173_gc_heap_uses_gc_tunables_guard_size_mismatch() -> Result<()> {
    if std::mem::size_of::<usize>() < std::mem::size_of::<u64>()
        || std::env::var("WASMTIME_TEST_NO_HOG_MEMORY").is_ok()
    {
        return Ok(());
    }

    let _ = env_logger::try_init();

    let mut config = Config::new();
    config.wasm_gc(true);
    config.wasm_function_references(true);
    config.collector(Collector::DeferredReferenceCounting);
    // Set `memory_reservation=0` and a large `gc_heap_guard_size` while leaving
    // `memory_guard_size` at its default. Before the fix, the GC heap would use
    // `memory_guard_size` (small) instead of `gc_heap_guard_size` (large), causing
    // Cranelift-generated code that relies on virtual memory protection for
    // accesses within the gc_heap_guard_size to segfault.
    config.memory_reservation(0);
    config.gc_heap_guard_size(1 << 29); // 512MB GC heap guard
    config.memory_guard_size(32 * 1024 * 1024); // 32MB linear memory guard

    let engine = Engine::new(&config)?;

    let module = Module::new(
        &engine,
        r#"
        (module
            (type $arr (array (mut i32)))
            (func (export "run") (result i32)
                (local $i i32)
                (local $a (ref $arr))
                (local.set $a (array.new_default $arr (i32.const 64)))
                (block $break
                    (loop $loop
                        (br_if $break (i32.ge_u (local.get $i) (i32.const 1000)))
                        (array.set $arr
                            (local.get $a)
                            (i32.const 0)
                            (i32.add
                                (array.get $arr (local.get $a) (i32.const 0))
                                (i32.const 1)))
                        (local.set $i (i32.add (local.get $i) (i32.const 1)))
                        (br $loop)
                    )
                )
                (array.get $arr (local.get $a) (i32.const 0))
            )
        )
        "#,
    )?;

    let mut store = Store::new(&engine, ());
    let instance = Instance::new(&mut store, &module, &[])?;
    let run = instance.get_typed_func::<(), i32>(&mut store, "run")?;
    let result = run.call(&mut store, ())?;
    assert_eq!(result, 1000);

    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn copying_collector_gc_zeal_counter_stress() -> Result<()> {
    // Many allocations with different `gc_zeal_alloc_counter` values to stress
    // GC timing.
    for counter in [2, 3, 5, 7, 10] {
        let (mut store, engine) = copying_store_with_gc_zeal(counter)?;
        log::debug!("Testing with gc_zeal_alloc_counter = {counter}");

        let module = Module::new(
            &engine,
            r#"
            (module
                (type $box (struct (field i32)))

                (func (export "test") (result i32)
                    (local $keep (ref null $box))
                    (local $i i32)

                    (local.set $keep (struct.new $box (i32.const 12345)))

                    (local.set $i (i32.const 0))
                    (block $done
                        (loop $loop
                            (br_if $done (i32.ge_u (local.get $i) (i32.const 30)))
                            (drop (struct.new $box (local.get $i)))
                            (local.set $i (i32.add (local.get $i) (i32.const 1)))
                            (br $loop)
                        )
                    )

                    (struct.get $box 0 (local.get $keep))
                )
            )
            "#,
        )?;
        let instance = Instance::new(&mut store, &module, &[])?;
        let test = instance.get_typed_func::<(), i32>(&mut store, "test")?;
        let result = test.call(&mut store, ())?;
        assert_eq!(result, 12345, "failed with gc_zeal_counter={counter}");
    }

    Ok(())
}

#[test]
fn anyref_from_eqref_rooted() -> Result<()> {
    let mut store = gc_store()?;
    let eq = EqRef::from_i31(&mut store, I31::wrapping_u32(7));
    // From<Rooted<EqRef>> for Rooted<AnyRef>
    let any: Rooted<AnyRef> = eq.into();
    assert!(any.is_i31(&store)?);
    Ok(())
}

#[test]
fn anyref_from_owned_eqref() -> Result<()> {
    let mut store = gc_store()?;
    let eq = EqRef::from_i31(&mut store, I31::wrapping_u32(3));
    let owned: OwnedRooted<EqRef> = eq.to_owned_rooted(&mut store)?;
    // From<OwnedRooted<EqRef>> for OwnedRooted<AnyRef>
    let any_owned: OwnedRooted<AnyRef> = owned.into();
    let any = any_owned.to_rooted(&mut store);
    assert!(any.is_i31(&store)?);
    Ok(())
}

#[test]
fn anyref_from_structref_rooted() -> Result<()> {
    let mut store = gc_store()?;
    let struct_ty = StructType::new(
        store.engine(),
        [FieldType::new(Mutability::Const, ValType::I32.into())],
    )?;
    let pre = StructRefPre::new(&mut store, struct_ty);
    let s = StructRef::new(&mut store, &pre, &[Val::I32(42)])?;
    // From<Rooted<StructRef>> for Rooted<AnyRef>
    let any: Rooted<AnyRef> = s.into();
    assert!(any.is_struct(&store)?);
    Ok(())
}

#[test]
fn anyref_from_owned_structref() -> Result<()> {
    let mut store = gc_store()?;
    let struct_ty = StructType::new(
        store.engine(),
        [FieldType::new(Mutability::Const, ValType::I32.into())],
    )?;
    let pre = StructRefPre::new(&mut store, struct_ty);
    let s = StructRef::new(&mut store, &pre, &[Val::I32(1)])?;
    let owned = s.to_owned_rooted(&mut store)?;
    // From<OwnedRooted<StructRef>> for OwnedRooted<AnyRef>
    let any_owned: OwnedRooted<AnyRef> = owned.into();
    let any = any_owned.to_rooted(&mut store);
    assert!(any.is_struct(&store)?);
    Ok(())
}

#[test]
fn anyref_from_arrayref_rooted() -> Result<()> {
    let mut store = gc_store()?;
    let array_ty = ArrayType::new(
        store.engine(),
        FieldType::new(Mutability::Const, ValType::I32.into()),
    );
    let pre = ArrayRefPre::new(&mut store, array_ty);
    let a = ArrayRef::new(&mut store, &pre, &Val::I32(0), 2)?;
    // From<Rooted<ArrayRef>> for Rooted<AnyRef>
    let any: Rooted<AnyRef> = a.into();
    assert!(any.is_array(&store)?);
    Ok(())
}

#[test]
fn anyref_from_owned_arrayref() -> Result<()> {
    let mut store = gc_store()?;
    let array_ty = ArrayType::new(
        store.engine(),
        FieldType::new(Mutability::Const, ValType::I32.into()),
    );
    let pre = ArrayRefPre::new(&mut store, array_ty);
    let a = ArrayRef::new(&mut store, &pre, &Val::I32(0), 1)?;
    let owned = a.to_owned_rooted(&mut store)?;
    // From<OwnedRooted<ArrayRef>> for OwnedRooted<AnyRef>
    let any_owned: OwnedRooted<AnyRef> = owned.into();
    let any = any_owned.to_rooted(&mut store);
    assert!(any.is_array(&store)?);
    Ok(())
}

#[test]
fn anyref_ty_struct() -> Result<()> {
    let mut store = gc_store()?;
    let struct_ty = StructType::new(
        store.engine(),
        [FieldType::new(Mutability::Const, ValType::I32.into())],
    )?;
    let pre = StructRefPre::new(&mut store, struct_ty);
    let s = StructRef::new(&mut store, &pre, &[Val::I32(0)])?;
    let any: Rooted<AnyRef> = s.into();
    let ty = any.ty(&store)?;
    assert!(matches!(ty, HeapType::ConcreteStruct(_)));
    Ok(())
}

#[test]
fn anyref_ty_extern_converted() -> Result<()> {
    let mut store = gc_store()?;
    // ExternRef converted to AnyRef has HeapType::Any
    let ext = ExternRef::new(&mut store, 42u32)?;
    let any = AnyRef::convert_extern(&mut store, ext)?;
    let ty = any.ty(&store)?;
    assert!(matches!(ty, HeapType::Any));
    Ok(())
}

#[test]
fn anyref_to_raw_struct() -> Result<()> {
    let mut store = gc_store()?;
    let struct_ty = StructType::new(
        store.engine(),
        [FieldType::new(Mutability::Const, ValType::I32.into())],
    )?;
    let pre = StructRefPre::new(&mut store, struct_ty);
    let s = StructRef::new(&mut store, &pre, &[Val::I32(99)])?;
    let any: Rooted<AnyRef> = s.into();
    // to_raw on a non-i31 anyref exercises the else-branch in to_raw
    let raw = any.to_raw(&mut store)?;
    assert_ne!(raw, 0);
    Ok(())
}

#[test]
fn externref_data_some() -> Result<()> {
    let mut store = gc_store()?;
    let rooted = ExternRef::new(&mut store, 100u32)?;
    let data = rooted.data(&store)?;
    let val = data.unwrap().downcast_ref::<u32>().unwrap();
    assert_eq!(*val, 100u32);
    Ok(())
}

#[test]
fn externref_data_mut_some() -> Result<()> {
    let mut store = gc_store()?;
    let rooted = ExternRef::new(&mut store, 1u32)?;
    {
        let data = rooted.data_mut(&mut store)?;
        *data.unwrap().downcast_mut::<u32>().unwrap() = 999;
    }
    let val = rooted.data(&store)?.unwrap().downcast_ref::<u32>().unwrap();
    assert_eq!(*val, 999u32);
    Ok(())
}

#[test]
fn externref_data_none_when_converted_from_anyref() -> Result<()> {
    let mut store = gc_store()?;
    let any = AnyRef::from_i31(&mut store, I31::wrapping_u32(5));
    // ExternRef wrapping an anyref has no host data
    let ext = ExternRef::convert_any(&mut store, any)?;
    let data = ext.data(&store)?;
    assert!(data.is_none());
    Ok(())
}

#[test]
fn externref_data_mut_none_when_converted_from_anyref() -> Result<()> {
    let mut store = gc_store()?;
    let any = AnyRef::from_i31(&mut store, I31::wrapping_u32(5));
    let ext = ExternRef::convert_any(&mut store, any)?;
    let data = ext.data_mut(&mut store)?;
    assert!(data.is_none());
    Ok(())
}

#[test]
fn owned_rooted_clone() -> Result<()> {
    let mut store = gc_store()?;
    let ext = ExternRef::new(&mut store, 77u32)?;
    let owned = ext.to_owned_rooted(&mut store)?;
    let clone = owned.clone();
    // Both owned and clone should refer to the same ExternRef
    let r1 = owned.to_rooted(&mut store);
    let r2 = clone.to_rooted(&mut store);
    assert!(Rooted::ref_eq(&store, &r1, &r2)?);
    Ok(())
}

#[test]
fn anyref_gc_is_eqref() -> Result<()> {
    let mut store = gc_store()?;
    let any = AnyRef::from_i31(&mut store, I31::wrapping_u32(3));
    assert!(any.is_eqref(&store)?);
    let struct_ty = StructType::new(store.engine(), [])?;
    let pre = StructRefPre::new(&mut store, struct_ty);
    let s = StructRef::new(&mut store, &pre, &[])?;
    let any: Rooted<AnyRef> = s.into();
    assert!(any.is_eqref(&store)?);
    // extern-converted anyref is not eqref
    let ext = ExternRef::new(&mut store, 42u32)?;
    let any = AnyRef::convert_extern(&mut store, ext)?;
    assert!(!any.is_eqref(&store)?);
    Ok(())
}

#[test]
fn anyref_gc_as_eqref() -> Result<()> {
    let mut store = gc_store()?;
    let any = AnyRef::from_i31(&mut store, I31::wrapping_u32(99));
    assert!(any.as_eqref(&mut store)?.is_some());
    let ext = ExternRef::new(&mut store, 42u32)?;
    let any = AnyRef::convert_extern(&mut store, ext)?;
    assert!(any.as_eqref(&mut store)?.is_none());
    Ok(())
}

#[test]
fn anyref_unwrap_eqref() -> Result<()> {
    let mut store = gc_store()?;
    let any = AnyRef::from_i31(&mut store, I31::wrapping_u32(5));
    let _eq = any.unwrap_eqref(&mut store)?;
    Ok(())
}

#[test]
fn anyref_gc_is_i31() -> Result<()> {
    let mut store = gc_store()?;
    let any = AnyRef::from_i31(&mut store, I31::wrapping_u32(10));
    assert!(any.is_i31(&store)?);
    let struct_ty = StructType::new(store.engine(), [])?;
    let pre = StructRefPre::new(&mut store, struct_ty);
    let s = StructRef::new(&mut store, &pre, &[])?;
    let any: Rooted<AnyRef> = s.into();
    assert!(!any.is_i31(&store)?);
    Ok(())
}

#[test]
fn anyref_gc_as_i31() -> Result<()> {
    let mut store = gc_store()?;
    let any = AnyRef::from_i31(&mut store, I31::wrapping_u32(55));
    assert_eq!(any.as_i31(&store)?.unwrap().get_u32(), 55);
    let struct_ty = StructType::new(store.engine(), [])?;
    let pre = StructRefPre::new(&mut store, struct_ty);
    let s = StructRef::new(&mut store, &pre, &[])?;
    let any: Rooted<AnyRef> = s.into();
    assert!(any.as_i31(&store)?.is_none());
    Ok(())
}

#[test]
fn anyref_unwrap_i31() -> Result<()> {
    let mut store = gc_store()?;
    let any = AnyRef::from_i31(&mut store, I31::wrapping_u32(9));
    let i31 = any.unwrap_i31(&store)?;
    assert_eq!(i31.get_u32(), 9);
    Ok(())
}

#[test]
fn anyref_gc_is_struct() -> Result<()> {
    let mut store = gc_store()?;
    let struct_ty = StructType::new(store.engine(), [])?;
    let pre = StructRefPre::new(&mut store, struct_ty);
    let s = StructRef::new(&mut store, &pre, &[])?;
    let any: Rooted<AnyRef> = s.into();
    assert!(any.is_struct(&store)?);
    let any = AnyRef::from_i31(&mut store, I31::wrapping_u32(0));
    assert!(!any.is_struct(&store)?);
    Ok(())
}

#[test]
fn anyref_gc_as_struct() -> Result<()> {
    let mut store = gc_store()?;
    let struct_ty = StructType::new(store.engine(), [])?;
    let pre = StructRefPre::new(&mut store, struct_ty);
    let s = StructRef::new(&mut store, &pre, &[])?;
    let any: Rooted<AnyRef> = s.into();
    assert!(any.as_struct(&store)?.is_some());
    let any = AnyRef::from_i31(&mut store, I31::wrapping_u32(0));
    assert!(any.as_struct(&store)?.is_none());
    Ok(())
}

#[test]
fn anyref_unwrap_struct() -> Result<()> {
    let mut store = gc_store()?;
    let struct_ty = StructType::new(store.engine(), [])?;
    let pre = StructRefPre::new(&mut store, struct_ty);
    let s = StructRef::new(&mut store, &pre, &[])?;
    let any: Rooted<AnyRef> = s.into();
    let _s2 = any.unwrap_struct(&store)?;
    Ok(())
}

#[test]
fn anyref_gc_is_array() -> Result<()> {
    let mut store = gc_store()?;
    let array_ty = ArrayType::new(
        store.engine(),
        FieldType::new(Mutability::Const, ValType::I32.into()),
    );
    let pre = ArrayRefPre::new(&mut store, array_ty);
    let a = ArrayRef::new(&mut store, &pre, &Val::I32(0), 1)?;
    let any: Rooted<AnyRef> = a.into();
    assert!(any.is_array(&store)?);
    let any = AnyRef::from_i31(&mut store, I31::wrapping_u32(0));
    assert!(!any.is_array(&store)?);
    Ok(())
}

#[test]
fn anyref_gc_as_array() -> Result<()> {
    let mut store = gc_store()?;
    let array_ty = ArrayType::new(
        store.engine(),
        FieldType::new(Mutability::Const, ValType::I32.into()),
    );
    let pre = ArrayRefPre::new(&mut store, array_ty);
    let a = ArrayRef::new(&mut store, &pre, &Val::I32(0), 1)?;
    let any: Rooted<AnyRef> = a.into();
    assert!(any.as_array(&store)?.is_some());
    let any = AnyRef::from_i31(&mut store, I31::wrapping_u32(0));
    assert!(any.as_array(&store)?.is_none());
    Ok(())
}

#[test]
fn anyref_unwrap_array() -> Result<()> {
    let mut store = gc_store()?;
    let array_ty = ArrayType::new(
        store.engine(),
        FieldType::new(Mutability::Const, ValType::I32.into()),
    );
    let pre = ArrayRefPre::new(&mut store, array_ty);
    let a = ArrayRef::new(&mut store, &pre, &Val::I32(0), 1)?;
    let any: Rooted<AnyRef> = a.into();
    let _a2 = any.unwrap_array(&store)?;
    Ok(())
}

#[test]
fn anyref_matches_ty_any_and_eq() -> Result<()> {
    let mut store = gc_store()?;
    let any = AnyRef::from_i31(&mut store, I31::wrapping_u32(1));
    assert!(any.matches_ty(&store, &HeapType::Any)?);
    assert!(any.matches_ty(&store, &HeapType::Eq)?);
    assert!(!any.matches_ty(&store, &HeapType::Extern)?);
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_func_returns_anyref() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "get_any") (result anyref)
            (ref.i31 (i32.const 42)))
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f = instance.get_typed_func::<(), Option<Rooted<AnyRef>>>(&mut store, "get_any")?;
    let result = f.call(&mut store, ())?;
    let any = result.unwrap();
    let i31 = any.as_i31(&store)?.unwrap();
    assert_eq!(i31.get_u32(), 42);
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_func_takes_anyref() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "take_any") (param anyref) (result i32)
            (ref.test i31ref (local.get 0)))
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f = instance.get_typed_func::<Option<Rooted<AnyRef>>, i32>(&mut store, "take_any")?;
    let any = AnyRef::from_i31(&mut store, I31::wrapping_u32(5));
    let result = f.call(&mut store, Some(any))?;
    assert_eq!(result, 1);
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_func_returns_eqref() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "get_eq") (result eqref)
            (ref.i31 (i32.const 7)))
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f = instance.get_typed_func::<(), Option<Rooted<EqRef>>>(&mut store, "get_eq")?;
    let result = f.call(&mut store, ())?;
    let eq = result.unwrap();
    let i31 = eq.as_i31(&store)?.unwrap();
    assert_eq!(i31.get_u32(), 7);
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_func_takes_eqref() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "take_eq") (param eqref) (result i32)
            (ref.test i31ref (local.get 0)))
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f = instance.get_typed_func::<Option<Rooted<EqRef>>, i32>(&mut store, "take_eq")?;
    let any = EqRef::from_i31(&mut store, I31::wrapping_u32(3));
    let result = f.call(&mut store, Some(any))?;
    assert_eq!(result, 1);
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_func_returns_externref() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "get_ext") (result externref)
            (ref.null extern))
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f = instance.get_typed_func::<(), Option<Rooted<ExternRef>>>(&mut store, "get_ext")?;
    let result = f.call(&mut store, ())?;
    assert!(result.is_none());
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_func_takes_externref() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "take_ext") (param externref) (result i32)
            (ref.is_null (local.get 0)))
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f = instance.get_typed_func::<Option<Rooted<ExternRef>>, i32>(&mut store, "take_ext")?;
    let result = f.call(&mut store, None)?;
    assert_eq!(result, 1);
    Ok(())
}

#[test]
fn anyref_ty_array() -> Result<()> {
    let mut store = gc_store()?;
    let array_ty = ArrayType::new(
        store.engine(),
        FieldType::new(Mutability::Const, ValType::I32.into()),
    );
    let pre = ArrayRefPre::new(&mut store, array_ty);
    let a = ArrayRef::new(&mut store, &pre, &Val::I32(0), 1)?;
    let any: Rooted<AnyRef> = a.into();
    let ty = any.ty(&store)?;
    assert!(matches!(ty, HeapType::ConcreteArray(_)));
    Ok(())
}

#[test]
fn eqref_ty_i31() -> Result<()> {
    let mut store = gc_store()?;
    let eq = EqRef::from_i31(&mut store, I31::wrapping_u32(10));
    let ty = eq.ty(&store)?;
    assert!(matches!(ty, HeapType::I31));
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_func_nonnull_returns_anyref() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "get") (result (ref any))
            (ref.i31 (i32.const 11)))
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f = instance.get_typed_func::<(), Rooted<AnyRef>>(&mut store, "get")?;
    let any = f.call(&mut store, ())?;
    assert!(any.is_i31(&store)?);
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_func_nonnull_takes_anyref() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "take") (param (ref any)) (result i32)
            (ref.test i31ref (local.get 0)))
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f = instance.get_typed_func::<Rooted<AnyRef>, i32>(&mut store, "take")?;
    let any = AnyRef::from_i31(&mut store, I31::wrapping_u32(5));
    let r = f.call(&mut store, any)?;
    assert_eq!(r, 1);
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_func_nonnull_returns_eqref() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "get") (result (ref eq))
            (ref.i31 (i32.const 22)))
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f = instance.get_typed_func::<(), Rooted<EqRef>>(&mut store, "get")?;
    let eq = f.call(&mut store, ())?;
    assert!(eq.is_i31(&store)?);
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_func_nonnull_takes_eqref() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "take") (param (ref eq)) (result i32)
            (ref.test i31ref (local.get 0)))
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f = instance.get_typed_func::<Rooted<EqRef>, i32>(&mut store, "take")?;
    let eq = EqRef::from_i31(&mut store, I31::wrapping_u32(3));
    let r = f.call(&mut store, eq)?;
    assert_eq!(r, 1);
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_func_nonnull_returns_externref() -> Result<()> {
    let mut store = gc_store()?;
    // Use a WASM helper: store an externref in a table, return it
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (import "" "ext" (global (ref extern)))
          (func (export "get") (result (ref extern))
            (global.get 0))
        )
        "#,
    )?;
    let ext = ExternRef::new(&mut store, 99u32)?;
    let global_ty = GlobalType::new(
        ValType::Ref(RefType::new(false, HeapType::Extern)),
        Mutability::Const,
    );
    let global = Global::new(&mut store, global_ty, Val::ExternRef(Some(ext)))?;
    let instance = Instance::new(&mut store, &module, &[global.into()])?;
    let f = instance.get_typed_func::<(), Rooted<ExternRef>>(&mut store, "get")?;
    let ext2 = f.call(&mut store, ())?;
    let data = ext2.data(&store)?.unwrap();
    assert_eq!(*data.downcast_ref::<u32>().unwrap(), 99u32);
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_func_nonnull_takes_externref() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "take") (param (ref extern)) (result i32)
            (i32.const 1))
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f = instance.get_typed_func::<Rooted<ExternRef>, i32>(&mut store, "take")?;
    let ext = ExternRef::new(&mut store, 1u32)?;
    let r = f.call(&mut store, ext)?;
    assert_eq!(r, 1);
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn externref_null_data_is_none() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "get_null") (result externref)
            (ref.null extern))
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f = instance.get_typed_func::<(), Option<Rooted<ExternRef>>>(&mut store, "get_null")?;
    let result = f.call(&mut store, ())?;
    assert!(result.is_none());
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_i31ref_roundtrip() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "nonnull") (param (ref i31)) (result (ref i31)) local.get 0)
          (func (export "nullable") (param i31ref) (result i31ref) local.get 0)
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f = instance.get_typed_func::<I31, I31>(&mut store, "nonnull")?;
    assert_eq!(f.call(&mut store, I31::wrapping_u32(77))?.get_u32(), 77);
    let f2 = instance.get_typed_func::<Option<I31>, Option<I31>>(&mut store, "nullable")?;
    assert_eq!(
        f2.call(&mut store, Some(I31::wrapping_u32(33)))?
            .unwrap()
            .get_u32(),
        33
    );
    assert!(f2.call(&mut store, None)?.is_none());
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_owned_anyref_roundtrip() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "nonnull") (param (ref any)) (result (ref any)) local.get 0)
          (func (export "nullable") (param anyref) (result anyref) local.get 0)
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f = instance
        .get_typed_func::<OwnedRooted<AnyRef>, OwnedRooted<AnyRef>>(&mut store, "nonnull")?;
    let owned = AnyRef::from_i31(&mut store, I31::wrapping_u32(7)).to_owned_rooted(&mut store)?;
    let result = f.call(&mut store, owned)?;
    assert!(result.to_rooted(&mut store).is_i31(&store)?);
    let f2 = instance.get_typed_func::<Option<OwnedRooted<AnyRef>>, Option<OwnedRooted<AnyRef>>>(
        &mut store, "nullable",
    )?;
    let owned2 = AnyRef::from_i31(&mut store, I31::wrapping_u32(3)).to_owned_rooted(&mut store)?;
    assert!(f2.call(&mut store, Some(owned2))?.is_some());
    assert!(f2.call(&mut store, None)?.is_none());
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_owned_eqref_roundtrip() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "nonnull") (param (ref eq)) (result (ref eq)) local.get 0)
          (func (export "nullable") (param eqref) (result eqref) local.get 0)
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f =
        instance.get_typed_func::<OwnedRooted<EqRef>, OwnedRooted<EqRef>>(&mut store, "nonnull")?;
    let owned = EqRef::from_i31(&mut store, I31::wrapping_u32(5)).to_owned_rooted(&mut store)?;
    let result = f.call(&mut store, owned)?;
    assert!(result.to_rooted(&mut store).is_i31(&store)?);
    let f2 = instance.get_typed_func::<Option<OwnedRooted<EqRef>>, Option<OwnedRooted<EqRef>>>(
        &mut store, "nullable",
    )?;
    let owned2 = EqRef::from_i31(&mut store, I31::wrapping_u32(2)).to_owned_rooted(&mut store)?;
    assert!(f2.call(&mut store, Some(owned2))?.is_some());
    assert!(f2.call(&mut store, None)?.is_none());
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_owned_externref_roundtrip() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "nonnull") (param (ref extern)) (result (ref extern)) local.get 0)
          (func (export "nullable") (param externref) (result externref) local.get 0)
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f = instance
        .get_typed_func::<OwnedRooted<ExternRef>, OwnedRooted<ExternRef>>(&mut store, "nonnull")?;
    let owned = ExternRef::new(&mut store, 42u32)?.to_owned_rooted(&mut store)?;
    let result = f.call(&mut store, owned)?;
    let rooted = result.to_rooted(&mut store);
    assert_eq!(
        *rooted.data(&store)?.unwrap().downcast_ref::<u32>().unwrap(),
        42u32
    );
    let f2 = instance
        .get_typed_func::<Option<OwnedRooted<ExternRef>>, Option<OwnedRooted<ExternRef>>>(
            &mut store, "nullable",
        )?;
    let owned2 = ExternRef::new(&mut store, 1u32)?.to_owned_rooted(&mut store)?;
    assert!(f2.call(&mut store, Some(owned2))?.is_some());
    assert!(f2.call(&mut store, None)?.is_none());
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_owned_structref_roundtrip() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "nonnull") (param (ref struct)) (result (ref struct)) local.get 0)
          (func (export "nullable") (param structref) (result structref) local.get 0)
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let struct_ty = StructType::new(store.engine(), [])?;
    let pre = StructRefPre::new(&mut store, struct_ty);
    let f = instance
        .get_typed_func::<OwnedRooted<StructRef>, OwnedRooted<StructRef>>(&mut store, "nonnull")?;
    let owned = StructRef::new(&mut store, &pre, &[])?.to_owned_rooted(&mut store)?;
    let result = f.call(&mut store, owned)?;
    let _ = result.to_rooted(&mut store);
    let f2 = instance
        .get_typed_func::<Option<OwnedRooted<StructRef>>, Option<OwnedRooted<StructRef>>>(
            &mut store, "nullable",
        )?;
    let owned2 = StructRef::new(&mut store, &pre, &[])?.to_owned_rooted(&mut store)?;
    assert!(f2.call(&mut store, Some(owned2))?.is_some());
    assert!(f2.call(&mut store, None)?.is_none());
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_owned_arrayref_roundtrip() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "nonnull") (param (ref array)) (result (ref array)) local.get 0)
          (func (export "nullable") (param arrayref) (result arrayref) local.get 0)
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let array_ty = ArrayType::new(
        store.engine(),
        FieldType::new(Mutability::Const, ValType::I32.into()),
    );
    let pre = ArrayRefPre::new(&mut store, array_ty);
    let f = instance
        .get_typed_func::<OwnedRooted<ArrayRef>, OwnedRooted<ArrayRef>>(&mut store, "nonnull")?;
    let owned = ArrayRef::new(&mut store, &pre, &Val::I32(0), 3)?.to_owned_rooted(&mut store)?;
    let result = f.call(&mut store, owned)?;
    assert_eq!(result.to_rooted(&mut store).len(&store)?, 3);
    let f2 = instance
        .get_typed_func::<Option<OwnedRooted<ArrayRef>>, Option<OwnedRooted<ArrayRef>>>(
            &mut store, "nullable",
        )?;
    let owned2 = ArrayRef::new(&mut store, &pre, &Val::I32(0), 1)?.to_owned_rooted(&mut store)?;
    assert!(f2.call(&mut store, Some(owned2))?.is_some());
    assert!(f2.call(&mut store, None)?.is_none());
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_option_noneref() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "roundtrip") (param (ref null none)) (result (ref null none))
            local.get 0)
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f = instance.get_typed_func::<Option<NoneRef>, Option<NoneRef>>(&mut store, "roundtrip")?;
    assert!(f.call(&mut store, None)?.is_none());
    Ok(())
}

#[test]
#[cfg_attr(miri, ignore)]
fn typed_option_noextern() -> Result<()> {
    let mut store = gc_store()?;
    let module = Module::new(
        store.engine(),
        r#"
        (module
          (func (export "roundtrip") (param (ref null noextern)) (result (ref null noextern))
            local.get 0)
        )
        "#,
    )?;
    let instance = Instance::new(&mut store, &module, &[])?;
    let f =
        instance.get_typed_func::<Option<NoExtern>, Option<NoExtern>>(&mut store, "roundtrip")?;
    assert!(f.call(&mut store, None)?.is_none());
    Ok(())
}

/// A test that performs a GC without actually compiling or running any Wasm
/// functions so we can run this test under MIRI.
#[test]
// FIXME: need to fold this into other miri testing which precompiles
#[cfg_attr(miri, ignore)]
fn miri_gc_smoke_test() -> Result<()> {
    for collector in [
        Collector::Copying,
        Collector::Null,
        Collector::DeferredReferenceCounting,
    ] {
        eprintln!("===== Collector: {collector:?} =====");

        let mut config = Config::new();
        config.wasm_gc(true);
        config.wasm_function_references(true);
        config.wasm_exceptions(true);
        config.collector(collector);

        let engine = Engine::new(&config)?;

        let struct_ty = StructType::new(
            &engine,
            [FieldType::new(Mutability::Const, StorageType::I8)],
        )?;
        let table_ty = TableType::new(RefType::ANYREF, 1, None);
        let global_ty = GlobalType::new(RefType::ANYREF.into(), Mutability::Var);
        let exn_ty = ExnType::new(&engine, [ValType::I32])?;
        let func_ty = FuncType::new(&engine, Some(ValType::I32), None);
        let tag_ty = TagType::new(func_ty);

        let module = Module::new(
            &engine,
            r#"
                (module
                    (type $s (struct (field i8)))

                    (global (export "g") (ref null $s) (struct.new $s (i32.const 1)))

                    (table $t (export "t") 1 (ref null $s))
                    (elem (table $t) (i32.const 0) (ref null $s) (struct.new $s (i32.const 2)))

                    ;; Can't actually do anything with this without running Wasm,
                    ;; but have it here anyways in case it trips anything in MIRI.
                    (elem anyref (struct.new $s (i32.const 0xff)))
                )
            "#,
        )?;

        let mut store = Store::new(&engine, ());

        let pre = StructRefPre::new(&mut store, struct_ty);
        let instance = Instance::new(&mut store, &module, &[])?;

        // Host table root.
        let table = {
            let mut store = RootScope::new(&mut store);
            let table = Table::new(&mut store, table_ty, Ref::Any(None))?;
            let s = StructRef::new(&mut store, &pre, &[Val::I32(3)])?;
            table.set(&mut store, 0, s.into())?;
            table
        };

        // Host global root.
        let global = {
            let mut store = RootScope::new(&mut store);
            let s = StructRef::new(&mut store, &pre, &[Val::I32(4)])?;
            Global::new(&mut store, global_ty, s.into())?
        };

        // Rooted<T> root.
        let rooted = StructRef::new(&mut store, &pre, &[Val::I32(5)])?;

        // OwnedRooted<T> root.
        let owned_rooted = {
            let mut store = RootScope::new(&mut store);
            StructRef::new(&mut store, &pre, &[Val::I32(6)])?.to_owned_rooted(&mut store)?
        };

        // Pending exception root.
        {
            let mut store = RootScope::new(&mut store);
            let pre = ExnRefPre::new(&mut store, exn_ty);
            let tag = Tag::new(&mut store, &tag_ty)?;
            let e = ExnRef::new(&mut store, &pre, &tag, &[Val::I32(7)])?;
            let _ = store.as_context_mut().throw::<()>(e);
        }

        // Do a GC!
        store.gc(None)?;

        let assert_field_value = |store: &mut Store<_>, val: Val, field: i32| -> Result<()> {
            let s = val.anyref().unwrap().unwrap().unwrap_struct(&store)?;
            let f = s.field(store, 0)?;
            assert_eq!(f.unwrap_i32(), field);
            Ok(())
        };

        // Instance global.
        {
            let global = instance.get_global(&mut store, "g").unwrap();
            let global_val = global.get(&mut store);
            assert_field_value(&mut store, global_val, 1)?;
        }

        // Instance table.
        {
            let table = instance.get_table(&mut store, "t").unwrap();
            let table_val = table.get(&mut store, 0).unwrap();
            assert_field_value(&mut store, table_val.into(), 2)?;
        }

        // Host table.
        let table_val = table.get(&mut store, 0).unwrap();
        assert_field_value(&mut store, table_val.into(), 3)?;

        // Host global.
        let global_val = global.get(&mut store);
        assert_field_value(&mut store, global_val, 4)?;

        // Rooted<T>.
        assert_field_value(&mut store, rooted.into(), 5)?;

        // OwnedRooted<T>.
        let owned_rooted = owned_rooted.to_rooted(&mut store);
        assert_field_value(&mut store, owned_rooted.into(), 6)?;

        // Pending exception.
        let e = store.take_pending_exception().unwrap();
        let e_val = e.field(&mut store, 0)?;
        assert_eq!(e_val.unwrap_i32(), 7);
    }

    Ok(())
}

#[tokio::test]
#[cfg_attr(miri, ignore)]
async fn copying_collector_async_gc_yields() -> Result<()> {
    let _ = env_logger::try_init();

    if crate::no_hog_memory() {
        return Ok(());
    }

    let mut config = Config::new();
    config.wasm_gc(true);
    config.wasm_function_references(true);
    config.collector(Collector::Copying);

    let engine = Engine::new(&config)?;

    const NUM_OBJECTS: usize = 100_000;
    const OBJECTS_PER_INCREMENT: usize = 16_384;

    let module = Module::new(
        &engine,
        r#"
            (module
                (type $node (struct (field i32) (field (ref null $node))))
                (global $list (export "list") (mut (ref null $node)) (ref.null $node))

                (func (export "build") (param $n i32)
                    (local $i i32)
                    (local.set $i (i32.const 0))
                    (block $done
                        (loop $loop
                            (br_if $done (i32.ge_u (local.get $i) (local.get $n)))
                            (global.set $list
                                (struct.new $node
                                    (local.get $i)
                                    (global.get $list)
                                )
                            )
                            (local.set $i (i32.add (local.get $i) (i32.const 1)))
                            (br $loop)
                        )
                    )
                )
            )
        "#,
    )?;

    let mut store = Store::new(&engine, ());
    let instance = Instance::new(&mut store, &module, &[])?;
    let build = instance.get_typed_func::<i32, ()>(&mut store, "build")?;
    build.call(&mut store, NUM_OBJECTS as i32)?;

    let gc_future = store.gc_async(None);
    let (result, pending_count) = CountPending::new(Box::pin(gc_future)).await;
    result?;

    let min_expected = NUM_OBJECTS / OBJECTS_PER_INCREMENT;
    assert!(
        pending_count >= min_expected,
        "expected at least {min_expected} yields but got {pending_count}",
    );

    Ok(())
}

// Cancel the wrapped future once a host-triggered collection is observed to be
// `skip` pending-polls underway (so we are past root processing / the semispace
// flip and into the actual copying), simulating an async cancellation that
// drops a `call_async` future while a GC is in progress.
struct CancelDuringGc<F> {
    future: F,
    in_gc: Arc<AtomicUsize>,
    skip: u32,
    seen_in_gc: u32,
}

impl<F: Future> Future for CancelDuringGc<F> {
    type Output = Option<F::Output>;

    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
        let me = unsafe { self.get_unchecked_mut() };
        let future = unsafe { Pin::new_unchecked(&mut me.future) };
        match future.poll(cx) {
            Poll::Ready(val) => Poll::Ready(Some(val)),
            Poll::Pending => {
                if me.in_gc.load(SeqCst) != 0 {
                    me.seen_in_gc += 1;
                    if me.seen_in_gc >= me.skip {
                        return Poll::Ready(None);
                    }
                }
                Poll::Pending
            }
        }
    }
}

// Test that cancelling a Wasm-execution future mid-collection and then calling
// into Wasm again doesn't trigger or observe any GC heap corruption.
#[tokio::test]
#[cfg_attr(miri, ignore)]
async fn issue_13516_copying_collector_cancellation_during_gc() -> Result<()> {
    let _ = env_logger::try_init();

    if crate::no_hog_memory() {
        return Ok(());
    }

    let mut config = Config::new();
    config.wasm_gc(true);
    config.wasm_function_references(true);
    config.collector(Collector::Copying);

    let engine = Engine::new(&config)?;
    let mut store = Store::new(&engine, ());

    // Build a linked list of `N` nodes rooted in a global so a collection has a
    // non-trivial worklist to copy (giving a wide window in which to cancel),
    // then trigger a host collection.
    const N: i32 = 100_000;
    let module = Module::new(
        &engine,
        r#"
            (module
                (import "" "gc" (func $gc))

                (type $node (struct (field (ref null $node))))

                (global $list (mut (ref null $node)) (ref.null $node))

                (func (export "run") (param $n i32)
                    (block $done
                        (loop $loop
                            (br_if $done (i32.eqz (local.get $n)))
                            (global.set $list (struct.new $node (global.get $list)))
                            (local.set $n (i32.sub (local.get $n) (i32.const 1)))
                            (br $loop)
                        )
                    )
                    (call $gc)
                )
            )
        "#,
    )?;

    // A host function that performs an async collection, flagging while it runs.
    let in_gc = Arc::new(AtomicUsize::new(0));
    let mut linker = Linker::new(&engine);
    linker.func_wrap_async("", "gc", {
        let in_gc = in_gc.clone();
        move |mut caller: Caller<'_, ()>, _: ()| {
            let in_gc = in_gc.clone();
            Box::new(async move {
                in_gc.fetch_add(1, SeqCst);
                let r = caller.gc_async(None).await;
                in_gc.store(0, SeqCst);
                r
            })
        }
    })?;

    let instance = linker.instantiate_async(&mut store, &module).await?;
    let run = instance.get_typed_func::<i32, ()>(&mut store, "run")?;

    // Drive a call that triggers a collection, then cancel it mid-collection.
    let cancelled = CancelDuringGc {
        future: run.call_async(&mut store, N),
        in_gc: in_gc.clone(),
        skip: 8,
        seen_in_gc: 0,
    }
    .await
    .is_none();
    assert!(
        cancelled,
        "the collection should have been cancelled mid-flight"
    );

    // Reusing the store after a mid-collection cancellation must not corrupt the
    // GC heap.
    run.call_async(&mut store, N).await?;

    Ok(())
}