use std::ops::Deref;
use std::ptr::NonNull;
use std::sync::atomic::AtomicU64;
use crate::any::AnyView;
use crate::derive::ObjectRef;
use crate::type_traits::AnyCompatible;
use tvm_ffi_sys::TVMFFITypeIndex as TypeIndex;
use tvm_ffi_sys::{COMBINED_REF_COUNT_BOTH_ONE, TVMFFIAny, TVMFFIGetTypeInfo, TVMFFIObject};
static TYPE_REGISTRATION: std::sync::Mutex<()> = std::sync::Mutex::new(());
#[doc(hidden)]
pub fn lock_type_registration() -> std::sync::MutexGuard<'static, ()> {
TYPE_REGISTRATION
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
#[doc(hidden)]
pub unsafe fn assert_registered_type_matches<T: ObjectCore>(
type_index: i32,
parent: i32,
binding: bool,
) {
let info = unsafe { &*TVMFFIGetTypeInfo(type_index) };
let depth = T::TYPE_DEPTH;
let same_parent = unsafe {
info.type_depth == depth
&& depth > 0
&& (**info.type_acenstors.add(depth as usize - 1)).type_index == parent
};
let size = std::mem::size_of::<T>();
let same_size = info.metadata.is_null() || {
let registered = unsafe { (*info.metadata).total_size as usize };
size == registered || (binding && size < registered)
};
assert!(
same_parent && same_size,
"Type key {} is already registered with another parent, depth or size",
T::TYPE_KEY
);
}
#[repr(C)]
pub struct Object {
header: TVMFFIObject,
}
impl Object {
#[allow(clippy::new_without_default)]
pub fn new() -> Self {
Self {
header: TVMFFIObject::new(),
}
}
}
unsafe impl ObjectCore for Object {
type Parent = ();
const TYPE_KEY: &'static str = "ffi.Object";
const TYPE_DEPTH: i32 = 0;
#[inline]
fn type_index() -> i32 {
TypeIndex::kTVMFFIStaticObjectBegin as i32
}
#[inline]
unsafe fn object_header_mut(this: &mut Self) -> &mut TVMFFIObject {
&mut this.header
}
}
#[repr(C)]
pub struct ObjectArc<T: ObjectCore> {
ptr: std::ptr::NonNull<T>,
_phantom: std::marker::PhantomData<T>,
}
impl ObjectArc<Object> {
pub(crate) unsafe fn new_opaque_unchecked<T>(data: T, type_index: i32) -> Self {
unsafe {
Self {
ptr: allocate_object(data, type_index).cast(),
_phantom: std::marker::PhantomData,
}
}
}
}
impl<T: ObjectCore> ObjectArc<T> {
pub fn new(data: T) -> Self {
let type_index = T::type_index();
unsafe {
Self {
ptr: allocate_object(data, type_index),
_phantom: std::marker::PhantomData,
}
}
}
pub unsafe fn new_with_extra_items<U>(data: T) -> Self
where
T: ObjectCoreWithExtraItems<ExtraItem = U>,
{
let data = std::mem::ManuallyDrop::new(data);
unsafe {
assert_eq!(std::mem::align_of::<T>() % std::mem::align_of::<U>(), 0);
assert_eq!(std::mem::size_of::<T>() % std::mem::align_of::<U>(), 0);
let extra_items_count = T::extra_items_count(&data);
let layout = unsafe_::extra_items_layout::<T, U>(extra_items_count);
let raw_data_ptr = std::alloc::alloc(layout);
if raw_data_ptr.is_null() {
std::alloc::handle_alloc_error(layout);
}
let ptr = raw_data_ptr as *mut T;
std::ptr::write(ptr, std::mem::ManuallyDrop::into_inner(data));
std::ptr::write(
ptr as *mut TVMFFIObject,
TVMFFIObject {
combined_ref_count: AtomicU64::new(COMBINED_REF_COUNT_BOTH_ONE),
type_index: T::type_index(),
__padding: 0,
deleter: Some(unsafe_::object_deleter_for_new_with_extra_items::<T, U>),
},
);
Self {
ptr: std::ptr::NonNull::new_unchecked(ptr),
_phantom: std::marker::PhantomData,
}
}
}
#[inline]
pub(crate) unsafe fn is_null(this: *const Self) -> bool {
unsafe { (*std::ptr::addr_of!((*this).ptr).cast::<*const T>()).is_null() }
}
#[inline(always)]
pub unsafe fn from_raw(ptr: *const T) -> Self {
unsafe {
Self {
ptr: std::ptr::NonNull::new_unchecked(ptr as *mut T),
_phantom: std::marker::PhantomData,
}
}
}
#[inline(always)]
pub fn into_raw(this: Self) -> *const T {
let droped_this = std::mem::ManuallyDrop::new(this);
droped_this.ptr.as_ptr() as *const T
}
#[inline]
pub fn as_ptr(this: &Self) -> *const T {
this.ptr.as_ptr() as *const T
}
#[inline]
pub fn as_mut_ptr(this: &mut Self) -> *mut T {
this.ptr.as_ptr()
}
#[inline]
pub fn strong_count(this: &Self) -> usize {
unsafe { unsafe_::strong_count(this.ptr.as_ptr().cast::<TVMFFIObject>()) }
}
#[inline]
pub fn is_unique(this: &Self) -> bool {
unsafe { unsafe_::is_unique(this.ptr.as_ptr().cast::<TVMFFIObject>()) }
}
#[inline]
pub fn weak_count(this: &Self) -> usize {
unsafe { unsafe_::weak_count(this.ptr.as_ptr().cast::<TVMFFIObject>()) - 1 }
}
}
impl<T: ObjectCore> Deref for ObjectArc<T> {
type Target = T;
#[inline]
fn deref(&self) -> &Self::Target {
unsafe { self.ptr.as_ref() }
}
}
impl<T: ObjectCore> Drop for ObjectArc<T> {
#[inline(always)]
fn drop(&mut self) {
unsafe { unsafe_::dec_ref(self.ptr.as_ptr().cast::<TVMFFIObject>()) }
}
}
impl<T: ObjectCore> Clone for ObjectArc<T> {
#[inline]
fn clone(&self) -> Self {
unsafe { unsafe_::inc_ref(self.ptr.as_ptr().cast::<TVMFFIObject>()) }
Self {
ptr: self.ptr,
_phantom: std::marker::PhantomData,
}
}
}
pub unsafe trait ImmutableObject: ObjectCore {}
unsafe impl<T: ImmutableObject + Send + Sync> Send for ObjectArc<T> {}
unsafe impl<T: ImmutableObject + Send + Sync> Sync for ObjectArc<T> {}
#[repr(C)]
#[derive(ObjectRef, Clone)]
#[object_ref(skip_deref)]
pub struct ObjectRef {
data: ObjectArc<Object>,
}
#[diagnostic::on_unimplemented(message = "`{Self}` is not an object type")]
pub unsafe trait ObjectCore: Sized + 'static {
type Parent;
const TYPE_KEY: &'static str;
const TYPE_DEPTH: i32;
#[doc(hidden)]
const TYPE_FINAL: bool = false;
#[doc(hidden)]
const TYPE_IMMUTABLE: bool = false;
#[doc(hidden)]
const TYPE_HAS_WRITABLE_FIELD: bool = false;
fn type_index() -> i32;
#[doc(hidden)]
fn type_index_available() -> bool {
true
}
unsafe fn object_header_mut(this: &mut Self) -> &mut TVMFFIObject;
}
#[doc(hidden)]
#[diagnostic::on_unimplemented(message = "`{Self}` does not derive from `{A}`")]
pub unsafe trait ObjectDerivesFrom<A, Depth> {}
#[doc(hidden)]
pub mod marker {
pub enum Itself {}
pub struct Parent<Depth>(std::marker::PhantomData<fn() -> Depth>);
}
unsafe impl<T: ObjectCore> ObjectDerivesFrom<T, marker::Itself> for T {}
unsafe impl<T: ObjectCore, A, Depth> ObjectDerivesFrom<A, marker::Parent<Depth>> for T where
T::Parent: ObjectDerivesFrom<A, Depth>
{
}
pub unsafe trait ObjectCoreWithExtraItems: ObjectCore {
type ExtraItem;
fn extra_items_count(this: &Self) -> usize;
unsafe fn extra_items_ptr(this: *mut Self) -> *mut Self::ExtraItem {
unsafe { this.cast::<u8>().add(std::mem::size_of::<Self>()).cast() }
}
unsafe fn extra_items<'a>(this: *const Self) -> &'a [Self::ExtraItem] {
unsafe {
std::slice::from_raw_parts(
Self::extra_items_ptr(this.cast_mut()),
Self::extra_items_count(&*this),
)
}
}
}
pub unsafe trait ObjectRefCore: Sized + Clone + AnyCompatible {
type ContainerType: ObjectCore;
fn data(this: &Self) -> &ObjectArc<Self::ContainerType>;
fn into_data(this: Self) -> ObjectArc<Self::ContainerType>;
unsafe fn from_data(data: ObjectArc<Self::ContainerType>) -> Self;
#[inline]
fn same_as<Other: ObjectRefCore>(&self, other: &Other) -> bool {
ObjectArc::as_ptr(Self::data(self)).cast::<()>()
== ObjectArc::as_ptr(Other::data(other)).cast::<()>()
}
#[inline]
fn type_key(&self) -> &'static str {
let type_index =
unsafe { (*ObjectArc::as_ptr(Self::data(self)).cast::<TVMFFIObject>()).type_index };
registered_type_key(type_index)
}
#[inline(always)]
fn is<T: AnyCompatible>(&self) -> bool {
let mut any_data = TVMFFIAny::new();
unsafe {
Self::copy_to_any_view(self, &mut any_data);
T::check_any_strict(&any_data)
}
}
#[inline(always)]
fn downcast_ref<T: crate::type_traits::DowncastRefFromObject>(&self) -> Option<&T> {
unsafe {
let place = &*(Self::data(self) as *const ObjectArc<Self::ContainerType>
as *const *mut TVMFFIObject);
T::downcast_ref_from_object(place)
}
}
#[inline(always)]
fn downcast<B>(self) -> std::result::Result<B, Self>
where
B: ObjectRefCore,
{
let mut any_data = TVMFFIAny::new();
unsafe {
Self::copy_to_any_view(&self, &mut any_data);
debug_assert!(any_data.type_index >= TypeIndex::kTVMFFIStaticObjectBegin as i32);
std::hint::assert_unchecked(
any_data.type_index >= TypeIndex::kTVMFFIStaticObjectBegin as i32,
);
if B::check_any_strict(&any_data) {
Self::move_to_any(self, &mut any_data);
Ok(B::move_from_any_after_check(&mut any_data))
} else {
Err(self)
}
}
}
}
pub unsafe trait ObjectRefAnyCompatible: ObjectRefCore {
#[doc(hidden)]
const MATCH_ANY_EXACT: bool = false;
#[inline]
fn extra_check(obj: &Self::ContainerType) -> bool {
let _ = obj;
true
}
#[inline]
fn convert_fallback(value: AnyView<'_>) -> Option<Self> {
let _ = value;
None
}
fn custom_type_str() -> String {
Self::ContainerType::TYPE_KEY.into()
}
fn custom_mismatch_type_info(value: AnyView<'_>) -> String {
crate::type_traits::runtime_type_key(value.type_index())
}
}
#[doc(hidden)]
#[diagnostic::on_unimplemented(
message = "`{Self}` promises more than its object type (generic or custom checks)",
note = "write `#[object_ref(unsafe parent = ...)]` and keep that promise for every value"
)]
pub unsafe trait PlainObjectRef: ObjectRefCore {}
unsafe impl<T: ObjectRefAnyCompatible> AnyCompatible for T {
const MATCH_ANY_EXACT: bool = <T as ObjectRefAnyCompatible>::MATCH_ANY_EXACT
&& <T::ContainerType as ObjectCore>::TYPE_FINAL;
#[inline]
fn match_any_exact_type_index() -> i32 {
T::ContainerType::type_index()
}
#[inline]
unsafe fn is_undefined(field: *const Self) -> bool {
unsafe { ObjectArc::is_null(field.cast::<ObjectArc<T::ContainerType>>()) }
}
#[inline(always)]
unsafe fn copy_to_any_view(src: &Self, data: &mut TVMFFIAny) {
unsafe { write_object_cell(ObjectArc::as_ptr(T::data(src)), data) };
}
#[inline(always)]
unsafe fn move_to_any(src: Self, data: &mut TVMFFIAny) {
unsafe { write_object_cell(ObjectArc::into_raw(T::into_data(src)), data) };
}
#[inline(always)]
unsafe fn check_any_strict(data: &TVMFFIAny) -> bool {
unsafe {
is_instance_of::<T::ContainerType>(data.type_index)
&& T::extra_check(&*data.data_union.v_obj.cast::<T::ContainerType>())
}
}
#[inline]
unsafe fn copy_from_any_view_after_check(data: &TVMFFIAny) -> Self {
unsafe { unsafe_::inc_ref(data.data_union.v_obj) };
unsafe { T::from_data(ObjectArc::from_raw(data.data_union.v_obj.cast())) }
}
#[inline(always)]
unsafe fn move_from_any_after_check(data: &mut TVMFFIAny) -> Self {
unsafe { T::from_data(ObjectArc::from_raw(data.data_union.v_obj.cast())) }
}
#[inline]
unsafe fn try_cast_from_any_view(data: &TVMFFIAny) -> Result<Self, ()> {
if unsafe { Self::check_any_strict(data) } {
return Ok(unsafe { Self::copy_from_any_view_after_check(data) });
}
if data.type_index == TypeIndex::kTVMFFIObjectRValueRef as i32 {
return Err(());
}
unsafe { T::convert_fallback(AnyView::from_raw_ffi_any(*data)).ok_or(()) }
}
fn get_mismatch_type_info(data: &TVMFFIAny) -> String {
T::custom_mismatch_type_info(unsafe { AnyView::from_raw_ffi_any(*data) })
}
fn type_str() -> String {
T::custom_type_str()
}
}
#[derive(Clone)]
pub struct ObjectIdentity {
data: ObjectArc<Object>,
}
impl ObjectIdentity {
pub fn of<T: ObjectRefCore>(value: &T) -> Self {
unsafe {
let ptr = ObjectArc::as_ptr(T::data(value)) as *mut TVMFFIObject;
unsafe_::inc_ref(ptr);
Self {
data: ObjectArc::from_raw(ptr.cast::<Object>()),
}
}
}
#[inline]
fn as_ptr(&self) -> *const TVMFFIObject {
ObjectArc::as_ptr(&self.data).cast::<TVMFFIObject>()
}
}
impl PartialEq for ObjectIdentity {
#[inline]
fn eq(&self, other: &Self) -> bool {
self.as_ptr() == other.as_ptr()
}
}
impl Eq for ObjectIdentity {}
impl std::hash::Hash for ObjectIdentity {
#[inline]
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
self.as_ptr().hash(state);
}
}
impl std::fmt::Debug for ObjectIdentity {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_tuple("ObjectIdentity")
.field(&self.as_ptr())
.finish()
}
}
#[inline(always)]
pub fn is_instance_of<Target: ObjectCore>(object_type_index: i32) -> bool {
let target_type_index = Target::type_index();
if object_type_index == target_type_index {
return true;
}
if Target::TYPE_FINAL {
return false;
}
let object_begin = TypeIndex::kTVMFFIStaticObjectBegin as i32;
if object_type_index < object_begin || target_type_index < object_begin {
return false;
}
if const { Target::TYPE_DEPTH == 0 } {
return true;
}
if object_type_index < target_type_index {
return false;
}
unsafe {
let object_info = TVMFFIGetTypeInfo(object_type_index);
if object_info.is_null() {
return false;
}
let target_depth = Target::TYPE_DEPTH;
if (*object_info).type_depth <= target_depth {
return false;
}
let ancestor = *(*object_info).type_acenstors.add(target_depth as usize);
!ancestor.is_null() && (*ancestor).type_index == target_type_index
}
}
pub fn fmt_object_ref<T: ObjectRefCore>(
this: &T,
f: &mut std::fmt::Formatter<'_>,
) -> std::fmt::Result {
let object = ObjectArc::as_ptr(T::data(this)).cast::<TVMFFIObject>();
let type_index = unsafe { (*object).type_index };
write!(f, "{}@{:p}", type_key_of_index(type_index), object)
}
#[inline(always)]
pub(crate) fn root_try_into<B>(value: ObjectRef) -> crate::error::Result<B>
where
B: ObjectRefCore,
{
value
.downcast::<B>()
.map_err(crate::error::Error::downcast_failed::<B>)
}
#[inline(always)]
pub(crate) fn root_try_into_ref<B>(value: &ObjectRef) -> crate::error::Result<B>
where
B: ObjectRefCore + crate::type_traits::DowncastRefFromObject,
{
match value.downcast_ref::<B>() {
Some(found) => Ok(found.clone()),
None => Err(crate::error::Error::downcast_failed::<B>(value.clone())),
}
}
#[inline(always)]
pub(crate) fn into_root_object_ref<S: ObjectRefCore>(value: S) -> ObjectRef {
let data = S::into_data(value);
unsafe {
ObjectRef::from_data(ObjectArc::from_raw(
ObjectArc::into_raw(data).cast::<Object>(),
))
}
}
#[inline]
pub(crate) fn registered_type_key(type_index: i32) -> &'static str {
unsafe { (*TVMFFIGetTypeInfo(type_index)).type_key.as_str() }
}
pub(crate) fn type_key_of_index(type_index: i32) -> String {
match crate::reflection::type_key(type_index) {
Some(key) => key.to_string(),
None => format!("<type_index {type_index}>"),
}
}
#[inline(always)]
unsafe fn write_object_cell<T: ObjectCore>(ptr: *const T, data: &mut TVMFFIAny) {
let object = ptr as *mut TVMFFIObject;
data.type_index = unsafe { (*object).type_index };
unsafe {
std::hint::assert_unchecked(data.type_index >= TypeIndex::kTVMFFIStaticObjectBegin as i32)
};
data.small_str_len = 0;
data.data_union.v_obj = object;
}
unsafe fn allocate_object<T>(data: T, type_index: i32) -> NonNull<T> {
let layout = std::alloc::Layout::new::<T>();
let allocation = unsafe { std::alloc::alloc(layout) };
if allocation.is_null() {
std::alloc::handle_alloc_error(layout);
}
let ptr = allocation.cast::<T>();
unsafe { std::ptr::write(ptr, data) };
unsafe {
std::ptr::write(
ptr.cast::<TVMFFIObject>(),
TVMFFIObject {
combined_ref_count: AtomicU64::new(COMBINED_REF_COUNT_BOTH_ONE),
type_index,
__padding: 0,
deleter: Some(unsafe_::object_deleter_for_new::<T>),
},
)
};
unsafe { NonNull::new_unchecked(ptr) }
}
#[doc(hidden)]
pub mod unsafe_ {
use tvm_ffi_sys::{
COMBINED_REF_COUNT_BOTH_ONE, COMBINED_REF_COUNT_MASK_U32, COMBINED_REF_COUNT_STRONG_ONE,
COMBINED_REF_COUNT_WEAK_ONE,
};
use std::ffi::c_void;
use std::sync::atomic::{Ordering, fence};
use tvm_ffi_sys::TVMFFIObject;
use tvm_ffi_sys::TVMFFIObjectDeleterFlagBitMask::{
kTVMFFIObjectDeleterFlagBitMaskBoth, kTVMFFIObjectDeleterFlagBitMaskStrong,
kTVMFFIObjectDeleterFlagBitMaskWeak,
};
#[inline]
pub(crate) unsafe fn inc_ref(handle: *mut TVMFFIObject) {
let obj = unsafe { &*handle };
obj.combined_ref_count.fetch_add(1, Ordering::Relaxed);
}
#[inline]
pub(crate) unsafe fn dec_ref(handle: *mut TVMFFIObject) {
let old_combined_count = unsafe {
(*handle)
.combined_ref_count
.fetch_sub(COMBINED_REF_COUNT_STRONG_ONE, Ordering::Release)
};
if old_combined_count == COMBINED_REF_COUNT_BOTH_ONE {
fence(Ordering::Acquire);
if let Some(deleter) = unsafe { (*handle).deleter } {
unsafe {
deleter(
handle.cast::<c_void>(),
kTVMFFIObjectDeleterFlagBitMaskBoth as i32,
)
};
}
} else if (old_combined_count & COMBINED_REF_COUNT_MASK_U32)
== COMBINED_REF_COUNT_STRONG_ONE
{
fence(Ordering::Acquire);
if let Some(deleter) = unsafe { (*handle).deleter } {
unsafe {
deleter(
handle.cast::<c_void>(),
kTVMFFIObjectDeleterFlagBitMaskStrong as i32,
)
};
}
let old_weak_count = unsafe {
(*handle)
.combined_ref_count
.fetch_sub(COMBINED_REF_COUNT_WEAK_ONE, Ordering::Release)
};
if old_weak_count == COMBINED_REF_COUNT_WEAK_ONE {
fence(Ordering::Acquire);
if let Some(deleter) = unsafe { (*handle).deleter } {
unsafe {
deleter(
handle.cast::<c_void>(),
kTVMFFIObjectDeleterFlagBitMaskWeak as i32,
)
};
}
}
}
}
#[inline]
pub(crate) unsafe fn strong_count(handle: *mut TVMFFIObject) -> usize {
let obj = unsafe { &*handle };
(obj.combined_ref_count.load(Ordering::Relaxed) & COMBINED_REF_COUNT_MASK_U32) as usize
}
#[inline]
pub(crate) unsafe fn is_unique(handle: *mut TVMFFIObject) -> bool {
unsafe {
(*handle).combined_ref_count.load(Ordering::Acquire) == COMBINED_REF_COUNT_BOTH_ONE
}
}
#[inline]
pub(crate) unsafe fn weak_count(handle: *mut TVMFFIObject) -> usize {
let obj = unsafe { &*handle };
(obj.combined_ref_count.load(Ordering::Relaxed) >> 32) as usize
}
pub(crate) unsafe extern "C" fn object_deleter_for_new<T>(ptr: *mut c_void, flags: i32) {
let obj = ptr as *mut T;
if flags & kTVMFFIObjectDeleterFlagBitMaskStrong as i32 != 0 {
unsafe { std::ptr::drop_in_place(obj) };
}
if flags & kTVMFFIObjectDeleterFlagBitMaskWeak as i32 != 0 {
unsafe { std::alloc::dealloc(ptr as *mut u8, std::alloc::Layout::new::<T>()) };
}
}
pub(crate) fn extra_items_layout<T, U>(count: usize) -> std::alloc::Layout {
use std::alloc::Layout;
let min_size = std::mem::size_of::<TVMFFIObject>() + std::mem::size_of::<u64>();
Layout::array::<U>(count)
.and_then(|items| Layout::new::<T>().extend(items))
.and_then(|(layout, _)| {
Layout::from_size_align(layout.size().max(min_size), layout.align())
})
.expect("capacity overflow")
}
pub(crate) unsafe extern "C" fn object_deleter_for_new_with_extra_items<T, U>(
ptr: *mut c_void,
flags: i32,
) where
T: super::ObjectCoreWithExtraItems<ExtraItem = U>,
{
let obj = ptr as *mut T;
if flags == kTVMFFIObjectDeleterFlagBitMaskBoth as i32 {
let extra_items_count = unsafe { T::extra_items_count(&(*obj)) };
unsafe { std::ptr::drop_in_place(obj) };
unsafe {
std::alloc::dealloc(
ptr as *mut u8,
extra_items_layout::<T, U>(extra_items_count),
)
};
} else {
let count_slot =
unsafe { (ptr as *mut u8).add(std::mem::size_of::<TVMFFIObject>()) as *mut u64 };
if flags & kTVMFFIObjectDeleterFlagBitMaskStrong as i32 != 0 {
let extra_items_count = unsafe { T::extra_items_count(&(*obj)) };
unsafe { std::ptr::drop_in_place(obj) };
unsafe { std::ptr::write(count_slot, extra_items_count as u64) };
}
if flags & kTVMFFIObjectDeleterFlagBitMaskWeak as i32 != 0 {
let extra_items_count = unsafe { std::ptr::read(count_slot) as usize };
unsafe {
std::alloc::dealloc(
ptr as *mut u8,
extra_items_layout::<T, U>(extra_items_count),
)
};
}
}
}
}