luau_vm/memory/
mod.rs

1pub(crate) mod allocator;
2mod page;
3
4pub(crate) use page::LuaPageList;
5pub use page::{GcoPageWalk, LuaPage, RawLuaPage};
6
7use core::alloc::Layout;
8use core::mem::{offset_of, size_of};
9use core::ptr::{self, NonNull};
10
11use crate::debug::DebugRuntime;
12use crate::gc::{GcHandle, GcObject, RawGcObject};
13use crate::handle::RawHandle;
14use crate::handle::sealed::Sealed;
15use crate::thread::Thread;
16use crate::types::LUA_TNIL;
17use crate::{VmError, VmErrorResult};
18
19pub const MAX_SMALL_SIZE: usize = 1024;
20pub const MAX_SMALL_SIZE_USED: usize = 1024;
21pub const LARGE_PAGE_THRESHOLD: usize = 512;
22const EXTERNAL_ALLOCATOR_METADATA_REDUCTION: usize = 24;
23pub const PAGE_SMALL_SIZE: usize = 16 * 1024 - EXTERNAL_ALLOCATOR_METADATA_REDUCTION;
24pub const PAGE_LARGE_SIZE: usize = 32 * 1024 - EXTERNAL_ALLOCATOR_METADATA_REDUCTION;
25/// `kBlockHeader`
26pub const BLOCK_HEADER_SIZE: usize =
27    if core::mem::size_of::<f64>() > core::mem::size_of::<*const ()>() {
28        core::mem::size_of::<f64>()
29    } else {
30        core::mem::size_of::<*const ()>()
31    };
32
33struct SizeClassConfig {
34    size_of_class: [i32; crate::memory::LUA_SIZE_CLASSES],
35    class_for_size: [i8; MAX_SMALL_SIZE + 1],
36    class_count: usize,
37}
38
39impl SizeClassConfig {
40    /// `SizeClassConfig::SizeClassConfig`
41    const fn new() -> Self {
42        let mut size_of_class = [0; crate::memory::LUA_SIZE_CLASSES];
43        let mut class_for_size = [-1; MAX_SMALL_SIZE + 1];
44        let mut class_count = 0usize;
45
46        let mut size = 8i32;
47        while size < 64 {
48            size_of_class[class_count] = size;
49            class_count += 1;
50            size += 8;
51        }
52
53        size = 64;
54        while size < 256 {
55            size_of_class[class_count] = size;
56            class_count += 1;
57            size += 16;
58        }
59
60        size = 256;
61        while size < 512 {
62            size_of_class[class_count] = size;
63            class_count += 1;
64            size += 32;
65        }
66
67        size = 512;
68        while size <= 1024 {
69            size_of_class[class_count] = size;
70            class_count += 1;
71            size += 64;
72        }
73
74        let mut klass = 0usize;
75        while klass < class_count {
76            class_for_size[size_of_class[klass] as usize] = klass as i8;
77            klass += 1;
78        }
79
80        let mut size = MAX_SMALL_SIZE - 1;
81        loop {
82            if class_for_size[size] < 0 {
83                class_for_size[size] = class_for_size[size + 1];
84            }
85
86            if size == 0 {
87                break;
88            }
89            size -= 1;
90        }
91
92        Self {
93            size_of_class,
94            class_for_size,
95            class_count,
96        }
97    }
98}
99
100const SIZE_CLASS_CONFIG: SizeClassConfig = SizeClassConfig::new();
101
102/// `sizeclass`
103pub fn size_class(size: usize) -> i32 {
104    if size.wrapping_sub(1) < MAX_SMALL_SIZE_USED {
105        i32::from(SIZE_CLASS_CONFIG.class_for_size[size])
106    } else {
107        -1
108    }
109}
110
111/// `SizeClassConfig::sizeOfClass`
112pub fn size_of_class(size_class: usize) -> i32 {
113    debug_assert!(size_class < SIZE_CLASS_CONFIG.class_count);
114    SIZE_CLASS_CONFIG.size_of_class[size_class]
115}
116
117pub const LUA_PAGE_PADDING: usize = if size_of::<*const ()>() == 8 { 8 } else { 12 };
118pub const GCO_LINK_OFFSET: usize =
119    (size_of::<RawGcObject>() + size_of::<*const ()>() - 1) & !(size_of::<*const ()>() - 1);
120
121/// `LUA_SIZECLASSES`
122pub(crate) const LUA_SIZE_CLASSES: usize = 40;
123
124/// `LUA_MEMORY_CATEGORIES`
125pub(crate) const LUA_MEMORY_CATEGORIES: usize = 256;
126
127const SMALL_BLOCK_ALIGNMENT: usize = BLOCK_HEADER_SIZE;
128
129/// Unstable VM allocation and object-visitation capability.
130///
131/// # Safety
132///
133/// Layouts, sizes, categories, pages, and pointers must describe allocations
134/// owned by this thread's VM. Reallocation and free operations require the
135/// exact original layout contract and invalidate every outstanding view into
136/// the affected storage.
137#[allow(
138    clippy::missing_safety_doc,
139    reason = "all methods share the capability-level safety contract"
140)]
141pub trait MemoryRuntime: Sealed {
142    unsafe fn new_raw(&self, layout: Layout, memcat: u8) -> VmErrorResult<*mut u8>;
143    unsafe fn new_gco_(&self, size: usize, memcat: u8) -> VmErrorResult<GcObject>;
144    unsafe fn free_raw(&self, block: *mut u8, old_layout: Layout, memcat: u8);
145
146    unsafe fn new_array<T>(&self, count: usize, memcat: u8) -> VmErrorResult<*mut T> {
147        unsafe {
148            let layout = self.array_layout::<T>(count)?;
149            self.new_raw(layout, memcat).map(|block| block.cast())
150        }
151    }
152
153    unsafe fn free_array<T>(&self, block: *mut T, count: usize, memcat: u8) {
154        unsafe {
155            self.free_raw(
156                block.cast::<u8>(),
157                self.array_layout::<T>(count)
158                    .expect("previously allocated array layout remains valid"),
159                memcat,
160            );
161        }
162    }
163
164    unsafe fn free_gco(&self, block: GcObject, old_size: usize, memcat: u8, page: LuaPage);
165
166    unsafe fn realloc_raw(
167        &self,
168        block: *mut u8,
169        old_layout: Layout,
170        new_layout: Layout,
171        memcat: u8,
172    ) -> VmErrorResult<*mut u8>;
173
174    unsafe fn realloc_array<T>(
175        &self,
176        block: *mut T,
177        old_count: usize,
178        new_count: usize,
179        memcat: u8,
180    ) -> VmErrorResult<*mut T> {
181        unsafe {
182            let old_layout = self
183                .array_layout::<T>(old_count)
184                .expect("previously allocated array layout remains valid");
185            let new_layout = self.array_layout::<T>(new_count)?;
186            self.realloc_raw(block.cast::<u8>(), old_layout, new_layout, memcat)
187                .map(|block| block.cast())
188        }
189    }
190
191    unsafe fn too_big<T>(&self) -> VmErrorResult<T>;
192
193    unsafe fn array_layout<T>(&self, count: usize) -> VmErrorResult<Layout> {
194        match Layout::array::<T>(count) {
195            Ok(layout) => Ok(layout),
196            Err(_) => unsafe { self.too_big() },
197        }
198    }
199
200    unsafe fn visit_gco(
201        &self,
202        context: *mut (),
203        visitor: unsafe fn(*mut (), LuaPage, GcObject) -> bool,
204    );
205}
206
207impl MemoryRuntime for Thread {
208    /// `luaM_new_`
209    unsafe fn new_raw(&self, layout: Layout, memcat: u8) -> VmErrorResult<*mut u8> {
210        unsafe {
211            let size = layout.size();
212            if size == 0 {
213                return Ok(ptr::null_mut());
214            }
215
216            let size_class_index = if layout.align() <= SMALL_BLOCK_ALIGNMENT {
217                size_class(size)
218            } else {
219                -1
220            };
221            let block = if size_class_index >= 0 {
222                self.new_block(size_class_index).map(NonNull::as_ptr)
223            } else {
224                let global = self.global();
225                global
226                    .allocator()
227                    .as_ref()
228                    .allocate(layout)
229                    .map(NonNull::as_ptr)
230            };
231
232            let block = match block {
233                Some(block) if !block.is_null() || size == 0 => block,
234                Some(_) | None => return Err(VmError::Memory),
235            };
236
237            let global_handle = self.global();
238            let global = global_handle.as_ptr().as_mut().unwrap_unchecked();
239            global.total_bytes += size;
240            global.memcat_bytes[memcat as usize] += size;
241
242            if let Some(on_allocate) = global.cb.on_allocate {
243                on_allocate(self, 0, size);
244            }
245
246            Ok(block)
247        }
248    }
249
250    /// `luaM_newgco_`
251    unsafe fn new_gco_(&self, size: usize, memcat: u8) -> VmErrorResult<GcObject> {
252        debug_assert!(size >= GCO_LINK_OFFSET + core::mem::size_of::<*const ()>());
253
254        unsafe {
255            let size_class_index = size_class(size);
256            let block = if size_class_index >= 0 {
257                self.new_gco_block(size_class_index)
258            } else {
259                self.large_gco_page(size).map(|(_, block)| block)
260            };
261
262            let Some(block) = block else {
263                return Err(VmError::Memory);
264            };
265
266            let global_handle = self.global();
267            let global = global_handle.as_ptr().as_mut().unwrap_unchecked();
268            global.total_bytes += size;
269            global.memcat_bytes[memcat as usize] += size;
270
271            if let Some(on_allocate) = global.cb.on_allocate {
272                on_allocate(self, 0, size);
273            }
274
275            Ok(block)
276        }
277    }
278
279    /// `luaM_free_`
280    unsafe fn free_raw(&self, block: *mut u8, old_layout: Layout, memcat: u8) {
281        let old_size = old_layout.size();
282        debug_assert!((old_size == 0) == block.is_null());
283
284        unsafe {
285            if old_size == 0 {
286                return;
287            }
288
289            let size_class_index = if old_layout.align() <= SMALL_BLOCK_ALIGNMENT {
290                size_class(old_size)
291            } else {
292                -1
293            };
294            if size_class_index >= 0 {
295                if let Some(block) = NonNull::new(block) {
296                    self.free_block(size_class_index, block);
297                }
298            } else {
299                let global = self.global();
300                global
301                    .allocator()
302                    .as_ref()
303                    .deallocate(NonNull::new_unchecked(block), old_layout);
304            }
305
306            let global_handle = self.global();
307            let global = global_handle.as_ptr().as_mut().unwrap_unchecked();
308            global.total_bytes -= old_size;
309            global.memcat_bytes[memcat as usize] -= old_size;
310        }
311    }
312
313    /// `luaM_freegco_`
314    unsafe fn free_gco(&self, block: GcObject, old_size: usize, memcat: u8, page: LuaPage) {
315        unsafe {
316            let size_class_index = size_class(old_size);
317            if size_class_index >= 0 {
318                block.as_ptr().as_mut().unwrap_unchecked().tt = LUA_TNIL as u8;
319                self.free_gco_block(
320                    size_class_index,
321                    NonNull::new_unchecked(block.as_ptr()),
322                    page,
323                );
324            } else {
325                debug_assert!(page.as_ptr().as_ref().unwrap_unchecked().busy_blocks == 1);
326                debug_assert!(
327                    page.as_ptr().as_ref().unwrap_unchecked().block_size as usize == old_size
328                );
329                debug_assert_eq!(block.as_ptr().cast::<u8>(), page.data_ptr_mut());
330
331                let global = self.global();
332                self.free_page(Some(global.all_gco_page_list()), page);
333            }
334
335            let global_handle = self.global();
336            let global = global_handle.as_ptr().as_mut().unwrap_unchecked();
337            global.total_bytes -= old_size;
338            global.memcat_bytes[memcat as usize] -= old_size;
339        }
340    }
341
342    /// `luaM_realloc_`
343    unsafe fn realloc_raw(
344        &self,
345        block: *mut u8,
346        old_layout: Layout,
347        new_layout: Layout,
348        memcat: u8,
349    ) -> VmErrorResult<*mut u8> {
350        unsafe {
351            let old_size = old_layout.size();
352            let new_size = new_layout.size();
353            debug_assert!((old_size == 0) == block.is_null());
354
355            if old_size == 0 {
356                return self.new_raw(new_layout, memcat);
357            }
358
359            if new_size == 0 {
360                self.free_raw(block, old_layout, memcat);
361                let global = self.global();
362                if let Some(on_allocate) =
363                    global.as_ptr().as_mut().unwrap_unchecked().cb.on_allocate
364                {
365                    on_allocate(self, old_size, new_size);
366                }
367                return Ok(ptr::null_mut());
368            }
369
370            let new_class = if new_layout.align() <= SMALL_BLOCK_ALIGNMENT {
371                size_class(new_size)
372            } else {
373                -1
374            };
375            let old_class = if old_layout.align() <= SMALL_BLOCK_ALIGNMENT {
376                size_class(old_size)
377            } else {
378                -1
379            };
380
381            let result = if new_class >= 0 || old_class >= 0 {
382                let result = if new_class >= 0 {
383                    self.new_block(new_class).map(NonNull::as_ptr)
384                } else {
385                    let global = self.global();
386                    global
387                        .allocator()
388                        .as_ref()
389                        .allocate(new_layout)
390                        .map(NonNull::as_ptr)
391                };
392
393                let result = match result {
394                    Some(result) if !result.is_null() || new_size == 0 => result,
395                    Some(_) | None => return Err(VmError::Memory),
396                };
397
398                if old_size > 0 && new_size > 0 {
399                    ptr::copy_nonoverlapping(block, result, old_size.min(new_size));
400                }
401
402                if old_class >= 0 {
403                    if let Some(block) = NonNull::new(block) {
404                        self.free_block(old_class, block);
405                    }
406                } else {
407                    let global = self.global();
408                    global
409                        .allocator()
410                        .as_ref()
411                        .deallocate(NonNull::new_unchecked(block), old_layout);
412                }
413
414                result
415            } else {
416                let global = self.global();
417                global
418                    .allocator()
419                    .as_ref()
420                    .reallocate(NonNull::new_unchecked(block), old_layout, new_layout)
421                    .map(NonNull::as_ptr)
422                    .ok_or(VmError::Memory)?
423            };
424
425            let global_handle = self.global();
426            let global = global_handle.as_ptr().as_mut().unwrap_unchecked();
427            if new_size >= old_size {
428                global.total_bytes += new_size - old_size;
429                global.memcat_bytes[memcat as usize] += new_size - old_size;
430            } else {
431                global.total_bytes -= old_size - new_size;
432                global.memcat_bytes[memcat as usize] -= old_size - new_size;
433            }
434
435            if let Some(on_allocate) = global.cb.on_allocate {
436                on_allocate(self, old_size, new_size);
437            }
438
439            Ok(result)
440        }
441    }
442
443    /// `luaM_toobig`
444    unsafe fn too_big<T>(&self) -> VmErrorResult<T> {
445        unsafe { crate::run_error!(self, "memory allocation error: block too big") }
446    }
447
448    /// `luaM_visitgco`
449    unsafe fn visit_gco(
450        &self,
451        context: *mut (),
452        visitor: unsafe fn(*mut (), LuaPage, GcObject) -> bool,
453    ) {
454        unsafe {
455            let global = self.global();
456            let mut current =
457                NonNull::new(global.as_ptr().as_ref().unwrap_unchecked().all_gco_pages);
458
459            while let Some(page) = current {
460                let page = LuaPage::from_raw(page);
461                current = page
462                    .next_page()
463                    .map(|next| NonNull::new_unchecked(next.as_ptr()));
464                page.visit_page(context, visitor);
465            }
466        }
467    }
468}
469
470impl Thread {
471    /// Typed allocation for the VM's closed set of GC object kinds.
472    pub(crate) unsafe fn new_gco<T: GcHandle>(&self, size: usize, memcat: u8) -> VmErrorResult<T> {
473        unsafe {
474            self.new_gco_(size, memcat)
475                .map(|object| T::from_gc_object(object))
476        }
477    }
478
479    /// `newpage`
480    unsafe fn new_page(
481        &self,
482        page_set: Option<LuaPageList>,
483        page_size: i32,
484        block_size: i32,
485        block_count: i32,
486    ) -> Option<LuaPage> {
487        debug_assert!(
488            page_size as usize - offset_of!(RawLuaPage, data)
489                >= block_size as usize * block_count as usize
490        );
491
492        let global = unsafe { self.global() };
493        let layout =
494            Layout::from_size_align(page_size as usize, core::mem::align_of::<RawLuaPage>())
495                .expect("valid page allocation layout");
496        let raw = unsafe { global.allocator().as_ref().allocate(layout)? };
497        let page = unsafe { LuaPage::from_raw(raw.cast::<RawLuaPage>()) };
498
499        unsafe {
500            let page_mut = page.as_ptr().as_mut().unwrap_unchecked();
501            page_mut.prev = ptr::null_mut();
502            page_mut.next = ptr::null_mut();
503            page_mut.list_prev = ptr::null_mut();
504            page_mut.list_next = ptr::null_mut();
505            page_mut.page_size = page_size;
506            page_mut.block_size = block_size;
507            page_mut.free_list = ptr::null_mut();
508            page_mut.free_next = (block_count - 1) * block_size;
509            page_mut.busy_blocks = 0;
510
511            if let Some(page_set) = page_set {
512                page_set.link_page(page);
513            }
514        }
515
516        Some(page)
517    }
518
519    /// `newclasspage`
520    unsafe fn new_class_page(
521        &self,
522        free_page_set: LuaPageList,
523        page_set: Option<LuaPageList>,
524        size_class_index: u8,
525        store_metadata: bool,
526    ) -> Option<LuaPage> {
527        let size_of_class = size_of_class(size_class_index as usize);
528        let page_size = if size_of_class as usize > LARGE_PAGE_THRESHOLD {
529            PAGE_LARGE_SIZE
530        } else {
531            PAGE_SMALL_SIZE
532        } as i32;
533        let block_size = size_of_class
534            + if store_metadata {
535                BLOCK_HEADER_SIZE as i32
536            } else {
537                0
538            };
539        let block_count = (page_size as usize - offset_of!(RawLuaPage, data)) / block_size as usize;
540
541        let page = unsafe { self.new_page(page_set, page_size, block_size, block_count as i32)? };
542        unsafe {
543            debug_assert!(free_page_set.get().is_null());
544            free_page_set.set(page.as_ptr());
545        }
546        Some(page)
547    }
548
549    /// `freepage`
550    unsafe fn free_page(&self, page_set: Option<LuaPageList>, page: LuaPage) {
551        let global = unsafe { self.global() };
552
553        unsafe {
554            if let Some(page_set) = page_set {
555                page_set.unlink_page(page);
556            }
557
558            let layout = Layout::from_size_align(
559                page.as_ptr().as_ref().unwrap_unchecked().page_size as usize,
560                core::mem::align_of::<RawLuaPage>(),
561            )
562            .expect("valid page allocation layout");
563            global
564                .allocator()
565                .as_ref()
566                .deallocate(NonNull::new_unchecked(page.as_ptr().cast()), layout);
567        }
568    }
569
570    /// `freeclasspage`
571    unsafe fn free_class_page(
572        &self,
573        free_page_set: LuaPageList,
574        page_set: Option<LuaPageList>,
575        page: LuaPage,
576    ) {
577        unsafe {
578            {
579                let page_ref = page.as_ptr().as_mut().unwrap_unchecked();
580                if let Some(mut next) = NonNull::new(page_ref.next) {
581                    next.as_mut().prev = page_ref.prev;
582                }
583
584                if let Some(mut prev) = NonNull::new(page_ref.prev) {
585                    prev.as_mut().next = page_ref.next;
586                } else if free_page_set.get() == page.as_ptr() {
587                    free_page_set.set(page_ref.next);
588                }
589            }
590
591            self.free_page(page_set, page);
592        }
593    }
594
595    /// `newblock`
596    unsafe fn new_block(&self, size_class_index: i32) -> Option<NonNull<u8>> {
597        unsafe {
598            let global = self.global();
599            let index = size_class_index as usize;
600            let free_pages = global.free_page_list(index);
601            let all_pages = global.all_page_list();
602            let page = match NonNull::new(free_pages.get()) {
603                Some(page) => LuaPage::from_raw(page),
604                None => {
605                    self.new_class_page(free_pages, Some(all_pages), size_class_index as u8, true)?
606                }
607            };
608
609            debug_assert!(page.as_ptr().as_ref().unwrap_unchecked().prev.is_null());
610            debug_assert!(
611                !page
612                    .as_ptr()
613                    .as_ref()
614                    .unwrap_unchecked()
615                    .free_list
616                    .is_null()
617                    || page.as_ptr().as_ref().unwrap_unchecked().free_next >= 0
618            );
619
620            let block = {
621                let page_ref = page.as_ptr().as_mut().unwrap_unchecked();
622                let block = if page_ref.free_next >= 0 {
623                    let block = page.block(page_ref.free_next);
624                    page_ref.free_next -= page_ref.block_size;
625                    page_ref.busy_blocks += 1;
626                    block
627                } else {
628                    let block = page_ref.free_list;
629                    page_ref.free_list = *LuaPage::metadata_slot(block);
630                    page_ref.busy_blocks += 1;
631                    block
632                };
633
634                *LuaPage::metadata_slot(block) = page.as_ptr().cast::<u8>();
635
636                if page_ref.free_list.is_null() && page_ref.free_next < 0 {
637                    free_pages.set(page_ref.next);
638                    if let Some(mut next) = NonNull::new(page_ref.next) {
639                        next.as_mut().prev = ptr::null_mut();
640                    }
641                    page_ref.next = ptr::null_mut();
642                }
643
644                block
645            };
646
647            Some(NonNull::new_unchecked(block.add(BLOCK_HEADER_SIZE)))
648        }
649    }
650
651    /// `newgcoblock`
652    unsafe fn new_gco_block(&self, size_class_index: i32) -> Option<GcObject> {
653        unsafe {
654            let global = self.global();
655            let index = size_class_index as usize;
656            let free_pages = global.free_gco_page_list(index);
657            let all_gco_pages = global.all_gco_page_list();
658            let page = match NonNull::new(free_pages.get()) {
659                Some(page) => LuaPage::from_raw(page),
660                None => self.new_class_page(
661                    free_pages,
662                    Some(all_gco_pages),
663                    size_class_index as u8,
664                    false,
665                )?,
666            };
667
668            debug_assert!(page.as_ptr().as_ref().unwrap_unchecked().prev.is_null());
669            debug_assert!(
670                !page
671                    .as_ptr()
672                    .as_ref()
673                    .unwrap_unchecked()
674                    .free_list
675                    .is_null()
676                    || page.as_ptr().as_ref().unwrap_unchecked().free_next >= 0
677            );
678
679            let block = {
680                let page_ref = page.as_ptr().as_mut().unwrap_unchecked();
681                let block = if page_ref.free_next >= 0 {
682                    let block = page.block(page_ref.free_next);
683                    page_ref.free_next -= page_ref.block_size;
684                    page_ref.busy_blocks += 1;
685                    block
686                } else {
687                    let block = page_ref.free_list;
688                    page_ref.free_list = *LuaPage::free_gco_link_slot(block);
689                    page_ref.busy_blocks += 1;
690                    block
691                };
692
693                if page_ref.free_list.is_null() && page_ref.free_next < 0 {
694                    free_pages.set(page_ref.next);
695                    if let Some(mut next) = NonNull::new(page_ref.next) {
696                        next.as_mut().prev = ptr::null_mut();
697                    }
698                    page_ref.next = ptr::null_mut();
699                }
700
701                block
702            };
703
704            Some(GcObject::from_raw(NonNull::new_unchecked(block.cast())))
705        }
706    }
707
708    /// `freeblock`
709    unsafe fn free_block(&self, size_class_index: i32, block: NonNull<u8>) {
710        unsafe {
711            let global = self.global();
712            let index = size_class_index as usize;
713            let free_pages = global.free_page_list(index);
714            let all_pages = global.all_page_list();
715            let raw_block = block.as_ptr().sub(BLOCK_HEADER_SIZE);
716            let page = LuaPage::from_raw(NonNull::new_unchecked(
717                (*LuaPage::metadata_slot(raw_block)).cast::<RawLuaPage>(),
718            ));
719
720            let should_free_page = {
721                let page_ref = page.as_ptr().as_mut().unwrap_unchecked();
722
723                if page_ref.free_list.is_null() && page_ref.free_next < 0 {
724                    debug_assert!(page_ref.prev.is_null());
725                    debug_assert!(page_ref.next.is_null());
726
727                    page_ref.next = free_pages.get();
728                    if let Some(mut next) = NonNull::new(page_ref.next) {
729                        next.as_mut().prev = page.as_ptr();
730                    }
731                    free_pages.set(page.as_ptr());
732                }
733
734                *LuaPage::metadata_slot(raw_block) = page_ref.free_list.cast::<u8>();
735                page_ref.free_list = raw_block;
736                page_ref.busy_blocks -= 1;
737                page_ref.busy_blocks == 0
738            };
739
740            if should_free_page {
741                self.free_class_page(free_pages, Some(all_pages), page);
742            }
743        }
744    }
745
746    /// `freegcoblock`
747    unsafe fn free_gco_block(
748        &self,
749        size_class_index: i32,
750        block: NonNull<RawGcObject>,
751        page: LuaPage,
752    ) {
753        unsafe {
754            let global = self.global();
755            let index = size_class_index as usize;
756            let free_pages = global.free_gco_page_list(index);
757            let all_gco_pages = global.all_gco_page_list();
758            let should_free_page = {
759                let page_ref = page.as_ptr().as_mut().unwrap_unchecked();
760
761                if page_ref.free_list.is_null() && page_ref.free_next < 0 {
762                    debug_assert!(page_ref.prev.is_null());
763                    debug_assert!(page_ref.next.is_null());
764
765                    page_ref.next = free_pages.get();
766                    if let Some(mut next) = NonNull::new(page_ref.next) {
767                        next.as_mut().prev = page.as_ptr();
768                    }
769                    free_pages.set(page.as_ptr());
770                }
771
772                *LuaPage::free_gco_link_slot(block.as_ptr().cast()) =
773                    page_ref.free_list.cast::<u8>();
774                page_ref.free_list = block.as_ptr().cast();
775                page_ref.busy_blocks -= 1;
776                page_ref.busy_blocks == 0
777            };
778
779            if should_free_page {
780                self.free_class_page(free_pages, Some(all_gco_pages), page);
781            }
782        }
783    }
784
785    /// `largegcopage`
786    unsafe fn large_gco_page(&self, size: usize) -> Option<(LuaPage, GcObject)> {
787        unsafe {
788            let global = self.global();
789            let all_gco_pages = global.all_gco_page_list();
790            let page = self.new_page(
791                Some(all_gco_pages),
792                (offset_of!(RawLuaPage, data) + size) as i32,
793                size as i32,
794                1,
795            )?;
796
797            let page_ref = page.as_ptr().as_mut().unwrap_unchecked();
798            let block = page.data_ptr_mut();
799            page_ref.free_next -= page_ref.block_size;
800            page_ref.busy_blocks += 1;
801
802            Some((
803                page,
804                GcObject::from_raw(NonNull::new_unchecked(block.cast::<RawGcObject>())),
805            ))
806        }
807    }
808}
luau_vm/memory/mod.rs

luau_vm/memory/
mod.rs
