#![allow(non_upper_case_globals, dead_code)]
use std::collections::{BTreeMap, HashMap, HashSet};
use std::fmt;
use std::mem;
use crate::codegen::MachineInstr;
pub const X86_CORO_FRAME_ALIGNMENT: u32 = 16;
pub const X86_CORO_MIN_FRAME_SIZE: u32 = 64;
pub const X86_CORO_FRAME_HEADER_SIZE: u32 = 40;
pub const X86_CORO_PROMISE_ALIGNMENT: u32 = 16;
pub const X86_CORO_MAX_SUSPEND_POINTS: usize = 256;
pub const X86_CORO_MAX_SPILLS: usize = 512;
pub const X86_CORO_STATE_NOT_STARTED: u32 = 0;
pub const X86_CORO_STATE_DONE: u32 = 0xFFFFFFFF;
pub const X86_CORO_STATE_RESERVED: u32 = 2;
pub const X86_CORO_HEAP_ELISION_MAX_SIZE: u32 = 4096;
pub const X86_CORO_HANDLE_ALIGNMENT: u32 = 8;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum X86CoroIntrinsicKind {
CoroId,
CoroIdRetcon,
CoroIdRetconOnce,
CoroBegin,
CoroEnd,
CoroSuspend,
CoroSuspendRetcon,
CoroFree,
CoroAlloc,
CoroFrame,
CoroSave,
CoroPromise,
CoroSize,
CoroAlign,
CoroResume,
CoroDestroy,
CoroDone,
}
impl fmt::Display for X86CoroIntrinsicKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let s = match self {
Self::CoroId => "coro.id",
Self::CoroIdRetcon => "coro.id.retcon",
Self::CoroIdRetconOnce => "coro.id.retcon.once",
Self::CoroBegin => "coro.begin",
Self::CoroEnd => "coro.end",
Self::CoroSuspend => "coro.suspend",
Self::CoroSuspendRetcon => "coro.suspend.retcon",
Self::CoroFree => "coro.free",
Self::CoroAlloc => "coro.alloc",
Self::CoroFrame => "coro.frame",
Self::CoroSave => "coro.save",
Self::CoroPromise => "coro.promise",
Self::CoroSize => "coro.size",
Self::CoroAlign => "coro.align",
Self::CoroResume => "coro.resume",
Self::CoroDestroy => "coro.destroy",
Self::CoroDone => "coro.done",
};
write!(f, "{}", s)
}
}
#[derive(Debug, Clone)]
pub struct X86CoroFrameLayout {
pub total_size: u32,
pub alignment: u32,
pub resume_fn_offset: u32,
pub destroy_fn_offset: u32,
pub index_offset: u32,
pub promise_offset: u32,
pub promise_size: u32,
pub num_params: u32,
pub params_offset: u32,
pub num_locals: u32,
pub locals_offset: u32,
pub num_spills: u32,
pub spills_offset: u32,
pub is_heap: bool,
pub unelided_size: u32,
}
impl Default for X86CoroFrameLayout {
fn default() -> Self {
Self {
total_size: X86_CORO_MIN_FRAME_SIZE,
alignment: X86_CORO_FRAME_ALIGNMENT,
resume_fn_offset: 0,
destroy_fn_offset: 8,
index_offset: 16,
promise_offset: 24,
promise_size: 0,
num_params: 0,
params_offset: 0,
num_locals: 0,
locals_offset: 0,
num_spills: 0,
spills_offset: 0,
is_heap: true,
unelided_size: X86_CORO_MIN_FRAME_SIZE,
}
}
}
impl X86CoroFrameLayout {
pub fn compute(
promise_size: u32,
promise_alignment: u32,
num_params: u32,
params_total_size: u32,
num_locals: u32,
locals_total_size: u32,
num_spills: u32,
spills_total_size: u32,
) -> Self {
let mut layout = Self::default();
let mut offset = 24u32;
let align = promise_alignment.max(layout.alignment);
offset = (offset + align - 1) & !(align - 1);
layout.promise_offset = offset;
layout.promise_size = promise_size;
offset += promise_size;
if num_params > 0 {
layout.num_params = num_params;
layout.params_offset = offset;
offset += params_total_size;
}
if num_locals > 0 {
layout.num_locals = num_locals;
layout.locals_offset = offset;
offset += locals_total_size;
}
if num_spills > 0 {
layout.num_spills = num_spills;
layout.spills_offset = offset;
offset += spills_total_size;
}
layout.total_size = (offset + layout.alignment - 1) & !(layout.alignment - 1);
layout.unelided_size = layout.total_size;
layout.alignment = layout.alignment.max(align);
layout
}
pub fn aligned_size(&self) -> u32 {
(self.total_size + self.alignment - 1) & !(self.alignment - 1)
}
pub fn generate_read_state(&self, frame_reg: u16) -> Vec<u8> {
self.generate_mov_from_frame(frame_reg, self.index_offset, 4)
}
pub fn generate_set_state(&self, frame_reg: u16, new_state: u32) -> Vec<u8> {
let mut seq = Vec::new();
if frame_reg >= 8 {
seq.push(0x41); }
seq.push(0xC7); let modrm = if frame_reg == 4 {
0x44 } else {
(0x40u16 | (frame_reg & 0x07)) as u8 };
seq.push(modrm);
if frame_reg == 4 {
seq.push(0x24); }
let disp = self.index_offset as u8;
seq.push(disp);
seq.extend_from_slice(&new_state.to_le_bytes());
seq
}
pub fn generate_load_resume_fn(&self, frame_reg: u16, dest_reg: u16) -> Vec<u8> {
let mut seq = Vec::new();
let rex: u8 =
0x48 | if dest_reg >= 8 { 0x04 } else { 0 } | if frame_reg >= 8 { 0x01 } else { 0 };
seq.push(rex);
seq.push(0x8B); let modrm = (0x40u16 | ((dest_reg & 0x07) << 3) | (frame_reg & 0x07)) as u8;
seq.push(modrm);
let disp = self.resume_fn_offset as u8;
seq.push(disp);
seq
}
pub fn generate_load_destroy_fn(&self, frame_reg: u16, dest_reg: u16) -> Vec<u8> {
let mut seq = Vec::new();
let rex: u8 =
0x48 | if dest_reg >= 8 { 0x04 } else { 0 } | if frame_reg >= 8 { 0x01 } else { 0 };
seq.push(rex);
seq.push(0x8B); let modrm = (0x40u16 | ((dest_reg & 0x07) << 3) | (frame_reg & 0x07)) as u8;
seq.push(modrm);
let disp = self.destroy_fn_offset as u8;
seq.push(disp);
seq
}
fn generate_mov_from_frame(&self, frame_reg: u16, offset: u32, size: u32) -> Vec<u8> {
let mut seq = Vec::new();
let rex: u8 = if size == 8 { 0x48 } else { 0 };
seq.push(rex);
seq.push(if size == 8 { 0x8B } else { 0x8B }); seq.push((0x40u16 | (frame_reg & 0x07)) as u8);
seq.push(offset as u8);
seq
}
}
#[derive(Debug, Clone, Copy)]
pub struct X86CoroSuspendFlags {
pub is_final: bool,
pub may_unwind: bool,
pub is_retcon: bool,
pub can_elide: bool,
}
impl Default for X86CoroSuspendFlags {
fn default() -> Self {
Self {
is_final: false,
may_unwind: false,
is_retcon: false,
can_elide: false,
}
}
}
#[derive(Debug, Clone)]
pub struct X86CoroSuspendPoint {
pub id: u32,
pub state_value: u32,
pub resume_offset: u32,
pub flags: X86CoroSuspendFlags,
pub spills: Vec<X86CoroSpillSlot>,
pub reloads: Vec<X86CoroSpillSlot>,
pub passes_handle: bool,
}
#[derive(Debug, Clone)]
pub struct X86CoroSpillSlot {
pub id: u32,
pub frame_offset: u32,
pub size: u32,
pub alignment: u32,
pub original_value: Option<u64>,
pub owner_suspend_id: u32,
pub is_gc_pointer: bool,
pub dwarf_reg: Option<u16>,
}
#[derive(Debug, Clone)]
pub struct X86CoroHandle {
pub frame_ptr: u64,
pub handle_size: u32,
pub is_valid: bool,
}
impl X86CoroHandle {
pub fn new(frame_ptr: u64) -> Self {
Self {
frame_ptr,
handle_size: 8,
is_valid: true,
}
}
pub fn null() -> Self {
Self {
frame_ptr: 0,
handle_size: 8,
is_valid: false,
}
}
pub fn generate_resume_call(&self, layout: &X86CoroFrameLayout) -> Vec<u8> {
let mut seq = Vec::new();
seq.push(0x48); seq.push(0x8B); seq.push(0x00); seq.push(0xFF); seq.push(0xD0); seq
}
pub fn generate_destroy_call(&self, layout: &X86CoroFrameLayout) -> Vec<u8> {
let mut seq = Vec::new();
seq.push(0x48);
seq.push(0x8B);
seq.push(0x40); seq.push(0x08); seq.push(0xFF);
seq.push(0xD0);
seq
}
}
#[derive(Debug, Clone)]
pub struct X86CoroRampFunction {
pub name: String,
pub frame_layout: X86CoroFrameLayout,
pub heap_elided: bool,
pub parameters: Vec<X86CoroParameter>,
pub promise_type: Option<X86CoroPromiseType>,
pub body_address: Option<u64>,
pub prologue_size: u32,
}
#[derive(Debug, Clone)]
pub struct X86CoroParameter {
pub name: String,
pub size: u32,
pub frame_offset: u32,
pub by_value: bool,
pub register: Option<u16>,
}
#[derive(Debug, Clone)]
pub struct X86CoroPromiseType {
pub size: u32,
pub alignment: u32,
pub type_name: String,
pub has_get_return_object: bool,
pub has_initial_suspend: bool,
pub has_final_suspend: bool,
pub has_unhandled_exception: bool,
pub has_return_value: bool,
}
impl X86CoroRampFunction {
pub fn new(name: &str, frame_layout: X86CoroFrameLayout) -> Self {
Self {
name: name.to_string(),
frame_layout,
heap_elided: false,
parameters: Vec::new(),
promise_type: None,
body_address: None,
prologue_size: 0,
}
}
pub fn add_parameter(&mut self, param: X86CoroParameter) {
self.parameters.push(param);
}
pub fn set_promise(&mut self, promise: X86CoroPromiseType) {
self.promise_type = Some(promise);
}
pub fn generate_prologue(&self) -> Vec<u8> {
let mut seq = Vec::new();
seq.push(0x55);
seq.push(0x48);
seq.push(0x89);
seq.push(0xE5);
seq.push(0x53);
if self.frame_layout.is_heap && !self.heap_elided {
seq.push(0xBF); seq.extend_from_slice(&self.frame_layout.total_size.to_le_bytes());
seq.push(0xBE); seq.extend_from_slice(&self.frame_layout.alignment.to_le_bytes());
seq.push(0xE8);
seq.extend_from_slice(&0u32.to_le_bytes()); } else {
seq.push(0x48);
seq.push(0x81);
seq.push(0xEC);
seq.extend_from_slice(&self.frame_layout.total_size.to_le_bytes());
}
seq.push(0x48);
seq.push(0x89);
seq.push(0xC3);
for param in &self.parameters {
if let Some(reg) = param.register {
let rex = 0x48u8 | if reg >= 8 { 0x04 } else { 0 };
seq.push(rex);
seq.push(0x89); let modrm = (0x83u16 | ((reg & 0x07) << 3)) as u8; seq.push(modrm);
seq.extend_from_slice(¶m.frame_offset.to_le_bytes());
}
}
seq.push(0xC7); seq.push(0x43); seq.push(self.frame_layout.index_offset as u8);
seq.extend_from_slice(&X86_CORO_STATE_NOT_STARTED.to_le_bytes());
seq
}
pub fn generate_epilogue(&self) -> Vec<u8> {
let mut seq = Vec::new();
seq.push(0x48);
seq.push(0x89);
seq.push(0xD8); seq.push(0x5B);
seq.push(0x5D);
seq.push(0xC3);
seq
}
}
#[derive(Debug, Clone)]
pub struct X86CoroResumeFunction {
pub name: String,
pub frame_layout: X86CoroFrameLayout,
pub suspend_points: Vec<X86CoroSuspendPoint>,
pub symmetric_transfer: bool,
}
impl X86CoroResumeFunction {
pub fn new(name: &str, frame_layout: X86CoroFrameLayout) -> Self {
Self {
name: name.to_string(),
frame_layout,
suspend_points: Vec::new(),
symmetric_transfer: false,
}
}
pub fn add_suspend_point(&mut self, sp: X86CoroSuspendPoint) -> u32 {
let state = sp.state_value;
self.suspend_points.push(sp);
state
}
pub fn generate_body(&self) -> Vec<u8> {
let mut seq = Vec::new();
seq.push(0x55);
seq.push(0x48);
seq.push(0x89);
seq.push(0xE5);
seq.push(0x53);
seq.push(0x48);
seq.push(0x89);
seq.push(0xFB);
seq.push(0x8B);
seq.push(0x43);
seq.push(self.frame_layout.index_offset as u8);
let num_points = self.suspend_points.len() as u32;
seq.push(0x83);
seq.push(0xF8); seq.push((num_points.max(1) - 1) as u8);
seq.push(0x77); seq.push(0x20);
for sp in &self.suspend_points {
let state = sp.state_value;
if state < 128 {
seq.push(0x83); seq.push(0xF8);
seq.push(state as u8);
} else {
seq.push(0x3D); seq.extend_from_slice(&state.to_le_bytes());
}
seq.push(0x74); seq.push(0x08);
seq.push(0xE9);
seq.extend_from_slice(&0u32.to_le_bytes()); }
seq.push(0x5B); seq.push(0x5D); seq.push(0xC3);
seq
}
pub fn generate_resume_point(&self, sp: &X86CoroSuspendPoint) -> Vec<u8> {
let mut seq = Vec::new();
for reload in &sp.reloads {
if let Some(reg) = reload.dwarf_reg {
let rex: u8 = 0x48 | if reg >= 8 { 0x04 } else { 0 };
seq.push(rex);
seq.push(0x8B); let modrm = (0x83u16 | ((reg & 0x07) << 3)) as u8; seq.push(modrm);
seq.extend_from_slice(&reload.frame_offset.to_le_bytes());
}
}
seq
}
}
#[derive(Debug, Clone)]
pub struct X86CoroDestroyFunction {
pub name: String,
pub frame_layout: X86CoroFrameLayout,
pub destructors: Vec<X86CoroDestructor>,
pub needs_deallocation: bool,
}
#[derive(Debug, Clone)]
pub struct X86CoroDestructor {
pub state_threshold: u32,
pub object_offset: u32,
pub destructor_fn: Option<u64>,
pub object_size: u32,
}
impl X86CoroDestroyFunction {
pub fn new(name: &str, frame_layout: X86CoroFrameLayout) -> Self {
Self {
name: name.to_string(),
frame_layout,
destructors: Vec::new(),
needs_deallocation: true,
}
}
pub fn add_destructor(&mut self, dtor: X86CoroDestructor) {
self.destructors.push(dtor);
}
pub fn generate_body(&self) -> Vec<u8> {
let mut seq = Vec::new();
seq.push(0x55);
seq.push(0x48);
seq.push(0x89);
seq.push(0xE5);
seq.push(0x53);
seq.push(0x48);
seq.push(0x89);
seq.push(0xFB);
seq.push(0x8B);
seq.push(0x43);
seq.push(self.frame_layout.index_offset as u8);
for dtor in self.destructors.iter().rev() {
seq.push(0x3D); seq.extend_from_slice(&dtor.state_threshold.to_le_bytes());
seq.push(0x7C); seq.push(0x0F);
seq.push(0x48);
seq.push(0x8D);
seq.push(0xBB); seq.extend_from_slice(&dtor.object_offset.to_le_bytes());
seq.push(0xE8);
seq.extend_from_slice(&0u32.to_le_bytes()); }
if self.needs_deallocation && self.frame_layout.is_heap {
seq.push(0x48);
seq.push(0x89);
seq.push(0xDF);
seq.push(0xE8);
seq.extend_from_slice(&0u32.to_le_bytes()); }
seq.push(0x5B); seq.push(0x5D); seq.push(0xC3);
seq
}
}
#[derive(Debug, Clone)]
pub struct X86CoroSymmetricTransfer {
pub enabled: bool,
pub target_handle_reg: Option<u16>,
pub own_frame_reg: Option<u16>,
}
impl X86CoroSymmetricTransfer {
pub fn new() -> Self {
Self {
enabled: false,
target_handle_reg: None,
own_frame_reg: None,
}
}
pub fn enable(&mut self) {
self.enabled = true;
}
pub fn generate_tail_call_sequence(
&self,
target_handle_reg: u16,
layout: &X86CoroFrameLayout,
) -> Vec<u8> {
let mut seq = Vec::new();
let rex: u8 = 0x48 | if target_handle_reg >= 8 { 0x01 } else { 0 };
seq.push(rex);
seq.push(0x8B); seq.push((target_handle_reg & 0x07) as u8); seq.push(0xFF);
seq.push(0xE0);
seq
}
pub fn is_eligible(&self, sp: &X86CoroSuspendPoint) -> bool {
if !self.enabled {
return false;
}
if self.target_handle_reg.is_none() {
return false;
}
if sp.flags.is_final {
return false;
}
true
}
}
#[derive(Debug, Clone)]
pub struct X86CoroHeapElision {
pub enabled: bool,
pub elided: bool,
pub max_frame_size: u32,
pub elision_failure_reason: Option<String>,
}
impl Default for X86CoroHeapElision {
fn default() -> Self {
Self {
enabled: true,
elided: false,
max_frame_size: X86_CORO_HEAP_ELISION_MAX_SIZE,
elision_failure_reason: None,
}
}
}
impl X86CoroHeapElision {
pub fn try_elide(
&mut self,
frame_size: u32,
does_not_escape: bool,
is_recursive: bool,
) -> bool {
if !self.enabled {
self.elision_failure_reason = Some("elision disabled".to_string());
return false;
}
if !does_not_escape {
self.elision_failure_reason =
Some("coroutine handle escapes caller's scope".to_string());
return false;
}
if frame_size > self.max_frame_size {
self.elision_failure_reason = Some(format!(
"frame size {} exceeds max {}",
frame_size, self.max_frame_size
));
return false;
}
if is_recursive {
self.elision_failure_reason =
Some("recursive coroutines may cause unbounded stack growth".to_string());
return false;
}
self.elided = true;
self.elision_failure_reason = None;
true
}
pub fn generate_alloca_sequence(&self, frame_size: u32) -> Vec<u8> {
let mut seq = Vec::new();
seq.push(0x48);
seq.push(0x81); seq.push(0xEC); seq.extend_from_slice(&frame_size.to_le_bytes());
seq.push(0x48);
seq.push(0x89);
seq.push(0xE0);
seq
}
}
pub struct X86CoroABI {
pub handle_reg: u16,
pub return_reg: u16,
pub frame_ptr_reg: u16,
pub scratch_reg: u16,
pub callee_saved_regs: Vec<u16>,
pub caller_saved_regs: Vec<u16>,
}
impl Default for X86CoroABI {
fn default() -> Self {
Self {
handle_reg: 7, return_reg: 0, frame_ptr_reg: 3, scratch_reg: 0, callee_saved_regs: vec![3, 6, 12, 13, 14, 15], caller_saved_regs: vec![0, 2, 1, 4, 5, 8, 9, 10, 11], }
}
}
impl X86CoroABI {
pub fn generate_indirect_resume_call(&self, frame_ptr_reg: u16) -> Vec<u8> {
let mut seq = Vec::new();
let rex = 0x48u8;
seq.push(rex);
seq.push(0x8B); seq.push((frame_ptr_reg & 0x07) as u8); seq.push(0xFF);
seq.push(0xD0);
seq
}
pub fn generate_coro_prologue(&self) -> Vec<u8> {
let mut seq = Vec::new();
seq.push(0x55);
seq.push(0x48);
seq.push(0x89);
seq.push(0xE5);
seq.push(0x53);
seq.push(0x48);
seq.push(0x89);
seq.push(0xFB);
seq
}
pub fn generate_coro_epilogue(&self) -> Vec<u8> {
let mut seq = Vec::new();
seq.push(0x5B);
seq.push(0x5D);
seq.push(0xC3);
seq
}
pub fn generate_promise_access(&self, frame_ptr_reg: u16, promise_offset: u32) -> Vec<u8> {
let mut seq = Vec::new();
seq.push(0x48);
seq.push(0x8D); let modrm = (0x80u16 | (frame_ptr_reg & 0x07)) as u8; seq.push(modrm);
seq.extend_from_slice(&promise_offset.to_le_bytes());
seq
}
}
#[derive(Debug, Clone)]
pub struct X86CoroLoweringConfig {
pub symmetric_transfer: bool,
pub heap_elision: bool,
pub heap_elision_max_size: u32,
pub split_coroutine: bool,
pub target_triple: String,
pub emit_debug_info: bool,
}
impl Default for X86CoroLoweringConfig {
fn default() -> Self {
Self {
symmetric_transfer: true,
heap_elision: true,
heap_elision_max_size: X86_CORO_HEAP_ELISION_MAX_SIZE,
split_coroutine: true,
target_triple: "x86_64-unknown-linux-gnu".to_string(),
emit_debug_info: false,
}
}
}
pub struct X86CoroLowering {
pub config: X86CoroLoweringConfig,
frame_layout: X86CoroFrameLayout,
ramp_function: Option<X86CoroRampFunction>,
resume_function: Option<X86CoroResumeFunction>,
destroy_function: Option<X86CoroDestroyFunction>,
symmetric_transfer: X86CoroSymmetricTransfer,
heap_elision: X86CoroHeapElision,
abi: X86CoroABI,
coroutine_name: Option<String>,
suspend_points: Vec<X86CoroSuspendPoint>,
spill_slots: Vec<X86CoroSpillSlot>,
parameters: Vec<X86CoroParameter>,
promise_info: Option<X86CoroPromiseType>,
}
impl X86CoroLowering {
pub fn new(config: X86CoroLoweringConfig) -> Self {
let mut st = X86CoroSymmetricTransfer::new();
if config.symmetric_transfer {
st.enable();
}
let mut he = X86CoroHeapElision::default();
he.enabled = config.heap_elision;
he.max_frame_size = config.heap_elision_max_size;
Self {
config,
frame_layout: X86CoroFrameLayout::default(),
ramp_function: None,
resume_function: None,
destroy_function: None,
symmetric_transfer: st,
heap_elision: he,
abi: X86CoroABI::default(),
coroutine_name: None,
suspend_points: Vec::new(),
spill_slots: Vec::new(),
parameters: Vec::new(),
promise_info: None,
}
}
pub fn begin_coroutine(&mut self, name: &str) {
self.coroutine_name = Some(name.to_string());
self.suspend_points.clear();
self.spill_slots.clear();
self.parameters.clear();
}
pub fn set_promise(&mut self, size: u32, alignment: u32, type_name: &str) {
self.promise_info = Some(X86CoroPromiseType {
size,
alignment,
type_name: type_name.to_string(),
has_get_return_object: true,
has_initial_suspend: true,
has_final_suspend: true,
has_unhandled_exception: true,
has_return_value: true,
});
}
pub fn add_parameter(&mut self, name: &str, size: u32, by_value: bool, register: Option<u16>) {
self.parameters.push(X86CoroParameter {
name: name.to_string(),
size,
frame_offset: 0, by_value,
register,
});
}
pub fn add_spill(
&mut self,
size: u32,
alignment: u32,
owner_suspend_id: u32,
is_gc_pointer: bool,
dwarf_reg: Option<u16>,
) -> u32 {
let id = self.spill_slots.len() as u32;
self.spill_slots.push(X86CoroSpillSlot {
id,
frame_offset: 0, size,
alignment,
original_value: None,
owner_suspend_id,
is_gc_pointer,
dwarf_reg,
});
id
}
pub fn add_suspend_point(
&mut self,
is_final: bool,
may_unwind: bool,
spills: Vec<u32>,
reloads: Vec<u32>,
) -> u32 {
let state_value = self.suspend_points.len() as u32 + X86_CORO_STATE_RESERVED;
let id = self.suspend_points.len() as u32;
let spill_slots: Vec<X86CoroSpillSlot> = spills
.iter()
.filter_map(|&sid| self.spill_slots.iter().find(|s| s.id == sid).cloned())
.collect();
let reload_slots: Vec<X86CoroSpillSlot> = reloads
.iter()
.filter_map(|&sid| self.spill_slots.iter().find(|s| s.id == sid).cloned())
.collect();
self.suspend_points.push(X86CoroSuspendPoint {
id,
state_value,
resume_offset: 0, flags: X86CoroSuspendFlags {
is_final,
may_unwind,
is_retcon: false,
can_elide: false,
},
spills: spill_slots,
reloads: reload_slots,
passes_handle: false,
});
state_value
}
pub fn compute_frame_layout(&mut self) -> &X86CoroFrameLayout {
let promise_size = self.promise_info.as_ref().map(|p| p.size).unwrap_or(0);
let promise_align = self
.promise_info
.as_ref()
.map(|p| p.alignment)
.unwrap_or(X86_CORO_PROMISE_ALIGNMENT);
let num_params = self.parameters.len() as u32;
let params_total: u32 = self.parameters.iter().map(|p| p.size).sum();
let num_spills = self.spill_slots.len() as u32;
let spills_total: u32 = self.spill_slots.iter().map(|s| s.size).sum();
self.frame_layout = X86CoroFrameLayout::compute(
promise_size,
promise_align,
num_params,
params_total,
0, 0,
num_spills,
spills_total,
);
let mut param_offset = self.frame_layout.params_offset;
for param in &mut self.parameters {
param.frame_offset = param_offset;
param_offset += param.size;
}
let mut spill_offset = self.frame_layout.spills_offset;
for spill in &mut self.spill_slots {
spill.frame_offset = spill_offset;
spill_offset += spill.size;
}
&self.frame_layout
}
pub fn try_heap_elision(&mut self, does_not_escape: bool, is_recursive: bool) -> bool {
let frame_size = self.frame_layout.total_size;
let result = self
.heap_elision
.try_elide(frame_size, does_not_escape, is_recursive);
if result {
self.frame_layout.is_heap = false;
}
result
}
pub fn build_ramp(&mut self) -> &X86CoroRampFunction {
let name = format!("{}_ramp", self.coroutine_name.as_deref().unwrap_or("coro"));
let mut ramp = X86CoroRampFunction::new(&name, self.frame_layout.clone());
ramp.heap_elided = self.heap_elision.elided;
ramp.parameters = self.parameters.clone();
ramp.promise_type = self.promise_info.clone();
ramp.prologue_size = ramp.generate_prologue().len() as u32;
self.ramp_function = Some(ramp);
self.ramp_function.as_ref().unwrap()
}
pub fn build_resume(&mut self) -> &X86CoroResumeFunction {
let name = format!(
"{}_resume",
self.coroutine_name.as_deref().unwrap_or("coro")
);
let mut resume = X86CoroResumeFunction::new(&name, self.frame_layout.clone());
resume.symmetric_transfer = self.symmetric_transfer.enabled;
for sp in &self.suspend_points {
resume.add_suspend_point(sp.clone());
}
self.resume_function = Some(resume);
self.resume_function.as_ref().unwrap()
}
pub fn build_destroy(&mut self) -> &X86CoroDestroyFunction {
let name = format!(
"{}_destroy",
self.coroutine_name.as_deref().unwrap_or("coro")
);
let mut destroy = X86CoroDestroyFunction::new(&name, self.frame_layout.clone());
destroy.needs_deallocation = self.frame_layout.is_heap;
for sp in self.suspend_points.iter().rev() {
destroy.add_destructor(X86CoroDestructor {
state_threshold: sp.state_value,
object_offset: 0, destructor_fn: None,
object_size: 0,
});
}
self.destroy_function = Some(destroy);
self.destroy_function.as_ref().unwrap()
}
pub fn finalize(&mut self) -> X86CoroLoweringResult {
self.compute_frame_layout();
let ramp = self.build_ramp().clone();
let resume = self.build_resume().clone();
let destroy = self.build_destroy().clone();
X86CoroLoweringResult {
coroutine_name: self.coroutine_name.clone().unwrap_or_default(),
frame_layout: self.frame_layout.clone(),
ramp_function: ramp,
resume_function: resume,
destroy_function: destroy,
heap_elided: self.heap_elision.elided,
symmetric_transfer_enabled: self.symmetric_transfer.enabled,
spill_slots: self.spill_slots.clone(),
suspend_points: self.suspend_points.clone(),
}
}
pub fn get_abi(&self) -> &X86CoroABI {
&self.abi
}
}
#[derive(Debug, Clone)]
pub struct X86CoroLoweringResult {
pub coroutine_name: String,
pub frame_layout: X86CoroFrameLayout,
pub ramp_function: X86CoroRampFunction,
pub resume_function: X86CoroResumeFunction,
pub destroy_function: X86CoroDestroyFunction,
pub heap_elided: bool,
pub symmetric_transfer_enabled: bool,
pub spill_slots: Vec<X86CoroSpillSlot>,
pub suspend_points: Vec<X86CoroSuspendPoint>,
}
impl X86CoroLoweringResult {
pub fn total_code_size(&self) -> u32 {
let ramp_prologue = self.ramp_function.generate_prologue().len() as u32;
let ramp_epilogue = self.ramp_function.generate_epilogue().len() as u32;
let resume_body = self.resume_function.generate_body().len() as u32;
let destroy_body = self.destroy_function.generate_body().len() as u32;
ramp_prologue + ramp_epilogue + resume_body + destroy_body
}
pub fn frame_size(&self) -> u32 {
self.frame_layout.aligned_size()
}
pub fn is_valid(&self) -> bool {
!self.coroutine_name.is_empty() && self.frame_layout.total_size > 0
}
}
#[derive(Debug, Clone)]
pub struct X86CoroCleanupFunction {
pub name: String,
pub frame_layout: X86CoroFrameLayout,
pub destructors: Vec<X86CoroDestructor>,
}
impl X86CoroCleanupFunction {
pub fn new(name: &str, frame_layout: X86CoroFrameLayout) -> Self {
Self {
name: name.to_string(),
frame_layout,
destructors: Vec::new(),
}
}
pub fn add_destructor(&mut self, dtor: X86CoroDestructor) {
self.destructors.push(dtor);
}
pub fn generate_body(&self) -> Vec<u8> {
let mut seq = Vec::new();
seq.push(0x55); seq.push(0x48);
seq.push(0x89);
seq.push(0xE5); seq.push(0x53);
seq.push(0x48);
seq.push(0x89);
seq.push(0xFB);
seq.push(0x8B);
seq.push(0x43);
seq.push(self.frame_layout.index_offset as u8);
for dtor in self.destructors.iter().rev() {
if dtor.state_threshold < 128 {
seq.push(0x83);
seq.push(0xF8);
seq.push(dtor.state_threshold as u8);
} else {
seq.push(0x3D);
seq.extend_from_slice(&dtor.state_threshold.to_le_bytes());
}
seq.push(0x7C); seq.push(0x0A);
seq.push(0x48);
seq.push(0x8D);
seq.push(0xBB);
seq.extend_from_slice(&dtor.object_offset.to_le_bytes());
seq.push(0xE8);
seq.extend_from_slice(&0u32.to_le_bytes());
}
seq.push(0x5B);
seq.push(0x5D);
seq.push(0xC3);
seq
}
}
#[derive(Debug, Clone)]
pub struct X86CoroSplittingPass {
pub config: X86CoroLoweringConfig,
suspend_states: Vec<X86CoroSuspendState>,
}
#[derive(Debug, Clone)]
pub struct X86CoroSuspendState {
pub suspend: X86CoroSuspendPoint,
pub live_values: Vec<X86CoroSpillSlot>,
pub before_block: Option<u64>,
pub after_block: Option<u64>,
pub requires_full_save: bool,
}
impl X86CoroSplittingPass {
pub fn new(config: X86CoroLoweringConfig) -> Self {
Self {
config,
suspend_states: Vec::new(),
}
}
pub fn register_suspend(
&mut self,
suspend: X86CoroSuspendPoint,
live_values: Vec<X86CoroSpillSlot>,
requires_full_save: bool,
) {
self.suspend_states.push(X86CoroSuspendState {
suspend,
live_values,
before_block: None,
after_block: None,
requires_full_save,
});
}
pub fn split(&self, coro_name: &str) -> X86CoroSplitResult {
let num_suspends = self.suspend_states.len();
let num_full_saves = self
.suspend_states
.iter()
.filter(|s| s.requires_full_save)
.count();
X86CoroSplitResult {
coroutine_name: coro_name.to_string(),
num_suspend_points: num_suspends,
num_full_saves,
requires_cleanup_fn: true,
has_dynamic_allocation: true,
}
}
}
#[derive(Debug, Clone)]
pub struct X86CoroSplitResult {
pub coroutine_name: String,
pub num_suspend_points: usize,
pub num_full_saves: usize,
pub requires_cleanup_fn: bool,
pub has_dynamic_allocation: bool,
}
#[derive(Debug, Clone)]
pub struct X86CoroDebugInfo {
pub frame_base_reg: u16,
pub variable_remappings: Vec<X86CoroVarRemapping>,
pub suspend_locations: Vec<X86CoroSuspendLocation>,
}
#[derive(Debug, Clone)]
pub struct X86CoroVarRemapping {
pub name: String,
pub original_offset: i32,
pub frame_offset: u32,
pub size: u32,
}
#[derive(Debug, Clone)]
pub struct X86CoroSuspendLocation {
pub suspend_id: u32,
pub line: u32,
pub column: u32,
}
pub struct X86CoroElisionAnalysis {
pub stored_in_global: bool,
pub passed_to_thread: bool,
pub returned_from_non_ramp: bool,
pub symmetric_transfer_preserves_elision: bool,
}
impl Default for X86CoroElisionAnalysis {
fn default() -> Self {
Self {
stored_in_global: false,
passed_to_thread: false,
returned_from_non_ramp: false,
symmetric_transfer_preserves_elision: true,
}
}
}
impl X86CoroElisionAnalysis {
pub fn can_elide(&self) -> bool {
!self.stored_in_global && !self.passed_to_thread && !self.returned_from_non_ramp
}
pub fn mark_escapes(&mut self) {
self.stored_in_global = true;
}
pub fn check_elision_conditions(&self, frame_size: u32, max_size: u32) -> Option<String> {
if !self.can_elide() {
return Some("coroutine escapes".to_string());
}
if frame_size > max_size {
return Some(format!("frame too large: {} > {}", frame_size, max_size));
}
None
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_frame_layout_computation() {
let layout = X86CoroFrameLayout::compute(
32, 8, 2, 16, 0, 0, 4, 32, );
assert!(layout.total_size >= X86_CORO_MIN_FRAME_SIZE);
assert_eq!(layout.alignment, 16);
assert_eq!(layout.resume_fn_offset, 0);
assert_eq!(layout.destroy_fn_offset, 8);
assert_eq!(layout.index_offset, 16);
assert!(layout.promise_offset >= 24);
assert_eq!(layout.promise_size, 32);
assert_eq!(layout.num_params, 2);
assert_eq!(layout.num_spills, 4);
}
#[test]
fn test_suspend_point_state_values() {
let mut lowering = X86CoroLowering::new(X86CoroLoweringConfig::default());
lowering.begin_coroutine("test_coro");
let s0 = lowering.add_suspend_point(false, false, vec![], vec![]);
let s1 = lowering.add_suspend_point(false, false, vec![], vec![]);
let s2 = lowering.add_suspend_point(true, false, vec![], vec![]);
assert_eq!(s0, X86_CORO_STATE_RESERVED); assert_eq!(s1, X86_CORO_STATE_RESERVED + 1); assert_eq!(s2, X86_CORO_STATE_RESERVED + 2); }
#[test]
fn test_ramp_prologue_generation() {
let mut lowering = X86CoroLowering::new(X86CoroLoweringConfig::default());
lowering.begin_coroutine("test_coro");
lowering.set_promise(16, 8, "promise_type");
lowering.add_parameter("x", 8, true, Some(5));
lowering.compute_frame_layout();
let ramp = lowering.build_ramp();
let prologue = ramp.generate_prologue();
assert!(prologue.len() > 10);
assert_eq!(prologue[0], 0x55);
}
#[test]
fn test_ramp_epilogue_generation() {
let layout = X86CoroFrameLayout::default();
let ramp = X86CoroRampFunction::new("test", layout);
let epilogue = ramp.generate_epilogue();
assert_eq!(epilogue.last(), Some(&0xC3));
}
#[test]
fn test_resume_function_generation() {
let layout = X86CoroFrameLayout::default();
let mut resume = X86CoroResumeFunction::new("test_resume", layout);
let sp = X86CoroSuspendPoint {
id: 0,
state_value: 2,
resume_offset: 0,
flags: X86CoroSuspendFlags::default(),
spills: vec![],
reloads: vec![],
passes_handle: false,
};
resume.add_suspend_point(sp);
let body = resume.generate_body();
assert!(body.len() > 5);
assert_eq!(body[0], 0x55);
}
#[test]
fn test_destroy_function_generation() {
let layout = X86CoroFrameLayout::default();
let mut destroy = X86CoroDestroyFunction::new("test_destroy", layout);
destroy.add_destructor(X86CoroDestructor {
state_threshold: 2,
object_offset: 32,
destructor_fn: Some(0x401000),
object_size: 16,
});
let body = destroy.generate_body();
assert!(body.len() > 5);
assert_eq!(body.last(), Some(&0xC3));
}
#[test]
fn test_symmetric_transfer_eligibility() {
let mut st = X86CoroSymmetricTransfer::new();
assert!(!st.is_eligible(&X86CoroSuspendPoint {
id: 0,
state_value: 2,
resume_offset: 0,
flags: X86CoroSuspendFlags::default(),
spills: vec![],
reloads: vec![],
passes_handle: false,
}));
st.enable();
st.target_handle_reg = Some(11);
let sp = X86CoroSuspendPoint {
id: 1,
state_value: 3,
resume_offset: 0,
flags: X86CoroSuspendFlags::default(),
spills: vec![],
reloads: vec![],
passes_handle: false,
};
assert!(st.is_eligible(&sp));
let final_sp = X86CoroSuspendPoint {
id: 2,
state_value: 4,
resume_offset: 0,
flags: X86CoroSuspendFlags {
is_final: true,
..Default::default()
},
spills: vec![],
reloads: vec![],
passes_handle: false,
};
assert!(!st.is_eligible(&final_sp)); }
#[test]
fn test_heap_elision_success() {
let mut he = X86CoroHeapElision::default();
let result = he.try_elide(256, true, false);
assert!(result);
assert!(he.elided);
assert!(he.elision_failure_reason.is_none());
}
#[test]
fn test_heap_elision_failure_escapes() {
let mut he = X86CoroHeapElision::default();
let result = he.try_elide(256, false, false);
assert!(!result);
assert!(he.elision_failure_reason.is_some());
}
#[test]
fn test_heap_elision_failure_large() {
let mut he = X86CoroHeapElision::default();
let result = he.try_elide(8192, true, false);
assert!(!result);
assert!(he.elision_failure_reason.is_some());
}
#[test]
fn test_heap_elision_failure_recursive() {
let mut he = X86CoroHeapElision::default();
let result = he.try_elide(256, true, true);
assert!(!result);
}
#[test]
fn test_coro_abi_prologue_epilogue() {
let abi = X86CoroABI::default();
let prologue = abi.generate_coro_prologue();
let epilogue = abi.generate_coro_epilogue();
assert_eq!(prologue[0], 0x55);
assert_eq!(epilogue.last(), Some(&0xC3));
}
#[test]
fn test_full_lowering_pipeline() {
let mut lowering = X86CoroLowering::new(X86CoroLoweringConfig::default());
lowering.begin_coroutine("my_coro");
lowering.set_promise(24, 8, "MyPromise");
lowering.add_parameter("arg0", 8, true, Some(7)); lowering.add_parameter("arg1", 4, true, Some(6));
let s0 = lowering.add_spill(8, 8, 0, false, Some(0)); let s1 = lowering.add_spill(8, 8, 0, true, Some(3));
lowering.add_suspend_point(false, false, vec![s0], vec![s0]);
lowering.add_suspend_point(true, false, vec![s0, s1], vec![s0, s1]);
let elided = lowering.try_heap_elision(true, false);
let result = lowering.finalize();
assert_eq!(result.coroutine_name, "my_coro");
assert!(result.frame_layout.total_size > 0);
assert!(result.is_valid());
assert_eq!(result.suspend_points.len(), 2);
assert_eq!(result.spill_slots.len(), 2);
let code_size = result.total_code_size();
assert!(code_size > 0);
}
#[test]
fn test_handle_null() {
let handle = X86CoroHandle::null();
assert!(!handle.is_valid);
assert_eq!(handle.frame_ptr, 0);
}
#[test]
fn test_handle_valid() {
let handle = X86CoroHandle::new(0x7FFF00001000);
assert!(handle.is_valid);
assert_eq!(handle.frame_ptr, 0x7FFF00001000);
}
#[test]
fn test_intrinsic_display() {
assert_eq!(X86CoroIntrinsicKind::CoroId.to_string(), "coro.id");
assert_eq!(X86CoroIntrinsicKind::CoroBegin.to_string(), "coro.begin");
assert_eq!(
X86CoroIntrinsicKind::CoroSuspend.to_string(),
"coro.suspend"
);
assert_eq!(X86CoroIntrinsicKind::CoroResume.to_string(), "coro.resume");
assert_eq!(
X86CoroIntrinsicKind::CoroDestroy.to_string(),
"coro.destroy"
);
}
#[test]
fn test_frame_state_read_write() {
let layout = X86CoroFrameLayout::default();
let write_seq = layout.generate_set_state(3, 3); assert!(write_seq.len() >= 7);
let read_seq = layout.generate_read_state(3); assert!(read_seq.len() >= 3);
}
#[test]
fn test_spill_slot_creation() {
let mut lowering = X86CoroLowering::new(X86CoroLoweringConfig::default());
lowering.begin_coroutine("spill_test");
let id = lowering.add_spill(8, 8, 0, true, Some(0));
assert_eq!(id, 0);
let id2 = lowering.add_spill(16, 16, 0, false, Some(2));
assert_eq!(id2, 1);
lowering.compute_frame_layout();
assert!(lowering.spill_slots[0].frame_offset > 0);
assert!(lowering.spill_slots[1].frame_offset > lowering.spill_slots[0].frame_offset);
}
#[test]
fn test_cleanup_function_generation() {
let layout = X86CoroFrameLayout::default();
let mut cleanup = X86CoroCleanupFunction::new("test_cleanup", layout);
cleanup.add_destructor(X86CoroDestructor {
state_threshold: 2,
object_offset: 32,
destructor_fn: Some(0x500000),
object_size: 16,
});
let body = cleanup.generate_body();
assert!(body.len() > 10);
assert_eq!(body[0], 0x55);
assert_eq!(body.last(), Some(&0xC3));
}
#[test]
fn test_splitting_pass() {
let config = X86CoroLoweringConfig::default();
let mut pass = X86CoroSplittingPass::new(config);
let sp1 = X86CoroSuspendPoint {
id: 0,
state_value: 2,
resume_offset: 0,
flags: X86CoroSuspendFlags::default(),
spills: vec![],
reloads: vec![],
passes_handle: false,
};
pass.register_suspend(sp1, vec![], false);
let result = pass.split("test_split");
assert_eq!(result.num_suspend_points, 1);
assert!(result.requires_cleanup_fn);
}
#[test]
fn test_elision_analysis() {
let analysis = X86CoroElisionAnalysis::default();
assert!(analysis.can_elide());
assert!(analysis.check_elision_conditions(256, 4096).is_none());
}
#[test]
fn test_elision_analysis_escape() {
let mut analysis = X86CoroElisionAnalysis::default();
analysis.mark_escapes();
assert!(!analysis.can_elide());
}
#[test]
fn test_elision_too_large() {
let analysis = X86CoroElisionAnalysis::default();
let reason = analysis.check_elision_conditions(8192, 4096);
assert!(reason.is_some());
assert!(reason.unwrap().contains("too large"));
}
#[test]
fn test_coro_result_validity() {
let layout = X86CoroFrameLayout::default();
let result = X86CoroLoweringResult {
coroutine_name: "".to_string(),
frame_layout: layout.clone(),
ramp_function: X86CoroRampFunction::new("", layout.clone()),
resume_function: X86CoroResumeFunction::new("", layout.clone()),
destroy_function: X86CoroDestroyFunction::new("", layout.clone()),
heap_elided: false,
symmetric_transfer_enabled: false,
spill_slots: vec![],
suspend_points: vec![],
};
assert!(!result.is_valid());
}
#[test]
fn test_abi_registers() {
let abi = X86CoroABI::default();
assert_eq!(abi.handle_reg, 7);
assert_eq!(abi.return_reg, 0);
assert_eq!(abi.frame_ptr_reg, 3);
assert_eq!(abi.callee_saved_regs.len(), 6);
}
#[test]
fn test_indirect_resume_call_gen() {
let abi = X86CoroABI::default();
let seq = abi.generate_indirect_resume_call(3);
assert!(seq.len() >= 4);
assert!(seq.contains(&0x8B));
assert!(seq.contains(&0xFF));
}
#[test]
fn test_tail_call_sequence() {
let st = X86CoroSymmetricTransfer::new();
let layout = X86CoroFrameLayout::default();
let seq = st.generate_tail_call_sequence(DW_R11 as u16, &layout);
assert!(seq.len() >= 4);
assert_eq!(seq[seq.len() - 2], 0xFF);
assert_eq!(seq[seq.len() - 1], 0xE0);
}
}