luaur_code_gen/functions/
lower_impl.rs1use crate::enums::include_ir_prefix::IncludeIrPrefix;
2use crate::enums::ir_block_kind::IrBlockKind;
3use crate::enums::ir_cmd::IrCmd;
4use crate::enums::ir_op_kind::IrOpKind;
5use crate::functions::any_argument_match::any_argument_match;
6use crate::functions::get_next_block::get_next_block;
7use crate::functions::is_pseudo::is_pseudo;
8use crate::functions::jit_rng_random::jit_rng_random;
9use crate::functions::to_string_detailed_ir_dump::to_string_detailed as to_string_detailed_inst;
10use crate::functions::to_string_detailed_ir_dump_alt_b::to_string_detailed as to_string_detailed_block;
11use crate::functions::to_string_ir_dump_alt_f::to_string_string_bytecode_types_c_char as to_string_bytecode_types;
12use crate::records::assembly_builder_a_64::AssemblyBuilderA64;
13use crate::records::assembly_builder_x_64::AssemblyBuilderX64;
14use crate::records::assembly_options::AssemblyOptions;
15use crate::records::bytecode_types::LBC_TYPE_ANY;
16use crate::records::ir_block::{kBlockNoStartPc, IrBlock};
17use crate::records::ir_function::IrFunction;
18use crate::records::ir_lowering_a_64::IrLoweringA64;
19use crate::records::ir_lowering_x_64::IrLoweringX64;
20use crate::records::ir_op::IrOp;
21use crate::records::ir_to_string_context::IrToStringContext;
22use crate::records::label::Label;
23
24const K_BLOCK_FLAG_SAFE_ENV_CHECK: u8 = 1 << 0;
25
26pub unsafe fn lower_impl_x_64(
27 build: &mut AssemblyBuilderX64,
28 lowering: &mut IrLoweringX64,
29 function: &mut IrFunction,
30 sorted_blocks: &[u32],
31 bytecodeid: i32,
32 options: &AssemblyOptions,
33) -> bool {
34 let mut bc_locations = vec![u32::MAX; function.instructions.len() + 1];
35
36 for i in 0..function.bc_mapping.len() {
37 let ir_location = function.bc_mapping[i].ir_location;
38
39 if ir_location != u32::MAX {
40 bc_locations[ir_location as usize] = i as u32;
41 }
42 }
43
44 let output_enabled = options.include_assembly || options.include_ir;
45
46 let mut text_size = build.text.len();
47 let mut code_size = build.get_code_size();
48 let mut seen_fallback = false;
49
50 let mut dummy = IrBlock::default();
51 dummy.start = u32::MAX;
52
53 debug_assert!(sorted_blocks[0] == 0);
54 debug_assert!(function.entry_block == 0);
55
56 for i in 0..sorted_blocks.len() {
57 let block_index = sorted_blocks[i];
58 let block_ptr = function.blocks.as_mut_ptr().add(block_index as usize);
59
60 if (*block_ptr).kind == IrBlockKind::Dead {
61 continue;
62 }
63
64 debug_assert!((*block_ptr).start != u32::MAX);
65 debug_assert!((*block_ptr).finish != u32::MAX);
66 debug_assert!(
67 !seen_fallback
68 || (*block_ptr).kind == IrBlockKind::Fallback
69 || (*block_ptr).kind == IrBlockKind::ExitSync
70 );
71
72 if ((*block_ptr).kind == IrBlockKind::Fallback
73 || (*block_ptr).kind == IrBlockKind::ExitSync)
74 && !seen_fallback
75 {
76 text_size = build.text.len();
77 code_size = build.get_code_size();
78 seen_fallback = true;
79 }
80
81 if options.include_ir {
82 if options.include_ir_prefix == IncludeIrPrefix::Yes {
83 build.log_append(format_args!("# "));
84 }
85
86 let mut ctx = IrToStringContext {
87 result: &mut build.text,
88 blocks: &function.blocks,
89 constants: &function.constants,
90 cfg: &function.cfg,
91 vm_exit_info: &function.vm_exit_info,
92 proto: function.proto.cast(),
93 };
94 to_string_detailed_block(
95 &mut ctx,
96 &*block_ptr,
97 block_index,
98 options.include_use_info,
99 options.include_cfg_info,
100 options.include_reg_flow_info,
101 );
102 }
103
104 function.valid_restore_op_blocks.push(block_index);
105
106 build.set_label_label(&mut (*block_ptr).label);
107
108 if block_index == function.entry_block {
109 function.entry_location = build.get_label_offset(&(*block_ptr).label);
110 }
111
112 lowering.start_block(&*block_ptr);
113
114 let next_block_ptr = {
115 let next_block = get_next_block(function, sorted_blocks, &mut dummy, i);
116 next_block as *mut IrBlock
117 };
118
119 if (*block_ptr).expected_next_block != u32::MAX {
120 debug_assert!(
121 function.get_block_index(&*next_block_ptr) == (*block_ptr).expected_next_block
122 );
123 }
124
125 if ((*block_ptr).flags & K_BLOCK_FLAG_SAFE_ENV_CHECK) != 0 {
126 if options.include_ir {
127 if options.include_ir_prefix == IncludeIrPrefix::Yes {
128 build.log_append(format_args!("# "));
129 }
130
131 build.log_append(format_args!(
132 " implicit CHECK_SAFE_ENV exit({})\n",
133 (*block_ptr).startpc
134 ));
135 }
136
137 debug_assert!((*block_ptr).startpc != kBlockNoStartPc);
138 lowering.check_safe_env(
139 IrOp {
140 kind_and_index: IrOpKind::VmExit as u32
141 | ((*block_ptr).startpc << IrOp::INDEX_SHIFT),
142 },
143 IrLoweringX64::kInvalidInstIdx,
144 &*next_block_ptr,
145 );
146 }
147
148 for index in (*block_ptr).start..=(*block_ptr).finish {
149 debug_assert!((index as usize) < function.instructions.len());
150
151 let bc_location = bc_locations[index as usize];
152
153 if output_enabled && options.annotator.is_some() && bc_location != u32::MAX {
154 if let Some(annotator) = options.annotator {
155 annotator(
156 options.annotator_context,
157 &mut build.text,
158 bytecodeid,
159 bc_location as i32,
160 );
161 }
162
163 let bc_types = function.get_bytecode_types_at(bc_location as i32);
164
165 if bc_types.result != LBC_TYPE_ANY
166 || bc_types.a != LBC_TYPE_ANY
167 || bc_types.b != LBC_TYPE_ANY
168 || bc_types.c != LBC_TYPE_ANY
169 {
170 to_string_bytecode_types(
171 &mut build.text,
172 &bc_types,
173 options.compilation_options.userdata_types,
174 );
175
176 build.log_append(format_args!("\n"));
177 }
178 }
179
180 if bc_location != u32::MAX {
181 let label = if index == (*block_ptr).start {
182 (*block_ptr).label
183 } else {
184 let mut label = Label::default();
185 build.set_label(&mut label);
186 label
187 };
188
189 function.bc_mapping[bc_location as usize].asm_location =
190 build.get_label_offset(&label);
191 }
192
193 let inst_ptr = function.instructions.as_mut_ptr().add(index as usize);
194
195 if is_pseudo((*inst_ptr).cmd) {
196 if let Some(hint) = function.find_store_location_hint(index) {
197 lowering.regs.curr_inst_idx = index;
198 lowering.value_tracker.process_store_location_hint(hint);
199 lowering.regs.curr_inst_idx = IrLoweringX64::kInvalidInstIdx;
200 }
201
202 debug_assert!((*inst_ptr).use_count == 0);
203 continue;
204 }
205
206 debug_assert!((*inst_ptr).last_use == 0 || (*inst_ptr).use_count != 0);
207
208 if options.include_ir {
209 if options.include_ir_prefix == IncludeIrPrefix::Yes {
210 build.log_append(format_args!("# "));
211 }
212
213 let mut ctx = IrToStringContext {
214 result: &mut build.text,
215 blocks: &function.blocks,
216 constants: &function.constants,
217 cfg: &function.cfg,
218 vm_exit_info: &function.vm_exit_info,
219 proto: function.proto.cast(),
220 };
221 to_string_detailed_inst(
222 &mut ctx,
223 &*block_ptr,
224 block_index,
225 &mut *inst_ptr,
226 index,
227 options.include_use_info,
228 );
229 }
230
231 lowering.lower_inst(&mut *inst_ptr, index, &*next_block_ptr);
232
233 if lowering.has_error() {
234 for j in (i + 1)..sorted_blocks.len() {
235 let abandoned_ptr = function.blocks.as_mut_ptr().add(sorted_blocks[j] as usize);
236
237 build.set_label_label(&mut (*abandoned_ptr).label);
238 }
239
240 lowering.ir_lowering_x_64_finish_function();
241
242 return false;
243 }
244 }
245
246 lowering.finish_block(&*block_ptr, &*next_block_ptr);
247
248 if function.jit_rng_state != 0 {
249 let term_inst_ptr = function
250 .instructions
251 .as_ptr()
252 .add((*block_ptr).finish as usize);
253 let next_start = (*next_block_ptr).start;
254
255 let block_falls_through = any_argument_match(&*term_inst_ptr, |op| {
256 op.kind() == IrOpKind::Block
257 && function.blocks[op.index() as usize].start == next_start
258 });
259
260 if !(block_falls_through
261 && (*term_inst_ptr).cmd == IrCmd::JUMP
262 && (*next_block_ptr).use_count == 1)
263 {
264 let max_nop_bytes = if block_falls_through { 4 } else { 8 };
265 let nop_bytes = jit_rng_random(&mut function.jit_rng_state) % max_nop_bytes;
266
267 if nop_bytes > 0 {
268 build.nop(nop_bytes);
269 }
270 }
271 }
272
273 if options.include_ir && options.include_ir_prefix == IncludeIrPrefix::Yes {
274 build.log_append(format_args!("#\n"));
275 }
276
277 if (*block_ptr).expected_next_block == u32::MAX {
278 function.valid_restore_op_blocks.clear();
279 }
280 }
281
282 if !seen_fallback {
283 text_size = build.text.len();
284 code_size = build.get_code_size();
285 }
286
287 lowering.ir_lowering_x_64_finish_function();
288
289 if output_enabled && !options.include_outlined_code && text_size < build.text.len() {
290 build.text.truncate(text_size);
291
292 if options.include_assembly {
293 build.log_append(format_args!(
294 "; skipping {} bytes of outlined code\n",
295 build
296 .get_code_size()
297 .wrapping_sub(code_size)
298 .wrapping_mul(core::mem::size_of::<u8>() as u32)
299 ));
300 }
301 }
302
303 true
304}
305
306pub unsafe fn lower_impl_a_64(
307 build: &mut AssemblyBuilderA64,
308 lowering: &mut IrLoweringA64,
309 function: &mut IrFunction,
310 sorted_blocks: &[u32],
311 bytecodeid: i32,
312 options: &AssemblyOptions,
313) -> bool {
314 let mut bc_locations = vec![u32::MAX; function.instructions.len() + 1];
315
316 for i in 0..function.bc_mapping.len() {
317 let ir_location = function.bc_mapping[i].ir_location;
318
319 if ir_location != u32::MAX {
320 bc_locations[ir_location as usize] = i as u32;
321 }
322 }
323
324 let output_enabled = options.include_assembly || options.include_ir;
325
326 let mut text_size = build.text.len();
327 let mut code_size = build.get_code_size();
328 let mut seen_fallback = false;
329
330 let mut dummy = IrBlock::default();
331 dummy.start = u32::MAX;
332
333 debug_assert!(sorted_blocks[0] == 0);
334 debug_assert!(function.entry_block == 0);
335
336 for i in 0..sorted_blocks.len() {
337 let block_index = sorted_blocks[i];
338 let block_ptr = function.blocks.as_mut_ptr().add(block_index as usize);
339
340 if (*block_ptr).kind == IrBlockKind::Dead {
341 continue;
342 }
343
344 debug_assert!((*block_ptr).start != u32::MAX);
345 debug_assert!((*block_ptr).finish != u32::MAX);
346 debug_assert!(
347 !seen_fallback
348 || (*block_ptr).kind == IrBlockKind::Fallback
349 || (*block_ptr).kind == IrBlockKind::ExitSync
350 );
351
352 if ((*block_ptr).kind == IrBlockKind::Fallback
353 || (*block_ptr).kind == IrBlockKind::ExitSync)
354 && !seen_fallback
355 {
356 text_size = build.text.len();
357 code_size = build.get_code_size();
358 seen_fallback = true;
359 }
360
361 if options.include_ir {
362 if options.include_ir_prefix == IncludeIrPrefix::Yes {
363 build.log_append(format_args!("# "));
364 }
365
366 let mut ctx = IrToStringContext {
367 result: &mut build.text,
368 blocks: &function.blocks,
369 constants: &function.constants,
370 cfg: &function.cfg,
371 vm_exit_info: &function.vm_exit_info,
372 proto: function.proto.cast(),
373 };
374 to_string_detailed_block(
375 &mut ctx,
376 &*block_ptr,
377 block_index,
378 options.include_use_info,
379 options.include_cfg_info,
380 options.include_reg_flow_info,
381 );
382 }
383
384 function.valid_restore_op_blocks.push(block_index);
385
386 build.set_label_label(&mut (*block_ptr).label);
387
388 if block_index == function.entry_block {
389 function.entry_location = build.get_label_offset(&(*block_ptr).label);
390 }
391
392 lowering.ir_lowering_a_64_start_block(&*block_ptr);
393
394 let next_block_ptr = {
395 let next_block = get_next_block(function, sorted_blocks, &mut dummy, i);
396 next_block as *mut IrBlock
397 };
398
399 if (*block_ptr).expected_next_block != u32::MAX {
400 debug_assert!(
401 function.get_block_index(&*next_block_ptr) == (*block_ptr).expected_next_block
402 );
403 }
404
405 if ((*block_ptr).flags & K_BLOCK_FLAG_SAFE_ENV_CHECK) != 0 {
406 if options.include_ir {
407 if options.include_ir_prefix == IncludeIrPrefix::Yes {
408 build.log_append(format_args!("# "));
409 }
410
411 build.log_append(format_args!(
412 " implicit CHECK_SAFE_ENV exit({})\n",
413 (*block_ptr).startpc
414 ));
415 }
416
417 debug_assert!((*block_ptr).startpc != kBlockNoStartPc);
418 lowering.ir_lowering_a_64_check_safe_env(
419 IrOp {
420 kind_and_index: IrOpKind::VmExit as u32
421 | ((*block_ptr).startpc << IrOp::INDEX_SHIFT),
422 },
423 IrLoweringA64::kInvalidInstIdx,
424 &*next_block_ptr,
425 );
426 }
427
428 for index in (*block_ptr).start..=(*block_ptr).finish {
429 debug_assert!((index as usize) < function.instructions.len());
430
431 let bc_location = bc_locations[index as usize];
432
433 if output_enabled && options.annotator.is_some() && bc_location != u32::MAX {
434 if let Some(annotator) = options.annotator {
435 annotator(
436 options.annotator_context,
437 &mut build.text,
438 bytecodeid,
439 bc_location as i32,
440 );
441 }
442
443 let bc_types = function.get_bytecode_types_at(bc_location as i32);
444
445 if bc_types.result != LBC_TYPE_ANY
446 || bc_types.a != LBC_TYPE_ANY
447 || bc_types.b != LBC_TYPE_ANY
448 || bc_types.c != LBC_TYPE_ANY
449 {
450 to_string_bytecode_types(
451 &mut build.text,
452 &bc_types,
453 options.compilation_options.userdata_types,
454 );
455
456 build.log_append(format_args!("\n"));
457 }
458 }
459
460 if bc_location != u32::MAX {
461 let label = if index == (*block_ptr).start {
462 (*block_ptr).label
463 } else {
464 build.set_label()
465 };
466
467 function.bc_mapping[bc_location as usize].asm_location =
468 build.get_label_offset(&label);
469 }
470
471 let inst_ptr = function.instructions.as_mut_ptr().add(index as usize);
472
473 if is_pseudo((*inst_ptr).cmd) {
474 if let Some(hint) = function.find_store_location_hint(index) {
475 lowering.regs.curr_inst_idx = index;
476 lowering.value_tracker.process_store_location_hint(hint);
477 lowering.regs.curr_inst_idx = IrLoweringA64::kInvalidInstIdx;
478 }
479
480 debug_assert!((*inst_ptr).use_count == 0);
481 continue;
482 }
483
484 debug_assert!((*inst_ptr).last_use == 0 || (*inst_ptr).use_count != 0);
485
486 if options.include_ir {
487 if options.include_ir_prefix == IncludeIrPrefix::Yes {
488 build.log_append(format_args!("# "));
489 }
490
491 let mut ctx = IrToStringContext {
492 result: &mut build.text,
493 blocks: &function.blocks,
494 constants: &function.constants,
495 cfg: &function.cfg,
496 vm_exit_info: &function.vm_exit_info,
497 proto: function.proto.cast(),
498 };
499 to_string_detailed_inst(
500 &mut ctx,
501 &*block_ptr,
502 block_index,
503 &mut *inst_ptr,
504 index,
505 options.include_use_info,
506 );
507 }
508
509 lowering.ir_lowering_a_64_lower_inst(&mut *inst_ptr, index, &*next_block_ptr);
510
511 if lowering.ir_lowering_a_64_has_error() {
512 for j in (i + 1)..sorted_blocks.len() {
513 let abandoned_ptr = function.blocks.as_mut_ptr().add(sorted_blocks[j] as usize);
514
515 build.set_label_label(&mut (*abandoned_ptr).label);
516 }
517
518 lowering.ir_lowering_a_64_finish_function();
519
520 return false;
521 }
522 }
523
524 lowering.ir_lowering_a_64_finish_block(&*block_ptr, &*next_block_ptr);
525
526 if function.jit_rng_state != 0 {
527 let term_inst_ptr = function
528 .instructions
529 .as_ptr()
530 .add((*block_ptr).finish as usize);
531 let next_start = (*next_block_ptr).start;
532
533 let block_falls_through = any_argument_match(&*term_inst_ptr, |op| {
534 op.kind() == IrOpKind::Block
535 && function.blocks[op.index() as usize].start == next_start
536 });
537
538 if !(block_falls_through
539 && (*term_inst_ptr).cmd == IrCmd::JUMP
540 && (*next_block_ptr).use_count == 1)
541 {
542 let max_nop_bytes = if block_falls_through { 4 } else { 8 };
543 let nop_bytes = jit_rng_random(&mut function.jit_rng_state) % max_nop_bytes;
544
545 if nop_bytes > 0 {
546 build.nop(nop_bytes);
547 }
548 }
549 }
550
551 if options.include_ir && options.include_ir_prefix == IncludeIrPrefix::Yes {
552 build.log_append(format_args!("#\n"));
553 }
554
555 if (*block_ptr).expected_next_block == u32::MAX {
556 function.valid_restore_op_blocks.clear();
557 }
558 }
559
560 if !seen_fallback {
561 text_size = build.text.len();
562 code_size = build.get_code_size();
563 }
564
565 lowering.ir_lowering_a_64_finish_function();
566
567 if output_enabled && !options.include_outlined_code && text_size < build.text.len() {
568 build.text.truncate(text_size);
569
570 if options.include_assembly {
571 build.log_append(format_args!(
572 "; skipping {} bytes of outlined code\n",
573 build
574 .get_code_size()
575 .wrapping_sub(code_size)
576 .wrapping_mul(core::mem::size_of::<u32>() as u32)
577 ));
578 }
579 }
580
581 true
582}