1use super::context::{CodeGenError, EbpfContext, Result};
7use aya_ebpf_bindings::bindings::bpf_func_id::{
8 BPF_FUNC_get_current_pid_tgid, BPF_FUNC_ktime_get_ns, BPF_FUNC_map_lookup_elem,
9 BPF_FUNC_perf_event_output, BPF_FUNC_probe_read_user, BPF_FUNC_probe_read_user_str,
10 BPF_FUNC_ringbuf_output,
11};
12use ghostscope_dwarf::MemoryAccessSize;
13use ghostscope_platform::register_mapping;
14use ghostscope_protocol::trace_event::VariableStatus;
15use inkwell::types::{BasicType, BasicTypeEnum};
16use inkwell::values::{BasicMetadataValueEnum, BasicValueEnum, IntValue, PointerValue};
17use inkwell::AddressSpace;
18
19struct ProbeReadResult<'ctx> {
20 loaded_i64: IntValue<'ctx>,
21 combined_fail: IntValue<'ctx>,
22 not_found: IntValue<'ctx>,
23}
24
25impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> {
26 fn get_probe_read_scratch_buffer(
27 &mut self,
28 result_size: usize,
29 name_prefix: &str,
30 ) -> Result<PointerValue<'ctx>> {
31 if result_size <= 4 {
32 if let Some(key_alloca) = self.pm_key_alloca {
33 let i32_type = self.context.i32_type();
42 let key_arr_ty = i32_type.array_type(4);
43 let zero = i32_type.const_zero();
44 return unsafe {
45 self.builder
46 .build_gep(
47 key_arr_ty,
48 key_alloca,
49 &[zero, zero],
50 &format!("{name_prefix}_scratch_i8"),
51 )
52 .map_err(|e| CodeGenError::LLVMError(e.to_string()))
53 };
54 }
55 }
56
57 let buffer_name = format!("_temp_read_buffer_{result_size}");
58 let global_buffer = match self.module.get_global(&buffer_name) {
59 Some(existing) => existing.as_pointer_value(),
60 None => {
61 let array_type = self.context.i8_type().array_type(result_size as u32);
62 let global =
63 self.module
64 .add_global(array_type, Some(AddressSpace::default()), &buffer_name);
65 global.set_initializer(&array_type.const_zero());
66 global.as_pointer_value()
67 }
68 };
69 Ok(global_buffer)
70 }
71
72 pub fn lookup_proc_pid_alias(
73 &mut self,
74 runtime_pid: IntValue<'ctx>,
75 name_prefix: &str,
76 ) -> Result<IntValue<'ctx>> {
77 let Some(map_global) = self.module.get_global("pid_aliases") else {
78 return Ok(runtime_pid);
79 };
80
81 let i32_type = self.context.i32_type();
82 let ptr_type = self.context.ptr_type(AddressSpace::default());
83 let map_ptr = map_global.as_pointer_value();
84 let map_ptr_cast = self
85 .builder
86 .build_bit_cast(map_ptr, ptr_type, &format!("{name_prefix}_map_ptr"))
87 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
88
89 let key_alloca = self.pm_key_alloca.ok_or_else(|| {
90 CodeGenError::LLVMError("pm_key not allocated in entry block".to_string())
91 })?;
92 let key_arr_ty = i32_type.array_type(4);
93 let zero = i32_type.const_zero();
94 let key_ptr = unsafe {
95 self.builder
96 .build_gep(
97 key_arr_ty,
98 key_alloca,
99 &[zero, zero],
100 &format!("{name_prefix}_alias_key_ptr"),
101 )
102 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
103 };
104 self.builder
105 .build_store(key_ptr, runtime_pid)
106 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
107 let key_arg = self
108 .builder
109 .build_bit_cast(key_ptr, ptr_type, &format!("{name_prefix}_alias_key_arg"))
110 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
111
112 let lookup_id = self
113 .context
114 .i64_type()
115 .const_int(BPF_FUNC_map_lookup_elem as u64, false);
116 let lookup_fn_type = ptr_type.fn_type(&[ptr_type.into(), ptr_type.into()], false);
117 let lookup_fn_ptr = self
118 .builder
119 .build_int_to_ptr(lookup_id, ptr_type, &format!("{name_prefix}_lookup_fn"))
120 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
121 let lookup_args: Vec<BasicMetadataValueEnum> = vec![map_ptr_cast.into(), key_arg.into()];
122 let value_ptr_any = self
123 .builder
124 .build_indirect_call(
125 lookup_fn_type,
126 lookup_fn_ptr,
127 &lookup_args,
128 &format!("{name_prefix}_alias_lookup"),
129 )
130 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
131 .try_as_basic_value()
132 .left()
133 .ok_or_else(|| {
134 CodeGenError::LLVMError("pid_aliases lookup returned void".to_string())
135 })?;
136
137 let value_ptr = match value_ptr_any {
138 BasicValueEnum::PointerValue(p) => p,
139 _ => {
140 return Err(CodeGenError::LLVMError(
141 "pid_aliases lookup did not return pointer".to_string(),
142 ));
143 }
144 };
145
146 let helper_fn = self
147 .builder
148 .get_insert_block()
149 .unwrap()
150 .get_parent()
151 .unwrap();
152 let alias_hit_block = self
153 .context
154 .append_basic_block(helper_fn, &format!("{name_prefix}_alias_hit"));
155 let alias_miss_block = self
156 .context
157 .append_basic_block(helper_fn, &format!("{name_prefix}_alias_miss"));
158 let alias_cont_block = self
159 .context
160 .append_basic_block(helper_fn, &format!("{name_prefix}_alias_cont"));
161
162 let i64_type = self.context.i64_type();
163 let value_ptr_int = self
164 .builder
165 .build_ptr_to_int(
166 value_ptr,
167 i64_type,
168 &format!("{name_prefix}_alias_value_ptr_int"),
169 )
170 .map_err(|e| CodeGenError::Builder(e.to_string()))?;
171 let is_hit = self
172 .builder
173 .build_int_compare(
174 inkwell::IntPredicate::NE,
175 value_ptr_int,
176 i64_type.const_zero(),
177 &format!("{name_prefix}_alias_found"),
178 )
179 .map_err(|e| CodeGenError::Builder(e.to_string()))?;
180 self.builder
181 .build_conditional_branch(is_hit, alias_hit_block, alias_miss_block)
182 .map_err(|e| CodeGenError::Builder(e.to_string()))?;
183
184 self.builder.position_at_end(alias_hit_block);
185 let alias_value = self
186 .builder
187 .build_load(i32_type, value_ptr, &format!("{name_prefix}_alias_value"))
188 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
189 .into_int_value();
190 self.builder
191 .build_unconditional_branch(alias_cont_block)
192 .map_err(|e| CodeGenError::Builder(e.to_string()))?;
193 let alias_hit_end = self.builder.get_insert_block().unwrap();
194
195 self.builder.position_at_end(alias_miss_block);
196 self.builder
197 .build_unconditional_branch(alias_cont_block)
198 .map_err(|e| CodeGenError::Builder(e.to_string()))?;
199 let alias_miss_end = self.builder.get_insert_block().unwrap();
200
201 self.builder.position_at_end(alias_cont_block);
202 let alias_phi = self
203 .builder
204 .build_phi(i32_type, &format!("{name_prefix}_alias_pid"))
205 .map_err(|e| CodeGenError::Builder(e.to_string()))?;
206 alias_phi.add_incoming(&[
207 (&alias_value, alias_hit_end),
208 (&runtime_pid, alias_miss_end),
209 ]);
210
211 Ok(alias_phi.as_basic_value().into_int_value())
212 }
213
214 pub fn get_or_create_i8_buffer(
216 &mut self,
217 size: u32,
218 name_prefix: &str,
219 ) -> (
220 inkwell::types::ArrayType<'ctx>,
221 inkwell::values::PointerValue<'ctx>,
222 ) {
223 let array_ty = self.context.i8_type().array_type(size);
224 let name = format!("{name_prefix}_{size}");
225 let global_ptr = match self.module.get_global(&name) {
226 Some(g) => g.as_pointer_value(),
227 None => {
228 let g = self
229 .module
230 .add_global(array_ty, Some(AddressSpace::default()), &name);
231 g.set_initializer(&array_ty.const_zero());
232 g.as_pointer_value()
233 }
234 };
235 (array_ty, global_ptr)
236 }
237
238 pub fn read_user_cstr_into_buffer(
241 &mut self,
242 src_addr: inkwell::values::IntValue<'ctx>,
243 size: u32,
244 name_prefix: &str,
245 ) -> Result<(
246 inkwell::values::PointerValue<'ctx>,
247 inkwell::values::IntValue<'ctx>,
248 inkwell::types::ArrayType<'ctx>,
249 )> {
250 let (arr_ty, buf_global) = self.get_or_create_i8_buffer(size, name_prefix);
251
252 let ptr_ty = self.context.ptr_type(AddressSpace::default());
253 let dst_ptr = self
255 .builder
256 .build_bit_cast(buf_global, ptr_ty, "dst_ptr")
257 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
258 let src_ptr = self
259 .builder
260 .build_int_to_ptr(src_addr, ptr_ty, "src_ptr")
261 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
262
263 let i64_ty = self.context.i64_type();
265 let i32_ty = self.context.i32_type();
266 let args: [inkwell::values::BasicValueEnum; 3] = [
267 dst_ptr,
268 i32_ty.const_int(size as u64, false).into(),
269 inkwell::values::BasicValueEnum::PointerValue(src_ptr),
270 ];
271 let ret = self.create_bpf_helper_call(
272 BPF_FUNC_probe_read_user_str as u64,
273 &args,
274 i64_ty.into(),
275 "probe_read_user_str",
276 )?;
277 let len = if let inkwell::values::BasicValueEnum::IntValue(iv) = ret {
278 iv
279 } else {
280 return Err(CodeGenError::LLVMError(
281 "probe_read_user_str did not return integer".to_string(),
282 ));
283 };
284 Ok((buf_global, len, arr_ty))
285 }
286
287 pub fn read_user_bytes_into_buffer(
290 &mut self,
291 src_addr: inkwell::values::IntValue<'ctx>,
292 size: u32,
293 name_prefix: &str,
294 ) -> Result<(
295 inkwell::values::PointerValue<'ctx>,
296 inkwell::values::IntValue<'ctx>,
297 inkwell::types::ArrayType<'ctx>,
298 )> {
299 let (arr_ty, buf_global) = self.get_or_create_i8_buffer(size, name_prefix);
300 let ptr_ty = self.context.ptr_type(AddressSpace::default());
301 let i32_ty = self.context.i32_type();
302 let i64_ty = self.context.i64_type();
303 let dst_ptr = self
305 .builder
306 .build_bit_cast(buf_global, ptr_ty, "dst_ptr")
307 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
308 let src_ptr = self
309 .builder
310 .build_int_to_ptr(src_addr, ptr_ty, "src_ptr")
311 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
312
313 let args: [inkwell::values::BasicValueEnum; 3] = [
314 dst_ptr,
315 i32_ty.const_int(size as u64, false).into(),
316 inkwell::values::BasicValueEnum::PointerValue(src_ptr),
317 ];
318 let ret = self.create_bpf_helper_call(
320 BPF_FUNC_probe_read_user as u64,
321 &args,
322 i64_ty.into(),
323 "probe_read_user",
324 )?;
325 let status = if let inkwell::values::BasicValueEnum::IntValue(iv) = ret {
326 iv
327 } else {
328 return Err(CodeGenError::LLVMError(
329 "probe_read_user did not return integer".to_string(),
330 ));
331 };
332 Ok((buf_global, status, arr_ty))
333 }
334 pub fn generate_runtime_address_from_offsets(
337 &mut self,
338 link_addr: IntValue<'ctx>,
339 section_type: u8,
340 module_cookie: u64,
341 ) -> Result<(IntValue<'ctx>, IntValue<'ctx>)> {
342 const BPF_FUNC_GET_NS_CURRENT_PID_TGID: u64 = 120;
343 const BPF_PIDNS_INFO_SIZE: u64 = 8; let i64_type = self.context.i64_type();
346 let ptr_type = self.context.ptr_type(AddressSpace::default());
347
348 let map_global = self
350 .module
351 .get_global("proc_module_offsets")
352 .ok_or_else(|| {
353 CodeGenError::LLVMError("proc_module_offsets map not found".to_string())
354 })?;
355 let map_ptr = map_global.as_pointer_value();
356 let map_ptr_cast = self
357 .builder
358 .build_bit_cast(map_ptr, ptr_type, "map_ptr")
359 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
360
361 let i32_type = self.context.i32_type();
364 let key_arr_ty = i32_type.array_type(4);
365 let key_alloca = self.pm_key_alloca.ok_or_else(|| {
366 CodeGenError::LLVMError("pm_key not allocated in entry block".to_string())
367 })?;
368 let zero = i32_type.const_zero();
370 let base_i32_ptr = unsafe {
371 self.builder
372 .build_gep(key_arr_ty, key_alloca, &[zero, zero], "pm_key_i32_ptr")
373 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
374 };
375
376 let helper_id = i64_type.const_int(BPF_FUNC_get_current_pid_tgid as u64, false);
380 let helper_fn_type = i64_type.fn_type(&[], false);
381 let helper_fn_ptr = self
382 .builder
383 .build_int_to_ptr(helper_id, ptr_type, "get_pid_fn")
384 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
385 let pid_tgid = self
386 .builder
387 .build_indirect_call(helper_fn_type, helper_fn_ptr, &[], "pid_tgid")
388 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
389 .try_as_basic_value()
390 .left()
391 .ok_or_else(|| {
392 CodeGenError::LLVMError("get_current_pid_tgid returned void".to_string())
393 })?;
394 let host_tgid = if let BasicValueEnum::IntValue(v) = pid_tgid {
395 let shifted = self
397 .builder
398 .build_right_shift(v, i64_type.const_int(32, false), false, "pid_shift")
399 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
400 self.builder
401 .build_int_truncate(shifted, i32_type, "pid32")
402 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
403 } else {
404 return Err(CodeGenError::LLVMError(
405 "pid_tgid is not IntValue".to_string(),
406 ));
407 };
408
409 let ns_spec = self
410 .compile_options
411 .proc_offsets_pid_ns
412 .and_then(|pid_ns| pid_ns.helper_dev_inode());
413
414 let runtime_pid = if let Some((pid_ns_dev, pid_ns_inode)) = ns_spec {
415 self.builder
417 .build_store(key_alloca, key_arr_ty.const_zero())
418 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
419 let pidns_info_ptr = self
420 .builder
421 .build_bit_cast(key_alloca, ptr_type, "offset_pidns_info_ptr")
422 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
423 let helper_args = [
424 i64_type.const_int(pid_ns_dev, false).into(),
425 i64_type.const_int(pid_ns_inode, false).into(),
426 pidns_info_ptr,
427 i64_type.const_int(BPF_PIDNS_INFO_SIZE, false).into(),
428 ];
429 let helper_ret = self.create_bpf_helper_call(
430 BPF_FUNC_GET_NS_CURRENT_PID_TGID,
431 &helper_args,
432 i64_type.into(),
433 "offset_ns_pid_tgid_ret",
434 )?;
435 let helper_ret = match helper_ret {
436 BasicValueEnum::IntValue(v) => v,
437 _ => {
438 return Err(CodeGenError::LLVMError(
439 "bpf_get_ns_current_pid_tgid did not return integer".to_string(),
440 ));
441 }
442 };
443 let helper_ok = self
444 .builder
445 .build_int_compare(
446 inkwell::IntPredicate::EQ,
447 helper_ret,
448 i64_type.const_zero(),
449 "offset_ns_helper_ok",
450 )
451 .map_err(|e| CodeGenError::Builder(e.to_string()))?;
452 let ns_tgid_ptr = unsafe {
453 self.builder.build_gep(
454 key_arr_ty,
455 key_alloca,
456 &[i32_type.const_zero(), i32_type.const_int(1, false)],
457 "offset_ns_tgid_ptr",
458 )
459 }
460 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
461 let ns_tgid = self
462 .builder
463 .build_load(i32_type, ns_tgid_ptr, "offset_ns_tgid")
464 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
465 .into_int_value();
466 self.builder
471 .build_select(helper_ok, ns_tgid, host_tgid, "offset_pid_key")
472 .map_err(|e| CodeGenError::Builder(e.to_string()))?
473 .into_int_value()
474 } else {
475 host_tgid
476 };
477 let pid = self.lookup_proc_pid_alias(runtime_pid, "offset_pid")?;
478
479 self.builder
481 .build_store(base_i32_ptr, pid)
482 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
483
484 let idx1 = i32_type.const_int(1, false);
486 let pad_ptr = unsafe {
487 self.builder
488 .build_gep(self.context.i32_type(), base_i32_ptr, &[idx1], "pad_ptr")
489 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
490 };
491 self.builder
492 .build_store(pad_ptr, self.context.i32_type().const_zero())
493 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
494
495 let cookie_lo = i32_type.const_int(module_cookie & 0xffff_ffff, false);
497 let cookie_hi = i32_type.const_int(module_cookie >> 32, false);
498 let idx2 = i32_type.const_int(2, false);
499 let idx3 = i32_type.const_int(3, false);
500 let cookie_lo_ptr = unsafe {
502 self.builder
503 .build_gep(
504 self.context.i32_type(),
505 base_i32_ptr,
506 &[idx2],
507 "cookie_lo_ptr",
508 )
509 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
510 };
511 let cookie_hi_ptr = unsafe {
512 self.builder
513 .build_gep(
514 self.context.i32_type(),
515 base_i32_ptr,
516 &[idx3],
517 "cookie_hi_ptr",
518 )
519 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
520 };
521 self.builder
522 .build_store(cookie_lo_ptr, cookie_lo)
523 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
524 self.builder
525 .build_store(cookie_hi_ptr, cookie_hi)
526 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
527
528 let lookup_id = i64_type.const_int(BPF_FUNC_map_lookup_elem as u64, false);
530 let lookup_fn_type = ptr_type.fn_type(&[ptr_type.into(), ptr_type.into()], false);
531 let lookup_fn_ptr = self
532 .builder
533 .build_int_to_ptr(lookup_id, ptr_type, "lookup_fn")
534 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
535 let key_arg = self
537 .builder
538 .build_bit_cast(key_alloca, ptr_type, "key_arg")
539 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
540 let args: Vec<BasicMetadataValueEnum> = vec![map_ptr_cast.into(), key_arg.into()];
541 let val_ptr_any = self
542 .builder
543 .build_indirect_call(lookup_fn_type, lookup_fn_ptr, &args, "val_ptr_any")
544 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
545 .try_as_basic_value()
546 .left()
547 .ok_or_else(|| CodeGenError::LLVMError("map_lookup_elem returned void".to_string()))?;
548
549 let null_ptr = ptr_type.const_null();
550 let found_block = self.context.append_basic_block(
551 self.builder
552 .get_insert_block()
553 .unwrap()
554 .get_parent()
555 .unwrap(),
556 "found_offsets",
557 );
558 let miss_block = self.context.append_basic_block(
559 self.builder
560 .get_insert_block()
561 .unwrap()
562 .get_parent()
563 .unwrap(),
564 "miss_offsets",
565 );
566 let cont_block = self.context.append_basic_block(
567 self.builder
568 .get_insert_block()
569 .unwrap()
570 .get_parent()
571 .unwrap(),
572 "cont_offsets",
573 );
574
575 let val_ptr = if let BasicValueEnum::PointerValue(p) = val_ptr_any {
577 p
578 } else {
579 null_ptr
580 };
581 let is_null = self
582 .builder
583 .build_int_compare(
584 inkwell::IntPredicate::EQ,
585 self.builder
586 .build_ptr_to_int(val_ptr, i64_type, "val_ptr_i64")
587 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?,
588 i64_type.const_zero(),
589 "is_null_offsets",
590 )
591 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
592 self.builder
593 .build_conditional_branch(is_null, miss_block, found_block)
594 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
595
596 self.builder.position_at_end(found_block);
598 let i64_ptr_ty = self.context.ptr_type(AddressSpace::default());
601 let val_u64_ptr = self
602 .builder
603 .build_pointer_cast(val_ptr, i64_ptr_ty, "val_u64_ptr")
604 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
605 let load_field = |idx: u64,
606 ctx: &mut EbpfContext<'ctx, 'dw>,
607 base: PointerValue<'ctx>|
608 -> Result<IntValue<'ctx>> {
609 let idx_i32 = ctx.context.i32_type().const_int(idx, false);
611 let field_ptr = unsafe {
612 ctx.builder
613 .build_gep(ctx.context.i64_type(), base, &[idx_i32], "field_ptr_i64")
614 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
615 };
616 let loaded = ctx
617 .builder
618 .build_load(ctx.context.i64_type(), field_ptr, "loaded_offset")
619 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
620 if let BasicValueEnum::IntValue(iv) = loaded {
621 Ok(iv)
622 } else {
623 Err(CodeGenError::LLVMError("offset load failed".to_string()))
624 }
625 };
626 let st = section_type;
627 let off_text = load_field(0, self, val_u64_ptr)?;
628 let off_rodata = load_field(1, self, val_u64_ptr)?;
629 let off_data = load_field(2, self, val_u64_ptr)?;
630 let off_bss = load_field(3, self, val_u64_ptr)?;
631 let st_val = i32_type.const_int(st as u64, false);
636 let eq_text = self
637 .builder
638 .build_int_compare(
639 inkwell::IntPredicate::EQ,
640 st_val,
641 i32_type.const_int(0, false),
642 "is_text",
643 )
644 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
645 let eq_ro = self
646 .builder
647 .build_int_compare(
648 inkwell::IntPredicate::EQ,
649 st_val,
650 i32_type.const_int(1, false),
651 "is_ro",
652 )
653 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
654 let eq_da = self
655 .builder
656 .build_int_compare(
657 inkwell::IntPredicate::EQ,
658 st_val,
659 i32_type.const_int(2, false),
660 "is_da",
661 )
662 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
663
664 let tmp_any = self
665 .builder
666 .build_select(eq_da, off_data, off_bss, "sel_data_bss")
667 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
668 let tmp = tmp_any.into_int_value();
669
670 let tmp2_any = self
671 .builder
672 .build_select(eq_ro, off_rodata, tmp, "sel_rodata_else")
673 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
674 let tmp2 = tmp2_any.into_int_value();
675
676 let off_final_any = self
677 .builder
678 .build_select(eq_text, off_text, tmp2, "sel_text_else")
679 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
680 let off_final = off_final_any.into_int_value();
681 let rt_addr = self
682 .builder
683 .build_int_add(link_addr, off_final, "runtime_addr")
684 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
685 self.builder
686 .build_unconditional_branch(cont_block)
687 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
688
689 self.builder.position_at_end(miss_block);
691 self.builder
692 .build_unconditional_branch(cont_block)
693 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
694
695 self.builder.position_at_end(cont_block);
697 let phi = self
698 .builder
699 .build_phi(i64_type, "addr_phi")
700 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
701 phi.add_incoming(&[(&rt_addr, found_block), (&link_addr, miss_block)]);
702 let final_addr = phi.as_basic_value().into_int_value();
703
704 let i1_type = self.context.bool_type();
706 let one = i1_type.const_int(1, false);
707 let zero = i1_type.const_int(0, false);
708 let flag_phi = self
709 .builder
710 .build_phi(i1_type, "off_found_phi")
711 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
712 flag_phi.add_incoming(&[(&one, found_block), (&zero, miss_block)]);
713 let found_flag = flag_phi.as_basic_value().into_int_value();
714
715 self.store_offsets_found_flag(found_flag)?;
716 Ok((final_addr, found_flag))
717 }
718 pub fn load_register_value(
720 &mut self,
721 reg_num: u16,
722 pt_regs_ptr: PointerValue<'ctx>,
723 ) -> Result<BasicValueEnum<'ctx>> {
724 if let Some(cached_value) = self.register_cache.get(®_num) {
726 return Ok((*cached_value).into());
727 }
728
729 let pt_regs_offset = self.dwarf_reg_to_pt_regs_offset(reg_num)?;
731
732 let i64_type = self.context.i64_type();
734 let offset_value = i64_type.const_int(pt_regs_offset as u64, false);
735
736 let reg_ptr = unsafe {
737 self.builder
738 .build_gep(
739 i64_type,
740 pt_regs_ptr,
741 &[offset_value],
742 &format!("reg_{reg_num}_ptr"),
743 )
744 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
745 };
746
747 let reg_value = self
749 .builder
750 .build_load(i64_type, reg_ptr, &format!("reg_{reg_num}"))
751 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
752
753 if let BasicValueEnum::IntValue(int_val) = reg_value {
754 self.register_cache.insert(reg_num, int_val);
756 Ok(reg_value)
757 } else {
758 Err(CodeGenError::RegisterMappingError(format!(
759 "Failed to load register {reg_num} as integer"
760 )))
761 }
762 }
763
764 fn probe_read_user_core(
765 &mut self,
766 addr: IntValue<'ctx>,
767 size: MemoryAccessSize,
768 name_suffix: &str,
769 ) -> Result<ProbeReadResult<'ctx>> {
770 let i64_type = self.context.i64_type();
771 let ptr_type = self.context.ptr_type(AddressSpace::default());
772 let offsets_found = self.load_offsets_found_flag()?;
773 let not_found = self
774 .builder
775 .build_not(offsets_found, "offsets_miss")
776 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
777
778 let result_size = size.bytes();
779 let scratch_buffer = self.get_probe_read_scratch_buffer(result_size, name_suffix)?;
780 let dst_ptr = self
781 .builder
782 .build_bit_cast(scratch_buffer, ptr_type, "dst_ptr")
783 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
784 let base_src_ptr = self
785 .builder
786 .build_int_to_ptr(addr, ptr_type, "src_ptr")
787 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
788 let null_ptr = ptr_type.const_null();
789 let src_ptr = self
790 .builder
791 .build_select::<BasicValueEnum<'ctx>, _>(
792 offsets_found,
793 base_src_ptr.into(),
794 null_ptr.into(),
795 "src_or_null",
796 )
797 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
798 .into_pointer_value();
799
800 let i32_type = self.context.i32_type();
801 let helper_id = i64_type.const_int(BPF_FUNC_probe_read_user as u64, false);
802 let helper_fn_type =
803 i32_type.fn_type(&[ptr_type.into(), i32_type.into(), ptr_type.into()], false);
804 let helper_fn_ptr = self
805 .builder
806 .build_int_to_ptr(helper_id, ptr_type, "probe_read_user_fn")
807 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
808 let size_val = i32_type.const_int(result_size as u64, false);
809 let zero_i32 = i32_type.const_zero();
810 let effective_size = self
811 .builder
812 .build_select::<BasicValueEnum<'ctx>, _>(
813 offsets_found,
814 size_val.into(),
815 zero_i32.into(),
816 "size_or_zero",
817 )
818 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
819 .into_int_value();
820 let call_args: Vec<BasicMetadataValueEnum> =
821 vec![dst_ptr.into(), effective_size.into(), src_ptr.into()];
822
823 let call_site = self
824 .builder
825 .build_indirect_call(
826 helper_fn_type,
827 helper_fn_ptr,
828 &call_args,
829 "probe_read_result",
830 )
831 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
832 let ret_iv = call_site.try_as_basic_value().left().ok_or_else(|| {
833 CodeGenError::LLVMError("Expected integer return from helper".to_string())
834 })?;
835 let ret_i32 = ret_iv.into_int_value();
836 let read_fail = self
837 .builder
838 .build_int_compare(
839 inkwell::IntPredicate::NE,
840 ret_i32,
841 i32_type.const_zero(),
842 "read_fail",
843 )
844 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
845 let combined_fail = self
846 .builder
847 .build_or(read_fail, not_found, "combined_fail")
848 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
849
850 let result_type: BasicTypeEnum = match size {
851 MemoryAccessSize::U8 => self.context.i8_type().into(),
852 MemoryAccessSize::U16 => self.context.i16_type().into(),
853 MemoryAccessSize::U32 => self.context.i32_type().into(),
854 MemoryAccessSize::U64 => self.context.i64_type().into(),
855 };
856 let typed_ptr = self
857 .builder
858 .build_bit_cast(
859 scratch_buffer,
860 self.context.ptr_type(AddressSpace::default()),
861 "typed_ptr",
862 )
863 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
864 let loaded_value = self
865 .builder
866 .build_load(result_type, typed_ptr.into_pointer_value(), "loaded_value")
867 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
868 let loaded_i64 = if let BasicValueEnum::IntValue(int_val) = loaded_value {
869 if int_val.get_type().get_bit_width() < 64 {
870 self.builder
871 .build_int_z_extend(int_val, i64_type, "extended")
872 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
873 } else {
874 int_val
875 }
876 } else {
877 return Err(CodeGenError::MemoryAccessError(
878 "Expected integer value from memory read".to_string(),
879 ));
880 };
881
882 Ok(ProbeReadResult {
883 loaded_i64,
884 combined_fail,
885 not_found,
886 })
887 }
888
889 fn update_any_fail_flag(
890 &mut self,
891 combined_fail: IntValue<'ctx>,
892 name_suffix: &str,
893 ) -> Result<()> {
894 let i8_type = self.context.i8_type();
895 let fail_i8 = self
896 .builder
897 .build_int_z_extend(combined_fail, i8_type, &format!("fail_i8_{name_suffix}"))
898 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
899 let fail_ptr = self.get_or_create_flag_global("_gs_any_fail");
900 let cur_fail = self
901 .builder
902 .build_load(i8_type, fail_ptr, &format!("cur_fail_{name_suffix}"))
903 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
904 .into_int_value();
905 let new_fail = self
906 .builder
907 .build_or(cur_fail, fail_i8, &format!("fail_or_miss_{name_suffix}"))
908 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
909 self.builder
910 .build_store(fail_ptr, new_fail)
911 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
912 Ok(())
913 }
914
915 pub(crate) fn store_variable_read_status(
916 &mut self,
917 status_ptr: PointerValue<'ctx>,
918 combined_fail: IntValue<'ctx>,
919 not_found: IntValue<'ctx>,
920 name_suffix: &str,
921 ) -> Result<()> {
922 let cur_status = self
923 .builder
924 .build_load(
925 self.context.i8_type(),
926 status_ptr,
927 &format!("cur_status_{name_suffix}"),
928 )
929 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
930 .into_int_value();
931 let is_ok = self
932 .builder
933 .build_int_compare(
934 inkwell::IntPredicate::EQ,
935 cur_status,
936 self.context.i8_type().const_zero(),
937 &format!("status_is_ok_{name_suffix}"),
938 )
939 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
940 let desired_status = self
941 .builder
942 .build_select::<BasicValueEnum<'ctx>, _>(
943 not_found,
944 self.context
945 .i8_type()
946 .const_int(VariableStatus::OffsetsUnavailable as u64, false)
947 .into(),
948 self.context
949 .i8_type()
950 .const_int(VariableStatus::ReadError as u64, false)
951 .into(),
952 &format!("desired_read_status_{name_suffix}"),
953 )
954 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
955 let should_store = self
956 .builder
957 .build_and(
958 is_ok,
959 combined_fail,
960 &format!("should_store_read_status_{name_suffix}"),
961 )
962 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
963 let new_status = self
964 .builder
965 .build_select::<BasicValueEnum<'ctx>, _>(
966 should_store,
967 desired_status,
968 cur_status.into(),
969 &format!("new_status_{name_suffix}"),
970 )
971 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
972 self.builder
973 .build_store(status_ptr, new_status)
974 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
975 Ok(())
976 }
977
978 pub fn generate_memory_read(
980 &mut self,
981 addr: IntValue<'ctx>,
982 size: MemoryAccessSize,
983 status_ptr: Option<PointerValue<'ctx>>,
984 ) -> Result<BasicValueEnum<'ctx>> {
985 let zero_const = self.context.i64_type().const_zero();
986 let ProbeReadResult {
987 loaded_i64,
988 combined_fail,
989 not_found,
990 } = self.probe_read_user_core(addr, size, "probe_read_user")?;
991
992 if let Some(status_ptr) = status_ptr {
993 self.store_variable_read_status(
994 status_ptr,
995 combined_fail,
996 not_found,
997 "probe_read_user",
998 )?;
999 }
1000 self.update_any_fail_flag(combined_fail, "probe_read_user")?;
1001
1002 let zero_bv: BasicValueEnum = zero_const.into();
1003 let val_bv: BasicValueEnum = loaded_i64.into();
1004 self.builder
1005 .build_select::<BasicValueEnum<'ctx>, _>(
1006 combined_fail,
1007 zero_bv,
1008 val_bv,
1009 "value_or_zero",
1010 )
1011 .map_err(|e| CodeGenError::LLVMError(e.to_string()))
1012 }
1013
1014 pub fn generate_memory_read_with_status(
1017 &mut self,
1018 addr: IntValue<'ctx>,
1019 size: MemoryAccessSize,
1020 ) -> Result<BasicValueEnum<'ctx>> {
1021 let zero_const = self.context.i64_type().const_zero();
1022 let ProbeReadResult {
1023 loaded_i64,
1024 combined_fail,
1025 ..
1026 } = self.probe_read_user_core(addr, size, "probe_read_user_cf")?;
1027
1028 let cur_block = self.builder.get_insert_block().unwrap();
1029 let func = cur_block.get_parent().unwrap();
1030 let set_block = self.context.append_basic_block(func, "set_cond_err");
1031 let cont_block = self.context.append_basic_block(func, "read_cont");
1032 self.builder
1033 .build_conditional_branch(combined_fail, set_block, cont_block)
1034 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
1035 self.builder.position_at_end(set_block);
1036 let _ = self.set_condition_error_if_unset(2u8);
1037 let _ = self.set_condition_error_addr_if_unset(addr);
1038 self.builder
1039 .build_unconditional_branch(cont_block)
1040 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
1041 self.builder.position_at_end(cont_block);
1042
1043 let zero_bv: BasicValueEnum = zero_const.into();
1044 let val_bv: BasicValueEnum = loaded_i64.into();
1045 let sel_bv = self
1046 .builder
1047 .build_select::<BasicValueEnum<'ctx>, _>(combined_fail, zero_bv, val_bv, "val_or_zero")
1048 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
1049
1050 self.update_any_fail_flag(combined_fail, "probe_read_user_cf")?;
1051 Ok(sel_bv)
1052 }
1053 pub fn dwarf_reg_to_pt_regs_offset(&self, dwarf_reg: u16) -> Result<usize> {
1055 let byte_offset = register_mapping::dwarf_reg_to_pt_regs_byte_offset(dwarf_reg)
1057 .ok_or_else(|| {
1058 CodeGenError::RegisterMappingError(format!(
1059 "Unsupported DWARF register: {dwarf_reg}"
1060 ))
1061 })?;
1062
1063 let u64_index = byte_offset / core::mem::size_of::<u64>();
1065 Ok(u64_index)
1066 }
1067
1068 pub fn create_bpf_helper_call(
1071 &mut self,
1072 helper_id: u64,
1073 args: &[BasicValueEnum<'ctx>],
1074 return_type: BasicTypeEnum<'ctx>,
1075 call_name: &str,
1076 ) -> Result<BasicValueEnum<'ctx>> {
1077 use inkwell::types::BasicMetadataTypeEnum;
1078
1079 let arg_types: Vec<BasicMetadataTypeEnum> =
1081 args.iter().map(|arg| arg.get_type().into()).collect();
1082 let fn_type = return_type.fn_type(&arg_types, false);
1083
1084 let i64_type = self.context.i64_type();
1086 let ptr_type = self.context.ptr_type(AddressSpace::default());
1087
1088 let helper_id_val = i64_type.const_int(helper_id, false);
1089 let helper_fn_ptr = self
1090 .builder
1091 .build_int_to_ptr(helper_id_val, ptr_type, "helper_fn")
1092 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
1093
1094 let metadata_args: Vec<BasicMetadataValueEnum> =
1096 args.iter().map(|arg| (*arg).into()).collect();
1097
1098 let call_result = self
1100 .builder
1101 .build_indirect_call(fn_type, helper_fn_ptr, &metadata_args, call_name)
1102 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
1103
1104 Ok(call_result.try_as_basic_value().left().unwrap_or_else(|| {
1106 return_type.const_zero()
1108 }))
1109 }
1110
1111 pub fn get_current_timestamp(&mut self) -> Result<IntValue<'ctx>> {
1113 let i64_type = self.context.i64_type();
1114
1115 let timestamp = self.create_bpf_helper_call(
1117 BPF_FUNC_ktime_get_ns as u64,
1118 &[],
1119 i64_type.into(),
1120 "timestamp",
1121 )?;
1122
1123 if let BasicValueEnum::IntValue(int_val) = timestamp {
1124 Ok(int_val)
1125 } else {
1126 Err(CodeGenError::LLVMError(
1127 "bpf_ktime_get_ns did not return integer".to_string(),
1128 ))
1129 }
1130 }
1131
1132 pub fn get_current_pid_tgid(&mut self) -> Result<IntValue<'ctx>> {
1134 let i64_type = self.context.i64_type();
1135
1136 let pid_tgid = self.create_bpf_helper_call(
1138 BPF_FUNC_get_current_pid_tgid as u64,
1139 &[],
1140 i64_type.into(),
1141 "pid_tgid",
1142 )?;
1143
1144 if let BasicValueEnum::IntValue(int_val) = pid_tgid {
1145 Ok(int_val)
1146 } else {
1147 Err(CodeGenError::LLVMError(
1148 "bpf_get_current_pid_tgid did not return integer".to_string(),
1149 ))
1150 }
1151 }
1152
1153 pub fn create_event_output(&mut self, data: PointerValue<'ctx>, size: u64) -> Result<()> {
1156 match self.compile_options.event_map_type {
1157 crate::EventMapType::RingBuf => self.create_ringbuf_output_internal(data, size),
1158 crate::EventMapType::PerfEventArray => {
1159 self.create_perf_event_output_internal(data, size)
1160 }
1161 }
1162 }
1163
1164 fn create_ringbuf_output_internal(
1166 &mut self,
1167 data: PointerValue<'ctx>,
1168 size: u64,
1169 ) -> Result<()> {
1170 let i64_type = self.context.i64_type();
1171
1172 let ringbuf_global = self
1174 .map_manager
1175 .get_ringbuf_map(&self.module, "ringbuf")
1176 .map_err(|e| {
1177 CodeGenError::MemoryAccessError(format!("Failed to get ringbuf map: {e}"))
1178 })?;
1179
1180 let args = [
1182 ringbuf_global.into(),
1183 data.into(),
1184 i64_type.const_int(size, false).into(),
1185 i64_type.const_zero().into(), ];
1187
1188 let _result = self.create_bpf_helper_call(
1189 BPF_FUNC_ringbuf_output as u64,
1190 &args,
1191 i64_type.into(),
1192 "ringbuf_output",
1193 )?;
1194
1195 Ok(())
1196 }
1197
1198 pub fn create_ringbuf_output_dynamic(
1200 &mut self,
1201 data: PointerValue<'ctx>,
1202 size: IntValue<'ctx>,
1203 ) -> Result<()> {
1204 let i64_type = self.context.i64_type();
1205
1206 let ringbuf_global = self
1208 .map_manager
1209 .get_ringbuf_map(&self.module, "ringbuf")
1210 .map_err(|e| {
1211 CodeGenError::MemoryAccessError(format!("Failed to get ringbuf map: {e}"))
1212 })?;
1213
1214 let args = [
1216 ringbuf_global.into(),
1217 data.into(),
1218 size.into(),
1219 i64_type.const_zero().into(), ];
1221
1222 let _result = self.create_bpf_helper_call(
1223 BPF_FUNC_ringbuf_output as u64,
1224 &args,
1225 i64_type.into(),
1226 "ringbuf_output",
1227 )?;
1228
1229 Ok(())
1230 }
1231
1232 pub fn lookup_percpu_value_ptr(
1234 &mut self,
1235 map_name: &str,
1236 key_const: u32,
1237 ) -> Result<PointerValue<'ctx>> {
1238 let ptr_ty = self.context.ptr_type(AddressSpace::default());
1239 let i32_ty = self.context.i32_type();
1240 let map_global = self
1241 .map_manager
1242 .get_map(&self.module, map_name)
1243 .map_err(|e| CodeGenError::LLVMError(format!("Map not found {map_name}: {e}")))?;
1244 let map_ptr = self
1245 .builder
1246 .build_bit_cast(map_global, ptr_ty, "map_ptr")
1247 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
1248
1249 let key_arr_ty = i32_ty.array_type(4);
1251 let key_alloca = self.pm_key_alloca.ok_or_else(|| {
1252 CodeGenError::LLVMError("pm_key not allocated in entry block".to_string())
1253 })?;
1254 let zero = i32_ty.const_zero();
1255 let base_i32_ptr = unsafe {
1256 self.builder
1257 .build_gep(key_arr_ty, key_alloca, &[zero, zero], "percpu_key_i32_ptr")
1258 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?
1259 };
1260 self.builder
1261 .build_store(base_i32_ptr, i32_ty.const_int(key_const as u64, false))
1262 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
1263 let key_ptr = self
1264 .builder
1265 .build_bit_cast(base_i32_ptr, ptr_ty, "key_ptr")
1266 .map_err(|e| CodeGenError::LLVMError(e.to_string()))?;
1267
1268 let ret = self.create_bpf_helper_call(
1270 BPF_FUNC_map_lookup_elem as u64,
1271 &[map_ptr, key_ptr],
1272 ptr_ty.into(),
1273 "map_lookup_elem",
1274 )?;
1275 let val_ptr = if let BasicValueEnum::PointerValue(p) = ret {
1276 p
1277 } else {
1278 return Err(CodeGenError::LLVMError(
1279 "map_lookup_elem did not return pointer".to_string(),
1280 ));
1281 };
1282 Ok(val_ptr)
1283 }
1284
1285 fn create_perf_event_output_internal(
1287 &mut self,
1288 data: PointerValue<'ctx>,
1289 size: u64,
1290 ) -> Result<()> {
1291 let size_val = self.context.i64_type().const_int(size, false);
1292 self.create_perf_event_output_dynamic(data, size_val)
1293 }
1294
1295 pub fn create_perf_event_output_dynamic(
1297 &mut self,
1298 data: PointerValue<'ctx>,
1299 size: IntValue<'ctx>,
1300 ) -> Result<()> {
1301 let i64_type = self.context.i64_type();
1302
1303 let ctx_param = self
1305 .builder
1306 .get_insert_block()
1307 .and_then(|bb| bb.get_parent())
1308 .and_then(|func| func.get_first_param())
1309 .ok_or_else(|| {
1310 CodeGenError::LLVMError("Failed to get context parameter".to_string())
1311 })?;
1312
1313 let events_global = self
1315 .map_manager
1316 .get_perf_map(&self.module, "events")
1317 .map_err(|e| {
1318 CodeGenError::MemoryAccessError(format!("Failed to get perf event map: {e}"))
1319 })?;
1320
1321 let args = [
1324 ctx_param,
1325 events_global.into(),
1326 i64_type.const_int(0xFFFFFFFF_u64, false).into(), data.into(),
1328 size.into(),
1329 ];
1330
1331 let _result = self.create_bpf_helper_call(
1332 BPF_FUNC_perf_event_output as u64,
1333 &args,
1334 i64_type.into(),
1335 "perf_event_output",
1336 )?;
1337
1338 Ok(())
1339 }
1340}