1mod array;
4mod flags;
5mod hash;
6mod lru;
7mod queue;
8pub(crate) mod stream;
9use alloc::{
10 boxed::Box,
11 string::{String, ToString},
12 sync::Arc,
13 vec,
14 vec::Vec,
15};
16use core::{any::Any, ffi::CStr, fmt::Debug, ops::Range};
17
18use lock_api::{Mutex, MutexGuard, RawMutex};
19
20use crate::{
21 BpfError, BpfResult as Result, KernelAuxiliaryOps, PollWaker,
22 linux_bpf::{BpfMapType, bpf_attr},
23 map::flags::BpfMapCreateFlags,
24};
25
26pub type BpfCallBackFn = fn(key: &[u8], value: &[u8], ctx: *const u8) -> i32;
28
29pub trait BpfMapCommonOps: Send + Sync + Debug + Any {
31 fn lookup_elem(&mut self, _key: &[u8]) -> Result<Option<&[u8]>> {
35 Err(BpfError::EPERM)
36 }
37 fn update_elem(&mut self, _key: &[u8], _value: &[u8], _flags: u64) -> Result<()> {
41 Err(BpfError::EPERM)
42 }
43 fn delete_elem(&mut self, _key: &[u8]) -> Result<()> {
47 Err(BpfError::EPERM)
48 }
49 fn for_each_elem(&mut self, _cb: BpfCallBackFn, _ctx: *const u8, _flags: u64) -> Result<u32> {
54 Err(BpfError::EPERM)
55 }
56 fn lookup_and_delete_elem(&mut self, _key: &[u8], _value: &mut [u8]) -> Result<()> {
59 Err(BpfError::EPERM)
60 }
61
62 fn lookup_percpu_elem(&mut self, _key: &[u8], _cpu: u32) -> Result<Option<&[u8]>> {
64 Err(BpfError::EPERM)
65 }
66 fn get_next_key(&self, _key: Option<&[u8]>, _next_key: &mut [u8]) -> Result<()> {
70 Err(BpfError::EPERM)
71 }
72
73 fn push_elem(&mut self, _value: &[u8], _flags: u64) -> Result<()> {
75 Err(BpfError::EPERM)
76 }
77
78 fn pop_elem(&mut self, _value: &mut [u8]) -> Result<()> {
80 Err(BpfError::EPERM)
81 }
82
83 fn peek_elem(&self, _value: &mut [u8]) -> Result<()> {
85 Err(BpfError::EPERM)
86 }
87
88 fn freeze(&self) -> Result<()> {
92 Err(BpfError::EPERM)
93 }
94
95 fn map_values_ptr_range(&self) -> Result<Range<usize>> {
99 Err(BpfError::EPERM)
100 }
101
102 fn map_mem_usage(&self) -> Result<usize>;
104
105 fn map_mmap(&self, _offset: usize, _size: usize) -> Result<Vec<usize>> {
107 Err(BpfError::EPERM)
108 }
109
110 fn readable(&self) -> bool {
112 false
113 }
114
115 fn writable(&self) -> bool {
117 false
118 }
119
120 fn as_any(&self) -> &dyn Any;
122 fn as_any_mut(&mut self) -> &mut dyn Any;
124}
125
126pub trait PerCpuVariantsOps: Sync + Send + Debug + 'static {
128 fn create<T: Clone + Sync + Send + 'static>(value: T) -> Option<Box<dyn PerCpuVariants<T>>>;
130 fn num_cpus() -> u32;
132}
133
134#[allow(clippy::mut_from_ref)]
136pub trait PerCpuVariants<T: Clone + Sync + Send>: Sync + Send + Debug {
137 fn get(&self) -> &T;
139 fn get_mut(&self) -> &mut T;
141 unsafe fn force_get(&self, cpu: u32) -> &T;
148 unsafe fn force_get_mut(&self, cpu: u32) -> &mut T;
155}
156
157bitflags::bitflags! {
158 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
160 pub struct BpfMapUpdateElemFlags: u64 {
161 const BPF_ANY = 0;
163 const BPF_NOEXIST = 1;
165 const BPF_EXISTS = 2;
167 const BPF_F_LOCK = 4;
169 }
170}
171
172#[derive(Debug, Clone, Default)]
174pub struct BpfMapMeta {
175 pub map_type: BpfMapType,
177 pub key_size: u32,
179 pub value_size: u32,
181 pub max_entries: u32,
183 pub map_flags: BpfMapCreateFlags,
185 pub _map_name: String,
187}
188
189impl TryFrom<&bpf_attr> for BpfMapMeta {
190 type Error = BpfError;
191 fn try_from(attr: &bpf_attr) -> Result<Self> {
192 let u = unsafe { &attr.__bindgen_anon_1 };
193 let map_name_slice = unsafe {
194 core::slice::from_raw_parts(u.map_name.as_ptr() as *const u8, u.map_name.len())
195 };
196 let map_name = CStr::from_bytes_until_nul(map_name_slice)
197 .map_err(|_| BpfError::EINVAL)?
198 .to_str()
199 .map_err(|_| BpfError::EINVAL)?
200 .to_string();
201 let map_type = BpfMapType::try_from(u.map_type).map_err(|_| BpfError::EINVAL)?;
202
203 let map_flags = BpfMapCreateFlags::from_bits(u.map_flags).ok_or(BpfError::EINVAL)?;
204 Ok(BpfMapMeta {
205 map_type,
206 key_size: u.key_size,
207 value_size: u.value_size,
208 max_entries: u.max_entries,
209 map_flags,
210 _map_name: map_name,
211 })
212 }
213}
214
215#[derive(Debug)]
217pub struct UnifiedMap<L: RawMutex + 'static> {
218 inner_map: Mutex<L, Box<dyn BpfMapCommonOps>>,
219 map_meta: BpfMapMeta,
220}
221
222impl<L: RawMutex + 'static> UnifiedMap<L> {
223 fn new(map_meta: BpfMapMeta, map: Box<dyn BpfMapCommonOps>) -> Self {
224 Self {
225 inner_map: Mutex::new(map),
226 map_meta,
227 }
228 }
229 pub fn map(&self) -> MutexGuard<'_, L, Box<dyn BpfMapCommonOps>> {
231 self.inner_map.lock()
232 }
233
234 pub fn map_mut(&self) -> MutexGuard<'_, L, Box<dyn BpfMapCommonOps>> {
236 self.inner_map.lock()
237 }
238
239 pub fn map_meta(&self) -> &BpfMapMeta {
241 &self.map_meta
242 }
243}
244
245pub fn bpf_map_create<F: KernelAuxiliaryOps, T: PerCpuVariantsOps + 'static>(
251 map_meta: BpfMapMeta,
252 poll_waker: Option<Arc<dyn PollWaker>>,
253) -> Result<UnifiedMap<F::MapLock>> {
254 log::trace!("The map attr is {:#?}", map_meta);
255 let map: Box<dyn BpfMapCommonOps> = match map_meta.map_type {
256 BpfMapType::BPF_MAP_TYPE_ARRAY => {
257 let array_map = array::ArrayMap::new(&map_meta)?;
258 Box::new(array_map)
259 }
260 BpfMapType::BPF_MAP_TYPE_PERCPU_ARRAY => {
261 let per_cpu_array_map = array::PerCpuArrayMap::<T>::new(&map_meta)?;
262 Box::new(per_cpu_array_map)
263 }
264 BpfMapType::BPF_MAP_TYPE_PERF_EVENT_ARRAY => {
265 let perf_event_array_map = array::PerfEventArrayMap::new(&map_meta, T::num_cpus())?;
266 Box::new(perf_event_array_map)
267 }
268
269 BpfMapType::BPF_MAP_TYPE_CPUMAP
270 | BpfMapType::BPF_MAP_TYPE_DEVMAP
271 | BpfMapType::BPF_MAP_TYPE_DEVMAP_HASH => {
272 log::error!("bpf map type {:?} not implemented", map_meta.map_type);
273 Err(BpfError::EPERM)?
274 }
275 BpfMapType::BPF_MAP_TYPE_HASH => {
276 let hash_map = hash::BpfHashMap::new(&map_meta)?;
277 Box::new(hash_map)
278 }
279 BpfMapType::BPF_MAP_TYPE_PERCPU_HASH => {
280 let per_cpu_hash_map = hash::PerCpuHashMap::<T>::new(&map_meta)?;
281 Box::new(per_cpu_hash_map)
282 }
283 BpfMapType::BPF_MAP_TYPE_QUEUE => {
284 let queue_map = queue::QueueMap::new(&map_meta)?;
285 Box::new(queue_map)
286 }
287 BpfMapType::BPF_MAP_TYPE_STACK => {
288 let stack_map = queue::StackMap::new(&map_meta)?;
289 Box::new(stack_map)
290 }
291 BpfMapType::BPF_MAP_TYPE_LRU_HASH => {
292 let lru_hash_map = lru::LruMap::new(&map_meta)?;
293 Box::new(lru_hash_map)
294 }
295 BpfMapType::BPF_MAP_TYPE_LRU_PERCPU_HASH => {
296 let lru_per_cpu_hash_map = lru::PerCpuLruMap::<T>::new(&map_meta)?;
297 Box::new(lru_per_cpu_hash_map)
298 }
299 BpfMapType::BPF_MAP_TYPE_RINGBUF => {
300 let poll_waker = poll_waker.ok_or(BpfError::EINVAL)?;
301 let ringbuf_map = stream::RingBufMap::<F>::new(&map_meta, poll_waker)?;
302 Box::new(ringbuf_map)
303 }
304 _ => {
305 log::error!("bpf map type {:?} not implemented", map_meta.map_type);
306 Err(BpfError::EPERM)?
307 }
308 };
309 let unified_map = UnifiedMap::new(map_meta, map);
310 Ok(unified_map)
311}
312
313#[derive(Debug, Clone, Copy)]
315pub struct BpfMapUpdateArg {
316 pub map_fd: u32,
318 pub key: u64,
320 pub value: u64,
322 pub flags: u64,
324}
325
326impl From<&bpf_attr> for BpfMapUpdateArg {
327 fn from(attr: &bpf_attr) -> Self {
328 let u = unsafe { &attr.__bindgen_anon_2 };
329 let map_fd = u.map_fd;
330 let key = u.key;
331 let value = unsafe { u.__bindgen_anon_1.value };
332 let flags = u.flags;
333 BpfMapUpdateArg {
334 map_fd,
335 key,
336 value,
337 flags,
338 }
339 }
340}
341
342#[derive(Debug, Clone, Copy)]
344pub struct BpfMapGetNextKeyArg {
345 pub map_fd: u32,
347 pub key: Option<u64>,
349 pub next_key: u64,
351}
352
353impl From<&bpf_attr> for BpfMapGetNextKeyArg {
354 fn from(attr: &bpf_attr) -> Self {
355 unsafe {
356 let u = &attr.__bindgen_anon_2;
357 BpfMapGetNextKeyArg {
358 map_fd: u.map_fd,
359 key: if u.key != 0 { Some(u.key) } else { None },
360 next_key: u.__bindgen_anon_1.next_key,
361 }
362 }
363 }
364}
365
366pub fn bpf_map_update_elem<F: KernelAuxiliaryOps, L: RawMutex + 'static>(
370 arg: BpfMapUpdateArg,
371) -> Result<()> {
372 F::get_unified_map_from_fd::<_, _>(arg.map_fd, |unified_map| {
373 let meta = unified_map.map_meta();
374 let key_size = meta.key_size as usize;
375 let value_size = meta.value_size as usize;
376 let mut key = vec![0u8; key_size];
377 let mut value = vec![0u8; value_size];
378 F::copy_from_user(arg.key as *const u8, key_size, &mut key)?;
379 F::copy_from_user(arg.value as *const u8, value_size, &mut value)?;
380 unified_map.map_mut().update_elem(&key, &value, arg.flags)
381 })
382}
383
384pub fn bpf_map_freeze<F: KernelAuxiliaryOps, L: RawMutex + 'static>(map_fd: u32) -> Result<()> {
386 F::get_unified_map_from_fd(map_fd, |unified_map| unified_map.map().freeze())
387}
388
389pub fn bpf_lookup_elem<F: KernelAuxiliaryOps, L: RawMutex + 'static>(
393 arg: BpfMapUpdateArg,
394) -> Result<()> {
395 F::get_unified_map_from_fd(arg.map_fd, |unified_map| {
397 let meta = unified_map.map_meta();
398 let key_size = meta.key_size as usize;
399 let value_size = meta.value_size as usize;
400 let mut key = vec![0u8; key_size];
401 F::copy_from_user(arg.key as *const u8, key_size, &mut key)?;
402 let r_value = {
403 let mut map = unified_map.map_mut();
404 map.lookup_elem(&key)?.map(|v| v.to_vec())
405 };
406 if let Some(r_value) = r_value {
407 F::copy_to_user(arg.value as *mut u8, value_size, &r_value)?;
408 Ok(())
409 } else {
410 Err(BpfError::ENOENT)
411 }
412 })
413}
414pub fn bpf_map_get_next_key<F: KernelAuxiliaryOps, L: RawMutex + 'static>(
422 arg: BpfMapGetNextKeyArg,
423) -> Result<()> {
424 F::get_unified_map_from_fd(arg.map_fd, |unified_map| {
426 let meta = unified_map.map_meta();
427 let key_size = meta.key_size as usize;
428 let mut next_key = vec![0u8; key_size];
429 if let Some(key_ptr) = arg.key {
430 let mut key = vec![0u8; key_size];
431 F::copy_from_user(key_ptr as *const u8, key_size, &mut key)?;
432 unified_map
433 .map_mut()
434 .get_next_key(Some(&key), &mut next_key)?;
435 } else {
436 unified_map.map_mut().get_next_key(None, &mut next_key)?;
437 };
438 F::copy_to_user(arg.next_key as *mut u8, key_size, &next_key)?;
439 Ok(())
440 })
441}
442
443pub fn bpf_map_delete_elem<F: KernelAuxiliaryOps, L: RawMutex + 'static>(
452 arg: BpfMapUpdateArg,
453) -> Result<()> {
454 F::get_unified_map_from_fd(arg.map_fd, |unified_map| {
456 let meta = unified_map.map_meta();
457 let key_size = meta.key_size as usize;
458 let mut key = vec![0u8; key_size];
459 F::copy_from_user(arg.key as *const u8, key_size, &mut key)?;
460 unified_map.map_mut().delete_elem(&key)
461 })
462}
463
464pub fn bpf_map_lookup_batch<F: KernelAuxiliaryOps>(_arg: BpfMapUpdateArg) -> Result<usize> {
468 Err(BpfError::EPERM)
470}
471
472pub fn bpf_map_lookup_and_delete_elem<F: KernelAuxiliaryOps, L: RawMutex + 'static>(
496 arg: BpfMapUpdateArg,
497) -> Result<()> {
498 F::get_unified_map_from_fd(arg.map_fd, |unified_map| {
500 let meta = unified_map.map_meta();
501 let key_size = meta.key_size as usize;
502 let value_size = meta.value_size as usize;
503 let mut key = vec![0u8; key_size];
504 let mut value = vec![0u8; value_size];
505 F::copy_from_user(arg.key as *const u8, key_size, &mut key)?;
506 unified_map
507 .map_mut()
508 .lookup_and_delete_elem(&key, &mut value)?;
509 F::copy_to_user(arg.value as *mut u8, value_size, &value)?;
510 Ok(())
511 })
512}
513
514#[cfg(test)]
515mod tests {
516 use alloc::{boxed::Box, vec::Vec};
517 use core::fmt::Debug;
518
519 use super::{PerCpuVariants, PerCpuVariantsOps};
520
521 #[derive(Debug)]
522 pub struct DummyPerCpuCreator;
523
524 #[derive(Debug)]
525 pub struct DummyPerCpuCreatorFalse;
526
527 pub struct DummyPerCpuVariants<T>(Vec<T>);
528
529 impl<T> Debug for DummyPerCpuVariants<T> {
530 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
531 f.debug_tuple("DummyPerCpuVariants").finish()
532 }
533 }
534
535 impl<T: Clone + Sync + Send> PerCpuVariants<T> for DummyPerCpuVariants<T> {
536 fn get(&self) -> &T {
537 &self.0[0]
538 }
539
540 fn get_mut(&self) -> &mut T {
541 unsafe { &mut *(self.0.as_ptr() as *mut T) }
542 }
543
544 unsafe fn force_get(&self, cpu: u32) -> &T {
545 &self.0[cpu as usize]
546 }
547
548 unsafe fn force_get_mut(&self, cpu: u32) -> &mut T {
549 let ptr = self.0.as_ptr();
550 let ptr = unsafe { ptr.add(cpu as usize) } as *mut T;
551 unsafe { &mut *ptr }
552 }
553 }
554
555 impl PerCpuVariantsOps for DummyPerCpuCreator {
556 fn create<T: Clone + Sync + Send + 'static>(
557 value: T,
558 ) -> Option<Box<dyn PerCpuVariants<T>>> {
559 let mut vec = Vec::new();
560 for _ in 0..Self::num_cpus() {
561 vec.push(value.clone());
562 }
563 Some(Box::new(DummyPerCpuVariants(vec)))
564 }
565
566 fn num_cpus() -> u32 {
567 1
568 }
569 }
570
571 impl PerCpuVariantsOps for DummyPerCpuCreatorFalse {
572 fn create<T: Clone + Sync + Send + 'static>(
573 _value: T,
574 ) -> Option<Box<dyn PerCpuVariants<T>>> {
575 None
576 }
577
578 fn num_cpus() -> u32 {
579 0
580 }
581 }
582}