vpp_plugin/vlib/buffer.rs
1//! vlib buffer abstraction
2//!
3//! This module contains abstractions around VPP's `vlib_buffer_t` structure.
4//! It provides safe access to buffer fields and operations.
5//! It also includes buffer allocation and deallocation functions.
6//! The goal is to provide a safe and ergonomic interface for working with VPP buffers.
7
8use std::{fmt, hint::assert_unchecked, mem::ManuallyDrop, mem::MaybeUninit};
9
10use arrayvec::ArrayVec;
11use bitflags::bitflags;
12
13use crate::{
14 bindings::{
15 CLIB_LOG2_CACHE_LINE_BYTES, VLIB_BUFFER_EXT_HDR_VALID, VLIB_BUFFER_IS_TRACED,
16 VLIB_BUFFER_MIN_CHAIN_SEG_SIZE, VLIB_BUFFER_NEXT_PRESENT, VLIB_BUFFER_PRE_DATA_SIZE,
17 VLIB_BUFFER_TOTAL_LENGTH_VALID, vlib_add_trace, vlib_buffer_func_main, vlib_buffer_t,
18 vlib_buffer_t__bindgen_ty_1, vlib_buffer_t__bindgen_ty_1__bindgen_ty_1__bindgen_ty_1,
19 vlib_helper_buffer_alloc, vlib_helper_buffer_free,
20 },
21 const_assert,
22 vlib::{
23 self, MainRef,
24 node::{ErrorCounters, Node, NodeRuntimeRef, VectorBufferIndex},
25 },
26 vppinfra::{
27 cache::{prefetch_load, prefetch_store},
28 likely,
29 },
30};
31
32/// VPP buffer index
33#[repr(transparent)]
34#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
35pub struct BufferIndex(u32);
36
37impl BufferIndex {
38 /// Construct a new `BufferIndex`
39 pub const fn new(buffer: u32) -> Self {
40 Self(buffer)
41 }
42}
43
44impl From<u32> for BufferIndex {
45 fn from(value: u32) -> BufferIndex {
46 Self(value)
47 }
48}
49
50impl From<BufferIndex> for u32 {
51 fn from(value: BufferIndex) -> Self {
52 value.0
53 }
54}
55
56impl VectorBufferIndex for BufferIndex {
57 fn as_u32_slice(slice: &[Self]) -> &[u32] {
58 // SAFETY: BufferIndex is a repr(transparent) wrapper around u32 so the src and dst slice
59 // types have the same memory layout
60 unsafe { std::mem::transmute::<&[BufferIndex], &[u32]>(slice) }
61 }
62}
63
64bitflags! {
65 /// vlib buffer flags
66 #[repr(transparent)]
67 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
68 pub struct BufferFlags: u32 {
69 /// Trace this buffer
70 const IS_TRACED = VLIB_BUFFER_IS_TRACED;
71 /// This is one buffer in a chain of buffers
72 const NEXT_PRESENT = VLIB_BUFFER_NEXT_PRESENT;
73 /// Total length is valid
74 const TOTAL_LENGTH_VALID = VLIB_BUFFER_TOTAL_LENGTH_VALID;
75 /// Contains external buffer manager header
76 const EXT_HDR_VALID = VLIB_BUFFER_EXT_HDR_VALID;
77
78 // Flags can be extended by user/vnet
79 const _ = !0;
80 }
81}
82
83/// Construct a user buffer flag
84///
85/// `n` must be less than 29 and greater than 0.
86pub const fn vlib_buffer_flag_user(n: u32) -> u32 {
87 assert!(n < 29 && n > 0);
88 1 << (32 - n)
89}
90
91/// Reference to a VPP buffer
92///
93/// A `&mut BufferRef<FeatureData>` is equivalent to a `vlib_buffer_t *` in C (a `*mut
94/// vlib_buffer_t` in Rust).
95#[repr(transparent)]
96pub struct BufferRef<FeatureData>(foreign_types::Opaque, std::marker::PhantomData<FeatureData>);
97
98impl<FeatureData> BufferRef<FeatureData> {
99 /// Create a `&BufferRef` from a raw pointer
100 ///
101 /// # Safety
102 ///
103 /// - The pointer must be a valid and properly initialised `vlib_buffer_t`.
104 /// - The pointer must stay valid and the contents must not be mutated for the duration of the
105 /// lifetime of the returned object.
106 #[inline(always)]
107 pub unsafe fn from_ptr<'a>(ptr: *mut vlib_buffer_t) -> &'a Self {
108 // SAFETY: The safety requirements are documented in the function's safety comment.
109 unsafe { &*(ptr as *mut _) }
110 }
111
112 /// Create a `&mut BufferRef` from a raw pointer
113 ///
114 /// # Safety
115 ///
116 /// - The pointer must be a valid and properly initialised `vlib_buffer_t`.
117 /// - The pointer must stay valid and the contents must not be mutated for the duration of the
118 /// lifetime of the returned object.
119 #[inline(always)]
120 pub unsafe fn from_ptr_mut<'a>(ptr: *mut vlib_buffer_t) -> &'a mut Self {
121 // SAFETY: The safety requirements are documented in the function's safety comment.
122 unsafe { &mut *(ptr as *mut _) }
123 }
124
125 /// Returns the raw pointer to the underlying `vlib_buffer_t`
126 #[inline(always)]
127 pub fn as_ptr(&self) -> *mut vlib_buffer_t {
128 self as *const _ as *mut _
129 }
130
131 #[inline(always)]
132 fn as_details(&self) -> &vlib_buffer_t__bindgen_ty_1 {
133 // SAFETY: since the reference to self is valid, so must be the pointer and it's safe to
134 // use the __bindgen_anon_1 union arm since the union is just present to force alignment
135 // Creation preconditions mean there are no aliased accesses to the buffer so it's fine
136 // to take a reference
137 unsafe { (*self.as_ptr()).__bindgen_anon_1.as_ref() }
138 }
139
140 #[inline(always)]
141 fn as_details_mut(&mut self) -> &mut vlib_buffer_t__bindgen_ty_1 {
142 // SAFETY: since the reference to self is valid, so must be the pointer and it's safe to
143 // use the __bindgen_anon_1 union arm since the union is just present to force alignment.
144 // Creation preconditions mean there are no aliased accesses to the buffer so it's fine
145 // to take a reference
146 unsafe { (*self.as_ptr()).__bindgen_anon_1.as_mut() }
147 }
148
149 #[inline(always)]
150 pub(crate) fn as_metadata(&self) -> &vlib_buffer_t__bindgen_ty_1__bindgen_ty_1__bindgen_ty_1 {
151 // SAFETY: since the reference to self is valid, so must be the pointer and it's safe to
152 // use the __bindgen_anon_1 union arm since the union is just present to force alignment
153 // Creation preconditions mean there are no aliased accesses to the buffer so it's fine
154 // to take a reference
155 unsafe { self.as_details().__bindgen_anon_1.__bindgen_anon_1.as_ref() }
156 }
157
158 #[inline(always)]
159 pub(crate) fn as_metadata_mut(
160 &mut self,
161 ) -> &mut vlib_buffer_t__bindgen_ty_1__bindgen_ty_1__bindgen_ty_1 {
162 // SAFETY: since the reference to self is valid, so must be the pointer and it's safe to
163 // use the __bindgen_anon_1 union arm since the union is just present to force alignment.
164 // Creation preconditions mean there are no aliased accesses to the buffer so it's fine
165 // to take a reference
166 unsafe {
167 self.as_details_mut()
168 .__bindgen_anon_1
169 .__bindgen_anon_1
170 .as_mut()
171 }
172 }
173
174 fn data(&self) -> *const u8 {
175 self.as_details().data.as_ptr()
176 }
177
178 fn current_data_offset(&self) -> i16 {
179 self.as_metadata().current_data
180 }
181
182 fn current_data_offset_mut(&mut self) -> &mut i16 {
183 &mut self.as_metadata_mut().current_data
184 }
185
186 /// Current length
187 ///
188 /// Typically, this is the amount of packet data remaining from [`Self::current_ptr_mut`].
189 pub fn current_length(&self) -> u16 {
190 self.as_metadata().current_length
191 }
192
193 fn current_length_mut(&mut self) -> &mut u16 {
194 &mut self.as_metadata_mut().current_length
195 }
196
197 /// Get the flags set for this buffer
198 #[inline(always)]
199 pub fn flags(&self) -> BufferFlags {
200 BufferFlags::from_bits_retain(self.as_metadata().flags)
201 }
202
203 /// Set the flags for this buffer
204 ///
205 /// # Safety
206 ///
207 /// [`BufferFlags::NEXT_PRESENT`] must not be set unless there is a next buffer in the chain.
208 /// [`BufferFlags::EXT_HDR_VALID`] must not be set or cleared unless the external buffer manager header is valid
209 /// or not valid respectively.
210 ///
211 #[inline(always)]
212 pub unsafe fn set_flags(&mut self, flags: BufferFlags) {
213 self.as_metadata_mut().flags = flags.bits()
214 }
215
216 /// Get a pointer to the current data
217 ///
218 /// This corresponds to the VPP C API `vlib_buffer_get_current`.
219 ///
220 /// # Usage guidance
221 ///
222 /// Note that the pointer returned may point to uninitialised data depending on the context.
223 /// In addition, depending on the context, the amount of remaining data may be less than
224 /// expected.
225 /// Finally, if remaining data is sufficient and it's initialised it may not have been validated
226 /// so care must be taken in determining whether or not lengths in the headers can be trusted.
227 pub fn current_ptr_mut(&mut self) -> *mut u8 {
228 let data = self.data().cast_mut();
229 let current_data = self.current_data_offset();
230
231 debug_assert!(current_data >= -(VLIB_BUFFER_PRE_DATA_SIZE as i16));
232
233 // SAFETY: current_data is asserted to be valid and point into valid (but possibly
234 // unintialised) data or pre_data.
235 unsafe { data.offset(current_data as isize) }
236 }
237
238 /// Check if the buffer has space to advance `l` bytes
239 ///
240 /// This corresponds to the VPP C API `vlib_buffer_has_space`.
241 pub fn has_space(&self, l: i16) -> bool {
242 self.current_length() >= l as u16
243 }
244
245 /// Advance the current data pointer by `l` bytes
246 ///
247 /// This corresponds to the VPP C API `vlib_buffer_advance`.
248 ///
249 /// # Safety
250 ///
251 /// - If `l` is positive, the buffer must have at least `l` bytes of data remaining.
252 /// - If `l` is negative, the current data offset must be at least `-l` bytes from the start of
253 /// the buffer's data area (including pre-data).
254 pub unsafe fn advance(&mut self, l: i16) {
255 debug_assert!(l < 0 || self.current_length() >= l as u16);
256 debug_assert!(
257 l >= 0 || self.current_data_offset() + VLIB_BUFFER_PRE_DATA_SIZE as i16 >= -l
258 );
259
260 *self.current_data_offset_mut() += l;
261 if l >= 0 {
262 *self.current_length_mut() -= l as u16;
263 } else {
264 *self.current_length_mut() += -l as u16;
265 }
266
267 debug_assert!(
268 !self.flags().contains(BufferFlags::NEXT_PRESENT)
269 || self.current_length() >= VLIB_BUFFER_MIN_CHAIN_SEG_SIZE as u16
270 );
271 }
272
273 /// Append uninitialised data to the end of the current data.
274 ///
275 /// Returns a pointer to the start of the newly appended uninitialised data.
276 ///
277 /// This corresponds to the VPP C function `vlib_buffer_put_uninit`.
278 ///
279 /// # Safety
280 ///
281 /// The current data plus the space requested must not exceed the data size of the buffer
282 /// given during allocation. See [`super::MainRef::buffer_default_data_size`] for buffers
283 /// allocated by [`super::MainRef::alloc_buffer`].
284 ///
285 /// The caller must ensure that the data is correctly initialised before passing the buffer to
286 /// code that assumes it is correctly initialised, such as enqueing the buffer another node.
287 pub unsafe fn put_uninit(&mut self, size: u16) -> *mut u8 {
288 let p = self.tail_mut();
289 *self.current_length_mut() += size;
290 p
291 }
292
293 /// Get a pointer to the end of the current data
294 ///
295 /// This corresponds to the VPP C function `vlib_buffer_get_tail`.
296 pub fn tail_mut(&mut self) -> *mut u8 {
297 let data = self.data().cast_mut();
298 let current_data = self.current_data_offset();
299
300 debug_assert!(current_data >= -(VLIB_BUFFER_PRE_DATA_SIZE as i16));
301
302 // SAFETY: current_data and current_length are asserted to be valid and `current_data +
303 // current_length` asserted to point to the end of valid (but possibly unintialised) data
304 // or pre_data.
305 unsafe {
306 let ptr = data.offset(current_data as isize);
307 ptr.add(self.current_length() as usize)
308 }
309 }
310
311 /// Add trace data to this buffer
312 pub fn add_trace<N: Node>(
313 &mut self,
314 vm: &MainRef,
315 node: &NodeRuntimeRef<N>,
316 ) -> &mut MaybeUninit<N::TraceData> {
317 // SAFETY: pointers are valid and the uninitialised data that is returned cannot be read
318 // by safe code
319 unsafe {
320 &mut *(vlib_add_trace(
321 vm.as_ptr(),
322 node.as_ptr(),
323 self.as_ptr(),
324 std::mem::size_of::<N::TraceData>() as u32,
325 ) as *mut MaybeUninit<N::TraceData>)
326 }
327 }
328
329 /// Set an error reason
330 ///
331 /// This is typically done before sending the packet to the `drop` node, where it use the
332 /// value to display the reason in traces and automatically increment the per-node, per-error
333 /// counter for the error.
334 pub fn set_error<N: Node>(&mut self, node: &NodeRuntimeRef<N>, error: N::Errors) {
335 // SAFETY: vlib_node_runtime_t::errors is sized according to the number of error values
336 // and it is a precondition of the Errors trait that the value return by into_u16() cannot
337 // be greater than or equal to the declared number of error values.
338 unsafe {
339 let error_value = (*node.as_ptr()).errors.add(error.into_u16() as usize);
340 self.as_metadata_mut().error = *error_value;
341 }
342 }
343
344 /// Get the total length of the buffer chain not including the first buffer
345 #[inline(always)]
346 pub fn total_length_not_including_first_buffer(&self) -> u32 {
347 debug_assert!(self.flags().contains(BufferFlags::TOTAL_LENGTH_VALID));
348 self.as_details().total_length_not_including_first_buffer
349 }
350
351 /// Get the total length of the buffer chain from the current offset
352 ///
353 /// Note that this doesn't take into account any bytes that have been [`Self::advance()`]d
354 /// over.
355 #[inline(always)]
356 pub fn length_in_chain(&self, vm: &vlib::MainRef) -> u64 {
357 let len = self.current_length();
358
359 if likely(!self.flags().contains(BufferFlags::NEXT_PRESENT)) {
360 return len as u64;
361 }
362
363 if likely(self.flags().contains(BufferFlags::TOTAL_LENGTH_VALID)) {
364 return len as u64 + self.total_length_not_including_first_buffer() as u64;
365 }
366
367 // SAFETY: The buffer pointer is valid and the function is called in a valid context.
368 unsafe {
369 crate::bindings::vlib_buffer_length_in_chain_slow_path(vm.as_ptr(), self.as_ptr())
370 }
371 }
372
373 /// Hint to the CPU to prefetch the buffer header for read access.
374 ///
375 /// This is a performance hint that attempts to bring the buffer header into the CPU cache
376 /// prior to reading fields from it. It does not affect program semantics and may be a no-op
377 /// on some platforms. Use this when you will shortly read header fields and want to reduce
378 /// cache miss latency.
379 pub fn prefetch_header_load(&self) {
380 prefetch_load(self.as_ptr());
381 }
382
383 /// Hint to the CPU to prefetch the buffer header for write access.
384 ///
385 /// Similar to `prefetch_header_load` but indicates imminent writes to the header. This is a
386 /// performance optimization only and does not change observable behaviour other than timing.
387 pub fn prefetch_header_store(&self) {
388 prefetch_store(self.as_ptr());
389 }
390
391 /// Hint to the CPU to prefetch the buffer data area for read access.
392 ///
393 /// This brings the buffer's data into cache in preparation for reading the packet payload.
394 /// It is a non-semantic performance hint and may be ignored on some architectures. Use this
395 /// when you will shortly read packet data and want to reduce cache miss latency.
396 pub fn prefetch_data_load(&self) {
397 prefetch_load(&self.as_details().data);
398 }
399
400 /// Hint to the CPU to prefetch the buffer data area for write access.
401 ///
402 /// Similar to `prefetch_data_load` but indicates the caller will write into the data area.
403 /// This is a cache-warming hint to reduce latency on subsequent stores.
404 pub fn prefetch_data_store(&self) {
405 prefetch_store(&self.as_details().data);
406 }
407}
408
409/// Owned buffer (with context)
410///
411/// The `&MainRef` context is necessary to be able to free the buffer on drop.
412pub struct BufferWithContext<'a> {
413 buffer: u32,
414 vm: &'a MainRef,
415}
416
417impl<'a> BufferWithContext<'a> {
418 /// Creates a `BufferWithContext` directly from a buffer index and a main reference
419 ///
420 /// # Safety
421 /// - The buffer index must be valid and the caller must have ownership of the buffer it
422 /// corresponds to.
423 pub unsafe fn from_parts(buffer: BufferIndex, vm: &'a MainRef) -> Self {
424 Self {
425 buffer: buffer.0,
426 vm,
427 }
428 }
429
430 /// Decomposes a `BufferWithContext` into its component parts
431 ///
432 /// After calling this the caller is responsible for ensuring the buffer gets freed, either
433 /// by calling [`BufferWithContext::from_parts`] or by passing it into another function which
434 /// takes ownership of it and eventually causes it to be freed.
435 pub fn into_parts(self) -> (BufferIndex, &'a MainRef) {
436 let me = ManuallyDrop::new(self);
437 (BufferIndex(me.buffer), me.vm)
438 }
439
440 /// Get a mutable reference to the buffer
441 pub fn as_buffer_ref(&mut self) -> &mut BufferRef<()> {
442 let from = &[self.buffer];
443 let mut b: ArrayVec<_, 1> = ArrayVec::new();
444 // SAFETY: capacity of b equals the length of from, `self.buffer` is a valid index and we
445 // force FeatureData to `()` since it isn't known and the buffer cannot be part of a
446 // feature arc.
447 unsafe {
448 self.vm.get_buffers(from, &mut b);
449 }
450 b.remove(0)
451 }
452}
453
454impl Drop for BufferWithContext<'_> {
455 fn drop(&mut self) {
456 // SAFETY: we have a reference to MainRef so the pointer must be valid, we pass in a
457 // pointer to buffers consistent with the number of buffers passed in, and self.buffer
458 // is a valid buffer index that we have ownership of.
459 unsafe {
460 vlib_helper_buffer_free(self.vm.as_ptr(), &mut self.buffer, 1);
461 }
462 }
463}
464
465/// Buffer allocation error
466#[derive(Copy, Clone, PartialEq, Eq, Debug)]
467pub struct BufferAllocError;
468
469impl fmt::Display for BufferAllocError {
470 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
471 write!(f, "buffer allocation error")
472 }
473}
474
475impl std::error::Error for BufferAllocError {}
476
477/// u64 x 8
478///
479/// This type exists to strongly hint to the compiler that it should emit vector instructions.
480///
481/// In the future, the implementation might be changed to use standard library portable SIMD once
482/// stabilised (https://github.com/rust-lang/rust/issues/86656), or use arch-specific intrinsics
483/// (if evidenced by high-enough performance improvement).
484#[allow(non_camel_case_types)]
485pub(crate) struct u64x8([u64; 8]);
486
487impl u64x8 {
488 /// Construct a `u64x8` from an array of 8 `u64`s
489 #[inline(always)]
490 pub(crate) fn from_array(a: [u64; 8]) -> Self {
491 Self(a)
492 }
493
494 /// Construct a `u64x8` from a pointer to 8 `u32`s
495 #[inline(always)]
496 pub(crate) unsafe fn from_u32_ptr(ptr: *const u32) -> Self {
497 // SAFETY: The caller must ensure the pointer is valid for reading 8 u32 values.
498 unsafe {
499 Self([
500 *ptr.add(0) as u64,
501 *ptr.add(1) as u64,
502 *ptr.add(2) as u64,
503 *ptr.add(3) as u64,
504 *ptr.add(4) as u64,
505 *ptr.add(5) as u64,
506 *ptr.add(6) as u64,
507 *ptr.add(7) as u64,
508 ])
509 }
510 }
511
512 /// Shift each element to the left by a given constant value, assigning the result to `self`
513 #[inline(always)]
514 pub(crate) fn shift_elements_left<const OFFSET: u32>(&mut self) {
515 for a in &mut self.0 {
516 *a <<= OFFSET;
517 }
518 }
519
520 /// Add a given value to each element, returning a new u64x8 with the result
521 #[inline(always)]
522 pub(crate) fn add_u64(&self, value: u64) -> Self {
523 Self::from_array([
524 self.0[0] + value,
525 self.0[1] + value,
526 self.0[2] + value,
527 self.0[3] + value,
528 self.0[4] + value,
529 self.0[5] + value,
530 self.0[6] + value,
531 self.0[7] + value,
532 ])
533 }
534
535 /// Write 8 contiguous elements starting from `ptr`
536 #[inline(always)]
537 pub(crate) unsafe fn store(&self, ptr: *mut u64) {
538 // SAFETY: The caller must ensure the pointer is valid for writing 8 u64 values.
539 unsafe {
540 *ptr.add(0) = self.0[0];
541 *ptr.add(1) = self.0[1];
542 *ptr.add(2) = self.0[2];
543 *ptr.add(3) = self.0[3];
544 *ptr.add(4) = self.0[4];
545 *ptr.add(5) = self.0[5];
546 *ptr.add(6) = self.0[6];
547 *ptr.add(7) = self.0[7];
548 }
549 }
550}
551
552/// Round a value up to the next multiple of the given power-of-two
553const fn next_multiple_of_pow2(val: usize, pow2: usize) -> usize {
554 debug_assert!(pow2.is_power_of_two());
555 (val + pow2 - 1) & !(pow2 - 1)
556}
557
558impl MainRef {
559 /// Get pointers to buffers for the given buffer indices, writing them into the provided `to` arrayvec.
560 ///
561 /// This is similar to `vlib_get_buffers` in the C API.
562 ///
563 /// Note that although it would be more idiomatic to return an `ArrayVec` directly, this
564 /// method takes a mutable reference to an `ArrayVec` to avoid an unnecessary copy when
565 /// returning.
566 ///
567 /// # Safety
568 ///
569 /// - The caller must ensure that `to` has enough capacity to hold all the buffers
570 /// corresponding to the indices in `from_indices`.
571 /// - Each index in `from_indices` must be valid and the caller must have ownership of the
572 /// buffer it corresponds to.
573 /// - Each buffer's `current_config_index` must be consistent with the `FeatureData` type. If
574 /// they are not known (i.e. because the caller the node isn't being executed in a feature
575 /// arc), FeatureData should be a zero-sized type such as `()`.
576 /// - The capacity of `from_indices` must be a multiple of 8 (note though that the length is
577 /// allowed not to be). In other words, it must be valid to read multiples of 8 from the
578 /// underlying memory (possibly returning uninitialised or stale data) without faulting.
579 #[inline(always)]
580 pub unsafe fn get_buffers<'a, 'me, 'buf: 'me, FeatureData, const N: usize>(
581 &'me self,
582 from_indices: &'a [u32],
583 to: &mut ArrayVec<&'buf mut BufferRef<FeatureData>, N>,
584 ) {
585 // SAFETY: The safety requirements are documented in the function's safety comment.
586 unsafe {
587 debug_assert!(from_indices.len() <= N);
588 assert_unchecked(from_indices.len() <= N);
589 // The vector operations below compute a pointer in terms of u64, i.e. assume that u64
590 // is a pointer-sized type the same as usize.
591 const_assert!(std::mem::size_of::<usize>() == std::mem::size_of::<u64>());
592
593 #[cfg(debug_assertions)]
594 for from_index in from_indices {
595 let buffer_mem_size = (*(*self.as_ptr()).buffer_main).buffer_mem_size;
596 debug_assert!(
597 ((*from_index << CLIB_LOG2_CACHE_LINE_BYTES) as u64) < buffer_mem_size
598 );
599 }
600
601 let buffer_mem_start = (*(*self.as_ptr()).buffer_main).buffer_mem_start;
602
603 // Check for the ArrayVec capacity being a multiple of 8 and if so the later
604 // implementation can perform a write of 8 elements at a time without worrying about
605 // writing beyond the end of the ArrayVec. If not, then fall back to a generic
606 // implementation. This check will be evaluated at compile time and one implementation
607 // or the other chosen.
608 if !N.is_multiple_of(8) {
609 let base = buffer_mem_start as *const i8;
610 for from_index in from_indices.iter() {
611 let ptr = base.add((*from_index << CLIB_LOG2_CACHE_LINE_BYTES) as usize)
612 as *mut vlib_buffer_t;
613 to.push_unchecked(BufferRef::from_ptr_mut(ptr));
614 }
615 return;
616 }
617
618 let mut len = from_indices.len();
619 len = next_multiple_of_pow2(len, 8);
620
621 let mut from_index = from_indices.as_ptr();
622 let mut to_ptr = to.as_mut_ptr();
623
624 while len >= 64 {
625 let mut from_index_x8_1 = u64x8::from_u32_ptr(from_index);
626 let mut from_index_x8_2 = u64x8::from_u32_ptr(from_index.add(8));
627 let mut from_index_x8_3 = u64x8::from_u32_ptr(from_index.add(2 * 8));
628 let mut from_index_x8_4 = u64x8::from_u32_ptr(from_index.add(3 * 8));
629 let mut from_index_x8_5 = u64x8::from_u32_ptr(from_index.add(4 * 8));
630 let mut from_index_x8_6 = u64x8::from_u32_ptr(from_index.add(5 * 8));
631 let mut from_index_x8_7 = u64x8::from_u32_ptr(from_index.add(6 * 8));
632 let mut from_index_x8_8 = u64x8::from_u32_ptr(from_index.add(7 * 8));
633
634 from_index_x8_1.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
635 from_index_x8_2.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
636 from_index_x8_3.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
637 from_index_x8_4.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
638 from_index_x8_5.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
639 from_index_x8_6.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
640 from_index_x8_7.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
641 from_index_x8_8.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
642
643 let buf_ptr_x8_1 = from_index_x8_1.add_u64(buffer_mem_start);
644 let buf_ptr_x8_2 = from_index_x8_2.add_u64(buffer_mem_start);
645 let buf_ptr_x8_3 = from_index_x8_3.add_u64(buffer_mem_start);
646 let buf_ptr_x8_4 = from_index_x8_4.add_u64(buffer_mem_start);
647 let buf_ptr_x8_5 = from_index_x8_5.add_u64(buffer_mem_start);
648 let buf_ptr_x8_6 = from_index_x8_6.add_u64(buffer_mem_start);
649 let buf_ptr_x8_7 = from_index_x8_7.add_u64(buffer_mem_start);
650 let buf_ptr_x8_8 = from_index_x8_8.add_u64(buffer_mem_start);
651
652 buf_ptr_x8_1.store(to_ptr as *mut u64);
653 buf_ptr_x8_2.store(to_ptr.add(8) as *mut u64);
654 buf_ptr_x8_3.store(to_ptr.add(2 * 8) as *mut u64);
655 buf_ptr_x8_4.store(to_ptr.add(3 * 8) as *mut u64);
656 buf_ptr_x8_5.store(to_ptr.add(4 * 8) as *mut u64);
657 buf_ptr_x8_6.store(to_ptr.add(5 * 8) as *mut u64);
658 buf_ptr_x8_7.store(to_ptr.add(6 * 8) as *mut u64);
659 buf_ptr_x8_8.store(to_ptr.add(7 * 8) as *mut u64);
660
661 to_ptr = to_ptr.add(64);
662 from_index = from_index.add(64);
663 len -= 64;
664 }
665
666 if likely(len >= 32) {
667 let mut from_index_x8_1 = u64x8::from_u32_ptr(from_index);
668 let mut from_index_x8_2 = u64x8::from_u32_ptr(from_index.add(8));
669 let mut from_index_x8_3 = u64x8::from_u32_ptr(from_index.add(2 * 8));
670 let mut from_index_x8_4 = u64x8::from_u32_ptr(from_index.add(3 * 8));
671
672 from_index_x8_1.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
673 from_index_x8_2.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
674 from_index_x8_3.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
675 from_index_x8_4.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
676
677 let buf_ptr_x8_1 = from_index_x8_1.add_u64(buffer_mem_start);
678 let buf_ptr_x8_2 = from_index_x8_2.add_u64(buffer_mem_start);
679 let buf_ptr_x8_3 = from_index_x8_3.add_u64(buffer_mem_start);
680 let buf_ptr_x8_4 = from_index_x8_4.add_u64(buffer_mem_start);
681
682 buf_ptr_x8_1.store(to_ptr as *mut u64);
683 buf_ptr_x8_2.store(to_ptr.add(8) as *mut u64);
684 buf_ptr_x8_3.store(to_ptr.add(2 * 8) as *mut u64);
685 buf_ptr_x8_4.store(to_ptr.add(3 * 8) as *mut u64);
686
687 to_ptr = to_ptr.add(32);
688 from_index = from_index.add(32);
689 len -= 32;
690 }
691
692 if likely(len >= 16) {
693 let mut from_index_x8_1 = u64x8::from_u32_ptr(from_index);
694 let mut from_index_x8_2 = u64x8::from_u32_ptr(from_index.add(8));
695
696 from_index_x8_1.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
697 from_index_x8_2.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
698
699 let buf_ptr_x8_1 = from_index_x8_1.add_u64(buffer_mem_start);
700 let buf_ptr_x8_2 = from_index_x8_2.add_u64(buffer_mem_start);
701
702 buf_ptr_x8_1.store(to_ptr as *mut u64);
703 buf_ptr_x8_2.store(to_ptr.add(8) as *mut u64);
704
705 to_ptr = to_ptr.add(16);
706 from_index = from_index.add(16);
707 len -= 16;
708 }
709
710 if likely(len > 0) {
711 let mut from_index_x8 = u64x8::from_u32_ptr(from_index);
712 from_index_x8.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
713 let buf_ptr_x8 = from_index_x8.add_u64(buffer_mem_start);
714 buf_ptr_x8.store(to_ptr as *mut u64);
715 }
716
717 to.set_len(from_indices.len());
718 }
719 }
720
721 /// Enqueues a slice of buffer indices to a next node
722 ///
723 /// This corresponds to the VPP C function `vlib_buffer_enqueue_to_next`.
724 ///
725 /// # Safety
726 ///
727 /// - The length of the from and next slices must match.
728 /// - The next node must have a `Vector` type of `u32` (or the C equivalent).
729 /// - The next node must have a `Scalar` type of `()` (or the C equivalent).
730 /// - The next node must have an `Aux` type of `()` (or the C equivalent).
731 /// - `vlib_buffer_func_main` must have been filled in with valid function pointers (which
732 /// will be done by VPP at initialisation time).
733 /// - The buffer state, such as `current_data` and `length` must be set according to the
734 /// preconditions of the next node.
735 /// - Each entry in the `from` slice must be a valid index to a buffer.
736 /// - Each entry in the `nexts` slice must be a valid next node index.
737 #[inline(always)]
738 pub unsafe fn buffer_enqueue_to_next<N, V: VectorBufferIndex>(
739 &self,
740 node: &mut NodeRuntimeRef<N>,
741 from: &[V],
742 nexts: &[u16],
743 ) {
744 debug_assert_eq!(from.len(), nexts.len());
745 // SAFETY: the caller asserts the function preconditions are true
746 unsafe {
747 (vlib_buffer_func_main
748 .buffer_enqueue_to_next_fn
749 .unwrap_unchecked())(
750 self.as_ptr(),
751 node.as_ptr(),
752 VectorBufferIndex::as_u32_slice(from).as_ptr().cast_mut(),
753 nexts.as_ptr() as *mut u16,
754 from.len() as u64,
755 )
756 }
757 }
758
759 /// Allocate a single buffer
760 ///
761 /// This corresponds to the VPP C API of `vlib_alloc_buffers`.
762 pub fn alloc_buffer(&self) -> Result<BufferWithContext<'_>, BufferAllocError> {
763 // SAFETY: we have a reference to self so the pointer must also be valid, we pass in a
764 // buffer pointer that is consistent with the number of buffers asked for, and on exit
765 // of the function either the buffer value is filled in with a valid index we have
766 // ownership of or not depending on the return value of the function.
767 unsafe {
768 let mut buffer = 0;
769 let res = vlib_helper_buffer_alloc(self.as_ptr(), &mut buffer, 1);
770 if res == 1 {
771 Ok(BufferWithContext::from_parts(buffer.into(), self))
772 } else {
773 Err(BufferAllocError)
774 }
775 }
776 }
777
778 /// Get the default data size for allocated buffers
779 ///
780 /// This corresponds to the VPP C API `vlib_buffer_get_default_data_size`.
781 pub fn buffer_default_data_size(&self) -> u32 {
782 // SAFETY: we have a reference to self so the pointer must also be valid, and MainRef
783 // creation preconditions mean that the buffer_main point must also be valid.
784 unsafe { (*(*self.as_ptr()).buffer_main).default_data_size }
785 }
786}
787
788#[cfg(test)]
789mod tests {
790 use arrayvec::ArrayVec;
791
792 use crate::{
793 bindings::{CLIB_LOG2_CACHE_LINE_BYTES, vlib_buffer_main_t, vlib_buffer_t, vlib_main_t},
794 vlib::{MainRef, node::FRAME_SIZE},
795 };
796
797 #[test]
798 fn get_buffers() {
799 let buffer = vlib_buffer_t::default();
800 // This is picked deliberately to not be 128 - 8 - 1 to be the worst case in terms of maximising the code
801 // paths that need to be taken
802 const BUFFERS_N: usize = 119;
803 let buffers = [buffer; BUFFERS_N];
804 let buffer_indices: ArrayVec<u32, 128> = (0..buffers.len() as u32)
805 .map(|n| {
806 n * (std::mem::size_of::<vlib_buffer_t>() as u32 >> CLIB_LOG2_CACHE_LINE_BYTES)
807 })
808 .collect();
809 let mut buffer_main = vlib_buffer_main_t {
810 buffer_mem_start: std::ptr::addr_of!(buffers) as u64,
811 buffer_mem_size: std::mem::size_of_val(&buffers) as u64,
812 ..vlib_buffer_main_t::default()
813 };
814 let mut main = vlib_main_t {
815 buffer_main: std::ptr::addr_of_mut!(buffer_main),
816 ..vlib_main_t::default()
817 };
818 // SAFETY: pointers used by MainRef::get_buffers are initialised correctly and valid for
819 // the duration of the call.
820 unsafe {
821 let mut to = ArrayVec::new();
822 let main_ref = MainRef::from_ptr_mut(std::ptr::addr_of_mut!(main));
823 main_ref.get_buffers::<(), FRAME_SIZE>(&buffer_indices, &mut to);
824 assert_eq!(to.len(), BUFFERS_N);
825 for (i, buf_ref) in to.iter().enumerate() {
826 assert!(
827 buf_ref.as_ptr().cast_const() == std::ptr::addr_of!(buffers[i]),
828 "Buffer index {i} pointers don't match: {:p} expected {:p}",
829 buf_ref.as_ptr(),
830 std::ptr::addr_of!(buffers[i])
831 );
832 }
833 }
834 }
835}