Skip to main content

vpp_plugin/vlib/
buffer.rs

1//! vlib buffer abstraction
2//!
3//! This module contains abstractions around VPP's `vlib_buffer_t` structure.
4//! It provides safe access to buffer fields and operations.
5//! It also includes buffer allocation and deallocation functions.
6//! The goal is to provide a safe and ergonomic interface for working with VPP buffers.
7
8use std::{fmt, hint::assert_unchecked, mem::ManuallyDrop, mem::MaybeUninit};
9
10use arrayvec::ArrayVec;
11use bitflags::bitflags;
12
13use crate::{
14    bindings::{
15        CLIB_LOG2_CACHE_LINE_BYTES, VLIB_BUFFER_EXT_HDR_VALID, VLIB_BUFFER_IS_TRACED,
16        VLIB_BUFFER_MIN_CHAIN_SEG_SIZE, VLIB_BUFFER_NEXT_PRESENT, VLIB_BUFFER_PRE_DATA_SIZE,
17        VLIB_BUFFER_TOTAL_LENGTH_VALID, vlib_add_trace, vlib_buffer_func_main, vlib_buffer_t,
18        vlib_buffer_t__bindgen_ty_1, vlib_buffer_t__bindgen_ty_1__bindgen_ty_1__bindgen_ty_1,
19        vlib_helper_buffer_alloc, vlib_helper_buffer_free,
20    },
21    const_assert,
22    vlib::{
23        self, MainRef,
24        node::{ErrorCounters, Node, NodeRuntimeRef, VectorBufferIndex},
25    },
26    vppinfra::{
27        cache::{prefetch_load, prefetch_store},
28        likely,
29    },
30};
31
32/// VPP buffer index
33#[repr(transparent)]
34#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
35pub struct BufferIndex(u32);
36
37impl BufferIndex {
38    /// Construct a new `BufferIndex`
39    pub const fn new(buffer: u32) -> Self {
40        Self(buffer)
41    }
42}
43
44impl From<u32> for BufferIndex {
45    fn from(value: u32) -> BufferIndex {
46        Self(value)
47    }
48}
49
50impl From<BufferIndex> for u32 {
51    fn from(value: BufferIndex) -> Self {
52        value.0
53    }
54}
55
56impl VectorBufferIndex for BufferIndex {
57    fn as_u32_slice(slice: &[Self]) -> &[u32] {
58        // SAFETY: BufferIndex is a repr(transparent) wrapper around u32 so the src and dst slice
59        // types have the same memory layout
60        unsafe { std::mem::transmute::<&[BufferIndex], &[u32]>(slice) }
61    }
62}
63
64bitflags! {
65    /// vlib buffer flags
66    #[repr(transparent)]
67    #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
68    pub struct BufferFlags: u32 {
69        /// Trace this buffer
70        const IS_TRACED = VLIB_BUFFER_IS_TRACED;
71        /// This is one buffer in a chain of buffers
72        const NEXT_PRESENT = VLIB_BUFFER_NEXT_PRESENT;
73        /// Total length is valid
74        const TOTAL_LENGTH_VALID = VLIB_BUFFER_TOTAL_LENGTH_VALID;
75        /// Contains external buffer manager header
76        const EXT_HDR_VALID = VLIB_BUFFER_EXT_HDR_VALID;
77
78        // Flags can be extended by user/vnet
79        const _ = !0;
80    }
81}
82
83/// Construct a user buffer flag
84///
85/// `n` must be less than 29 and greater than 0.
86pub const fn vlib_buffer_flag_user(n: u32) -> u32 {
87    assert!(n < 29 && n > 0);
88    1 << (32 - n)
89}
90
91/// Reference to a VPP buffer
92///
93/// A `&mut BufferRef<FeatureData>` is equivalent to a `vlib_buffer_t *` in C (a `*mut
94/// vlib_buffer_t` in Rust).
95#[repr(transparent)]
96pub struct BufferRef<FeatureData>(foreign_types::Opaque, std::marker::PhantomData<FeatureData>);
97
98impl<FeatureData> BufferRef<FeatureData> {
99    /// Create a `&BufferRef` from a raw pointer
100    ///
101    /// # Safety
102    ///
103    /// - The pointer must be a valid and properly initialised `vlib_buffer_t`.
104    /// - The pointer must stay valid and the contents must not be mutated for the duration of the
105    ///   lifetime of the returned object.
106    #[inline(always)]
107    pub unsafe fn from_ptr<'a>(ptr: *mut vlib_buffer_t) -> &'a Self {
108        // SAFETY: The safety requirements are documented in the function's safety comment.
109        unsafe { &*(ptr as *mut _) }
110    }
111
112    /// Create a `&mut BufferRef` from a raw pointer
113    ///
114    /// # Safety
115    ///
116    /// - The pointer must be a valid and properly initialised `vlib_buffer_t`.
117    /// - The pointer must stay valid and the contents must not be mutated for the duration of the
118    ///   lifetime of the returned object.
119    #[inline(always)]
120    pub unsafe fn from_ptr_mut<'a>(ptr: *mut vlib_buffer_t) -> &'a mut Self {
121        // SAFETY: The safety requirements are documented in the function's safety comment.
122        unsafe { &mut *(ptr as *mut _) }
123    }
124
125    /// Returns the raw pointer to the underlying `vlib_buffer_t`
126    #[inline(always)]
127    pub fn as_ptr(&self) -> *mut vlib_buffer_t {
128        self as *const _ as *mut _
129    }
130
131    #[inline(always)]
132    fn as_details(&self) -> &vlib_buffer_t__bindgen_ty_1 {
133        // SAFETY: since the reference to self is valid, so must be the pointer and it's safe to
134        // use the __bindgen_anon_1 union arm since the union is just present to force alignment
135        // Creation preconditions mean there are no aliased accesses to the buffer so it's fine
136        // to take a reference
137        unsafe { (*self.as_ptr()).__bindgen_anon_1.as_ref() }
138    }
139
140    #[inline(always)]
141    fn as_details_mut(&mut self) -> &mut vlib_buffer_t__bindgen_ty_1 {
142        // SAFETY: since the reference to self is valid, so must be the pointer and it's safe to
143        // use the __bindgen_anon_1 union arm since the union is just present to force alignment.
144        // Creation preconditions mean there are no aliased accesses to the buffer so it's fine
145        // to take a reference
146        unsafe { (*self.as_ptr()).__bindgen_anon_1.as_mut() }
147    }
148
149    #[inline(always)]
150    pub(crate) fn as_metadata(&self) -> &vlib_buffer_t__bindgen_ty_1__bindgen_ty_1__bindgen_ty_1 {
151        // SAFETY: since the reference to self is valid, so must be the pointer and it's safe to
152        // use the __bindgen_anon_1 union arm since the union is just present to force alignment
153        // Creation preconditions mean there are no aliased accesses to the buffer so it's fine
154        // to take a reference
155        unsafe { self.as_details().__bindgen_anon_1.__bindgen_anon_1.as_ref() }
156    }
157
158    #[inline(always)]
159    pub(crate) fn as_metadata_mut(
160        &mut self,
161    ) -> &mut vlib_buffer_t__bindgen_ty_1__bindgen_ty_1__bindgen_ty_1 {
162        // SAFETY: since the reference to self is valid, so must be the pointer and it's safe to
163        // use the __bindgen_anon_1 union arm since the union is just present to force alignment.
164        // Creation preconditions mean there are no aliased accesses to the buffer so it's fine
165        // to take a reference
166        unsafe {
167            self.as_details_mut()
168                .__bindgen_anon_1
169                .__bindgen_anon_1
170                .as_mut()
171        }
172    }
173
174    fn data(&self) -> *const u8 {
175        self.as_details().data.as_ptr()
176    }
177
178    fn current_data_offset(&self) -> i16 {
179        self.as_metadata().current_data
180    }
181
182    fn current_data_offset_mut(&mut self) -> &mut i16 {
183        &mut self.as_metadata_mut().current_data
184    }
185
186    /// Current length
187    ///
188    /// Typically, this is the amount of packet data remaining from [`Self::current_ptr_mut`].
189    pub fn current_length(&self) -> u16 {
190        self.as_metadata().current_length
191    }
192
193    fn current_length_mut(&mut self) -> &mut u16 {
194        &mut self.as_metadata_mut().current_length
195    }
196
197    /// Get the flags set for this buffer
198    #[inline(always)]
199    pub fn flags(&self) -> BufferFlags {
200        BufferFlags::from_bits_retain(self.as_metadata().flags)
201    }
202
203    /// Set the flags for this buffer
204    ///
205    /// # Safety
206    ///
207    /// [`BufferFlags::NEXT_PRESENT`] must not be set unless there is a next buffer in the chain.
208    /// [`BufferFlags::EXT_HDR_VALID`] must not be set or cleared unless the external buffer manager header is valid
209    /// or not valid respectively.
210    ///
211    #[inline(always)]
212    pub unsafe fn set_flags(&mut self, flags: BufferFlags) {
213        self.as_metadata_mut().flags = flags.bits()
214    }
215
216    /// Get a pointer to the current data
217    ///
218    /// This corresponds to the VPP C API `vlib_buffer_get_current`.
219    ///
220    /// # Usage guidance
221    ///
222    /// Note that the pointer returned may point to uninitialised data depending on the context.
223    /// In addition, depending on the context, the amount of remaining data may be less than
224    /// expected.
225    /// Finally, if remaining data is sufficient and it's initialised it may not have been validated
226    /// so care must be taken in determining whether or not lengths in the headers can be trusted.
227    pub fn current_ptr_mut(&mut self) -> *mut u8 {
228        let data = self.data().cast_mut();
229        let current_data = self.current_data_offset();
230
231        debug_assert!(current_data >= -(VLIB_BUFFER_PRE_DATA_SIZE as i16));
232
233        // SAFETY: current_data is asserted to be valid and point into valid (but possibly
234        // unintialised) data or pre_data.
235        unsafe { data.offset(current_data as isize) }
236    }
237
238    /// Check if the buffer has space to advance `l` bytes
239    ///
240    /// This corresponds to the VPP C API `vlib_buffer_has_space`.
241    pub fn has_space(&self, l: i16) -> bool {
242        self.current_length() >= l as u16
243    }
244
245    /// Advance the current data pointer by `l` bytes
246    ///
247    /// This corresponds to the VPP C API `vlib_buffer_advance`.
248    ///
249    /// # Safety
250    ///
251    /// - If `l` is positive, the buffer must have at least `l` bytes of data remaining.
252    /// - If `l` is negative, the current data offset must be at least `-l` bytes from the start of
253    ///   the buffer's data area (including pre-data).
254    pub unsafe fn advance(&mut self, l: i16) {
255        debug_assert!(l < 0 || self.current_length() >= l as u16);
256        debug_assert!(
257            l >= 0 || self.current_data_offset() + VLIB_BUFFER_PRE_DATA_SIZE as i16 >= -l
258        );
259
260        *self.current_data_offset_mut() += l;
261        if l >= 0 {
262            *self.current_length_mut() -= l as u16;
263        } else {
264            *self.current_length_mut() += -l as u16;
265        }
266
267        debug_assert!(
268            !self.flags().contains(BufferFlags::NEXT_PRESENT)
269                || self.current_length() >= VLIB_BUFFER_MIN_CHAIN_SEG_SIZE as u16
270        );
271    }
272
273    /// Append uninitialised data to the end of the current data.
274    ///
275    /// Returns a pointer to the start of the newly appended uninitialised data.
276    ///
277    /// This corresponds to the VPP C function `vlib_buffer_put_uninit`.
278    ///
279    /// # Safety
280    ///
281    /// The current data plus the space requested must not exceed the data size of the buffer
282    /// given during allocation. See [`super::MainRef::buffer_default_data_size`] for buffers
283    /// allocated by [`super::MainRef::alloc_buffer`].
284    ///
285    /// The caller must ensure that the data is correctly initialised before passing the buffer to
286    /// code that assumes it is correctly initialised, such as enqueing the buffer another node.
287    pub unsafe fn put_uninit(&mut self, size: u16) -> *mut u8 {
288        let p = self.tail_mut();
289        *self.current_length_mut() += size;
290        p
291    }
292
293    /// Get a pointer to the end of the current data
294    ///
295    /// This corresponds to the VPP C function `vlib_buffer_get_tail`.
296    pub fn tail_mut(&mut self) -> *mut u8 {
297        let data = self.data().cast_mut();
298        let current_data = self.current_data_offset();
299
300        debug_assert!(current_data >= -(VLIB_BUFFER_PRE_DATA_SIZE as i16));
301
302        // SAFETY: current_data and current_length are asserted to be valid and `current_data +
303        // current_length` asserted to point to the end of valid (but possibly unintialised) data
304        // or pre_data.
305        unsafe {
306            let ptr = data.offset(current_data as isize);
307            ptr.add(self.current_length() as usize)
308        }
309    }
310
311    /// Add trace data to this buffer
312    pub fn add_trace<N: Node>(
313        &mut self,
314        vm: &MainRef,
315        node: &NodeRuntimeRef<N>,
316    ) -> &mut MaybeUninit<N::TraceData> {
317        // SAFETY: pointers are valid and the uninitialised data that is returned cannot be read
318        // by safe code
319        unsafe {
320            &mut *(vlib_add_trace(
321                vm.as_ptr(),
322                node.as_ptr(),
323                self.as_ptr(),
324                std::mem::size_of::<N::TraceData>() as u32,
325            ) as *mut MaybeUninit<N::TraceData>)
326        }
327    }
328
329    /// Set an error reason
330    ///
331    /// This is typically done before sending the packet to the `drop` node, where it use the
332    /// value to display the reason in traces and automatically increment the per-node, per-error
333    /// counter for the error.
334    pub fn set_error<N: Node>(&mut self, node: &NodeRuntimeRef<N>, error: N::Errors) {
335        // SAFETY: vlib_node_runtime_t::errors is sized according to the number of error values
336        // and it is a precondition of the Errors trait that the value return by into_u16() cannot
337        // be greater than or equal to the declared number of error values.
338        unsafe {
339            let error_value = (*node.as_ptr()).errors.add(error.into_u16() as usize);
340            self.as_metadata_mut().error = *error_value;
341        }
342    }
343
344    /// Get the total length of the buffer chain not including the first buffer
345    #[inline(always)]
346    pub fn total_length_not_including_first_buffer(&self) -> u32 {
347        debug_assert!(self.flags().contains(BufferFlags::TOTAL_LENGTH_VALID));
348        self.as_details().total_length_not_including_first_buffer
349    }
350
351    /// Get the total length of the buffer chain from the current offset
352    ///
353    /// Note that this doesn't take into account any bytes that have been [`Self::advance()`]d
354    /// over.
355    #[inline(always)]
356    pub fn length_in_chain(&self, vm: &vlib::MainRef) -> u64 {
357        let len = self.current_length();
358
359        if likely(!self.flags().contains(BufferFlags::NEXT_PRESENT)) {
360            return len as u64;
361        }
362
363        if likely(self.flags().contains(BufferFlags::TOTAL_LENGTH_VALID)) {
364            return len as u64 + self.total_length_not_including_first_buffer() as u64;
365        }
366
367        // SAFETY: The buffer pointer is valid and the function is called in a valid context.
368        unsafe {
369            crate::bindings::vlib_buffer_length_in_chain_slow_path(vm.as_ptr(), self.as_ptr())
370        }
371    }
372
373    /// Hint to the CPU to prefetch the buffer header for read access.
374    ///
375    /// This is a performance hint that attempts to bring the buffer header into the CPU cache
376    /// prior to reading fields from it. It does not affect program semantics and may be a no-op
377    /// on some platforms. Use this when you will shortly read header fields and want to reduce
378    /// cache miss latency.
379    pub fn prefetch_header_load(&self) {
380        prefetch_load(self.as_ptr());
381    }
382
383    /// Hint to the CPU to prefetch the buffer header for write access.
384    ///
385    /// Similar to `prefetch_header_load` but indicates imminent writes to the header. This is a
386    /// performance optimization only and does not change observable behaviour other than timing.
387    pub fn prefetch_header_store(&self) {
388        prefetch_store(self.as_ptr());
389    }
390
391    /// Hint to the CPU to prefetch the buffer data area for read access.
392    ///
393    /// This brings the buffer's data into cache in preparation for reading the packet payload.
394    /// It is a non-semantic performance hint and may be ignored on some architectures. Use this
395    /// when you will shortly read packet data and want to reduce cache miss latency.
396    pub fn prefetch_data_load(&self) {
397        prefetch_load(&self.as_details().data);
398    }
399
400    /// Hint to the CPU to prefetch the buffer data area for write access.
401    ///
402    /// Similar to `prefetch_data_load` but indicates the caller will write into the data area.
403    /// This is a cache-warming hint to reduce latency on subsequent stores.
404    pub fn prefetch_data_store(&self) {
405        prefetch_store(&self.as_details().data);
406    }
407}
408
409/// Owned buffer (with context)
410///
411/// The `&MainRef` context is necessary to be able to free the buffer on drop.
412pub struct BufferWithContext<'a> {
413    buffer: u32,
414    vm: &'a MainRef,
415}
416
417impl<'a> BufferWithContext<'a> {
418    /// Creates a `BufferWithContext` directly from a buffer index and a main reference
419    ///
420    /// # Safety
421    /// - The buffer index must be valid and the caller must have ownership of the buffer it
422    ///   corresponds to.
423    pub unsafe fn from_parts(buffer: BufferIndex, vm: &'a MainRef) -> Self {
424        Self {
425            buffer: buffer.0,
426            vm,
427        }
428    }
429
430    /// Decomposes a `BufferWithContext` into its component parts
431    ///
432    /// After calling this the caller is responsible for ensuring the buffer gets freed, either
433    /// by calling [`BufferWithContext::from_parts`] or by passing it into another function which
434    /// takes ownership of it and eventually causes it to be freed.
435    pub fn into_parts(self) -> (BufferIndex, &'a MainRef) {
436        let me = ManuallyDrop::new(self);
437        (BufferIndex(me.buffer), me.vm)
438    }
439
440    /// Get a mutable reference to the buffer
441    pub fn as_buffer_ref(&mut self) -> &mut BufferRef<()> {
442        let from = &[self.buffer];
443        let mut b: ArrayVec<_, 1> = ArrayVec::new();
444        // SAFETY: capacity of b equals the length of from, `self.buffer` is a valid index and we
445        // force FeatureData to `()` since it isn't known and the buffer cannot be part of a
446        // feature arc.
447        unsafe {
448            self.vm.get_buffers(from, &mut b);
449        }
450        b.remove(0)
451    }
452}
453
454impl Drop for BufferWithContext<'_> {
455    fn drop(&mut self) {
456        // SAFETY: we have a reference to MainRef so the pointer must be valid, we pass in a
457        // pointer to buffers consistent with the number of buffers passed in, and self.buffer
458        // is a valid buffer index that we have ownership of.
459        unsafe {
460            vlib_helper_buffer_free(self.vm.as_ptr(), &mut self.buffer, 1);
461        }
462    }
463}
464
465/// Buffer allocation error
466#[derive(Copy, Clone, PartialEq, Eq, Debug)]
467pub struct BufferAllocError;
468
469impl fmt::Display for BufferAllocError {
470    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
471        write!(f, "buffer allocation error")
472    }
473}
474
475impl std::error::Error for BufferAllocError {}
476
477/// u64 x 8
478///
479/// This type exists to strongly hint to the compiler that it should emit vector instructions.
480///
481/// In the future, the implementation might be changed to use standard library portable SIMD once
482/// stabilised (https://github.com/rust-lang/rust/issues/86656), or use arch-specific intrinsics
483/// (if evidenced by high-enough performance improvement).
484#[allow(non_camel_case_types)]
485pub(crate) struct u64x8([u64; 8]);
486
487impl u64x8 {
488    /// Construct a `u64x8` from an array of 8 `u64`s
489    #[inline(always)]
490    pub(crate) fn from_array(a: [u64; 8]) -> Self {
491        Self(a)
492    }
493
494    /// Construct a `u64x8` from a pointer to 8 `u32`s
495    #[inline(always)]
496    pub(crate) unsafe fn from_u32_ptr(ptr: *const u32) -> Self {
497        // SAFETY: The caller must ensure the pointer is valid for reading 8 u32 values.
498        unsafe {
499            Self([
500                *ptr.add(0) as u64,
501                *ptr.add(1) as u64,
502                *ptr.add(2) as u64,
503                *ptr.add(3) as u64,
504                *ptr.add(4) as u64,
505                *ptr.add(5) as u64,
506                *ptr.add(6) as u64,
507                *ptr.add(7) as u64,
508            ])
509        }
510    }
511
512    /// Shift each element to the left by a given constant value, assigning the result to `self`
513    #[inline(always)]
514    pub(crate) fn shift_elements_left<const OFFSET: u32>(&mut self) {
515        for a in &mut self.0 {
516            *a <<= OFFSET;
517        }
518    }
519
520    /// Add a given value to each element, returning a new u64x8 with the result
521    #[inline(always)]
522    pub(crate) fn add_u64(&self, value: u64) -> Self {
523        Self::from_array([
524            self.0[0] + value,
525            self.0[1] + value,
526            self.0[2] + value,
527            self.0[3] + value,
528            self.0[4] + value,
529            self.0[5] + value,
530            self.0[6] + value,
531            self.0[7] + value,
532        ])
533    }
534
535    /// Write 8 contiguous elements starting from `ptr`
536    #[inline(always)]
537    pub(crate) unsafe fn store(&self, ptr: *mut u64) {
538        // SAFETY: The caller must ensure the pointer is valid for writing 8 u64 values.
539        unsafe {
540            *ptr.add(0) = self.0[0];
541            *ptr.add(1) = self.0[1];
542            *ptr.add(2) = self.0[2];
543            *ptr.add(3) = self.0[3];
544            *ptr.add(4) = self.0[4];
545            *ptr.add(5) = self.0[5];
546            *ptr.add(6) = self.0[6];
547            *ptr.add(7) = self.0[7];
548        }
549    }
550}
551
552/// Round a value up to the next multiple of the given power-of-two
553const fn next_multiple_of_pow2(val: usize, pow2: usize) -> usize {
554    debug_assert!(pow2.is_power_of_two());
555    (val + pow2 - 1) & !(pow2 - 1)
556}
557
558impl MainRef {
559    /// Get pointers to buffers for the given buffer indices, writing them into the provided `to` arrayvec.
560    ///
561    /// This is similar to `vlib_get_buffers` in the C API.
562    ///
563    /// Note that although it would be more idiomatic to return an `ArrayVec` directly, this
564    /// method takes a mutable reference to an `ArrayVec` to avoid an unnecessary copy when
565    /// returning.
566    ///
567    /// # Safety
568    ///
569    /// - The caller must ensure that `to` has enough capacity to hold all the buffers
570    ///   corresponding to the indices in `from_indices`.
571    /// - Each index in `from_indices` must be valid and the caller must have ownership of the
572    ///   buffer it corresponds to.
573    /// - Each buffer's `current_config_index` must be consistent with the `FeatureData` type. If
574    ///   they are not known (i.e. because the caller the node isn't being executed in a feature
575    ///   arc), FeatureData should be a zero-sized type such as `()`.
576    /// - The capacity of `from_indices` must be a multiple of 8 (note though that the length is
577    ///   allowed not to be). In other words, it must be valid to read multiples of 8 from the
578    ///   underlying memory (possibly returning uninitialised or stale data) without faulting.
579    #[inline(always)]
580    pub unsafe fn get_buffers<'a, 'me, 'buf: 'me, FeatureData, const N: usize>(
581        &'me self,
582        from_indices: &'a [u32],
583        to: &mut ArrayVec<&'buf mut BufferRef<FeatureData>, N>,
584    ) {
585        // SAFETY: The safety requirements are documented in the function's safety comment.
586        unsafe {
587            debug_assert!(from_indices.len() <= N);
588            assert_unchecked(from_indices.len() <= N);
589            // The vector operations below compute a pointer in terms of u64, i.e. assume that u64
590            // is a pointer-sized type the same as usize.
591            const_assert!(std::mem::size_of::<usize>() == std::mem::size_of::<u64>());
592
593            #[cfg(debug_assertions)]
594            for from_index in from_indices {
595                let buffer_mem_size = (*(*self.as_ptr()).buffer_main).buffer_mem_size;
596                debug_assert!(
597                    ((*from_index << CLIB_LOG2_CACHE_LINE_BYTES) as u64) < buffer_mem_size
598                );
599            }
600
601            let buffer_mem_start = (*(*self.as_ptr()).buffer_main).buffer_mem_start;
602
603            // Check for the ArrayVec capacity being a multiple of 8 and if so the later
604            // implementation can perform a write of 8 elements at a time without worrying about
605            // writing beyond the end of the ArrayVec. If not, then fall back to a generic
606            // implementation. This check will be evaluated at compile time and one implementation
607            // or the other chosen.
608            if !N.is_multiple_of(8) {
609                let base = buffer_mem_start as *const i8;
610                for from_index in from_indices.iter() {
611                    let ptr = base.add((*from_index << CLIB_LOG2_CACHE_LINE_BYTES) as usize)
612                        as *mut vlib_buffer_t;
613                    to.push_unchecked(BufferRef::from_ptr_mut(ptr));
614                }
615                return;
616            }
617
618            let mut len = from_indices.len();
619            len = next_multiple_of_pow2(len, 8);
620
621            let mut from_index = from_indices.as_ptr();
622            let mut to_ptr = to.as_mut_ptr();
623
624            while len >= 64 {
625                let mut from_index_x8_1 = u64x8::from_u32_ptr(from_index);
626                let mut from_index_x8_2 = u64x8::from_u32_ptr(from_index.add(8));
627                let mut from_index_x8_3 = u64x8::from_u32_ptr(from_index.add(2 * 8));
628                let mut from_index_x8_4 = u64x8::from_u32_ptr(from_index.add(3 * 8));
629                let mut from_index_x8_5 = u64x8::from_u32_ptr(from_index.add(4 * 8));
630                let mut from_index_x8_6 = u64x8::from_u32_ptr(from_index.add(5 * 8));
631                let mut from_index_x8_7 = u64x8::from_u32_ptr(from_index.add(6 * 8));
632                let mut from_index_x8_8 = u64x8::from_u32_ptr(from_index.add(7 * 8));
633
634                from_index_x8_1.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
635                from_index_x8_2.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
636                from_index_x8_3.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
637                from_index_x8_4.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
638                from_index_x8_5.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
639                from_index_x8_6.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
640                from_index_x8_7.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
641                from_index_x8_8.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
642
643                let buf_ptr_x8_1 = from_index_x8_1.add_u64(buffer_mem_start);
644                let buf_ptr_x8_2 = from_index_x8_2.add_u64(buffer_mem_start);
645                let buf_ptr_x8_3 = from_index_x8_3.add_u64(buffer_mem_start);
646                let buf_ptr_x8_4 = from_index_x8_4.add_u64(buffer_mem_start);
647                let buf_ptr_x8_5 = from_index_x8_5.add_u64(buffer_mem_start);
648                let buf_ptr_x8_6 = from_index_x8_6.add_u64(buffer_mem_start);
649                let buf_ptr_x8_7 = from_index_x8_7.add_u64(buffer_mem_start);
650                let buf_ptr_x8_8 = from_index_x8_8.add_u64(buffer_mem_start);
651
652                buf_ptr_x8_1.store(to_ptr as *mut u64);
653                buf_ptr_x8_2.store(to_ptr.add(8) as *mut u64);
654                buf_ptr_x8_3.store(to_ptr.add(2 * 8) as *mut u64);
655                buf_ptr_x8_4.store(to_ptr.add(3 * 8) as *mut u64);
656                buf_ptr_x8_5.store(to_ptr.add(4 * 8) as *mut u64);
657                buf_ptr_x8_6.store(to_ptr.add(5 * 8) as *mut u64);
658                buf_ptr_x8_7.store(to_ptr.add(6 * 8) as *mut u64);
659                buf_ptr_x8_8.store(to_ptr.add(7 * 8) as *mut u64);
660
661                to_ptr = to_ptr.add(64);
662                from_index = from_index.add(64);
663                len -= 64;
664            }
665
666            if likely(len >= 32) {
667                let mut from_index_x8_1 = u64x8::from_u32_ptr(from_index);
668                let mut from_index_x8_2 = u64x8::from_u32_ptr(from_index.add(8));
669                let mut from_index_x8_3 = u64x8::from_u32_ptr(from_index.add(2 * 8));
670                let mut from_index_x8_4 = u64x8::from_u32_ptr(from_index.add(3 * 8));
671
672                from_index_x8_1.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
673                from_index_x8_2.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
674                from_index_x8_3.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
675                from_index_x8_4.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
676
677                let buf_ptr_x8_1 = from_index_x8_1.add_u64(buffer_mem_start);
678                let buf_ptr_x8_2 = from_index_x8_2.add_u64(buffer_mem_start);
679                let buf_ptr_x8_3 = from_index_x8_3.add_u64(buffer_mem_start);
680                let buf_ptr_x8_4 = from_index_x8_4.add_u64(buffer_mem_start);
681
682                buf_ptr_x8_1.store(to_ptr as *mut u64);
683                buf_ptr_x8_2.store(to_ptr.add(8) as *mut u64);
684                buf_ptr_x8_3.store(to_ptr.add(2 * 8) as *mut u64);
685                buf_ptr_x8_4.store(to_ptr.add(3 * 8) as *mut u64);
686
687                to_ptr = to_ptr.add(32);
688                from_index = from_index.add(32);
689                len -= 32;
690            }
691
692            if likely(len >= 16) {
693                let mut from_index_x8_1 = u64x8::from_u32_ptr(from_index);
694                let mut from_index_x8_2 = u64x8::from_u32_ptr(from_index.add(8));
695
696                from_index_x8_1.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
697                from_index_x8_2.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
698
699                let buf_ptr_x8_1 = from_index_x8_1.add_u64(buffer_mem_start);
700                let buf_ptr_x8_2 = from_index_x8_2.add_u64(buffer_mem_start);
701
702                buf_ptr_x8_1.store(to_ptr as *mut u64);
703                buf_ptr_x8_2.store(to_ptr.add(8) as *mut u64);
704
705                to_ptr = to_ptr.add(16);
706                from_index = from_index.add(16);
707                len -= 16;
708            }
709
710            if likely(len > 0) {
711                let mut from_index_x8 = u64x8::from_u32_ptr(from_index);
712                from_index_x8.shift_elements_left::<CLIB_LOG2_CACHE_LINE_BYTES>();
713                let buf_ptr_x8 = from_index_x8.add_u64(buffer_mem_start);
714                buf_ptr_x8.store(to_ptr as *mut u64);
715            }
716
717            to.set_len(from_indices.len());
718        }
719    }
720
721    /// Enqueues a slice of buffer indices to a next node
722    ///
723    /// This corresponds to the VPP C function `vlib_buffer_enqueue_to_next`.
724    ///
725    /// # Safety
726    ///
727    /// - The length of the from and next slices must match.
728    /// - The next node must have a `Vector` type of `u32` (or the C equivalent).
729    /// - The next node must have a `Scalar` type of `()` (or the C equivalent).
730    /// - The next node must have an `Aux` type of `()` (or the C equivalent).
731    /// - `vlib_buffer_func_main` must have been filled in with valid function pointers (which
732    ///   will be done by VPP at initialisation time).
733    /// - The buffer state, such as `current_data` and `length` must be set according to the
734    ///   preconditions of the next node.
735    /// - Each entry in the `from` slice must be a valid index to a buffer.
736    /// - Each entry in the `nexts` slice must be a valid next node index.
737    #[inline(always)]
738    pub unsafe fn buffer_enqueue_to_next<N, V: VectorBufferIndex>(
739        &self,
740        node: &mut NodeRuntimeRef<N>,
741        from: &[V],
742        nexts: &[u16],
743    ) {
744        debug_assert_eq!(from.len(), nexts.len());
745        // SAFETY: the caller asserts the function preconditions are true
746        unsafe {
747            (vlib_buffer_func_main
748                .buffer_enqueue_to_next_fn
749                .unwrap_unchecked())(
750                self.as_ptr(),
751                node.as_ptr(),
752                VectorBufferIndex::as_u32_slice(from).as_ptr().cast_mut(),
753                nexts.as_ptr() as *mut u16,
754                from.len() as u64,
755            )
756        }
757    }
758
759    /// Allocate a single buffer
760    ///
761    /// This corresponds to the VPP C API of `vlib_alloc_buffers`.
762    pub fn alloc_buffer(&self) -> Result<BufferWithContext<'_>, BufferAllocError> {
763        // SAFETY: we have a reference to self so the pointer must also be valid, we pass in a
764        // buffer pointer that is consistent with the number of buffers asked for, and on exit
765        // of the function either the buffer value is filled in with a valid index we have
766        // ownership of or not depending on the return value of the function.
767        unsafe {
768            let mut buffer = 0;
769            let res = vlib_helper_buffer_alloc(self.as_ptr(), &mut buffer, 1);
770            if res == 1 {
771                Ok(BufferWithContext::from_parts(buffer.into(), self))
772            } else {
773                Err(BufferAllocError)
774            }
775        }
776    }
777
778    /// Get the default data size for allocated buffers
779    ///
780    /// This corresponds to the VPP C API `vlib_buffer_get_default_data_size`.
781    pub fn buffer_default_data_size(&self) -> u32 {
782        // SAFETY: we have a reference to self so the pointer must also be valid, and MainRef
783        // creation preconditions mean that the buffer_main point must also be valid.
784        unsafe { (*(*self.as_ptr()).buffer_main).default_data_size }
785    }
786}
787
788#[cfg(test)]
789mod tests {
790    use arrayvec::ArrayVec;
791
792    use crate::{
793        bindings::{CLIB_LOG2_CACHE_LINE_BYTES, vlib_buffer_main_t, vlib_buffer_t, vlib_main_t},
794        vlib::{MainRef, node::FRAME_SIZE},
795    };
796
797    #[test]
798    fn get_buffers() {
799        let buffer = vlib_buffer_t::default();
800        // This is picked deliberately to not be 128 - 8 - 1 to be the worst case in terms of maximising the code
801        // paths that need to be taken
802        const BUFFERS_N: usize = 119;
803        let buffers = [buffer; BUFFERS_N];
804        let buffer_indices: ArrayVec<u32, 128> = (0..buffers.len() as u32)
805            .map(|n| {
806                n * (std::mem::size_of::<vlib_buffer_t>() as u32 >> CLIB_LOG2_CACHE_LINE_BYTES)
807            })
808            .collect();
809        let mut buffer_main = vlib_buffer_main_t {
810            buffer_mem_start: std::ptr::addr_of!(buffers) as u64,
811            buffer_mem_size: std::mem::size_of_val(&buffers) as u64,
812            ..vlib_buffer_main_t::default()
813        };
814        let mut main = vlib_main_t {
815            buffer_main: std::ptr::addr_of_mut!(buffer_main),
816            ..vlib_main_t::default()
817        };
818        // SAFETY: pointers used by MainRef::get_buffers are initialised correctly and valid for
819        // the duration of the call.
820        unsafe {
821            let mut to = ArrayVec::new();
822            let main_ref = MainRef::from_ptr_mut(std::ptr::addr_of_mut!(main));
823            main_ref.get_buffers::<(), FRAME_SIZE>(&buffer_indices, &mut to);
824            assert_eq!(to.len(), BUFFERS_N);
825            for (i, buf_ref) in to.iter().enumerate() {
826                assert!(
827                    buf_ref.as_ptr().cast_const() == std::ptr::addr_of!(buffers[i]),
828                    "Buffer index {i} pointers don't match: {:p} expected {:p}",
829                    buf_ref.as_ptr(),
830                    std::ptr::addr_of!(buffers[i])
831                );
832            }
833        }
834    }
835}