axvirtio_common/pci/transport/
mod.rs

1//! VirtIO PCI common configuration transport.
2//!
3//! This module owns the transport state that is independent of a particular
4//! PCI implementation: feature negotiation, queue configuration, device
5//! configuration access, and queue notification.  The caller supplies guest
6//! memory only while processing a queue notification.  In particular, merely
7//! programming queue registers does not dereference guest memory.
8
9use alloc::{format, sync::Arc};
10
11use ax_sync::SpinLock;
12use axdevice_base::{AccessWidth, DeviceError, DeviceResult};
13
14use crate::{
15    GuestMemory, NoGuestMemoryAccessor, VirtioDeviceID, VirtioError, VirtioQueue, map_virtio_error,
16    pci::{InterruptTransition, VirtioPciInterruptCoordinator},
17};
18
19mod access;
20mod queue;
21mod reset;
22mod state;
23mod transition;
24
25pub use state::ActivityPermit;
26use state::{QueueActivity, QueueState, TransportState};
27use transition::InterruptPublicationKind;
28pub use transition::{
29    InterruptPublicationRequest, InterruptTransitionIntent, InterruptTransitionRequest,
30    QueueNotification, VirtioQueueGeneration,
31};
32
33pub(super) const COMMON_CONFIG_SIZE: u64 = 0x38;
34pub(super) const NOTIFY_CONFIG_OFFSET: u64 = 0x100;
35pub(super) const ISR_CONFIG_OFFSET: u64 = 0x200;
36pub(super) const DEVICE_CONFIG_OFFSET: u64 = 0x300;
37pub(super) const RESET_DRAIN_SPIN_LIMIT: usize = 1 << 20;
38
39pub(super) const DEVICE_FEATURE_SELECT: u64 = 0x00;
40pub(super) const DEVICE_FEATURE: u64 = 0x04;
41pub(super) const DRIVER_FEATURE_SELECT: u64 = 0x08;
42pub(super) const DRIVER_FEATURE: u64 = 0x0c;
43pub(super) const MSIX_CONFIG: u64 = 0x10;
44pub(super) const NUM_QUEUES: u64 = 0x12;
45pub(super) const DEVICE_STATUS: u64 = 0x14;
46pub(super) const CONFIG_GENERATION: u64 = 0x15;
47pub(super) const QUEUE_SELECT: u64 = 0x16;
48pub(super) const QUEUE_SIZE: u64 = 0x18;
49pub(super) const QUEUE_MSIX_VECTOR: u64 = 0x1a;
50pub(super) const QUEUE_ENABLE: u64 = 0x1c;
51pub(super) const QUEUE_NOTIFY_OFF: u64 = 0x1e;
52pub(super) const QUEUE_DESC: u64 = 0x20;
53pub(super) const QUEUE_DRIVER: u64 = 0x28;
54pub(super) const QUEUE_DEVICE: u64 = 0x30;
55
56/// Result of a device-specific queue notification.
57#[derive(Debug, Clone, Copy, PartialEq, Eq)]
58pub enum QueueNotifyOutcome {
59    /// The device had no request to process.
60    Idle,
61    /// The device completed synchronously.
62    Completed {
63        /// Whether the used-ring update requires a guest interrupt.
64        notify: bool,
65    },
66    /// The device handed work to an asynchronous backend.
67    Deferred {
68        /// Whether the eventual used-ring update requires a guest interrupt.
69        notify: bool,
70    },
71}
72
73/// Device-specific operations needed by the common VirtIO PCI transport.
74///
75/// The trait deliberately has no PCI or operating-system types beyond the
76/// typed access width and guest-memory capability.  An endpoint adapter can
77/// therefore expose this transport through any PCI resource and interrupt
78/// implementation.
79pub trait VirtioDeviceCore: Send + Sync {
80    /// VirtIO device type advertised by the endpoint.
81    fn device_type(&self) -> VirtioDeviceID;
82
83    /// Feature bits offered by the endpoint.
84    fn device_features(&self) -> u64;
85
86    /// Maximum number of queues exposed by the endpoint.
87    fn queue_num_max(&self) -> u16 {
88        1
89    }
90
91    /// Maximum size of each queue.
92    fn queue_size_max(&self) -> u16;
93
94    /// Size of the device-specific configuration space.
95    fn device_config_size(&self) -> u32;
96
97    /// Read the device-specific configuration space.
98    fn read_device_config(&self, offset: u64, width: AccessWidth) -> DeviceResult<u64>;
99
100    /// Write the device-specific configuration space.
101    fn write_device_config(&self, offset: u64, width: AccessWidth, value: u64) -> DeviceResult;
102
103    /// Process one queue notification using the caller's scoped memory grant.
104    fn notify_queue(
105        &self,
106        queue: &mut VirtioQueue<NoGuestMemoryAccessor>,
107        memory: &mut dyn GuestMemory,
108    ) -> DeviceResult<QueueNotifyOutcome>;
109
110    /// Whether queue processing can complete asynchronously.
111    fn requires_deferred_processing(&self) -> bool {
112        false
113    }
114
115    /// Reset device-specific state.
116    fn reset(&self) -> DeviceResult {
117        Ok(())
118    }
119}
120
121/// Side effect produced by a common-config write.
122pub enum VirtioPciWriteOutcome {
123    /// No work is required by the endpoint adapter.
124    None,
125    /// The guest notified a queue and the core processed it.
126    QueueNotified(QueueNotification),
127    /// The transport was reset by writing zero to device status.
128    Reset {
129        /// Physical interrupt-line transition required after reset.
130        interrupt: InterruptTransition,
131    },
132    /// Queue processing failed after admission and requires a device reset.
133    Fault {
134        /// The processing error returned to the guest-facing dispatcher.
135        error: DeviceError,
136        /// Config-change ISR publication retained through terminal IRQ handling.
137        publication: InterruptPublicationRequest,
138    },
139}
140
141/// Common VirtIO PCI transport state machine.
142pub struct VirtioPciTransport<D: VirtioDeviceCore> {
143    core: D,
144    state: SpinLock<TransportState>,
145    interrupts: Arc<VirtioPciInterruptCoordinator>,
146    activity: Arc<QueueActivity>,
147    device_config_size: u32,
148    #[cfg(test)]
149    notify_admission_hook: SpinLock<Option<Arc<dyn Fn() + Send + Sync>>>,
150    #[cfg(test)]
151    reset_before_core_hook: SpinLock<Option<Arc<dyn Fn() + Send + Sync>>>,
152}
153
154impl<D: VirtioDeviceCore> VirtioPciTransport<D> {
155    /// Creates a transport with all queues disabled and unconfigured.
156    ///
157    /// # Errors
158    ///
159    /// Returns an error when the core does not expose exactly one queue, has
160    /// a non-power-of-two queue size, or uses asynchronous processing. Commit
161    /// 4 deliberately supports only the synchronous single-queue path; a
162    /// multi-queue or asynchronous adapter belongs to a later integration.
163    pub fn try_new(core: D) -> DeviceResult<Self> {
164        let queue_num_max = core.queue_num_max();
165        let queue_size_max = core.queue_size_max();
166        if queue_num_max != 1 {
167            return Err(DeviceError::InvalidInput {
168                operation: "create VirtIO PCI transport",
169                detail: "commit 4 supports exactly one queue".into(),
170            });
171        }
172        if !queue_size_max.is_power_of_two() {
173            return Err(DeviceError::InvalidInput {
174                operation: "create VirtIO PCI transport",
175                detail: "queue size must be a power of two".into(),
176            });
177        }
178        if core.requires_deferred_processing() {
179            return Err(DeviceError::Unsupported {
180                operation: "create VirtIO PCI transport",
181                detail: "deferred queue processing is not supported by the synchronous PCI adapter"
182                    .into(),
183            });
184        }
185        Ok(Self {
186            device_config_size: core.device_config_size(),
187            state: SpinLock::new(TransportState::new(queue_num_max, queue_size_max)),
188            interrupts: Arc::new(VirtioPciInterruptCoordinator::new()),
189            activity: Arc::new(QueueActivity::new()),
190            core,
191            #[cfg(test)]
192            notify_admission_hook: SpinLock::new(None),
193            #[cfg(test)]
194            reset_before_core_hook: SpinLock::new(None),
195        })
196    }
197
198    /// Returns the device-specific core.
199    pub fn core(&self) -> &D {
200        &self.core
201    }
202
203    /// Returns the device type advertised by the core.
204    pub fn device_type(&self) -> VirtioDeviceID {
205        self.core.device_type()
206    }
207
208    /// Returns the advertised device feature bits.
209    pub fn device_features(&self) -> u64 {
210        self.core.device_features()
211    }
212
213    /// Returns the current device status.
214    pub fn status(&self) -> u8 {
215        self.state.lock().status
216    }
217
218    /// Returns the negotiated driver feature bits.
219    pub fn driver_features(&self) -> u64 {
220        self.state.lock().driver_features
221    }
222
223    /// Returns the generation of the current queue configuration lifetime.
224    pub fn queue_generation(&self) -> VirtioQueueGeneration {
225        VirtioQueueGeneration(self.state.lock().queue_generation)
226    }
227
228    /// Returns whether the transport has a pending interrupt status bit.
229    pub fn interrupt_pending(&self) -> bool {
230        self.interrupts.pending()
231    }
232
233    /// Records a virtqueue or configuration interrupt in the PCI ISR state.
234    #[cfg(test)]
235    pub(crate) fn record_interrupt(&self, configuration_change: bool) -> InterruptTransition {
236        if configuration_change {
237            self.interrupts.record_config_change()
238        } else {
239            self.interrupts.record_queue_completion(true)
240        }
241    }
242
243    /// Installs a test-only pause between queue snapshot and activity admission.
244    ///
245    /// This makes the reset/reconfiguration interleaving deterministic without
246    /// exposing a production scheduling hook.
247    #[cfg(test)]
248    pub(crate) fn set_notify_admission_hook<F>(&self, hook: F)
249    where
250        F: Fn() + Send + Sync + 'static,
251    {
252        *self.notify_admission_hook.lock() = Some(Arc::new(hook));
253    }
254
255    #[cfg(test)]
256    pub(super) fn run_notify_admission_hook(&self) {
257        let hook = self.notify_admission_hook.lock().clone();
258        if let Some(hook) = hook {
259            hook();
260        }
261    }
262
263    /// Installs a test-only pause after activity has drained and before the
264    /// device-specific reset begins. This makes reset handoff tests
265    /// deterministic without exposing a production scheduling hook.
266    #[cfg(test)]
267    pub(crate) fn set_reset_before_core_hook<F>(&self, hook: F)
268    where
269        F: Fn() + Send + Sync + 'static,
270    {
271        *self.reset_before_core_hook.lock() = Some(Arc::new(hook));
272    }
273
274    #[cfg(test)]
275    pub(super) fn run_reset_before_core_hook(&self) {
276        let hook = self.reset_before_core_hook.lock().clone();
277        if let Some(hook) = hook {
278            hook();
279        }
280    }
281
282    /// Updates the logical PCI Command.INTx Disable state and returns the
283    /// resulting physical-line transition intent.
284    ///
285    /// This operation is intentionally infallible and does not acquire
286    /// activity or IRQ-transition admission. The endpoint adapter uses it
287    /// while holding its per-function Command revision lock; the returned
288    /// intent must be admitted and executed only after that lock is released.
289    pub fn update_interrupt_disabled_logical(&self, disabled: bool) -> InterruptTransitionIntent {
290        // Capture the queue generation before changing the coordinator.  If a
291        // reset wins before activity admission, this intent must be rejected
292        // rather than acquiring a permit from the reopened generation.
293        let generation = self.queue_generation();
294        let transition = self.interrupts.set_disabled(disabled);
295        InterruptTransitionIntent::new(transition, generation)
296    }
297
298    /// Acquires control activity for a previously committed interrupt
299    /// transition intent.
300    ///
301    /// Logical Command state is already committed when this method runs. If
302    /// reset has closed activity admission, only the unexecuted physical
303    /// intent is cancelled; the logical disabled state remains available for
304    /// the next reset or synchronization point.
305    pub fn admit_interrupt_transition(
306        &self,
307        intent: InterruptTransitionIntent,
308    ) -> DeviceResult<Option<InterruptTransitionRequest>> {
309        let Some(activity) = self.activity.acquire(intent.generation()) else {
310            // The logical command state is committed before admission is
311            // acquired. If reset closed admission, cancel only the
312            // unexecuted physical intent; the desired state remains in
313            // the coordinator for the reset owner to preserve/retry.
314            self.interrupts.cancel_transition(intent.transition());
315            return Ok(None);
316        };
317
318        // Activity admission and reset close are linearized by the same gate,
319        // but the generation advances only when the reset core state is
320        // committed. Revalidate after acquiring the permit so an intent that
321        // waited through a completed reset cannot reach the endpoint IRQ
322        // callback.
323        if self.queue_generation() != intent.generation() {
324            self.interrupts
325                .suppress_stale_transition(intent.transition());
326            drop(activity);
327            return Ok(None);
328        }
329
330        Ok(Some(InterruptTransitionRequest::new(
331            Arc::clone(&self.interrupts),
332            intent.transition(),
333            Some(activity),
334        )))
335    }
336
337    /// Applies PCI Command.INTx Disable and returns a line transition intent.
338    pub fn set_interrupt_disabled(
339        &self,
340        disabled: bool,
341    ) -> DeviceResult<InterruptTransitionRequest> {
342        let intent = self.update_interrupt_disabled_logical(disabled);
343        self.admit_interrupt_transition(intent)?
344            .ok_or(DeviceError::InvalidState {
345                operation: "update VirtIO PCI interrupt state",
346                detail: "transport reset is in progress or the transition is stale".into(),
347            })
348    }
349
350    /// Commits an interrupt-line operation executed through the endpoint
351    /// context without turning a host line failure into a guest access error.
352    pub fn complete_interrupt_transition(
353        &self,
354        transition: InterruptTransition,
355        success: bool,
356    ) -> InterruptTransition {
357        self.interrupts.complete_transition(transition, success)
358    }
359
360    /// Suppresses a transition whose endpoint IRQ admission became stale.
361    ///
362    /// Admission closure is not a physical-line failure. Only release the
363    /// matching in-flight transition; preserve any retry state that was
364    /// already recorded by a real line operation failure.
365    pub fn suppress_stale_interrupt_transition(&self, transition: InterruptTransition) {
366        self.interrupts.suppress_stale_transition(transition);
367    }
368
369    /// Returns a retry intent for a previously failed line synchronization.
370    pub fn resynchronize_interrupt(&self) -> InterruptTransition {
371        self.interrupts.resynchronize()
372    }
373
374    fn acquire_control_activity(&self) -> DeviceResult<ActivityPermit> {
375        self.activity
376            .acquire(self.queue_generation())
377            .ok_or(DeviceError::InvalidState {
378                operation: "access VirtIO PCI transport control state",
379                detail: "transport reset is in progress".into(),
380            })
381    }
382}
383
384fn require_width(actual: AccessWidth, expected: AccessWidth) -> DeviceResult {
385    if actual == expected {
386        Ok(())
387    } else {
388        Err(DeviceError::InvalidWidth { expected, actual })
389    }
390}
391
392fn access_in_region(offset: u64, width: AccessWidth, start: u64, length: u64) -> bool {
393    offset >= start
394        && offset
395            .checked_add(width.size() as u64)
396            .is_some_and(|end| end <= start + length)
397}
398
399fn feature_word(features: u64, selector: u32) -> DeviceResult<u64> {
400    if selector > 1 {
401        Ok(0)
402    } else {
403        Ok((features >> (selector * 32)) & u32::MAX as u64)
404    }
405}
406
407fn invalid_queue(index: u16) -> DeviceError {
408    DeviceError::InvalidInput {
409        operation: "virtio-pci queue",
410        detail: format!("queue index {index} is not exposed"),
411    }
412}
413
414fn map_pci_error(error: VirtioError) -> DeviceError {
415    map_virtio_error(error, "virtio-pci queue")
416}
417
418fn reject_processing_queue(queue: &QueueState) -> DeviceResult {
419    if queue.processing {
420        Err(DeviceError::ResourceBusy {
421            operation: "configure VirtIO queue",
422            resource: "queue processing lease".into(),
423        })
424    } else {
425        Ok(())
426    }
427}
428
429#[cfg(test)]
430mod tests;
axvirtio_common/pci/transport/mod.rs

axvirtio_common/pci/transport/
mod.rs
