1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
//! Regression: the guest's runtime APIC ID must equal the declared
//! (CPUID/MADT) apic_id for a SPARSE topology.
//!
//! ktstr advertises `apic_id(topo, cpu_id)` in CPUID leaf 1/0xB and the MADT,
//! which is SPARSE for non-power-of-2 core/thread counts (gaps in the
//! bit-field encoding). KVM hardwires the in-kernel LAPIC x2apic_id to the KVM
//! `vcpu_id` (arch/x86/kvm/lapic.c `kvm_apic_set_x2apic_id`, read-only,
//! `WARN_ON_ONCE(id != vcpu_id)`), and the guest's runtime apic_id =
//! `read_apic_id()` = that `vcpu_id` (cpu/topology_common.c uses it as
//! authoritative, warning if the CPUID disagrees). So unless ktstr creates
//! each vCPU with `vcpu_id = apic_id(topo, cpu_id)`, the guest runs DENSE
//! apic_ids (= cpu_id) that diverge from the advertised sparse ones: sparse
//! APIC IDs (including any > 255) become unrouteable, AP bringup to the
//! gap/overflow IDs fails, and an MSI/IPI dest never matches the intended
//! vCPU. The fix is `create_vcpu(apic_id(topo, cpu_id))` in
//! `src/vmm/x86_64/kvm.rs`.
//!
//! This pins the invariant on a MINIMAL sparse topology — 2 LLC x 3 core x 1
//! thread = 6 vCPUs; 3 cores need a 2-bit field, so `apic_id = (llc<<2)|core`
//! = {0,1,2,4,5,6} (gap at 3, max 6 > 5) — so it runs on any host without a
//! \>255-vCPU box. The >255 ext-dest path itself is exercised by the
//! `wide_smp_*_irq_e2e` tests on a >=252-CPU host; this is the CI-runnable
//! root-cause guard. `/proc/cpuinfo` exposes both `apicid` (runtime,
//! `read_apic_id()`) and `initial apicid` (CPUID leaf 1) per CPU; the fix
//! makes them equal.
//!
//! Run: cargo run --bin cargo-ktstr -- ktstr test --kernel ../linux \
//! -- -E 'test(sparse_topology_runtime_apicid_matches_declared)' \
//! --success-output immediate
// `apicid` / `initial apicid` in /proc/cpuinfo are emitted only by x86's
// show_cpuinfo_core (under CONFIG_SMP); aarch64 has no such field, so on an
// arm64 guest read_apicids() yields 0 and this test would misreport an
// AP-bringup failure (the failure observed on the arm64 CI runner). The whole
// test is an x86 LAPIC/MADT/x2apic invariant — the aarch64 GIC affinity model
// has no apic_id notion — so gate the file to x86_64. A sparse arm64 MPIDR
// invariant, if warranted, would be a separate test.
#![cfg(target_arch = "x86_64")]
use anyhow::{Result, ensure};
use ktstr::assert::AssertResult;
use ktstr::ktstr_test;
use ktstr::scenario::Ctx;
use std::collections::BTreeSet;
use std::fs::read_to_string;
/// Parse `/proc/cpuinfo` into `(runtime apicid, initial apicid)` pairs, one
/// per processor block. `apicid` is `read_apic_id()` (the in-kernel LAPIC ID
/// an MSI/IPI resolves against); `initial apicid` is the CPUID leaf-1 value
/// ktstr advertised.
fn read_apicids() -> Result<Vec<(u32, u32)>> {
let text = read_to_string("/proc/cpuinfo")?;
let mut pairs = Vec::new();
let mut apicid: Option<u32> = None;
let mut initial: Option<u32> = None;
let mut flush = |apicid: &mut Option<u32>, initial: &mut Option<u32>| {
if let (Some(a), Some(i)) = (apicid.take(), initial.take()) {
pairs.push((a, i));
}
};
for line in text.lines() {
if line.trim().is_empty() {
// Blank line separates processor blocks.
flush(&mut apicid, &mut initial);
continue;
}
let Some((key, val)) = line.split_once(':') else {
continue;
};
match key.trim() {
"apicid" => apicid = Some(val.trim().parse()?),
"initial apicid" => initial = Some(val.trim().parse()?),
_ => {}
}
}
// Flush a trailing block with no terminating blank line.
flush(&mut apicid, &mut initial);
Ok(pairs)
}
#[ktstr_test(llcs = 2, cores = 3, threads = 1, no_perf_mode, duration_s = 4)]
fn sparse_topology_runtime_apicid_matches_declared(ctx: &Ctx) -> Result<AssertResult> {
let total = ctx.topo.total_cpus();
ensure!(
total == 6,
"expected 6 vCPUs (2 LLC x 3 core x 1 thread), got {total}"
);
let pairs = read_apicids()?;
// All vCPUs must come online. Pre-fix, the BSP's INIT-SIPI to a sparse
// MADT apic_id (e.g. 6) matches no LAPIC (KVM x2apic_ids are dense
// {0..5}), so that AP never boots and the count is short.
ensure!(
pairs.len() == total,
"all {total} vCPUs must report an apicid in /proc/cpuinfo, got {} — \
sparse APIC IDs unrouteable, so AP bringup to the gap/overflow IDs failed",
pairs.len()
);
let declared: BTreeSet<u32> = pairs.iter().map(|(_, i)| *i).collect();
let max_declared = declared.iter().copied().max().unwrap_or(0);
// Guard: the topology must be genuinely SPARSE, else the invariant below
// holds trivially (dense apic_id == cpu_id) and the test is vacuous. The
// 2x3x1 encoding declares a max apic_id of 6 over 6 CPUs, so a declared
// value > total-1 proves the bit-field gap is present.
ensure!(
max_declared > (total as u32) - 1,
"topology is not sparse (max declared apic_id {max_declared} <= {}); \
the test would be vacuous — pick a non-power-of-2 core/thread count",
(total as u32) - 1
);
// The invariant: each vCPU's runtime apic_id (what an MSI/IPI resolves
// against) must equal the declared CPUID apic_id. With vcpu_id=cpu_id the
// runtime set is dense {0..5} while declared is {0,1,2,4,5,6}.
eprintln!(
"SPARSE_APICID total={total} pairs={pairs:?} declared={declared:?} max_declared={max_declared}"
);
for (apicid, initial) in &pairs {
ensure!(
apicid == initial,
"a vCPU's runtime apic_id {apicid} != its declared CPUID apic_id {initial}: \
the in-kernel LAPIC ID (KVM vcpu_id) diverges from the advertised apic_id, \
so sparse (incl. >255) APIC IDs are unrouteable"
);
}
Ok(AssertResult::pass())
}