use crate::error::IronError;
use serde::{Deserialize, Serialize};
#[cfg(any(all(target_os = "linux", feature = "nvidia"), test))]
type Bdf = (u32, u8, u8, u8);
#[cfg(any(all(target_os = "linux", feature = "nvidia"), test))]
fn parse_bdf(text: &str) -> Option<Bdf> {
let mut parts = text.trim().split(':');
let domain = u32::from_str_radix(parts.next()?, 16).ok()?;
let bus = u8::from_str_radix(parts.next()?, 16).ok()?;
let (device, function) = parts.next()?.split_once('.')?;
if parts.next().is_some() {
return None;
}
Some((
domain,
bus,
u8::from_str_radix(device, 16).ok()?,
u8::from_str_radix(function, 16).ok()?,
))
}
#[cfg(any(target_os = "linux", test))]
#[derive(Default)]
struct NvmlPairFacts {
nvlink: std::collections::HashSet<(usize, usize)>,
ancestor: std::collections::HashMap<(usize, usize), GpuInterconnectType>,
}
#[cfg(all(target_os = "linux", feature = "nvidia"))]
const NVLINK_MAX_LINKS: u32 = 18;
#[cfg(all(target_os = "linux", feature = "nvidia"))]
fn interconnect_of_topology(
level: nvml_wrapper::enum_wrappers::device::TopologyLevel,
) -> Option<GpuInterconnectType> {
use nvml_wrapper::enum_wrappers::device::TopologyLevel;
match level {
TopologyLevel::Internal => Some(GpuInterconnectType::SameBoard),
TopologyLevel::Single | TopologyLevel::Multiple => {
Some(GpuInterconnectType::PciePeerToPeer)
}
TopologyLevel::HostBridge | TopologyLevel::Node => {
Some(GpuInterconnectType::PcieThroughHostBridge)
}
TopologyLevel::System => Some(GpuInterconnectType::PcieThroughCpu),
}
}
#[cfg(all(target_os = "linux", feature = "nvidia"))]
fn read_nvml_pairs(gpus: &[GpuTopologyNode]) -> Option<NvmlPairFacts> {
let nvml = nvml_wrapper::Nvml::init().ok()?;
let count = nvml.device_count().ok()?;
let position: std::collections::HashMap<Bdf, usize> = gpus
.iter()
.enumerate()
.filter_map(|(i, gpu)| Some((parse_bdf(&gpu.bdf)?, i)))
.collect();
let mut indexed: Vec<(usize, u32)> = Vec::new();
for index in 0..count {
let Ok(device) = nvml.device_by_index(index) else {
continue;
};
let Ok(pci) = device.pci_info() else {
continue;
};
if let Some(&pos) = parse_bdf(&pci.bus_id).and_then(|key| position.get(&key)) {
indexed.push((pos, index));
}
}
let mut facts = NvmlPairFacts::default();
for &(pos, index) in &indexed {
let Ok(device) = nvml.device_by_index(index) else {
continue;
};
for link_index in 0..NVLINK_MAX_LINKS {
let link = device.link_wrapper_for(link_index);
if !matches!(link.is_active(), Ok(true)) {
continue;
}
let Ok(remote) = link.remote_pci_info() else {
continue;
};
if let Some(&other) = parse_bdf(&remote.bus_id).and_then(|key| position.get(&key)) {
if other != pos {
facts.nvlink.insert((pos.min(other), pos.max(other)));
}
}
}
}
for (x, &(pos_a, index_a)) in indexed.iter().enumerate() {
for &(pos_b, index_b) in &indexed[x + 1..] {
let (Ok(a), Ok(b)) = (nvml.device_by_index(index_a), nvml.device_by_index(index_b))
else {
continue;
};
let Ok(level) = a.topology_common_ancestor(b) else {
continue;
};
if let Some(kind) = interconnect_of_topology(level) {
facts
.ancestor
.insert((pos_a.min(pos_b), pos_a.max(pos_b)), kind);
}
}
}
Some(facts)
}
#[cfg(all(target_os = "linux", not(feature = "nvidia")))]
fn read_nvml_pairs(_gpus: &[GpuTopologyNode]) -> Option<NvmlPairFacts> {
None
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum GpuInterconnectType {
NvLink,
XGmi,
PciePeerToPeer,
PcieThroughCpu,
PcieThroughHostBridge,
SameBoard,
None,
Unknown,
}
impl std::fmt::Display for GpuInterconnectType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::NvLink => write!(f, "NVLink"),
Self::XGmi => write!(f, "xGMI"),
Self::PciePeerToPeer => write!(f, "PCIe P2P"),
Self::PcieThroughCpu => write!(f, "PCIe (via CPU)"),
Self::PcieThroughHostBridge => write!(f, "PCIe (via Host Bridge)"),
Self::SameBoard => write!(f, "Same Board"),
Self::None => write!(f, "None"),
Self::Unknown => write!(f, "Unknown"),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GpuTopologyNode {
pub bdf: String,
pub index: u32,
pub name: String,
pub vendor: String,
pub numa_node: Option<u32>,
pub pcie_gen: Option<u8>,
pub pcie_width: Option<u8>,
pub pcie_speed_gts: Option<f64>,
pub pcie_bandwidth_gbs: Option<f64>,
pub iommu_group: Option<u32>,
pub driver: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GpuLink {
pub from_gpu: u32,
pub to_gpu: u32,
pub link_type: GpuInterconnectType,
pub num_links: Option<u32>,
pub bandwidth_per_link_gbs: Option<f64>,
pub total_bandwidth_gbs: Option<f64>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GpuTopologyOverview {
pub gpus: Vec<GpuTopologyNode>,
pub links: Vec<GpuLink>,
pub gpu_count: u32,
pub numa_node_count: u32,
pub has_nvlink: bool,
pub has_xgmi: bool,
pub same_numa: Option<bool>,
pub recommendations: Vec<String>,
}
pub struct GpuTopologyMonitor {
overview: GpuTopologyOverview,
}
impl GpuTopologyMonitor {
pub fn new() -> Result<Self, IronError> {
let overview = Self::scan()?;
Ok(Self { overview })
}
pub fn refresh(&mut self) -> Result<(), IronError> {
self.overview = Self::scan()?;
Ok(())
}
pub fn overview(&self) -> &GpuTopologyOverview {
&self.overview
}
pub fn gpus(&self) -> &[GpuTopologyNode] {
&self.overview.gpus
}
pub fn links(&self) -> &[GpuLink] {
&self.overview.links
}
pub fn links_for_gpu(&self, gpu_index: u32) -> Vec<&GpuLink> {
self.overview
.links
.iter()
.filter(|l| l.from_gpu == gpu_index || l.to_gpu == gpu_index)
.collect()
}
pub fn gpus_on_numa(&self, node: u32) -> Vec<&GpuTopologyNode> {
self.overview
.gpus
.iter()
.filter(|g| g.numa_node == Some(node))
.collect()
}
#[cfg(target_os = "linux")]
fn scan() -> Result<GpuTopologyOverview, IronError> {
let pci_path = std::path::Path::new("/sys/bus/pci/devices");
if !pci_path.exists() {
return Ok(Self::empty_overview());
}
let entries = std::fs::read_dir(pci_path).map_err(IronError::Io)?;
let mut gpus = Vec::new();
let mut gpu_index = 0u32;
for entry in entries.flatten() {
let path = entry.path();
let class = Self::read_sysfs(&path.join("class")).unwrap_or_default();
let class_val = u32::from_str_radix(class.trim_start_matches("0x"), 16).unwrap_or(0);
let class_major = class_val >> 16;
if class_major != 0x03 {
continue;
}
let bdf = entry.file_name().to_string_lossy().to_string();
let vendor_id = Self::read_sysfs(&path.join("vendor")).unwrap_or_default();
let device_id = Self::read_sysfs(&path.join("device")).unwrap_or_default();
let vendor = match vendor_id.as_str() {
"0x10de" => "nvidia",
"0x1002" => "amd",
"0x8086" => "intel",
_ => "unknown",
};
let name = format!("{}:{} GPU {}", vendor, device_id, gpu_index);
let numa_node =
Self::read_sysfs_i32(&path.join("numa_node")).and_then(|n| u32::try_from(n).ok());
let (pcie_gen, pcie_width, pcie_speed_gts) = Self::read_pcie_info(&path);
let pcie_bandwidth_gbs = pcie_bandwidth(pcie_gen, pcie_width);
let iommu_group = std::fs::read_link(path.join("iommu_group"))
.ok()
.and_then(|p| {
p.file_name()
.and_then(|n| n.to_str().and_then(|s| s.parse().ok()))
});
let driver = std::fs::read_link(path.join("driver"))
.ok()
.and_then(|p| p.file_name().map(|n| n.to_string_lossy().to_string()))
.unwrap_or_default();
gpus.push(GpuTopologyNode {
bdf,
index: gpu_index,
name,
vendor: vendor.to_string(),
numa_node,
pcie_gen,
pcie_width,
pcie_speed_gts,
pcie_bandwidth_gbs,
iommu_group,
driver,
});
gpu_index += 1;
}
let nvml_pairs = read_nvml_pairs(&gpus);
let mut links = Vec::new();
for i in 0..gpus.len() {
for j in (i + 1)..gpus.len() {
let link_type =
Self::detect_link_type(&gpus[i], &gpus[j], (i, j), nvml_pairs.as_ref());
let (num_links, bw_per_link) = Self::link_bandwidth(&link_type, &gpus[i], &gpus[j]);
links.push(GpuLink {
from_gpu: gpus[i].index,
to_gpu: gpus[j].index,
link_type,
num_links,
bandwidth_per_link_gbs: bw_per_link,
total_bandwidth_gbs: num_links.zip(bw_per_link).map(|(n, bw)| n as f64 * bw),
});
}
}
let gpu_count = gpus.len() as u32;
let numa_nodes: std::collections::HashSet<u32> =
gpus.iter().filter_map(|g| g.numa_node).collect();
let numa_node_count = numa_nodes.len() as u32;
let has_nvlink = links
.iter()
.any(|l| l.link_type == GpuInterconnectType::NvLink);
let has_xgmi = links
.iter()
.any(|l| l.link_type == GpuInterconnectType::XGmi);
let same_numa = if numa_nodes.len() > 1 {
Some(false)
} else if gpus.iter().all(|g| g.numa_node.is_some()) {
Some(true)
} else {
None
};
let interconnect_read = nvml_pairs.is_some() || gpus.iter().all(|g| g.vendor != "nvidia");
let mut recs = Vec::new();
if gpu_count > 1 && numa_nodes.len() > 1 {
recs.push("GPUs span multiple NUMA nodes — consider NUMA-aware scheduling".into());
}
if gpu_count > 1 && !has_nvlink && !has_xgmi && interconnect_read {
recs.push("No high-bandwidth GPU interconnect (NVLink/xGMI) — GPU communication will use PCIe".into());
}
Ok(GpuTopologyOverview {
gpus,
links,
gpu_count,
numa_node_count,
has_nvlink,
has_xgmi,
same_numa,
recommendations: recs,
})
}
#[cfg(target_os = "linux")]
fn read_pcie_info(path: &std::path::Path) -> (Option<u8>, Option<u8>, Option<f64>) {
let speed_str = Self::read_sysfs(&path.join("current_link_speed")).unwrap_or_default();
let width_str = Self::read_sysfs(&path.join("current_link_width")).unwrap_or_default();
parse_link(&speed_str, &width_str)
}
#[cfg(any(target_os = "linux", test))]
fn detect_link_type(
gpu_a: &GpuTopologyNode,
gpu_b: &GpuTopologyNode,
pair: (usize, usize),
nvml: Option<&NvmlPairFacts>,
) -> GpuInterconnectType {
if nvml.is_some_and(|facts| facts.nvlink.contains(&pair)) {
return GpuInterconnectType::NvLink;
}
let numa_pair = (gpu_a.numa_node, gpu_b.numa_node);
let different_numa = matches!(numa_pair, (Some(a), Some(b)) if a != b);
#[cfg(target_os = "linux")]
if gpu_a.vendor == "amd"
&& gpu_b.vendor == "amd"
&& matches!(numa_pair, (Some(a), Some(b)) if a == b)
&& xgmi_pair_in_one_hive(
Self::read_xgmi_hive_id(&gpu_a.bdf),
Self::read_xgmi_hive_id(&gpu_b.bdf),
)
{
return GpuInterconnectType::XGmi;
}
if let Some(kind) = nvml.and_then(|facts| facts.ancestor.get(&pair)) {
return *kind;
}
if different_numa {
GpuInterconnectType::PcieThroughCpu
} else {
GpuInterconnectType::Unknown
}
}
#[cfg(target_os = "linux")]
fn link_bandwidth(
link_type: &GpuInterconnectType,
gpu_a: &GpuTopologyNode,
gpu_b: &GpuTopologyNode,
) -> (Option<u32>, Option<f64>) {
match link_type {
GpuInterconnectType::NvLink | GpuInterconnectType::XGmi => (None, None),
GpuInterconnectType::PciePeerToPeer | GpuInterconnectType::PcieThroughCpu => {
let bw = gpu_a
.pcie_bandwidth_gbs
.zip(gpu_b.pcie_bandwidth_gbs)
.map(|(a, b)| a.min(b));
(Some(1), bw)
}
_ => (None, None),
}
}
#[cfg(target_os = "linux")]
fn read_sysfs(path: &std::path::Path) -> Option<String> {
std::fs::read_to_string(path)
.ok()
.map(|s| s.trim().to_string())
}
#[cfg(target_os = "linux")]
fn read_sysfs_i32(path: &std::path::Path) -> Option<i32> {
std::fs::read_to_string(path)
.ok()
.and_then(|s| s.trim().parse().ok())
}
#[cfg(target_os = "linux")]
fn read_xgmi_hive_id(bdf: &str) -> Option<u64> {
let path = std::path::Path::new("/sys/bus/pci/devices")
.join(bdf)
.join("xgmi_hive_info")
.join("xgmi_hive_id");
Self::read_sysfs(&path).and_then(|s| parse_xgmi_hive_id(&s))
}
#[cfg(not(target_os = "linux"))]
fn scan() -> Result<GpuTopologyOverview, IronError> {
Err(IronError::UnsupportedPlatform(
"GPU interconnect topology is read from `/sys/class/drm` and `nvidia-smi topo`, which this platform does not expose"
.into(),
))
}
fn empty_overview() -> GpuTopologyOverview {
GpuTopologyOverview {
gpus: Vec::new(),
links: Vec::new(),
gpu_count: 0,
numa_node_count: 0,
has_nvlink: false,
has_xgmi: false,
same_numa: Some(true),
recommendations: Vec::new(),
}
}
}
impl Default for GpuTopologyMonitor {
fn default() -> Self {
Self::new().unwrap_or(Self {
overview: Self::empty_overview(),
})
}
}
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
fn parse_link(speed: &str, width: &str) -> (Option<u8>, Option<u8>, Option<f64>) {
let width = width.trim_start_matches('x').parse::<u8>().ok();
let (generation, speed_gts) = if speed.contains("64.0") {
(6, 64.0)
} else if speed.contains("32.0") {
(5, 32.0)
} else if speed.contains("16.0") {
(4, 16.0)
} else if speed.contains("8.0") {
(3, 8.0)
} else if speed.contains("5.0") {
(2, 5.0)
} else if speed.contains("2.5") {
(1, 2.5)
} else {
return (None, width, None);
};
(Some(generation), width, Some(speed_gts))
}
#[cfg(any(target_os = "linux", test))]
fn parse_xgmi_hive_id(text: &str) -> Option<u64> {
let text = text.trim();
match text.strip_prefix("0x") {
Some(hex) => u64::from_str_radix(hex, 16).ok(),
None => text.parse().ok(),
}
}
#[cfg(any(target_os = "linux", test))]
fn xgmi_pair_in_one_hive(a: Option<u64>, b: Option<u64>) -> bool {
matches!((a, b), (Some(x), Some(y)) if x != 0 && x == y)
}
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
fn pcie_bandwidth(generation: Option<u8>, width: Option<u8>) -> Option<f64> {
let width = width? as f64;
let per_lane = match generation? {
1 => 0.25,
2 => 0.5,
3 => 0.985, 4 => 1.969,
5 => 3.938,
6 => 7.563,
_ => return None,
};
Some(per_lane * width)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_interconnect_display() {
assert_eq!(GpuInterconnectType::NvLink.to_string(), "NVLink");
assert_eq!(GpuInterconnectType::XGmi.to_string(), "xGMI");
assert_eq!(GpuInterconnectType::PciePeerToPeer.to_string(), "PCIe P2P");
}
#[test]
fn test_links_for_gpu() {
let overview = GpuTopologyOverview {
gpus: vec![
GpuTopologyNode {
bdf: "0000:01:00.0".into(),
index: 0,
name: "GPU0".into(),
vendor: "nvidia".into(),
numa_node: Some(0),
pcie_gen: Some(4),
pcie_width: Some(16),
pcie_speed_gts: Some(16.0),
pcie_bandwidth_gbs: Some(31.5),
iommu_group: Some(1),
driver: "nvidia".into(),
},
GpuTopologyNode {
bdf: "0000:41:00.0".into(),
index: 1,
name: "GPU1".into(),
vendor: "nvidia".into(),
numa_node: Some(1),
pcie_gen: Some(4),
pcie_width: Some(16),
pcie_speed_gts: Some(16.0),
pcie_bandwidth_gbs: Some(31.5),
iommu_group: Some(2),
driver: "nvidia".into(),
},
],
links: vec![GpuLink {
from_gpu: 0,
to_gpu: 1,
link_type: GpuInterconnectType::PcieThroughCpu,
num_links: Some(1),
bandwidth_per_link_gbs: Some(31.5),
total_bandwidth_gbs: Some(31.5),
}],
gpu_count: 2,
numa_node_count: 2,
has_nvlink: false,
has_xgmi: false,
same_numa: Some(false),
recommendations: Vec::new(),
};
let monitor = GpuTopologyMonitor { overview };
assert_eq!(monitor.links_for_gpu(0).len(), 1);
assert_eq!(monitor.links_for_gpu(1).len(), 1);
}
#[test]
fn test_numa_grouping() {
let overview = GpuTopologyOverview {
gpus: vec![
GpuTopologyNode {
bdf: "0000:01:00.0".into(),
index: 0,
name: "GPU0".into(),
vendor: "amd".into(),
numa_node: Some(0),
pcie_gen: Some(4),
pcie_width: Some(16),
pcie_speed_gts: Some(16.0),
pcie_bandwidth_gbs: Some(31.5),
iommu_group: None,
driver: "amdgpu".into(),
},
GpuTopologyNode {
bdf: "0000:02:00.0".into(),
index: 1,
name: "GPU1".into(),
vendor: "amd".into(),
numa_node: Some(0),
pcie_gen: Some(4),
pcie_width: Some(16),
pcie_speed_gts: Some(16.0),
pcie_bandwidth_gbs: Some(31.5),
iommu_group: None,
driver: "amdgpu".into(),
},
],
links: Vec::new(),
gpu_count: 2,
numa_node_count: 1,
has_nvlink: false,
has_xgmi: false,
same_numa: Some(true),
recommendations: Vec::new(),
};
let monitor = GpuTopologyMonitor { overview };
assert_eq!(monitor.gpus_on_numa(0).len(), 2);
}
#[test]
fn test_monitor_default() {
let monitor = GpuTopologyMonitor::default();
let _overview = monitor.overview();
}
#[test]
fn test_unknown_link_speed_is_not_read() {
let (generation, width, speed) = parse_link("unknown", "x16");
assert_eq!(generation, None);
assert_eq!(width, Some(16));
assert_eq!(speed, None);
assert_eq!(pcie_bandwidth(generation, width), None);
let (generation, width, _) = parse_link("8.0 GT/s PCIe", "x0");
assert_eq!(generation, Some(3));
assert_eq!(pcie_bandwidth(generation, width), Some(0.0));
}
fn gpu(index: u32, vendor: &str, numa: Option<u32>) -> GpuTopologyNode {
GpuTopologyNode {
bdf: format!("0000:{:02x}:00.0", index + 1),
index,
name: format!("GPU{index}"),
vendor: vendor.into(),
numa_node: numa,
pcie_gen: None,
pcie_width: None,
pcie_speed_gts: None,
pcie_bandwidth_gbs: None,
iommu_group: None,
driver: String::new(),
}
}
#[test]
fn nvml_and_sysfs_addresses_compare_equal() {
assert_eq!(parse_bdf("00000000:01:00.0"), parse_bdf("0000:01:00.0"));
assert_eq!(parse_bdf("0000:41:00.0"), Some((0, 0x41, 0, 0)));
assert_eq!(parse_bdf("0000:41:00"), None);
}
#[test]
fn nvidia_pair_on_one_numa_node_is_unknown_without_nvml() {
let (a, b) = (gpu(0, "nvidia", Some(0)), gpu(1, "nvidia", Some(0)));
assert_eq!(
GpuTopologyMonitor::detect_link_type(&a, &b, (0, 1), None),
GpuInterconnectType::Unknown
);
}
#[test]
fn nvlink_needs_an_active_link_reported_by_nvml() {
let (a, b) = (gpu(0, "nvidia", Some(0)), gpu(1, "nvidia", Some(0)));
let mut facts = NvmlPairFacts::default();
facts.nvlink.insert((0, 1));
assert_eq!(
GpuTopologyMonitor::detect_link_type(&a, &b, (0, 1), Some(&facts)),
GpuInterconnectType::NvLink
);
let empty = NvmlPairFacts::default();
assert_eq!(
GpuTopologyMonitor::detect_link_type(&a, &b, (0, 1), Some(&empty)),
GpuInterconnectType::Unknown
);
}
#[test]
fn nvml_topology_level_names_the_path_between_gpus() {
let (a, b) = (gpu(0, "nvidia", Some(0)), gpu(1, "nvidia", Some(0)));
let mut facts = NvmlPairFacts::default();
facts
.ancestor
.insert((0, 1), GpuInterconnectType::PciePeerToPeer);
assert_eq!(
GpuTopologyMonitor::detect_link_type(&a, &b, (0, 1), Some(&facts)),
GpuInterconnectType::PciePeerToPeer
);
}
#[test]
fn different_numa_nodes_still_cross_the_cpu() {
let (a, b) = (gpu(0, "nvidia", Some(0)), gpu(1, "nvidia", Some(1)));
assert_eq!(
GpuTopologyMonitor::detect_link_type(&a, &b, (0, 1), None),
GpuInterconnectType::PcieThroughCpu
);
let c = gpu(2, "nvidia", None);
assert_eq!(
GpuTopologyMonitor::detect_link_type(&a, &c, (0, 2), None),
GpuInterconnectType::Unknown
);
}
#[test]
fn xgmi_pair_needs_both_hive_ids_read_and_equal() {
assert!(xgmi_pair_in_one_hive(Some(7), Some(7)));
assert!(!xgmi_pair_in_one_hive(Some(7), Some(8)));
assert!(!xgmi_pair_in_one_hive(Some(7), None));
assert!(!xgmi_pair_in_one_hive(None, Some(7)));
assert!(!xgmi_pair_in_one_hive(None, None));
assert!(!xgmi_pair_in_one_hive(Some(0), Some(0)));
}
#[test]
fn xgmi_hive_id_parses_decimal_and_hex() {
assert_eq!(
parse_xgmi_hive_id("1234567890123456789\n"),
Some(1234567890123456789)
);
assert_eq!(parse_xgmi_hive_id("0x1a2b\n"), Some(0x1a2b));
assert_eq!(parse_xgmi_hive_id(""), None);
assert_eq!(parse_xgmi_hive_id("not a hive"), None);
}
#[test]
fn test_serialization() {
let node = GpuTopologyNode {
bdf: "0000:01:00.0".into(),
index: 0,
name: "GPU0".into(),
vendor: "nvidia".into(),
numa_node: Some(0),
pcie_gen: Some(5),
pcie_width: Some(16),
pcie_speed_gts: Some(32.0),
pcie_bandwidth_gbs: Some(63.0),
iommu_group: Some(10),
driver: "nvidia".into(),
};
let json = serde_json::to_string(&node).unwrap();
assert!(json.contains("0000:01:00.0"));
let _: GpuTopologyNode = serde_json::from_str(&json).unwrap();
}
}