use std::collections::{HashMap, HashSet};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use bytes::{Buf, Bytes};
use tokio::sync::RwLock;
use tracing::{debug, info, warn};
use super::compound::CompoundBuilder;
use super::session::{ClientIdentity, Session};
use crate::error::{NfsError, Result};
use crate::rpc;
use crate::rpc::auth::Auth;
const DS_TEARDOWN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
const DS_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
#[repr(u32)]
#[derive(Debug, Clone, Copy, PartialEq)]
pub(crate) enum LayoutType {
NfsV41Files = 1,
Osd2Objects = 2,
BlockVolume = 3,
}
#[repr(u32)]
#[derive(Debug, Clone, Copy, PartialEq)]
pub(crate) enum IoMode {
Read = 1,
ReadWrite = 2,
}
#[derive(Debug, Clone)]
#[allow(dead_code)] pub(crate) struct LayoutSegment {
pub offset: u64,
pub length: u64,
pub iomode: IoMode,
pub layout_type: LayoutType,
pub content: LayoutContent,
}
#[derive(Debug, Clone)]
#[allow(dead_code)] pub(crate) enum LayoutContent {
FilesLayout {
device_id: [u8; 16],
stripe_unit: u32,
is_dense: bool,
first_stripe_index: u32,
pattern_offset: u64,
fh_list: Vec<Bytes>,
},
Opaque(Bytes),
}
#[derive(Debug, Clone)]
#[allow(dead_code)] pub(crate) struct Layout {
pub stateid: [u8; 16],
pub return_on_close: bool,
pub segments: Vec<LayoutSegment>,
}
#[derive(Debug, Clone)]
pub(crate) struct DeviceInfo {
pub stripe_indices: Vec<u32>,
pub ds_addrs: Vec<Vec<SocketAddr>>,
}
#[derive(Debug, Clone)]
pub(crate) struct StripeChunk {
pub ds_index: u32,
pub file_offset: u64,
pub ds_offset: u64,
pub length: u32,
}
#[derive(Clone)]
pub(crate) struct DsConnection {
pub client: rpc::Client,
pub session: Arc<Session>,
}
impl DsConnection {
pub async fn destroy(&self, auth: &Auth) {
let builder = CompoundBuilder::new("ds_destroy_session").destroy_session(self.session.id());
let mut buf = Vec::new();
builder.encode_with_header(auth, &mut buf);
if let Err(e) = self.client.call(buf, 1, DS_TEARDOWN_TIMEOUT).await {
debug!(error = %e, "DS DESTROY_SESSION failed (may already be destroyed)");
}
let builder =
CompoundBuilder::new("ds_destroy_clientid").destroy_client_id(self.session.client_id());
let mut buf = Vec::new();
builder.encode_with_header(auth, &mut buf);
let _ = self.client.call(buf, 1, DS_TEARDOWN_TIMEOUT).await;
}
}
pub(crate) struct LayoutManager {
layouts: RwLock<HashMap<Bytes, Layout>>,
data_servers: RwLock<HashMap<SocketAddr, DsConnection>>,
device_cache: RwLock<HashMap<[u8; 16], DeviceInfo>>,
dirty: RwLock<HashMap<Bytes, (u64, u64)>>,
degenerate_logged: AtomicBool,
unreachable_ds: RwLock<HashSet<SocketAddr>>,
noresvport: bool,
}
impl LayoutManager {
pub fn new(noresvport: bool) -> Self {
Self {
layouts: RwLock::new(HashMap::new()),
data_servers: RwLock::new(HashMap::new()),
device_cache: RwLock::new(HashMap::new()),
dirty: RwLock::new(HashMap::new()),
degenerate_logged: AtomicBool::new(false),
unreachable_ds: RwLock::new(HashSet::new()),
noresvport,
}
}
pub async fn mark_ds_unreachable(&self, addr: SocketAddr) {
let mut set = self.unreachable_ds.write().await;
if set.insert(addr) {
warn!(addr = %addr, "marking pNFS data server unreachable, affected files fall back to MDS I/O");
}
}
pub async fn is_ds_unreachable(&self, addr: &SocketAddr) -> bool {
let set = self.unreachable_ds.read().await;
set.contains(addr)
}
pub fn should_log_degenerate(&self) -> bool {
self.degenerate_logged
.compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed)
.is_ok()
}
pub async fn store_layout(&self, fh: &Bytes, layout: Layout) {
let mut map = self.layouts.write().await;
debug!(
fh_len = fh.len(),
segments = layout.segments.len(),
"layout stored"
);
map.insert(fh.clone(), layout);
}
pub async fn get_layout(&self, fh: &Bytes) -> Option<Layout> {
let map = self.layouts.read().await;
map.get(fh).cloned()
}
pub async fn remove_layout(&self, fh: &Bytes) -> Option<Layout> {
let mut map = self.layouts.write().await;
map.remove(fh)
}
pub async fn drain_layouts(&self) -> Vec<(Bytes, Layout)> {
let mut map = self.layouts.write().await;
map.drain().collect()
}
pub async fn clear(&self) {
let mut map = self.layouts.write().await;
let mut devices = self.device_cache.write().await;
let mut dirty = self.dirty.write().await;
map.clear();
devices.clear();
dirty.clear();
}
pub async fn mark_dirty(&self, fh: &Bytes, start: u64, end: u64) {
let mut dirty = self.dirty.write().await;
dirty
.entry(fh.clone())
.and_modify(|(s, e)| {
*s = (*s).min(start);
*e = (*e).max(end);
})
.or_insert((start, end));
}
pub async fn take_dirty(&self, fh: &Bytes) -> Option<(u64, u64)> {
let mut dirty = self.dirty.write().await;
dirty.remove(fh)
}
pub async fn store_device(&self, device_id: [u8; 16], info: DeviceInfo) {
let mut cache = self.device_cache.write().await;
debug!(device_id = ?device_id, ds_count = info.ds_addrs.len(), "device info cached");
cache.insert(device_id, info);
}
pub async fn get_device(&self, device_id: &[u8; 16]) -> Option<DeviceInfo> {
let cache = self.device_cache.read().await;
cache.get(device_id).cloned()
}
pub async fn get_data_server(
&self,
addr: SocketAddr,
auth: &Auth,
client_identity: &ClientIdentity,
) -> Result<DsConnection> {
{
let servers = self.data_servers.read().await;
if let Some(conn) = servers.get(&addr) {
return Ok(conn.clone());
}
}
if self.is_ds_unreachable(&addr).await {
return Err(NfsError::Rpc(format!(
"pNFS data server {addr} marked unreachable"
)));
}
let connect_and_establish = async {
let mux = rpc::StreamMux::connect(addr, self.noresvport).await?;
let new_client = rpc::Client::new(mux, None);
let session = Session::establish_ds(&new_client, auth, client_identity).await?;
Ok::<DsConnection, NfsError>(DsConnection {
client: new_client,
session: Arc::new(session),
})
};
let conn = match tokio::time::timeout(DS_CONNECT_TIMEOUT, connect_and_establish).await {
Ok(Ok(conn)) => conn,
Ok(Err(e)) => {
self.mark_ds_unreachable(addr).await;
return Err(e);
}
Err(_) => {
self.mark_ds_unreachable(addr).await;
return Err(NfsError::Rpc(format!(
"pNFS data server {addr} connect timed out"
)));
}
};
let mut servers = self.data_servers.write().await;
if let Some(existing) = servers.get(&addr) {
let existing = existing.clone();
drop(servers);
let auth = auth.clone();
tokio::spawn(async move { conn.destroy(&auth).await });
return Ok(existing);
}
servers.insert(addr, conn.clone());
info!(addr = %addr, "connected to pNFS data server (session established)");
Ok(conn)
}
pub async fn remove_data_server(&self, addr: SocketAddr) -> Option<DsConnection> {
let mut servers = self.data_servers.write().await;
servers.remove(&addr)
}
pub async fn drain_data_servers(&self) -> Vec<(SocketAddr, DsConnection)> {
let mut servers = self.data_servers.write().await;
servers.drain().collect()
}
}
pub(crate) fn is_degenerate_device(device: &DeviceInfo, server_addr: &SocketAddr) -> bool {
!device.ds_addrs.is_empty()
&& device
.ds_addrs
.iter()
.all(|paths| paths.first() == Some(server_addr))
}
fn addr_prefix_len(a: &SocketAddr, b: &SocketAddr) -> u32 {
match (a.ip(), b.ip()) {
(IpAddr::V4(x), IpAddr::V4(y)) => (u32::from(x) ^ u32::from(y)).leading_zeros(),
(IpAddr::V6(x), IpAddr::V6(y)) => (u128::from(x) ^ u128::from(y)).leading_zeros(),
_ => 0,
}
}
pub(crate) fn sort_multipath_by_affinity(info: &mut DeviceInfo, server_addr: &SocketAddr) {
for paths in &mut info.ds_addrs {
paths.sort_by_key(|addr| {
let exact = *addr == *server_addr;
let same_family = addr.is_ipv4() == server_addr.is_ipv4();
(
!exact,
!same_family,
u32::MAX - addr_prefix_len(addr, server_addr),
)
});
}
}
pub(crate) fn decode_layoutget_response(data: &mut Bytes) -> Result<Layout> {
if data.remaining() < 4 {
return Err(NfsError::Xdr(
"LAYOUTGET return_on_close truncated".to_string(),
));
}
let return_on_close = data.get_u32() != 0;
if data.remaining() < 16 {
return Err(NfsError::Xdr("LAYOUTGET stateid truncated".to_string()));
}
let mut stateid = [0u8; 16];
data.copy_to_slice(&mut stateid);
if data.remaining() < 4 {
return Err(NfsError::Xdr(
"LAYOUTGET segments array truncated".to_string(),
));
}
let num_segments = data.get_u32() as usize;
if num_segments > 1024 {
return Err(NfsError::Xdr(format!(
"LAYOUTGET has {} segments, max 1024",
num_segments
)));
}
let mut segments = Vec::with_capacity(num_segments);
for _ in 0..num_segments {
if data.remaining() < 24 {
return Err(NfsError::Xdr(
"layout4 segment header truncated".to_string(),
));
}
let offset = data.get_u64();
let length = data.get_u64();
let iomode_val = data.get_u32();
let layout_type_val = data.get_u32();
let iomode = match iomode_val {
1 => IoMode::Read,
2 => IoMode::ReadWrite,
_ => IoMode::Read,
};
let layout_type = match layout_type_val {
1 => LayoutType::NfsV41Files,
2 => LayoutType::Osd2Objects,
3 => LayoutType::BlockVolume,
_ => LayoutType::NfsV41Files,
};
if data.remaining() < 4 {
return Err(NfsError::Xdr("layout_content length truncated".to_string()));
}
let content_len = data.get_u32() as usize;
let padded = (content_len + 3) & !3;
if data.remaining() < padded {
return Err(NfsError::Xdr("layout_content data truncated".to_string()));
}
let mut content_data = data.split_to(content_len);
let pad = padded - content_len;
if data.remaining() >= pad {
data.advance(pad);
}
let content = if layout_type == LayoutType::NfsV41Files {
decode_files_layout(&mut content_data)?
} else {
LayoutContent::Opaque(content_data)
};
segments.push(LayoutSegment {
offset,
length,
iomode,
layout_type,
content,
});
}
Ok(Layout {
stateid,
return_on_close,
segments,
})
}
fn decode_files_layout(data: &mut Bytes) -> Result<LayoutContent> {
if data.remaining() < 16 {
return Err(NfsError::Xdr("files_layout deviceid truncated".to_string()));
}
let mut device_id = [0u8; 16];
data.copy_to_slice(&mut device_id);
if data.remaining() < 4 {
return Err(NfsError::Xdr("files_layout nfl_util truncated".to_string()));
}
let nfl_util = data.get_u32();
let stripe_unit = nfl_util & 0x3FFF_FFFF;
let is_dense = (nfl_util & 0x4000_0000) != 0;
if data.remaining() < 4 {
return Err(NfsError::Xdr(
"files_layout first_stripe_index truncated".to_string(),
));
}
let first_stripe_index = data.get_u32();
if data.remaining() < 8 {
return Err(NfsError::Xdr(
"files_layout pattern_offset truncated".to_string(),
));
}
let pattern_offset = data.get_u64();
if data.remaining() < 4 {
return Err(NfsError::Xdr(
"files_layout fh_list length truncated".to_string(),
));
}
let num_fhs = data.get_u32() as usize;
if num_fhs > 4096 {
return Err(NfsError::Xdr(format!(
"files_layout has {} FHs, max 4096",
num_fhs
)));
}
let mut fh_list = Vec::with_capacity(num_fhs);
for _ in 0..num_fhs {
if data.remaining() < 4 {
return Err(NfsError::Xdr("files_layout fh truncated".to_string()));
}
let fh_len = data.get_u32() as usize;
let padded = (fh_len + 3) & !3;
if data.remaining() < padded {
return Err(NfsError::Xdr("files_layout fh data truncated".to_string()));
}
let fh = data.slice(..fh_len);
data.advance(padded);
fh_list.push(fh);
}
Ok(LayoutContent::FilesLayout {
device_id,
stripe_unit,
is_dense,
first_stripe_index,
pattern_offset,
fh_list,
})
}
pub(crate) fn stripe_ds_index(
offset: u64,
stripe_unit: u32,
first_stripe_index: u32,
num_ds: u32,
) -> u32 {
let su = stripe_unit as u64;
let stripe_num = offset / su;
((stripe_num + first_stripe_index as u64) % num_ds as u64) as u32
}
pub(crate) fn ds_offset(
file_offset: u64,
stripe_unit: u32,
_first_stripe_index: u32,
num_ds: u32,
is_dense: bool,
) -> u64 {
let su = stripe_unit as u64;
let stripe_num = file_offset / su;
let offset_in_stripe = file_offset % su;
if is_dense {
let ds_stripe_num = stripe_num / num_ds as u64;
ds_stripe_num * su + offset_in_stripe
} else {
let _ = (su, stripe_num, offset_in_stripe); file_offset
}
}
pub(crate) fn split_into_stripes(
offset: u64,
count: u32,
stripe_unit: u32,
is_dense: bool,
first_stripe_index: u32,
num_ds: u32,
pattern_offset: u64,
) -> Vec<StripeChunk> {
let mut chunks = Vec::new();
if stripe_unit == 0 || num_ds == 0 {
return chunks;
}
let su = stripe_unit as u64;
let end = offset + count as u64;
let mut pos = offset;
while pos < end {
let stripe_end = ((pos / su) + 1) * su;
let chunk_end = end.min(stripe_end);
let chunk_len = (chunk_end - pos) as u32;
let adjusted = pos.saturating_sub(pattern_offset);
let ds_idx = stripe_ds_index(adjusted, stripe_unit, first_stripe_index, num_ds);
let ds_off = ds_offset(adjusted, stripe_unit, first_stripe_index, num_ds, is_dense);
chunks.push(StripeChunk {
ds_index: ds_idx,
file_offset: pos,
ds_offset: ds_off,
length: chunk_len,
});
pos = chunk_end;
}
chunks
}
fn parse_netaddr4(r_netid: &str, r_addr: &str) -> Result<SocketAddr> {
let parse_u8 = |s: &str| -> Result<u8> {
s.parse::<u8>()
.map_err(|_| NfsError::Xdr(format!("invalid address octet: {}", s)))
};
if r_netid == "tcp" {
let parts: Vec<&str> = r_addr.split('.').collect();
if parts.len() != 6 {
return Err(NfsError::Xdr(format!(
"invalid tcp r_addr (expected 6 dot-separated fields): {}",
r_addr
)));
}
let ip = Ipv4Addr::new(
parse_u8(parts[0])?,
parse_u8(parts[1])?,
parse_u8(parts[2])?,
parse_u8(parts[3])?,
);
let port = parse_u8(parts[4])? as u16 * 256 + parse_u8(parts[5])? as u16;
Ok(SocketAddr::new(IpAddr::V4(ip), port))
} else if r_netid == "tcp6" {
let dot2 = {
let last = r_addr.rfind('.').ok_or_else(|| {
NfsError::Xdr(format!("invalid tcp6 r_addr (missing port): {}", r_addr))
})?;
r_addr[..last].rfind('.').ok_or_else(|| {
NfsError::Xdr(format!(
"invalid tcp6 r_addr (missing port octets): {}",
r_addr
))
})?
};
let ip_part = &r_addr[..dot2];
let port_part = &r_addr[dot2 + 1..];
let port_octets: Vec<&str> = port_part.split('.').collect();
if port_octets.len() != 2 {
return Err(NfsError::Xdr(format!(
"invalid tcp6 r_addr port part: {}",
r_addr
)));
}
let p1 = parse_u8(port_octets[0])? as u16;
let p2 = parse_u8(port_octets[1])? as u16;
let port = p1 * 256 + p2;
let ip: Ipv6Addr = ip_part
.parse()
.map_err(|_| NfsError::Xdr(format!("invalid IPv6 address: {}", ip_part)))?;
Ok(SocketAddr::new(IpAddr::V6(ip), port))
} else {
Err(NfsError::Xdr(format!("unsupported netid: {}", r_netid)))
}
}
fn read_xdr_string(data: &mut Bytes) -> Result<String> {
if data.remaining() < 4 {
return Err(NfsError::Xdr("XDR string length truncated".to_string()));
}
let len = data.get_u32() as usize;
let padded = (len + 3) & !3;
if data.remaining() < padded {
return Err(NfsError::Xdr("XDR string data truncated".to_string()));
}
let s = std::str::from_utf8(&data[..len])
.map_err(|_| NfsError::Xdr("XDR string not valid UTF-8".to_string()))?
.to_string();
data.advance(padded);
Ok(s)
}
pub(crate) fn decode_getdeviceinfo_response(data: &mut Bytes) -> Result<DeviceInfo> {
if data.remaining() < 4 {
return Err(NfsError::Xdr(
"GETDEVICEINFO layout_type truncated".to_string(),
));
}
let layout_type = data.get_u32();
if layout_type != 1 {
return Err(NfsError::Xdr(format!(
"GETDEVICEINFO unsupported layout type: {}",
layout_type
)));
}
if data.remaining() < 4 {
return Err(NfsError::Xdr(
"GETDEVICEINFO da_addr_body length truncated".to_string(),
));
}
let body_len = data.get_u32() as usize;
let padded = (body_len + 3) & !3;
if data.remaining() < padded {
return Err(NfsError::Xdr(
"GETDEVICEINFO da_addr_body truncated".to_string(),
));
}
let mut body = data.split_to(body_len);
let pad = padded - body_len;
if data.remaining() >= pad {
data.advance(pad);
}
if body.remaining() < 4 {
return Err(NfsError::Xdr(
"GETDEVICEINFO stripe_indices length truncated".to_string(),
));
}
let num_indices = body.get_u32() as usize;
if num_indices > 4096 {
return Err(NfsError::Xdr(format!(
"GETDEVICEINFO has {} stripe_indices, max 4096",
num_indices
)));
}
if body.remaining() < num_indices * 4 {
return Err(NfsError::Xdr(
"GETDEVICEINFO stripe_indices data truncated".to_string(),
));
}
let mut stripe_indices = Vec::with_capacity(num_indices);
for _ in 0..num_indices {
stripe_indices.push(body.get_u32());
}
if body.remaining() < 4 {
return Err(NfsError::Xdr(
"GETDEVICEINFO multipath_ds_list length truncated".to_string(),
));
}
let num_ds = body.get_u32() as usize;
if num_ds > 4096 {
return Err(NfsError::Xdr(format!(
"GETDEVICEINFO has {} data servers, max 4096",
num_ds
)));
}
let mut ds_addrs = Vec::with_capacity(num_ds);
for _ in 0..num_ds {
if body.remaining() < 4 {
return Err(NfsError::Xdr(
"GETDEVICEINFO multipath_list length truncated".to_string(),
));
}
let num_addrs = body.get_u32() as usize;
if num_addrs > 256 {
return Err(NfsError::Xdr(format!(
"GETDEVICEINFO DS has {} addresses, max 256",
num_addrs
)));
}
let mut addrs = Vec::with_capacity(num_addrs);
for _ in 0..num_addrs {
let r_netid = read_xdr_string(&mut body)?;
let r_addr = read_xdr_string(&mut body)?;
let addr = parse_netaddr4(&r_netid, &r_addr)?;
addrs.push(addr);
}
ds_addrs.push(addrs);
}
Ok(DeviceInfo {
stripe_indices,
ds_addrs,
})
}
#[cfg(test)]
mod tests {
use super::*;
fn put_u32(buf: &mut Vec<u8>, v: u32) {
buf.extend_from_slice(&v.to_be_bytes());
}
fn put_u64(buf: &mut Vec<u8>, v: u64) {
buf.extend_from_slice(&v.to_be_bytes());
}
#[tokio::test]
async fn layout_manager_store_and_get() {
let mgr = LayoutManager::new(false);
let fh = Bytes::from_static(b"file1");
let layout = Layout {
stateid: [1u8; 16],
return_on_close: true,
segments: vec![],
};
mgr.store_layout(&fh, layout).await;
let got = mgr.get_layout(&fh).await.unwrap();
assert_eq!(got.stateid, [1u8; 16]);
assert!(got.return_on_close);
}
#[tokio::test]
async fn layout_manager_remove() {
let mgr = LayoutManager::new(false);
let fh = Bytes::from_static(b"file2");
mgr.store_layout(
&fh,
Layout {
stateid: [2u8; 16],
return_on_close: false,
segments: vec![],
},
)
.await;
let removed = mgr.remove_layout(&fh).await;
assert!(removed.is_some());
assert!(mgr.get_layout(&fh).await.is_none());
}
#[test]
fn degenerate_device_single_ds_is_mds() {
let mds: SocketAddr = "10.0.0.1:2049".parse().unwrap();
let device = DeviceInfo {
stripe_indices: vec![0],
ds_addrs: vec![vec![mds]],
};
assert!(is_degenerate_device(&device, &mds));
}
#[test]
fn degenerate_device_other_ds() {
let mds: SocketAddr = "10.0.0.1:2049".parse().unwrap();
let other: SocketAddr = "10.0.0.2:2049".parse().unwrap();
let device = DeviceInfo {
stripe_indices: vec![0, 1],
ds_addrs: vec![vec![mds], vec![other]],
};
assert!(!is_degenerate_device(&device, &mds));
}
#[test]
fn degenerate_device_empty_ds_list() {
let mds: SocketAddr = "10.0.0.1:2049".parse().unwrap();
let device = DeviceInfo {
stripe_indices: vec![],
ds_addrs: vec![],
};
assert!(!is_degenerate_device(&device, &mds));
}
#[test]
fn degenerate_device_port_mismatch() {
let mds: SocketAddr = "10.0.0.1:2049".parse().unwrap();
let same_ip_other_port: SocketAddr = "10.0.0.1:20490".parse().unwrap();
let device = DeviceInfo {
stripe_indices: vec![0],
ds_addrs: vec![vec![same_ip_other_port]],
};
assert!(!is_degenerate_device(&device, &mds));
}
#[test]
fn multipath_sort_exact_match_first() {
let mds: SocketAddr = "10.128.61.201:2049".parse().unwrap();
let mut info = DeviceInfo {
stripe_indices: vec![0],
ds_addrs: vec![vec![
"192.168.13.132:2049".parse().unwrap(),
"10.128.61.201:2049".parse().unwrap(),
]],
};
sort_multipath_by_affinity(&mut info, &mds);
assert_eq!(info.ds_addrs[0][0], mds);
assert!(is_degenerate_device(&info, &mds));
}
#[test]
fn multipath_sort_prefix_affinity() {
let mds: SocketAddr = "10.128.61.201:2049".parse().unwrap();
let near: SocketAddr = "10.128.61.200:2049".parse().unwrap();
let far: SocketAddr = "192.168.13.131:2049".parse().unwrap();
let mut info = DeviceInfo {
stripe_indices: vec![0],
ds_addrs: vec![vec![far, near]],
};
sort_multipath_by_affinity(&mut info, &mds);
assert_eq!(info.ds_addrs[0][0], near);
assert!(!is_degenerate_device(&info, &mds));
}
#[test]
fn multipath_sort_keeps_v4_over_v6_mismatch() {
let mds: SocketAddr = "10.128.61.201:2049".parse().unwrap();
let v6: SocketAddr = "[2001:db8::1]:2049".parse().unwrap();
let v4: SocketAddr = "172.16.0.1:2049".parse().unwrap();
let mut info = DeviceInfo {
stripe_indices: vec![0],
ds_addrs: vec![vec![v6, v4]],
};
sort_multipath_by_affinity(&mut info, &mds);
assert_eq!(info.ds_addrs[0][0], v4);
}
#[tokio::test]
async fn unreachable_ds_mark_and_check() {
let mgr = LayoutManager::new(false);
let addr: SocketAddr = "192.168.13.131:2049".parse().unwrap();
assert!(!mgr.is_ds_unreachable(&addr).await);
mgr.mark_ds_unreachable(addr).await;
assert!(mgr.is_ds_unreachable(&addr).await);
mgr.mark_ds_unreachable(addr).await;
assert!(mgr.is_ds_unreachable(&addr).await);
}
#[tokio::test]
async fn degenerate_log_only_once() {
let mgr = LayoutManager::new(false);
assert!(mgr.should_log_degenerate());
assert!(!mgr.should_log_degenerate());
}
#[tokio::test]
async fn dirty_range_merge() {
let mgr = LayoutManager::new(false);
let fh = Bytes::from_static(b"dirty1");
mgr.mark_dirty(&fh, 4096, 8192).await;
mgr.mark_dirty(&fh, 0, 4096).await;
mgr.mark_dirty(&fh, 16384, 20480).await;
assert_eq!(mgr.take_dirty(&fh).await, Some((0, 20480)));
}
#[tokio::test]
async fn dirty_take_removes() {
let mgr = LayoutManager::new(false);
let fh = Bytes::from_static(b"dirty2");
mgr.mark_dirty(&fh, 100, 200).await;
assert_eq!(mgr.take_dirty(&fh).await, Some((100, 200)));
assert_eq!(mgr.take_dirty(&fh).await, None);
}
#[tokio::test]
async fn dirty_cleared_on_clear() {
let mgr = LayoutManager::new(false);
let fh = Bytes::from_static(b"dirty3");
mgr.mark_dirty(&fh, 0, 1).await;
mgr.clear().await;
assert_eq!(mgr.take_dirty(&fh).await, None);
}
#[tokio::test]
async fn layout_manager_clear() {
let mgr = LayoutManager::new(false);
mgr.store_layout(
&Bytes::from_static(b"a"),
Layout {
stateid: [0u8; 16],
return_on_close: false,
segments: vec![],
},
)
.await;
mgr.store_layout(
&Bytes::from_static(b"b"),
Layout {
stateid: [0u8; 16],
return_on_close: false,
segments: vec![],
},
)
.await;
mgr.clear().await;
assert!(mgr.get_layout(&Bytes::from_static(b"a")).await.is_none());
assert!(mgr.get_layout(&Bytes::from_static(b"b")).await.is_none());
}
#[test]
fn decode_layoutget_empty_segments() {
let mut buf = Vec::new();
put_u32(&mut buf, 1); buf.extend_from_slice(&[5u8; 16]); put_u32(&mut buf, 0);
let mut bytes = Bytes::from(buf);
let layout = decode_layoutget_response(&mut bytes).unwrap();
assert!(layout.return_on_close);
assert_eq!(layout.stateid, [5u8; 16]);
assert!(layout.segments.is_empty());
}
#[test]
fn decode_layoutget_one_segment_opaque() {
let mut buf = Vec::new();
put_u32(&mut buf, 0); buf.extend_from_slice(&[7u8; 16]); put_u32(&mut buf, 1); put_u64(&mut buf, 0); put_u64(&mut buf, 0xFFFFFFFFFFFFFFFF); put_u32(&mut buf, 2); put_u32(&mut buf, 2); let content = vec![0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x11, 0x22];
put_u32(&mut buf, content.len() as u32);
buf.extend_from_slice(&content);
let mut bytes = Bytes::from(buf);
let layout = decode_layoutget_response(&mut bytes).unwrap();
assert_eq!(layout.segments.len(), 1);
assert_eq!(layout.segments[0].iomode, IoMode::ReadWrite);
assert!(matches!(
layout.segments[0].content,
LayoutContent::Opaque(_)
));
}
#[test]
fn decode_files_layout_basic() {
let mut buf = Vec::new();
buf.extend_from_slice(&[0xAAu8; 16]); put_u32(&mut buf, 65536); put_u32(&mut buf, 0); put_u64(&mut buf, 0); put_u32(&mut buf, 2); put_u32(&mut buf, 4);
buf.extend_from_slice(&[1, 2, 3, 4]);
put_u32(&mut buf, 4);
buf.extend_from_slice(&[5, 6, 7, 8]);
let mut bytes = Bytes::from(buf);
let content = decode_files_layout(&mut bytes).unwrap();
match content {
LayoutContent::FilesLayout {
device_id,
stripe_unit,
is_dense,
first_stripe_index,
pattern_offset,
fh_list,
} => {
assert_eq!(device_id, [0xAAu8; 16]);
assert_eq!(stripe_unit, 65536);
assert!(!is_dense);
assert_eq!(first_stripe_index, 0);
assert_eq!(pattern_offset, 0);
assert_eq!(fh_list.len(), 2);
assert_eq!(fh_list[0].as_ref(), &[1, 2, 3, 4]);
assert_eq!(fh_list[1].as_ref(), &[5, 6, 7, 8]);
}
_ => panic!("expected FilesLayout"),
}
}
#[test]
fn decode_layoutget_truncated() {
let buf = vec![0u8; 5]; let mut bytes = Bytes::from(buf);
assert!(decode_layoutget_response(&mut bytes).is_err());
}
#[test]
fn iomode_values() {
assert_eq!(IoMode::Read as u32, 1);
assert_eq!(IoMode::ReadWrite as u32, 2);
}
#[test]
fn layout_type_values() {
assert_eq!(LayoutType::NfsV41Files as u32, 1);
assert_eq!(LayoutType::Osd2Objects as u32, 2);
assert_eq!(LayoutType::BlockVolume as u32, 3);
}
#[test]
fn decode_files_layout_dense_flag() {
let mut buf = Vec::new();
buf.extend_from_slice(&[0xBBu8; 16]); let nfl_util: u32 = 0x4000_0000 | 4096;
put_u32(&mut buf, nfl_util);
put_u32(&mut buf, 2); put_u64(&mut buf, 1024); put_u32(&mut buf, 1); put_u32(&mut buf, 4);
buf.extend_from_slice(&[9, 10, 11, 12]);
let mut bytes = Bytes::from(buf);
let content = decode_files_layout(&mut bytes).unwrap();
match content {
LayoutContent::FilesLayout {
stripe_unit,
is_dense,
first_stripe_index,
pattern_offset,
..
} => {
assert_eq!(stripe_unit, 4096);
assert!(is_dense);
assert_eq!(first_stripe_index, 2);
assert_eq!(pattern_offset, 1024);
}
_ => panic!("expected FilesLayout"),
}
}
#[tokio::test]
async fn device_cache_store_and_get() {
let mgr = LayoutManager::new(false);
let dev_id = [0xCCu8; 16];
let info = DeviceInfo {
stripe_indices: vec![0],
ds_addrs: vec![vec!["10.0.0.1:2049".parse().unwrap()]],
};
mgr.store_device(dev_id, info).await;
let got = mgr.get_device(&dev_id).await;
assert!(got.is_some());
let got = got.unwrap();
assert_eq!(got.ds_addrs.len(), 1);
assert_eq!(
got.ds_addrs[0][0],
"10.0.0.1:2049".parse::<SocketAddr>().unwrap()
);
}
#[tokio::test]
async fn device_cache_cleared_on_clear() {
let mgr = LayoutManager::new(false);
let dev_id = [0xDDu8; 16];
mgr.store_device(
dev_id,
DeviceInfo {
stripe_indices: vec![],
ds_addrs: vec![],
},
)
.await;
mgr.clear().await;
assert!(mgr.get_device(&dev_id).await.is_none());
}
#[test]
fn test_parse_netaddr4_basic() {
let addr = parse_netaddr4("tcp", "192.168.1.1.8.1").unwrap();
assert_eq!(
addr,
SocketAddr::new(IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1)), 2049)
);
}
#[test]
fn test_parse_netaddr4_high_port() {
let addr = parse_netaddr4("tcp", "10.0.0.1.31.144").unwrap();
assert_eq!(
addr,
SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), 8080)
);
}
#[test]
fn test_parse_netaddr4_tcp6() {
let addr = parse_netaddr4("tcp6", "2001:db8::1.8.1").unwrap();
assert_eq!(addr.port(), 2049);
assert_eq!(addr.ip().to_string(), "2001:db8::1");
}
#[test]
fn test_parse_netaddr4_unsupported_netid() {
assert!(parse_netaddr4("udp", "10.0.0.1.8.1").is_err());
}
#[test]
fn test_parse_netaddr4_invalid_addr() {
assert!(parse_netaddr4("tcp", "10.0.0.1.8").is_err()); assert!(parse_netaddr4("tcp", "10.0.0.1.8.1.2").is_err()); assert!(parse_netaddr4("tcp", "999.0.0.1.8.1").is_err()); }
#[test]
fn parse_netaddr4_ipv4() {
let addr = parse_netaddr4("tcp", "192.168.1.1.8.1").unwrap();
assert_eq!(addr.port(), 8 * 256 + 1); assert_eq!(addr.ip().to_string(), "192.168.1.1");
}
#[test]
fn parse_netaddr4_ipv6_loopback() {
let addr = parse_netaddr4("tcp6", "::1.8.1").unwrap();
assert_eq!(addr.port(), 2049);
assert!(addr.ip().is_loopback());
}
#[test]
fn parse_netaddr4_invalid_netid() {
assert!(parse_netaddr4("udp", "1.2.3.4.0.1").is_err());
}
#[test]
fn parse_netaddr4_ipv4_invalid() {
assert!(parse_netaddr4("tcp", "999.1.1.1.0.1").is_err()); }
#[test]
fn test_stripe_ds_index_basic() {
assert_eq!(stripe_ds_index(0, 4096, 0, 4), 0);
assert_eq!(stripe_ds_index(4096, 4096, 0, 4), 1);
assert_eq!(stripe_ds_index(8192, 4096, 0, 4), 2);
assert_eq!(stripe_ds_index(16384, 4096, 0, 4), 0);
}
#[test]
fn test_stripe_ds_index_with_first_stripe_offset() {
assert_eq!(stripe_ds_index(0, 4096, 2, 4), 2);
assert_eq!(stripe_ds_index(4096, 4096, 2, 4), 3);
assert_eq!(stripe_ds_index(8192, 4096, 2, 4), 0);
}
#[test]
fn test_ds_offset_dense() {
assert_eq!(ds_offset(0, 4096, 0, 4, true), 0);
assert_eq!(ds_offset(12345, 4096, 0, 4, true), 57);
assert_eq!(ds_offset(16384, 4096, 0, 4, true), 4096);
}
#[test]
fn test_ds_offset_sparse() {
assert_eq!(ds_offset(0, 4096, 0, 4, false), 0);
assert_eq!(ds_offset(4096, 4096, 0, 4, false), 4096);
assert_eq!(ds_offset(8192, 4096, 0, 4, false), 8192);
assert_eq!(ds_offset(16484, 4096, 0, 4, false), 16484);
}
#[test]
fn test_split_into_stripes_single_stripe() {
let chunks = split_into_stripes(0, 1000, 4096, false, 0, 4, 0);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].ds_index, 0);
assert_eq!(chunks[0].file_offset, 0);
assert_eq!(chunks[0].length, 1000);
}
#[test]
fn test_split_into_stripes_crosses_boundary() {
let chunks = split_into_stripes(3000, 2000, 4096, false, 0, 4, 0);
assert_eq!(chunks.len(), 2);
assert_eq!(chunks[0].ds_index, 0);
assert_eq!(chunks[0].file_offset, 3000);
assert_eq!(chunks[0].length, 1096);
assert_eq!(chunks[1].ds_index, 1);
assert_eq!(chunks[1].file_offset, 4096);
assert_eq!(chunks[1].length, 904);
}
#[test]
fn test_split_into_stripes_multiple() {
let chunks = split_into_stripes(0, 12288, 4096, false, 0, 4, 0);
assert_eq!(chunks.len(), 3);
assert_eq!(chunks[0].ds_index, 0);
assert_eq!(chunks[1].ds_index, 1);
assert_eq!(chunks[2].ds_index, 2);
for c in &chunks {
assert_eq!(c.length, 4096);
}
}
fn put_xdr_string(buf: &mut Vec<u8>, s: &str) {
let bytes = s.as_bytes();
put_u32(buf, bytes.len() as u32);
buf.extend_from_slice(bytes);
let pad = (4 - (bytes.len() % 4)) % 4;
for _ in 0..pad {
buf.push(0);
}
}
#[test]
fn test_decode_getdeviceinfo_single_ds() {
let mut body = Vec::new();
put_u32(&mut body, 1);
put_u32(&mut body, 0);
put_u32(&mut body, 1);
put_u32(&mut body, 1);
put_xdr_string(&mut body, "tcp");
put_xdr_string(&mut body, "192.168.1.1.8.1");
let mut buf = Vec::new();
put_u32(&mut buf, 1); put_u32(&mut buf, body.len() as u32);
buf.extend_from_slice(&body);
let pad = (4 - (body.len() % 4)) % 4;
buf.extend(std::iter::repeat_n(0, pad));
let mut bytes = Bytes::from(buf);
let info = decode_getdeviceinfo_response(&mut bytes).unwrap();
assert_eq!(info.ds_addrs.len(), 1);
assert_eq!(info.ds_addrs[0].len(), 1);
assert_eq!(
info.ds_addrs[0][0],
SocketAddr::new(IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1)), 2049)
);
}
#[test]
fn test_decode_getdeviceinfo_multiple_ds() {
let mut body = Vec::new();
put_u32(&mut body, 2);
put_u32(&mut body, 0);
put_u32(&mut body, 1);
put_u32(&mut body, 2);
put_u32(&mut body, 1);
put_xdr_string(&mut body, "tcp");
put_xdr_string(&mut body, "10.0.0.1.8.1"); put_u32(&mut body, 2);
put_xdr_string(&mut body, "tcp");
put_xdr_string(&mut body, "10.0.0.2.8.1"); put_xdr_string(&mut body, "tcp");
put_xdr_string(&mut body, "10.0.0.3.8.1");
let mut buf = Vec::new();
put_u32(&mut buf, 1);
put_u32(&mut buf, body.len() as u32);
buf.extend_from_slice(&body);
let pad = (4 - (body.len() % 4)) % 4;
buf.extend(std::iter::repeat_n(0, pad));
let mut bytes = Bytes::from(buf);
let info = decode_getdeviceinfo_response(&mut bytes).unwrap();
assert_eq!(info.ds_addrs.len(), 2);
assert_eq!(info.ds_addrs[0].len(), 1);
assert_eq!(info.ds_addrs[1].len(), 2);
assert_eq!(
info.ds_addrs[1][0],
SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)), 2049)
);
assert_eq!(
info.ds_addrs[1][1],
SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 3)), 2049)
);
}
#[test]
fn test_decode_getdeviceinfo_unsupported_layout_type() {
let mut buf = Vec::new();
put_u32(&mut buf, 2); put_u32(&mut buf, 0);
let mut bytes = Bytes::from(buf);
assert!(decode_getdeviceinfo_response(&mut bytes).is_err());
}
#[test]
fn test_decode_getdeviceinfo_truncated() {
let buf = vec![0u8; 2]; let mut bytes = Bytes::from(buf);
assert!(decode_getdeviceinfo_response(&mut bytes).is_err());
}
#[test]
fn ds_offset_dense_multi_ds() {
assert_eq!(ds_offset(0, 4096, 0, 3, true), 0);
assert_eq!(ds_offset(4096, 4096, 0, 3, true), 0);
assert_eq!(ds_offset(8192, 4096, 0, 3, true), 0);
assert_eq!(ds_offset(12288, 4096, 0, 3, true), 4096);
assert_eq!(ds_offset(4097, 4096, 0, 3, true), 1);
}
#[test]
fn split_into_stripes_pattern_offset_zero() {
let chunks = split_into_stripes(0, 8192, 4096, false, 0, 2, 0);
assert_eq!(chunks.len(), 2);
assert_eq!(chunks[0].ds_index, 0);
assert_eq!(chunks[1].ds_index, 1);
}
#[test]
fn split_into_stripes_pattern_offset_nonzero() {
let chunks = split_into_stripes(4096, 4096, 4096, false, 0, 2, 4096);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].ds_index, 0);
assert_eq!(chunks[0].ds_offset, 0);
}
}