use std::path::{Path, PathBuf};
use super::map_stream::{PeerUpdate, StateUpdate, state_update_from_frame};
use crate::NodeCapMap;
pub const NODE_ATTR_CACHE_NETWORK_MAPS: &str = "cache-network-maps";
pub const NODE_ATTR_DISABLE_CACHE_NETWORK_MAPS: &str = "disable-cache-network-maps";
pub const NETMAP_CACHE_FILE: &str = "netmap.json";
const NETMAP_CACHE_TMP_FILE: &str = "netmap.json.tmp";
pub fn netmap_caching_enabled(cap_map: &NodeCapMap) -> bool {
!cap_map.contains_key(NODE_ATTR_DISABLE_CACHE_NETWORK_MAPS)
&& cap_map.contains_key(NODE_ATTR_CACHE_NETWORK_MAPS)
}
#[derive(Debug, Clone)]
pub struct NetmapCache {
dir: PathBuf,
}
impl NetmapCache {
pub fn new(dir: impl Into<PathBuf>) -> Self {
Self { dir: dir.into() }
}
pub fn path(&self) -> PathBuf {
self.dir.join(NETMAP_CACHE_FILE)
}
pub async fn observe(&self, update: &StateUpdate, frame: &[u8]) {
let Some(node) = update.node.as_ref() else {
return;
};
if !netmap_caching_enabled(&node.cap_map) {
self.discard().await;
return;
}
if !matches!(update.peer_update, Some(PeerUpdate::Full(_))) {
return;
}
if let Err(e) = self.store(frame).await {
tracing::warn!(error = %e, path = %self.path().display(), "writing netmap cache");
}
}
async fn store(&self, frame: &[u8]) -> std::io::Result<()> {
create_dir_private(&self.dir).await?;
let tmp = self.dir.join(NETMAP_CACHE_TMP_FILE);
write_private(&tmp, frame).await?;
tokio::fs::rename(&tmp, self.path()).await
}
async fn load(&self) -> Option<Vec<u8>> {
match self.read_cached().await {
Ok(bytes) => Some(bytes),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => None,
Err(e) => {
tracing::warn!(error = %e, path = %self.path().display(), "reading netmap cache");
None
}
}
}
async fn read_cached(&self) -> std::io::Result<Vec<u8>> {
let dir = tokio::fs::symlink_metadata(&self.dir).await?;
ensure_private(&self.dir, &dir, Entry::Dir)?;
read_private(&self.path()).await
}
pub async fn discard(&self) {
match tokio::fs::remove_file(self.path()).await {
Ok(()) => tracing::debug!(path = %self.path().display(), "discarded netmap cache"),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => {
tracing::warn!(error = %e, path = %self.path().display(), "discarding netmap cache");
}
}
}
pub async fn load_state_update(&self) -> Option<StateUpdate> {
let frame = self.load().await?;
let mut update = state_update_from_frame(&frame)?;
update.session_handle = None;
update.seq = 0;
update.ping = None;
if update
.tka
.as_ref()
.is_some_and(crate::TkaStatus::is_enabled)
{
tracing::info!(
"cached netmap was taken under Tailnet Lock; replaying it without its peers (no \
synced authority to verify their key signatures against at cold start)"
);
update.peer_update = None;
update.peer_patches.clear();
}
Some(update)
}
}
async fn create_dir_private(dir: &Path) -> std::io::Result<()> {
match tokio::fs::symlink_metadata(dir).await {
Ok(meta) => return ensure_private(dir, &meta, Entry::Dir),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(e),
}
let mut builder = tokio::fs::DirBuilder::new();
builder.recursive(true);
#[cfg(unix)]
{
builder.mode(0o700);
}
builder.create(dir).await?;
let meta = tokio::fs::symlink_metadata(dir).await?;
ensure_private(dir, &meta, Entry::Dir)
}
async fn write_private(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
use tokio::io::AsyncWriteExt as _;
if let Err(e) = tokio::fs::remove_file(path).await
&& e.kind() != std::io::ErrorKind::NotFound
{
return Err(e);
}
let mut opts = tokio::fs::OpenOptions::new();
opts.write(true).create_new(true);
#[cfg(unix)]
{
opts.mode(0o600);
}
let mut file = opts.open(path).await?;
file.write_all(bytes).await?;
file.flush().await
}
async fn read_private(path: &Path) -> std::io::Result<Vec<u8>> {
use tokio::io::AsyncReadExt as _;
let mut opts = tokio::fs::OpenOptions::new();
opts.read(true);
#[cfg(unix)]
{
opts.custom_flags(libc::O_NOFOLLOW);
}
let mut file = opts.open(path).await?;
ensure_private(path, &file.metadata().await?, Entry::File)?;
let mut bytes = Vec::new();
file.read_to_end(&mut bytes).await?;
Ok(bytes)
}
#[derive(Clone, Copy)]
enum Entry {
Dir,
File,
}
fn ensure_private(path: &Path, meta: &std::fs::Metadata, kind: Entry) -> std::io::Result<()> {
let (ok, want) = match kind {
Entry::Dir => (meta.is_dir(), "a directory"),
Entry::File => (meta.is_file(), "a regular file"),
};
if !ok {
return Err(refused(path, &alloc::format!("not {want}")));
}
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt as _;
let euid = unsafe { libc::geteuid() };
if meta.uid() != euid {
return Err(refused(path, "owned by another user"));
}
if meta.mode() & 0o077 != 0 {
return Err(refused(path, "readable or writable by other users"));
}
}
Ok(())
}
fn refused(path: &Path, why: &str) -> std::io::Error {
std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
alloc::format!("refusing the netmap cache at {}: {why}", path.display()),
)
}
#[cfg(test)]
mod tests {
use futures_util::StreamExt as _;
use super::{super::map_stream::map_stream, *};
fn frame(body: &str) -> Vec<u8> {
let compressed = ruzstd::encoding::compress_to_vec(
body.as_bytes(),
ruzstd::encoding::CompressionLevel::Fastest,
);
let mut buf = (compressed.len() as u32).to_le_bytes().to_vec();
buf.extend_from_slice(&compressed);
buf
}
fn scratch_dir(label: &str) -> PathBuf {
let dir =
std::env::temp_dir().join(format!("ts-rs-netmap-cache-{}-{label}", std::process::id()));
std::fs::remove_dir_all(&dir).ok();
dir
}
fn cache_at(dir: &Path) -> NetmapCache {
NetmapCache::new(dir)
}
fn full_netmap(cap_map: &str) -> String {
full_netmap_with(cap_map, "")
}
fn full_netmap_with(cap_map: &str, extra_fields: &str) -> String {
format!(
r#"{{
{extra_fields}
"MapSessionHandle": "sess-1",
"Seq": 9,
"Node": {{
"ID": 1,
"StableID": "self-1",
"Name": "self.example.ts.net.",
"Addresses": ["100.64.0.1/32"],
"CapMap": {cap_map}
}},
"Peers": [{{
"ID": 2,
"StableID": "peer-2",
"Name": "peer.example.ts.net.",
"Addresses": ["100.64.0.2/32"],
"Endpoints": ["192.0.2.7:41641"],
"HomeDERP": 3
}}],
"DERPMap": {{ "Regions": {{ "3": {{
"RegionID": 3,
"RegionCode": "tst",
"RegionName": "Test",
"Nodes": []
}} }} }},
"PingRequest": {{
"URL": "https://control.example/ping/abc",
"URLIsNoise": false,
"Types": "disco"
}}
}}"#
)
}
async fn poll_one(body: &str, cache: &NetmapCache) {
let buf = frame(body);
let mut stream = core::pin::pin!(map_stream(&buf[..], Some(cache.clone())));
stream.next().await.expect("one netmap");
}
#[tokio::test]
async fn cached_netmap_is_used_on_cold_start() {
let dir = scratch_dir("cold-start");
let cache = NetmapCache::new(&dir);
poll_one(&full_netmap(r#"{"cache-network-maps": null}"#), &cache).await;
let replayed = NetmapCache::new(&dir)
.load_state_update()
.await
.expect("a cached netmap must be replayable on cold start");
let node = replayed.node.as_ref().expect("self node");
assert_eq!(node.stable_id.0, "self-1");
assert!(
netmap_caching_enabled(&node.cap_map),
"the replayed self node must carry the attribute that made it cacheable"
);
let Some(PeerUpdate::Full(peers)) = replayed.peer_update.as_ref() else {
panic!(
"the cached netmap must replay a full peer set, got {:?}",
replayed.peer_update
);
};
assert_eq!(peers.len(), 1);
assert_eq!(peers[0].stable_id.0, "peer-2");
assert_eq!(
peers[0].underlay_addresses,
vec!["192.0.2.7:41641".parse().unwrap()],
"the peer's endpoints are the point of the cache: they are what a cold start dials"
);
assert!(
replayed.derp.is_some(),
"the DERP map must survive the round trip; without it a cold start has no relay"
);
assert_eq!(replayed.session_handle, None);
assert_eq!(replayed.seq, 0);
assert!(replayed.ping.is_none(), "a stale ping must not be replayed");
std::fs::remove_dir_all(&dir).ok();
}
#[tokio::test]
async fn disable_cache_network_maps_suppresses_the_cache() {
let dir = scratch_dir("disable-attr");
let cache = NetmapCache::new(&dir);
poll_one(
&full_netmap(r#"{"cache-network-maps": null, "disable-cache-network-maps": null}"#),
&cache,
)
.await;
assert!(
!cache.path().exists(),
"disable-cache-network-maps must suppress the write even when the enabling \
attribute is also granted"
);
assert!(
NetmapCache::new(&dir).load_state_update().await.is_none(),
"a suppressed cache must leave a cold start with nothing to replay"
);
std::fs::remove_dir_all(&dir).ok();
}
#[tokio::test]
async fn withdrawing_the_attribute_discards_an_existing_cache() {
for (label, cap_map) in [
("dropped", "{}"),
(
"overridden",
r#"{"cache-network-maps": null, "disable-cache-network-maps": null}"#,
),
] {
let dir = scratch_dir(&format!("withdraw-{label}"));
let cache = NetmapCache::new(&dir);
poll_one(&full_netmap(r#"{"cache-network-maps": null}"#), &cache).await;
assert!(cache.path().exists(), "{label}: the grant must cache first");
poll_one(&full_netmap(cap_map), &cache).await;
assert!(
!cache.path().exists(),
"{label}: withdrawing the grant must discard the cached netmap"
);
std::fs::remove_dir_all(&dir).ok();
}
}
#[tokio::test]
async fn no_attribute_never_caches() {
let dir = scratch_dir("no-attr");
let cache = NetmapCache::new(&dir);
poll_one(&full_netmap("{}"), &cache).await;
assert!(
!dir.exists(),
"an ungranted node must not so much as create the cache directory"
);
std::fs::remove_dir_all(&dir).ok();
}
#[tokio::test]
async fn delta_frames_do_not_replace_the_cached_full_netmap() {
let dir = scratch_dir("delta");
let cache = NetmapCache::new(&dir);
poll_one(&full_netmap(r#"{"cache-network-maps": null}"#), &cache).await;
let cached = std::fs::read(cache.path()).expect("cached netmap");
poll_one(
r#"{
"Seq": 10,
"Node": { "ID": 1, "StableID": "self-1", "CapMap": {"cache-network-maps": null} },
"PeersChanged": [{ "ID": 3, "StableID": "peer-3", "Name": "late.example.ts.net." }]
}"#,
&cache,
)
.await;
assert_eq!(
std::fs::read(cache.path()).expect("cached netmap"),
cached,
"a delta frame must not overwrite the cached full netmap"
);
std::fs::remove_dir_all(&dir).ok();
}
#[cfg(unix)]
#[tokio::test]
async fn cached_netmap_is_written_private() {
use std::os::unix::fs::PermissionsExt as _;
let dir = scratch_dir("perms");
let cache = NetmapCache::new(&dir);
poll_one(&full_netmap(r#"{"cache-network-maps": null}"#), &cache).await;
let file = std::fs::metadata(cache.path()).expect("cached netmap");
assert_eq!(
file.permissions().mode() & 0o777,
0o600,
"the cached netmap must be readable only by this user"
);
let parent = std::fs::metadata(&dir).expect("cache dir");
assert_eq!(
parent.permissions().mode() & 0o777,
0o700,
"the cache directory must be private too"
);
std::fs::remove_dir_all(&dir).ok();
}
#[tokio::test]
async fn peers_cached_under_tailnet_lock_are_not_replayed() {
let dir = scratch_dir("tka-locked");
let cache = NetmapCache::new(&dir);
poll_one(
&full_netmap_with(
r#"{"cache-network-maps": null}"#,
r#""TKAInfo": { "Head": "s7ovkkqcbxlaqedbmdyrhqzhqu", "Disabled": false },"#,
),
&cache,
)
.await;
let replayed = NetmapCache::new(&dir)
.load_state_update()
.await
.expect("a locked tailnet still replays the netmap");
assert!(
replayed.peer_update.is_none(),
"peers cached under Tailnet Lock must not be replayed unverified, got {:?}",
replayed.peer_update
);
assert!(replayed.peer_patches.is_empty());
assert!(
replayed.node.is_some() && replayed.derp.is_some(),
"the rest of the netmap carries no peer identity and must still replay"
);
}
#[tokio::test]
async fn peers_cached_with_the_lock_disabled_still_replay() {
let dir = scratch_dir("tka-disabled");
let cache = NetmapCache::new(&dir);
poll_one(
&full_netmap_with(
r#"{"cache-network-maps": null}"#,
r#""TKAInfo": { "Head": "s7ovkkqcbxlaqedbmdyrhqzhqu", "Disabled": true },"#,
),
&cache,
)
.await;
let replayed = NetmapCache::new(&dir)
.load_state_update()
.await
.expect("a cached netmap");
assert!(
matches!(replayed.peer_update, Some(PeerUpdate::Full(ref p)) if p.len() == 1),
"a disabled lock enforces nothing, so its peers replay: {:?}",
replayed.peer_update
);
}
#[cfg(unix)]
#[tokio::test]
async fn a_shared_cache_directory_is_refused() {
use std::os::unix::fs::PermissionsExt as _;
let dir = scratch_dir("shared-dir");
std::fs::create_dir_all(&dir).expect("scratch dir");
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)).expect("chmod");
poll_one(
&full_netmap(r#"{"cache-network-maps": null}"#),
&cache_at(&dir),
)
.await;
assert!(
!cache_at(&dir).path().exists(),
"a netmap must not be written into a directory other users can read"
);
std::fs::remove_dir_all(&dir).ok();
}
#[cfg(unix)]
#[tokio::test]
async fn a_cache_in_a_shared_directory_is_not_replayed() {
use std::os::unix::fs::PermissionsExt as _;
let dir = scratch_dir("shared-dir-load");
std::fs::create_dir_all(&dir).expect("scratch dir");
std::fs::write(
dir.join(NETMAP_CACHE_FILE),
full_netmap(r#"{"cache-network-maps": null}"#),
)
.expect("plant a netmap");
std::fs::set_permissions(
dir.join(NETMAP_CACHE_FILE),
std::fs::Permissions::from_mode(0o600),
)
.expect("chmod");
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).expect("chmod");
assert!(
cache_at(&dir).load_state_update().await.is_some(),
"control: a private cache replays"
);
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o777)).expect("chmod");
assert!(
cache_at(&dir).load_state_update().await.is_none(),
"a netmap another local user could have swapped must not be replayed"
);
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).ok();
std::fs::remove_dir_all(&dir).ok();
}
#[cfg(unix)]
#[tokio::test]
async fn a_shared_or_symlinked_cache_file_is_not_replayed() {
use std::os::unix::fs::PermissionsExt as _;
let dir = scratch_dir("shared-file");
std::fs::create_dir_all(&dir).expect("scratch dir");
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).expect("chmod");
let body = full_netmap(r#"{"cache-network-maps": null}"#);
std::fs::write(dir.join(NETMAP_CACHE_FILE), &body).expect("plant a netmap");
std::fs::set_permissions(
dir.join(NETMAP_CACHE_FILE),
std::fs::Permissions::from_mode(0o644),
)
.expect("chmod");
assert!(
cache_at(&dir).load_state_update().await.is_none(),
"a world-readable cache file is not one this node wrote; it must not be replayed"
);
let elsewhere = dir.join("planted.json");
std::fs::write(&elsewhere, &body).expect("plant a netmap");
std::fs::set_permissions(&elsewhere, std::fs::Permissions::from_mode(0o600))
.expect("chmod");
std::fs::remove_file(dir.join(NETMAP_CACHE_FILE)).expect("clear");
std::os::unix::fs::symlink(&elsewhere, dir.join(NETMAP_CACHE_FILE)).expect("symlink");
assert!(
cache_at(&dir).load_state_update().await.is_none(),
"the cache path must be a regular file, never a redirect to one"
);
std::fs::remove_dir_all(&dir).ok();
}
#[cfg(unix)]
#[tokio::test]
async fn a_stale_temporary_file_never_lends_the_netmap_its_mode() {
use std::os::unix::fs::PermissionsExt as _;
let dir = scratch_dir("stale-tmp");
std::fs::create_dir_all(&dir).expect("scratch dir");
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).expect("chmod");
let tmp = dir.join(NETMAP_CACHE_TMP_FILE);
std::fs::write(&tmp, b"half a netmap").expect("stale temporary file");
std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o666)).expect("chmod");
poll_one(
&full_netmap(r#"{"cache-network-maps": null}"#),
&cache_at(&dir),
)
.await;
let cached = cache_at(&dir).path();
assert_eq!(
std::fs::metadata(&cached)
.expect("cached netmap")
.permissions()
.mode()
& 0o777,
0o600,
"the netmap must land in a file this write created, at this write's mode"
);
assert!(
cache_at(&dir).load_state_update().await.is_some(),
"and it must still be the netmap"
);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn caching_is_granted_only_by_the_enabling_attribute_alone() {
let cap = |caps: &[&str]| -> NodeCapMap {
caps.iter().map(|c| ((*c).to_owned(), Vec::new())).collect()
};
assert!(netmap_caching_enabled(&cap(&[
NODE_ATTR_CACHE_NETWORK_MAPS
])));
assert!(!netmap_caching_enabled(&cap(&[])));
assert!(!netmap_caching_enabled(&cap(&[
NODE_ATTR_DISABLE_CACHE_NETWORK_MAPS
])));
assert!(!netmap_caching_enabled(&cap(&[
NODE_ATTR_CACHE_NETWORK_MAPS,
NODE_ATTR_DISABLE_CACHE_NETWORK_MAPS
])));
}
}