1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
70use std::net::SocketAddr;
71use std::path::{Path, PathBuf};
72use std::sync::Mutex;
73use std::time::Duration;
74
75use borg::collective::{self, Config, Route};
76use borg::msg::{Line, V1_RATE_MS};
77use iroh::Endpoint;
78use iroh::endpoint::{Builder, ConnectOptions, Connection, ConnectionError, Incoming, IncomingAddr, PortmapperConfig, RecvStream, SendStream, presets};
79use iroh::tls::CaTlsConfig;
80use serde::{Deserialize, Serialize};
81
82use crate::addr::{self, Hint, Relay, Target};
83use crate::identity::Identity;
84use crate::invite::{self, Invite, Ticket};
85use crate::lines::{self, Hash, REASONING, SYNCED, logs};
86use crate::membership::Membership;
87use crate::origin::{self, Origin};
88use crate::recon::{self, Recon};
89use crate::{from_hex, peers, to_hex};
90use crate::{live, steer};
91use borg::protocol::PeerVersion;
92
93mod vectors;
94
95pub const ALPN: &[u8] = b"borg/mesh/15";
101pub const V14_ALPN: &[u8] = b"borg/mesh/14";
104pub const V13_ALPN: &[u8] = b"borg/mesh/13";
109pub const LEGACY_ALPN: &[u8] = b"borg/mesh/12";
112fn older() -> Vec<Vec<u8>> {
116 let below = version(LEGACY_ALPN).unwrap_or(0);
117 (1..below).rev().map(|v| format!("borg/mesh/{v}").into_bytes()).collect()
118}
119
120const MISMATCH_CLOSE: u32 = 2;
124
125#[derive(Clone, Copy)]
127struct Peer<'a> {
128 key: &'a str,
129 alpn: &'a [u8],
130}
131
132fn spoken(alpn: &[u8]) -> bool {
135 alpn == ALPN || alpn == V14_ALPN || alpn == V13_ALPN || alpn == LEGACY_ALPN
136}
137
138const PLAYBOOKS: &str = "playbooks.jsonl";
142
143const STEPS: [&str; 7] = [SYNCED[0], SYNCED[1], SYNCED[2], PLAYBOOKS, SYNCED[3], SYNCED[4], SYNCED[5]];
145
146fn synced(alpn: &[u8]) -> impl Iterator<Item = &'static str> {
150 let (absorbs, messages) = (alpn == ALPN, alpn != LEGACY_ALPN);
151 STEPS.into_iter().filter(move |f| (absorbs || *f != lines::ABSORBED) && (messages || *f != borg::msg::FILE))
152}
153
154fn version(alpn: &[u8]) -> Option<u32> {
156 std::str::from_utf8(alpn).ok()?.strip_prefix("borg/mesh/")?.parse().ok()
157}
158fn reads(alpn: &[u8]) -> u8 {
162 if alpn == ALPN || alpn == V14_ALPN {
163 2
164 } else if alpn == V13_ALPN {
165 1
166 } else {
167 0
168 }
169}
170pub const JOIN_ALPN: &[u8] = b"borg/join/1";
172const MAX_FRAME: usize = 256 << 20;
175const MAX_GIVE: usize = 64 << 20;
178pub const REPLICAS: usize = 2;
181const MAX_ASK: usize = 1000;
184const LEGACY_GOAL_LOGS: usize = 7;
187const V1_KEEP: usize = 500;
194
195#[derive(Debug, Clone)]
199pub struct Place {
200 pub config: PathBuf,
202 pub base: PathBuf,
204 pub dir: String,
206 pub collective: Option<String>,
209}
210
211impl Place {
212 pub fn from_env(dir: &str) -> Place {
213 Place { config: collective::config_path(), base: borg::learn::home(), dir: dir.into(), collective: None }
214 }
215
216 pub fn named(&self, name: &str) -> Place {
218 Place { collective: Some(name.into()), ..self.clone() }
219 }
220
221 pub fn route(&self) -> Route {
223 self.resolve().1
224 }
225
226 fn resolve(&self) -> (Option<Config>, Route) {
227 match collective::load_from(&self.config) {
228 Ok(cfg) => {
229 let r = match &self.collective {
230 None => collective::route(cfg.as_ref(), &self.dir, &self.base),
231 Some(name) => match cfg.as_ref() {
232 None if name == "default" => collective::route(None, &self.dir, &self.base),
233 None => Route::Off,
234 Some(c) => c.collectives.iter().find(|e| e.name == *name).map_or(Route::Off, |e| Route::Collective {
235 name: e.name.clone(),
236 home: collective::store(&self.base, &e.name),
237 share: e.share,
238 }),
239 },
240 };
241 (cfg, r)
242 }
243 Err(_) => (None, Route::Off),
244 }
245 }
246
247 pub fn relay(&self) -> Result<Relay, String> {
249 let (cfg, route) = self.resolve();
250 let configured = match (&cfg, &route) {
251 (Some(c), Route::Collective { name, .. }) => c.collectives.iter().find(|e| e.name == *name).and_then(|e| e.relay.clone()),
252 _ => None,
253 };
254 Relay::choose(configured.as_deref())
255 }
256
257 pub fn gate(&self) -> Result<(String, PathBuf), String> {
259 let (cfg, route) = self.resolve();
260 if let (Some(name), Route::Off) = (&self.collective, &route) {
261 return Err(format!("collective {name:?} is not in {}", self.config.display()));
262 }
263 let (name, home) = gate(&route)?;
264 if let Some(local) = collective::local_owner_of(cfg.as_ref(), home, &self.base) {
265 return Err(format!("store {} also belongs to local collective {local:?}: refusing to sync", home.display()));
266 }
267 Ok((name.to_string(), home.to_path_buf()))
268 }
269}
270
271pub fn gate(route: &Route) -> Result<(&str, &Path), String> {
273 match route {
274 Route::Collective { name, home, .. } if collective::may_sync(route) => Ok((name, home)),
275 Route::Collective { name, .. } => Err(format!("collective {name:?} is local: it never leaves this machine, refusing to sync")),
276 Route::Off => Err("this repo belongs to no collective (off): nothing may sync".into()),
277 }
278}
279
280#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
282pub struct Report {
283 pub sent: usize,
285 pub received: usize,
286 pub rejected: usize,
288 pub quarantined: usize,
290 pub rounds: usize,
292 pub fps: usize,
293 pub hashes: usize,
294 pub replicated: usize,
296 pub answered: usize,
298 pub presence: usize,
300 pub steered: usize,
302 pub dropped: usize,
303 pub joined: usize,
305}
306
307#[derive(Debug, Clone, Serialize, Deserialize)]
309pub struct Signed {
310 pub line: String,
311 pub origin: Origin,
312}
313
314#[derive(Debug, Serialize, Deserialize)]
315#[serde(tag = "t", rename_all = "snake_case")]
316enum Msg {
317 Hello {
321 collective: String,
322 listen: Option<String>,
323 #[serde(default)]
324 relay: Option<String>,
325 #[serde(default, skip_serializing_if = "Option::is_none")]
326 borg: Option<String>,
327 },
328 Refused {
329 reason: String,
330 },
331 Join {
332 invite: Invite,
333 listen: Option<String>,
334 relay: Option<String>,
335 },
336 Welcome {
337 docs: Vec<Membership>,
338 },
339 Membership {
340 docs: Vec<Membership>,
341 },
342 Round {
343 round: recon::Round,
344 },
345 Lines {
346 lines: Vec<Signed>,
347 },
348 Goals {
350 ids: Vec<String>,
351 },
352 Offer {
353 hashes: Vec<String>,
354 },
355 Want {
356 hashes: Vec<String>,
357 },
358 Ask {
359 commits: Vec<String>,
360 },
361 Live {
362 presence: Vec<borg::presence::Remote>,
363 #[serde(default)]
364 directives: Directives,
365 },
366}
367
368#[derive(Debug, Default)]
371struct Directives;
372
373impl Serialize for Directives {
374 fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
375 s.collect_seq(std::iter::empty::<()>())
376 }
377}
378
379impl<'de> Deserialize<'de> for Directives {
380 fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
381 serde::de::IgnoredAny::deserialize(d).map(|_| Directives)
382 }
383}
384
385fn ca_trust() -> CaTlsConfig {
390 let system = CaTlsConfig::system();
391 match system.server_cert_verifier(iroh::tls::default_provider()) {
392 Ok(_) => system,
393 Err(_) => CaTlsConfig::embedded(),
394 }
395}
396
397pub struct Node {
399 place: Place,
400 home: PathBuf,
402 me: Identity,
403 endpoint: Endpoint,
404 store: Mutex<()>,
406 announce: bool,
408 relay: Relay,
410 legacy: Mutex<HashMap<(String, String, usize), BTreeSet<Hash>>>,
414 speaks: Speaks,
416 announcing: Mutex<()>,
419}
420
421#[derive(Debug, Clone, Copy)]
425struct Speaks {
426 alpn: &'static [u8],
427 borg: Option<&'static str>,
429 tell: bool,
430}
431
432impl Speaks {
433 fn this() -> Speaks {
434 Speaks { alpn: ALPN, borg: Some(env!("CARGO_PKG_VERSION")), tell: tells(std::env::var("BORG_ANNOUNCE").ok().as_deref()) }
435 }
436}
437
438fn tells(var: Option<&str>) -> bool {
440 var != Some("off")
441}
442
443impl Node {
444 pub async fn bind(place: &Place, addr: SocketAddr) -> Result<Node, String> {
447 Node::open(place, Some(addr), true, |b| b).await
448 }
449
450 pub async fn joining(place: &Place, addr: SocketAddr) -> Result<Node, String> {
452 Node::open(place, Some(addr), false, |b| b).await
453 }
454
455 pub async fn open(place: &Place, addr: Option<SocketAddr>, member: bool, tweak: impl FnOnce(Builder) -> Builder) -> Result<Node, String> {
459 let (_, home) = place.gate()?;
460 let relay = place.relay()?;
461 let me = Identity::load_or_create(&home)?;
462 if member {
463 members(&home)?;
464 }
465 let builder = match &relay {
466 Relay::Default => Endpoint::builder(presets::N0),
468 Relay::Custom(_) => Endpoint::builder(presets::Minimal).relay_mode(relay.mode()),
470 Relay::Disabled => Endpoint::builder(presets::Minimal).relay_mode(relay.mode()).portmapper_config(PortmapperConfig::Disabled),
472 };
473 let mut builder = builder
474 .secret_key(me.iroh_secret())
475 .alpns([vec![ALPN.to_vec(), JOIN_ALPN.to_vec(), V14_ALPN.to_vec(), V13_ALPN.to_vec(), LEGACY_ALPN.to_vec()], older()].concat())
476 .clear_ip_transports()
477 .ca_tls_config(ca_trust());
478 if let Some(a) = addr {
479 builder = builder.bind_addr(a).map_err(|e| e.to_string())?;
480 }
481 let endpoint = tweak(builder).bind().await.map_err(|e| format!("bind {}: {e}", addr.map(|a| a.to_string()).unwrap_or("relay".into())))?;
482 Ok(Node {
483 place: place.clone(),
484 home,
485 me,
486 endpoint,
487 store: Mutex::new(()),
488 announce: true,
489 relay,
490 legacy: Mutex::default(),
491 speaks: Speaks::this(),
492 announcing: Mutex::new(()),
493 })
494 }
495
496 pub async fn online(&self, within: Duration) -> bool {
499 self.relay == Relay::Disabled || tokio::time::timeout(within, self.endpoint.online()).await.is_ok()
500 }
501
502 pub fn home_relay(&self) -> Option<String> {
504 self.endpoint.addr().relay_urls().next().map(|u| u.to_string())
505 }
506
507 pub fn hints(&self) -> Vec<Hint> {
510 let mut h: Vec<Hint> = self.tickets().iter().filter_map(|t| Target::parse(t).ok()).flat_map(|t| t.hints).collect();
511 h.extend(self.home_relay().and_then(|u| addr::hints(&u).ok()).into_iter().flatten());
512 h
513 }
514
515 pub fn relay(&self) -> &Relay {
516 &self.relay
517 }
518
519 pub fn quiet(mut self) -> Node {
522 self.announce = false;
523 self
524 }
525
526 pub fn gate(&self) -> Result<(String, PathBuf), String> {
528 let (name, home) = self.place.gate()?;
529 if home != self.home {
530 return Err(format!("{} now routes to store {}, this node serves {}: restart it", self.place.dir, home.display(), self.home.display()));
531 }
532 Ok((name, home))
533 }
534
535 pub fn home(&self) -> &Path {
536 &self.home
537 }
538
539 pub fn public(&self) -> String {
540 self.me.public()
541 }
542
543 fn addr(&self) -> Option<SocketAddr> {
544 self.endpoint.bound_sockets().first().copied()
545 }
546
547 pub fn ticket(&self) -> String {
549 self.tickets().into_iter().next().unwrap_or_else(|| format!("{}@{}", self.me.public(), self.addr().map(|a| a.to_string()).unwrap_or_default()))
550 }
551
552 pub fn tickets(&self) -> Vec<String> {
556 let Some(bound) = self.addr() else { return Vec::new() };
557 let addrs: Vec<SocketAddr> = if bound.ip().is_unspecified() {
558 let mut v: Vec<SocketAddr> = self
559 .endpoint
560 .addr()
561 .ip_addrs()
562 .filter(|a| !a.ip().is_unspecified() && a.is_ipv4() == bound.is_ipv4())
563 .map(|a| SocketAddr::new(a.ip(), bound.port()))
564 .collect();
565 v.sort_by_key(|a| (a.ip().is_loopback(), *a));
566 v.dedup();
567 v
568 } else {
569 vec![bound]
570 };
571 addrs.into_iter().map(|a| format!("{}@{a}", self.me.public())).collect()
572 }
573
574 fn listen(&self) -> Option<String> {
576 self.announce.then(|| self.addr()).flatten().map(|a| a.to_string())
577 }
578
579 pub fn enforce(&self) -> Result<usize, String> {
582 let _w = self.store.lock().unwrap_or_else(|e| e.into_inner());
583 let m = members(&self.home)?;
584 steer::enforce(&self.home, &m, borg::learn::now_ms());
585 origin::apply_if_new(&self.home, &m)
586 }
587
588 pub fn stamp(&self) -> Result<usize, String> {
590 self.local().stamp()
591 }
592
593 fn local(&self) -> Store<'_> {
595 Store { home: &self.home, lock: &self.store, me: &self.me }
596 }
597
598 fn synced(&self, remote: &str, alpn: &[u8], started: i64, borg: Option<String>) {
605 {
606 let _w = self.store.lock().unwrap_or_else(|e| e.into_inner());
607 let _ = borg::presence::note_synced(&self.home, remote, started);
608 let _ = borg::presence::forget_mismatch(&self.home, remote);
609 let _ = borg::presence::note_reads(&self.home, remote, reads(alpn));
610 if let Some(protocol) = version(alpn) {
611 let now = borg::learn::now_ms();
612 let engineer = engineer_of(&self.home, remote, now);
613 let v = PeerVersion { borg, protocol, ms: now, engineer };
614 let _ = borg::protocol::note_version(&self.home, remote, &v);
615 }
616 }
617 self.announce(alpn);
618 }
619
620 fn announce(&self, alpn: &[u8]) -> Option<borg::msg::Message> {
628 let (ours, theirs) = (version(self.speaks.alpn)?, version(alpn)?);
629 if !self.speaks.tell || theirs >= ours {
630 return None;
631 }
632 let _one = self.announcing.lock().unwrap_or_else(|e| e.into_inner());
633 if borg::protocol::announced(&self.home, ours) {
634 return None;
635 }
636 let email: String = borg::learn::engineer(&self.place.dir).chars().filter(|c| c.is_ascii_alphanumeric() || "-_.@+".contains(*c)).collect();
639 let (by, who) = match email.as_str() {
640 "" => (borg::msg::ENGINEER.to_string(), format!("the engineer of node {}", borg::msg::short_node(&self.me.public()))),
641 e => (format!("{}:{e}", borg::msg::ENGINEER), e.to_string()),
642 };
643 let name = if email.is_empty() { borg::msg::ENGINEER.to_string() } else { email };
644 let from = borg::msg::Sender { by, name, harness: "terminal".into(), node: self.me.public() };
645 let now = borg::learn::now_ms();
646 if v1_slot_held(&borg::msg::read(&self.home), &from, now) {
647 return None;
648 }
649 let text = borg::protocol::announcement(ours, theirs, &who, self.speaks.borg.unwrap_or("older than 0.7"));
650 borg::msg::send(&self.home, borg::msg::Message::new(borg::msg::TELL, &from, borg::msg::ALL, None, now, &text), now).ok()
651 }
652
653 fn mismatch(&self, conn: &Connection, remote: &str) -> String {
657 let m = borg::presence::Mismatch {
658 theirs: String::from_utf8_lossy(conn.alpn()).into_owned(),
659 ours: String::from_utf8_lossy(ALPN).into_owned(),
660 ms: borg::learn::now_ms(),
661 };
662 let (line, told) = (m.line(remote), m.told(&self.me.public()));
663 {
664 let _w = self.store.lock().unwrap_or_else(|e| e.into_inner());
665 let _ = borg::presence::note_mismatch(&self.home, remote, m);
666 }
667 conn.close(MISMATCH_CLOSE.into(), told.as_bytes());
668 line
669 }
670
671 fn refused(&self, conn: &Connection, remote: &str) -> Option<String> {
676 let Some(ConnectionError::ApplicationClosed(c)) = conn.close_reason() else { return None };
677 if u64::from(c.error_code) != u64::from(MISMATCH_CLOSE) {
678 return None;
679 }
680 let said = borg::presence::Mismatch::from_told(&String::from_utf8_lossy(&c.reason), borg::learn::now_ms())?;
681 let ours = version(ALPN)?;
682 if borg::protocol::number(&said.theirs)? <= ours {
683 return None;
684 }
685 let m = borg::presence::Mismatch { ours: String::from_utf8_lossy(ALPN).into_owned(), ..said };
686 let line = m.line(remote);
687 let _w = self.store.lock().unwrap_or_else(|e| e.into_inner());
688 let _ = borg::presence::note_mismatch(&self.home, remote, m);
689 Some(line)
690 }
691
692 pub(crate) fn unreachable(&self, remote: &str) {
694 let _w = self.store.lock().unwrap_or_else(|e| e.into_inner());
695 let _ = borg::presence::forget_synced(&self.home, remote);
696 }
697
698 fn borg(&self) -> Option<String> {
700 self.speaks.borg.map(str::to_string)
701 }
702
703 fn announced(&self) -> Option<String> {
705 self.announce.then(|| self.home_relay()).flatten()
706 }
707
708 pub async fn sync(&self, target: &str) -> Result<Report, String> {
710 self.fetch(target, &[]).await
711 }
712
713 pub async fn fetch(&self, target: &str, commits: &[String]) -> Result<Report, String> {
716 self.dial(target, commits, ALPN, [vec![V14_ALPN.to_vec(), V13_ALPN.to_vec(), LEGACY_ALPN.to_vec()], older()].concat()).await
717 }
718
719 async fn dial(&self, target: &str, commits: &[String], alpn: &[u8], also: Vec<Vec<u8>>) -> Result<Report, String> {
721 let (name, home) = self.gate()?;
722 let (name, home) = (name.as_str(), home.as_path());
723 let membership = members(home)?;
724 let t = Target::parse(target)?;
725 if membership.role_of(&t.key).is_none() {
726 return Err(format!("{} is not a member of {name}: refusing to connect", t.key));
727 }
728 let started = borg::learn::now_ms();
729 let connecting = self.endpoint.connect_with_opts(t.endpoint_addr(&self.relay)?, alpn, ConnectOptions::new().with_additional_alpns(also));
730 let conn = connecting.await.map_err(|e| format!("connect {target}: {e}"))?.await.map_err(|e| format!("connect {target}: {e}"))?;
731 if !spoken(conn.alpn()) {
732 return Err(self.mismatch(&conn, &t.key));
733 }
734 let result = async {
735 let remote = authorise(&conn, &membership, name)?;
736 let (mut tx, mut rx) = conn.open_bi().await.map_err(|e| e.to_string())?;
737 let hello = Msg::Hello { collective: name.into(), listen: self.listen(), relay: self.announced(), borg: self.borg() };
738 let Msg::Hello { collective, borg, .. } = swap(true, &mut tx, &mut rx, hello).await? else {
739 return Err("protocol: expected hello".into());
740 };
741 if collective != name {
742 return Err(format!("peer answered for collective {collective:?}, not {name:?}"));
743 }
744 let report = self.exchange(Peer { key: &remote, alpn: conn.alpn() }, true, &mut tx, &mut rx, membership, commits).await?;
745 let _ = tx.finish();
746 self.synced(&remote, conn.alpn(), started, borg);
747 Ok(report)
748 }
749 .await;
750 let result = result.map_err(|e| self.refused(&conn, &t.key).unwrap_or(e));
751 conn.close(0u32.into(), if result.is_ok() { b"done" } else { b"error" });
752 vectors::reindexed(&self.home, result).await
753 }
754
755 pub async fn join(&self, ticket: &Ticket) -> Result<(Membership, Report), String> {
759 let (name, home) = self.gate()?;
760 let (name, home) = (name.as_str(), home.as_path());
761 if ticket.invite.collective != name {
762 return Err(format!("the ticket is for collective {:?}, this node is {name:?}", ticket.invite.collective));
763 }
764 let admin = ticket.target();
765 let conn = self.endpoint.connect(admin.endpoint_addr(&self.relay)?, JOIN_ALPN).await.map_err(|e| format!("connect {}: {e}", admin.key))?;
766 let me = self.public();
767 let result = async {
768 let (mut tx, mut rx) = conn.open_bi().await.map_err(|e| e.to_string())?;
769 let ask = Msg::Join { invite: ticket.invite.clone(), listen: self.listen(), relay: self.announced() };
770 let Msg::Welcome { docs } = swap(true, &mut tx, &mut rx, ask).await? else { return Err("protocol: expected welcome".into()) };
771 let _ = tx.finish();
772 let _w = self.store.lock().unwrap_or_else(|e| e.into_inner());
773 let installed = match origin::join(home, name, &docs) {
774 Ok((m, _)) => m,
775 Err(e) => Membership::load(home)?.filter(|m| m.role_of(&me).is_some()).ok_or(e)?,
777 };
778 if installed.role_of(&me).is_none() {
779 return Err(format!("the admin's membership of {name} does not list this node ({me})"));
780 }
781 steer::enforce(home, &installed, borg::learn::now_ms());
782 Ok(installed)
783 }
784 .await;
785 conn.close(0u32.into(), if result.is_ok() { b"done" } else { b"error" });
786 let installed = result?;
787 peers::remember(home, &admin.key, &addr::join_hints(&admin.hints))?;
788 let report = self.sync(&admin.to_string()).await.map_err(|e| format!("joined {name}, but the first sync failed: {e}"))?;
789 Ok((installed, report))
790 }
791
792 async fn admit(&self, conn: &Connection, observed: Option<SocketAddr>) -> Result<Report, String> {
796 let (name, home) = self.gate()?;
797 let (name, home) = (name.as_str(), home.as_path());
798 let joiner = to_hex(conn.remote_id().as_bytes());
799 let (mut tx, mut rx) = conn.accept_bi().await.map_err(|e| e.to_string())?;
800 let Msg::Join { invite, listen, relay } = recv(&mut rx).await? else { return Err("protocol: expected join".into()) };
801 let verdict = {
802 let _w = self.store.lock().unwrap_or_else(|e| e.into_inner());
803 invite::redeem(home, &self.me, name, &invite, &joiner, borg::learn::now_ms())
804 };
805 let m = match verdict {
806 Ok(m) => m,
807 Err(reason) => {
808 send(&mut tx, &Msg::Refused { reason: reason.clone() }).await?;
809 let _ = tx.finish();
810 let _ = tokio::time::timeout(Duration::from_secs(5), conn.closed()).await;
812 return Err(format!("join by {joiner}: {reason}"));
813 }
814 };
815 steer::enforce(home, &m, borg::learn::now_ms());
816 heard(home, &joiner, listen, relay, observed);
817 send(&mut tx, &Msg::Welcome { docs: Membership::history(home) }).await?;
818 let _ = tx.finish();
819 conn.closed().await; Ok(Report { joined: 1, ..Report::default() })
821 }
822
823 pub async fn incoming(&self) -> Option<Incoming> {
825 self.endpoint.accept().await
826 }
827
828 pub async fn accept(&self) -> Result<Report, String> {
830 self.handle(self.incoming().await.ok_or("endpoint closed")?).await
831 }
832
833 pub async fn handle(&self, incoming: Incoming) -> Result<Report, String> {
835 let observed = match incoming.remote_addr() {
837 IncomingAddr::Ip(a) => Some(a),
838 _ => None,
839 };
840 let started = borg::learn::now_ms();
841 let conn = incoming.await.map_err(|e| format!("handshake: {e}"))?;
842 if !spoken(conn.alpn()) && conn.alpn() != JOIN_ALPN {
843 let (name, home) = self.gate()?;
844 let remote = authorise(&conn, &members(&home)?, &name)?;
845 return Err(self.mismatch(&conn, &remote));
846 }
847 let session = async {
848 let (name, home) = self.gate()?;
849 let (name, home) = (name.as_str(), home.as_path());
850 let membership = members(home)?;
851 let remote = authorise(&conn, &membership, name)?;
852 let (mut tx, mut rx) = conn.accept_bi().await.map_err(|e| e.to_string())?;
853 let Msg::Hello { collective, listen, relay, borg } = recv(&mut rx).await? else { return Err("protocol: expected hello".into()) };
854 if collective != name {
855 let reason = format!("this node serves collective {name:?}, not {collective:?}");
856 send(&mut tx, &Msg::Refused { reason: reason.clone() }).await?;
857 return Err(reason);
858 }
859 heard(home, &remote, listen, relay, observed);
860 send(&mut tx, &Msg::Hello { collective: name.into(), listen: self.listen(), relay: self.announced(), borg: self.borg() }).await?;
861 let report = self.exchange(Peer { key: &remote, alpn: conn.alpn() }, false, &mut tx, &mut rx, membership, &[]).await?;
862 let _ = tx.finish();
863 conn.closed().await; self.synced(&remote, conn.alpn(), started, borg);
865 Ok(report)
866 };
867 let result = if conn.alpn() == JOIN_ALPN {
868 self.admit(&conn, observed).await
869 } else {
870 session.await.map_err(|e| {
871 let member = self.gate().ok().and_then(|(name, home)| authorise(&conn, &members(&home).ok()?, &name).ok());
872 member.and_then(|remote| self.refused(&conn, &remote)).unwrap_or(e)
873 })
874 };
875 if let Err(e) = &result {
876 conn.close(1u32.into(), e.as_bytes());
877 }
878 vectors::reindexed(&self.home, result).await
879 }
880
881 pub async fn close(&self) {
882 self.endpoint.close().await;
883 }
884
885 async fn exchange(
887 &self,
888 peer: Peer<'_>,
889 dialler: bool,
890 tx: &mut SendStream,
891 rx: &mut RecvStream,
892 mine: Membership,
893 ask: &[String],
894 ) -> Result<Report, String> {
895 let Peer { key: remote, alpn } = peer;
896 let home = self.home.as_path();
897 let mut report = Report::default();
898 let Msg::Membership { docs } = swap(dialler, tx, rx, Msg::Membership { docs: Membership::history(home) }).await? else {
899 return Err("protocol: expected membership".into());
900 };
901 let mine = {
902 let _w = self.store.lock().unwrap_or_else(|e| e.into_inner());
903 let (held, _) = Membership::adopt(home, &docs).unwrap_or((mine, false));
904 report.quarantined += origin::apply_if_new(home, &held)?;
905 steer::enforce(home, &held, borg::learn::now_ms()); held
907 };
908 if mine.role_of(remote).is_none() {
909 return Err(format!("{remote} is no longer a member of {}", mine.collective));
910 }
911 self.stamp()?;
912 let origins = origin::load(home);
913
914 for file in synced(alpn) {
916 if file == PLAYBOOKS {
917 self.skip(dialler, tx, rx, (remote.to_string(), file.to_string(), 0)).await?;
918 } else {
919 self.reconcile(dialler, tx, rx, &home.join(file), &origins, &mine, alpn == V13_ALPN, &mut report).await?;
920 }
921 }
922
923 let Msg::Goals { ids } = swap(dialler, tx, rx, Msg::Goals { ids: Vec::new() }).await? else {
925 return Err("protocol: expected goals".into());
926 };
927 for goal in legacy_goals(ids) {
928 for log in 0..LEGACY_GOAL_LOGS {
929 self.skip(dialler, tx, rx, (remote.to_string(), goal.clone(), log)).await?;
930 }
931 }
932
933 let path = home.join(REASONING);
935 let held = lines::read(&path);
936 let set = origin::admitted(held.clone(), &origins, &mine);
937 self.gate()?;
938 let me = self.me.public();
939 let offer: BTreeSet<Hash> =
940 set.keys().filter(|h| origins.get(*h).is_some_and(|o| o.key == me) && replicas(h, &mine, &me).iter().any(|k| k == remote)).copied().collect();
941 let Msg::Offer { hashes } = swap(dialler, tx, rx, Msg::Offer { hashes: offer.iter().map(|h| to_hex(h)).collect() }).await? else {
942 return Err("protocol: expected offer".into());
943 };
944 let offered: BTreeSet<Hash> = hashes.iter().map(|h| from_hex::<32>(h)).collect::<Result<_, _>>()?;
945 let want: BTreeSet<Hash> = offered.into_iter().filter(|h| !held.contains_key(h)).collect();
946 let Msg::Want { hashes } = swap(dialler, tx, rx, Msg::Want { hashes: want.iter().map(|h| to_hex(h)).collect() }).await? else {
947 return Err("protocol: expected want".into());
948 };
949 let wanted: Vec<Hash> = hashes.iter().map(|h| from_hex::<32>(h)).collect::<Result<Vec<_>, _>>()?.into_iter().filter(|h| offer.contains(h)).collect();
950 let give = signed(&set, &origins, wanted.iter());
951 report.replicated += give.len();
952 self.gate()?; let Msg::Lines { lines: got } = swap(dialler, tx, rx, Msg::Lines { lines: give }).await? else {
954 return Err("protocol: expected lines".into());
955 };
956 self.store_lines(&path, got, &want, &mine, Some(remote), &mut report)?;
957
958 let ask: Vec<String> = ask.iter().take(MAX_ASK).cloned().collect();
960 let Msg::Ask { commits: theirs } = swap(dialler, tx, rx, Msg::Ask { commits: ask.clone() }).await? else {
961 return Err("protocol: expected ask".into());
962 };
963 let held = lines::read(&path);
964 let set = origin::admitted(held.clone(), &origin::load(home), &mine);
965 let answer: Vec<&Hash> = set.iter().filter(|(_, l)| about(&theirs, l)).map(|(h, _)| h).take(MAX_ASK).collect();
966 let give = signed(&set, &origin::load(home), answer.into_iter());
967 report.answered += give.len();
968 self.gate()?; let Msg::Lines { lines: got } = swap(dialler, tx, rx, Msg::Lines { lines: give }).await? else {
970 return Err("protocol: expected lines".into());
971 };
972 let want: BTreeSet<Hash> = got.iter().filter(|s| about(&ask, &s.line)).map(|s| lines::hash(&s.line)).filter(|h| !held.contains_key(h)).collect();
973 self.store_lines(&path, got, &want, &mine, None, &mut report)?;
974
975 let now = borg::learn::now_ms();
977 let live = Msg::Live { presence: live::outgoing(home, &self.me, now), directives: Directives };
978 self.gate()?; let Msg::Live { presence, .. } = swap(dialler, tx, rx, live).await? else {
980 return Err("protocol: expected live".into());
981 };
982 let _w = self.store.lock().unwrap_or_else(|e| e.into_inner());
983 report.presence += live::receive(home, &mine, &me, presence, now)?;
984 Ok(report)
985 }
986
987 #[allow(clippy::too_many_arguments)] async fn reconcile(
992 &self,
993 dialler: bool,
994 tx: &mut SendStream,
995 rx: &mut RecvStream,
996 path: &Path,
997 origins: &HashMap<Hash, Origin>,
998 mine: &Membership,
999 v1: bool,
1000 report: &mut Report,
1001 ) -> Result<(), String> {
1002 let set = offered(path, origins, mine, borg::learn::now_ms(), v1);
1003 self.gate()?; let mut r = Recon::new(&set);
1005 loop {
1006 let m = r.message();
1007 let Msg::Round { round } = swap(dialler, tx, rx, Msg::Round { round: m.clone() }).await? else {
1008 return Err("protocol: expected a reconciliation round".into());
1009 };
1010 if r.absorb(&m, round)? {
1011 break;
1012 }
1013 }
1014 (report.rounds, report.fps, report.hashes) = (report.rounds + r.cost.rounds, report.fps + r.cost.fps, report.hashes + r.cost.hashes);
1015 let give = frame(&set, origins, r.give.iter());
1016 report.sent += give.len();
1017 self.gate()?; let Msg::Lines { lines: got } = swap(dialler, tx, rx, Msg::Lines { lines: give }).await? else {
1019 return Err("protocol: expected lines".into());
1020 };
1021 self.store_lines(path, got, &r.want, mine, None, report)
1022 }
1023
1024 async fn skip(&self, dialler: bool, tx: &mut SendStream, rx: &mut RecvStream, log: (String, String, usize)) -> Result<(), String> {
1031 let mut want = BTreeSet::new();
1032 if dialler {
1033 let given = self.legacy.lock().unwrap_or_else(|e| e.into_inner()).get(&log).cloned().unwrap_or_default();
1034 let given: BTreeMap<Hash, ()> = given.into_iter().map(|h| (h, ())).collect();
1035 let mut r = Recon::new(&given);
1036 loop {
1037 let m = r.message();
1038 let Msg::Round { round } = swap(true, tx, rx, Msg::Round { round: m.clone() }).await? else {
1039 return Err("protocol: expected a reconciliation round".into());
1040 };
1041 if r.absorb(&m, round)? {
1042 break;
1043 }
1044 }
1045 want = r.want;
1046 } else {
1047 let Msg::Round { round } = recv(rx).await? else {
1048 return Err("protocol: expected a reconciliation round".into());
1049 };
1050 send(tx, &Msg::Round { round }).await?;
1051 }
1052 let Msg::Lines { lines: got } = swap(dialler, tx, rx, Msg::Lines { lines: Vec::new() }).await? else {
1053 return Err("protocol: expected lines".into());
1054 };
1055 let got: Vec<Hash> = got.iter().map(|s| lines::hash(&s.line)).filter(|h| want.contains(h)).collect();
1056 if !got.is_empty() {
1057 self.legacy.lock().unwrap_or_else(|e| e.into_inner()).entry(log).or_default().extend(got);
1058 }
1059 Ok(())
1060 }
1061
1062 fn store_lines(
1064 &self,
1065 path: &Path,
1066 got: Vec<Signed>,
1067 want: &BTreeSet<Hash>,
1068 m: &Membership,
1069 author: Option<&str>,
1070 report: &mut Report,
1071 ) -> Result<(), String> {
1072 self.local().store_lines(path, got, want, m, author, report)
1073 }
1074
1075 #[cfg(test)]
1077 fn store_messages(
1078 &self,
1079 path: &Path,
1080 keep: BTreeMap<Hash, Signed>,
1081 decided: (&str, Stored),
1082 m: &Membership,
1083 now: i64,
1084 report: &mut Report,
1085 ) -> Result<steer::Admitted, String> {
1086 self.local().store_messages(path, keep, decided, m, now, report)
1087 }
1088}
1089
1090pub struct Store<'a> {
1095 pub home: &'a Path,
1097 pub lock: &'a Mutex<()>,
1099 pub me: &'a Identity,
1101}
1102
1103impl Store<'_> {
1104 pub fn stamp(&self) -> Result<usize, String> {
1106 let _w = self.lock.lock().unwrap_or_else(|e| e.into_inner());
1107 borg::outcome::publish(self.home);
1109 origin::stamp(self.home, self.me, &logs(self.home), borg::learn::now_ms())
1110 }
1111
1112 pub fn store_lines(
1116 &self,
1117 path: &Path,
1118 got: Vec<Signed>,
1119 want: &BTreeSet<Hash>,
1120 m: &Membership,
1121 author: Option<&str>,
1122 report: &mut Report,
1123 ) -> Result<(), String> {
1124 let (file, now) = (log_name(path), borg::learn::now_ms());
1125 let mut keep: BTreeMap<Hash, Signed> = BTreeMap::new();
1126 for s in got {
1127 let h = lines::hash(&s.line);
1128 let ok = want.contains(&h)
1129 && s.origin.verify(&s.line).is_ok()
1130 && m.admits(&s.origin.key, s.origin.ms)
1131 && author.is_none_or(|a| a == s.origin.key)
1132 && lines::travels(file, &s.line, &s.origin.key, now);
1133 if ok {
1134 keep.insert(h, s);
1135 } else {
1136 report.rejected += 1;
1137 }
1138 }
1139 let _w = self.lock.lock().unwrap_or_else(|e| e.into_inner());
1140 if file != borg::msg::FILE {
1141 let known = origin::load(self.home);
1142 origin::append(self.home, &keep.iter().filter(|(h, _)| !known.contains_key(*h)).map(|(_, s)| s.origin.clone()).collect::<Vec<_>>())?;
1143 report.received += lines::merge(path, &keep.into_values().map(|s| s.line).collect::<Vec<_>>())?;
1144 return Ok(());
1145 }
1146 let text = std::fs::read_to_string(path).unwrap_or_default();
1149 let decided = stored(&text, &keep, now);
1150 let a = self.store_messages(path, keep, (&text, decided), m, now, report)?;
1151 (report.steered, report.dropped) = (report.steered + a.new, report.dropped + a.dropped);
1152 drop(_w);
1153 if a.new > 0 {
1155 borg::veto::apply_messages(self.home, now);
1156 }
1157 Ok(())
1158 }
1159
1160 fn store_messages(
1168 &self,
1169 path: &Path,
1170 mut keep: BTreeMap<Hash, Signed>,
1171 decided: (&str, Stored),
1172 m: &Membership,
1173 now: i64,
1174 report: &mut Report,
1175 ) -> Result<steer::Admitted, String> {
1176 let (read, first) = decided;
1177 if first.taken.is_empty() && first.body.is_none() {
1178 report.rejected += first.refused;
1179 return Ok(steer::Admitted::default());
1180 }
1181 let known = origin::load(self.home);
1182 borg::live::locked(path, || {
1183 let text = std::fs::read_to_string(path).unwrap_or_default();
1184 let d = if text == read { first } else { stored(&text, &keep, now) };
1185 keep.retain(|h, _| d.taken.contains(h));
1186 report.rejected += d.refused;
1187 let new: Vec<borg::msg::Message> = keep.values().filter_map(|s| serde_json::from_str(&s.line).ok()).collect();
1188 origin::append(self.home, &keep.iter().filter(|(h, _)| !known.contains_key(*h)).map(|(_, s)| s.origin.clone()).collect::<Vec<_>>())?;
1189 let io = |e: std::io::Error| format!("{}: {e}", path.display());
1190 let before = borg::msg::mark_of(path);
1191 let talk = keep.values().any(|s| !matches!(Line::parse(&s.line), Some(Line::Seen(_))));
1192 match &d.body {
1193 Some(body) => {
1194 borg::steer::rewrite(path, body).map_err(io)?;
1195 borg::msg::wrote(path, before.as_deref(), borg::msg::Change::rewrite(&text, body, now), talk);
1196 }
1197 None if keep.is_empty() => {}
1198 None => {
1199 let open = !text.is_empty() && !text.ends_with('\n');
1201 let out: String = std::iter::once(if open { "\n" } else { "" }).chain(keep.values().flat_map(|s| [s.line.as_str(), "\n"])).collect();
1202 std::fs::OpenOptions::new()
1203 .create(true)
1204 .append(true)
1205 .open(path)
1206 .and_then(|mut f| std::io::Write::write_all(&mut f, out.as_bytes()))
1207 .map_err(io)?;
1208 borg::msg::wrote(path, before.as_deref(), borg::msg::Change::Appended(out.len()), talk);
1209 }
1210 }
1211 report.received += keep.len();
1212 steer::admit(self.home, m, &self.me.public(), &new, now).map_err(|e| format!("admissions: {e}"))
1214 })
1215 }
1216}
1217
1218struct Stored {
1220 taken: BTreeSet<Hash>,
1222 refused: usize,
1224 body: Option<String>,
1226}
1227
1228fn stored(text: &str, keep: &BTreeMap<Hash, Signed>, now: i64) -> Stored {
1234 let file: Vec<(Hash, &str)> = text.lines().map(|l| (lines::hash(l), l)).collect();
1235 let held: HashSet<&Hash> = file.iter().map(|(h, _)| h).collect();
1236 let new: Vec<(&Hash, &Signed)> = keep.iter().filter(|(h, _)| !held.contains(h)).collect();
1237 let lacked = new.len();
1238 let paced: Vec<(&Hash, &Signed)> = if new.is_empty() {
1239 new
1240 } else {
1241 let all: BTreeMap<Hash, String> = file.iter().map(|(h, l)| (*h, l.to_string())).chain(new.iter().map(|(h, s)| (**h, s.line.clone()))).collect();
1242 let w = window(&all);
1243 new.into_iter().filter(|(h, _)| w.contains(*h)).collect()
1244 };
1245 let (kept, _) = borg::msg::bound(file.iter().map(|(_, l)| *l).chain(paced.iter().map(|(_, s)| s.line.as_str())), now);
1246 let set: HashSet<&str> = kept.iter().copied().collect();
1247 let taken: BTreeSet<Hash> = paced.iter().filter(|(_, s)| set.contains(s.line.as_str())).map(|(h, _)| **h).collect();
1248 let body = file.iter().any(|(_, l)| !set.contains(l)).then(|| kept.iter().map(|l| format!("{l}\n")).collect());
1249 Stored { refused: lacked - taken.len(), taken, body }
1250}
1251
1252pub fn offered(path: &Path, origins: &HashMap<Hash, Origin>, m: &Membership, now: i64, v1: bool) -> BTreeMap<Hash, String> {
1256 let mut set = travelling(path, origins, m, now);
1257 if log_name(path) == borg::msg::FILE {
1258 let w = window(&lines::read(path));
1259 set.retain(|h, _| w.contains(h));
1260 if v1 {
1261 let w = v1_window(&set, v1_share(m));
1262 set.retain(|h, _| w.contains(h));
1263 }
1264 }
1265 set
1266}
1267
1268fn travelling(path: &Path, origins: &HashMap<Hash, Origin>, m: &Membership, now: i64) -> BTreeMap<Hash, String> {
1271 let file = log_name(path);
1272 let mut set = origin::admitted(lines::read(path), origins, m);
1273 set.retain(|h, l| origins.get(h).is_some_and(|o| lines::travels(file, l, &o.key, now)));
1274 set
1275}
1276
1277fn window(set: &BTreeMap<Hash, String>) -> BTreeSet<Hash> {
1286 let (hashes, all): (Vec<Hash>, Vec<Line>) = set.iter().filter_map(|(h, l)| Line::parse(l).map(|x| (*h, x))).unzip();
1287 let admitted = borg::msg::within_windows(&all);
1288 hashes
1289 .into_iter()
1290 .zip(&all)
1291 .filter(|(_, l)| match l {
1292 Line::Seen(_) => true,
1293 Line::Message(m) => m.kind.privileged() || admitted.contains(m.id.as_str()),
1294 })
1295 .map(|(h, _)| h)
1296 .collect()
1297}
1298
1299fn v1_share(m: &Membership) -> usize {
1302 (V1_KEEP / m.members.len().max(1)).max(1)
1303}
1304
1305fn v1_slot_held(messages: &[borg::msg::Message], from: &borg::msg::Sender, now: i64) -> bool {
1309 messages
1310 .iter()
1311 .any(|m| m.v == 1 && !m.kind.privileged() && m.node == from.node && m.by == from.by && m.to == borg::msg::ALL && now.saturating_sub(m.ms) < V1_RATE_MS)
1312}
1313
1314fn v1_window(set: &BTreeMap<Hash, String>, per_node: usize) -> BTreeSet<Hash> {
1319 let mut all: Vec<(i64, Hash, borg::msg::Message)> = set
1320 .iter()
1321 .filter_map(|(h, l)| serde_json::from_str::<borg::msg::Message>(l).ok().filter(|m| m.v == 1 && m.well_formed()).map(|m| (m.ms, *h, m)))
1322 .collect();
1323 all.sort_by_key(|x| (x.0, x.1));
1324 let mut last: HashMap<(String, String, String), i64> = HashMap::new();
1325 let mut paced: Vec<(Hash, String, bool)> = Vec::new();
1326 for (ms, h, m) in all {
1327 if m.kind.privileged() {
1328 paced.push((h, m.node, true));
1329 continue;
1330 }
1331 let slot = last.entry((m.node.clone(), m.by, m.to)).or_insert(i64::MIN);
1332 if ms.saturating_sub(*slot) >= V1_RATE_MS {
1333 *slot = ms;
1334 paced.push((h, m.node, false));
1335 }
1336 }
1337 let mut per: HashMap<(String, bool), usize> = HashMap::new();
1338 let mut total = [0usize; 2];
1339 paced
1340 .into_iter()
1341 .rev()
1342 .filter(|(_, node, steers)| {
1343 let n = per.entry((node.clone(), *steers)).or_default();
1344 *n += 1;
1345 *n <= per_node
1346 })
1347 .filter(|(_, _, steers)| {
1348 let t = &mut total[usize::from(*steers)];
1349 *t += 1;
1350 *t <= V1_KEEP
1351 })
1352 .map(|(h, _, _)| h)
1353 .collect()
1354}
1355
1356fn log_name(path: &Path) -> &str {
1358 path.file_name().and_then(|f| f.to_str()).unwrap_or_default()
1359}
1360
1361fn legacy_goals(ids: Vec<String>) -> BTreeSet<String> {
1364 let valid = |id: &String| (1..=40).contains(&id.len()) && id.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-');
1365 ids.into_iter().filter(valid).collect()
1366}
1367
1368fn reachable(listen: &str, observed: Option<SocketAddr>) -> Option<SocketAddr> {
1371 let listen: SocketAddr = listen.parse().ok()?;
1372 let ip = match observed {
1373 Some(o) => o.ip().to_canonical(),
1374 None => listen.ip(),
1375 };
1376 let at = SocketAddr::new(ip, listen.port());
1377 (!at.ip().is_unspecified() && at.port() != 0).then_some(at)
1378}
1379
1380fn engineer_of(home: &Path, key: &str, now: i64) -> Option<String> {
1383 let seen = borg::presence::remote(home, now).into_iter().filter(|r| r.key == key && !r.p.engineer.is_empty()).max_by_key(|r| r.p.ms);
1384 seen.map(|r| r.p.engineer).or_else(|| borg::protocol::versions(home).remove(key).and_then(|v| v.engineer))
1385}
1386
1387fn heard(home: &Path, key: &str, listen: Option<String>, relay: Option<String>, observed: Option<SocketAddr>) {
1390 let direct = listen.and_then(|l| reachable(&l, observed)).map(|a| a.to_string());
1391 let hints: Vec<String> = direct.into_iter().chain(relay.filter(|r| addr::hints(r).is_ok())).collect();
1392 let _ = peers::remember(home, key, &hints.join(","));
1393}
1394
1395fn about(commits: &[String], line: &str) -> bool {
1398 let Ok(v) = serde_json::from_str::<serde_json::Value>(line) else { return false };
1399 ["commit", "fix"].iter().filter_map(|k| v.get(k)?.as_str()).any(|c| asked(commits, c))
1400}
1401
1402fn asked(asked: &[String], commit: &str) -> bool {
1404 asked.iter().any(|a| a == commit || (a.len() >= 7 && commit.starts_with(a.as_str())))
1405}
1406
1407pub fn frame<'a>(set: &BTreeMap<Hash, String>, origins: &HashMap<Hash, Origin>, hashes: impl Iterator<Item = &'a Hash>) -> Vec<Signed> {
1412 within_budget(signed(set, origins, hashes))
1413}
1414
1415fn within_budget(lines: Vec<Signed>) -> Vec<Signed> {
1417 let mut used = 0;
1418 lines
1419 .into_iter()
1420 .enumerate()
1421 .take_while(|(i, s)| {
1422 used += s.line.len();
1423 *i == 0 || used <= MAX_GIVE
1424 })
1425 .map(|(_, s)| s)
1426 .collect()
1427}
1428
1429fn signed<'a>(set: &BTreeMap<Hash, String>, origins: &HashMap<Hash, Origin>, hashes: impl Iterator<Item = &'a Hash>) -> Vec<Signed> {
1430 hashes.filter_map(|h| Some(Signed { line: set.get(h)?.clone(), origin: origins.get(h)?.clone() })).collect()
1431}
1432
1433pub fn replicas(h: &Hash, m: &Membership, author: &str) -> Vec<String> {
1437 let mut keys: Vec<(Hash, &str)> = m
1438 .members
1439 .iter()
1440 .filter(|x| x.key != author)
1441 .map(|x| {
1442 let mut s = blake3::Hasher::new();
1443 s.update(h);
1444 s.update(x.key.as_bytes());
1445 (*s.finalize().as_bytes(), x.key.as_str())
1446 })
1447 .collect();
1448 keys.sort_by(|a, b| b.cmp(a));
1449 keys.into_iter().take(REPLICAS).map(|(_, k)| k.to_string()).collect()
1450}
1451
1452fn members(home: &Path) -> Result<Membership, String> {
1453 Membership::load(home)?
1454 .ok_or_else(|| format!("no membership in {}: `borg mesh init` (admin) or `borg mesh join <file>`", crate::membership::path(home).display()))
1455}
1456
1457fn authorise(conn: &Connection, membership: &Membership, name: &str) -> Result<String, String> {
1459 let remote = to_hex(conn.remote_id().as_bytes());
1460 match membership.role_of(&remote) {
1461 Some(_) => Ok(remote),
1462 None => Err(format!("rejected {remote}: not a member of {name}")),
1463 }
1464}
1465
1466async fn swap(dialler: bool, tx: &mut SendStream, rx: &mut RecvStream, m: Msg) -> Result<Msg, String> {
1468 if dialler {
1469 send(tx, &m).await?;
1470 recv(rx).await
1471 } else {
1472 let theirs = recv(rx).await?;
1473 send(tx, &m).await?;
1474 Ok(theirs)
1475 }
1476}
1477
1478async fn send(tx: &mut SendStream, m: &Msg) -> Result<(), String> {
1479 let body = serde_json::to_vec(m).map_err(|e| e.to_string())?;
1480 tx.write_all(&(body.len() as u32).to_be_bytes()).await.map_err(|e| format!("send: {e}"))?;
1481 tx.write_all(&body).await.map_err(|e| format!("send: {e}"))
1482}
1483
1484async fn recv(rx: &mut RecvStream) -> Result<Msg, String> {
1485 let mut len = [0u8; 4];
1486 rx.read_exact(&mut len).await.map_err(|e| format!("receive: {e}"))?;
1487 let len = u32::from_be_bytes(len) as usize;
1488 if len > MAX_FRAME {
1489 return Err(format!("frame of {len} bytes exceeds {MAX_FRAME}"));
1490 }
1491 let mut body = vec![0u8; len];
1492 rx.read_exact(&mut body).await.map_err(|e| format!("receive: {e}"))?;
1493 match serde_json::from_slice(&body).map_err(|e| format!("protocol: {e}"))? {
1494 Msg::Refused { reason } => Err(format!("refused by peer: {reason}")),
1495 m => Ok(m),
1496 }
1497}
1498
1499#[cfg(test)]
1500mod tests {
1501 use super::*;
1502 use serde_json::json;
1503
1504 #[test]
1506 fn lines_about_the_asked_commits() {
1507 let ask = vec!["d0d0d0d".to_string(), "f1f1f1f1f1".to_string()];
1508 let excerpt = json!({"kind": "reasoning", "commit": "d0d0d0d0d0"}).to_string();
1509 let link = |c: &str, f: &str| json!({"kind": "regress", "commit": c, "fix": f}).to_string();
1510 assert!(about(&ask, &excerpt) && about(&ask, &link("aaaaaaaaaa", "f1f1f1f1f1")) && about(&ask, &link("d0d0d0d0d0", "bbbbbbbbbb")));
1511 assert!(!about(&ask, &link("aaaaaaaaaa", "bbbbbbbbbb")) && !about(&ask, "not json"));
1512 }
1513
1514 #[test]
1516 fn a_frame_gives_lines_up_to_its_budget() {
1517 let me = Identity::load_or_create(&std::env::temp_dir().join(format!("borg-mesh-budget-{}", std::process::id()))).unwrap();
1518 let line = |n: usize| {
1519 let l = "x".repeat(n);
1520 Signed { origin: Origin::new(&me, &lines::hash(&l), 1), line: l }
1521 };
1522 let give = within_budget(vec![line(MAX_GIVE / 2), line(MAX_GIVE / 2), line(1)]);
1523 assert_eq!(give.len(), 2);
1524 assert_eq!(within_budget(vec![line(MAX_GIVE + 1), line(1)]).len(), 1);
1525 assert!(within_budget(Vec::new()).is_empty());
1526 }
1527
1528 fn node(tag: &str, share: &str) -> Place {
1531 let base = std::env::temp_dir().join(format!("borg-mesh-sync-{tag}-{}", std::process::id()));
1532 let _ = std::fs::remove_dir_all(&base);
1533 std::fs::create_dir_all(home_of(&base)).unwrap();
1534 let p = Place { config: base.join("collectives.json"), base: base.clone(), dir: base.join("repo").display().to_string(), collective: None };
1535 configure(&p, share);
1536 p
1537 }
1538
1539 fn configure(p: &Place, share: &str) {
1540 let cfg = json!({"default": "off", "collectives": [{"name": "work", "paths": [p.dir], "share": share, "relay": "disabled"}]});
1541 std::fs::write(&p.config, cfg.to_string()).unwrap();
1542 }
1543
1544 fn home_of(base: &Path) -> PathBuf {
1545 base.join("collectives").join("work")
1546 }
1547
1548 fn home(p: &Place) -> PathBuf {
1549 home_of(&p.base)
1550 }
1551
1552 fn lesson(id: &str, version: u32, q: &str) -> String {
1553 json!({"id": id, "origin_ms": 1, "question": q, "version": version}).to_string()
1554 }
1555
1556 fn write(home: &Path, file: &str, lines: &[String]) {
1557 std::fs::write(home.join(file), lines.iter().map(|l| format!("{l}\n")).collect::<String>()).unwrap();
1558 }
1559
1560 const LOCAL: &str = "127.0.0.1:0";
1561
1562 fn team(tag: &str) -> (Place, Place) {
1564 let (a, b) = (node(&format!("{tag}-alice"), "team"), node(&format!("{tag}-bob"), "team"));
1565 let alice = Identity::load_or_create(&home(&a)).unwrap();
1566 let bob = Identity::load_or_create(&home(&b)).unwrap();
1567 let mut m = Membership::found("work", &alice);
1568 m.invite(&alice, &bob.public(), "member").unwrap();
1569 m.save(&home(&a)).unwrap();
1570 m.save(&home(&b)).unwrap();
1571 (a, b)
1572 }
1573
1574 fn rollup(node: &str, version: u64, turns: u64) -> String {
1575 json!({"id": format!("stat:{node}:dev@x:2026-10-12"), "version": version, "day": "2026-10-12", "node": node,
1576 "engineer": "dev@x", "counts": {"turn_end": turns}, "usd": 0.0, "dur_ms": 0})
1577 .to_string()
1578 }
1579
1580 #[tokio::test(flavor = "multi_thread")]
1581 async fn usage_rollups_sync_and_the_latest_version_wins() {
1582 let (pa, pb) = team("stats");
1583 let (a, b) = (home(&pa), home(&pb));
1584 write(&a, "stats.jsonl", &[rollup("na", 1, 3), rollup("na", 2, 5)]);
1585 write(&b, "stats.jsonl", &[rollup("nb", 1, 7)]);
1586 let alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
1587 let bob = Node::bind(&pb, LOCAL.parse().unwrap()).await.unwrap();
1588 let ticket = alice.ticket();
1589 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
1590 let (served, synced) = (served.unwrap(), synced.unwrap());
1591 assert_eq!((synced.received, served.received), (2, 1), "both of alice's lines reach bob, bob's one reaches alice");
1592 for h in [&a, &b] {
1593 let r = borg::stats::rollups(h);
1594 assert_eq!(r.len(), 2, "one bucket per node");
1595 assert_eq!(r["stat:na:dev@x:2026-10-12"].counts["turn_end"], 5, "the newest version of alice's bucket");
1596 assert_eq!(r["stat:nb:dev@x:2026-10-12"].counts["turn_end"], 7);
1597 }
1598 }
1599
1600 #[tokio::test(flavor = "multi_thread")]
1603 async fn a_leftover_playbooks_log_is_not_synced() {
1604 let (pa, pb) = team("playbooks");
1605 let (a, b) = (home(&pa), home(&pb));
1606 let edit = json!({"pb": "p1", "edit": "e1", "step": "s1", "text": "run the parser tests", "ms": 1}).to_string();
1607 write(&a, PLAYBOOKS, std::slice::from_ref(&edit));
1608 let h = lines::hash(&edit);
1609 origin::append(&a, &[Origin::new(&Identity::load_or_create(&a).unwrap(), &h, 1)]).unwrap();
1610 let c = json!({"id": "cabc", "text": "locks are taken in id order", "lesson": "L1", "by": "lead@x", "ms": 5, "source": "veto"}).to_string();
1611 write(&a, borg::veto::FILE, std::slice::from_ref(&c));
1612 let alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
1613 let bob = Node::bind(&pb, LOCAL.parse().unwrap()).await.unwrap();
1614 let ticket = alice.ticket();
1615 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
1616 let (served, synced) = (served.unwrap(), synced.unwrap());
1617 assert_eq!((synced.received, served.sent), (1, 1), "only the constraint, the step after the playbooks', moves");
1618 assert!(holds(&b, borg::veto::FILE, &c));
1619 assert!(!b.join(PLAYBOOKS).exists(), "no playbook reaches bob");
1620 assert!(holds(&a, PLAYBOOKS, &edit), "alice's leftover stays untouched");
1621 alice.close().await;
1622 bob.close().await;
1623 }
1624
1625 #[tokio::test(flavor = "multi_thread")]
1626 async fn a_ratified_constraint_reaches_the_team_signed() {
1627 let (pa, pb) = team("constraints");
1628 let (a, b) = (home(&pa), home(&pb));
1629 write(&a, "lessons.jsonl", &[lesson("L1", 1, "lock order?")]);
1630 let c = json!({"id": "cabc", "text": "locks are taken in id order", "lesson": "L1", "by": "lead@x", "ms": 5, "source": "veto"}).to_string();
1631 write(&a, borg::veto::FILE, std::slice::from_ref(&c));
1632 let alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
1633 let bob = Node::bind(&pb, LOCAL.parse().unwrap()).await.unwrap();
1634 let ticket = alice.ticket();
1635 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
1636 served.unwrap();
1637 assert!(synced.unwrap().received >= 2, "the lesson and the constraint");
1638 assert_eq!(lines::read(&b.join(borg::veto::FILE)).into_values().collect::<Vec<_>>(), std::slice::from_ref(&c));
1639 assert!(origin::load(&b).contains_key(&lines::hash(&c)), "it arrived with its signed origin");
1640 let standing: Vec<String> = borg::conscience::load(&b.join("lessons.jsonl"), None).into_iter().filter(|c| c.standing()).map(|c| c.question).collect();
1641 assert_eq!(standing, ["locks are taken in id order"], "bob's sessions get it as a standing lesson");
1642 alice.close().await;
1643 bob.close().await;
1644 }
1645
1646 fn absorb(id: &str, kind: &str, ms: i64, note: Option<&str>) -> String {
1648 let mut v = json!({"id": id, "kind": kind, "ms": ms, "by": "lead@x"});
1649 if let Some(n) = note {
1650 (v["note"], v["version"]) = (json!(n), json!(1));
1651 }
1652 v.to_string()
1653 }
1654
1655 #[tokio::test(flavor = "multi_thread")]
1658 async fn absorbs_reach_the_team_signed_and_a_malformed_one_stays_home() {
1659 let (pa, pb) = team("absorbs");
1660 let (a, b) = (home(&pa), home(&pb));
1661 let (held, released) = (absorb("L1", "absorb", 5, Some("skills/review@2026-10-30")), absorb("L2", "release", 6, None));
1662 let oversize = absorb("L3", "absorb", 7, Some(&"n".repeat(borg::absorb::NOTE_CHARS + 1)));
1663 write(&a, lines::ABSORBED, &[held.clone(), released.clone(), oversize.clone()]);
1664 let bobs = absorb("L4", "absorb", 8, Some("a team's skills repo"));
1665 write(&b, lines::ABSORBED, std::slice::from_ref(&bobs));
1666 let alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
1667 let bob = Node::bind(&pb, LOCAL.parse().unwrap()).await.unwrap();
1668 let ticket = alice.ticket();
1669 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
1670 let (served, synced) = (served.unwrap(), synced.unwrap());
1671 assert_eq!((synced.received, synced.rejected, served.received), (2, 0, 1), "the two well-formed lines, the oversize one never offered");
1672 assert!(holds(&b, lines::ABSORBED, &held) && holds(&b, lines::ABSORBED, &released) && !holds(&b, lines::ABSORBED, &oversize));
1673 assert!(holds(&a, lines::ABSORBED, &bobs), "bob's absorb reaches alice");
1674 assert!(origin::load(&b).contains_key(&lines::hash(&held)), "it arrived with its signed origin");
1675 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
1676 let quiet = Report { rounds: SYNCED.len(), fps: SYNCED.len(), ..Report::default() };
1677 assert_eq!((synced.unwrap(), served.unwrap()), (quiet, quiet), "resync is quiet");
1678 alice.close().await;
1679 bob.close().await;
1680 }
1681
1682 #[test]
1685 fn a_malformed_or_oversize_absorb_from_a_peer_is_refused() {
1686 let (pa, pb) = team("absorb-refused");
1687 let (a, b) = (home(&pa), home(&pb));
1688 let (alice, bob) = (Identity::load_or_create(&a).unwrap(), Identity::load_or_create(&b).unwrap());
1689 let ok = absorb("L1", "absorb", 5, Some("skills/review@2026-10-30"));
1690 let bad = [
1691 absorb("L1", "absorb", 5, Some(&"n".repeat(borg::absorb::NOTE_CHARS + 1))),
1692 absorb("L1", "forget", 5, None),
1693 json!({"id": "L1", "kind": "release", "ms": 5}).to_string(),
1694 ];
1695 let now = borg::learn::now_ms();
1696 let got: Vec<Signed> =
1697 std::iter::once(&ok).chain(&bad).map(|l| Signed { origin: Origin::new(&alice, &lines::hash(l), now), line: l.clone() }).collect();
1698 let want: BTreeSet<Hash> = got.iter().map(|s| lines::hash(&s.line)).collect();
1699 let lock = Mutex::new(());
1700 let store = Store { home: &b, lock: &lock, me: &bob };
1701 let mut report = Report::default();
1702 store.store_lines(&b.join(lines::ABSORBED), got, &want, &members(&b).unwrap(), None, &mut report).unwrap();
1703 assert_eq!((report.received, report.rejected), (1, bad.len()));
1704 assert_eq!(lines::read(&b.join(lines::ABSORBED)).into_values().collect::<Vec<_>>(), [ok]);
1705 }
1706
1707 #[tokio::test(flavor = "multi_thread")]
1710 async fn a_mesh_14_peer_syncs_everything_but_the_absorbs_both_ways() {
1711 let p = crew("v14", &["alice", "old1", "old2"]);
1712 let (a, o1, o2) = (home(&p[0]), home(&p[1]), home(&p[2]));
1713 let keys: Vec<String> = [&a, &o1, &o2].iter().map(|h| Identity::load_or_create(h).unwrap().public()).collect();
1714 let now = borg::learn::now_ms();
1715 write(&a, "lessons.jsonl", &[lesson("L1", 1, "lock order?")]);
1716 let o = json!({"kind": "asked_then_changed", "session": "s1", "lessons": ["L1"], "ms": 1, "repo": "app"}).to_string();
1717 write(&a, "outcomes.jsonl", &[o]);
1718 assert_eq!(borg::outcome::publish(&a), 1);
1719 let tell = message(&a, &keys[0], ALL_TO, now - 100, "migrations are mine today");
1720 let ours = absorb("L1", "absorb", 5, Some("skills/review@2026-10-30"));
1721 write(&a, lines::ABSORBED, std::slice::from_ref(&ours));
1722 let mut theirs = Vec::new();
1723 for (i, h) in [(1, &o1), (2, &o2)] {
1724 write(h, "lessons.jsonl", &[lesson(&format!("L{}", i + 1), 1, "retry budget?")]);
1725 write(h, lines::ABSORBED, &[absorb(&format!("L{}", i + 1), "absorb", 6, Some("a team's skills repo"))]);
1726 theirs.push(message(h, &keys[i], ALL_TO, now - 50, "from a 0.6 node"));
1727 }
1728 let mut alice = Node::bind(&p[0], LOCAL.parse().unwrap()).await.unwrap();
1729 alice.speaks.tell = false; let old = |i: usize| Node::open(&p[i], Some(LOCAL.parse().unwrap()), true, |x| x.alpns(vec![V14_ALPN.to_vec()]));
1731 let (old1, old2) = (old(1).await.unwrap(), old(2).await.unwrap());
1732 let ticket = alice.ticket();
1734 let (served, dialled) = tokio::join!(alice.accept(), old1.dial(&ticket, &[], V14_ALPN, Vec::new()));
1735 served.unwrap();
1736 dialled.unwrap();
1737 let old_ticket = old2.ticket();
1739 let (served, dialled) = tokio::join!(old2.accept(), alice.sync(&old_ticket));
1740 served.unwrap();
1741 dialled.unwrap();
1742 let team = borg::outcome::TEAM_FILE;
1743 for o in [&o1, &o2] {
1744 assert!(holds(o, "lessons.jsonl", &lesson("L1", 1, "lock order?")), "lessons travel");
1745 assert_eq!(lines::read(&o.join(team)), lines::read(&a.join(team)), "outcomes travel");
1746 assert!(holds(o, borg::msg::FILE, &tell), "messages travel");
1747 assert!(!holds(o, lines::ABSORBED, &ours), "no absorb reaches a 0.6 node");
1748 }
1749 assert!(theirs.iter().all(|l| holds(&a, borg::msg::FILE, l)), "their messages reach this node");
1750 assert!(holds(&a, "lessons.jsonl", &lesson("L2", 1, "retry budget?")) && holds(&a, "lessons.jsonl", &lesson("L3", 1, "retry budget?")));
1751 assert_eq!(lines::read(&a.join(lines::ABSORBED)).into_values().collect::<Vec<_>>(), [ours], "and none of theirs reaches it");
1752 assert_eq!(borg::presence::reads(&a).get(&keys[1]).copied(), Some(2), "a 0.6 node reads every message line");
1753 let (served, dialled) = tokio::join!(alice.accept(), old1.dial(&ticket, &[], V14_ALPN, Vec::new()));
1756 assert_eq!((dialled.unwrap().received, served.unwrap().sent), (2, 2));
1757 assert!(!holds(&o1, lines::ABSORBED, &absorb("L3", "absorb", 6, Some("a team's skills repo"))));
1758 let (served, dialled) = tokio::join!(alice.accept(), old1.dial(&ticket, &[], V14_ALPN, Vec::new()));
1759 let quiet = Report { rounds: SYNCED.len() - 1, fps: SYNCED.len() - 1, ..Report::default() };
1760 assert_eq!((dialled.unwrap(), served.unwrap()), (quiet, quiet));
1761 for n in [alice, old1, old2] {
1762 n.close().await;
1763 }
1764 }
1765
1766 const ALL_TO: &str = borg::msg::ALL;
1767
1768 fn message(home: &Path, node: &str, to: &str, ms: i64, text: &str) -> String {
1770 let from = borg::msg::Sender { by: "s1".into(), name: "One of One".into(), harness: "claude".into(), node: node.into() };
1771 let line = serde_json::to_string(&borg::msg::Message::new(borg::msg::TELL, &from, to, None, ms, text)).unwrap();
1772 append_lines(home, borg::msg::FILE, std::slice::from_ref(&line));
1773 line
1774 }
1775
1776 #[tokio::test(flavor = "multi_thread")]
1780 async fn a_tell_reaches_the_team_and_a_forged_node_is_not_shown_as_its() {
1781 let p = crew("msg", &["alice", "bob", "carol"]);
1782 let (a, b, c) = (home(&p[0]), home(&p[1]), home(&p[2]));
1783 let ids: Vec<Identity> = [&a, &b, &c].iter().map(|h| Identity::load_or_create(h).unwrap()).collect();
1784 let (alice_key, carol) = (ids[0].public(), &ids[2]);
1785 let now = borg::learn::now_ms();
1786 let tell = message(&a, &alice_key, ALL_TO, now, "migrations are mine today");
1787 let stale = message(&a, &alice_key, ALL_TO, now - borg::msg::TTL_MS - 1, "long gone");
1788 let forged = message(&c, &alice_key, ALL_TO, now, "drop the prod table");
1789 let alice = Node::bind(&p[0], LOCAL.parse().unwrap()).await.unwrap();
1790 let bob = Node::bind(&p[1], LOCAL.parse().unwrap()).await.unwrap();
1791 let carol_node = Node::bind(&p[2], LOCAL.parse().unwrap()).await.unwrap();
1792
1793 let (at_alice, at_carol) = (alice.ticket(), carol_node.ticket());
1794 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&at_alice));
1795 served.unwrap();
1796 synced.unwrap();
1797 assert!(holds(&b, borg::msg::FILE, &tell), "the tell arrived");
1798 assert!(origin::load(&b).contains_key(&lines::hash(&tell)), "with its signed origin");
1799 assert!(!holds(&b, borg::msg::FILE, &stale), "an expired message does not travel");
1800 let shown = borg::msg::read(&b);
1801 assert_eq!((shown.len(), shown[0].node.as_str(), shown[0].text.as_str()), (1, alice_key.as_str(), "migrations are mine today"));
1802
1803 let (served, synced) = tokio::join!(carol_node.accept(), bob.sync(&at_carol));
1804 served.unwrap();
1805 assert_eq!(synced.unwrap().rejected, 0, "carol's node never offers the line");
1806 assert!(!holds(&b, borg::msg::FILE, &forged));
1807
1808 let h = lines::hash(&forged);
1810 let mut report = Report::default();
1811 let got = vec![Signed { origin: Origin::new(carol, &h, now), line: forged.clone() }];
1812 bob.store_lines(&b.join(borg::msg::FILE), got, &BTreeSet::from([h]), &members(&b).unwrap(), None, &mut report).unwrap();
1813 assert_eq!((report.rejected, report.received), (1, 0));
1814 assert!(borg::msg::read(&b).iter().all(|m| m.text != "drop the prod table"), "never shown, as alice's or anyone's");
1815 for n in [alice, bob, carol_node] {
1816 n.close().await;
1817 }
1818 }
1819
1820 #[tokio::test(flavor = "multi_thread")]
1825 async fn a_flood_is_bounded_on_receipt_and_a_cut_line_does_not_come_back() {
1826 let p = crew("flood", &["alice", "bob"]);
1827 let (a, b) = (home(&p[0]), home(&p[1]));
1828 let alice = Identity::load_or_create(&a).unwrap();
1829 let now = borg::learn::now_ms();
1830 let (long, n) = ("w".repeat(borg::msg::WIRE_TEXT), borg::msg::WINDOW_CAP + 6);
1831 let lines: Vec<String> = (0..n).map(|i| message(&a, &alice.public(), &format!("s-{i:03}"), now - (n - i) as i64 * 1000, &long)).collect();
1832 let fits = borg::msg::MEMBER_BYTES / (lines[0].len() + 1);
1833 let got: Vec<Signed> = lines.iter().map(|l| Signed { origin: Origin::new(&alice, &lines::hash(l), now), line: l.clone() }).collect();
1834 let want: BTreeSet<Hash> = got.iter().map(|s| lines::hash(&s.line)).collect();
1835 let bob = Node::bind(&p[1], LOCAL.parse().unwrap()).await.unwrap();
1836 let path = b.join(borg::msg::FILE);
1837 let mut report = Report::default();
1838 bob.store_lines(&path, got.clone(), &want, &members(&b).unwrap(), None, &mut report).unwrap();
1839 assert_eq!((report.received, report.rejected), (fits, n - fits), "the share's newest");
1840 assert!(!holds(&b, borg::msg::FILE, &lines[n - fits - 1]) && holds(&b, borg::msg::FILE, &lines[n - fits]), "the oldest past the share");
1841
1842 let mut twice = Report::default();
1843 bob.store_lines(&path, got, &want, &members(&b).unwrap(), None, &mut twice).unwrap();
1844 assert_eq!((twice.received, twice.rejected), (0, n - fits), "nothing cut comes back");
1845 assert_eq!(lines::read(&path).len(), fits);
1846 assert_eq!(offered(&path, &origin::load(&b), &members(&b).unwrap(), now, false).len(), fits);
1847 bob.close().await;
1848 }
1849
1850 #[tokio::test(flavor = "multi_thread")]
1854 async fn a_nodes_own_lines_never_cut_what_its_peers_keep() {
1855 let (pa, pb) = team("local-share");
1856 let (a, b) = (home(&pa), home(&pb));
1857 let alice = Identity::load_or_create(&a).unwrap().public();
1858 let bob = Identity::load_or_create(&b).unwrap().public();
1859 let now = borg::learn::now_ms();
1860 let long = "w".repeat(borg::msg::WIRE_TEXT);
1861 let n = borg::msg::WINDOW_CAP / 2 + borg::msg::WINDOW_CAP / 8;
1862 let lines_to = |to: &str, at: i64| -> Vec<String> { (0..n).map(|i| line_of(&alice, &format!("s-{i:03}"), to, at + i as i64, &long).1).collect() };
1863 let out = lines_to(&format!("r@{bob}"), now - 20_000);
1864 let here = lines_to(&format!("local@{alice}"), now - 10_000);
1865 let size = out[0].len().max(here[0].len()) + 1;
1866 assert!(n * size <= borg::msg::MEMBER_BYTES && 2 * n * size > borg::msg::MEMBER_BYTES, "each kind alone fits a share, both do not");
1867 append_lines(&a, borg::msg::FILE, &out);
1868 let (alice_node, bob_node) = (Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap(), Node::bind(&pb, LOCAL.parse().unwrap()).await.unwrap());
1869 let ticket = alice_node.ticket();
1870 let (served, synced) = tokio::join!(alice_node.accept(), bob_node.sync(&ticket));
1871 assert_eq!((served.unwrap().rejected, synced.unwrap().rejected), (0, 0));
1872 assert!(out.iter().all(|l| holds(&b, borg::msg::FILE, l)), "bob keeps every tell");
1873 append_lines(&a, borg::msg::FILE, &here);
1875 let first: borg::msg::Message = serde_json::from_str(&here[0]).unwrap();
1876 borg::msg::ack(&a, "s-001", &[&first.id], now).unwrap();
1877 assert!(out.iter().chain(&here).all(|l| holds(&a, borg::msg::FILE, l)), "alice keeps what travels and what stays");
1878 for _ in 0..2 {
1879 let (served, synced) = tokio::join!(alice_node.accept(), bob_node.sync(&ticket));
1880 assert_eq!((served.unwrap().rejected, synced.unwrap().rejected), (0, 0), "nothing offered back, nothing refused");
1881 }
1882 alice_node.close().await;
1883 bob_node.close().await;
1884 }
1885
1886 #[tokio::test(flavor = "multi_thread")]
1890 async fn a_line_written_between_the_decision_and_the_lock_is_kept() {
1891 let (pa, pb) = team("race");
1892 let (a, b) = (home(&pa), home(&pb));
1893 let alice = Identity::load_or_create(&a).unwrap();
1894 let bob = Identity::load_or_create(&b).unwrap().public();
1895 let now = borg::learn::now_ms();
1896 let expired = line_of(&alice.public(), "s1", &format!("r0@{bob}"), now - borg::msg::TTL_MS - borg::steer::SLACK_MS - 1, "gone").1;
1897 let burst: Vec<(borg::msg::Message, String)> =
1899 (0..borg::msg::WINDOW_CAP).map(|i| line_of(&alice.public(), "s1", &format!("r1@{bob}"), now - 2000 + i as i64, &format!("t{i}"))).collect();
1900 let next = line_of(&alice.public(), "s1", &format!("r1@{bob}"), now - 3000, "earlier").1;
1901 let other = line_of(&alice.public(), "s2", &format!("r2@{bob}"), now - 1000, "other").1;
1902 append_lines(&b, borg::msg::FILE, &std::iter::once(expired.clone()).chain(burst.iter().map(|(_, l)| l.clone())).collect::<Vec<_>>());
1903 let path = b.join(borg::msg::FILE);
1904 let keep: BTreeMap<Hash, Signed> =
1905 [&next, &other].into_iter().map(|l| (lines::hash(l), Signed { origin: Origin::new(&alice, &lines::hash(l), now), line: l.clone() })).collect();
1906 let read = std::fs::read_to_string(&path).unwrap();
1907 let decided = stored(&read, &keep, now);
1908 assert_eq!((decided.taken.len(), decided.refused, decided.body.is_some()), (1, 1, true), "one more unread tell to r1 than any window lets through");
1909 let seen = seen_of(&bob, "r1", now - 500, &[&burst[0].0.id]);
1911 append_lines(&b, borg::msg::FILE, std::slice::from_ref(&seen));
1912 let bob_node = Node::bind(&pb, LOCAL.parse().unwrap()).await.unwrap();
1913 let mut report = Report::default();
1914 bob_node.store_messages(&path, keep, (&read, decided), &members(&b).unwrap(), now, &mut report).unwrap();
1915 assert!(holds(&b, borg::msg::FILE, &seen), "the acknowledgement written in between stays");
1916 assert!(holds(&b, borg::msg::FILE, &next) && holds(&b, borg::msg::FILE, &other), "and it let the earlier tell in");
1917 assert!(!holds(&b, borg::msg::FILE, &expired), "the bound still pruned");
1918 assert_eq!((report.received, report.rejected), (2, 0));
1919 bob_node.close().await;
1920 }
1921
1922 fn line_of(node: &str, by: &str, to: &str, ms: i64, text: &str) -> (borg::msg::Message, String) {
1924 let from = borg::msg::Sender { by: by.into(), name: "One of One".into(), harness: "claude".into(), node: node.into() };
1925 let m = borg::msg::Message::new(borg::msg::TELL, &from, to, None, ms, text);
1926 let line = serde_json::to_string(&m).unwrap();
1927 (m, line)
1928 }
1929
1930 fn seen_of(node: &str, by: &str, ms: i64, ids: &[&str]) -> String {
1932 serde_json::to_string(&borg::msg::Seen::new(by, node, ms, ids)).unwrap()
1933 }
1934
1935 #[tokio::test(flavor = "multi_thread")]
1938 async fn a_seen_line_from_the_recipients_node_opens_the_senders_window() {
1939 let p = crew("seen", &["alice", "bob", "carol"]);
1940 let (a, b, c) = (home(&p[0]), home(&p[1]), home(&p[2]));
1941 let keys: Vec<String> = [&a, &b, &c].iter().map(|h| Identity::load_or_create(h).unwrap().public()).collect();
1942 let now = borg::learn::now_ms();
1943 let to = format!("s2@{}", keys[1]);
1944 let from = borg::msg::Sender { by: "s1".into(), name: "One of One".into(), harness: "claude".into(), node: keys[0].clone() };
1945 let tell = |ms: i64, text: &str| borg::msg::send(&a, borg::msg::Message::new(borg::msg::TELL, &from, &to, None, ms, text), ms);
1946 let x = tell(now, "one").unwrap();
1947 assert!(matches!(tell(now + 1, "two"), Err(borg::msg::SendError::Window { outstanding: 1, .. })));
1948 let (alice, bob, carol) = (
1949 Node::bind(&p[0], LOCAL.parse().unwrap()).await.unwrap(),
1950 Node::bind(&p[1], LOCAL.parse().unwrap()).await.unwrap(),
1951 Node::bind(&p[2], LOCAL.parse().unwrap()).await.unwrap(),
1952 );
1953 let ticket = alice.ticket();
1954 let forged = seen_of(&keys[2], "s2", now, &[&x.id]);
1956 append_lines(&c, borg::msg::FILE, std::slice::from_ref(&forged));
1957 let (served, synced) = tokio::join!(alice.accept(), carol.sync(&ticket));
1958 served.unwrap();
1959 synced.unwrap();
1960 assert!(holds(&a, borg::msg::FILE, &forged), "a seen line travels under its node");
1961 assert!(matches!(tell(now + 2, "two"), Err(borg::msg::SendError::Window { .. })), "but s2 is bob's");
1962 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
1964 served.unwrap();
1965 synced.unwrap();
1966 let caps = borg::harness::by_name("claude").unwrap().caps();
1967 let out = borg::msg::deliver(&b, caps, "prompt", &json!({"session_id": "s2"}), Some(now - 10), now + 3, None).expect("shown");
1968 assert_eq!(out["shown_ids"], json!([x.id]));
1969 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
1970 served.unwrap();
1971 assert_eq!(synced.unwrap().rejected, 0);
1972 tell(now + 4, "two").expect("acknowledged on bob's node");
1973 assert_eq!(borg::msg::read(&b), vec![x], "no seen line is read as a message");
1974 for n in [alice, bob, carol] {
1975 n.close().await;
1976 }
1977 }
1978
1979 #[tokio::test(flavor = "multi_thread")]
1984 async fn a_tell_admitted_when_written_travels_after_what_opened_its_window_is_pruned() {
1985 use borg::msg::{Author, Message, SendError, Sender, TELL, TTL_MS};
1986 let p = crew("pruned", &["alice", "bob", "carol"]);
1987 let (a, b, c) = (home(&p[0]), home(&p[1]), home(&p[2]));
1988 let key = Identity::load_or_create(&a).unwrap().public();
1989 let now = borg::learn::now_ms();
1990 let s1 = Sender { by: "s1".into(), name: "One of One".into(), harness: "claude".into(), node: key.clone() };
1991 let to = format!("s2@{key}");
1992 let tell = |ms: i64, text: &str| borg::msg::send(&a, Message::new(TELL, &s1, &to, None, ms, text), ms);
1993 let t = now - TTL_MS - borg::steer::SLACK_MS - 60_000;
1995 let first = tell(t, "first").unwrap();
1996 borg::msg::ack(&a, "s2", &[&first.id], t + 1).unwrap();
1997 let t2 = t + 2 * borg::steer::SLACK_MS;
1998 let (second, third) = (tell(t2, "second").unwrap(), tell(t2 + 1, "third").unwrap());
1999 assert!(matches!(tell(t2 + 2, "fourth"), Err(SendError::Window { outstanding: 2, window: 2, .. })), "the acknowledgement doubled it");
2000 borg::msg::ack(&a, "s7", &[&second.id], now).unwrap();
2002 let held = borg::msg::lines(&a);
2003 assert!(held.iter().all(|l| l.message().is_none_or(|m| m.id != first.id)), "the first is pruned");
2004 let w = borg::msg::window_for(&held, Author { node: &key, by: "s1" }, &to, now);
2005 assert!(w.admits(&second.id) && !w.admits(&third.id), "a replay over what is left admits less than send did");
2006 let (alice, bob, carol) = (
2007 Node::bind(&p[0], LOCAL.parse().unwrap()).await.unwrap(),
2008 Node::bind(&p[1], LOCAL.parse().unwrap()).await.unwrap(),
2009 Node::bind(&p[2], LOCAL.parse().unwrap()).await.unwrap(),
2010 );
2011 let line = |m: &Message| serde_json::to_string(m).unwrap();
2012 let (from_alice, from_bob) = (alice.ticket(), bob.ticket());
2013 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&from_alice));
2014 served.unwrap();
2015 assert_eq!(synced.unwrap().rejected, 0);
2016 assert!(holds(&b, borg::msg::FILE, &line(&second)) && holds(&b, borg::msg::FILE, &line(&third)), "bob takes both from alice");
2017 let (served, synced) = tokio::join!(bob.accept(), carol.sync(&from_bob));
2018 served.unwrap();
2019 assert_eq!(synced.unwrap().rejected, 0);
2020 assert!(holds(&c, borg::msg::FILE, &line(&second)) && holds(&c, borg::msg::FILE, &line(&third)), "and carol both from bob");
2021 for n in [alice, bob, carol] {
2022 n.close().await;
2023 }
2024 }
2025
2026 #[tokio::test(flavor = "multi_thread")]
2029 async fn a_merged_seen_line_leaves_the_logs_talk_and_a_merged_tell_moves_it() {
2030 let p = crew("talk", &["alice", "bob"]);
2031 let (a, b) = (home(&p[0]), home(&p[1]));
2032 let keys: Vec<String> = [&a, &b].iter().map(|h| Identity::load_or_create(h).unwrap().public()).collect();
2033 let now = borg::learn::now_ms();
2034 let sender = |node: &str, by: &str| borg::msg::Sender { by: by.into(), name: "One of One".into(), harness: "claude".into(), node: node.into() };
2035 let x = borg::msg::Message::new(borg::msg::TELL, &sender(&keys[0], "s1"), &format!("s2@{}", keys[1]), None, now, "one");
2036 let x = borg::msg::send(&a, x, now).unwrap();
2037 let (alice, bob) = (Node::bind(&p[0], LOCAL.parse().unwrap()).await.unwrap(), Node::bind(&p[1], LOCAL.parse().unwrap()).await.unwrap());
2038 let ticket = alice.ticket();
2039 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
2040 served.unwrap();
2041 synced.unwrap();
2042 let caps = borg::harness::by_name("claude").unwrap().caps();
2043 let out = borg::msg::deliver(&b, caps, "prompt", &json!({"session_id": "s2"}), Some(now - 10), now + 1, None).expect("shown on bob's node");
2044 assert_eq!(out["shown_ids"], json!([x.id]));
2045 let (log, talk) = (borg::msg::mark_of(&borg::msg::path(&a)), borg::msg::talk_mark(&a));
2046 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
2047 served.unwrap();
2048 assert_eq!(synced.unwrap().rejected, 0);
2049 assert!(
2050 lines::read(&borg::msg::path(&a)).values().any(|l| matches!(Line::parse(l), Some(Line::Seen(s)) if s.ids == [x.id.clone()])),
2051 "the seen line merged"
2052 );
2053 assert_ne!(borg::msg::mark_of(&borg::msg::path(&a)), log, "the log moved");
2054 assert_eq!(borg::msg::talk_mark(&a), talk, "its talk did not");
2055 let reply = borg::msg::Message::new(borg::msg::TELL, &sender(&keys[1], "s2"), &format!("s1@{}", keys[0]), Some(&x.id), now + 2, "done");
2057 let reply = borg::msg::send(&b, reply, now + 2).unwrap();
2058 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
2059 served.unwrap();
2060 synced.unwrap();
2061 assert_ne!(borg::msg::talk_mark(&a), talk, "a tell moves it");
2062 let out = borg::msg::deliver(&a, caps, "prompt", &json!({"session_id": "s1"}), Some(now - 10), now + 3, None).expect("shown on alice's node");
2063 assert_eq!(out["shown_ids"], json!([reply.id]));
2064 for n in [alice, bob] {
2065 n.close().await;
2066 }
2067 }
2068
2069 #[tokio::test(flavor = "multi_thread")]
2072 async fn a_mesh_13_peer_gets_only_v1_lines_in_v1_pacing_both_ways() {
2073 let p = crew("v13", &["alice", "old1", "old2"]);
2074 let (a, o1, o2) = (home(&p[0]), home(&p[1]), home(&p[2]));
2075 let keys: Vec<String> = [&a, &o1, &o2].iter().map(|h| Identity::load_or_create(h).unwrap().public()).collect();
2076 let now = borg::learn::now_ms();
2077 let to = format!("s-r@{}", keys[0]);
2078 let (first, first_line) = line_of(&keys[0], "s1", &to, now - 3000, "first");
2079 let seen = seen_of(&keys[0], "s-r", now - 2500, &[&first.id]);
2080 let (second, second_line) = line_of(&keys[0], "s1", &to, now - 2000, "second");
2081 let (long, long_line) = line_of(&keys[0], "s1", &format!("s-x@{}", keys[0]), now - 1000, &"w".repeat(borg::msg::V1_TEXT + 1));
2082 assert_eq!((first.v, second.v, long.v), (1, 1, 2));
2083 append_lines(&a, borg::msg::FILE, &[first_line.clone(), seen.clone(), second_line.clone(), long_line.clone()]);
2084 let theirs: Vec<String> = [(&o1, &keys[1]), (&o2, &keys[2])].iter().map(|(h, k)| message(h, k, ALL_TO, now - 100, "from an older node")).collect();
2085 let alice = Node::bind(&p[0], LOCAL.parse().unwrap()).await.unwrap();
2086 let old = |i: usize| Node::open(&p[i], Some(LOCAL.parse().unwrap()), true, |x| x.alpns(vec![V13_ALPN.to_vec()]));
2087 let (old1, old2) = (old(1).await.unwrap(), old(2).await.unwrap());
2088 let ticket = alice.ticket();
2090 let (served, dialled) = tokio::join!(alice.accept(), old1.dial(&ticket, &[], V13_ALPN, Vec::new()));
2091 served.unwrap();
2092 dialled.unwrap();
2093 let old_ticket = old2.ticket();
2095 let (served, dialled) = tokio::join!(old2.accept(), alice.sync(&old_ticket));
2096 served.unwrap();
2097 dialled.unwrap();
2098 for o in [&o1, &o2] {
2099 assert!(holds(o, borg::msg::FILE, &first_line), "a v1 line travels");
2100 assert!(!holds(o, borg::msg::FILE, &second_line), "0.5.0's pacing: one per sender and recipient per rate slot");
2101 assert!(!holds(o, borg::msg::FILE, &long_line) && !holds(o, borg::msg::FILE, &seen), "no v2 text, no seen line");
2102 let v1 = |l: &String| serde_json::from_str::<borg::msg::Message>(l).is_ok_and(|m| m.v == 1);
2103 assert!(lines::read(&o.join(borg::msg::FILE)).values().all(v1), "every line a 0.5.0 reader reads");
2104 }
2105 assert!(theirs.iter().all(|l| holds(&a, borg::msg::FILE, l)), "their lines reach this node");
2106 for n in [alice, old1, old2] {
2107 n.close().await;
2108 }
2109 }
2110
2111 #[tokio::test(flavor = "multi_thread")]
2115 async fn two_tells_a_rate_slot_apart_reach_a_mesh_13_node() {
2116 let p = crew("v13-rate", &["alice", "old"]);
2117 let (a, o) = (home(&p[0]), home(&p[1]));
2118 let keys: Vec<String> = [&a, &o].iter().map(|h| Identity::load_or_create(h).unwrap().public()).collect();
2119 let now = borg::learn::now_ms();
2120 let from = borg::msg::Sender { by: "s1".into(), name: "One of One".into(), harness: "claude".into(), node: keys[0].clone() };
2121 let to = format!("s9@{}", keys[1]);
2122 let tell = |ms: i64, text: &str| borg::msg::send(&a, borg::msg::Message::new(borg::msg::TELL, &from, &to, None, ms, text), ms);
2123 let first = tell(now - V1_RATE_MS - 1000, "first").unwrap();
2124 assert!(matches!(tell(now - V1_RATE_MS, "too soon"), Err(borg::msg::SendError::Window { pace: borg::msg::Pace::Rate, .. })));
2125 let second = tell(now - 500, "a slot later").unwrap();
2126 let alice = Node::bind(&p[0], LOCAL.parse().unwrap()).await.unwrap();
2127 let old = Node::open(&p[1], Some(LOCAL.parse().unwrap()), true, |x| x.alpns(vec![V13_ALPN.to_vec()])).await.unwrap();
2128 let ticket = alice.ticket();
2129 let (served, dialled) = tokio::join!(alice.accept(), old.dial(&ticket, &[], V13_ALPN, Vec::new()));
2130 served.unwrap();
2131 dialled.unwrap();
2132 let line = |m: &borg::msg::Message| serde_json::to_string(m).unwrap();
2133 assert!(holds(&o, borg::msg::FILE, &line(&first)) && holds(&o, borg::msg::FILE, &line(&second)), "both offered and taken");
2134 for n in [alice, old] {
2135 n.close().await;
2136 }
2137 }
2138
2139 #[tokio::test(flavor = "multi_thread")]
2144 async fn a_text_a_mesh_13_node_cannot_read_is_refused_for_its_sessions() {
2145 let p = crew("v13-long", &["alice", "old"]);
2146 let (a, o) = (home(&p[0]), home(&p[1]));
2147 let keys: Vec<String> = [&a, &o].iter().map(|h| Identity::load_or_create(h).unwrap().public()).collect();
2148 let now = borg::learn::now_ms();
2149 let from = borg::msg::Sender { by: "s1".into(), name: "One of One".into(), harness: "claude".into(), node: keys[0].clone() };
2150 let tell = |to: &str, ms: i64, text: &str| borg::msg::send(&a, borg::msg::Message::new(borg::msg::TELL, &from, to, None, ms, text), ms);
2151 let long = "w".repeat(borg::msg::V1_TEXT + 1);
2152 let alice = Node::bind(&p[0], LOCAL.parse().unwrap()).await.unwrap();
2153 let old = Node::open(&p[1], Some(LOCAL.parse().unwrap()), true, |x| x.alpns(vec![V13_ALPN.to_vec()])).await.unwrap();
2154 let ticket = alice.ticket();
2155 let (served, dialled) = tokio::join!(alice.accept(), old.dial(&ticket, &[], V13_ALPN, Vec::new()));
2156 served.unwrap();
2157 dialled.unwrap();
2158 assert_eq!(borg::presence::reads(&a).get(&keys[1]), Some(&1));
2159 let to = format!("s9@{}", keys[1]);
2160 let refused = tell(&to, now, &long).unwrap_err();
2161 assert!(matches!(&refused, borg::msg::SendError::Reads { v: 1, .. }), "{refused:?}");
2162 assert!(refused.to_string().contains(&format!("at most {} characters", borg::msg::V1_TEXT)), "{refused}");
2163 tell(&to, now + 1, "short").expect("a v1 text goes, and nothing held the window");
2164 tell(ALL_TO, now + 2, &long).expect("the other members read it");
2165 let (served, dialled) = tokio::join!(alice.accept(), old.dial(&ticket, &[], ALPN, Vec::new()));
2166 served.unwrap();
2167 dialled.unwrap();
2168 tell(&format!("s8@{}", keys[1]), now + 3, &long).expect("upgraded");
2169 alice.close().await;
2170 old.close().await;
2171 }
2172
2173 #[tokio::test(flavor = "multi_thread")]
2177 async fn a_membership_change_seen_at_different_times_keeps_the_messages() {
2178 let (pa, pb) = team("members");
2179 let (a, b) = (home(&pa), home(&pb));
2180 let alice_id = Identity::load_or_create(&a).unwrap();
2181 let bob_key = Identity::load_or_create(&b).unwrap().public();
2182 let now = borg::learn::now_ms();
2183 let n = V1_KEEP / 2 + 50;
2184 let sent: Vec<String> = (0..n).map(|i| message(&a, &alice_id.public(), &format!("s-{i:03}@{bob_key}"), now - (n - i) as i64, "hi")).collect();
2185 let alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
2186 let bob = Node::bind(&pb, LOCAL.parse().unwrap()).await.unwrap();
2187 let (ticket, bob_ticket) = (alice.ticket(), bob.ticket());
2188 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
2189 served.unwrap();
2190 assert_eq!(synced.unwrap().rejected, 0);
2191 let carol = Identity::load_or_create(&a.join("carol")).unwrap();
2193 let mut m = Membership::load(&a).unwrap().unwrap();
2194 m.invite(&alice_id, &carol.public(), "member").unwrap();
2195 m.save(&a).unwrap();
2196 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
2197 assert_eq!((served.unwrap().rejected, synced.unwrap().rejected), (0, 0));
2198 let (served, synced) = tokio::join!(bob.accept(), alice.sync(&bob_ticket));
2199 assert_eq!((served.unwrap().rejected, synced.unwrap().rejected), (0, 0));
2200 for h in [&a, &b] {
2201 assert!(sent.iter().all(|l| holds(h, borg::msg::FILE, l)), "every message kept");
2202 }
2203 assert_eq!(Membership::load(&b).unwrap().unwrap().members.len(), 3, "bob saw the change");
2204 alice.close().await;
2205 bob.close().await;
2206 }
2207
2208 #[tokio::test(flavor = "multi_thread")]
2209 async fn team_outcomes_sync_and_count_once() {
2210 let (pa, pb) = team("outcomes");
2211 let (a, b) = (home(&pa), home(&pb));
2212 write(&a, "lessons.jsonl", &[lesson("L1", 1, "lock order?")]);
2213 let o = |kind: &str, ms| json!({"kind": kind, "session": "s1", "lessons": ["L1"], "ms": ms, "repo": "app", "groups": [["L1", "L2"]]}).to_string();
2214 write(&a, "outcomes.jsonl", &[o("asked", 1), o("asked_then_changed", 2)]);
2215 assert_eq!(borg::outcome::publish(&a), 2);
2216 let alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
2217 let bob = Node::bind(&pb, LOCAL.parse().unwrap()).await.unwrap();
2218 let ticket = alice.ticket();
2219 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
2220 served.unwrap();
2221 synced.unwrap();
2222 let team = borg::outcome::TEAM_FILE;
2223 assert_eq!(lines::read(&b.join(team)), lines::read(&a.join(team)), "bob holds alice's outcomes");
2224 let text = std::fs::read_to_string(b.join(team)).unwrap();
2225 assert!(!text.contains("\"s1\"") && !text.contains("app") && !text.contains("groups"), "redacted: {text}");
2226 assert!(!b.join("outcomes.jsonl").exists(), "the raw log stays with alice");
2227 let l1 = [borg::conscience::Challenge { id: "L1".into(), ..Default::default() }];
2228 let changed = |h: &Path| borg::memory::derive(&borg::outcome::read_team(&h.join("lessons.jsonl")), &l1, None)["L1"].changed;
2229 assert_eq!(changed(&b), 1, "alice's session counts on bob");
2230 assert_eq!(changed(&a), 1, "alice's own published copy does not count twice");
2231 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
2232 let quiet = Report { rounds: SYNCED.len(), fps: SYNCED.len(), ..Report::default() };
2233 assert_eq!((synced.unwrap(), served.unwrap()), (quiet, quiet), "resync is quiet");
2234 alice.close().await;
2235 bob.close().await;
2236 }
2237
2238 #[tokio::test(flavor = "multi_thread")]
2239 async fn two_members_end_with_the_union_and_resync_moves_only_digests() {
2240 let (pa, pb) = team("union");
2241 let (a, b) = (home(&pa), home(&pb));
2242 write(&a, "lessons.jsonl", &[lesson("L1", 1, "lock order?"), lesson("L2", 1, "retry budget?")]);
2243 write(&b, "lessons.jsonl", &[lesson("L1", 1, "lock order?"), lesson("L1", 2, "lock order, also in tests?"), lesson("L3", 1, "idempotent?")]);
2244
2245 let carol = Identity::load_or_create(&home(&node("union-carol", "team"))).unwrap().public();
2247 let mut m = Membership::load(&a).unwrap().unwrap();
2248 m.invite(&Identity::load_or_create(&a).unwrap(), &carol, "member").unwrap();
2249 m.save(&a).unwrap();
2250
2251 let alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
2252 let bob = Node::bind(&pb, LOCAL.parse().unwrap()).await.unwrap();
2253 let ticket = alice.ticket();
2254 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
2255 let (served, synced) = (served.unwrap(), synced.unwrap());
2256 assert_eq!((synced.received, synced.sent), (1, 2), "bob gets L2, gives L1v2 and L3");
2257 assert_eq!((served.received, served.sent), (2, 1));
2258 assert_eq!(Membership::load(&b).unwrap().unwrap().role_of(&carol), Some("member"), "bob adopted the newer membership");
2259 for file in SYNCED {
2260 assert_eq!(lines::read(&a.join(file)), lines::read(&b.join(file)), "{file}: both hold the union");
2261 }
2262 let asked = |h: &Path| {
2263 let mut v: Vec<(String, String)> = borg::conscience::load(&h.join("lessons.jsonl"), None).into_iter().map(|c| (c.id, c.question)).collect();
2264 v.sort();
2265 v
2266 };
2267 assert_eq!(asked(&a), asked(&b));
2268 assert!(asked(&a).contains(&("L1".into(), "lock order, also in tests?".into())), "the newer version wins on alice too");
2269
2270 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
2271 let quiet = Report { rounds: SYNCED.len(), fps: SYNCED.len(), ..Report::default() };
2273 assert_eq!(synced.unwrap(), quiet, "re-sync: one fingerprint per log, nothing new");
2274 assert_eq!(served.unwrap(), quiet);
2275 alice.close().await;
2276 bob.close().await;
2277 }
2278
2279 #[tokio::test(flavor = "multi_thread")]
2282 async fn an_episode_is_learned_only_where_it_was_captured() {
2283 let (pa, pb) = team("episodes");
2284 let (a, b) = (home(&pa), home(&pb));
2285 let e1 = json!({"id": "e1", "ms": 1, "repo": "r", "session": "s", "task": "t", "agent": "a",
2286 "rejected_tool": null, "correction": "check TOCTOU on the flag", "engineer": "ann@x"});
2287 write(&a, "episodes.jsonl", &[e1.to_string()]);
2288 let calls = std::cell::Cell::new(0);
2289 let ask = |_: &str| {
2290 calls.set(calls.get() + 1);
2291 Some(json!({"is_lesson": true, "question": "Is the check-then-act atomic?", "situations": [], "consequence": "race", "same_as": ""}))
2292 };
2293
2294 let alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
2295 let bob = Node::bind(&pb, LOCAL.parse().unwrap()).await.unwrap();
2296 let ticket = alice.ticket();
2297 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
2298 served.unwrap();
2299 synced.unwrap();
2300 assert!(!b.join("episodes.jsonl").exists(), "alice's raw episode stays with alice");
2301 assert_eq!(borg::learn::learn_in(&b, None, &ask, false, 2), (0, 0, 0));
2302 assert_eq!(calls.get(), 0, "bob asks no model for alice's episode");
2303
2304 assert_eq!(borg::learn::learn_in(&a, None, &ask, false, 2), (1, 0, 0));
2305 assert_eq!(calls.get(), 1, "alice learns e1 once");
2306 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
2307 served.unwrap();
2308 assert_eq!(synced.unwrap().received, 1, "bob receives the lesson");
2309 let got = borg::learn::lessons(&b.join("lessons.jsonl"));
2310 assert_eq!(
2311 (got.len(), got[0].id.as_str(), got[0].episodes.clone(), got[0].engineers.clone()),
2312 (1, "Le1", vec!["e1".to_string()], vec!["ann@x".to_string()])
2313 );
2314 assert_eq!(borg::learn::learn_in(&b, None, &ask, false, 3), (0, 0, 0));
2315 assert_eq!(calls.get(), 1, "and still learns nothing again");
2316 alice.close().await;
2317 bob.close().await;
2318 }
2319
2320 #[tokio::test(flavor = "multi_thread")]
2324 async fn a_member_on_an_older_protocol_is_named_both_ways() {
2325 const OLD: &[u8] = b"borg/mesh/5";
2326 let (pa, pb) = team("protocol");
2327 let (a, b) = (home(&pa), home(&pb));
2328 let bob_key = Identity::load_or_create(&b).unwrap().public();
2329 let alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
2330 let old_bob = Node::open(&pb, Some(LOCAL.parse().unwrap()), true, |x| x.alpns(vec![OLD.to_vec()])).await.unwrap();
2331
2332 let serve_old = async {
2334 let conn = old_bob.incoming().await?.await.ok()?;
2335 Some((conn.alpn().to_vec(), conn.closed().await.to_string()))
2336 };
2337 let old_ticket = old_bob.ticket();
2338 let (seen, synced) = tokio::join!(serve_old, alice.sync(&old_ticket));
2339 let err = synced.unwrap_err();
2340 let ours = String::from_utf8_lossy(ALPN);
2341 let line = format!("peer {} speaks borg/mesh/5, this node {ours}: nothing syncs with it; upgrade it", &bob_key[..12]);
2342 assert_eq!(err, line);
2343 let (alpn, closed) = seen.expect("the old node completed the handshake");
2344 assert_eq!(alpn, OLD);
2345 let alice_key = Identity::load_or_create(&a).unwrap().public();
2346 let told = format!("peer {} speaks {ours}, this node borg/mesh/5: nothing syncs with it; upgrade this node", &alice_key[..12]);
2347 assert!(closed.contains(&told), "the old node is told why, as it reads it: {closed}");
2348 assert_eq!(borg::presence::mismatches(&a)[&bob_key].theirs, "borg/mesh/5");
2349
2350 let _ = std::fs::remove_file(borg::presence::protocols_path(&a));
2352 let addr = Target::parse(&alice.ticket()).unwrap().endpoint_addr(&old_bob.relay).unwrap();
2353 let dial_old = async {
2354 let conn = old_bob.endpoint.connect(addr, OLD).await.map_err(|e| e.to_string())?;
2355 Ok::<String, String>(conn.closed().await.to_string())
2356 };
2357 let (served, dialled) = tokio::join!(alice.accept(), dial_old);
2358 assert_eq!(served.unwrap_err(), line);
2359 assert!(dialled.unwrap().contains(&told));
2360 assert_eq!(borg::presence::mismatches(&a)[&bob_key].ours, ours);
2361 old_bob.close().await;
2362
2363 let bob = Node::bind(&pb, LOCAL.parse().unwrap()).await.unwrap();
2365 let ticket = alice.ticket();
2366 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
2367 served.unwrap();
2368 synced.unwrap();
2369 assert!(borg::presence::mismatches(&a).is_empty());
2370 alice.close().await;
2371 bob.close().await;
2372 }
2373
2374 #[test]
2375 fn older_protocols_are_every_version_below_the_spoken_ones() {
2376 let n = version(LEGACY_ALPN).expect("borg/mesh/<n>");
2377 assert_eq!(
2378 (version(V13_ALPN), version(V14_ALPN), version(ALPN)),
2379 (Some(n + 1), Some(n + 2), Some(n + 3)),
2380 "this protocol, 0.6's, 0.4.0's and 0.3.0's are the four spoken"
2381 );
2382 assert!(spoken(ALPN) && spoken(V14_ALPN) && spoken(V13_ALPN) && spoken(LEGACY_ALPN));
2383 assert_eq!((reads(ALPN), reads(V14_ALPN), reads(V13_ALPN), reads(LEGACY_ALPN)), (2, 2, 1, 0));
2384 let o = older();
2385 let below = format!("borg/mesh/{}", n - 1);
2386 assert_eq!((o.len(), o[0].as_slice(), o[o.len() - 1].as_slice()), (n as usize - 1, below.as_bytes(), b"borg/mesh/1".as_slice()));
2387 assert!(o.iter().all(|a| !spoken(a)));
2388 assert_eq!(synced(ALPN).filter(|f| *f != PLAYBOOKS).collect::<Vec<_>>(), SYNCED, "every synced log, in order");
2389 assert_eq!(synced(ALPN).collect::<Vec<_>>(), [&OLD_SYNCED[..], &[borg::msg::FILE, lines::ABSORBED]].concat(), "0.6's list, then the absorbs");
2390 assert_eq!(synced(V14_ALPN).collect::<Vec<_>>(), [&OLD_SYNCED[..], &[borg::msg::FILE]].concat(), "0.6's list, in its order");
2391 assert_eq!(synced(V13_ALPN).collect::<Vec<_>>(), synced(V14_ALPN).collect::<Vec<_>>(), "0.4.0's list is 0.6's");
2392 assert_eq!(synced(LEGACY_ALPN).collect::<Vec<_>>(), OLD_SYNCED, "0.3.0's list, in its order");
2393 }
2394
2395 #[test]
2398 fn every_spoken_protocol_has_a_note() {
2399 let ours = version(ALPN).unwrap();
2400 assert_eq!(borg::protocol::current(), ours, "add borg/mesh/{ours} to borg protocol.rs NOTES, first");
2401 let oldest = version(LEGACY_ALPN).unwrap();
2402 for n in oldest + 1..=ours {
2403 let alpn = format!("borg/mesh/{n}");
2404 assert!(spoken(alpn.as_bytes()), "{alpn} is spoken");
2405 assert!(borg::protocol::note(n).is_some(), "{alpn} has a NOTES line");
2406 }
2407 assert_eq!(borg::protocol::announcement_head(ours), format!("{} is in use:", String::from_utf8_lossy(ALPN)));
2408 }
2409
2410 #[test]
2413 fn a_hello_without_a_version_is_an_older_node() {
2414 let old: Msg = serde_json::from_str(r#"{"t":"hello","collective":"work","listen":null,"relay":null}"#).unwrap();
2415 assert!(matches!(old, Msg::Hello { borg: None, .. }));
2416 let older: Msg = serde_json::from_str(r#"{"t":"hello","collective":"work","listen":"127.0.0.1:1"}"#).unwrap();
2417 assert!(matches!(older, Msg::Hello { borg: None, relay: None, .. }), "before the relay too");
2418 let ours = Msg::Hello { collective: "work".into(), listen: None, relay: None, borg: Speaks::this().borg.map(str::to_string) };
2419 let text = serde_json::to_string(&ours).unwrap();
2420 assert!(text.contains(&format!(r#""borg":"{}""#, env!("CARGO_PKG_VERSION"))), "{text}");
2421 let posing = serde_json::to_string(&Msg::Hello { collective: "work".into(), listen: None, relay: None, borg: None }).unwrap();
2422 assert!(!posing.contains("borg"), "the old shape: {posing}");
2423 assert!(tells(None) && tells(Some("on")) && !tells(Some("off")), "BORG_ANNOUNCE=off turns it off");
2424 }
2425
2426 fn announcements(home: &Path, n: u32) -> Vec<borg::msg::Message> {
2428 let head = borg::protocol::announcement_head(n);
2429 borg::msg::read(home).into_iter().filter(|m| m.text.starts_with(&head)).collect()
2430 }
2431
2432 #[tokio::test(flavor = "multi_thread")]
2436 async fn an_older_protocol_is_announced_once_and_retried_after_a_refusal() {
2437 let (pa, _) = team("announce");
2438 let a = home(&pa);
2439 let mut alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
2440 let ours = version(ALPN).unwrap();
2441 assert!(alice.announce(ALPN).is_none(), "a peer on this protocol");
2442 assert!(alice.announce(JOIN_ALPN).is_none(), "no session protocol");
2443 alice.speaks.tell = false;
2444 assert!(alice.announce(V14_ALPN).is_none(), "BORG_ANNOUNCE=off");
2445 alice.speaks.tell = true;
2446
2447 let from = borg::msg::Sender { by: borg::msg::ENGINEER.into(), name: borg::msg::ENGINEER.into(), harness: "terminal".into(), node: alice.public() };
2449 let now = borg::learn::now_ms();
2450 let mut filled = 0;
2451 while borg::msg::send(&a, borg::msg::Message::new(borg::msg::TELL, &from, ALL_TO, None, now + filled, &format!("note {filled}")), now + filled).is_ok()
2452 {
2453 filled += 1;
2454 assert!(filled < 1000, "the window bounds a sender");
2455 }
2456 assert!(alice.announce(V14_ALPN).is_none(), "refused by the window");
2457 assert!(announcements(&a, ours).is_empty());
2458 std::fs::remove_file(borg::msg::path(&a)).unwrap(); let freeze = borg::msg::send(&a, borg::msg::Message::new(borg::msg::TELL, &from, ALL_TO, None, now - 60_000, "freeze"), now - 60_000).unwrap();
2463 borg::msg::ack(&a, "s1", &[&freeze.id], now - 30_000).unwrap();
2464 assert!(alice.announce(V13_ALPN).is_none(), "the engineer's v1 slot to all is held");
2465 assert!(announcements(&a, ours).is_empty());
2466 std::fs::remove_file(borg::msg::path(&a)).unwrap();
2467
2468 let sent = alice.announce(V13_ALPN).expect("announced");
2469 assert_eq!(
2470 (sent.kind, sent.to.as_str(), sent.by.as_str(), sent.node.as_str()),
2471 (borg::msg::TELL, ALL_TO, borg::msg::ENGINEER, alice.public().as_str())
2472 );
2473 assert_eq!(sent.v, 1, "a 0.4 or 0.5 node reads it: {}", sent.text);
2474 for missed in [14, 15] {
2475 assert!(sent.text.contains(borg::protocol::note(missed).unwrap()), "{}", sent.text);
2476 }
2477 assert!(sent.text.contains(env!("CARGO_PKG_VERSION")) && sent.text.contains("borg update"), "{}", sent.text);
2478 assert!(alice.announce(V14_ALPN).is_none() && alice.announce(V13_ALPN).is_none(), "held: once");
2479 assert_eq!(announcements(&a, ours).len(), 1);
2480 alice.close().await;
2481 }
2482
2483 #[tokio::test(flavor = "multi_thread")]
2485 async fn sessions_that_end_together_announce_once() {
2486 let (pa, _) = team("announce-race");
2487 let a = home(&pa);
2488 let alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
2489 let ours = version(ALPN).unwrap();
2490 let from = borg::msg::Sender { by: borg::msg::ENGINEER.into(), name: borg::msg::ENGINEER.into(), harness: "terminal".into(), node: alice.public() };
2492 let then = borg::learn::now_ms() - V1_RATE_MS - 60_000;
2493 let early = borg::msg::send(&a, borg::msg::Message::new(borg::msg::TELL, &from, ALL_TO, None, then, "early"), then).unwrap();
2494 borg::msg::ack(&a, "s1", &[&early.id], then + 1).unwrap();
2495 let go = std::sync::Barrier::new(8);
2496 std::thread::scope(|s| {
2497 for _ in 0..8 {
2498 s.spawn(|| {
2499 go.wait();
2500 alice.announce(V14_ALPN)
2501 });
2502 }
2503 });
2504 assert_eq!(announcements(&a, ours).len(), 1);
2505 alice.close().await;
2506 }
2507
2508 #[tokio::test(flavor = "multi_thread")]
2512 async fn a_member_on_a_newer_protocol_is_recorded_from_its_refusal() {
2513 let (pa, pb) = team("newer");
2514 let (a, b) = (home(&pa), home(&pb));
2515 let bob_key = Identity::load_or_create(&b).unwrap().public();
2516 let ours = String::from_utf8_lossy(ALPN).into_owned();
2517 let newer = format!("borg/mesh/{}", version(ALPN).unwrap() + 1);
2518 let alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
2519 let new_bob = Node::open(&pb, Some(LOCAL.parse().unwrap()), true, |x| x.alpns(vec![newer.clone().into_bytes(), ALPN.to_vec()])).await.unwrap();
2521 let told = borg::presence::Mismatch { theirs: ours.clone(), ours: newer.clone(), ms: 1 }.told(&bob_key);
2522 let line = format!("peer {} speaks {newer}, this node {ours}: nothing syncs with it; upgrade this node", &bob_key[..12]);
2523
2524 let refuse = async {
2526 let conn = new_bob.incoming().await?.await.ok()?;
2527 let agreed = conn.alpn().to_vec();
2528 conn.close(MISMATCH_CLOSE.into(), told.as_bytes());
2529 Some(agreed)
2530 };
2531 let ticket = new_bob.ticket();
2532 let (agreed, synced) = tokio::join!(refuse, alice.sync(&ticket));
2533 assert_eq!(agreed.as_deref(), Some(ALPN), "the handshake agreed on this node's protocol");
2534 assert_eq!(synced.unwrap_err(), line);
2535 let m = &borg::presence::mismatches(&a)[&bob_key];
2536 assert_eq!((m.theirs.as_str(), m.ours.as_str()), (newer.as_str(), ours.as_str()));
2537 let shown = a.join("shown");
2538 let n = borg::update::notice(None, &a, &shown).expect("the drone hears it");
2539 assert!(n.contains(&format!("Nothing syncs with peer {} ({newer}", &bob_key[..12])), "{n}");
2540
2541 let _ = std::fs::remove_file(borg::presence::protocols_path(&a));
2543 let addr = Target::parse(&alice.ticket()).unwrap().endpoint_addr(&new_bob.relay).unwrap();
2544 let dial_new = async {
2545 let opts = ConnectOptions::new().with_additional_alpns(vec![ALPN.to_vec()]);
2546 let conn = new_bob.endpoint.connect_with_opts(addr, newer.as_bytes(), opts).await.map_err(|e| e.to_string())?.await.map_err(|e| e.to_string())?;
2547 let agreed = conn.alpn().to_vec();
2548 conn.close(MISMATCH_CLOSE.into(), told.as_bytes());
2549 Ok::<Vec<u8>, String>(agreed)
2550 };
2551 let (served, dialled) = tokio::join!(alice.accept(), dial_new);
2552 assert_eq!(dialled.unwrap(), ALPN);
2553 assert_eq!(served.unwrap_err(), line);
2554 assert_eq!(borg::presence::mismatches(&a)[&bob_key].theirs, newer);
2555 assert!(borg::presence::Mismatch::from_told("error", 1).is_none());
2557 alice.close().await;
2558 new_bob.close().await;
2559 }
2560
2561 #[tokio::test(flavor = "multi_thread")]
2565 async fn a_member_on_an_older_protocol_hears_the_announcement_once() {
2566 let (pa, pb) = team("announce-sim");
2567 let (a, b) = (home(&pa), home(&pb));
2568 let (alice_key, bob_key) = (Identity::load_or_create(&a).unwrap().public(), Identity::load_or_create(&b).unwrap().public());
2569 let (ours, theirs) = (version(ALPN).unwrap(), version(V14_ALPN).unwrap());
2570 let alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
2571 let mut old_bob = Node::open(&pb, Some(LOCAL.parse().unwrap()), true, |x| x.alpns(vec![V14_ALPN.to_vec()])).await.unwrap();
2572 old_bob.speaks = Speaks { alpn: V14_ALPN, borg: None, tell: true };
2573
2574 let ticket = alice.ticket();
2576 let (served, dialled) = tokio::join!(alice.accept(), old_bob.dial(&ticket, &[], V14_ALPN, Vec::new()));
2577 served.unwrap();
2578 dialled.unwrap();
2579 let held = announcements(&a, ours);
2580 assert_eq!(held.len(), 1, "one announcement");
2581 assert!(held[0].text.contains(borg::protocol::note(ours).unwrap()), "{}", held[0].text);
2582 assert!(announcements(&b, ours).is_empty() && announcements(&b, theirs).is_empty(), "the old node announces nothing");
2583 let v = &borg::protocol::versions(&a)[&bob_key];
2584 assert_eq!((v.borg.as_deref(), v.protocol), (None, theirs), "no version: older than 0.7");
2585 let v = &borg::protocol::versions(&b)[&alice_key];
2586 assert_eq!((v.borg.as_deref(), v.protocol), (Some(env!("CARGO_PKG_VERSION")), theirs));
2587
2588 let old_ticket = old_bob.ticket();
2590 for _ in 0..2 {
2591 let (served, dialled) = tokio::join!(old_bob.accept(), alice.sync(&old_ticket));
2592 served.unwrap();
2593 dialled.unwrap();
2594 }
2595 assert_eq!(announcements(&a, ours), held, "still the one");
2596 assert_eq!(announcements(&b, ours), held, "the old node holds it, signed by this node");
2597 alice.close().await;
2598 old_bob.close().await;
2599 }
2600
2601 #[tokio::test(flavor = "multi_thread")]
2602 async fn an_unlisted_key_is_rejected_both_ways() {
2603 let (pa, pb) = team("unlisted");
2604 let a = home(&pa);
2605 write(&a, "lessons.jsonl", &[lesson("L1", 1, "secret sauce?")]);
2606 let pe = node("unlisted-eve", "team");
2608 let e = home(&pe);
2609 let eve = Identity::load_or_create(&e).unwrap();
2610 let mut forged = Membership::found("work", &eve);
2611 forged.invite(&eve, &Identity::load_or_create(&a).unwrap().public(), "member").unwrap();
2612 forged.save(&e).unwrap();
2613
2614 let alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
2615 let eve_node = Node::bind(&pe, LOCAL.parse().unwrap()).await.unwrap();
2616 let ticket = alice.ticket();
2617 let (served, synced) = tokio::join!(alice.accept(), eve_node.sync(&ticket));
2618 assert!(served.unwrap_err().contains("not a member"), "alice turns eve away");
2619 assert!(synced.is_err());
2620 assert!(!e.join("lessons.jsonl").exists(), "eve received nothing");
2621
2622 let bob = Node::bind(&pb, LOCAL.parse().unwrap()).await.unwrap();
2624 let err = bob.sync(&eve_node.ticket()).await.unwrap_err();
2625 assert!(err.contains("not a member") && err.contains("refusing to connect"), "{err}");
2626 for n in [alice, eve_node, bob] {
2627 n.close().await;
2628 }
2629 }
2630
2631 #[tokio::test(flavor = "multi_thread")]
2632 async fn a_local_collective_or_off_repo_never_syncs() {
2633 let (pa, _) = team("local");
2634 write(&home(&pa), "lessons.jsonl", &[lesson("L1", 1, "private thought?")]);
2635 configure(&pa, "local");
2636 assert!(!collective::may_sync(&pa.route()));
2637 let err = Node::bind(&pa, LOCAL.parse().unwrap()).await.err().expect("local must be refused");
2638 assert!(err.contains("is local") && err.contains("refusing to sync"), "{err}");
2639 let off = Place { dir: "/somewhere/else".into(), ..pa.clone() };
2640 let err = Node::bind(&off, LOCAL.parse().unwrap()).await.err().expect("off must be refused");
2641 assert!(err.contains("off"), "{err}");
2642 std::fs::write(&pa.config, "{ broken").unwrap();
2643 assert_eq!(pa.route(), Route::Off, "a broken config routes off");
2644 assert!(Node::bind(&pa, LOCAL.parse().unwrap()).await.is_err());
2645 let team = Route::Collective { name: "work".into(), home: home(&pa), share: collective::Share::Team };
2646 assert!(gate(&team).is_ok() && gate(&Route::Off).is_err());
2647 }
2648
2649 #[tokio::test(flavor = "multi_thread")]
2652 async fn serve_stops_syncing_when_the_collective_turns_local() {
2653 let (pa, pb) = team("turns-local");
2654 write(&home(&pa), "lessons.jsonl", &[lesson("L1", 1, "now private?")]);
2655 let alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
2656 let bob = Node::bind(&pb, LOCAL.parse().unwrap()).await.unwrap();
2657 let ticket = alice.ticket();
2658 configure(&pa, "local");
2659 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
2660 let err = served.unwrap_err();
2661 assert!(err.contains("is local") && err.contains("refusing to sync"), "{err}");
2662 assert!(synced.is_err());
2663 assert!(!home(&pb).join("lessons.jsonl").exists(), "bob received nothing");
2664 configure(&pb, "local");
2666 assert!(bob.sync(&ticket).await.unwrap_err().contains("is local"));
2667 alice.close().await;
2668 bob.close().await;
2669 }
2670
2671 #[test]
2674 fn a_store_shared_with_a_local_collective_is_refused() {
2675 let p = node("shared-store", "team");
2676 let cfg: Config = serde_json::from_value(json!({"collectives": [
2677 {"name": "work", "paths": [p.dir], "share": "team"},
2678 {"name": "work", "paths": ["/elsewhere"], "share": "local"}]}))
2679 .unwrap();
2680 let route = collective::route(Some(&cfg), &p.dir, &p.base);
2681 assert!(gate(&route).is_ok(), "the route alone looks shareable");
2682 assert_eq!(collective::local_owner_of(Some(&cfg), &home(&p), &p.base), Some("work"));
2683 std::fs::write(
2685 &p.config,
2686 serde_json::to_string(&json!({"collectives": [
2687 {"name": "work", "paths": [p.dir], "share": "team"},
2688 {"name": "work", "paths": ["/elsewhere"], "share": "local"}]}))
2689 .unwrap(),
2690 )
2691 .unwrap();
2692 assert!(p.gate().is_err());
2693 }
2694
2695 fn crew(tag: &str, names: &[&str]) -> Vec<Place> {
2697 let places: Vec<Place> = names.iter().map(|n| node(&format!("{tag}-{n}"), "team")).collect();
2698 let ids: Vec<Identity> = places.iter().map(|p| Identity::load_or_create(&home(p)).unwrap()).collect();
2699 let mut m = Membership::found("work", &ids[0]);
2700 for id in &ids[1..] {
2701 m.invite(&ids[0], &id.public(), "member").unwrap();
2702 }
2703 for p in &places {
2704 m.save(&home(p)).unwrap();
2705 }
2706 places
2707 }
2708
2709 fn append_lines(home: &Path, file: &str, lines: &[String]) {
2710 use std::io::Write;
2711 let mut f = std::fs::OpenOptions::new().create(true).append(true).open(home.join(file)).unwrap();
2712 for l in lines {
2713 writeln!(f, "{l}").unwrap();
2714 }
2715 }
2716
2717 fn holds(home: &Path, file: &str, line: &str) -> bool {
2718 lines::read(&home.join(file)).contains_key(&lines::hash(line))
2719 }
2720
2721 async fn eventually(what: &str, limit: std::time::Duration, f: impl Fn() -> bool) {
2722 let start = std::time::Instant::now();
2723 while !f() {
2724 assert!(start.elapsed() < limit, "timed out waiting for: {what}");
2725 tokio::time::sleep(std::time::Duration::from_millis(25)).await;
2726 }
2727 }
2728
2729 fn addr_of(n: &Node) -> String {
2730 n.ticket().split_once('@').unwrap().1.to_string()
2731 }
2732
2733 #[tokio::test(flavor = "multi_thread")]
2736 async fn serve_spreads_new_lessons_without_a_manual_sync() {
2737 use crate::daemon::{self, Event, Options};
2738 use std::sync::Arc;
2739 use std::time::Duration;
2740 let p = crew("daemon", &["alice", "bob"]);
2741 let (a, b) = (home(&p[0]), home(&p[1]));
2742 let alice = Arc::new(Node::bind(&p[0], LOCAL.parse().unwrap()).await.unwrap());
2743 let bob = Arc::new(Node::bind(&p[1], LOCAL.parse().unwrap()).await.unwrap());
2744 assert!(peers::remember(&b, &alice.public(), &addr_of(&alice)).unwrap());
2746 let opts = Options {
2747 interval: Duration::from_millis(300),
2748 poll: Duration::from_millis(50),
2749 backoff_max: Duration::from_millis(600),
2750 timeout: Duration::from_secs(5),
2751 grace: Duration::from_secs(1),
2752 };
2753 let synced = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2754 let mut stops = Vec::new();
2755 let mut loops = Vec::new();
2756 for n in [alice.clone(), bob.clone()] {
2757 let (tx, rx) = tokio::sync::oneshot::channel::<()>();
2758 stops.push(tx);
2759 let count = synced.clone();
2760 loops.push(tokio::spawn(daemon::serve(
2761 n,
2762 opts,
2763 async move {
2764 let _ = rx.await;
2765 },
2766 move |e| {
2767 if matches!(e, Event::Synced { result: Ok(_), .. }) {
2768 count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2769 }
2770 },
2771 )));
2772 }
2773 let l1 = lesson("L1", 1, "who owns the lock?");
2774 append_lines(&a, "lessons.jsonl", std::slice::from_ref(&l1));
2775 eventually("alice's lesson at bob", Duration::from_secs(10), || holds(&b, "lessons.jsonl", &l1)).await;
2776 let l2 = lesson("L2", 1, "is the retry idempotent?");
2777 append_lines(&b, "lessons.jsonl", std::slice::from_ref(&l2));
2778 eventually("bob's lesson at alice", Duration::from_secs(10), || holds(&a, "lessons.jsonl", &l2)).await;
2779 assert_eq!(peers::load(&a).get(&bob.public()), Some(&addr_of(&bob)), "alice learned bob's address from his hello");
2780 eventually("a successful sync round", Duration::from_secs(10), || synced.load(std::sync::atomic::Ordering::Relaxed) > 0).await;
2783
2784 configure(&p[0], "local");
2786 let l3 = lesson("L3", 1, "private now?");
2787 append_lines(&a, "lessons.jsonl", std::slice::from_ref(&l3));
2788 tokio::time::sleep(Duration::from_millis(1500)).await;
2789 assert!(!holds(&b, "lessons.jsonl", &l3), "a local collective's lesson never leaves");
2790
2791 for s in stops {
2792 let _ = s.send(());
2793 }
2794 for l in loops {
2795 tokio::time::timeout(Duration::from_secs(5), l).await.expect("serve stops within its grace").unwrap();
2796 }
2797 }
2798
2799 struct Serving {
2801 stops: Vec<tokio::sync::oneshot::Sender<()>>,
2802 loops: Vec<tokio::task::JoinHandle<()>>,
2803 failed: std::sync::Arc<std::sync::Mutex<Vec<String>>>,
2804 }
2805
2806 fn serving(nodes: Vec<(&'static str, std::sync::Arc<Node>)>, opts: crate::daemon::Options) -> Serving {
2807 let s = Serving { stops: Vec::new(), loops: Vec::new(), failed: Default::default() };
2808 nodes.into_iter().fold(s, |mut s, (who, n)| {
2809 s.add(who, n, opts);
2810 s
2811 })
2812 }
2813
2814 impl Serving {
2815 fn add(&mut self, who: &'static str, n: std::sync::Arc<Node>, opts: crate::daemon::Options) {
2816 use crate::daemon::{self, Event};
2817 let (tx, rx) = tokio::sync::oneshot::channel::<()>();
2818 self.stops.push(tx);
2819 let failed = self.failed.clone();
2820 self.loops.push(tokio::spawn(daemon::serve(
2821 n,
2822 opts,
2823 async move {
2824 let _ = rx.await;
2825 },
2826 move |e| {
2827 if matches!(e, Event::Served(Err(_)) | Event::Synced { result: Err(_), .. } | Event::Gate(Err(_))) {
2828 failed.lock().unwrap().push(format!("{who} at {}: {e:?}", borg::learn::now_ms()));
2829 }
2830 },
2831 )));
2832 }
2833
2834 async fn stop(self) {
2835 for s in self.stops {
2836 let _ = s.send(());
2837 }
2838 for l in self.loops {
2839 tokio::time::timeout(std::time::Duration::from_secs(10), l).await.expect("serve stops within its grace").unwrap();
2840 }
2841 }
2842 }
2843
2844 #[tokio::test(flavor = "multi_thread")]
2850 async fn a_peer_that_does_not_answer_holds_no_line_back() {
2851 use std::sync::Arc;
2852 use std::time::Duration;
2853 use tokio::sync::oneshot;
2854 let p = crew("deaf", &["alice", "bob", "carol"]);
2855 let (a, b) = (home(&p[0]), home(&p[1]));
2856 let alice = Arc::new(Node::bind(&p[0], LOCAL.parse().unwrap()).await.unwrap());
2857 let bob = Arc::new(Node::bind(&p[1], LOCAL.parse().unwrap()).await.unwrap());
2858 let carol = Node::bind(&p[2], LOCAL.parse().unwrap()).await.unwrap();
2859 let key = carol.public();
2860 assert!(peers::remember(&a, &key, &addr_of(&carol)).unwrap());
2861 assert!(peers::remember(&a, &bob.public(), &addr_of(&bob)).unwrap());
2862 borg::presence::note_synced(&a, &key, borg::learn::now_ms()).unwrap();
2863 let (held_tx, held) = oneshot::channel::<()>();
2864 let (release, release_rx) = oneshot::channel::<()>();
2865 let deaf = tokio::spawn(async move {
2866 let conn = carol.incoming().await.expect("alice dials carol").await.expect("handshake");
2867 let (_tx, mut rx) = conn.accept_bi().await.expect("alice's stream");
2868 assert!(matches!(recv(&mut rx).await, Ok(Msg::Hello { .. })), "alice's hello");
2869 let _ = held_tx.send(());
2870 let _ = release_rx.await;
2871 let open = conn.close_reason().is_none();
2872 conn.close(1u32.into(), b"deaf");
2873 carol.close().await;
2874 open
2875 });
2876 let opts =
2877 crate::daemon::Options { poll: Duration::from_millis(100), timeout: Duration::from_secs(600), grace: Duration::from_secs(1), ..Default::default() };
2878 let serve = serving(vec![("alice", alice.clone()), ("bob", bob.clone())], opts);
2879 let limit = Duration::from_secs(60); tokio::time::timeout(limit, held).await.expect("carol holds alice's dial").unwrap();
2881
2882 let l = rollup("n1", 1, 1);
2883 append_lines(&a, "stats.jsonl", std::slice::from_ref(&l));
2884 eventually("alice's line at bob", limit, || holds(&b, "stats.jsonl", &l)).await;
2885 assert!(serve.failed.lock().unwrap().iter().all(|f| !f.contains(&key)), "carol's dial has not ended");
2886 release.send(()).unwrap();
2887 assert!(deaf.await.unwrap(), "alice's dial to carol was still open when bob got the line");
2888
2889 eventually("alice's dial to carol failed", limit, || serve.failed.lock().unwrap().iter().any(|f| f.contains(&key))).await;
2890 eventually("carol's record dropped", limit, || !borg::presence::synced(&a).contains_key(&key)).await;
2891 serve.stop().await;
2892 }
2893
2894 #[tokio::test(flavor = "multi_thread")]
2897 async fn a_failed_dial_is_retried_soon_with_what_it_missed() {
2898 use std::sync::Arc;
2899 use std::time::Duration;
2900 let p = crew("retry", &["alice", "bob"]);
2901 let (a, b) = (home(&p[0]), home(&p[1]));
2902 let alice = Arc::new(Node::bind(&p[0], LOCAL.parse().unwrap()).await.unwrap());
2903 let bob = Arc::new(Node::bind(&p[1], LOCAL.parse().unwrap()).await.unwrap());
2904 assert!(peers::remember(&a, &bob.public(), &addr_of(&bob)).unwrap());
2905 let l1 = lesson("L1", 1, "who owns the lock?");
2906 append_lines(&a, "lessons.jsonl", std::slice::from_ref(&l1));
2907 let opts = crate::daemon::Options { timeout: Duration::from_secs(1), grace: Duration::from_secs(1), ..Default::default() };
2908 let mut serve = serving(vec![("alice", alice.clone())], opts);
2909 eventually("alice's dial to bob failed", Duration::from_secs(10), || !serve.failed.lock().unwrap().is_empty()).await;
2910 serve.add("bob", bob.clone(), opts);
2911 eventually("alice's lesson at bob", Duration::from_secs(10), || holds(&b, "lessons.jsonl", &l1)).await;
2912 serve.stop().await;
2913 }
2914
2915 #[test]
2918 fn legacy_goals_count_the_valid_distinct_ids() {
2919 let ids = ["g1", "../evil", "A B", "g0", "g1", "/abs", ""].map(String::from).to_vec();
2920 assert_eq!(legacy_goals(ids).into_iter().collect::<Vec<_>>(), ["g0", "g1"], "sorted, as the peer walks them");
2921 assert!(legacy_goals(vec!["x".repeat(41)]).is_empty());
2922 }
2923
2924 const OLD_SYNCED: [&str; 5] = ["lessons.jsonl", "stats.jsonl", "outcomes-team.jsonl", "playbooks.jsonl", "constraints.jsonl"];
2926
2927 struct OldGoal {
2929 logs: Vec<BTreeMap<Hash, String>>,
2930 origins: HashMap<Hash, Origin>,
2931 }
2932
2933 impl OldGoal {
2934 fn new(me: &Identity) -> OldGoal {
2935 let mut g = OldGoal { logs: vec![BTreeMap::new(); LEGACY_GOAL_LOGS], origins: HashMap::new() };
2936 for log in 0..LEGACY_GOAL_LOGS {
2937 g.add(me, log, &json!({"log": log, "n": 0}).to_string());
2938 }
2939 g
2940 }
2941
2942 fn add(&mut self, me: &Identity, log: usize, line: &str) {
2943 let h = lines::hash(line);
2944 self.origins.insert(h, Origin::new(me, &h, 1));
2945 self.logs[log].insert(h, line.into());
2946 }
2947 }
2948
2949 async fn send_raw(tx: &mut SendStream, v: &serde_json::Value) -> Result<(), String> {
2950 let body = serde_json::to_vec(v).map_err(|e| e.to_string())?;
2951 tx.write_all(&(body.len() as u32).to_be_bytes()).await.map_err(|e| e.to_string())?;
2952 tx.write_all(&body).await.map_err(|e| e.to_string())
2953 }
2954
2955 async fn old_exchange(n: &Node, dialler: bool, tx: &mut SendStream, rx: &mut RecvStream, goal: &OldGoal) -> Result<(usize, usize), String> {
2960 let home = n.home.as_path();
2961 let mine = members(home)?;
2962 let Msg::Membership { .. } = swap(dialler, tx, rx, Msg::Membership { docs: Membership::history(home) }).await? else { return Err("membership".into()) };
2963 n.stamp()?;
2964 let (origins, mut report) = (origin::load(home), Report::default());
2965 for file in OLD_SYNCED {
2966 n.reconcile(dialler, tx, rx, &home.join(file), &origins, &mine, false, &mut report).await?;
2967 }
2968 let ids = ["g1", "g1", "Bad Id"].map(String::from).to_vec();
2969 let Msg::Goals { ids: theirs } = swap(dialler, tx, rx, Msg::Goals { ids }).await? else { return Err("goals".into()) };
2970 assert!(theirs.is_empty(), "a new node names no goal");
2971 let mut gave = 0;
2972 for set in &goal.logs {
2973 let mut r = Recon::new(set);
2974 loop {
2975 let m = r.message();
2976 let Msg::Round { round } = swap(dialler, tx, rx, Msg::Round { round: m.clone() }).await? else { return Err("round".into()) };
2977 if r.absorb(&m, round)? {
2978 break;
2979 }
2980 }
2981 let give = within_budget(signed(set, &goal.origins, r.give.iter()));
2982 gave += give.len();
2983 let Msg::Lines { lines } = swap(dialler, tx, rx, Msg::Lines { lines: give }).await? else { return Err("lines".into()) };
2984 assert!(lines.is_empty(), "a new node gives no hive line");
2985 }
2986 for m in [
2987 Msg::Offer { hashes: vec![] },
2988 Msg::Want { hashes: vec![] },
2989 Msg::Lines { lines: vec![] },
2990 Msg::Ask { commits: vec![] },
2991 Msg::Lines { lines: vec![] },
2992 ] {
2993 swap(dialler, tx, rx, m).await?;
2994 }
2995 let now = borg::learn::now_ms();
2996 let directive = json!({"id": "d1", "collective": "work", "key": n.me.public(), "engineer": "old@x", "target": {"node": "", "session": ""},
2997 "kind": "pause", "text": "", "ms": now, "sig": "00"});
2998 let live = json!({"t": "live", "presence": live::outgoing(home, &n.me, now), "directives": [directive],
2999 "beats": [{"goal": "g1", "line": "{}", "key": n.me.public(), "sig": "00"}]});
3000 let theirs = if dialler {
3001 send_raw(tx, &live).await?;
3002 recv(rx).await?
3003 } else {
3004 let theirs = recv(rx).await?;
3005 send_raw(tx, &live).await?;
3006 theirs
3007 };
3008 let Msg::Live { presence, .. } = theirs else { return Err("live".into()) };
3009 Ok((gave, live::receive(home, &mine, &n.me.public(), presence, now)?))
3010 }
3011
3012 async fn old_dial(n: &Node, target: &str, goal: &OldGoal) -> Result<(usize, usize), String> {
3013 let t = Target::parse(target)?;
3014 let conn = n.endpoint.connect(t.endpoint_addr(&n.relay)?, LEGACY_ALPN).await.map_err(|e| e.to_string())?;
3015 let (mut tx, mut rx) = conn.open_bi().await.map_err(|e| e.to_string())?;
3016 swap(true, &mut tx, &mut rx, Msg::Hello { collective: "work".into(), listen: None, relay: None, borg: None }).await?;
3017 let got = old_exchange(n, true, &mut tx, &mut rx, goal).await;
3018 let _ = tx.finish();
3019 conn.close(0u32.into(), b"done");
3020 got
3021 }
3022
3023 async fn old_serve(n: &Node, goal: &OldGoal) -> Result<(usize, usize), String> {
3024 let conn = n.incoming().await.ok_or("closed")?.await.map_err(|e| e.to_string())?;
3025 let (mut tx, mut rx) = conn.accept_bi().await.map_err(|e| e.to_string())?;
3026 let Msg::Hello { .. } = recv(&mut rx).await? else { return Err("hello".into()) };
3027 send(&mut tx, &Msg::Hello { collective: "work".into(), listen: None, relay: None, borg: None }).await?;
3028 let got = old_exchange(n, false, &mut tx, &mut rx, goal).await;
3029 let _ = tx.finish();
3030 conn.closed().await;
3031 got
3032 }
3033
3034 #[tokio::test(flavor = "multi_thread")]
3039 async fn a_0_3_0_peer_syncs_in_both_roles_and_its_hive_lines_come_once() {
3040 let (pa, pb) = team("legacy");
3041 let (a, b) = (home(&pa), home(&pb));
3042 let (alice_id, bob_id) = (Identity::load_or_create(&a).unwrap(), Identity::load_or_create(&b).unwrap());
3043 let mut m = Membership::load(&a).unwrap().unwrap();
3044 m.invite(&alice_id, &bob_id.public(), "steer").unwrap();
3045 m.save(&a).unwrap();
3046 m.save(&b).unwrap();
3047 let (l1, l2) = (lesson("L1", 1, "lock order?"), lesson("L2", 1, "retry budget?"));
3048 write(&a, "lessons.jsonl", std::slice::from_ref(&l1));
3049 write(&b, "lessons.jsonl", std::slice::from_ref(&l2));
3050 let edit = json!({"pb": "p1", "edit": "e1", "step": "s1", "text": "run the parser tests", "ms": 1}).to_string();
3051 write(&a, PLAYBOOKS, std::slice::from_ref(&edit));
3052 origin::append(&a, &[Origin::new(&alice_id, &lines::hash(&edit), 1)]).unwrap();
3053 let now = borg::learn::now_ms();
3054 for (h, p, who) in [(&a, &pa, "alice"), (&b, &pb, "bob")] {
3055 let ev = json!({"session_id": format!("{who}-s1"), "cwd": p.dir, "prompt": format!("{who} works")});
3056 borg::presence::write(h, &borg::presence::update(None, "work", "claude", "prompt", &ev, now, |_| (format!("{who}@x"), "repo".into()))).unwrap();
3057 }
3058 let tell = message(&b, &bob_id.public(), ALL_TO, now, "the new node's note");
3059 let old = Node::open(&pa, Some(LOCAL.parse().unwrap()), true, |x| x.alpns(vec![LEGACY_ALPN.to_vec()])).await.unwrap();
3060 let new = Node::bind(&pb, LOCAL.parse().unwrap()).await.unwrap();
3061 let (old_at, new_at) = (old.ticket(), new.ticket());
3062 let mut goal = OldGoal::new(&alice_id);
3063
3064 let (old_got, new_got) = tokio::join!(old_serve(&old, &goal), new.sync(&old_at));
3066 let (new_got, (gave, presence)) = (new_got.unwrap(), old_got.unwrap());
3067 assert_eq!((gave, presence, new_got.presence), (LEGACY_GOAL_LOGS, 1, 1));
3068 assert!(holds(&a, "lessons.jsonl", &l2) && holds(&b, "lessons.jsonl", &l1), "lessons both ways");
3069
3070 let (old_got, new_got) = tokio::join!(old_dial(&old, &new_at, &goal), new.accept());
3073 let (new_got, (gave, _)) = (new_got.unwrap(), old_got.unwrap());
3074 assert_eq!((gave, new_got.steered, new_got.dropped), (0, 0, 0));
3075 assert!(!borg::steer::active(&b), "a 0.3.0 directive counts nowhere");
3076
3077 goal.add(&alice_id, 5, "{\"bundle\":\"new\"}");
3079 let (old_got, new_got) = tokio::join!(old_serve(&old, &goal), new.sync(&old_at));
3080 new_got.unwrap();
3081 assert_eq!(old_got.unwrap().0, 1, "what was given before is not sent again");
3082 assert!(!b.join("hive").exists(), "no hive file on the new node");
3083 assert!(!b.join(PLAYBOOKS).exists(), "the old node's playbook edit is followed, not kept");
3084 assert!(holds(&b, borg::msg::FILE, &tell) && !a.join(borg::msg::FILE).exists(), "messages stay off a 0.3.0 session");
3085 old.close().await;
3086 new.close().await;
3087 }
3088
3089 #[tokio::test(flavor = "multi_thread")]
3093 async fn a_revocation_spreads_and_the_revoked_node_is_refused() {
3094 let p = crew("revoke", &["alice", "bob", "carol"]);
3095 let (a, b, c) = (home(&p[0]), home(&p[1]), home(&p[2]));
3096 let alice = Node::bind(&p[0], LOCAL.parse().unwrap()).await.unwrap();
3097 let bob = Node::bind(&p[1], LOCAL.parse().unwrap()).await.unwrap();
3098 let carol = Node::bind(&p[2], LOCAL.parse().unwrap()).await.unwrap();
3099 let (ta, tb, tc) = (alice.ticket(), bob.ticket(), carol.ticket());
3100
3101 let early = lesson("Lc1", 1, "carol's useful lesson?");
3102 write(&c, "lessons.jsonl", std::slice::from_ref(&early));
3103 let (s, r) = tokio::join!(bob.accept(), carol.sync(&tb));
3104 assert_eq!((s.unwrap().received, r.unwrap().sent), (1, 1));
3105 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
3106
3107 let mut m = Membership::load(&a).unwrap().unwrap();
3108 m.revoke(&Identity::load_or_create(&a).unwrap(), &carol.public(), borg::learn::now_ms()).unwrap();
3109 m.save(&a).unwrap();
3110 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
3111
3112 let late = lesson("Lc2", 1, "carol's lesson after her revocation?");
3113 append_lines(&c, "lessons.jsonl", std::slice::from_ref(&late));
3114 let (s, r) = tokio::join!(bob.accept(), carol.sync(&tb));
3115 assert_eq!((s.unwrap().received, r.unwrap().sent), (1, 1), "bob has not heard of the revocation yet");
3116 assert!(holds(&b, "lessons.jsonl", &late));
3117
3118 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ta));
3119 let (served, synced) = (served.unwrap(), synced.unwrap());
3120 assert_eq!(Membership::load(&b).unwrap().unwrap().version, m.version, "bob learned the revocation from alice");
3121 assert_eq!(synced.quarantined, 1, "carol's late lesson is set aside before any data moves");
3122 assert!(!holds(&b, "lessons.jsonl", &late) && holds(&b.join("mesh"), "quarantine.jsonl", &late));
3123 assert_eq!(served.received, 1, "alice gets carol's earlier lesson only");
3124 assert!(holds(&a, "lessons.jsonl", &early) && !holds(&a, "lessons.jsonl", &late));
3125
3126 let (served, synced) = tokio::join!(bob.accept(), carol.sync(&tb));
3127 assert!(served.unwrap_err().contains("not a member"), "bob turns carol away");
3128 assert!(synced.is_err());
3129 let err = bob.sync(&tc).await.unwrap_err();
3130 assert!(err.contains("not a member") && err.contains("refusing to connect"), "{err}");
3131 for n in [alice, bob, carol] {
3132 n.close().await;
3133 }
3134 }
3135
3136 #[tokio::test(flavor = "multi_thread")]
3139 async fn catch_up_transfers_only_the_difference() {
3140 let (pa, pb) = team("catch-up");
3141 let (a, b) = (home(&pa), home(&pb));
3142 let common: Vec<String> = (0..2000).map(|i| lesson(&format!("L{i}"), 1, &format!("question {i}?"))).collect();
3143 write(&a, "lessons.jsonl", &common);
3144 write(&b, "lessons.jsonl", &common);
3145 append_lines(&a, "lessons.jsonl", &(0..3).map(|i| lesson(&format!("A{i}"), 1, "alice offline?")).collect::<Vec<_>>());
3146 append_lines(&b, "lessons.jsonl", &(0..2).map(|i| lesson(&format!("B{i}"), 1, "bob offline?")).collect::<Vec<_>>());
3147 let alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
3148 let bob = Node::bind(&pb, LOCAL.parse().unwrap()).await.unwrap();
3149 let ticket = alice.ticket();
3150 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
3151 let (served, synced) = (served.unwrap(), synced.unwrap());
3152 assert_eq!((synced.received, synced.sent, served.received, served.sent), (3, 2, 2, 3));
3153 let hashes = synced.hashes + served.hashes;
3154 let fps = synced.fps + served.fps;
3155 eprintln!("catch-up: 5 of 2005 lines moved, {hashes} hashes and {fps} fingerprints in {} rounds", synced.rounds);
3156 assert!(hashes <= 2 * crate::recon::LEAF * 5, "hashes ∝ difference, not history: {hashes}");
3157 assert!(fps <= 2 * (1 + 16 * 5 * synced.rounds), "{fps}");
3158 assert_eq!(lines::read(&a.join("lessons.jsonl")), lines::read(&b.join("lessons.jsonl")));
3159 alice.close().await;
3160 bob.close().await;
3161 }
3162
3163 #[tokio::test(flavor = "multi_thread")]
3166 async fn reasoning_goes_to_two_replicas() {
3167 let p = crew("replicas", &["alice", "bob", "carol", "dave"]);
3168 let a = home(&p[0]);
3169 let excerpt = json!({"commit": "abc", "task": "fix the lock", "proposal": "take it in order"}).to_string();
3170 write(&a, REASONING, std::slice::from_ref(&excerpt));
3171 let alice = Node::bind(&p[0], LOCAL.parse().unwrap()).await.unwrap();
3172 let m = Membership::load(&a).unwrap().unwrap();
3173 let chosen = replicas(&lines::hash(&excerpt), &m, &alice.public());
3174 assert_eq!(chosen.len(), 2);
3175 let mut holders = Vec::new();
3176 for q in &p[1..] {
3177 let n = Node::bind(q, LOCAL.parse().unwrap()).await.unwrap();
3178 let ticket = alice.ticket();
3179 let (served, synced) = tokio::join!(alice.accept(), n.sync(&ticket));
3180 served.unwrap();
3181 synced.unwrap();
3182 if holds(&home(q), REASONING, &excerpt) {
3183 holders.push(n.public());
3184 }
3185 n.close().await;
3186 }
3187 holders.sort();
3188 let mut want = chosen.clone();
3189 want.sort();
3190 assert_eq!(holders, want, "exactly the two replicas hold it");
3191 alice.close().await;
3192 }
3193
3194 #[tokio::test(flavor = "multi_thread")]
3197 async fn a_node_bound_to_any_address_is_dialled_back_where_it_was_seen() {
3198 let (pa, pb) = team("any-addr");
3199 let alice = Node::bind(&pa, "0.0.0.0:0".parse().unwrap()).await.unwrap();
3200 let bob = Node::bind(&pb, "0.0.0.0:0".parse().unwrap()).await.unwrap();
3201 let (pa_port, pb_port) = (alice.addr().unwrap().port(), bob.addr().unwrap().port());
3202 for t in alice.tickets() {
3203 assert!(!t.contains("@0.0.0.0:") && t.ends_with(&format!(":{pa_port}")), "{t}");
3204 }
3205 let ticket = format!("{}@127.0.0.1:{pa_port}", alice.public());
3206 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
3207 served.unwrap();
3208 synced.unwrap();
3209 assert_eq!(peers::load(&home(&pa)).get(&bob.public()), Some(&format!("127.0.0.1:{pb_port}")), "bob is remembered where alice saw him");
3210 let known = peers::load(&home(&pa));
3212 let back = format!("{}@{}", bob.public(), known[&bob.public()]);
3213 let (served, synced) = tokio::join!(bob.accept(), alice.sync(&back));
3214 served.unwrap();
3215 synced.unwrap();
3216 let quiet = Node::bind(&pb, "0.0.0.0:0".parse().unwrap()).await.unwrap().quiet();
3218 assert_eq!(quiet.listen(), None);
3219 assert_eq!(reachable("0.0.0.0:7878", Some("10.0.0.5:51000".parse().unwrap())), Some("10.0.0.5:7878".parse().unwrap()));
3220 assert_eq!(reachable("0.0.0.0:7878", None), None);
3221 for n in [alice, bob, quiet] {
3222 n.close().await;
3223 }
3224 }
3225
3226 #[tokio::test(flavor = "multi_thread")]
3231 async fn join_and_enforce_quarantine_what_the_new_membership_refuses() {
3232 let p = crew("join-q", &["alice", "bob", "carol", "dave"]);
3233 let (a, b, c, d) = (home(&p[0]), home(&p[1]), home(&p[2]), home(&p[3]));
3234 let bob = Node::bind(&p[1], LOCAL.parse().unwrap()).await.unwrap();
3235 let carol = Node::bind(&p[2], LOCAL.parse().unwrap()).await.unwrap();
3236 let dave = Node::bind(&p[3], LOCAL.parse().unwrap()).await.unwrap();
3237 let early = lesson("Lc1", 1, "carol's useful lesson?");
3238 write(&c, "lessons.jsonl", std::slice::from_ref(&early));
3239 carol.stamp().unwrap();
3240 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
3241 let (m, _) = origin::revoke(&a, &Identity::load_or_create(&a).unwrap(), &carol.public(), borg::learn::now_ms()).unwrap();
3242 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
3243 let late = lesson("Lc2", 1, "carol's lesson after her revocation?");
3244 append_lines(&c, "lessons.jsonl", std::slice::from_ref(&late));
3245 for (node, h) in [(&bob, &b), (&dave, &d)] {
3246 let t = node.ticket();
3247 let (s, r) = tokio::join!(node.accept(), carol.sync(&t));
3248 s.unwrap();
3249 r.unwrap();
3250 assert!(holds(h, "lessons.jsonl", &late) && holds(h, "lessons.jsonl", &early));
3251 }
3252
3253 let handed = std::fs::read_to_string(crate::membership::path(&a)).unwrap();
3254 let (got, moved) = origin::join(&b, "work", &crate::membership::parse_docs(&handed).unwrap()).unwrap();
3255 assert_eq!((got.version, moved), (m.version, 1));
3256 assert!(!holds(&b, "lessons.jsonl", &late) && holds(&b, "lessons.jsonl", &early), "quarantined at join");
3257 assert!(origin::join(&b, "work", &crate::membership::parse_docs(&handed).unwrap()).is_err(), "nothing newer the second time");
3258
3259 m.save(&d).unwrap(); assert_eq!(dave.enforce().unwrap(), 1);
3261 assert_eq!(dave.enforce().unwrap(), 0, "once per membership");
3262 assert!(!holds(&d, "lessons.jsonl", &late));
3263 for n in [bob, carol, dave] {
3264 n.close().await;
3265 }
3266 }
3267
3268 fn git(repo: &Path, args: &[&str]) -> String {
3269 let o = std::process::Command::new("git")
3270 .arg("-C")
3271 .arg(repo)
3272 .args(args)
3273 .env("GIT_CONFIG_GLOBAL", "/dev/null")
3274 .env("GIT_CONFIG_NOSYSTEM", "1")
3275 .output()
3276 .unwrap();
3277 assert!(o.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&o.stderr));
3278 String::from_utf8_lossy(&o.stdout).trim().to_string()
3279 }
3280
3281 #[tokio::test(flavor = "multi_thread")]
3284 async fn reasoning_is_fetched_from_a_replica_while_its_author_is_offline() {
3285 let p = crew("fetch", &["alice", "bob", "carol", "dave"]);
3286 let a = home(&p[0]);
3287 let repo = p[3].base.join("work");
3288 std::fs::create_dir_all(&repo).unwrap();
3289 git(&repo, &["init", "-q"]);
3290 git(&repo, &["-c", "user.email=ann@example.com", "-c", "user.name=Ann", "commit", "-q", "--allow-empty", "-m", "take the lock first"]);
3291 let commit = git(&repo, &["rev-parse", "HEAD"]);
3292 let excerpt = json!({"kind": "reasoning", "commit": commit, "repo": "work", "session": "s-9", "task": "fix the race",
3293 "proposal": "take the lock before reading the flag", "episodes": [], "ms": 1})
3294 .to_string();
3295 write(&a, REASONING, std::slice::from_ref(&excerpt));
3296 let alice = Node::bind(&p[0], LOCAL.parse().unwrap()).await.unwrap();
3297 let chosen = replicas(&lines::hash(&excerpt), &Membership::load(&a).unwrap().unwrap(), &alice.public());
3298 let mut others = Vec::new();
3299 for q in &p[1..] {
3300 let n = Node::bind(q, LOCAL.parse().unwrap()).await.unwrap();
3301 let t = alice.ticket();
3302 let (served, synced) = tokio::join!(alice.accept(), n.sync(&t));
3303 served.unwrap();
3304 synced.unwrap();
3305 others.push((q.clone(), n));
3306 }
3307 alice.close().await; let (holders, askers): (Vec<_>, Vec<_>) = others.into_iter().partition(|(_, n)| chosen.contains(&n.public()));
3309 let (asker_place, asker) = askers.into_iter().next().expect("one member is no replica");
3310 let holder = &holders[0].1;
3311 assert!(!holds(&home(&asker_place), REASONING, &excerpt));
3312
3313 let short = commit[..10].to_string();
3314 let t = holder.ticket();
3315 let (served, fetched) = tokio::join!(holder.accept(), asker.fetch(&t, std::slice::from_ref(&short)));
3316 assert_eq!(served.unwrap().answered, 1, "a replica answers for an excerpt it did not write");
3317 assert_eq!(fetched.unwrap().received, 1);
3318 let written = borg::notes::restore(&repo, &home(&asker_place), std::slice::from_ref(&commit)).unwrap();
3319 assert_eq!(written, vec![commit.clone()]);
3320 assert!(git(&repo, &["notes", "--ref=refs/notes/borg", "show", &commit]).contains("take the lock before reading the flag"));
3321
3322 let (served, synced) = tokio::join!(holder.accept(), asker.sync(&t));
3324 assert_eq!((served.unwrap().answered, synced.unwrap().received), (0, 0));
3325 asker.close().await;
3326 for (_, n) in holders {
3327 n.close().await;
3328 }
3329 }
3330
3331 fn steering(node: &str, kind: borg::msg::Kind, to: &str, ms: i64, text: &str) -> borg::msg::Message {
3333 let from = borg::msg::Sender { by: "engineer:bob@x".into(), name: "bob@x".into(), harness: "terminal".into(), node: node.into() };
3334 borg::msg::Message::new(kind, &from, to, None, ms, text)
3335 }
3336
3337 #[tokio::test(flavor = "multi_thread")]
3342 async fn presence_and_steering_travel_between_members() {
3343 use borg::msg::Kind;
3344 let (pa, pb) = team("live");
3345 let (a, b) = (home(&pa), home(&pb));
3346 let (alice_id, bob_id) = (Identity::load_or_create(&a).unwrap(), Identity::load_or_create(&b).unwrap());
3347 let (alice_key, bob_key) = (alice_id.public(), bob_id.public());
3348 let mut m = Membership::load(&a).unwrap().unwrap();
3349 m.invite(&alice_id, &bob_key, "steer").unwrap();
3350 m.save(&a).unwrap(); let observer = Identity::load_or_create(&a.join("observer")).unwrap();
3352 m.invite(&alice_id, &observer.public(), "observe").unwrap();
3353 m.save(&a).unwrap(); let now = borg::learn::now_ms();
3355 let ev = json!({"session_id": "alice-s1", "cwd": pa.dir, "prompt": "refactor the recon loop"});
3356 borg::presence::write(&a, &borg::presence::update(None, "work", "claude", "prompt", &ev, now, |_| ("alice@x".into(), "repo".into()))).unwrap();
3357
3358 let alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
3359 let bob = Node::bind(&pb, LOCAL.parse().unwrap()).await.unwrap();
3360 let ticket = alice.ticket();
3361 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
3362 served.unwrap();
3363 assert_eq!(synced.unwrap().presence, 1);
3364 let ds = borg::top::drones(&b, true, now);
3365 assert_eq!((ds.len(), ds[0].node.as_str(), ds[0].p.prompt.as_str()), (1, alice_key.as_str(), "refactor the recon loop"));
3366 let lessons = std::fs::read_to_string(b.join("lessons.jsonl")).unwrap_or_default();
3367 assert!(!lessons.contains("recon loop"), "never in lessons");
3368
3369 let to = format!("alice-s1@{alice_key}");
3370 let pre = json!({"session_id": "alice-s1", "cwd": pa.dir, "tool_name": "Bash"});
3371 let refused = |at: i64| borg::steer::around(&a, "pre_tool", &pre, Some(now), at, || None);
3372 let round = || async {
3373 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
3374 synced.unwrap();
3375 served.unwrap()
3376 };
3377 borg::msg::send(&b, steering(&bob_key, Kind::Pause, &to, now, ""), now).unwrap();
3378 let served = round().await;
3379 assert_eq!((served.steered, served.dropped), (1, 0));
3380 let deny = refused(now + 1).expect("paused");
3381 let why = deny["hookSpecificOutput"]["permissionDecisionReason"].as_str().unwrap();
3382 assert!(why.contains(&format!("paused by bob@x (node {}, role steer)", &bob_key[..12])), "{why}");
3383
3384 let by_observer = serde_json::to_string(&steering(&observer.public(), Kind::Pause, ALL_TO, now, "")).unwrap();
3386 let h = lines::hash(&by_observer);
3387 let mut report = Report::default();
3388 let got = vec![Signed { origin: Origin::new(&observer, &h, now), line: by_observer.clone() }];
3389 alice.store_lines(&a.join(borg::msg::FILE), got, &BTreeSet::from([h]), &members(&a).unwrap(), None, &mut report).unwrap();
3390 assert_eq!((report.received, report.steered, report.dropped), (1, 0, 1));
3391 let log = std::fs::read_to_string(borg::steer::dropped_path(&a)).unwrap();
3392 assert!(log.contains("role observe may not steer"), "{log}");
3393
3394 borg::msg::send(&b, steering(&bob_key, Kind::Resume, &to, now + 2, ""), now + 2).unwrap();
3396 assert_eq!(round().await.steered, 1);
3397 assert_eq!(refused(now + 3), None);
3398
3399 borg::msg::send(&b, steering(&bob_key, Kind::Pause, &to, now + 4, ""), now + 4).unwrap();
3401 assert_eq!(round().await.steered, 1);
3402 assert!(refused(now + 5).is_some(), "paused again");
3403 let mut m = Membership::load(&a).unwrap().unwrap();
3404 m.revoke(&alice_id, &bob_key, now + 5).unwrap();
3405 m.save(&a).unwrap();
3406 alice.enforce().unwrap();
3407 assert!(borg::steer::read_admitted(&a).iter().all(|x| x.node != bob_key), "the revoked node's admissions are gone");
3408 assert_eq!(refused(now + 6), None, "and the pause no longer holds");
3409 alice.close().await;
3410 bob.close().await;
3411 }
3412
3413 #[tokio::test(flavor = "multi_thread")]
3417 async fn a_veto_on_one_node_reaches_the_drones_node_once() {
3418 let (pa, pb) = team("veto");
3419 let (a, b) = (home(&pa), home(&pb));
3420 let (alice_id, bob_id) = (Identity::load_or_create(&a).unwrap(), Identity::load_or_create(&b).unwrap());
3421 let mut m = Membership::load(&a).unwrap().unwrap();
3422 m.invite(&alice_id, &bob_id.public(), "steer").unwrap();
3423 m.save(&a).unwrap();
3424 let now = borg::learn::now_ms();
3425 let ev = json!({"session_id": "alice-s1", "cwd": pa.dir, "prompt": "look merges up"});
3426 borg::presence::write(&a, &borg::presence::update(None, "work", "claude", "prompt", &ev, now, |_| ("alice@x".into(), "repo".into()))).unwrap();
3427 write(&a, "lessons.jsonl", &[lesson("L1", 1, "how are merges found?")]);
3428 let asked = json!({"kind": "asked", "session": "alice-s1", "lessons": ["L1"], "ms": now - 10, "repo": "repo"}).to_string();
3429 write(&a, "outcomes.jsonl", &[asked]);
3430 let alice = Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap();
3431 let bob = Node::bind(&pb, LOCAL.parse().unwrap()).await.unwrap();
3432 let ticket = alice.ticket();
3433 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
3434 let _ = (served.unwrap(), synced.unwrap());
3435
3436 let to = format!("alice-s1@{}", alice_id.public());
3437 let veto = steering(&bob_id.public(), borg::msg::Kind::Veto, &to, now, "never look merges up in the commit log");
3438 borg::msg::send(&b, veto, now).unwrap();
3439 assert!(!b.join("episodes.jsonl").exists(), "the issuing node records no veto of its own");
3440 for _ in 0..2 {
3441 let (served, synced) = tokio::join!(alice.accept(), bob.sync(&ticket));
3442 let _ = (served.unwrap(), synced.unwrap());
3443 }
3444 let vetoed =
3445 |h: &Path| borg::outcome::read(&h.join("outcomes.jsonl")).into_iter().filter(|o| o.kind == borg::outcome::VETOED && o.lessons == ["L1"]).count();
3446 assert_eq!(vetoed(&a), 1, "credited once on the drone's node, over two rounds");
3447 let eps = std::fs::read_to_string(a.join("episodes.jsonl")).unwrap();
3448 assert_eq!(eps.lines().count(), 1);
3449 assert!(eps.contains("\"kind\":\"veto\"") && eps.contains("bob@x") && eps.contains("never look merges up"), "{eps}");
3450 let stats: Vec<_> = borg::stats::events(&a).into_iter().filter(|e| e.kind == borg::stats::VETO).collect();
3451 assert_eq!((stats.len(), stats[0].why.as_str()), (1, "mesh"));
3452 let caps = borg::harness::by_name("claude").unwrap().caps();
3454 let prompt = json!({"session_id": "alice-s1", "cwd": pa.dir});
3455 let out = borg::msg::deliver(&a, caps, "prompt", &prompt, Some(now - 10), now + 1, None).expect("shown");
3456 let ctx = out["hookSpecificOutput"]["additionalContext"].as_str().unwrap();
3457 assert!(ctx.contains(&format!("[borg] Veto from bob@x (node {}, role steer)", &bob_id.public()[..12])), "{ctx}");
3458 assert_eq!(borg::msg::deliver(&a, caps, "prompt", &prompt, Some(now - 10), now + 2, None), None, "once");
3459 let l1 = [borg::conscience::Challenge { id: "L1".into(), ..Default::default() }];
3461 let belief = |h: &Path| borg::memory::derive(&borg::outcome::read_team(&h.join("lessons.jsonl")), &l1, None)["L1"].vetoed;
3462 assert_eq!((belief(&a), belief(&b)), (1, 1));
3463 assert!(!b.join("outcomes.jsonl").exists() && !b.join("episodes.jsonl").exists(), "bob's node holds only the team copy");
3464 alice.close().await;
3465 bob.close().await;
3466 }
3467
3468 #[tokio::test(flavor = "multi_thread")]
3469 async fn a_local_collectives_presence_and_steering_never_leave() {
3470 let p = node("live-private", "local");
3471 let h = home(&p);
3472 let now = borg::learn::now_ms();
3473 let ev = json!({"session_id": "s1", "cwd": p.dir, "prompt": "private diary"});
3474 borg::presence::write(&h, &borg::presence::update(None, "work", "claude", "prompt", &ev, now, |_| ("me@x".into(), "r".into()))).unwrap();
3475 let me = Identity::load_or_create(&h).unwrap().public();
3476 borg::msg::send(&h, steering(&me, borg::msg::Kind::Pause, ALL_TO, now, "lunch"), now).unwrap();
3477 let held = borg::steer::read_admitted(&h);
3478 assert_eq!((held.len(), held[0].role), (1, borg::steer::Role::Local), "this machine's engineer steers it");
3479 assert!(Node::bind(&p, LOCAL.parse().unwrap()).await.is_err(), "no socket at all");
3480 }
3481
3482 #[test]
3487 fn the_window_bounds_a_burst_and_never_a_steer() {
3488 use borg::msg::{Kind, Seen, V1_RATE_MS, WINDOW_CAP};
3489 let (node, recipient) = ("a".repeat(64), "b".repeat(64));
3490 let to = format!("s1@{recipient}");
3491 let steers = [steering(&node, Kind::Pause, &to, 1000, ""), steering(&node, Kind::Resume, &to, 1001, "")];
3492 let burst: Vec<borg::msg::Message> = (0..WINDOW_CAP + 2).map(|i| steering(&node, Kind::Tell, &to, 2000 + i as i64, &format!("t{i}"))).collect();
3493 let later = steering(&node, Kind::Tell, &to, 2000 + burst.len() as i64 + V1_RATE_MS, "a slot later");
3494 let line = |m: &borg::msg::Message| serde_json::to_string(m).unwrap();
3495 let mut set: BTreeMap<Hash, String> = steers.iter().chain(&burst).chain([&later]).map(line).map(|l| (lines::hash(&l), l)).collect();
3496 let within = |set: &BTreeMap<Hash, String>| {
3497 let w = window(set);
3498 steers.iter().chain(&burst).chain([&later]).map(|m| w.contains(&lines::hash(&line(m)))).collect::<Vec<bool>>()
3499 };
3500 let mut want = vec![true; steers.len() + burst.len() + 1];
3501 (want[2], want[3]) = (false, false);
3502 assert_eq!(within(&set), want, "steers in; of {} unread within a rate slot, the newest {WINDOW_CAP}", burst.len());
3503 let seen = serde_json::to_string(&Seen::new("s1", &recipient, 3000, &[&burst[0].id])).unwrap();
3504 set.insert(lines::hash(&seen), seen);
3505 want[2] = true;
3506 assert_eq!(within(&set), want, "the first acknowledged: in, and the second still one too many");
3507 }
3508
3509 #[test]
3513 fn the_v1_window_keeps_a_pause_and_offers_only_v1_lines() {
3514 use borg::msg::Kind;
3515 let (node, per) = ("a".repeat(64), 5);
3516 let to = |i: usize| format!("s{i}@{}", "b".repeat(64));
3517 let ms: Vec<borg::msg::Message> = std::iter::once(steering(&node, Kind::Pause, &to(0), 1000, ""))
3518 .chain((0..=per).map(|i| steering(&node, Kind::Tell, &to(i), 1001 + i as i64, "hi")))
3519 .collect();
3520 let long = steering(&node, Kind::Tell, &to(9), 2000, &"w".repeat(borg::msg::V1_TEXT + 1));
3521 let seen = seen_of(&node, "s1", 2001, &[&ms[1].id]);
3522 let mut set: BTreeMap<Hash, String> = ms.iter().chain([&long]).map(|m| serde_json::to_string(m).unwrap()).map(|l| (lines::hash(&l), l)).collect();
3523 set.insert(lines::hash(&seen), seen.clone());
3524 let w = v1_window(&set, per);
3525 let kept = |m: &borg::msg::Message| w.contains(&lines::hash(&serde_json::to_string(m).unwrap()));
3526 assert!(kept(&ms[0]), "the pause travels");
3527 assert_eq!((kept(&ms[1]), ms[2..].iter().filter(|m| kept(m)).count()), (false, per), "the oldest tell went");
3528 assert!(!kept(&long) && !w.contains(&lines::hash(&seen)), "a v2 text and a seen line are not 0.5.0's to read");
3529 }
3530
3531 #[test]
3538 fn what_send_wrote_stays_within_the_window_however_the_log_is_pruned() {
3539 use borg::msg::{Address, Kind, Message, SendError, Sender};
3540 let home = std::env::temp_dir().join(format!("borg-mesh-sent-{}", std::process::id()));
3541 let _ = std::fs::remove_dir_all(&home);
3542 std::fs::create_dir_all(&home).unwrap();
3543 let here = borg::msg::node(&home);
3544 let senders = ["s1", "s2", "engineer:e@x"];
3545 let recipients = [format!("r1@{here}"), format!("r1@{here}"), format!("r2@{here}"), format!("r3@{}", "d".repeat(64)), "all".into(), "engineer".into()];
3546 let mut seed = 11u64;
3547 let mut pick = |n: usize| {
3548 seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
3549 (seed >> 33) as usize % n
3550 };
3551 let start = 10 * borg::msg::TTL_MS;
3552 let (mut ms, mut sent, mut acks, mut refused) = (start, Vec::<Message>::new(), 0, 0);
3553 while ms < start + 3 * borg::msg::TTL_MS {
3554 ms += [1_000, 1_000, borg::msg::V1_RATE_MS, 3_600_000, 6 * 3_600_000][pick(5)];
3555 let from = Sender { by: senders[pick(senders.len())].into(), name: "One of One".into(), harness: "claude".into(), node: here.clone() };
3556 let earlier = (!sent.is_empty()).then(|| sent[sent.len() - 1 - pick(sent.len().min(4))].clone());
3558 let recipient = earlier.as_ref().and_then(|m| match Address::parse(&m.to) {
3560 Some(Address::Session { id, node: Some(n) }) if n == here => Some(id),
3561 Some(Address::All) => Some("r9".to_string()),
3562 _ => None,
3563 });
3564 match (pick(6), earlier, recipient) {
3565 (0 | 1, Some(m), Some(who)) => {
3566 borg::msg::ack(&home, &who, &[&m.id], ms).unwrap();
3567 acks += 1;
3568 }
3569 (2, Some(m), Some(who)) if !borg::msg::is_engineer(&m.by) => {
3570 let from = Sender { by: who, name: "Two of Two".into(), harness: "omp".into(), node: here.clone() };
3571 let _ = borg::msg::send(&home, Message::new(Kind::Tell, &from, &format!("{}@{here}", m.by), Some(&m.id), ms, "re"), ms);
3572 }
3573 _ => match borg::msg::send(&home, Message::new(Kind::Tell, &from, &recipients[pick(recipients.len())], None, ms, "hi"), ms) {
3574 Ok(m) => sent.push(m),
3575 Err(SendError::Window { .. }) => refused += 1,
3576 Err(e) => panic!("{e}"),
3577 },
3578 }
3579 }
3580 let held: Vec<String> = std::fs::read_to_string(borg::msg::path(&home)).unwrap().lines().map(String::from).collect();
3581 let talk = |l: &str| Line::parse(l).and_then(|x| x.message().cloned()).filter(|m| !m.kind.privileged());
3582 assert!(acks > 10 && refused > 10 && held.iter().filter(|l| talk(l).is_some()).count() < sent.len(), "{acks} acks, {refused} refused, pruned");
3583 let mut fewer = 0;
3584 for cut in (0..held.len()).step_by(5) {
3585 let set: BTreeMap<Hash, String> = held[cut..].iter().map(|l| (lines::hash(l), l.clone())).collect();
3586 let (w, parsed) = (window(&set), held[cut..].iter().filter_map(|l| Line::parse(l)).collect::<Vec<Line>>());
3587 for (l, m) in held[cut..].iter().filter_map(|l| talk(l).map(|m| (l, m))) {
3588 assert!(w.contains(&lines::hash(l)), "from line {cut} on: {l}");
3589 fewer += usize::from(!borg::msg::window_for(&parsed, m.author(), &m.to, ms).admits(&m.id));
3590 }
3591 }
3592 assert!(fewer > 0, "some stretch's replay admits less than send did");
3593 let _ = std::fs::remove_dir_all(&home);
3594 }
3595
3596 #[test]
3599 fn live_keeps_an_empty_directives_field_for_0_3_0() {
3600 let out = serde_json::to_value(Msg::Live { presence: Vec::new(), directives: Directives }).unwrap();
3601 assert_eq!(out, json!({"t": "live", "presence": [], "directives": []}));
3602 let theirs = json!({"t": "live", "presence": [], "directives": [{"kind": "pause", "sig": "00"}], "beats": []});
3603 assert!(matches!(serde_json::from_value::<Msg>(theirs).unwrap(), Msg::Live { .. }));
3604 assert!(matches!(serde_json::from_value::<Msg>(json!({"t": "live", "presence": []})).unwrap(), Msg::Live { .. }));
3605 }
3606
3607 fn answer(n: std::sync::Arc<Node>) {
3609 tokio::spawn(async move {
3610 while let Some(i) = n.incoming().await {
3611 let n = n.clone();
3612 tokio::spawn(async move {
3613 let _ = n.handle(i).await;
3614 });
3615 }
3616 });
3617 }
3618
3619 fn ticket(admin: &Node, role: &str, ttl_ms: i64, uses: u32) -> Ticket {
3620 let home = admin.home();
3621 let m = Membership::load(home).unwrap().unwrap();
3622 let invite = invite::issue(home, &admin.me, &m, role, ttl_ms, uses, borg::learn::now_ms()).unwrap();
3623 Ticket::decode(&Ticket { invite, relay: admin.relay.clone(), hints: admin.hints() }.encode()).unwrap()
3624 }
3625
3626 #[tokio::test(flavor = "multi_thread")]
3630 async fn three_nodes_join_by_ticket_and_sync_a_lesson() {
3631 let pa = crew("ticket", &["alice"]).remove(0);
3632 let (pb, pc) = (node("ticket-bob", "team"), node("ticket-carol", "team"));
3633 let l1 = lesson("L1", 1, "who owns the lock?");
3634 write(&home(&pa), "lessons.jsonl", std::slice::from_ref(&l1));
3635 let alice = std::sync::Arc::new(Node::bind(&pa, LOCAL.parse().unwrap()).await.unwrap());
3636 answer(alice.clone());
3637
3638 let one = ticket(&alice, "member", 60_000, 1);
3639 let bob = Node::joining(&pb, LOCAL.parse().unwrap()).await.unwrap().quiet();
3640 let (m, rep) = bob.join(&one).await.unwrap();
3641 assert_eq!(m.role_of(&bob.public()), Some("member"));
3642 assert_eq!(rep.received, 1, "bob syncs at once");
3643 assert!(holds(&home(&pb), "lessons.jsonl", &l1));
3644 assert_eq!(peers::load(&home(&pb)).get(&alice.public()), Some(&addr_of(&alice)), "bob knows where alice is");
3645 assert!(peers::load(&home(&pa)).contains_key(&bob.public()), "alice knows bob, by key at least");
3646
3647 let carol = Node::joining(&pc, LOCAL.parse().unwrap()).await.unwrap().quiet();
3648 let err = carol.join(&one).await.unwrap_err();
3649 assert!(err.contains("already used"), "{err}");
3650 assert!(Membership::load(&home(&pc)).unwrap().is_none() && !home(&pc).join("lessons.jsonl").exists(), "carol got nothing");
3651 assert!(bob.join(&one).await.is_ok(), "bob again (a lost answer): welcomed");
3652
3653 let gone = ticket(&alice, "member", 1, 1);
3654 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
3655 assert!(carol.join(&gone).await.unwrap_err().contains("expired"));
3656
3657 let two = ticket(&alice, "observe", 60_000, 1);
3658 let (m, rep) = carol.join(&two).await.unwrap();
3659 assert_eq!(m.role_of(&carol.public()), Some("observe"));
3660 assert_eq!(rep.received, 1);
3661 let held = Membership::load(&home(&pa)).unwrap().unwrap();
3662 assert_eq!((held.role_of(&bob.public()), held.role_of(&carol.public())), (Some("member"), Some("observe")));
3663 assert!(invite::pending(alice.home(), borg::learn::now_ms()).is_empty(), "both tickets spent");
3664
3665 assert_eq!(Membership::load(&home(&pb)).unwrap().unwrap().role_of(&carol.public()), None);
3667 bob.sync(&format!("{}@{}", alice.public(), addr_of(&alice))).await.unwrap();
3668 assert_eq!(Membership::load(&home(&pb)).unwrap().unwrap().role_of(&carol.public()), Some("observe"));
3669 for n in [bob, carol] {
3670 n.close().await;
3671 }
3672 alice.close().await;
3673 }
3674
3675 #[tokio::test(flavor = "multi_thread")]
3679 async fn join_and_sync_go_through_the_relay_when_no_direct_address_is_known() {
3680 let (_map, url, _server) = iroh::test_utils::run_relay_server().await.unwrap();
3681 let pa = crew("relay", &["alice"]).remove(0);
3682 let pb = node("relay-bob", "team");
3683 for p in [&pa, &pb] {
3684 let cfg = json!({"default": "off", "collectives": [{"name": "work", "paths": [p.dir], "share": "team", "relay": url.to_string()}]});
3685 std::fs::write(&p.config, cfg.to_string()).unwrap();
3686 }
3687 let l1 = lesson("L1", 1, "does this cross the NAT?");
3688 write(&home(&pa), "lessons.jsonl", std::slice::from_ref(&l1));
3689 let trust = |b: Builder| b.ca_tls_config(iroh::tls::CaTlsConfig::insecure_skip_verify());
3690 let alice = std::sync::Arc::new(Node::open(&pa, None, true, trust).await.unwrap());
3691 let bob = Node::open(&pb, None, false, trust).await.unwrap().quiet();
3692 assert!(matches!(alice.relay(), Relay::Custom(_)));
3693 assert!(alice.tickets().is_empty() && bob.tickets().is_empty(), "no direct address on either side");
3694 let wait = std::time::Duration::from_secs(10);
3695 assert!(alice.online(wait).await && bob.online(wait).await, "both reach the relay");
3696 answer(alice.clone());
3697
3698 let mut t = ticket(&alice, "member", 60_000, 1);
3699 t.hints.retain(|h| !matches!(h, Hint::Direct(_)));
3700 let (m, rep) = bob.join(&t).await.unwrap();
3701 assert_eq!(m.role_of(&bob.public()), Some("member"));
3702 assert_eq!(rep.received, 1);
3703 assert!(holds(&home(&pb), "lessons.jsonl", &l1), "the lesson crossed through the relay");
3704 let l2 = lesson("L2", 1, "and back?");
3706 append_lines(&home(&pb), "lessons.jsonl", std::slice::from_ref(&l2));
3707 assert_eq!(bob.sync(&alice.public()).await.unwrap().sent, 1);
3708 assert!(holds(&home(&pa), "lessons.jsonl", &l2));
3709 bob.close().await;
3710 alice.close().await;
3711 }
3712}