1use std::{
6 collections::{BTreeMap, HashSet},
7 io,
8 path::{Path, PathBuf},
9 sync::{
10 Arc, Mutex as StdMutex,
11 atomic::{AtomicBool, Ordering},
12 },
13 time::{Duration, Instant},
14};
15
16use bytes::Bytes;
17use arachne_routing::RoutingTable;
18use futures_util::StreamExt;
19use iroh::endpoint::{Connection, RecvStream, SendStream, VarInt};
20use iroh_blobs::{
21 Hash,
22 api::{
23 TempTag,
24 blobs::{AddPathOptions, ExportMode, ExportOptions, ImportMode},
25 remote::GetProgressItem,
26 },
27 get,
28 protocol::Request,
29 provider::{self, events::EventSender},
30 store::{fs::FsStore, gc_run_once},
31 util::{RecvStream as BlobReader, SendStream as BlobWriter},
32};
33use serde::{Deserialize, Serialize};
34use tokio::sync::{Mutex, OnceCell, Semaphore, watch};
35
36use crate::{ALPN, Error, PeerId, Result, WorkspaceId, connections::Connections, transport};
37
38pub(crate) const STREAM_KIND: u8 = 1;
39const ADMISSION_LIFETIME: Duration = Duration::from_secs(120);
40const IDLE_TIMEOUT: Duration = Duration::from_secs(60);
41
42#[derive(Clone, Debug, Serialize, Deserialize)]
46#[serde(deny_unknown_fields)]
47pub struct ResourceTicket {
48 pub hash: [u8; 32],
49 pub size: u64,
50 pub grant: [u8; 32],
51}
52
53struct ReadGrant {
54 workspace: WorkspaceId,
55 revision: u64,
56 peer: PeerId,
57 path: PathBuf,
58 expires: Instant,
59 tag: TempTag,
60}
61
62struct Shared {
63 connections: Connections,
64 routing: Arc<Mutex<RoutingTable>>,
65 store: OnceCell<(PathBuf, FsStore)>,
66 grants: StdMutex<BTreeMap<[u8; 32], Arc<ReadGrant>>>,
67 changed: watch::Sender<u64>,
68 serving: Semaphore,
69 downloading: Mutex<()>,
70 collecting: Mutex<()>,
71 closed: AtomicBool,
72}
73
74#[derive(Clone)]
75pub struct ResourceTransfers(Arc<Shared>);
76
77impl ResourceTransfers {
78 pub(crate) fn new(connections: Connections, routing: Arc<Mutex<RoutingTable>>) -> Self {
79 Self(Arc::new(Shared {
80 connections,
81 routing,
82 store: OnceCell::new(),
83 grants: StdMutex::new(BTreeMap::new()),
84 changed: watch::channel(0).0,
85 serving: Semaphore::new(4),
86 downloading: Mutex::new(()),
87 collecting: Mutex::new(()),
88 closed: AtomicBool::new(false),
89 }))
90 }
91
92 async fn store(&self, root: &Path) -> Result<FsStore> {
93 if !root.is_absolute() {
94 return Err(Error::Rejected);
95 }
96 let (path, store) = self
97 .0
98 .store
99 .get_or_try_init(|| async {
100 let store = FsStore::load(root).await.map_err(transport)?;
101 gc_run_once(&store, &mut HashSet::new())
104 .await
105 .map_err(transport)?;
106 Ok::<_, Error>((root.to_owned(), store))
107 })
108 .await?;
109 if path != root {
110 return Err(Error::Rejected);
111 }
112 Ok(store.clone())
113 }
114
115 async fn authorize(&self, workspace: WorkspaceId, revision: u64, peer: PeerId) -> Result<()> {
116 if self.0.closed.load(Ordering::Acquire) {
117 return Err(Error::Rejected);
118 }
119 let routing = self.0.routing.lock().await;
120 routing.authorizes_endpoint(workspace, revision, self.0.connections.id())?;
121 routing.authorizes_endpoint(workspace, revision, peer)?;
122 Ok(())
123 }
124
125 pub async fn prepare(
128 &self,
129 root: &Path,
130 path: &Path,
131 workspace: WorkspaceId,
132 revision: u64,
133 peer: PeerId,
134 ) -> Result<ResourceTicket> {
135 let generation = *self.0.changed.borrow();
136 self.authorize(workspace, revision, peer).await?;
137 if !path.is_absolute() {
138 return Err(Error::Rejected);
139 }
140 let metadata = tokio::fs::metadata(path).await.map_err(transport)?;
141 if !metadata.is_file() {
142 return Err(Error::Rejected);
143 }
144 let store = self.store(root).await?;
145 let tag = store
146 .add_path_with_opts(AddPathOptions {
147 path: path.to_owned(),
148 mode: ImportMode::TryReference,
149 format: iroh_blobs::BlobFormat::Raw,
150 })
151 .temp_tag()
152 .await
153 .map_err(transport)?;
154 self.authorize(workspace, revision, peer).await?;
155 let mut token = [0; 32];
156 getrandom::fill(&mut token).map_err(transport)?;
157 let ticket = ResourceTicket {
158 hash: *tag.hash().as_bytes(),
159 size: metadata.len(),
160 grant: token,
161 };
162 {
163 let mut grants = self.0.grants.lock().unwrap();
164 if generation != *self.0.changed.borrow() {
165 return Err(Error::Rejected);
166 }
167 grants
168 .retain(|_, grant| grant.expires > Instant::now() || Arc::strong_count(grant) > 1);
169 if grants.len() >= 64 {
170 return Err(Error::Backpressure);
171 }
172 grants.insert(
173 token,
174 Arc::new(ReadGrant {
175 workspace,
176 revision,
177 peer,
178 path: path.to_owned(),
179 expires: Instant::now() + ADMISSION_LIFETIME,
180 tag,
181 }),
182 );
183 }
184 self.collect(&store).await?;
185 Ok(ticket)
186 }
187
188 pub fn revoke(&self, path: Option<&Path>) {
191 let mut grants = self.0.grants.lock().unwrap();
192 grants.retain(|_, grant| path.is_some_and(|path| grant.path != path));
193 self.policy_changed();
194 }
195
196 pub(crate) fn policy_changed(&self) {
197 self.0
198 .changed
199 .send_modify(|generation| *generation = generation.wrapping_add(1));
200 }
201
202 async fn collect(&self, store: &FsStore) -> Result<()> {
203 let _serial = self.0.collecting.lock().await;
204 gc_run_once(store, &mut HashSet::new())
205 .await
206 .map_err(transport)
207 }
208
209 async fn admitted(&self, token: &[u8; 32], grant: &ReadGrant) -> Result<()> {
210 self.authorize(grant.workspace, grant.revision, grant.peer)
211 .await?;
212 if !self.0.grants.lock().unwrap().contains_key(token) {
213 return Err(Error::Rejected);
214 }
215 Ok(())
216 }
217
218 pub(crate) async fn serve(
219 &self,
220 connection: &Connection,
221 send: &mut SendStream,
222 recv: &mut RecvStream,
223 ) -> Result<()> {
224 let _slot = self
225 .0
226 .serving
227 .try_acquire()
228 .map_err(|_| Error::Backpressure)?;
229 let mut changes = self.0.changed.subscribe();
230 let mut token = [0; 32];
231 tokio::time::timeout(crate::TIMEOUT, recv.read_exact(&mut token))
232 .await
233 .map_err(|_| Error::Timeout("resource admission"))?
234 .map_err(transport)?;
235 let grant = self
236 .0
237 .grants
238 .lock()
239 .unwrap()
240 .get(&token)
241 .cloned()
242 .ok_or(Error::Rejected)?;
243 if grant.peer != *connection.remote_id().as_bytes() || grant.expires <= Instant::now() {
244 return Err(Error::Rejected);
245 }
246 self.admitted(&token, &grant).await?;
247 let (_, store) = self.0.store.get().ok_or(Error::Rejected)?;
248 let mut pair = provider::StreamPair::new(
249 connection.stable_id() as u64,
250 SizedReader {
251 inner: recv,
252 size: None,
253 header: [0; 8],
254 read: 0,
255 },
256 IdleWriter(send),
257 EventSender::default(),
258 );
259 let request = tokio::time::timeout(crate::TIMEOUT, pair.read_request())
260 .await
261 .map_err(|_| Error::Timeout("blob request"))?
262 .map_err(transport)?;
263 let Request::Get(request) = request else {
264 return Err(Error::Rejected);
265 };
266 if request.hash != grant.tag.hash() || !request.ranges.is_blob() {
269 return Err(Error::Rejected);
270 }
271 let transfer = provider::handle_get(pair, store.as_ref().clone(), request);
272 tokio::pin!(transfer);
273 loop {
274 tokio::select! {
275 result = &mut transfer => return result.map_err(transport),
276 result = changes.changed() => {
277 result.map_err(transport)?;
278 self.admitted(&token, &grant).await?;
279 }
280 }
281 }
282 }
283
284 #[allow(clippy::too_many_arguments)]
288 pub async fn fetch(
289 &self,
290 root: &Path,
291 target: &Path,
292 workspace: WorkspaceId,
293 revision: u64,
294 peer: PeerId,
295 ticket: ResourceTicket,
296 progress: watch::Sender<u64>,
297 ) -> Result<()> {
298 let _slot = self
299 .0
300 .downloading
301 .try_lock()
302 .map_err(|_| Error::Backpressure)?;
303 self.authorize(workspace, revision, peer).await?;
304 if !target.is_absolute() {
305 return Err(Error::Rejected);
306 }
307 let store = self.store(root).await?;
308 let hash = Hash::from_bytes(ticket.hash);
309 store
310 .tags()
311 .set(b"partial", hash)
312 .await
313 .map_err(transport)?;
314 self.collect(&store).await?;
315 let local = store.remote().local(hash).await.map_err(transport)?;
316 progress.send_replace(local.local_bytes());
317 if !local.is_complete() {
318 let mut changes = self.0.changed.subscribe();
319 let connection =
320 tokio::time::timeout(IDLE_TIMEOUT, self.0.connections.connect(peer, ALPN))
321 .await
322 .map_err(|_| Error::Timeout("resource connect"))??;
323 self.authorize(workspace, revision, peer).await?;
324 let (mut send, recv) = connection.open_bi().await.map_err(transport)?;
325 send.write_all(&[STREAM_KIND]).await.map_err(transport)?;
326 send.write_all(&ticket.grant).await.map_err(transport)?;
327 let reader = SizedReader {
328 inner: recv,
329 size: Some(ticket.size),
330 header: [0; 8],
331 read: 0,
332 };
333 let pair = get::StreamPair::new(connection.stable_id() as u64, reader, send);
334 let transfer = store.remote().fetch(pair, hash);
335 let mut events = transfer.stream();
336 let idle = tokio::time::sleep(IDLE_TIMEOUT);
337 tokio::pin!(idle);
338 let result = loop {
339 tokio::select! {
340 event = events.next() => match event {
341 Some(GetProgressItem::Progress(bytes)) => {
342 progress.send_replace(bytes);
343 idle.as_mut().reset(tokio::time::Instant::now() + IDLE_TIMEOUT);
344 }
345 Some(GetProgressItem::Done(_)) => break Ok(()),
346 Some(GetProgressItem::Error(error)) => break Err(transport(error)),
347 None => break Err(Error::Rejected),
348 },
349 _ = &mut idle => break Err(Error::Timeout("resource progress")),
350 result = changes.changed() => {
351 if let Err(error) = result.map_err(transport) { break Err(error); }
352 if let Err(error) = self.authorize(workspace, revision, peer).await { break Err(error); }
353 }
354 }
355 };
356 drop(events);
358 store.sync_db().await.map_err(transport)?;
359 result?;
360 }
361 self.authorize(workspace, revision, peer).await?;
362 store
363 .export_with_opts(ExportOptions {
364 hash,
365 target: target.to_owned(),
366 mode: ExportMode::TryReference,
367 })
368 .await
369 .map_err(transport)?;
370 if tokio::fs::metadata(target).await.map_err(transport)?.len() != ticket.size {
371 return Err(Error::Rejected);
372 }
373 store.tags().delete(b"partial").await.map_err(transport)?;
374 self.collect(&store).await?;
375 progress.send_replace(ticket.size);
376 Ok(())
377 }
378
379 pub async fn clear_partial(&self, root: &Path) -> Result<()> {
381 let _slot = self.0.downloading.lock().await;
382 if root.exists() {
383 let store = self.store(root).await?;
384 store.tags().delete(b"partial").await.map_err(transport)?;
385 self.collect(&store).await?;
386 }
387 Ok(())
388 }
389
390 pub(crate) async fn close(&self) {
391 self.stop();
392 if let Some((_, store)) = self.0.store.get() {
393 let _ = store.shutdown().await;
394 }
395 }
396
397 pub(crate) fn stop(&self) {
398 self.0.closed.store(true, Ordering::Release);
399 self.revoke(None);
400 }
401}
402
403struct SizedReader<R> {
406 inner: R,
407 size: Option<u64>,
408 header: [u8; 8],
409 read: usize,
410}
411
412impl<R> SizedReader<R> {
413 fn validate(&mut self, bytes: &[u8]) -> io::Result<()> {
414 let Some(size) = self.size else {
415 return Ok(());
416 };
417 let length = (8 - self.read).min(bytes.len());
418 self.header[self.read..self.read + length].copy_from_slice(&bytes[..length]);
419 self.read += length;
420 if self.read == 8 && u64::from_le_bytes(self.header) != size {
421 return Err(io::Error::new(
422 io::ErrorKind::InvalidData,
423 "resource length differs from grant",
424 ));
425 }
426 Ok(())
427 }
428}
429
430impl<R: BlobReader> BlobReader for SizedReader<R> {
431 async fn recv_bytes(&mut self, len: usize) -> io::Result<Bytes> {
432 let bytes = io_idle(self.inner.recv_bytes(len)).await?;
433 self.validate(&bytes)?;
434 Ok(bytes)
435 }
436 async fn recv_bytes_exact(&mut self, len: usize) -> io::Result<Bytes> {
437 let bytes = io_idle(self.inner.recv_bytes_exact(len)).await?;
438 self.validate(&bytes)?;
439 Ok(bytes)
440 }
441 async fn recv_exact(&mut self, target: &mut [u8]) -> io::Result<()> {
442 io_idle(self.inner.recv_exact(target)).await?;
443 self.validate(target)
444 }
445 fn stop(&mut self, code: VarInt) -> io::Result<()> {
446 BlobReader::stop(&mut self.inner, code)
447 }
448 fn id(&self) -> u64 {
449 BlobReader::id(&self.inner)
450 }
451}
452
453async fn io_idle<T>(operation: impl std::future::Future<Output = io::Result<T>>) -> io::Result<T> {
454 tokio::time::timeout(IDLE_TIMEOUT, operation)
455 .await
456 .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "resource stream stalled"))?
457}
458
459struct IdleWriter<'a>(&'a mut SendStream);
462impl BlobWriter for IdleWriter<'_> {
463 async fn send_bytes(&mut self, bytes: Bytes) -> io::Result<()> {
464 io_idle(BlobWriter::send_bytes(self.0, bytes)).await
465 }
466 async fn send(&mut self, bytes: &[u8]) -> io::Result<()> {
467 io_idle(BlobWriter::send(self.0, bytes)).await
468 }
469 async fn sync(&mut self) -> io::Result<()> {
470 io_idle(BlobWriter::sync(self.0)).await
471 }
472 fn reset(&mut self, code: VarInt) -> io::Result<()> {
473 BlobWriter::reset(self.0, code)
474 }
475 async fn stopped(&mut self) -> io::Result<Option<VarInt>> {
476 io_idle(BlobWriter::stopped(self.0)).await
477 }
478 fn id(&self) -> u64 {
479 BlobWriter::id(self.0)
480 }
481}
482
483#[tokio::test(start_paused = true)]
484async fn idle_deadline_is_not_a_transfer_duration_limit() {
485 let error = io_idle(std::future::pending::<io::Result<()>>())
486 .await
487 .unwrap_err();
488 assert_eq!(error.kind(), io::ErrorKind::TimedOut);
489 let started = tokio::time::Instant::now();
490 for _ in 0..3 {
491 io_idle(async {
492 tokio::time::sleep(IDLE_TIMEOUT / 2).await;
493 Ok(())
494 })
495 .await
496 .unwrap();
497 }
498 assert!(started.elapsed() > IDLE_TIMEOUT);
499}