Skip to main content

arachne_node/
resources.rs

1//! Authorized immutable byte transfers. Catalogs and recipient grants belong to
2//! the caller; this module binds a short-lived read capability to a verified
3//! workspace endpoint and runs the standard iroh-blobs range/verification engine.
4//! No file bytes enter the message journal and no file-size ceiling is imposed.
5use std::{
6    collections::{BTreeMap, HashSet},
7    io,
8    path::{Path, PathBuf},
9    sync::{
10        Arc, Mutex as StdMutex,
11        atomic::{AtomicBool, Ordering},
12    },
13    time::{Duration, Instant},
14};
15
16use bytes::Bytes;
17use arachne_routing::RoutingTable;
18use futures_util::StreamExt;
19use iroh::endpoint::{Connection, RecvStream, SendStream, VarInt};
20use iroh_blobs::{
21    Hash,
22    api::{
23        TempTag,
24        blobs::{AddPathOptions, ExportMode, ExportOptions, ImportMode},
25        remote::GetProgressItem,
26    },
27    get,
28    protocol::Request,
29    provider::{self, events::EventSender},
30    store::{fs::FsStore, gc_run_once},
31    util::{RecvStream as BlobReader, SendStream as BlobWriter},
32};
33use serde::{Deserialize, Serialize};
34use tokio::sync::{Mutex, OnceCell, Semaphore, watch};
35
36use crate::{ALPN, Error, PeerId, Result, WorkspaceId, connections::Connections, transport};
37
38pub(crate) const STREAM_KIND: u8 = 1;
39const ADMISSION_LIFETIME: Duration = Duration::from_secs(120);
40const IDLE_TIMEOUT: Duration = Duration::from_secs(60);
41
42/// Transmitted only inside an authenticated, recipient-scoped application
43/// message. The hash identifies bytes; neither a hash nor this token alone is
44/// authority. Serving also requires the bound endpoint and current membership.
45#[derive(Clone, Debug, Serialize, Deserialize)]
46#[serde(deny_unknown_fields)]
47pub struct ResourceTicket {
48    pub hash: [u8; 32],
49    pub size: u64,
50    pub grant: [u8; 32],
51}
52
53struct ReadGrant {
54    workspace: WorkspaceId,
55    revision: u64,
56    peer: PeerId,
57    path: PathBuf,
58    expires: Instant,
59    tag: TempTag,
60}
61
62struct Shared {
63    connections: Connections,
64    routing: Arc<Mutex<RoutingTable>>,
65    store: OnceCell<(PathBuf, FsStore)>,
66    grants: StdMutex<BTreeMap<[u8; 32], Arc<ReadGrant>>>,
67    changed: watch::Sender<u64>,
68    serving: Semaphore,
69    downloading: Mutex<()>,
70    collecting: Mutex<()>,
71    closed: AtomicBool,
72}
73
74#[derive(Clone)]
75pub struct ResourceTransfers(Arc<Shared>);
76
77impl ResourceTransfers {
78    pub(crate) fn new(connections: Connections, routing: Arc<Mutex<RoutingTable>>) -> Self {
79        Self(Arc::new(Shared {
80            connections,
81            routing,
82            store: OnceCell::new(),
83            grants: StdMutex::new(BTreeMap::new()),
84            changed: watch::channel(0).0,
85            serving: Semaphore::new(4),
86            downloading: Mutex::new(()),
87            collecting: Mutex::new(()),
88            closed: AtomicBool::new(false),
89        }))
90    }
91
92    async fn store(&self, root: &Path) -> Result<FsStore> {
93        if !root.is_absolute() {
94            return Err(Error::Rejected);
95        }
96        let (path, store) = self
97            .0
98            .store
99            .get_or_try_init(|| async {
100                let store = FsStore::load(root).await.map_err(transport)?;
101                // One resumable partial per workspace. Unreferenced old import
102                // indexes/outboards are reclaimed; immutable source files are external.
103                gc_run_once(&store, &mut HashSet::new())
104                    .await
105                    .map_err(transport)?;
106                Ok::<_, Error>((root.to_owned(), store))
107            })
108            .await?;
109        if path != root {
110            return Err(Error::Rejected);
111        }
112        Ok(store.clone())
113    }
114
115    async fn authorize(&self, workspace: WorkspaceId, revision: u64, peer: PeerId) -> Result<()> {
116        if self.0.closed.load(Ordering::Acquire) {
117            return Err(Error::Rejected);
118        }
119        let routing = self.0.routing.lock().await;
120        routing.authorizes_endpoint(workspace, revision, self.0.connections.id())?;
121        routing.authorizes_endpoint(workspace, revision, peer)?;
122        Ok(())
123    }
124
125    /// Caller has already checked this resource's public/direct audience. The
126    /// source must be an application-owned immutable snapshot, not a live file.
127    pub async fn prepare(
128        &self,
129        root: &Path,
130        path: &Path,
131        workspace: WorkspaceId,
132        revision: u64,
133        peer: PeerId,
134    ) -> Result<ResourceTicket> {
135        let generation = *self.0.changed.borrow();
136        self.authorize(workspace, revision, peer).await?;
137        if !path.is_absolute() {
138            return Err(Error::Rejected);
139        }
140        let metadata = tokio::fs::metadata(path).await.map_err(transport)?;
141        if !metadata.is_file() {
142            return Err(Error::Rejected);
143        }
144        let store = self.store(root).await?;
145        let tag = store
146            .add_path_with_opts(AddPathOptions {
147                path: path.to_owned(),
148                mode: ImportMode::TryReference,
149                format: iroh_blobs::BlobFormat::Raw,
150            })
151            .temp_tag()
152            .await
153            .map_err(transport)?;
154        self.authorize(workspace, revision, peer).await?;
155        let mut token = [0; 32];
156        getrandom::fill(&mut token).map_err(transport)?;
157        let ticket = ResourceTicket {
158            hash: *tag.hash().as_bytes(),
159            size: metadata.len(),
160            grant: token,
161        };
162        {
163            let mut grants = self.0.grants.lock().unwrap();
164            if generation != *self.0.changed.borrow() {
165                return Err(Error::Rejected);
166            }
167            grants
168                .retain(|_, grant| grant.expires > Instant::now() || Arc::strong_count(grant) > 1);
169            if grants.len() >= 64 {
170                return Err(Error::Backpressure);
171            }
172            grants.insert(
173                token,
174                Arc::new(ReadGrant {
175                    workspace,
176                    revision,
177                    peer,
178                    path: path.to_owned(),
179                    expires: Instant::now() + ADMISSION_LIFETIME,
180                    tag,
181                }),
182            );
183        }
184        self.collect(&store).await?;
185        Ok(ticket)
186    }
187
188    /// Withdraw before deleting a source, or pass None when serving preferences
189    /// change. Active streams are cancelled too, not just future admissions.
190    pub fn revoke(&self, path: Option<&Path>) {
191        let mut grants = self.0.grants.lock().unwrap();
192        grants.retain(|_, grant| path.is_some_and(|path| grant.path != path));
193        self.policy_changed();
194    }
195
196    pub(crate) fn policy_changed(&self) {
197        self.0
198            .changed
199            .send_modify(|generation| *generation = generation.wrapping_add(1));
200    }
201
202    async fn collect(&self, store: &FsStore) -> Result<()> {
203        let _serial = self.0.collecting.lock().await;
204        gc_run_once(store, &mut HashSet::new())
205            .await
206            .map_err(transport)
207    }
208
209    async fn admitted(&self, token: &[u8; 32], grant: &ReadGrant) -> Result<()> {
210        self.authorize(grant.workspace, grant.revision, grant.peer)
211            .await?;
212        if !self.0.grants.lock().unwrap().contains_key(token) {
213            return Err(Error::Rejected);
214        }
215        Ok(())
216    }
217
218    pub(crate) async fn serve(
219        &self,
220        connection: &Connection,
221        send: &mut SendStream,
222        recv: &mut RecvStream,
223    ) -> Result<()> {
224        let _slot = self
225            .0
226            .serving
227            .try_acquire()
228            .map_err(|_| Error::Backpressure)?;
229        let mut changes = self.0.changed.subscribe();
230        let mut token = [0; 32];
231        tokio::time::timeout(crate::TIMEOUT, recv.read_exact(&mut token))
232            .await
233            .map_err(|_| Error::Timeout("resource admission"))?
234            .map_err(transport)?;
235        let grant = self
236            .0
237            .grants
238            .lock()
239            .unwrap()
240            .get(&token)
241            .cloned()
242            .ok_or(Error::Rejected)?;
243        if grant.peer != *connection.remote_id().as_bytes() || grant.expires <= Instant::now() {
244            return Err(Error::Rejected);
245        }
246        self.admitted(&token, &grant).await?;
247        let (_, store) = self.0.store.get().ok_or(Error::Rejected)?;
248        let mut pair = provider::StreamPair::new(
249            connection.stable_id() as u64,
250            SizedReader {
251                inner: recv,
252                size: None,
253                header: [0; 8],
254                read: 0,
255            },
256            IdleWriter(send),
257            EventSender::default(),
258        );
259        let request = tokio::time::timeout(crate::TIMEOUT, pair.read_request())
260            .await
261            .map_err(|_| Error::Timeout("blob request"))?
262            .map_err(transport)?;
263        let Request::Get(request) = request else {
264            return Err(Error::Rejected);
265        };
266        // No push, collections, observe or get-many surface. A grant covers only
267        // one raw immutable blob, including its authenticated resume ranges.
268        if request.hash != grant.tag.hash() || !request.ranges.is_blob() {
269            return Err(Error::Rejected);
270        }
271        let transfer = provider::handle_get(pair, store.as_ref().clone(), request);
272        tokio::pin!(transfer);
273        loop {
274            tokio::select! {
275                result = &mut transfer => return result.map_err(transport),
276                result = changes.changed() => {
277                    result.map_err(transport)?;
278                    self.admitted(&token, &grant).await?;
279                }
280            }
281        }
282    }
283
284    /// Standard verified range download; retains at most one partial in this
285    /// workspace store. `target` is a private caller-owned temporary path.
286    /// The caller reserves disk/quota using the authenticated advertised size.
287    #[allow(clippy::too_many_arguments)]
288    pub async fn fetch(
289        &self,
290        root: &Path,
291        target: &Path,
292        workspace: WorkspaceId,
293        revision: u64,
294        peer: PeerId,
295        ticket: ResourceTicket,
296        progress: watch::Sender<u64>,
297    ) -> Result<()> {
298        let _slot = self
299            .0
300            .downloading
301            .try_lock()
302            .map_err(|_| Error::Backpressure)?;
303        self.authorize(workspace, revision, peer).await?;
304        if !target.is_absolute() {
305            return Err(Error::Rejected);
306        }
307        let store = self.store(root).await?;
308        let hash = Hash::from_bytes(ticket.hash);
309        store
310            .tags()
311            .set(b"partial", hash)
312            .await
313            .map_err(transport)?;
314        self.collect(&store).await?;
315        let local = store.remote().local(hash).await.map_err(transport)?;
316        progress.send_replace(local.local_bytes());
317        if !local.is_complete() {
318            let mut changes = self.0.changed.subscribe();
319            let connection =
320                tokio::time::timeout(IDLE_TIMEOUT, self.0.connections.connect(peer, ALPN))
321                    .await
322                    .map_err(|_| Error::Timeout("resource connect"))??;
323            self.authorize(workspace, revision, peer).await?;
324            let (mut send, recv) = connection.open_bi().await.map_err(transport)?;
325            send.write_all(&[STREAM_KIND]).await.map_err(transport)?;
326            send.write_all(&ticket.grant).await.map_err(transport)?;
327            let reader = SizedReader {
328                inner: recv,
329                size: Some(ticket.size),
330                header: [0; 8],
331                read: 0,
332            };
333            let pair = get::StreamPair::new(connection.stable_id() as u64, reader, send);
334            let transfer = store.remote().fetch(pair, hash);
335            let mut events = transfer.stream();
336            let idle = tokio::time::sleep(IDLE_TIMEOUT);
337            tokio::pin!(idle);
338            let result = loop {
339                tokio::select! {
340                    event = events.next() => match event {
341                        Some(GetProgressItem::Progress(bytes)) => {
342                            progress.send_replace(bytes);
343                            idle.as_mut().reset(tokio::time::Instant::now() + IDLE_TIMEOUT);
344                        }
345                        Some(GetProgressItem::Done(_)) => break Ok(()),
346                        Some(GetProgressItem::Error(error)) => break Err(transport(error)),
347                        None => break Err(Error::Rejected),
348                    },
349                    _ = &mut idle => break Err(Error::Timeout("resource progress")),
350                    result = changes.changed() => {
351                        if let Err(error) = result.map_err(transport) { break Err(error); }
352                        if let Err(error) = self.authorize(workspace, revision, peer).await { break Err(error); }
353                    }
354                }
355            };
356            // Flush verified partial ranges even when the peer disappears.
357            drop(events);
358            store.sync_db().await.map_err(transport)?;
359            result?;
360        }
361        self.authorize(workspace, revision, peer).await?;
362        store
363            .export_with_opts(ExportOptions {
364                hash,
365                target: target.to_owned(),
366                mode: ExportMode::TryReference,
367            })
368            .await
369            .map_err(transport)?;
370        if tokio::fs::metadata(target).await.map_err(transport)?.len() != ticket.size {
371            return Err(Error::Rejected);
372        }
373        store.tags().delete(b"partial").await.map_err(transport)?;
374        self.collect(&store).await?;
375        progress.send_replace(ticket.size);
376        Ok(())
377    }
378
379    /// Clear only the private resumable transfer, not retained/published files.
380    pub async fn clear_partial(&self, root: &Path) -> Result<()> {
381        let _slot = self.0.downloading.lock().await;
382        if root.exists() {
383            let store = self.store(root).await?;
384            store.tags().delete(b"partial").await.map_err(transport)?;
385            self.collect(&store).await?;
386        }
387        Ok(())
388    }
389
390    pub(crate) async fn close(&self) {
391        self.stop();
392        if let Some((_, store)) = self.0.store.get() {
393            let _ = store.shutdown().await;
394        }
395    }
396
397    pub(crate) fn stop(&self) {
398        self.0.closed.store(true, Ordering::Release);
399        self.revoke(None);
400    }
401}
402
403/// Check the standard blob size header before its decoder/storage sees it.
404/// Advertised length is a resource integrity boundary, not a file-size limit.
405struct SizedReader<R> {
406    inner: R,
407    size: Option<u64>,
408    header: [u8; 8],
409    read: usize,
410}
411
412impl<R> SizedReader<R> {
413    fn validate(&mut self, bytes: &[u8]) -> io::Result<()> {
414        let Some(size) = self.size else {
415            return Ok(());
416        };
417        let length = (8 - self.read).min(bytes.len());
418        self.header[self.read..self.read + length].copy_from_slice(&bytes[..length]);
419        self.read += length;
420        if self.read == 8 && u64::from_le_bytes(self.header) != size {
421            return Err(io::Error::new(
422                io::ErrorKind::InvalidData,
423                "resource length differs from grant",
424            ));
425        }
426        Ok(())
427    }
428}
429
430impl<R: BlobReader> BlobReader for SizedReader<R> {
431    async fn recv_bytes(&mut self, len: usize) -> io::Result<Bytes> {
432        let bytes = io_idle(self.inner.recv_bytes(len)).await?;
433        self.validate(&bytes)?;
434        Ok(bytes)
435    }
436    async fn recv_bytes_exact(&mut self, len: usize) -> io::Result<Bytes> {
437        let bytes = io_idle(self.inner.recv_bytes_exact(len)).await?;
438        self.validate(&bytes)?;
439        Ok(bytes)
440    }
441    async fn recv_exact(&mut self, target: &mut [u8]) -> io::Result<()> {
442        io_idle(self.inner.recv_exact(target)).await?;
443        self.validate(target)
444    }
445    fn stop(&mut self, code: VarInt) -> io::Result<()> {
446        BlobReader::stop(&mut self.inner, code)
447    }
448    fn id(&self) -> u64 {
449        BlobReader::id(&self.inner)
450    }
451}
452
453async fn io_idle<T>(operation: impl std::future::Future<Output = io::Result<T>>) -> io::Result<T> {
454    tokio::time::timeout(IDLE_TIMEOUT, operation)
455        .await
456        .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "resource stream stalled"))?
457}
458
459/// A peer keeping QUIC alive without reading must release scarce serve slots.
460/// This limits idle I/O, never the duration or size of a progressing transfer.
461struct IdleWriter<'a>(&'a mut SendStream);
462impl BlobWriter for IdleWriter<'_> {
463    async fn send_bytes(&mut self, bytes: Bytes) -> io::Result<()> {
464        io_idle(BlobWriter::send_bytes(self.0, bytes)).await
465    }
466    async fn send(&mut self, bytes: &[u8]) -> io::Result<()> {
467        io_idle(BlobWriter::send(self.0, bytes)).await
468    }
469    async fn sync(&mut self) -> io::Result<()> {
470        io_idle(BlobWriter::sync(self.0)).await
471    }
472    fn reset(&mut self, code: VarInt) -> io::Result<()> {
473        BlobWriter::reset(self.0, code)
474    }
475    async fn stopped(&mut self) -> io::Result<Option<VarInt>> {
476        io_idle(BlobWriter::stopped(self.0)).await
477    }
478    fn id(&self) -> u64 {
479        BlobWriter::id(self.0)
480    }
481}
482
483#[tokio::test(start_paused = true)]
484async fn idle_deadline_is_not_a_transfer_duration_limit() {
485    let error = io_idle(std::future::pending::<io::Result<()>>())
486        .await
487        .unwrap_err();
488    assert_eq!(error.kind(), io::ErrorKind::TimedOut);
489    let started = tokio::time::Instant::now();
490    for _ in 0..3 {
491        io_idle(async {
492            tokio::time::sleep(IDLE_TIMEOUT / 2).await;
493            Ok(())
494        })
495        .await
496        .unwrap();
497    }
498    assert!(started.elapsed() > IDLE_TIMEOUT);
499}