Skip to main content

fx_torrent/
stream.rs

1use crate::channel::{ChannelSender, Reply};
2use crate::storage::Storage;
3use crate::torrent_data::DataPool;
4use crate::{File, PiecePriority, Result, TorrentCommand, TorrentError, TorrentEvent};
5use futures::future::BoxFuture;
6use futures::task::AtomicWaker;
7use futures::{ready, FutureExt, Stream};
8use fx_callback::Subscription;
9use log::trace;
10use std::cmp::min;
11use std::fmt::Debug;
12use std::io;
13use std::ops::Range;
14use std::pin::Pin;
15use std::sync::Arc;
16use std::task::{Context, Poll};
17use tokio::select;
18use tokio_util::sync::CancellationToken;
19
20/// The internal stream buffer type.
21type Buffer = Range<usize>;
22
23/// Streams the contents of a single torrent file.
24///
25/// # Example Usage
26///
27/// ```rust
28/// # use futures::StreamExt;
29/// # use fx_torrent::Torrent;
30///
31/// # async fn example() {
32///     let torrent = Torrent::request()
33///         .build()
34///         .unwrap();
35///     let file = torrent.file(&0).await.unwrap();
36///
37///     let mut stream = torrent.stream(&file).await.unwrap();
38///     while let Some(bytes) = stream.next().await {
39///         // use the bytes here
40///     }
41/// # }
42/// ```
43#[derive(Debug)]
44pub struct FileStream {
45    file: File,
46    storage: Storage,
47    data_pool: DataPool,
48    command_sender: ChannelSender<TorrentCommand>,
49    cursor: usize,
50    stream_buffer_len: usize,
51    piece_len: usize,
52    waker: Arc<AtomicWaker>,
53    state: StreamState,
54    cancellation_token: CancellationToken,
55}
56
57impl FileStream {
58    pub(crate) fn new(
59        file: File,
60        piece_len: usize,
61        stream_buffer_len: usize,
62        storage: Storage,
63        data_pool: DataPool,
64        command_sender: ChannelSender<TorrentCommand>,
65        receiver: Subscription<TorrentEvent>,
66    ) -> Self {
67        let waker = Arc::new(AtomicWaker::new());
68        let cancellation_token = CancellationToken::new();
69
70        let event_waker = waker.clone();
71        let event_cancellation_token = cancellation_token.clone();
72        tokio::spawn(async move {
73            Self::run_event_loop(receiver, event_waker, event_cancellation_token).await;
74        });
75
76        Self {
77            file,
78            storage,
79            data_pool,
80            command_sender,
81            cursor: 0,
82            stream_buffer_len,
83            piece_len,
84            waker,
85            state: StreamState::Idle,
86            cancellation_token,
87        }
88    }
89
90    /// Returns the total number of bytes in the stream.
91    pub fn len(&self) -> usize {
92        self.file.len()
93    }
94
95    /// Reset the stream progress back to the start.
96    pub fn reset(&mut self) {
97        self.cursor = 0;
98        self.state = StreamState::Idle;
99    }
100
101    /// Seek the given offset within the stream.
102    pub fn seek(&mut self, offset: usize) -> Result<()> {
103        if offset > self.file.len() {
104            return Err(TorrentError::Io(io::Error::new(
105                io::ErrorKind::InvalidInput,
106                "offset exceeds file length",
107            )));
108        }
109
110        self.cursor = offset;
111        self.state = StreamState::Idle;
112        Ok(())
113    }
114
115    /// Returns the current byte range of the stream.
116    pub fn range(&self) -> Range<usize> {
117        self.cursor..self.file.len()
118    }
119
120    /// Returns the [Content-Range](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Content-Range)
121    /// information in bytes for the stream.
122    pub fn content_range(&self) -> String {
123        format!(
124            "bytes {}-{}/{}",
125            self.cursor,
126            self.file.len() - 1,
127            self.file.len()
128        )
129    }
130
131    fn next_buffer(&self) -> Buffer {
132        let buffer_end_byte = min(self.cursor + self.stream_buffer_len, self.file.len());
133        self.file.torrent_offset + self.cursor..self.file.torrent_offset + buffer_end_byte
134    }
135
136    fn prioritize_buffer(&self, bytes: Buffer) {
137        let command_sender = self.command_sender.clone();
138        let waker = self.waker.clone();
139
140        // execute the prioritization on a separate task
141        tokio::spawn(async move {
142            command_sender
143                .fire_and_forget(TorrentCommand::PrioritizeBytes {
144                    bytes,
145                    priority: PiecePriority::Now,
146                    response: Reply::empty(),
147                })
148                .await;
149
150            waker.wake();
151        });
152    }
153
154    async fn run_event_loop(
155        mut receiver: Subscription<TorrentEvent>,
156        waker: Arc<AtomicWaker>,
157        cancellation_token: CancellationToken,
158    ) {
159        loop {
160            select! {
161                _ = cancellation_token.cancelled() => break,
162                event = receiver.recv() => {
163                    match event {
164                        Err(_) => break,
165                        Ok(event) => {
166                            match &*event {
167                                TorrentEvent::PieceCompleted(_) | TorrentEvent::StateChanged(_) => {
168                                    waker.wake()
169                                }
170                                _ => {}
171                            }
172                        }
173                    }
174                }
175            }
176        }
177    }
178}
179
180impl Stream for FileStream {
181    type Item = Result<Vec<u8>>;
182
183    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
184        // early exit if the end of the file has been reached
185        if self.cursor >= self.file.len() {
186            return Poll::Ready(None);
187        }
188
189        self.waker.register(cx.waker());
190        loop {
191            match &mut self.state {
192                StreamState::Idle => {
193                    let buffer = self.next_buffer();
194                    let data_pool = self.data_pool.clone();
195
196                    let future = {
197                        let buffer = buffer.clone();
198                        async move { data_pool.has_bytes(buffer).await }
199                    }
200                    .boxed();
201                    trace!("Streaming buffer {:?}", buffer);
202                    self.state = StreamState::Checking {
203                        buffer,
204                        future,
205                        is_prioritized: false,
206                    };
207                }
208                StreamState::Checking {
209                    buffer,
210                    future,
211                    is_prioritized,
212                } => {
213                    let buffer = buffer.clone();
214                    let is_available = ready!(future.as_mut().poll(cx));
215
216                    if !is_available {
217                        if !*is_prioritized {
218                            let buffer = buffer.clone();
219                            self.prioritize_buffer(buffer);
220                        }
221
222                        self.state = StreamState::Waiting { buffer };
223                        return Poll::Pending;
224                    }
225
226                    let storage = self.storage.clone();
227                    let starting_piece = self.cursor / self.piece_len;
228                    let starting_offset = self.cursor % self.piece_len;
229                    let stream_buffer_len = self.stream_buffer_len;
230                    let future = async move {
231                        let mut buffer = vec![0u8; stream_buffer_len];
232
233                        match storage
234                            .read(&mut buffer, &starting_piece, starting_offset)
235                            .await
236                        {
237                            Err(e) => Err(TorrentError::Io(e)),
238                            Ok(bytes_read) => {
239                                if bytes_read != buffer.len() {
240                                    Err(TorrentError::Io(io::Error::new(
241                                        io::ErrorKind::UnexpectedEof,
242                                        format!(
243                                            "wanted {} bytes, but got {:?} instead",
244                                            stream_buffer_len, bytes_read
245                                        ),
246                                    )))
247                                } else {
248                                    Ok(buffer)
249                                }
250                            }
251                        }
252                    }
253                    .boxed();
254                    self.state = StreamState::Reading { future };
255                }
256                StreamState::Waiting { buffer } => {
257                    let buffer = buffer.clone();
258                    let data_pool = self.data_pool.clone();
259
260                    let future = {
261                        let buffer = buffer.clone();
262                        async move { data_pool.has_bytes(buffer).await }.boxed()
263                    };
264                    self.state = StreamState::Checking {
265                        buffer,
266                        future,
267                        is_prioritized: true,
268                    };
269                }
270                StreamState::Reading { future } => {
271                    let result = ready!(future.as_mut().poll(cx));
272                    if let Ok(bytes) = &result {
273                        self.cursor += bytes.len();
274                    }
275
276                    self.state = StreamState::Idle;
277                    return Poll::Ready(Some(result));
278                }
279            }
280        }
281    }
282
283    fn size_hint(&self) -> (usize, Option<usize>) {
284        if self.stream_buffer_len == 0 || self.len() == 0 {
285            return (0, Some(0));
286        }
287
288        let upper_bound = self.len().div_ceil(self.stream_buffer_len);
289        (0, Some(upper_bound))
290    }
291}
292
293impl Drop for FileStream {
294    fn drop(&mut self) {
295        self.cancellation_token.cancel();
296    }
297}
298
299enum StreamState {
300    Idle,
301    Checking {
302        buffer: Buffer,
303        future: BoxFuture<'static, bool>,
304        is_prioritized: bool,
305    },
306    Waiting {
307        buffer: Buffer,
308    },
309    Reading {
310        future: BoxFuture<'static, Result<Vec<u8>>>,
311    },
312}
313
314impl Debug for StreamState {
315    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
316        match self {
317            StreamState::Idle => f.write_str("StreamState::Idle"),
318            StreamState::Checking { .. } => f.write_str("StreamState::Checking"),
319            StreamState::Waiting { .. } => f.write_str("StreamState::Waiting"),
320            StreamState::Reading { .. } => f.write_str("StreamState::Reading"),
321        }
322    }
323}
324
325#[cfg(test)]
326mod tests {
327    use crate::operation::CreatePiecesAndFilesOperation;
328    use crate::storage::DiskStorage;
329    use crate::tests::copy_test_file;
330    use crate::tests::helpers::wait_for_torrent_pieces;
331    use futures::{Stream, StreamExt};
332    use std::time::Duration;
333    use tempfile::tempdir;
334    use tokio::time::timeout;
335
336    #[tokio::test]
337    async fn test_stream_next() {
338        init_logger!();
339        let temp_dir = tempdir().unwrap();
340        let temp_path = temp_dir.path().to_str().unwrap();
341        copy_test_file(
342            temp_path,
343            "piece-1_30.iso",
344            Some("debian-12.4.0-amd64-DVD-1.iso"),
345        );
346        let torrent = torrent!(
347            "debian-udp.torrent",
348            temp_path,
349            TorrentFlags::none(),
350            TorrentConfig::builder().build(),
351            vec![CreatePiecesAndFilesOperation::new().into(),],
352            vec![],
353            |params| DiskStorage::new(params.info_hash, params.path, params.data_pool).into(),
354            None
355        );
356        let data_pool = torrent.inner.data_pool().await.unwrap();
357        let (peer, _peer2) = tcp_peer_pair!(&torrent, vec![]);
358        torrent.inner.peer_connected(peer.clone().into()).await;
359
360        // wait for the pieces to be created
361        wait_for_torrent_pieces(&torrent).await;
362
363        // create the stream
364        let file = torrent.file(&0).await.unwrap();
365        let mut stream = torrent.stream(&file).await.unwrap();
366
367        // set piece 0 as completed
368        mark_piece_completed!(&stream.command_sender, 0, peer.addr(), "piece-1_30.iso");
369        assert_timeout!(
370            Duration::from_secs(2), // TODO: improve test performance
371            data_pool.is_piece_completed(&0).await
372        );
373
374        // get the next buffer, which should complete instantly
375        let result =
376            timeout!(Duration::from_millis(500), stream.next()).expect("expected buffer data");
377        match result {
378            Ok(_) => {
379                assert_eq!(
380                    stream.stream_buffer_len, stream.cursor,
381                    "expected the stream cursor to be updated"
382                );
383            }
384            _ => assert!(false, "expected Ok(), but got {:?}", result),
385        }
386
387        // try to get the next buffer, which should be blocked
388        let command_sender = stream.command_sender.clone();
389        let mut future = stream.next();
390        let result = timeout(Duration::from_millis(250), &mut future).await;
391        match result {
392            Err(_) => {}
393            _ => assert!(false, "expected Err(Elapsed), but got {:?}", result),
394        }
395
396        // complete the piece
397        mark_piece_completed!(&command_sender, 1, peer.addr(), "piece-1_30.iso");
398        assert_timeout!(
399            Duration::from_secs(2), // TODO: improve test performance
400            data_pool.is_piece_completed(&1).await
401        );
402
403        // try to complete the previous future instantly
404        let result = timeout!(Duration::from_millis(500), future).unwrap();
405        match result {
406            Ok(_) => {}
407            _ => assert!(false, "expected Ok(), but got {:?}", result),
408        }
409    }
410
411    #[tokio::test]
412    async fn test_len() {
413        init_logger!();
414        let temp_dir = tempdir().unwrap();
415        let temp_path = temp_dir.path().to_str().unwrap();
416        copy_test_file(
417            temp_path,
418            "piece-1_30.iso",
419            Some("debian-12.4.0-amd64-DVD-1.iso"),
420        );
421        let torrent = torrent!(
422            "debian-udp.torrent",
423            temp_path,
424            TorrentFlags::none(),
425            TorrentConfig::builder().build(),
426            vec![CreatePiecesAndFilesOperation::new().into(),],
427            vec![],
428            |params| DiskStorage::new(params.info_hash, params.path, params.data_pool).into(),
429            None
430        );
431
432        // wait for the pieces to be created
433        wait_for_torrent_pieces(&torrent).await;
434
435        // create the stream
436        let file = torrent.file(&0).await.unwrap();
437        let stream = torrent.stream(&file).await.unwrap();
438
439        let result = stream.len();
440        assert_eq!(file.len(), result, "expected the file length to match");
441    }
442
443    #[tokio::test]
444    async fn test_reset() {
445        init_logger!();
446        let temp_dir = tempdir().unwrap();
447        let temp_path = temp_dir.path().to_str().unwrap();
448        copy_test_file(
449            temp_path,
450            "piece-1_30.iso",
451            Some("debian-12.4.0-amd64-DVD-1.iso"),
452        );
453        let torrent = torrent!(
454            "debian-udp.torrent",
455            temp_path,
456            TorrentFlags::none(),
457            TorrentConfig::builder().build(),
458            vec![CreatePiecesAndFilesOperation::new().into(),],
459            vec![],
460            |params| DiskStorage::new(params.info_hash, params.path, params.data_pool).into(),
461            None
462        );
463        let data_pool = torrent.inner.data_pool().await.unwrap();
464        let (peer, _peer2) = tcp_peer_pair!(&torrent, vec![]);
465        torrent.inner.peer_connected(peer.clone().into()).await;
466
467        // wait for the pieces to be created
468        wait_for_torrent_pieces(&torrent).await;
469
470        // create the stream
471        let file = torrent.file(&0).await.unwrap();
472        let mut stream = torrent.stream(&file).await.unwrap();
473
474        // set the next piece as completed
475        mark_piece_completed!(&stream.command_sender, 0, peer.addr(), "piece-1_30.iso");
476        assert_timeout!(
477            Duration::from_secs(2), // TODO: improve test performance
478            data_pool.is_piece_completed(&0).await
479        );
480
481        // read the next stream buffer
482        let result = timeout!(Duration::from_millis(750), stream.next());
483        match result {
484            Some(Ok(_)) => {
485                assert_eq!(
486                    stream.stream_buffer_len, stream.cursor,
487                    "expected the stream cursor to be updated"
488                );
489            }
490            _ => assert!(false, "expected Some(Ok()), but got {:?}", result),
491        }
492
493        // reset the stream back to the start
494        stream.reset();
495        assert_eq!(0, stream.cursor, "expected the cursor to have been reset");
496    }
497
498    #[tokio::test]
499    async fn test_seek() {
500        init_logger!();
501        let temp_dir = tempdir().unwrap();
502        let temp_path = temp_dir.path().to_str().unwrap();
503        copy_test_file(
504            temp_path,
505            "piece-1_30.iso",
506            Some("debian-12.4.0-amd64-DVD-1.iso"),
507        );
508        let torrent = torrent!(
509            "debian-udp.torrent",
510            temp_path,
511            TorrentFlags::none(),
512            TorrentConfig::builder().build(),
513            vec![CreatePiecesAndFilesOperation::new().into(),],
514            vec![],
515            |params| DiskStorage::new(params.info_hash, params.path, params.data_pool).into(),
516            None
517        );
518        let data_pool = torrent.inner.data_pool().await.unwrap();
519        let (peer, _peer2) = tcp_peer_pair!(&torrent, vec![]);
520        torrent.inner.peer_connected(peer.clone().into()).await;
521
522        // wait for the pieces to be created
523        wait_for_torrent_pieces(&torrent).await;
524
525        // create the stream
526        let file = torrent.file(&0).await.unwrap();
527        let mut stream = torrent.stream(&file).await.unwrap();
528
529        // set the next piece as completed
530        mark_piece_completed!(&stream.command_sender, 0, peer.addr(), "piece-1_30.iso");
531        assert_timeout!(
532            Duration::from_secs(2), // TODO: improve test performance
533            data_pool.is_piece_completed(&0).await
534        );
535
536        // read the next stream buffer
537        let result = timeout!(Duration::from_millis(750), stream.next());
538        match result {
539            Some(Ok(_)) => {
540                assert_eq!(
541                    stream.stream_buffer_len, stream.cursor,
542                    "expected the stream cursor to be updated"
543                );
544            }
545            _ => assert!(false, "expected Some(Ok()), but got {:?}", result),
546        }
547
548        // seek an offset within the stream
549        stream.seek(stream.stream_buffer_len * 3).unwrap();
550        assert_eq!(
551            stream.stream_buffer_len * 3,
552            stream.cursor,
553            "expected the cursor to have been reset"
554        );
555    }
556
557    #[tokio::test]
558    async fn test_range() {
559        init_logger!();
560        let temp_dir = tempdir().unwrap();
561        let temp_path = temp_dir.path().to_str().unwrap();
562        copy_test_file(
563            temp_path,
564            "piece-1_30.iso",
565            Some("debian-12.4.0-amd64-DVD-1.iso"),
566        );
567        let torrent = torrent!(
568            "debian-udp.torrent",
569            temp_path,
570            TorrentFlags::none(),
571            TorrentConfig::builder().build(),
572            vec![CreatePiecesAndFilesOperation::new().into(),],
573            vec![],
574            |params| DiskStorage::new(params.info_hash, params.path, params.data_pool).into(),
575            None
576        );
577
578        // wait for the pieces to be created
579        wait_for_torrent_pieces(&torrent).await;
580
581        // create the stream
582        let file = torrent.file(&0).await.unwrap();
583        let mut stream = torrent.stream(&file).await.unwrap();
584
585        // seek an offset within the stream
586        let offset = stream.stream_buffer_len * 5;
587        stream.seek(offset).unwrap();
588        let result = stream.range();
589        assert_eq!(
590            offset..file.len(),
591            result,
592            "expected the stream range to match"
593        );
594
595        // reset the stream
596        stream.reset();
597        let result = stream.range();
598        assert_eq!(0..file.len(), result, "expected the stream range to match");
599    }
600
601    #[tokio::test]
602    async fn test_content_range() {
603        init_logger!();
604        let temp_dir = tempdir().unwrap();
605        let temp_path = temp_dir.path().to_str().unwrap();
606        copy_test_file(
607            temp_path,
608            "piece-1_30.iso",
609            Some("debian-12.4.0-amd64-DVD-1.iso"),
610        );
611        let torrent = torrent!(
612            "debian-udp.torrent",
613            temp_path,
614            TorrentFlags::none(),
615            TorrentConfig::builder().build(),
616            vec![CreatePiecesAndFilesOperation::new().into(),],
617            vec![],
618            |params| DiskStorage::new(params.info_hash, params.path, params.data_pool).into(),
619            None
620        );
621
622        // wait for the pieces to be created
623        wait_for_torrent_pieces(&torrent).await;
624
625        // create the stream
626        let file = torrent.file(&0).await.unwrap();
627        let mut stream = torrent.stream(&file).await.unwrap();
628
629        // seek an offset within the stream
630        let offset = stream.stream_buffer_len * 9;
631        stream.seek(offset).unwrap();
632        let result = stream.content_range();
633        assert_eq!(
634            format!("bytes {}-{}/{}", offset, file.len() - 1, file.len()),
635            result,
636            "expected the stream range to match"
637        );
638
639        // reset the stream
640        stream.reset();
641        let result = stream.content_range();
642        assert_eq!(
643            format!("bytes 0-{}/{}", file.len() - 1, file.len()),
644            result,
645            "expected the stream range to match"
646        );
647    }
648
649    #[tokio::test]
650    async fn test_size_hint() {
651        init_logger!();
652        let temp_dir = tempdir().unwrap();
653        let temp_path = temp_dir.path().to_str().unwrap();
654        copy_test_file(
655            temp_path,
656            "piece-1_30.iso",
657            Some("debian-12.4.0-amd64-DVD-1.iso"),
658        );
659        let torrent = torrent!(
660            "debian-udp.torrent",
661            temp_path,
662            TorrentFlags::none(),
663            TorrentConfig::builder().build(),
664            vec![CreatePiecesAndFilesOperation::new().into(),],
665            vec![],
666            |params| DiskStorage::new(params.info_hash, params.path, params.data_pool).into(),
667            None
668        );
669
670        // wait for the pieces to be created
671        wait_for_torrent_pieces(&torrent).await;
672
673        // create the stream
674        let file = torrent.file(&0).await.unwrap();
675        let stream = torrent.stream(&file).await.unwrap();
676
677        let result = stream.size_hint();
678        assert_eq!((0, Some(15237)), result, "expected the size hint to match");
679    }
680}