miasma 0.1.16

Trap malicious AI web scrapers in an endless pit of poisoned training data.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

use std::io;

use async_compression::{Level, tokio::bufread::GzipEncoder};
use bytes::Bytes;
use futures::{Stream, TryStreamExt};
use tokio::io::BufReader;
use tokio_util::io::{ReaderStream, StreamReader};

const COMPRESS_BUFFER_SIZE: usize = 1024 * 4;

/// Compresses the poison stream with gzip encoding.
pub fn gzip_stream(
    stream: impl Stream<Item = Result<Bytes, reqwest::Error>>,
) -> impl Stream<Item = Result<Bytes, io::Error>> {
    let stream = stream.map_err(io::Error::other);
    let reader = StreamReader::new(stream);
    let buf = BufReader::with_capacity(COMPRESS_BUFFER_SIZE, reader);
    let encoder = GzipEncoder::with_quality(buf, Level::Fastest);
    ReaderStream::new(encoder)
}