miasma 0.3.0

Trap malicious AI web scrapers in an endless pit of poisoned training data.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

use std::io;

use async_compression::{Level, tokio::bufread::GzipEncoder};
use futures::TryStreamExt;
use tokio::io::BufReader;
use tokio_util::io::{ReaderStream, StreamReader};

use crate::MiasmaStream;

const COMPRESS_BUFFER_SIZE: usize = 1024 * 4;

/// Compresses the poison stream with gzip encoding.
pub fn gzip_stream<E>(stream: impl MiasmaStream<E>) -> impl MiasmaStream<io::Error>
where
    E: Into<Box<dyn std::error::Error + Send + Sync>>,
{
    let stream = stream.map_err(io::Error::other);
    let reader = StreamReader::new(stream);
    let buf = BufReader::with_capacity(COMPRESS_BUFFER_SIZE, reader);
    let encoder = GzipEncoder::with_quality(
        buf,
        // CRANK IT !
        // we want the smallest response size possible...
        Level::Precise(i32::MAX),
    );
    ReaderStream::new(encoder)
}