miasma 0.1.13

Trap malicious AI web scrapers in an endless pit of poisoned training data.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

use bytes::Bytes;
use futures::Stream;
use reqwest::{Client, Error};
use std::{sync::LazyLock, time::Duration};
use url::Url;

static CLIENT: LazyLock<Client> = LazyLock::new(|| {
    reqwest::Client::builder()
        .gzip(true) // Poison Fountain serves gzipped data
        .timeout(Duration::from_secs(5))
        .user_agent(concat!(
            env!("CARGO_PKG_NAME"),
            "/",
            env!("CARGO_PKG_VERSION")
        ))
        .build()
        .expect("should be able to build client")
});

/// Fetch poisoned training data.
pub async fn stream_poison(
    poison_source: &Url,
) -> Result<impl Stream<Item = Result<Bytes, Error>>, Error> {
    let stream = CLIENT
        .get(poison_source.clone())
        .send()
        .await?
        .error_for_status()?
        .bytes_stream();

    // TODO: escape HTML sequences in the poison string
    // It's possible that the poison source will send JavaScript within `<script>` tags, which will
    // execute in browsers.
    // Probably a very niche case we shouldn't worry about, but worth documenting...
    Ok(stream)
}