gs-hf 0.4.6

Fork of hf-hub. This crates aims ease the interaction with huggingface It aims to be compatible with the huggingface_hub python package, but only implements a smaller subset of functions.
Documentation
//! A minimal in-process Hub lookalike for hermetic end-to-end tests.
//!
//! Serves just enough of the protocol the client relies on: the
//! `/{repo}/resolve/{revision}/{filename}` endpoint with `Range`
//! support, optional relative redirect hops, an optional absolute
//! "CDN" redirect (etag/commit headers on the redirect response,
//! `Content-Range` only on the CDN itself, like S3), and quoted etags.
//! Listens on loopback only, so it also runs inside the Nix sandbox.

use std::io::{BufRead, BufReader, Write};
use std::net::{SocketAddr, TcpListener, TcpStream};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::thread::JoinHandle;

pub(crate) struct Fixture {
    pub(crate) repo: &'static str,
    pub(crate) filename: &'static str,
    pub(crate) content: Vec<u8>,
    pub(crate) etag: &'static str,
    pub(crate) commit: &'static str,
    /// Number of relative redirect hops before the final Hub response.
    pub(crate) relative_redirects: usize,
    /// Whether the final Hub response redirects to an absolute CDN URL.
    pub(crate) cdn_redirect: bool,
}

pub(crate) struct BucketFixture {
    pub(crate) bucket_id: &'static str,
    /// Raw JSON array bodies served page by page by the tree endpoint,
    /// chained with `Link: rel="next"` headers.
    pub(crate) tree_pages: Vec<&'static str>,
    /// Files served by the resolve endpoint, keyed by percent-encoded path.
    pub(crate) files: Vec<(&'static str, Vec<u8>)>,
    /// Serve only half of the first CDN response body, then close the
    /// connection, so clients exercise their resume-with-Range path.
    pub(crate) truncate_first_cdn_response: AtomicBool,
}

#[derive(Debug)]
pub(crate) struct RequestLog {
    pub(crate) path: String,
    pub(crate) range: Option<String>,
    pub(crate) authorization: Option<String>,
}

enum ServerFixture {
    Repo(Fixture),
    Bucket(BucketFixture),
}

pub(crate) struct TestServer {
    addr: SocketAddr,
    requests: Arc<Mutex<Vec<RequestLog>>>,
    shutdown: Arc<AtomicBool>,
    handle: Option<JoinHandle<()>>,
}

impl TestServer {
    pub(crate) fn spawn(fixture: Fixture) -> Self {
        Self::spawn_fixture(ServerFixture::Repo(fixture))
    }

    pub(crate) fn spawn_bucket(fixture: BucketFixture) -> Self {
        Self::spawn_fixture(ServerFixture::Bucket(fixture))
    }

    fn spawn_fixture(fixture: ServerFixture) -> Self {
        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = listener.local_addr().unwrap();
        let requests = Arc::new(Mutex::new(Vec::new()));
        let shutdown = Arc::new(AtomicBool::new(false));

        let handle = {
            let requests = Arc::clone(&requests);
            let shutdown = Arc::clone(&shutdown);
            let fixture = Arc::new(fixture);
            let endpoint = format!("http://{addr}");
            std::thread::spawn(move || {
                for stream in listener.incoming() {
                    if shutdown.load(Ordering::SeqCst) {
                        break;
                    }
                    let Ok(stream) = stream else { break };
                    let requests = Arc::clone(&requests);
                    let fixture = Arc::clone(&fixture);
                    let endpoint = endpoint.clone();
                    // One thread per connection so concurrent chunk
                    // requests cannot deadlock on a sequential loop.
                    std::thread::spawn(move || {
                        handle_connection(stream, &fixture, &endpoint, &requests);
                    });
                }
            })
        };

        Self {
            addr,
            requests,
            shutdown,
            handle: Some(handle),
        }
    }

    pub(crate) fn endpoint(&self) -> String {
        format!("http://{}", self.addr)
    }

    pub(crate) fn requests(&self) -> Vec<RequestLog> {
        std::mem::take(&mut self.requests.lock().unwrap())
    }
}

impl Drop for TestServer {
    fn drop(&mut self) {
        self.shutdown.store(true, Ordering::SeqCst);
        // Wake the accept loop.
        let _ = TcpStream::connect(self.addr);
        if let Some(handle) = self.handle.take() {
            let _ = handle.join();
        }
    }
}

fn handle_connection(
    mut stream: TcpStream,
    fixture: &ServerFixture,
    endpoint: &str,
    requests: &Mutex<Vec<RequestLog>>,
) {
    let mut reader = BufReader::new(stream.try_clone().unwrap());
    let mut request_line = String::new();
    if reader.read_line(&mut request_line).is_err() || request_line.is_empty() {
        return;
    }
    let target = match request_line.split_whitespace().nth(1) {
        Some(target) => target.to_string(),
        None => return,
    };

    let mut range = None;
    let mut authorization = None;
    loop {
        let mut line = String::new();
        if reader.read_line(&mut line).is_err() {
            return;
        }
        let line = line.trim_end();
        if line.is_empty() {
            break;
        }
        if let Some((name, value)) = line.split_once(':') {
            match name.to_ascii_lowercase().as_str() {
                "range" => range = Some(value.trim().to_string()),
                "authorization" => authorization = Some(value.trim().to_string()),
                _ => {}
            }
        }
    }

    requests.lock().unwrap().push(RequestLog {
        path: target.clone(),
        range: range.clone(),
        authorization,
    });

    match fixture {
        ServerFixture::Repo(fixture) => {
            // Query parameters do not participate in repo routing.
            let path = target.split('?').next().unwrap();
            route(&mut stream, fixture, endpoint, path, range.as_deref());
        }
        ServerFixture::Bucket(fixture) => {
            route_bucket(&mut stream, fixture, endpoint, &target, range.as_deref());
        }
    }
    let _ = stream.flush();
}

fn route(
    stream: &mut TcpStream,
    fixture: &Fixture,
    endpoint: &str,
    path: &str,
    range: Option<&str>,
) {
    // Repo info requests. Like the Hub, an unknown repo is answered with
    // 401 rather than 404 so its existence is not leaked.
    if path.starts_with("/api/") {
        if path.contains(fixture.repo) {
            let body = format!(
                r#"{{"siblings":[{{"rfilename":"{}"}}],"sha":"{}"}}"#,
                fixture.filename, fixture.commit
            );
            return respond(stream, 200, &[], body.as_bytes());
        }
        return respond(
            stream,
            401,
            &[],
            br#"{"error":"Invalid username or password."}"#,
        );
    }

    // Match the resolve endpoint on repo prefix and filename suffix so
    // percent-encoded revisions (refs%2Fpr%2F5) route the same way.
    let is_resolve = path.starts_with(&format!("/{}/resolve/", fixture.repo))
        && path.ends_with(&format!("/{}", fixture.filename));

    let hop = path
        .strip_prefix("/_redirect/")
        .and_then(|hop| hop.parse::<usize>().ok());

    let is_final_hub_path = match hop {
        Some(hop) => hop >= fixture.relative_redirects,
        None => is_resolve && fixture.relative_redirects == 0,
    };

    if is_resolve && fixture.relative_redirects > 0 {
        return respond(
            stream,
            302,
            &[("location".into(), "/_redirect/1".into())],
            &[],
        );
    }
    if let Some(hop) = hop {
        if !is_final_hub_path {
            let location = format!("/_redirect/{}", hop + 1);
            return respond(stream, 302, &[("location".into(), location)], &[]);
        }
    }

    if path == "/cdn/blob" {
        return serve_range(stream, fixture, range, false);
    }

    if is_final_hub_path {
        if fixture.cdn_redirect {
            let headers = vec![
                ("location".into(), format!("{endpoint}/cdn/blob")),
                ("etag".into(), format!("\"{}\"", fixture.etag)),
                ("x-repo-commit".into(), fixture.commit.to_string()),
            ];
            return respond(stream, 302, &headers, &[]);
        }
        return serve_range(stream, fixture, range, true);
    }

    respond(stream, 404, &[], b"not found");
}

fn serve_range(stream: &mut TcpStream, fixture: &Fixture, range: Option<&str>, with_meta: bool) {
    let len = fixture.content.len();
    let (start, stop) = match range.and_then(parse_range) {
        // Clamp like S3 does: the tokio engine's last chunk may request
        // one byte past the end.
        Some((start, stop)) => (start, stop.min(len - 1)),
        None => (0, len - 1),
    };

    let mut headers = vec![(
        "content-range".to_string(),
        format!("bytes {start}-{stop}/{len}"),
    )];
    if with_meta {
        headers.push(("etag".into(), format!("\"{}\"", fixture.etag)));
        headers.push(("x-repo-commit".into(), fixture.commit.to_string()));
    }
    respond(stream, 206, &headers, &fixture.content[start..=stop]);
}

fn route_bucket(
    stream: &mut TcpStream,
    fixture: &BucketFixture,
    endpoint: &str,
    target: &str,
    range: Option<&str>,
) {
    let (path, query) = match target.split_once('?') {
        Some((path, query)) => (path, Some(query)),
        None => (target, None),
    };

    let tree = format!("/api/buckets/{}/tree", fixture.bucket_id);
    if path == tree || path.starts_with(&format!("{tree}/")) {
        // The cursor query parameter selects the page, mimicking the
        // Hub's server-generated pagination cursors.
        let page = query
            .into_iter()
            .flat_map(|query| query.split('&'))
            .find_map(|param| param.strip_prefix("cursor="))
            .map_or(0, |cursor| cursor.parse().unwrap());
        let mut headers = vec![];
        if page + 1 < fixture.tree_pages.len() {
            let next = format!("<{endpoint}{tree}?cursor={}>; rel=\"next\"", page + 1);
            headers.push(("link".to_string(), next));
        }
        return respond(stream, 200, &headers, fixture.tree_pages[page].as_bytes());
    }

    if let Some(encoded) = path.strip_prefix(&format!("/buckets/{}/resolve/", fixture.bucket_id)) {
        let location = format!("{endpoint}/cdn/bucket/{encoded}");
        return respond(stream, 302, &[("location".into(), location)], &[]);
    }

    if let Some(encoded) = path.strip_prefix("/cdn/bucket/") {
        if let Some((_, content)) = fixture.files.iter().find(|(path, _)| *path == encoded) {
            let truncate = fixture
                .truncate_first_cdn_response
                .swap(false, Ordering::SeqCst);
            return serve_bucket_range(stream, content, range, truncate);
        }
    }

    respond(stream, 404, &[], b"not found");
}

fn serve_bucket_range(stream: &mut TcpStream, content: &[u8], range: Option<&str>, truncate: bool) {
    let len = content.len();
    let (start, stop) = match range.and_then(parse_range) {
        Some((start, stop)) => (start, stop.min(len - 1)),
        None => (0, len - 1),
    };
    let headers = vec![(
        "content-range".to_string(),
        format!("bytes {start}-{stop}/{len}"),
    )];
    let body = &content[start..=stop];
    let served = if truncate {
        &body[..body.len() / 2]
    } else {
        body
    };
    respond_with_content_length(stream, 206, &headers, body.len(), served);
}

/// Parses `bytes=a-b` / `bytes=a-` range headers.
fn parse_range(range: &str) -> Option<(usize, usize)> {
    let (start, stop) = range.strip_prefix("bytes=")?.split_once('-')?;
    let start = start.parse().ok()?;
    let stop = if stop.is_empty() {
        usize::MAX
    } else {
        stop.parse().ok()?
    };
    Some((start, stop))
}

fn respond(stream: &mut TcpStream, status: u16, headers: &[(String, String)], body: &[u8]) {
    respond_with_content_length(stream, status, headers, body.len(), body);
}

/// Declares `content_length` but writes only `body`: with a shorter
/// body the client sees the connection close mid-download.
fn respond_with_content_length(
    stream: &mut TcpStream,
    status: u16,
    headers: &[(String, String)],
    content_length: usize,
    body: &[u8],
) {
    let reason = match status {
        206 => "Partial Content",
        302 => "Found",
        401 => "Unauthorized",
        404 => "Not Found",
        _ => "OK",
    };
    let mut response = format!("HTTP/1.1 {status} {reason}\r\n");
    for (name, value) in headers {
        response.push_str(&format!("{name}: {value}\r\n"));
    }
    response.push_str(&format!("content-length: {content_length}\r\n"));
    response.push_str("connection: close\r\n\r\n");
    let _ = stream.write_all(response.as_bytes());
    let _ = stream.write_all(body);
}