Skip to main content

shardline_hub_api/
lib.rs

1#![deny(unsafe_code)]
2#![cfg_attr(
3    test,
4    allow(
5        clippy::unwrap_used,
6        clippy::expect_used,
7        clippy::indexing_slicing,
8        clippy::shadow_unrelated,
9        clippy::let_underscore_must_use,
10        clippy::format_push_string,
11        clippy::vec_init_then_push,
12        clippy::useless_format,
13        clippy::same_item_push,
14        clippy::useless_vec,
15        clippy::str_to_string
16    )
17)]
18
19//! HuggingFace Hub API compatibility layer for Shardline.
20//!
21//! This crate provides an Axum-based HTTP API that makes Shardline a drop-in
22//! HuggingFace Hub alternative. Users can point `huggingface-cli` at a Shardline
23//! server and upload/download models as if it were the real Hub.
24//!
25//! # Quick start
26//!
27//! The commit-parsing helpers are pure and are the easiest place to start:
28//! they turn the Hub's NDJSON commit stream into typed instructions that the
29//! rest of the API applies to storage.
30//!
31//! ```
32//! use shardline_hub_api::commit::parse_ndjson_commit;
33//! use shardline_hub_api::commit::CommitInstruction;
34//! use base64::Engine;
35//!
36//! let content = base64::engine::general_purpose::STANDARD.encode(b"hello world");
37//! let body = format!(
38//!     "{{\"header\":{{\"summary\":\"add readme\",\"parentCommit\":\"\"}}}}\n\
39//!      {{\"file\":{{\"path\":\"README.md\",\"content\":\"{content}\"}}}}"
40//! );
41//!
42//! let commit = parse_ndjson_commit(&body)?;
43//! assert_eq!(commit.message, "add readme");
44//! assert!(matches!(
45//!     &commit.instructions[0],
46//!     CommitInstruction::InlineFile { path, content } if path == "README.md" && content == b"hello world"
47//! ));
48//! # Ok::<(), Box<dyn std::error::Error>>(())
49//! ```
50//!
51//! To serve the API itself, build a [`HubState`](routes::HubState) backed by a
52//! Shardline index store and object store, then mount [`hub_routes`] into your
53//! Axum application:
54//!
55//! ```no_run
56//! use axum::Router;
57//! use shardline_hub_api::hub_routes;
58//! use shardline_hub_api::routes::HubState;
59//!
60//! # fn example(state: HubState) {
61//! let app: Router = hub_routes(state, true);
62//! # }
63//! ```
64
65pub mod auth;
66pub mod commit;
67pub mod error;
68pub mod git;
69pub mod models;
70mod object_locations;
71mod parquet_preview;
72pub mod query;
73pub mod resolve;
74mod revision_access;
75mod router;
76pub mod routes;
77pub mod secrets;
78pub mod state;
79pub mod types;
80
81pub use router::{hub_routes, hub_routes_with_dataset_query};