use std::collections::{BTreeMap, BTreeSet};
use std::error::Error as StdError;
use std::fmt;
use std::fs::{self, File};
use std::io::{self, Read, Write};
use std::path::{Component, Path, PathBuf};
use std::str::FromStr;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use cap_std::ambient_authority;
use cap_std::fs::Dir as CapDir;
use md5::{Digest, Md5};
use reqwest::blocking::{Client, Response};
use reqwest::redirect::Policy;
use serde::{Deserialize, Serialize};
use serde_json::Value;
const POLY_HAVEN_API: &str = "https://api.polyhaven.com";
const AMBIENTCG_API: &str = "https://ambientcg.com/api/v3";
const SKETCHFAB_API: &str = "https://api.sketchfab.com/v3";
const GAZEBO_FUEL_API: &str = "https://fuel.gazebosim.org/1.0";
const GOOGLE_RESEARCH_OWNER: &str = "GoogleResearch";
const GOOGLE_SCANNED_COLLECTION: &str = "Scanned Objects by Google Research";
const FUEL_PAGE_SIZE: usize = 20;
const MAX_FUEL_PAGES: usize = 100;
const DEFAULT_DISCOVERY_LIMIT: usize = 20;
const MAX_DISCOVERY_LIMIT: usize = 100;
const MAX_REDIRECTS: usize = 5;
const DEFAULT_MAX_DOWNLOAD_BYTES: u64 = 1024 * 1024 * 1024;
const DOWNLOAD_PROGRESS_GRANULARITY: u64 = 1024 * 1024;
const MAX_API_RESPONSE_BYTES: u64 = 16 * 1024 * 1024;
const MAX_ERROR_RESPONSE_BYTES: u64 = 8 * 1024;
const MAX_MANIFEST_BYTES: u64 = 4 * 1024 * 1024;
const MAX_BUNDLE_FILES: usize = 128;
const MAX_BUNDLE_INCLUDE_DEPTH: usize = 8;
const MAX_BUNDLE_PATH_BYTES: usize = 512;
const MAX_BUNDLE_PATH_COMPONENTS: usize = 16;
const MANIFEST_NAME: &str = "assetforge-asset.json";
pub const POLY_HAVEN_LICENSE_URL: &str = "https://polyhaven.com/license";
pub const POLY_HAVEN_API_TERMS_URL: &str =
"https://github.com/Poly-Haven/Public-API/blob/master/ToS.md";
pub const AMBIENTCG_LICENSE_URL: &str = "https://docs.ambientcg.com/license/";
pub const AMBIENTCG_API_TERMS_URL: &str = "https://docs.ambientcg.com/api/";
pub const SKETCHFAB_API_TERMS_URL: &str = "https://sketchfab.com/developers/terms";
pub const SKETCHFAB_CC0_LICENSE_URL: &str = "https://creativecommons.org/publicdomain/zero/1.0/";
pub const GOOGLE_SCANNED_OBJECTS_LICENSE_URL: &str = "https://creativecommons.org/licenses/by/4.0/";
pub const GAZEBO_FUEL_API_TERMS_URL: &str = "https://gazebosim.org/api/fuel_tools/10/rest.html";
pub type Result<T> = std::result::Result<T, AssetForgeError>;
#[derive(Debug, thiserror::Error)]
pub enum AssetForgeError {
#[error("invalid asset target '{0}'")]
InvalidTarget(String),
#[error("invalid asset option '{0}'")]
InvalidOption(String),
#[error("{provider} downloads require {environment_variable} or an API token")]
AuthenticationRequired {
provider: &'static str,
environment_variable: &'static str,
},
#[error("{provider} API returned {status}: {body}")]
ProviderApi {
provider: &'static str,
status: u16,
body: String,
},
#[error("{provider} asset '{asset}' was not found")]
NotFound {
provider: &'static str,
asset: String,
},
#[error("{provider} asset '{asset}' has no {resolution} {format} bundle")]
BundleUnavailable {
provider: &'static str,
asset: String,
format: AssetFormat,
resolution: AssetResolution,
},
#[error("refusing untrusted asset download URL '{0}'")]
UntrustedDownloadUrl(String),
#[error("refusing unsafe asset file path '{0}'")]
UnsafePath(String),
#[error("asset bundle is {size} bytes, exceeding the {limit} byte limit")]
DownloadLimitExceeded {
size: u64,
limit: u64,
},
#[error("{label} is {size} bytes, exceeding the {limit} byte limit")]
ResponseLimitExceeded {
label: String,
size: u64,
limit: u64,
},
#[error("download verification failed for '{path}': {message}")]
VerificationFailed {
path: String,
message: String,
},
#[error("refusing to replace unrecognized output '{}': {reason}", path.display())]
UnrecognizedExistingOutput {
path: PathBuf,
reason: String,
},
#[error("asset '{asset}' belongs to {actual}, not {expected}")]
ProviderMismatch {
asset: String,
expected: AssetProvider,
actual: AssetProvider,
},
#[error("output path '{}' is not a directory", .0.display())]
OutputNotDirectory(PathBuf),
#[error("I/O error: {0}")]
Io(#[from] io::Error),
#[error("request error: {0}")]
Request(String),
#[error("JSON error: {0}")]
Json(#[from] serde_json::Error),
}
impl From<reqwest::Error> for AssetForgeError {
fn from(error: reqwest::Error) -> Self {
Self::Request(error_chain(&error))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum AssetProvider {
#[serde(rename = "polyhaven")]
PolyHaven,
#[serde(rename = "ambientcg")]
AmbientCG,
#[serde(rename = "sketchfab")]
Sketchfab,
#[serde(rename = "google-scanned-objects")]
GoogleScannedObjects,
}
impl AssetProvider {
pub const ALL: [Self; 4] = [
Self::PolyHaven,
Self::AmbientCG,
Self::Sketchfab,
Self::GoogleScannedObjects,
];
pub fn all() -> &'static [Self] {
&Self::ALL
}
pub fn slug(self) -> &'static str {
match self {
Self::PolyHaven => "polyhaven",
Self::AmbientCG => "ambientcg",
Self::Sketchfab => "sketchfab",
Self::GoogleScannedObjects => "google-scanned-objects",
}
}
pub fn label(self) -> &'static str {
match self {
Self::PolyHaven => "Poly Haven",
Self::AmbientCG => "ambientCG",
Self::Sketchfab => "Sketchfab CC0",
Self::GoogleScannedObjects => "Google Scanned Objects",
}
}
pub fn short_label(self) -> &'static str {
match self {
Self::PolyHaven => "PH",
Self::AmbientCG => "ACG",
Self::Sketchfab => "SF",
Self::GoogleScannedObjects => "GSO",
}
}
pub fn has_md5_verification(self) -> bool {
matches!(self, Self::PolyHaven)
}
pub fn from_slug(value: &str) -> Option<Self> {
match normalized_slug(value).as_str() {
"polyhaven" | "ph" => Some(Self::PolyHaven),
"ambientcg" | "acg" | "ambient" => Some(Self::AmbientCG),
"sketchfab" | "sf" => Some(Self::Sketchfab),
"googlescannedobjects" | "gso" | "googleobjects" => Some(Self::GoogleScannedObjects),
_ => None,
}
}
}
impl fmt::Display for AssetProvider {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.slug())
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum AssetTarget {
Asset(String),
Search(String),
Category(String),
Top,
Latest,
}
impl AssetTarget {
pub fn is_collection(&self) -> bool {
!matches!(self, Self::Asset(_))
}
}
pub fn parse_asset_target(input: &str) -> Result<AssetTarget> {
parse_asset_target_for_provider(input, AssetProvider::PolyHaven)
}
pub fn parse_asset_target_for_provider(
input: &str,
provider: AssetProvider,
) -> Result<AssetTarget> {
let normalized = input.trim();
if normalized.is_empty() {
return Err(AssetForgeError::InvalidTarget("empty target".to_string()));
}
match normalized.to_ascii_lowercase().as_str() {
"top" | "popular" | "trending" => return Ok(AssetTarget::Top),
"latest" | "newest" | "new" => return Ok(AssetTarget::Latest),
_ => {}
}
for (prefix, constructor) in [
("asset:", AssetTarget::Asset as fn(String) -> AssetTarget),
("search:", AssetTarget::Search),
("category:", AssetTarget::Category),
] {
if normalized
.get(..prefix.len())
.is_some_and(|head| head.eq_ignore_ascii_case(prefix))
{
return non_empty_target(&normalized[prefix.len()..], constructor);
}
}
if normalized.contains("://") {
return parse_provider_url(normalized, provider);
}
if normalized.chars().any(char::is_whitespace) {
return Ok(AssetTarget::Search(normalized.to_string()));
}
Ok(AssetTarget::Asset(normalized.to_string()))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum AssetFormat {
Auto,
Gltf,
Fbx,
Blend,
Usd,
Archive,
}
impl AssetFormat {
fn api_key(self) -> &'static str {
match self {
Self::Auto => "auto",
Self::Gltf => "gltf",
Self::Fbx => "fbx",
Self::Blend => "blend",
Self::Usd => "usd",
Self::Archive => "archive",
}
}
}
impl fmt::Display for AssetFormat {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.api_key())
}
}
impl FromStr for AssetFormat {
type Err = AssetForgeError;
fn from_str(value: &str) -> Result<Self> {
match normalized_slug(value).as_str() {
"auto" | "default" => Ok(Self::Auto),
"gltf" => Ok(Self::Gltf),
"fbx" => Ok(Self::Fbx),
"blend" | "blender" => Ok(Self::Blend),
"usd" | "usdc" => Ok(Self::Usd),
"archive" | "zip" | "package" => Ok(Self::Archive),
_ => Err(AssetForgeError::InvalidOption(value.to_string())),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum AssetResolution {
Auto,
Original,
#[serde(rename = "1k")]
OneK,
#[serde(rename = "2k")]
TwoK,
#[serde(rename = "4k")]
FourK,
#[serde(rename = "8k")]
EightK,
}
impl AssetResolution {
pub fn slug(self) -> &'static str {
match self {
Self::Auto => "auto",
Self::Original => "original",
Self::OneK => "1k",
Self::TwoK => "2k",
Self::FourK => "4k",
Self::EightK => "8k",
}
}
}
impl fmt::Display for AssetResolution {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.slug())
}
}
impl FromStr for AssetResolution {
type Err = AssetForgeError;
fn from_str(value: &str) -> Result<Self> {
match normalized_slug(value).as_str() {
"auto" | "default" => Ok(Self::Auto),
"original" | "source" => Ok(Self::Original),
"1k" | "1024" => Ok(Self::OneK),
"2k" | "2048" => Ok(Self::TwoK),
"4k" | "4096" => Ok(Self::FourK),
"8k" | "8192" => Ok(Self::EightK),
_ => Err(AssetForgeError::InvalidOption(value.to_string())),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct AssetRights {
pub license: String,
pub license_url: String,
pub attribution: String,
pub api_terms_url: String,
}
impl AssetRights {
fn poly_haven() -> Self {
Self {
license: "CC0-1.0".to_string(),
license_url: POLY_HAVEN_LICENSE_URL.to_string(),
attribution: "Poly Haven".to_string(),
api_terms_url: POLY_HAVEN_API_TERMS_URL.to_string(),
}
}
fn ambientcg() -> Self {
Self {
license: "CC0-1.0".to_string(),
license_url: AMBIENTCG_LICENSE_URL.to_string(),
attribution: "ambientCG".to_string(),
api_terms_url: AMBIENTCG_API_TERMS_URL.to_string(),
}
}
fn sketchfab(creator: &str) -> Self {
Self {
license: "CC0-1.0".to_string(),
license_url: SKETCHFAB_CC0_LICENSE_URL.to_string(),
attribution: format!("{creator} via Sketchfab"),
api_terms_url: SKETCHFAB_API_TERMS_URL.to_string(),
}
}
fn google_scanned_objects() -> Self {
Self {
license: "CC-BY-4.0".to_string(),
license_url: GOOGLE_SCANNED_OBJECTS_LICENSE_URL.to_string(),
attribution: "Google Research via Gazebo Fuel".to_string(),
api_terms_url: GAZEBO_FUEL_API_TERMS_URL.to_string(),
}
}
fn for_provider(provider: AssetProvider) -> Self {
match provider {
AssetProvider::PolyHaven => Self::poly_haven(),
AssetProvider::AmbientCG => Self::ambientcg(),
AssetProvider::Sketchfab => Self::sketchfab("Sketchfab creator"),
AssetProvider::GoogleScannedObjects => Self::google_scanned_objects(),
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct AssetSummary {
pub provider: AssetProvider,
pub id: String,
pub name: String,
pub description: String,
pub categories: Vec<String>,
pub tags: Vec<String>,
pub download_count: u64,
pub polycount: u64,
pub date_published: u64,
pub thumbnail_url: String,
pub source_url: String,
pub rights: AssetRights,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct AssetFile {
pub path: String,
pub size: u64,
pub url: String,
pub md5: String,
pub role: AssetFileRole,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum AssetFileRole {
Scene,
Dependency,
Package,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct AssetBundle {
pub provider: AssetProvider,
pub asset_id: String,
pub source_url: String,
pub rights: AssetRights,
pub format: AssetFormat,
pub resolution: AssetResolution,
pub files: Vec<AssetFile>,
pub total_bytes: u64,
}
#[derive(Debug, Clone)]
pub struct ArchiveOptions {
pub output: PathBuf,
pub format: AssetFormat,
pub resolution: AssetResolution,
pub skip_existing: bool,
pub max_download_bytes: u64,
}
impl Default for ArchiveOptions {
fn default() -> Self {
Self {
output: PathBuf::from("assets"),
format: AssetFormat::Auto,
resolution: AssetResolution::Auto,
skip_existing: false,
max_download_bytes: DEFAULT_MAX_DOWNLOAD_BYTES,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArchivedAsset {
pub asset_id: String,
pub output: PathBuf,
pub files_archived: usize,
pub bytes_written: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArchiveFailure {
pub asset_id: String,
pub message: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArchiveSummary {
pub selected: usize,
pub archived: usize,
pub skipped: usize,
pub files_archived: usize,
pub bytes_written: u64,
pub assets: Vec<ArchivedAsset>,
pub failed: Vec<ArchiveFailure>,
}
impl ArchiveSummary {
pub fn is_success(&self) -> bool {
self.failed.is_empty()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ArchiveProgressPhase {
Starting,
Downloading,
Completed,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ArchiveProgress {
pub phase: ArchiveProgressPhase,
pub total_assets: usize,
pub completed_assets: usize,
pub succeeded_assets: usize,
pub failed_assets: usize,
pub bytes_written: u64,
pub active_asset: Option<String>,
pub active_file: Option<String>,
pub active_bytes_written: u64,
pub active_total_bytes: Option<u64>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct AssetManifest {
pub format_version: String,
pub provider: AssetProvider,
pub asset: AssetSummary,
pub bundle: AssetBundle,
#[serde(default)]
pub provider_revision: String,
pub bytes_written: u64,
#[serde(default)]
pub file_blake3: BTreeMap<String, String>,
pub complete: bool,
}
#[derive(Clone)]
pub struct AssetForge {
provider: AssetProvider,
client: Client,
discovery_limit: usize,
api_token: Option<String>,
}
impl AssetForge {
pub fn with_user_agent(provider: AssetProvider, user_agent: &str) -> Result<Self> {
let user_agent = user_agent.trim();
if user_agent.is_empty() {
return Err(AssetForgeError::InvalidOption(
"User-Agent must not be empty".to_string(),
));
}
let client = Client::builder()
.user_agent(user_agent)
.redirect(Policy::none())
.timeout(Duration::from_secs(60))
.build()?;
Ok(Self {
provider,
client,
discovery_limit: DEFAULT_DISCOVERY_LIMIT,
api_token: None,
})
}
pub fn with_api_token(mut self, token: impl Into<String>) -> Self {
self.api_token = non_empty_string(token.into());
self
}
pub fn with_discovery_limit(mut self, limit: usize) -> Self {
self.discovery_limit = limit.clamp(1, MAX_DISCOVERY_LIMIT);
self
}
pub fn discover(&self, target: &AssetTarget) -> Result<Vec<AssetSummary>> {
match self.provider {
AssetProvider::PolyHaven => {
let catalog = self.poly_haven_catalog()?;
discover_catalog(catalog, target, self.discovery_limit)
}
AssetProvider::AmbientCG => self.ambientcg_discover(target),
AssetProvider::Sketchfab => self.sketchfab_discover(target),
AssetProvider::GoogleScannedObjects => self.google_scanned_discover(target),
}
}
pub fn bundle(
&self,
asset_id: &str,
format: AssetFormat,
resolution: AssetResolution,
) -> Result<AssetBundle> {
match self.provider {
AssetProvider::PolyHaven => {
let url = poly_haven_api_url("files", asset_id)?;
let value: Value = self.get_json(url.as_str())?;
bundle_from_value(asset_id, format, resolution, &value)
}
AssetProvider::AmbientCG => self.ambientcg_bundle(asset_id, format, resolution),
AssetProvider::Sketchfab => self.sketchfab_bundle(asset_id, format, resolution),
AssetProvider::GoogleScannedObjects => {
self.google_scanned_bundle(asset_id, format, resolution)
}
}
}
pub fn archive_assets(
&self,
assets: &[AssetSummary],
options: &ArchiveOptions,
) -> Result<ArchiveSummary> {
self.archive_assets_inner(assets, options, None)
}
pub fn archive_assets_with_progress<F>(
&self,
assets: &[AssetSummary],
options: &ArchiveOptions,
mut progress: F,
) -> Result<ArchiveSummary>
where
F: FnMut(ArchiveProgress),
{
self.archive_assets_inner(assets, options, Some(&mut progress))
}
fn archive_assets_inner(
&self,
assets: &[AssetSummary],
options: &ArchiveOptions,
mut progress: Option<&mut dyn FnMut(ArchiveProgress)>,
) -> Result<ArchiveSummary> {
prepare_output(&options.output)?;
let mut summary = ArchiveSummary {
selected: assets.len(),
..ArchiveSummary::default()
};
emit_progress(
&mut progress,
progress_snapshot(ArchiveProgressPhase::Starting, assets.len(), &summary),
);
for asset in assets {
let retained_bytes = summary.bytes_written;
let result = (|| -> Result<Option<ArchivedAsset>> {
if asset.provider != self.provider {
return Err(AssetForgeError::ProviderMismatch {
asset: asset.id.clone(),
expected: self.provider,
actual: asset.provider,
});
}
let asset_dir = options.output.join(asset_directory_name(&asset.id)?);
if options.skip_existing && asset_dir.join(MANIFEST_NAME).exists() {
let current_bundle =
self.bundle(&asset.id, options.format, options.resolution)?;
let current_revision = self.provider_revision(&asset.id)?;
if matching_complete_manifest(
&asset_dir,
self.provider,
&asset.id,
¤t_bundle,
¤t_revision,
)? {
return Ok(None);
}
}
ensure_replaceable_output(&asset_dir, self.provider, &asset.id)?;
let staging_dir = staging_path(&options.output, &asset.id)?;
fs::create_dir(&staging_dir)?;
let archive_result = (|| -> Result<ArchivedAsset> {
let (authoritative_asset, provider_revision) =
self.fetch_authoritative_asset(&asset.id)?;
let mut bundle = self.bundle(&asset.id, options.format, options.resolution)?;
if options.max_download_bytes > 0
&& bundle.total_bytes > options.max_download_bytes
{
return Err(AssetForgeError::DownloadLimitExceeded {
size: bundle.total_bytes,
limit: options.max_download_bytes,
});
}
let mut asset_bytes = 0u64;
let mut file_blake3 = BTreeMap::new();
for file in &mut bundle.files {
let output = staging_dir.join(path_from_manifest(&file.path)?);
let prior_asset_bytes = asset_bytes;
let active_asset = asset.id.clone();
let active_file = file.path.clone();
let advertised_size = (file.size > 0).then_some(file.size);
let remaining_limit = (options.max_download_bytes > 0)
.then_some(options.max_download_bytes.saturating_sub(asset_bytes));
let downloaded = download_file(
&self.client,
self.provider,
file,
&output,
remaining_limit,
|active_bytes| {
emit_progress(
&mut progress,
ArchiveProgress {
phase: ArchiveProgressPhase::Downloading,
total_assets: assets.len(),
completed_assets: summary.archived
+ summary.skipped
+ summary.failed.len(),
succeeded_assets: summary.archived,
failed_assets: summary.failed.len(),
bytes_written: retained_bytes
.saturating_add(prior_asset_bytes)
.saturating_add(active_bytes),
active_asset: Some(active_asset.clone()),
active_file: Some(active_file.clone()),
active_bytes_written: active_bytes,
active_total_bytes: advertised_size,
},
);
},
)?;
file.size = downloaded.bytes;
file_blake3.insert(file.path.clone(), downloaded.blake3);
asset_bytes = asset_bytes.saturating_add(downloaded.bytes);
}
bundle.total_bytes = asset_bytes;
let manifest = AssetManifest {
format_version: "2".to_string(),
provider: self.provider,
asset: authoritative_asset,
bundle: bundle.clone(),
provider_revision,
bytes_written: asset_bytes,
file_blake3,
complete: true,
};
write_manifest(&staging_dir, &manifest)?;
validate_manifest_identity(&manifest, self.provider, &asset.id)?;
verify_manifest_files(&staging_dir, &manifest)?;
Ok(ArchivedAsset {
asset_id: asset.id.clone(),
output: asset_dir.clone(),
files_archived: bundle.files.len(),
bytes_written: asset_bytes,
})
})();
match archive_result {
Ok(archived) => {
if let Err(error) = commit_staged_archive(&staging_dir, &asset_dir) {
let _ = fs::remove_dir_all(&staging_dir);
return Err(error);
}
Ok(Some(archived))
}
Err(error) => {
let _ = fs::remove_dir_all(&staging_dir);
Err(error)
}
}
})();
match result {
Ok(Some(archived)) => {
summary.archived += 1;
summary.files_archived += archived.files_archived;
summary.bytes_written =
summary.bytes_written.saturating_add(archived.bytes_written);
summary.assets.push(archived);
}
Ok(None) => summary.skipped += 1,
Err(error) => {
summary.failed.push(ArchiveFailure {
asset_id: asset.id.clone(),
message: error.to_string(),
});
}
}
emit_progress(
&mut progress,
progress_snapshot(ArchiveProgressPhase::Downloading, assets.len(), &summary),
);
}
emit_progress(
&mut progress,
progress_snapshot(ArchiveProgressPhase::Completed, assets.len(), &summary),
);
Ok(summary)
}
fn poly_haven_catalog(&self) -> Result<BTreeMap<String, PolyHavenAsset>> {
self.get_json(&format!("{POLY_HAVEN_API}/assets?t=models"))
}
fn poly_haven_asset(&self, asset_id: &str) -> Result<AssetSummary> {
let url = poly_haven_api_url("info", asset_id)?;
let asset: PolyHavenAsset = self.get_json(url.as_str())?;
Ok(asset_summary(asset_id, &asset))
}
fn fetch_authoritative_asset(&self, asset_id: &str) -> Result<(AssetSummary, String)> {
match self.provider {
AssetProvider::PolyHaven => Ok((self.poly_haven_asset(asset_id)?, String::new())),
AssetProvider::AmbientCG => {
let asset = self.ambientcg_asset_record(asset_id)?;
let revision = asset.release_date.clone();
Ok((ambientcg_asset_summary(&asset)?, revision))
}
AssetProvider::Sketchfab => {
let model = self.sketchfab_model(asset_id)?;
let revision = model.updated_at.clone();
Ok((sketchfab_asset_summary(&model)?, revision))
}
AssetProvider::GoogleScannedObjects => {
let model = self.google_scanned_model(asset_id)?;
let revision = model.updated_at.clone();
Ok((google_scanned_summary(&model)?, revision))
}
}
}
fn provider_revision(&self, asset_id: &str) -> Result<String> {
match self.provider {
AssetProvider::PolyHaven => Ok(String::new()),
AssetProvider::AmbientCG => Ok(self.ambientcg_asset_record(asset_id)?.release_date),
AssetProvider::Sketchfab => Ok(self.sketchfab_model(asset_id)?.updated_at),
AssetProvider::GoogleScannedObjects => {
Ok(self.google_scanned_model(asset_id)?.updated_at)
}
}
}
fn ambientcg_discover(&self, target: &AssetTarget) -> Result<Vec<AssetSummary>> {
if let AssetTarget::Asset(id) = target {
return Ok(vec![self.ambientcg_asset(id)?]);
}
let candidate_limit = match target {
AssetTarget::Search(_) => self.discovery_limit.saturating_mul(4).min(500),
_ => self.discovery_limit,
};
let mut url = reqwest::Url::parse(&format!("{AMBIENTCG_API}/assets"))
.map_err(|error| AssetForgeError::Request(error.to_string()))?;
{
let mut query = url.query_pairs_mut();
query
.append_pair("type", "3d-model")
.append_pair("limit", &candidate_limit.to_string())
.append_pair(
"include",
"type,title,url,tags,downloadStatistics,downloads,releaseDate",
);
match target {
AssetTarget::Search(value) => {
query
.append_pair("q", value)
.append_pair("sort", "downloads");
}
AssetTarget::Category(value) => {
query
.append_pair("category", value)
.append_pair("sort", "downloads");
}
AssetTarget::Top => {
query.append_pair("sort", "downloads");
}
AssetTarget::Latest => {
query.append_pair("sort", "latest");
}
AssetTarget::Asset(_) => unreachable!(),
}
}
let response: AmbientCGResponse = self.get_json(url.as_str())?;
let mut assets = response
.assets
.iter()
.map(ambientcg_asset_summary)
.collect::<Result<Vec<_>>>()?;
if let AssetTarget::Category(category) = target {
for asset in &mut assets {
asset.categories.push(category.clone());
}
}
rank_and_limit(assets, target, self.discovery_limit)
}
fn ambientcg_asset(&self, asset_id: &str) -> Result<AssetSummary> {
let asset = self.ambientcg_asset_record(asset_id)?;
ambientcg_asset_summary(&asset)
}
fn ambientcg_asset_record(&self, asset_id: &str) -> Result<AmbientCGAsset> {
let mut url = reqwest::Url::parse(&format!("{AMBIENTCG_API}/assets"))
.map_err(|error| AssetForgeError::Request(error.to_string()))?;
url.query_pairs_mut()
.append_pair("id", asset_id)
.append_pair("type", "3d-model")
.append_pair(
"include",
"type,title,url,tags,downloadStatistics,downloads,releaseDate",
);
let response: AmbientCGResponse = self.get_json(url.as_str())?;
let asset = response
.assets
.into_iter()
.find(|asset| asset.id.eq_ignore_ascii_case(asset_id))
.ok_or_else(|| AssetForgeError::NotFound {
provider: AssetProvider::AmbientCG.slug(),
asset: asset_id.to_string(),
})?;
if asset.id != asset_id || asset.asset_type != "3d-model" {
return Err(AssetForgeError::NotFound {
provider: AssetProvider::AmbientCG.slug(),
asset: asset_id.to_string(),
});
}
Ok(asset)
}
fn ambientcg_bundle(
&self,
asset_id: &str,
format: AssetFormat,
resolution: AssetResolution,
) -> Result<AssetBundle> {
let effective_format = match format {
AssetFormat::Auto | AssetFormat::Archive => AssetFormat::Archive,
_ => {
return Err(AssetForgeError::BundleUnavailable {
provider: AssetProvider::AmbientCG.slug(),
asset: asset_id.to_string(),
format,
resolution,
});
}
};
let effective_resolution = match resolution {
AssetResolution::Auto => AssetResolution::TwoK,
AssetResolution::OneK | AssetResolution::TwoK | AssetResolution::FourK => resolution,
AssetResolution::Original | AssetResolution::EightK => {
return Err(AssetForgeError::BundleUnavailable {
provider: AssetProvider::AmbientCG.slug(),
asset: asset_id.to_string(),
format,
resolution,
});
}
};
let asset = self.ambientcg_asset_record(asset_id)?;
let resolution_label = effective_resolution.slug().to_ascii_uppercase();
let preferred =
["HQ", "SQ", "LQ"].map(|quality| format!("{quality}-{resolution_label}-JPG"));
let download = preferred
.iter()
.find_map(|attributes| {
asset.downloads.iter().find(|download| {
download.extension.eq_ignore_ascii_case("zip")
&& download.attributes.eq_ignore_ascii_case(attributes)
})
})
.ok_or_else(|| AssetForgeError::BundleUnavailable {
provider: AssetProvider::AmbientCG.slug(),
asset: asset_id.to_string(),
format,
resolution,
})?;
trusted_download_url(AssetProvider::AmbientCG, &download.url)?;
let file_name = format!("{asset_id}_{}.zip", download.attributes);
Ok(AssetBundle {
provider: AssetProvider::AmbientCG,
asset_id: asset_id.to_string(),
source_url: asset.url.clone(),
rights: AssetRights::ambientcg(),
format: effective_format,
resolution: effective_resolution,
files: vec![AssetFile {
path: file_name,
size: download.size,
url: download.url.clone(),
md5: String::new(),
role: AssetFileRole::Package,
}],
total_bytes: download.size,
})
}
fn sketchfab_discover(&self, target: &AssetTarget) -> Result<Vec<AssetSummary>> {
if let AssetTarget::Asset(id) = target {
return Ok(vec![self.sketchfab_asset(id)?]);
}
let candidate_limit = match target {
AssetTarget::Search(_) => self.discovery_limit.saturating_mul(3).min(72),
AssetTarget::Category(_) | AssetTarget::Top => {
self.discovery_limit.saturating_mul(5).min(100)
}
AssetTarget::Latest => self.discovery_limit,
AssetTarget::Asset(_) => unreachable!(),
};
let mut url = reqwest::Url::parse(&format!("{SKETCHFAB_API}/search"))
.map_err(|error| AssetForgeError::Request(error.to_string()))?;
{
let mut query = url.query_pairs_mut();
query
.append_pair("type", "models")
.append_pair("downloadable", "true")
.append_pair("licenses", "cc0")
.append_pair("archives_flavours", "true")
.append_pair("count", &candidate_limit.min(24).to_string());
match target {
AssetTarget::Search(value) => {
query.append_pair("q", value);
}
AssetTarget::Category(value) => {
query
.append_pair("categories", value)
.append_pair("sort_by", "-viewCount");
}
AssetTarget::Top => {
query.append_pair("sort_by", "-viewCount");
}
AssetTarget::Latest => {
query.append_pair("sort_by", "-publishedAt");
}
AssetTarget::Asset(_) => unreachable!(),
}
}
let mut candidates = Vec::new();
let mut next = Some(url.to_string());
while candidates.len() < candidate_limit {
let Some(page_url) = next.take() else { break };
trusted_api_url(AssetProvider::Sketchfab, &page_url)?;
let page: SketchfabSearchResponse = self.get_json(&page_url)?;
candidates.extend(page.results.into_iter().filter(|model| {
model.is_downloadable
&& !model.is_age_restricted
&& (model.license.slug.eq_ignore_ascii_case("cc0")
|| model.license.label.to_ascii_lowercase().starts_with("cc0"))
}));
next = page.next;
}
candidates.truncate(candidate_limit);
let mut assets = Vec::with_capacity(candidates.len());
for candidate in candidates {
assets.push(self.sketchfab_asset(&candidate.uid)?);
}
rank_and_limit(assets, target, self.discovery_limit)
}
fn sketchfab_asset(&self, asset_id: &str) -> Result<AssetSummary> {
let model = self.sketchfab_model(asset_id)?;
sketchfab_asset_summary(&model)
}
fn sketchfab_model(&self, asset_id: &str) -> Result<SketchfabModel> {
let url = sketchfab_api_url("models", asset_id, None)?;
let model: SketchfabModel = self.get_json(url.as_str())?;
if model.uid != asset_id
|| !model.is_downloadable
|| model.is_age_restricted
|| !model.license.slug.eq_ignore_ascii_case("cc0")
{
return Err(AssetForgeError::NotFound {
provider: AssetProvider::Sketchfab.slug(),
asset: asset_id.to_string(),
});
}
Ok(model)
}
fn sketchfab_bundle(
&self,
asset_id: &str,
format: AssetFormat,
resolution: AssetResolution,
) -> Result<AssetBundle> {
if self.api_token.is_none() {
return Err(AssetForgeError::AuthenticationRequired {
provider: AssetProvider::Sketchfab.slug(),
environment_variable: "SKETCHFAB_API_TOKEN",
});
}
let effective_format = match format {
AssetFormat::Auto | AssetFormat::Gltf => AssetFormat::Gltf,
_ => {
return Err(AssetForgeError::BundleUnavailable {
provider: AssetProvider::Sketchfab.slug(),
asset: asset_id.to_string(),
format,
resolution,
});
}
};
let effective_resolution = match resolution {
AssetResolution::Auto | AssetResolution::Original => AssetResolution::Original,
_ => {
return Err(AssetForgeError::BundleUnavailable {
provider: AssetProvider::Sketchfab.slug(),
asset: asset_id.to_string(),
format,
resolution,
});
}
};
let summary = self.sketchfab_asset(asset_id)?;
let url = sketchfab_api_url("models", asset_id, Some("download"))?;
let response: SketchfabDownloadResponse = self.get_json(url.as_str())?;
let mut download = response
.gltf
.ok_or_else(|| AssetForgeError::BundleUnavailable {
provider: AssetProvider::Sketchfab.slug(),
asset: asset_id.to_string(),
format,
resolution,
})?;
trusted_download_url(AssetProvider::Sketchfab, &download.url)?;
if download.size == 0 {
download.size =
probe_download_size(&self.client, AssetProvider::Sketchfab, &download.url)?;
}
Ok(AssetBundle {
provider: AssetProvider::Sketchfab,
asset_id: asset_id.to_string(),
source_url: summary.source_url.clone(),
rights: summary.rights,
format: effective_format,
resolution: effective_resolution,
files: vec![AssetFile {
path: format!("{asset_id}.zip"),
size: download.size,
url: download.url,
md5: String::new(),
role: AssetFileRole::Package,
}],
total_bytes: download.size,
})
}
fn google_scanned_discover(&self, target: &AssetTarget) -> Result<Vec<AssetSummary>> {
if let AssetTarget::Asset(id) = target {
return Ok(vec![google_scanned_summary(
&self.google_scanned_model(id)?,
)?]);
}
let models = match target {
AssetTarget::Search(query) => self.google_scanned_search(query)?,
AssetTarget::Category(_) | AssetTarget::Top | AssetTarget::Latest => {
self.google_scanned_catalog()?
}
AssetTarget::Asset(_) => unreachable!(),
};
let mut assets = models
.iter()
.map(google_scanned_summary)
.collect::<Result<Vec<_>>>()?;
if let AssetTarget::Category(category) = target {
assets.retain(|asset| {
asset
.categories
.iter()
.any(|candidate| candidate.eq_ignore_ascii_case(category))
});
}
rank_and_limit(assets, target, self.discovery_limit)
}
fn google_scanned_search(&self, query: &str) -> Result<Vec<GoogleScannedModel>> {
let candidate_limit = self.discovery_limit.saturating_mul(5).min(100);
let mut models = Vec::new();
for page in 1..=MAX_FUEL_PAGES {
let mut url = reqwest::Url::parse(&format!("{GAZEBO_FUEL_API}/models"))
.map_err(|error| AssetForgeError::Request(error.to_string()))?;
url.query_pairs_mut()
.append_pair("q", query)
.append_pair("page", &page.to_string())
.append_pair("per_page", &FUEL_PAGE_SIZE.to_string());
let page_models: Vec<GoogleScannedModel> = self.get_json(url.as_str())?;
let page_len = page_models.len();
models.extend(page_models.into_iter().filter(valid_google_scanned_model));
if models.len() >= candidate_limit || page_len < FUEL_PAGE_SIZE {
break;
}
}
models.truncate(candidate_limit);
Ok(models)
}
fn google_scanned_catalog(&self) -> Result<Vec<GoogleScannedModel>> {
let mut models = Vec::new();
for page in 1..=MAX_FUEL_PAGES {
let mut url = reqwest::Url::parse(GAZEBO_FUEL_API)
.map_err(|error| AssetForgeError::Request(error.to_string()))?;
url.path_segments_mut()
.map_err(|_| AssetForgeError::Request("invalid Fuel API base URL".to_string()))?
.push(GOOGLE_RESEARCH_OWNER)
.push("collections")
.push(GOOGLE_SCANNED_COLLECTION)
.push("models");
url.query_pairs_mut()
.append_pair("page", &page.to_string())
.append_pair("per_page", &FUEL_PAGE_SIZE.to_string());
let page_models: Vec<GoogleScannedModel> = self.get_json(url.as_str())?;
let page_len = page_models.len();
models.extend(page_models.into_iter().filter(valid_google_scanned_model));
if page_len < FUEL_PAGE_SIZE {
return Ok(models);
}
}
Err(AssetForgeError::ResponseLimitExceeded {
label: "Google Scanned Objects catalog pages".to_string(),
size: MAX_FUEL_PAGES as u64,
limit: MAX_FUEL_PAGES as u64,
})
}
fn google_scanned_model(&self, asset_id: &str) -> Result<GoogleScannedModel> {
let url = fuel_model_url(asset_id, None)?;
let model: GoogleScannedModel = self.get_json(url.as_str())?;
if model.name != asset_id || !valid_google_scanned_model(&model) {
return Err(AssetForgeError::NotFound {
provider: AssetProvider::GoogleScannedObjects.slug(),
asset: asset_id.to_string(),
});
}
Ok(model)
}
fn google_scanned_bundle(
&self,
asset_id: &str,
format: AssetFormat,
resolution: AssetResolution,
) -> Result<AssetBundle> {
let effective_format = match format {
AssetFormat::Auto | AssetFormat::Archive => AssetFormat::Archive,
_ => {
return Err(AssetForgeError::BundleUnavailable {
provider: AssetProvider::GoogleScannedObjects.slug(),
asset: asset_id.to_string(),
format,
resolution,
});
}
};
let effective_resolution = match resolution {
AssetResolution::Auto | AssetResolution::Original => AssetResolution::Original,
_ => {
return Err(AssetForgeError::BundleUnavailable {
provider: AssetProvider::GoogleScannedObjects.slug(),
asset: asset_id.to_string(),
format,
resolution,
});
}
};
let model = self.google_scanned_model(asset_id)?;
let summary = google_scanned_summary(&model)?;
let url = fuel_model_url(asset_id, Some(&format!("tip/{asset_id}.zip")))?;
trusted_download_url(AssetProvider::GoogleScannedObjects, url.as_str())?;
Ok(AssetBundle {
provider: AssetProvider::GoogleScannedObjects,
asset_id: asset_id.to_string(),
source_url: summary.source_url,
rights: AssetRights::google_scanned_objects(),
format: effective_format,
resolution: effective_resolution,
files: vec![AssetFile {
path: format!("{asset_id}.zip"),
size: model.filesize,
url: url.to_string(),
md5: String::new(),
role: AssetFileRole::Package,
}],
total_bytes: model.filesize,
})
}
fn get_json<T>(&self, url: &str) -> Result<T>
where
T: for<'de> Deserialize<'de>,
{
trusted_api_url(self.provider, url)?;
let mut request = self.client.get(url);
if self.provider == AssetProvider::Sketchfab {
if let Some(token) = &self.api_token {
request = request.header("Authorization", format!("Token {token}"));
}
}
let response = request.send()?;
let status = response.status();
if !status.is_success() {
let body = bounded_error_body(response);
return Err(AssetForgeError::ProviderApi {
provider: self.provider.slug(),
status: status.as_u16(),
body,
});
}
let bytes = read_response_limited(response, MAX_API_RESPONSE_BYTES, "provider response")?;
Ok(serde_json::from_slice(&bytes)?)
}
}
#[derive(Debug, Clone, Deserialize)]
struct PolyHavenAsset {
#[serde(default)]
name: String,
#[serde(default)]
description: String,
#[serde(default)]
categories: Vec<String>,
#[serde(default)]
category: String,
#[serde(default)]
tags: Vec<String>,
#[serde(default)]
download_count: u64,
#[serde(default)]
polycount: u64,
#[serde(default)]
date_published: u64,
#[serde(default)]
thumbnail_url: String,
}
#[derive(Debug, Clone, Deserialize)]
struct PolyHavenFile {
size: u64,
url: String,
md5: String,
#[serde(default)]
include: BTreeMap<String, PolyHavenFile>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
struct AmbientCGResponse {
assets: Vec<AmbientCGAsset>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
struct AmbientCGAsset {
id: String,
#[serde(rename = "type")]
asset_type: String,
title: String,
url: String,
#[serde(default)]
tags: Vec<String>,
download_statistics: AmbientCGDownloadStats,
downloads: Vec<AmbientCGDownload>,
release_date: String,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
struct AmbientCGDownloadStats {
total: u64,
}
#[derive(Debug, Clone, Deserialize)]
struct AmbientCGDownload {
attributes: String,
extension: String,
url: String,
size: u64,
}
fn ambientcg_asset_summary(asset: &AmbientCGAsset) -> Result<AssetSummary> {
if asset.id.is_empty() || asset.title.is_empty() || asset.url.is_empty() {
return Err(AssetForgeError::Json(serde_json::Error::io(
io::Error::new(
io::ErrorKind::InvalidData,
"ambientCG response omitted required asset metadata",
),
)));
}
Ok(AssetSummary {
provider: AssetProvider::AmbientCG,
id: asset.id.clone(),
name: asset.title.clone(),
description: String::new(),
categories: Vec::new(),
tags: asset.tags.clone(),
download_count: asset.download_statistics.total,
polycount: 0,
date_published: parse_date_timestamp(&asset.release_date),
thumbnail_url: String::new(),
source_url: asset.url.clone(),
rights: AssetRights::ambientcg(),
})
}
#[derive(Debug, Clone, Deserialize)]
struct SketchfabSearchResponse {
next: Option<String>,
results: Vec<SketchfabSearchModel>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
struct SketchfabSearchModel {
uid: String,
is_downloadable: bool,
#[serde(default)]
is_age_restricted: bool,
license: SketchfabLicense,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
struct SketchfabModel {
uid: String,
name: String,
description: String,
categories: Vec<SketchfabCategory>,
tags: Vec<SketchfabTag>,
user: SketchfabUser,
license: SketchfabLicense,
viewer_url: String,
is_downloadable: bool,
#[serde(default)]
is_age_restricted: bool,
download_count: u64,
face_count: u64,
published_at: String,
updated_at: String,
thumbnails: SketchfabThumbnails,
}
#[derive(Debug, Clone, Deserialize)]
struct SketchfabTag {
name: String,
}
#[derive(Debug, Clone, Deserialize)]
struct SketchfabCategory {
name: String,
#[serde(default)]
slug: String,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
struct SketchfabUser {
username: String,
#[serde(default)]
display_name: String,
}
#[derive(Debug, Clone, Deserialize)]
struct SketchfabLicense {
label: String,
#[serde(default)]
slug: String,
}
#[derive(Debug, Clone, Deserialize)]
struct SketchfabThumbnails {
#[serde(default)]
images: Vec<SketchfabImage>,
}
#[derive(Debug, Clone, Deserialize)]
struct SketchfabImage {
url: String,
#[serde(default)]
width: u64,
}
#[derive(Debug, Clone, Deserialize)]
struct SketchfabDownloadResponse {
gltf: Option<SketchfabDownload>,
}
#[derive(Debug, Clone, Deserialize)]
struct SketchfabDownload {
url: String,
#[serde(default)]
size: u64,
}
fn sketchfab_asset_summary(model: &SketchfabModel) -> Result<AssetSummary> {
if model.uid.is_empty() || model.name.is_empty() || model.viewer_url.is_empty() {
return Err(AssetForgeError::Json(serde_json::Error::io(
io::Error::new(
io::ErrorKind::InvalidData,
"Sketchfab response omitted required model metadata",
),
)));
}
let creator = if model.user.display_name.trim().is_empty() {
model.user.username.as_str()
} else {
model.user.display_name.as_str()
};
let thumbnail_url = model
.thumbnails
.images
.iter()
.max_by_key(|image| image.width)
.map(|image| image.url.clone())
.unwrap_or_default();
Ok(AssetSummary {
provider: AssetProvider::Sketchfab,
id: model.uid.clone(),
name: model.name.clone(),
description: model.description.clone(),
categories: model
.categories
.iter()
.map(|category| {
if category.slug.is_empty() {
category.name.clone()
} else {
category.slug.clone()
}
})
.collect(),
tags: model.tags.iter().map(|tag| tag.name.clone()).collect(),
download_count: model.download_count,
polycount: model.face_count,
date_published: parse_date_timestamp(&model.published_at),
thumbnail_url,
source_url: model.viewer_url.clone(),
rights: AssetRights::sketchfab(creator),
})
}
#[derive(Debug, Clone, Deserialize)]
struct GoogleScannedModel {
#[serde(rename = "createdAt")]
created_at: String,
#[serde(rename = "updatedAt")]
updated_at: String,
name: String,
owner: String,
#[serde(default)]
description: String,
#[serde(default)]
downloads: u64,
filesize: u64,
license_id: u64,
license_name: String,
license_url: String,
permission: u64,
thumbnail_url: String,
#[serde(rename = "private")]
is_private: bool,
#[serde(default)]
categories: Vec<String>,
}
fn valid_google_scanned_model(model: &GoogleScannedModel) -> bool {
model.owner == GOOGLE_RESEARCH_OWNER
&& !model.is_private
&& model.permission == 0
&& model.license_id == 2
&& model
.license_name
.eq_ignore_ascii_case("Creative Commons Attribution 4.0 International")
&& matches!(
model.license_url.as_str(),
"http://creativecommons.org/licenses/by/4.0/"
| "https://creativecommons.org/licenses/by/4.0/"
)
&& model.filesize > 0
}
fn google_scanned_summary(model: &GoogleScannedModel) -> Result<AssetSummary> {
if !valid_google_scanned_model(model) {
return Err(AssetForgeError::VerificationFailed {
path: model.name.clone(),
message: "Google Scanned Object owner or CC BY 4.0 metadata is invalid".to_string(),
});
}
let source_url = fuel_model_url(&model.name, None)?.to_string();
let thumbnail_url = if model.thumbnail_url.starts_with('/') {
format!("{GAZEBO_FUEL_API}{}", model.thumbnail_url)
} else {
model.thumbnail_url.clone()
};
let display_name = model
.description
.lines()
.next()
.map(str::trim)
.filter(|name| !name.is_empty())
.map_or_else(|| model.name.replace('_', " "), str::to_string);
Ok(AssetSummary {
provider: AssetProvider::GoogleScannedObjects,
id: model.name.clone(),
name: display_name,
description: model.description.clone(),
categories: model.categories.clone(),
tags: Vec::new(),
download_count: model.downloads,
polycount: 0,
date_published: parse_date_timestamp(&model.created_at),
thumbnail_url,
source_url,
rights: AssetRights::google_scanned_objects(),
})
}
fn discover_catalog(
catalog: BTreeMap<String, PolyHavenAsset>,
target: &AssetTarget,
limit: usize,
) -> Result<Vec<AssetSummary>> {
if let AssetTarget::Asset(id) = target {
let Some(asset) = catalog.get(id) else {
return Err(AssetForgeError::NotFound {
provider: AssetProvider::PolyHaven.slug(),
asset: id.clone(),
});
};
return Ok(vec![asset_summary(id, asset)]);
}
let mut assets = catalog
.iter()
.map(|(id, asset)| asset_summary(id, asset))
.collect::<Vec<_>>();
if let AssetTarget::Category(category) = target {
assets.retain(|asset| {
asset
.categories
.iter()
.any(|candidate| candidate.eq_ignore_ascii_case(category))
|| catalog
.get(&asset.id)
.is_some_and(|source| source.category.eq_ignore_ascii_case(category))
});
}
rank_and_limit(assets, target, limit)
}
fn rank_and_limit(
mut assets: Vec<AssetSummary>,
target: &AssetTarget,
limit: usize,
) -> Result<Vec<AssetSummary>> {
match target {
AssetTarget::Search(query) => {
let mut ranked = assets
.into_iter()
.filter_map(|asset| relevance_score(&asset, query).map(|score| (score, asset)))
.collect::<Vec<_>>();
ranked.sort_by(|(left_score, left), (right_score, right)| {
right_score
.cmp(left_score)
.then_with(|| right.download_count.cmp(&left.download_count))
.then_with(|| left.id.cmp(&right.id))
});
assets = ranked.into_iter().map(|(_, asset)| asset).collect();
}
AssetTarget::Latest => assets.sort_by(|left, right| {
right
.date_published
.cmp(&left.date_published)
.then_with(|| right.download_count.cmp(&left.download_count))
.then_with(|| left.id.cmp(&right.id))
}),
AssetTarget::Category(_) | AssetTarget::Top => assets.sort_by(|left, right| {
right
.download_count
.cmp(&left.download_count)
.then_with(|| left.id.cmp(&right.id))
}),
AssetTarget::Asset(_) => {}
}
assets.truncate(limit);
Ok(assets)
}
fn relevance_score(asset: &AssetSummary, query: &str) -> Option<u64> {
let query = normalized_search_text(query);
let terms = query.split_whitespace().collect::<BTreeSet<_>>();
if terms.is_empty() {
return None;
}
let id = normalized_search_text(&asset.id);
let name = normalized_search_text(&asset.name);
let tags = normalized_search_text(&asset.tags.join(" "));
let categories = normalized_search_text(&asset.categories.join(" "));
let description = normalized_search_text(&asset.description);
let primary = format!("{id} {name} {tags} {categories}");
let all = format!("{primary} {description}");
if !terms.iter().all(|term| all.contains(term)) {
return None;
}
let score = if id == query || name == query {
600
} else if id.contains(&query) || name.contains(&query) {
500
} else if terms
.iter()
.all(|term| id.contains(term) || name.contains(term))
{
450
} else if terms.iter().all(|term| primary.contains(term)) {
400
} else if terms
.iter()
.any(|term| id.contains(term) || name.contains(term))
{
350
} else if terms
.iter()
.all(|term| tags.contains(term) || categories.contains(term))
{
300
} else {
100
};
Some(score)
}
fn normalized_search_text(value: &str) -> String {
value
.chars()
.map(|character| {
if character.is_alphanumeric() {
character.to_ascii_lowercase()
} else {
' '
}
})
.collect::<String>()
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
}
fn asset_summary(id: &str, asset: &PolyHavenAsset) -> AssetSummary {
AssetSummary {
provider: AssetProvider::PolyHaven,
id: id.to_string(),
name: asset.name.clone(),
description: asset.description.clone(),
categories: asset.categories.clone(),
tags: asset.tags.clone(),
download_count: asset.download_count,
polycount: asset.polycount,
date_published: asset.date_published,
thumbnail_url: asset.thumbnail_url.clone(),
source_url: format!("https://polyhaven.com/a/{id}"),
rights: AssetRights::poly_haven(),
}
}
fn bundle_from_value(
asset_id: &str,
format: AssetFormat,
resolution: AssetResolution,
value: &Value,
) -> Result<AssetBundle> {
let effective_format = match format {
AssetFormat::Auto => AssetFormat::Gltf,
AssetFormat::Archive => {
return Err(AssetForgeError::BundleUnavailable {
provider: AssetProvider::PolyHaven.slug(),
asset: asset_id.to_string(),
format,
resolution,
});
}
_ => format,
};
let effective_resolution = match resolution {
AssetResolution::Auto => AssetResolution::TwoK,
AssetResolution::Original => {
return Err(AssetForgeError::BundleUnavailable {
provider: AssetProvider::PolyHaven.slug(),
asset: asset_id.to_string(),
format,
resolution,
});
}
_ => resolution,
};
let entry = value
.get(effective_format.api_key())
.and_then(|formats| formats.get(effective_resolution.slug()))
.and_then(|resolution| resolution.get(effective_format.api_key()))
.ok_or_else(|| AssetForgeError::BundleUnavailable {
provider: AssetProvider::PolyHaven.slug(),
asset: asset_id.to_string(),
format,
resolution,
})?;
let scene: PolyHavenFile = serde_json::from_value(entry.clone())?;
let scene_url = trusted_download_url(AssetProvider::PolyHaven, &scene.url)?;
let scene_name = scene_url
.path_segments()
.and_then(|mut segments| segments.next_back())
.filter(|name| !name.is_empty())
.ok_or_else(|| AssetForgeError::UnsafePath(scene.url.clone()))?;
let scene_path = safe_relative_path(scene_name)?;
let mut files = Vec::new();
let mut seen_paths = BTreeMap::new();
push_bundle_file(
&mut files,
&mut seen_paths,
scene_path,
&scene,
AssetFileRole::Scene,
0,
)?;
let total_bytes = files
.iter()
.fold(0u64, |total, file| total.saturating_add(file.size));
Ok(AssetBundle {
provider: AssetProvider::PolyHaven,
asset_id: asset_id.to_string(),
source_url: format!("https://polyhaven.com/a/{asset_id}"),
rights: AssetRights::poly_haven(),
format: effective_format,
resolution: effective_resolution,
files,
total_bytes,
})
}
fn push_bundle_file(
files: &mut Vec<AssetFile>,
seen_paths: &mut BTreeMap<String, (String, String)>,
path: PathBuf,
file: &PolyHavenFile,
role: AssetFileRole,
depth: usize,
) -> Result<()> {
if depth > MAX_BUNDLE_INCLUDE_DEPTH {
return Err(AssetForgeError::ResponseLimitExceeded {
label: "bundle include depth".to_string(),
size: depth as u64,
limit: MAX_BUNDLE_INCLUDE_DEPTH as u64,
});
}
trusted_download_url(AssetProvider::PolyHaven, &file.url)?;
let path = manifest_path(&path)?;
validate_bundle_path(&path)?;
if file.md5.len() != 32 || !file.md5.bytes().all(|byte| byte.is_ascii_hexdigit()) {
return Err(AssetForgeError::VerificationFailed {
path,
message: "provider MD5 is not 32 hexadecimal characters".to_string(),
});
}
let path_key = path.to_lowercase();
match seen_paths.get(&path_key) {
Some((existing_path, existing_url))
if existing_path == &path && existing_url == &file.url =>
{
return Ok(());
}
Some(_) => return Err(AssetForgeError::UnsafePath(path)),
None => {
if files.len() >= MAX_BUNDLE_FILES {
return Err(AssetForgeError::ResponseLimitExceeded {
label: "bundle file count".to_string(),
size: files.len().saturating_add(1) as u64,
limit: MAX_BUNDLE_FILES as u64,
});
}
seen_paths.insert(path_key, (path.clone(), file.url.clone()));
}
}
{
files.push(AssetFile {
path,
size: file.size,
url: file.url.clone(),
md5: file.md5.to_ascii_lowercase(),
role,
});
}
for (include_path, dependency) in &file.include {
push_bundle_file(
files,
seen_paths,
safe_relative_path(include_path)?,
dependency,
AssetFileRole::Dependency,
depth.saturating_add(1),
)?;
}
Ok(())
}
struct DownloadedFile {
bytes: u64,
blake3: String,
}
fn download_file<F>(
client: &Client,
provider: AssetProvider,
asset_file: &AssetFile,
output: &Path,
max_bytes: Option<u64>,
mut progress: F,
) -> Result<DownloadedFile>
where
F: FnMut(u64),
{
trusted_download_url(provider, &asset_file.url)?;
if let Some(parent) = output.parent() {
fs::create_dir_all(parent)?;
}
let mut response = send_download_request(client, provider, &asset_file.url)?;
let status = response.status();
if !status.is_success() {
let body = bounded_error_body(response);
return Err(AssetForgeError::ProviderApi {
provider: provider.slug(),
status: status.as_u16(),
body,
});
}
if let Some(length) = response.content_length() {
if asset_file.size > 0 && length != asset_file.size {
return Err(verification_error(
asset_file,
format!(
"provider expected {} bytes but server advertised {length}",
asset_file.size
),
));
}
if max_bytes.is_some_and(|limit| length > limit) {
return Err(AssetForgeError::DownloadLimitExceeded {
size: length,
limit: max_bytes.unwrap_or_default(),
});
}
}
let partial = partial_path(output);
let mut file = File::create(&partial)?;
let mut md5 = Md5::new();
let mut blake3 = blake3::Hasher::new();
let mut bytes_written = 0u64;
let mut last_emit = 0u64;
let mut buffer = [0u8; 64 * 1024];
loop {
let read = match response.read(&mut buffer) {
Ok(read) => read,
Err(error) => {
drop(file);
let _ = fs::remove_file(&partial);
return Err(AssetForgeError::Request(error_chain(&error)));
}
};
if read == 0 {
break;
}
if let Err(error) = file.write_all(&buffer[..read]) {
drop(file);
let _ = fs::remove_file(&partial);
return Err(AssetForgeError::Io(error));
}
md5.update(&buffer[..read]);
blake3.update(&buffer[..read]);
bytes_written = bytes_written.saturating_add(read as u64);
if asset_file.size > 0 && bytes_written > asset_file.size {
drop(file);
let _ = fs::remove_file(&partial);
return Err(verification_error(
asset_file,
format!("received more than the expected {} bytes", asset_file.size),
));
}
if max_bytes.is_some_and(|limit| bytes_written > limit) {
drop(file);
let _ = fs::remove_file(&partial);
return Err(AssetForgeError::DownloadLimitExceeded {
size: bytes_written,
limit: max_bytes.unwrap_or_default(),
});
}
if bytes_written.saturating_sub(last_emit) >= DOWNLOAD_PROGRESS_GRANULARITY {
progress(bytes_written);
last_emit = bytes_written;
}
}
if asset_file.size > 0 && bytes_written != asset_file.size {
drop(file);
let _ = fs::remove_file(&partial);
return Err(verification_error(
asset_file,
format!(
"expected {} bytes but received {bytes_written}",
asset_file.size
),
));
}
let checksum = format!("{:x}", md5.finalize());
if provider.has_md5_verification() && !checksum.eq_ignore_ascii_case(&asset_file.md5) {
drop(file);
let _ = fs::remove_file(&partial);
return Err(verification_error(
asset_file,
format!("expected MD5 {} but received {checksum}", asset_file.md5),
));
}
if let Err(error) = file.flush().and_then(|()| file.sync_all()) {
drop(file);
let _ = fs::remove_file(&partial);
return Err(AssetForgeError::Io(error));
}
drop(file);
if last_emit != bytes_written {
progress(bytes_written);
}
if let Err(error) = fs::rename(&partial, output) {
let _ = fs::remove_file(&partial);
return Err(AssetForgeError::Io(error));
}
Ok(DownloadedFile {
bytes: bytes_written,
blake3: blake3.finalize().to_hex().to_string(),
})
}
fn send_download_request(client: &Client, provider: AssetProvider, url: &str) -> Result<Response> {
let mut current = trusted_download_url(provider, url)?;
for redirects in 0..=MAX_REDIRECTS {
let response = client.get(current.clone()).send()?;
if !response.status().is_redirection() {
trusted_download_url(provider, response.url().as_str())?;
return Ok(response);
}
if redirects == MAX_REDIRECTS {
return Err(AssetForgeError::ProviderApi {
provider: provider.slug(),
status: response.status().as_u16(),
body: "download exceeded the redirect limit".to_string(),
});
}
let location = response
.headers()
.get(reqwest::header::LOCATION)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| AssetForgeError::ProviderApi {
provider: provider.slug(),
status: response.status().as_u16(),
body: "redirect omitted a valid Location header".to_string(),
})?;
current = current
.join(location)
.map_err(|_| AssetForgeError::UntrustedDownloadUrl(location.to_string()))?;
trusted_download_url(provider, current.as_str())?;
}
unreachable!()
}
fn probe_download_size(client: &Client, provider: AssetProvider, url: &str) -> Result<u64> {
let mut current = trusted_download_url(provider, url)?;
for redirects in 0..=MAX_REDIRECTS {
let response = client
.get(current.clone())
.header(reqwest::header::RANGE, "bytes=0-0")
.send()?;
if response.status().is_redirection() {
if redirects == MAX_REDIRECTS {
return Err(AssetForgeError::ProviderApi {
provider: provider.slug(),
status: response.status().as_u16(),
body: "size probe exceeded the redirect limit".to_string(),
});
}
let location = response
.headers()
.get(reqwest::header::LOCATION)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| AssetForgeError::ProviderApi {
provider: provider.slug(),
status: response.status().as_u16(),
body: "redirect omitted a valid Location header".to_string(),
})?;
current = current
.join(location)
.map_err(|_| AssetForgeError::UntrustedDownloadUrl(location.to_string()))?;
trusted_download_url(provider, current.as_str())?;
continue;
}
if !response.status().is_success() {
let status = response.status().as_u16();
return Err(AssetForgeError::ProviderApi {
provider: provider.slug(),
status,
body: bounded_error_body(response),
});
}
if let Some(total) = response
.headers()
.get(reqwest::header::CONTENT_RANGE)
.and_then(|value| value.to_str().ok())
.and_then(|value| value.rsplit_once('/'))
.and_then(|(_, total)| total.parse::<u64>().ok())
{
return Ok(total);
}
if response.status() == reqwest::StatusCode::OK {
if let Some(length) = response.content_length() {
return Ok(length);
}
}
return Err(AssetForgeError::InvalidOption(
"provider did not advertise a downloadable bundle size".to_string(),
));
}
unreachable!()
}
fn verification_error(file: &AssetFile, message: String) -> AssetForgeError {
AssetForgeError::VerificationFailed {
path: file.path.clone(),
message,
}
}
fn poly_haven_api_url(endpoint: &str, asset_id: &str) -> Result<reqwest::Url> {
let mut url = reqwest::Url::parse(POLY_HAVEN_API)
.map_err(|_| AssetForgeError::InvalidTarget(asset_id.to_string()))?;
url.path_segments_mut()
.map_err(|_| AssetForgeError::InvalidTarget(asset_id.to_string()))?
.push(endpoint)
.push(asset_id);
Ok(url)
}
fn sketchfab_api_url(endpoint: &str, asset_id: &str, suffix: Option<&str>) -> Result<reqwest::Url> {
let mut url = reqwest::Url::parse(SKETCHFAB_API)
.map_err(|_| AssetForgeError::InvalidTarget(asset_id.to_string()))?;
let mut segments = url
.path_segments_mut()
.map_err(|_| AssetForgeError::InvalidTarget(asset_id.to_string()))?;
segments.push(endpoint).push(asset_id);
if let Some(suffix) = suffix {
segments.push(suffix);
}
drop(segments);
Ok(url)
}
fn fuel_model_url(asset_id: &str, suffix: Option<&str>) -> Result<reqwest::Url> {
let mut url = reqwest::Url::parse(GAZEBO_FUEL_API)
.map_err(|_| AssetForgeError::InvalidTarget(asset_id.to_string()))?;
let mut segments = url
.path_segments_mut()
.map_err(|_| AssetForgeError::InvalidTarget(asset_id.to_string()))?;
segments
.push(GOOGLE_RESEARCH_OWNER)
.push("models")
.push(asset_id);
if let Some(suffix) = suffix {
for segment in suffix.split('/').filter(|segment| !segment.is_empty()) {
segments.push(segment);
}
}
drop(segments);
Ok(url)
}
fn read_response_limited(mut response: Response, limit: u64, label: &str) -> Result<Vec<u8>> {
if let Some(length) = response.content_length() {
if length > limit {
return Err(AssetForgeError::ResponseLimitExceeded {
label: label.to_string(),
size: length,
limit,
});
}
}
let mut bytes = Vec::new();
response
.by_ref()
.take(limit.saturating_add(1))
.read_to_end(&mut bytes)?;
if bytes.len() as u64 > limit {
return Err(AssetForgeError::ResponseLimitExceeded {
label: label.to_string(),
size: bytes.len() as u64,
limit,
});
}
Ok(bytes)
}
fn bounded_error_body(response: Response) -> String {
match read_response_limited(
response,
MAX_ERROR_RESPONSE_BYTES,
"provider error response",
) {
Ok(bytes) => String::from_utf8_lossy(&bytes).into_owned(),
Err(AssetForgeError::ResponseLimitExceeded { .. }) => {
"<error response exceeded safety limit>".to_string()
}
Err(_) => "<unable to read body>".to_string(),
}
}
fn trusted_api_url(provider: AssetProvider, url: &str) -> Result<reqwest::Url> {
let parsed = reqwest::Url::parse(url)
.map_err(|_| AssetForgeError::UntrustedDownloadUrl(url.to_string()))?;
let host = parsed.host_str().unwrap_or_default();
let expected_host = match provider {
AssetProvider::PolyHaven => "api.polyhaven.com",
AssetProvider::AmbientCG => "ambientcg.com",
AssetProvider::Sketchfab => "api.sketchfab.com",
AssetProvider::GoogleScannedObjects => "fuel.gazebosim.org",
};
if !secure_https_url(&parsed) || host != expected_host {
return Err(AssetForgeError::UntrustedDownloadUrl(url.to_string()));
}
Ok(parsed)
}
fn trusted_download_url(provider: AssetProvider, url: &str) -> Result<reqwest::Url> {
let parsed = reqwest::Url::parse(url)
.map_err(|_| AssetForgeError::UntrustedDownloadUrl(url.to_string()))?;
let host = parsed.host_str().unwrap_or_default();
let trusted = match provider {
AssetProvider::PolyHaven => host == "dl.polyhaven.org",
AssetProvider::AmbientCG => {
host == "ambientcg.com" || host == "acg-download.struffelproductions.com"
}
AssetProvider::Sketchfab => {
host == "sketchfab.com"
|| host.ends_with(".sketchfab.com")
|| host == "sketchfab-prod-media.s3.amazonaws.com"
|| host == "sketchfab-prod-media.s3-accelerate.amazonaws.com"
}
AssetProvider::GoogleScannedObjects => host == "fuel.gazebosim.org",
};
if !secure_https_url(&parsed) || !trusted {
return Err(AssetForgeError::UntrustedDownloadUrl(url.to_string()));
}
Ok(parsed)
}
fn secure_https_url(url: &reqwest::Url) -> bool {
url.scheme() == "https"
&& url.port().is_none_or(|port| port == 443)
&& url.username().is_empty()
&& url.password().is_none()
}
fn parse_provider_url(input: &str, provider: AssetProvider) -> Result<AssetTarget> {
let url = reqwest::Url::parse(input)
.map_err(|_| AssetForgeError::InvalidTarget(input.to_string()))?;
if !secure_https_url(&url) {
return Err(AssetForgeError::InvalidTarget(input.to_string()));
}
match provider {
AssetProvider::PolyHaven => {
let host = url.host_str().unwrap_or_default();
if host != "polyhaven.com" && host != "www.polyhaven.com" {
return Err(AssetForgeError::InvalidTarget(input.to_string()));
}
let segments = url
.path_segments()
.map(|segments| segments.filter(|part| !part.is_empty()).collect::<Vec<_>>())
.unwrap_or_default();
if segments.len() != 2 || segments[0] != "a" {
return Err(AssetForgeError::InvalidTarget(input.to_string()));
}
non_empty_target(segments[1], AssetTarget::Asset)
}
AssetProvider::AmbientCG => {
let host = url.host_str().unwrap_or_default();
if host != "ambientcg.com" && host != "www.ambientcg.com" {
return Err(AssetForgeError::InvalidTarget(input.to_string()));
}
let segments = url
.path_segments()
.map(|segments| segments.filter(|part| !part.is_empty()).collect::<Vec<_>>())
.unwrap_or_default();
if segments.len() != 2 || segments[0] != "a" {
return Err(AssetForgeError::InvalidTarget(input.to_string()));
}
non_empty_target(segments[1], AssetTarget::Asset)
}
AssetProvider::Sketchfab => {
let host = url.host_str().unwrap_or_default();
if host != "sketchfab.com" && host != "www.sketchfab.com" {
return Err(AssetForgeError::InvalidTarget(input.to_string()));
}
let segments = url
.path_segments()
.map(|segments| segments.filter(|part| !part.is_empty()).collect::<Vec<_>>())
.unwrap_or_default();
if segments.len() == 2 && segments[0] == "models" {
return non_empty_target(segments[1], AssetTarget::Asset);
}
if segments.len() == 2 && segments[0] == "3d-models" {
let uid = segments[1]
.rsplit_once('-')
.map_or(segments[1], |(_, uid)| uid);
if uid.len() == 32 && uid.bytes().all(|byte| byte.is_ascii_hexdigit()) {
return non_empty_target(uid, AssetTarget::Asset);
}
}
Err(AssetForgeError::InvalidTarget(input.to_string()))
}
AssetProvider::GoogleScannedObjects => {
if url.host_str() != Some("fuel.gazebosim.org") {
return Err(AssetForgeError::InvalidTarget(input.to_string()));
}
let segments = url
.path_segments()
.map(|segments| segments.filter(|part| !part.is_empty()).collect::<Vec<_>>())
.unwrap_or_default();
if segments.len() >= 4
&& segments[0] == "1.0"
&& segments[1] == GOOGLE_RESEARCH_OWNER
&& segments[2] == "models"
{
return non_empty_target(segments[3], AssetTarget::Asset);
}
Err(AssetForgeError::InvalidTarget(input.to_string()))
}
}
}
fn non_empty_target(value: &str, constructor: fn(String) -> AssetTarget) -> Result<AssetTarget> {
let value = value.trim();
if value.is_empty() {
Err(AssetForgeError::InvalidTarget(value.to_string()))
} else {
Ok(constructor(value.to_string()))
}
}
fn prepare_output(output: &Path) -> Result<()> {
if output.exists() && !output.is_dir() {
return Err(AssetForgeError::OutputNotDirectory(output.to_path_buf()));
}
fs::create_dir_all(output)?;
Ok(())
}
fn asset_directory_name(value: &str) -> Result<String> {
let value = value.trim();
if value.is_empty()
|| value == "."
|| value == ".."
|| !value.chars().all(|character| {
character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.')
})
{
return Err(AssetForgeError::UnsafePath(value.to_string()));
}
Ok(value.to_string())
}
fn safe_relative_path(value: &str) -> Result<PathBuf> {
let path = Path::new(value);
if path.is_absolute() {
return Err(AssetForgeError::UnsafePath(value.to_string()));
}
let mut safe = PathBuf::new();
for component in path.components() {
match component {
Component::Normal(part) if !part.is_empty() => safe.push(part),
_ => return Err(AssetForgeError::UnsafePath(value.to_string())),
}
}
if safe.as_os_str().is_empty() {
return Err(AssetForgeError::UnsafePath(value.to_string()));
}
Ok(safe)
}
fn manifest_path(path: &Path) -> Result<String> {
let path = safe_relative_path(&path.to_string_lossy())?;
Ok(path
.components()
.map(|component| component.as_os_str().to_string_lossy())
.collect::<Vec<_>>()
.join("/"))
}
fn validate_bundle_path(path: &str) -> Result<()> {
let path = manifest_path(&safe_relative_path(path)?)?;
let component_count = Path::new(&path).components().count();
if path.len() > MAX_BUNDLE_PATH_BYTES || component_count > MAX_BUNDLE_PATH_COMPONENTS {
return Err(AssetForgeError::ResponseLimitExceeded {
label: "bundle path".to_string(),
size: path.len().max(component_count) as u64,
limit: MAX_BUNDLE_PATH_BYTES.max(MAX_BUNDLE_PATH_COMPONENTS) as u64,
});
}
let file_name = Path::new(&path)
.file_name()
.and_then(|name| name.to_str())
.unwrap_or_default();
if path.eq_ignore_ascii_case(MANIFEST_NAME)
|| path.eq_ignore_ascii_case(&format!("{MANIFEST_NAME}.tmp"))
|| file_name.to_ascii_lowercase().ends_with(".assetforge-part")
{
return Err(AssetForgeError::UnsafePath(path));
}
Ok(())
}
fn path_from_manifest(path: &str) -> Result<PathBuf> {
safe_relative_path(path)
}
fn partial_path(path: &Path) -> PathBuf {
let mut partial = path.to_path_buf();
let extension = path
.extension()
.and_then(|value| value.to_str())
.map_or_else(
|| "assetforge-part".to_string(),
|value| format!("{value}.assetforge-part"),
);
partial.set_extension(extension);
partial
}
fn write_manifest(asset_dir: &Path, manifest: &AssetManifest) -> Result<()> {
let final_path = asset_dir.join(MANIFEST_NAME);
let temporary_path = asset_dir.join(format!("{MANIFEST_NAME}.tmp"));
let payload = serde_json::to_vec_pretty(manifest)?;
if payload.len() as u64 > MAX_MANIFEST_BYTES {
return Err(AssetForgeError::ResponseLimitExceeded {
label: MANIFEST_NAME.to_string(),
size: payload.len() as u64,
limit: MAX_MANIFEST_BYTES,
});
}
let mut file = File::create(&temporary_path)?;
file.write_all(&payload)?;
file.flush()?;
file.sync_all()?;
drop(file);
fs::rename(temporary_path, final_path)?;
Ok(())
}
fn matching_complete_manifest(
asset_dir: &Path,
provider: AssetProvider,
asset_id: &str,
current_bundle: &AssetBundle,
current_revision: &str,
) -> Result<bool> {
if !asset_dir.exists() || !asset_dir.join(MANIFEST_NAME).exists() {
return Ok(false);
}
let directory = open_archive_directory(asset_dir)?;
let manifest = read_manifest_bounded(&directory)?;
validate_manifest_identity(&manifest, provider, asset_id)?;
if !bundles_match(&manifest.bundle, current_bundle) {
return Ok(false);
}
if !current_revision.is_empty() && manifest.provider_revision != current_revision {
return Ok(false);
}
verify_manifest_files_in_directory(&directory, &manifest)?;
Ok(true)
}
fn bundles_match(archived: &AssetBundle, current: &AssetBundle) -> bool {
if archived.provider != current.provider
|| archived.asset_id != current.asset_id
|| archived.format != current.format
|| archived.resolution != current.resolution
|| archived.files.len() != current.files.len()
{
return false;
}
if current.provider != AssetProvider::Sketchfab {
return archived == current;
}
archived
.files
.iter()
.zip(¤t.files)
.all(|(left, right)| left.path == right.path && left.role == right.role)
}
fn read_manifest_bounded(directory: &CapDir) -> Result<AssetManifest> {
let metadata = directory.symlink_metadata(MANIFEST_NAME)?;
if !metadata.file_type().is_file() || metadata.file_type().is_symlink() {
return Err(AssetForgeError::UnsafePath(MANIFEST_NAME.to_string()));
}
if metadata.len() > MAX_MANIFEST_BYTES {
return Err(AssetForgeError::ResponseLimitExceeded {
label: MANIFEST_NAME.to_string(),
size: metadata.len(),
limit: MAX_MANIFEST_BYTES,
});
}
let mut contents = Vec::with_capacity(metadata.len() as usize);
directory
.open(MANIFEST_NAME)?
.take(MAX_MANIFEST_BYTES.saturating_add(1))
.read_to_end(&mut contents)?;
Ok(serde_json::from_slice(&contents)?)
}
fn validate_manifest_identity(
manifest: &AssetManifest,
provider: AssetProvider,
asset_id: &str,
) -> Result<()> {
let valid_version = manifest.format_version == "2"
|| (manifest.format_version == "1" && provider == AssetProvider::PolyHaven);
let valid_source = match provider {
AssetProvider::PolyHaven => {
manifest.asset.source_url == format!("https://polyhaven.com/a/{asset_id}")
}
AssetProvider::AmbientCG => {
manifest.asset.source_url == format!("https://ambientcg.com/a/{asset_id}")
}
AssetProvider::Sketchfab => {
manifest
.asset
.source_url
.starts_with("https://sketchfab.com/3d-models/")
&& manifest.asset.source_url.ends_with(asset_id)
}
AssetProvider::GoogleScannedObjects => {
manifest.asset.source_url
== fuel_model_url(asset_id, None)
.map(|url| url.to_string())
.unwrap_or_default()
}
};
let valid_rights = match provider {
AssetProvider::PolyHaven
| AssetProvider::AmbientCG
| AssetProvider::GoogleScannedObjects => {
manifest.asset.rights == AssetRights::for_provider(provider)
}
AssetProvider::Sketchfab => {
manifest.asset.rights.license == "CC0-1.0"
&& manifest.asset.rights.license_url == SKETCHFAB_CC0_LICENSE_URL
&& manifest.asset.rights.api_terms_url == SKETCHFAB_API_TERMS_URL
&& manifest
.asset
.rights
.attribution
.ends_with(" via Sketchfab")
}
};
let valid = valid_version
&& manifest.complete
&& manifest.provider == provider
&& manifest.asset.provider == provider
&& manifest.bundle.provider == provider
&& manifest.asset.id == asset_id
&& manifest.bundle.asset_id == asset_id
&& valid_source
&& manifest.bundle.source_url == manifest.asset.source_url
&& valid_rights
&& manifest.bundle.rights == manifest.asset.rights;
if !valid {
return Err(AssetForgeError::VerificationFailed {
path: MANIFEST_NAME.to_string(),
message: "manifest identity or rights metadata is inconsistent".to_string(),
});
}
Ok(())
}
fn verify_manifest_files(asset_dir: &Path, manifest: &AssetManifest) -> Result<()> {
let directory = open_archive_directory(asset_dir)?;
verify_manifest_files_in_directory(&directory, manifest)
}
fn verify_manifest_files_in_directory(directory: &CapDir, manifest: &AssetManifest) -> Result<()> {
let mut seen_paths = BTreeSet::new();
let mut total_bytes = 0u64;
if manifest.bundle.files.is_empty() {
return Err(AssetForgeError::VerificationFailed {
path: MANIFEST_NAME.to_string(),
message: "bundle has no files".to_string(),
});
}
let check_md5 = manifest.provider.has_md5_verification();
for file in &manifest.bundle.files {
validate_bundle_path(&file.path)?;
trusted_download_url(manifest.provider, &file.url)?;
if !seen_paths.insert(file.path.clone()) {
return Err(verification_error(
file,
"duplicate manifest path".to_string(),
));
}
let path = path_from_manifest(&file.path)?;
let (actual_size, md5, blake3) = inspect_regular_file(directory, &path, file)?;
if actual_size != file.size {
return Err(verification_error(
file,
format!("expected {} bytes but found {actual_size}", file.size),
));
}
if check_md5 && !md5.eq_ignore_ascii_case(&file.md5) {
return Err(verification_error(
file,
format!("expected MD5 {} but found {md5}", file.md5),
));
}
if manifest.format_version == "2" {
let expected = manifest.file_blake3.get(&file.path).ok_or_else(|| {
verification_error(file, "manifest is missing a BLAKE3 digest".to_string())
})?;
if !blake3.eq_ignore_ascii_case(expected) {
return Err(verification_error(
file,
format!("expected BLAKE3 {expected} but found {blake3}"),
));
}
}
total_bytes = total_bytes.saturating_add(actual_size);
}
if total_bytes != manifest.bundle.total_bytes || total_bytes != manifest.bytes_written {
return Err(AssetForgeError::VerificationFailed {
path: MANIFEST_NAME.to_string(),
message: "manifest byte totals do not match archived files".to_string(),
});
}
Ok(())
}
fn inspect_regular_file(
directory: &CapDir,
relative: &Path,
file: &AssetFile,
) -> Result<(u64, String, String)> {
let components = relative.components().collect::<Vec<_>>();
let mut current = PathBuf::new();
for (index, component) in components.iter().enumerate() {
let Component::Normal(part) = component else {
return Err(AssetForgeError::UnsafePath(relative.display().to_string()));
};
current.push(part);
let metadata = directory.symlink_metadata(¤t)?;
if metadata.file_type().is_symlink() {
return Err(verification_error(
file,
format!("path component '{}' is a symlink", current.display()),
));
}
let is_last = index + 1 == components.len();
if (!is_last && !metadata.is_dir()) || (is_last && !metadata.is_file()) {
return Err(verification_error(
file,
format!(
"path component '{}' has the wrong file type",
current.display()
),
));
}
if is_last {
break;
}
}
let mut opened = directory.open(relative)?;
let metadata = opened.metadata()?;
if !metadata.is_file() {
return Err(verification_error(
file,
"archived path is not a regular file".to_string(),
));
}
let mut md5 = Md5::new();
let mut blake3 = blake3::Hasher::new();
let mut buffer = [0u8; 64 * 1024];
loop {
let read = opened.read(&mut buffer)?;
if read == 0 {
break;
}
md5.update(&buffer[..read]);
blake3.update(&buffer[..read]);
}
Ok((
metadata.len(),
format!("{:x}", md5.finalize()),
blake3.finalize().to_hex().to_string(),
))
}
fn open_archive_directory(path: &Path) -> Result<CapDir> {
let parent_path = path.parent().unwrap_or_else(|| Path::new("."));
let name = path
.file_name()
.ok_or_else(|| AssetForgeError::UnsafePath(path.display().to_string()))?;
let parent = CapDir::open_ambient_dir(parent_path, ambient_authority())?;
let directory = parent.open_dir(name)?;
if !directory.dir_metadata()?.is_dir() {
return Err(AssetForgeError::UnsafePath(path.display().to_string()));
}
Ok(directory)
}
fn ensure_replaceable_output(
asset_dir: &Path,
provider: AssetProvider,
asset_id: &str,
) -> Result<()> {
if !asset_dir.exists() {
return Ok(());
}
let directory = open_archive_directory(asset_dir).map_err(|error| {
AssetForgeError::UnrecognizedExistingOutput {
path: asset_dir.to_path_buf(),
reason: error.to_string(),
}
})?;
let manifest = read_manifest_bounded(&directory).map_err(|error| {
AssetForgeError::UnrecognizedExistingOutput {
path: asset_dir.to_path_buf(),
reason: error.to_string(),
}
})?;
validate_manifest_identity(&manifest, provider, asset_id).map_err(|error| {
AssetForgeError::UnrecognizedExistingOutput {
path: asset_dir.to_path_buf(),
reason: error.to_string(),
}
})
}
fn staging_path(output: &Path, asset_id: &str) -> Result<PathBuf> {
let nonce = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(|error| AssetForgeError::InvalidOption(error.to_string()))?
.as_nanos();
Ok(output.join(format!(
".assetforge-{}-{}-{nonce}.staging",
asset_directory_name(asset_id)?,
std::process::id()
)))
}
fn commit_staged_archive(staging: &Path, destination: &Path) -> Result<()> {
if !destination.exists() {
fs::rename(staging, destination)?;
return Ok(());
}
let file_name = destination
.file_name()
.and_then(|name| name.to_str())
.ok_or_else(|| AssetForgeError::UnsafePath(destination.display().to_string()))?;
let backup = destination.with_file_name(format!(
".assetforge-{file_name}-{}-backup",
std::process::id()
));
if backup.exists() {
return Err(AssetForgeError::UnrecognizedExistingOutput {
path: backup,
reason: "stale replacement backup exists".to_string(),
});
}
fs::rename(destination, &backup)?;
if let Err(error) = fs::rename(staging, destination) {
let _ = fs::rename(&backup, destination);
return Err(AssetForgeError::Io(error));
}
let _ = fs::remove_dir_all(backup);
Ok(())
}
fn progress_snapshot(
phase: ArchiveProgressPhase,
total_assets: usize,
summary: &ArchiveSummary,
) -> ArchiveProgress {
ArchiveProgress {
phase,
total_assets,
completed_assets: summary.archived + summary.skipped + summary.failed.len(),
succeeded_assets: summary.archived,
failed_assets: summary.failed.len(),
bytes_written: summary.bytes_written,
active_asset: None,
active_file: None,
active_bytes_written: 0,
active_total_bytes: None,
}
}
fn emit_progress(
progress: &mut Option<&mut dyn FnMut(ArchiveProgress)>,
snapshot: ArchiveProgress,
) {
if let Some(progress) = progress.as_deref_mut() {
progress(snapshot);
}
}
fn normalized_slug(value: &str) -> String {
value
.trim()
.chars()
.filter(|character| !matches!(character, '-' | '_' | ' ' | '\t' | '\n' | '\r'))
.flat_map(char::to_lowercase)
.collect()
}
fn non_empty_string(value: String) -> Option<String> {
let value = value.trim();
(!value.is_empty()).then(|| value.to_string())
}
fn parse_date_timestamp(value: &str) -> u64 {
let Some(date) = value.get(..10) else {
return 0;
};
let mut parts = date.split('-').filter_map(|part| part.parse::<i64>().ok());
let (Some(mut year), Some(month), Some(day), None) =
(parts.next(), parts.next(), parts.next(), parts.next())
else {
return 0;
};
if !(1..=12).contains(&month) || !(1..=31).contains(&day) {
return 0;
}
year -= i64::from(month <= 2);
let era = year.div_euclid(400);
let year_of_era = year - era * 400;
let adjusted_month = month + if month > 2 { -3 } else { 9 };
let day_of_year = (153 * adjusted_month + 2) / 5 + day - 1;
let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year;
let days_since_epoch = era * 146_097 + day_of_era - 719_468;
u64::try_from(days_since_epoch)
.unwrap_or_default()
.saturating_mul(86_400)
}
fn error_chain(error: &(dyn StdError + 'static)) -> String {
let mut message = error.to_string();
let mut source = error.source();
while let Some(error) = source {
message.push_str("; caused by: ");
message.push_str(&error.to_string());
source = error.source();
}
message
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
fn catalog_fixture() -> BTreeMap<String, PolyHavenAsset> {
serde_json::from_str(
r#"{
"Shelf_01": {
"name": "Shelf 01",
"description": "Weathered wooden shelving",
"categories": ["furniture", "shelves"],
"category": "Furniture/Storage Furniture/Shelving & Bookcases",
"tags": ["wood", "storage"],
"download_count": 100,
"polycount": 182,
"date_published": 10,
"thumbnail_url": "https://cdn.polyhaven.com/shelf.png"
},
"new_chair": {
"name": "New Chair",
"description": "Modern seating",
"categories": ["furniture", "seating"],
"category": "Furniture/Seating/Chairs",
"tags": ["chair"],
"download_count": 20,
"polycount": 500,
"date_published": 20,
"thumbnail_url": "https://cdn.polyhaven.com/chair.png"
}
}"#,
)
.unwrap()
}
#[test]
fn parses_poly_haven_targets() {
assert_eq!(
parse_asset_target("https://polyhaven.com/a/wooden_stool_02").unwrap(),
AssetTarget::Asset("wooden_stool_02".to_string())
);
assert_eq!(
parse_asset_target("search:wooden chair").unwrap(),
AssetTarget::Search("wooden chair".to_string())
);
assert_eq!(
parse_asset_target("category:furniture").unwrap(),
AssetTarget::Category("furniture".to_string())
);
assert_eq!(parse_asset_target("top").unwrap(), AssetTarget::Top);
assert!(parse_asset_target("https://example.com/a/chair").is_err());
assert!(parse_asset_target("https://polyhaven.com/models/chair").is_err());
assert_eq!(
parse_asset_target_for_provider(
"https://ambientcg.com/a/3DTreeStump001",
AssetProvider::AmbientCG,
)
.unwrap(),
AssetTarget::Asset("3DTreeStump001".to_string())
);
assert_eq!(
parse_asset_target_for_provider(
"https://sketchfab.com/3d-models/cargo-ship-45bf9e2128bd4c1b97ef3018e8163999",
AssetProvider::Sketchfab,
)
.unwrap(),
AssetTarget::Asset("45bf9e2128bd4c1b97ef3018e8163999".to_string())
);
}
#[test]
fn discovers_and_sorts_catalog_assets() {
let top = discover_catalog(catalog_fixture(), &AssetTarget::Top, 10).unwrap();
assert_eq!(top[0].id, "Shelf_01");
assert_eq!(top[0].provider, AssetProvider::PolyHaven);
assert_eq!(top[0].rights.license, "CC0-1.0");
let latest = discover_catalog(catalog_fixture(), &AssetTarget::Latest, 10).unwrap();
assert_eq!(latest[0].id, "new_chair");
let search = discover_catalog(
catalog_fixture(),
&AssetTarget::Search("storage".to_string()),
10,
)
.unwrap();
assert_eq!(search.len(), 1);
assert_eq!(search[0].id, "Shelf_01");
let category = discover_catalog(
catalog_fixture(),
&AssetTarget::Category("seating".to_string()),
10,
)
.unwrap();
assert_eq!(category[0].id, "new_chair");
let category_path = discover_catalog(
catalog_fixture(),
&AssetTarget::Category("Furniture/Seating/Chairs".to_string()),
10,
)
.unwrap();
assert_eq!(category_path[0].id, "new_chair");
}
#[test]
fn search_relevance_precedes_download_count() {
let summary = |id: &str, name: &str, description: &str, downloads: u64| AssetSummary {
provider: AssetProvider::Sketchfab,
id: id.to_string(),
name: name.to_string(),
description: description.to_string(),
categories: Vec::new(),
tags: Vec::new(),
download_count: downloads,
polycount: 0,
date_published: 0,
thumbnail_url: String::new(),
source_url: format!("https://sketchfab.com/3d-models/{name}-{id}"),
rights: AssetRights::sketchfab("Creator"),
};
let ranked = rank_and_limit(
vec![
summary(
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"Industrial Machine",
"Used for ship construction",
50_000,
),
summary(
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
"Cargo Ship",
"Ocean vessel",
200,
),
summary(
"cccccccccccccccccccccccccccccccc",
"Space Ship",
"Science fiction vessel",
1_000,
),
],
&AssetTarget::Search("ship".to_string()),
10,
)
.unwrap();
assert_eq!(ranked[0].name, "Space Ship");
assert_eq!(ranked[1].name, "Cargo Ship");
assert_eq!(ranked[2].name, "Industrial Machine");
}
#[test]
fn parses_current_ambientcg_v3_schema() {
let response: AmbientCGResponse = serde_json::from_str(
r#"{
"totalResults": 1,
"assets": [{
"id": "3DTreeStump001",
"type": "3d-model",
"title": "3D Tree Stump 001",
"url": "https://ambientcg.com/a/3DTreeStump001",
"tags": ["tree", "stump"],
"downloadStatistics": {"total": 29317},
"downloads": [{
"attributes": "HQ-2K-JPG",
"extension": "zip",
"url": "https://ambientcg.com/get?file=3DTreeStump001_HQ-2K-JPG.zip",
"size": 114813922
}],
"releaseDate": "2021-08-08"
}]
}"#,
)
.unwrap();
let summary = ambientcg_asset_summary(&response.assets[0]).unwrap();
assert_eq!(summary.id, "3DTreeStump001");
assert_eq!(summary.download_count, 29_317);
assert!(summary.date_published > 0);
assert_eq!(response.assets[0].downloads[0].attributes, "HQ-2K-JPG");
}
#[test]
fn parses_sketchfab_cc0_metadata_and_attribution() {
let model: SketchfabModel = serde_json::from_str(
r#"{
"uid": "45bf9e2128bd4c1b97ef3018e8163999",
"name": "Historic Ship",
"description": "A ship",
"categories": [{"name": "History", "slug": "history"}],
"tags": [{"name": "ship"}],
"user": {"username": "museum", "displayName": "Maritime Museum"},
"license": {"label": "CC0 Public Domain", "slug": "cc0"},
"viewerUrl": "https://sketchfab.com/3d-models/historic-ship-45bf9e2128bd4c1b97ef3018e8163999",
"isDownloadable": true,
"isAgeRestricted": false,
"downloadCount": 1234,
"faceCount": 40202,
"publishedAt": "2020-12-21T14:28:31.921139",
"updatedAt": "2025-10-03T15:28:36.313948",
"thumbnails": {"images": [{"url": "https://media.sketchfab.com/thumb.jpeg", "width": 720}]}
}"#,
)
.unwrap();
let summary = sketchfab_asset_summary(&model).unwrap();
assert_eq!(summary.download_count, 1_234);
assert_eq!(summary.tags, vec!["ship"]);
assert_eq!(summary.rights.attribution, "Maritime Museum via Sketchfab");
}
#[test]
fn validates_google_scanned_object_license_and_owner() {
let model: GoogleScannedModel = serde_json::from_str(
r#"{
"createdAt": "2020-09-03T17:19:00Z",
"updatedAt": "2026-07-10T17:00:51Z",
"name": "Boat_Shoe",
"owner": "GoogleResearch",
"description": "Boat Shoe",
"downloads": 2526,
"filesize": 6722193,
"license_id": 2,
"license_name": "Creative Commons Attribution 4.0 International",
"license_url": "http://creativecommons.org/licenses/by/4.0/",
"permission": 0,
"thumbnail_url": "/GoogleResearch/models/Boat_Shoe/tip/files/thumbnails/0.jpg",
"private": false,
"categories": ["Shoe"]
}"#,
)
.unwrap();
let summary = google_scanned_summary(&model).unwrap();
assert_eq!(summary.provider, AssetProvider::GoogleScannedObjects);
assert_eq!(summary.download_count, 2_526);
assert_eq!(summary.rights.license, "CC-BY-4.0");
let mut untrusted = model;
untrusted.owner = "SomeoneElse".to_string();
assert!(google_scanned_summary(&untrusted).is_err());
}
#[test]
fn resolves_scene_bundle_with_dependencies() {
let value: Value = serde_json::from_str(
r#"{
"gltf": {
"1k": {
"gltf": {
"size": 4,
"url": "https://dl.polyhaven.org/file/chair.gltf",
"md5": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"include": {
"chair.bin": {
"size": 8,
"url": "https://dl.polyhaven.org/file/chair.bin",
"md5": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
},
"textures/chair_diff.jpg": {
"size": 16,
"url": "https://dl.polyhaven.org/file/chair_diff.jpg",
"md5": "cccccccccccccccccccccccccccccccc"
}
}
}
}
}
}"#,
)
.unwrap();
let bundle =
bundle_from_value("chair", AssetFormat::Gltf, AssetResolution::OneK, &value).unwrap();
assert_eq!(bundle.files.len(), 3);
assert_eq!(bundle.total_bytes, 28);
assert_eq!(bundle.files[0].path, "chair.gltf");
assert_eq!(bundle.files[0].role, AssetFileRole::Scene);
assert!(
bundle
.files
.iter()
.any(|file| file.path == "textures/chair_diff.jpg")
);
}
#[test]
fn rejects_untrusted_downloads_and_unsafe_paths() {
assert!(
trusted_download_url(AssetProvider::PolyHaven, "http://dl.polyhaven.org/file/a")
.is_err()
);
assert!(
trusted_download_url(
AssetProvider::PolyHaven,
"https://dl.polyhaven.org.evil.test/file/a"
)
.is_err()
);
assert!(safe_relative_path("../secret").is_err());
assert!(safe_relative_path("/absolute").is_err());
assert_eq!(
safe_relative_path("textures/chair.jpg").unwrap(),
PathBuf::from("textures/chair.jpg")
);
}
#[test]
fn preserves_required_aliases_and_rejects_control_paths() {
let aliases: Value = serde_json::from_str(
r#"{
"gltf": {"1k": {"gltf": {
"size": 4,
"url": "https://dl.polyhaven.org/file/chair.gltf",
"md5": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"include": {
"first.bin": {
"size": 8,
"url": "https://dl.polyhaven.org/file/shared.bin",
"md5": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
},
"second.bin": {
"size": 8,
"url": "https://dl.polyhaven.org/file/shared.bin",
"md5": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
}
}
}}}
}"#,
)
.unwrap();
let bundle =
bundle_from_value("chair", AssetFormat::Gltf, AssetResolution::OneK, &aliases).unwrap();
assert_eq!(bundle.files.len(), 3);
assert_eq!(bundle.total_bytes, 20);
let reserved: Value = serde_json::from_str(
r#"{
"gltf": {"1k": {"gltf": {
"size": 4,
"url": "https://dl.polyhaven.org/file/chair.gltf",
"md5": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"include": {
"assetforge-asset.json": {
"size": 8,
"url": "https://dl.polyhaven.org/file/manifest.json",
"md5": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
}
}
}}}
}"#,
)
.unwrap();
assert!(matches!(
bundle_from_value("chair", AssetFormat::Gltf, AssetResolution::OneK, &reserved),
Err(AssetForgeError::UnsafePath(_))
));
}
#[test]
fn parses_format_and_resolution_aliases() {
assert_eq!("gltf".parse::<AssetFormat>().unwrap(), AssetFormat::Gltf);
assert_eq!(
"2048".parse::<AssetResolution>().unwrap(),
AssetResolution::TwoK
);
assert!("obj".parse::<AssetFormat>().is_err());
assert!("16k".parse::<AssetResolution>().is_err());
}
#[test]
fn serializes_stable_provider_and_resolution_slugs() {
assert_eq!(
serde_json::to_string(&AssetProvider::PolyHaven).unwrap(),
"\"polyhaven\""
);
assert_eq!(
serde_json::to_string(&AssetResolution::TwoK).unwrap(),
"\"2k\""
);
}
#[test]
fn skip_existing_verifies_every_archived_file() {
let temp = tempdir().unwrap();
let asset_dir = temp.path().join("chair");
fs::create_dir(&asset_dir).unwrap();
let bytes = b"gltf";
let checksum = format!("{:x}", Md5::digest(bytes));
fs::write(asset_dir.join("chair.gltf"), bytes).unwrap();
let rights = AssetRights::poly_haven();
let asset = AssetSummary {
provider: AssetProvider::PolyHaven,
id: "chair".to_string(),
name: "Chair".to_string(),
description: String::new(),
categories: vec!["furniture".to_string()],
tags: vec!["chair".to_string()],
download_count: 1,
polycount: 1,
date_published: 1,
thumbnail_url: String::new(),
source_url: "https://polyhaven.com/a/chair".to_string(),
rights: rights.clone(),
};
let file = AssetFile {
path: "chair.gltf".to_string(),
size: bytes.len() as u64,
url: "https://dl.polyhaven.org/file/chair.gltf".to_string(),
md5: checksum,
role: AssetFileRole::Scene,
};
let bundle = AssetBundle {
provider: AssetProvider::PolyHaven,
asset_id: "chair".to_string(),
source_url: asset.source_url.clone(),
rights,
format: AssetFormat::Gltf,
resolution: AssetResolution::OneK,
files: vec![file],
total_bytes: bytes.len() as u64,
};
write_manifest(
&asset_dir,
&AssetManifest {
format_version: "1".to_string(),
provider: AssetProvider::PolyHaven,
asset,
bundle: bundle.clone(),
provider_revision: String::new(),
bytes_written: bytes.len() as u64,
file_blake3: BTreeMap::new(),
complete: true,
},
)
.unwrap();
assert!(
matching_complete_manifest(&asset_dir, AssetProvider::PolyHaven, "chair", &bundle, "")
.unwrap()
);
fs::write(asset_dir.join("chair.gltf"), b"evil").unwrap();
assert!(
matching_complete_manifest(&asset_dir, AssetProvider::PolyHaven, "chair", &bundle, "")
.is_err()
);
}
#[test]
fn staged_commit_replaces_destination_without_partial_visibility() {
let temp = tempdir().unwrap();
let destination = temp.path().join("chair");
let staging = temp.path().join("staging");
fs::create_dir(&destination).unwrap();
fs::create_dir(&staging).unwrap();
fs::write(destination.join("old"), b"old").unwrap();
fs::write(staging.join("new"), b"new").unwrap();
commit_staged_archive(&staging, &destination).unwrap();
assert!(!staging.exists());
assert!(!destination.join("old").exists());
assert_eq!(fs::read(destination.join("new")).unwrap(), b"new");
}
#[test]
#[ignore = "requires Poly Haven network access and downloads a 1K glTF bundle"]
fn live_poly_haven_archive_and_verified_skip() {
let forge = AssetForge::with_user_agent(
AssetProvider::PolyHaven,
"assetforge-live-test/0.1 (Tknott95/AssetForge)",
)
.unwrap();
let assets = forge
.discover(&AssetTarget::Asset("Shelf_01".to_string()))
.unwrap();
assert_eq!(assets.len(), 1);
assert_eq!(assets[0].rights.attribution, "Poly Haven");
let temp = tempdir().unwrap();
let options = ArchiveOptions {
output: temp.path().to_path_buf(),
format: AssetFormat::Gltf,
resolution: AssetResolution::OneK,
skip_existing: false,
max_download_bytes: 256 * 1024 * 1024,
};
let mut progress_events = Vec::new();
let summary = forge
.archive_assets_with_progress(&assets, &options, |progress| {
progress_events.push(progress)
})
.unwrap();
assert!(summary.is_success(), "failures: {:?}", summary.failed);
assert_eq!(summary.archived, 1);
assert!(progress_events.iter().any(|event| {
event.phase == ArchiveProgressPhase::Downloading && event.active_bytes_written > 0
}));
let manifest =
read_manifest_bounded(&open_archive_directory(&temp.path().join("Shelf_01")).unwrap())
.unwrap();
assert!(manifest.complete);
assert_eq!(manifest.format_version, "2");
assert_eq!(manifest.file_blake3.len(), manifest.bundle.files.len());
assert_eq!(manifest.asset.rights.license, "CC0-1.0");
let skipped = forge
.archive_assets(
&assets,
&ArchiveOptions {
skip_existing: true,
..options
},
)
.unwrap();
assert_eq!(skipped.skipped, 1);
assert!(skipped.is_success());
}
#[test]
#[ignore = "requires ambientCG network access and downloads a 1K model archive"]
fn live_ambientcg_archive_follows_validated_redirect_and_reports_progress() {
let forge = AssetForge::with_user_agent(
AssetProvider::AmbientCG,
"assetforge-live-test/0.1 (Tknott95/AssetForge)",
)
.unwrap()
.with_discovery_limit(5);
let assets = forge
.discover(&AssetTarget::Asset("3DTreeStump001".to_string()))
.unwrap();
let temp = tempdir().unwrap();
let options = ArchiveOptions {
output: temp.path().to_path_buf(),
format: AssetFormat::Archive,
resolution: AssetResolution::OneK,
skip_existing: false,
max_download_bytes: 256 * 1024 * 1024,
};
let mut saw_bytes = false;
let summary = forge
.archive_assets_with_progress(&assets, &options, |progress| {
saw_bytes |= progress.active_bytes_written > 0;
})
.unwrap();
assert!(summary.is_success(), "failures: {:?}", summary.failed);
assert!(saw_bytes);
let manifest = read_manifest_bounded(
&open_archive_directory(&temp.path().join("3DTreeStump001")).unwrap(),
)
.unwrap();
assert_eq!(manifest.bundle.format, AssetFormat::Archive);
assert_eq!(manifest.bundle.resolution, AssetResolution::OneK);
assert_eq!(manifest.file_blake3.len(), 1);
}
#[test]
#[ignore = "requires public Sketchfab API access"]
fn live_sketchfab_cc0_search_is_relevance_ranked() {
let forge = AssetForge::with_user_agent(
AssetProvider::Sketchfab,
"assetforge-live-test/0.1 (Tknott95/AssetForge)",
)
.unwrap()
.with_discovery_limit(5);
let target = AssetTarget::Search("ship".to_string());
let assets = forge.discover(&target).unwrap();
assert!(!assets.is_empty());
assert!(assets.iter().all(|asset| {
asset.provider == AssetProvider::Sketchfab
&& asset.rights.license == "CC0-1.0"
&& asset.rights.attribution.ends_with(" via Sketchfab")
}));
let scores = assets
.iter()
.map(|asset| relevance_score(asset, "ship").unwrap())
.collect::<Vec<_>>();
assert!(scores[0] >= 400, "top result only matched description text");
assert!(scores.windows(2).all(|pair| pair[0] >= pair[1]));
assert!(assets.windows(2).all(|pair| {
let left = relevance_score(&pair[0], "ship").unwrap();
let right = relevance_score(&pair[1], "ship").unwrap();
left != right || pair[0].download_count >= pair[1].download_count
}));
assert!(matches!(
forge.bundle(&assets[0].id, AssetFormat::Auto, AssetResolution::Auto),
Err(AssetForgeError::AuthenticationRequired { .. })
));
}
#[test]
#[ignore = "requires Gazebo Fuel access and downloads one Google Scanned Object archive"]
fn live_google_scanned_object_archive_and_progress() {
let id = "Mens_Authentic_Original_Boat_Shoe_in_Navy_Leather_NHHQddDLQys";
let forge = AssetForge::with_user_agent(
AssetProvider::GoogleScannedObjects,
"assetforge-live-test/0.3 (Tknott95/AssetForge)",
)
.unwrap()
.with_discovery_limit(3);
let search = forge
.discover(&AssetTarget::Search("boat shoe".to_string()))
.unwrap();
assert!(!search.is_empty());
assert!(search.iter().all(|asset| {
asset.provider == AssetProvider::GoogleScannedObjects
&& asset.rights.license == "CC-BY-4.0"
}));
let assets = forge.discover(&AssetTarget::Asset(id.to_string())).unwrap();
let temp = tempdir().unwrap();
let options = ArchiveOptions {
output: temp.path().to_path_buf(),
format: AssetFormat::Auto,
resolution: AssetResolution::Auto,
skip_existing: false,
max_download_bytes: 32 * 1024 * 1024,
};
let mut saw_bytes = false;
let summary = forge
.archive_assets_with_progress(&assets, &options, |progress| {
saw_bytes |= progress.active_bytes_written > 0;
})
.unwrap();
assert!(summary.is_success(), "failures: {:?}", summary.failed);
assert!(saw_bytes);
let manifest =
read_manifest_bounded(&open_archive_directory(&temp.path().join(id)).unwrap()).unwrap();
assert_eq!(manifest.bundle.format, AssetFormat::Archive);
assert_eq!(manifest.bundle.resolution, AssetResolution::Original);
assert_eq!(manifest.asset.rights.license, "CC-BY-4.0");
assert_eq!(manifest.file_blake3.len(), 1);
}
}