flatland_client_lib/
content_pack.rs

1//! Sim content packs — versioned `assets/` trees published beside client gfx.
2
3use std::collections::BTreeMap;
4use std::path::{Path, PathBuf};
5
6use anyhow::Context;
7
8use crate::asset_backend::{mime_for_path, AssetBackendConfig, AssetStore};
9use crate::assets::{sha256_file, AssetBundleIndex, AssetFileEntry};
10
11/// Relative directories under `assets/` included in the sim pack (worker ContentBundle).
12pub const SIM_PACK_ASSET_DIRS: &[&str] = &[
13    "crafting",
14    "items",
15    "world",
16    "config",
17    "npcs",
18    "loot",
19    "combat",
20    "dialogue",
21    "quests",
22    "crops",
23    "gfx/sprites",
24    "paperdoll",
25];
26
27/// Build a file index for sim content under `repo_root/assets/…`.
28pub fn build_sim_pack_index(
29    repo_root: &Path,
30    publish_rev: u64,
31    published_at: &str,
32) -> anyhow::Result<AssetBundleIndex> {
33    let assets = repo_root.join("assets");
34    let mut files = BTreeMap::new();
35    for rel_dir in SIM_PACK_ASSET_DIRS {
36        let dir = assets.join(rel_dir);
37        if dir.is_dir() {
38            walk_into(&dir, &assets, &mut files)?;
39        }
40    }
41    // Publish marker so workers know the pack rev without a separate channel.
42    let marker = assets.join(".content-publish.json");
43    if marker.is_file() {
44        let meta = std::fs::metadata(&marker)?;
45        files.insert(
46            ".content-publish.json".to_string(),
47            AssetFileEntry {
48                sha256: sha256_file(&marker)?,
49                size: meta.len(),
50            },
51        );
52    }
53    if files.is_empty() {
54        anyhow::bail!(
55            "sim pack is empty — expected content under {}",
56            assets.display()
57        );
58    }
59    Ok(AssetBundleIndex {
60        publish_rev,
61        published_at: published_at.to_string(),
62        files,
63    })
64}
65
66fn walk_into(
67    dir: &Path,
68    assets_root: &Path,
69    files: &mut BTreeMap<String, AssetFileEntry>,
70) -> anyhow::Result<()> {
71    for entry in std::fs::read_dir(dir).with_context(|| format!("read_dir {}", dir.display()))? {
72        let entry = entry?;
73        let path = entry.path();
74        let name = entry.file_name();
75        let name_str = name.to_string_lossy();
76        if name_str.starts_with('.') || name_str == "node_modules" {
77            continue;
78        }
79        if name_str.ends_with(".map-locks.json") {
80            continue;
81        }
82        if path.is_dir() {
83            walk_into(&path, assets_root, files)?;
84            continue;
85        }
86        if !path.is_file() {
87            continue;
88        }
89        let rel = path
90            .strip_prefix(assets_root)
91            .with_context(|| format!("strip {}", path.display()))?
92            .to_string_lossy()
93            .replace('\\', "/");
94        let meta = std::fs::metadata(&path)?;
95        files.insert(
96            rel,
97            AssetFileEntry {
98                sha256: sha256_file(&path)?,
99                size: meta.len(),
100            },
101        );
102    }
103    Ok(())
104}
105
106/// Resolve an on-disk path for a sim pack relative key (`world/segments/…`).
107pub fn resolve_sim_pack_file(repo_root: &Path, relative: &str) -> PathBuf {
108    repo_root.join("assets").join(relative)
109}
110
111/// Upload sim pack files + `latest.json` via the configured asset store.
112pub async fn upload_sim_pack(
113    store: &dyn AssetStore,
114    cfg: &AssetBackendConfig,
115    repo_root: &Path,
116    index: &AssetBundleIndex,
117) -> anyhow::Result<()> {
118    for (rel, _) in &index.files {
119        let path = resolve_sim_pack_file(repo_root, rel);
120        let bytes = std::fs::read(&path)
121            .with_context(|| format!("read sim pack file {}", path.display()))?;
122        let key = cfg.sim_object_key(index.publish_rev, rel);
123        store
124            .put(&key, mime_for_path(&path), &bytes)
125            .await
126            .with_context(|| format!("upload sim {key}"))?;
127    }
128    let latest = serde_json::to_vec(index)?;
129    store
130        .put(&cfg.sim_index_object(), "application/json", &latest)
131        .await
132        .context("upload sim latest.json")?;
133    Ok(())
134}
135
136/// Download sim pack `publish_rev` into `dest_root/assets/…` (repo-shaped tree).
137pub async fn download_sim_pack(
138    store: &dyn AssetStore,
139    cfg: &AssetBackendConfig,
140    index: &AssetBundleIndex,
141    dest_root: &Path,
142) -> anyhow::Result<PathBuf> {
143    let assets_out = dest_root.join("assets");
144    std::fs::create_dir_all(&assets_out)?;
145
146    const CONCURRENCY: usize = 8;
147    let pending: Vec<(String, AssetFileEntry)> = index
148        .files
149        .iter()
150        .map(|(k, v)| (k.clone(), v.clone()))
151        .collect();
152    let total = pending.len();
153    let mut join_set = tokio::task::JoinSet::new();
154    let mut next = 0usize;
155    let done = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
156
157    // Local backend: read files from disk concurrently via blocking pool.
158    // Remote: each task uses its own HTTP client + config clone.
159    while next < pending.len() || !join_set.is_empty() {
160        while join_set.len() < CONCURRENCY && next < pending.len() {
161            let (rel, entry) = pending[next].clone();
162            next += 1;
163            let cfg = cfg.clone();
164            let assets_out = assets_out.clone();
165            let publish_rev = index.publish_rev;
166            let done = std::sync::Arc::clone(&done);
167            join_set.spawn(async move {
168                let store = crate::asset_backend::store_from_config(&cfg)?;
169                let key = cfg.sim_object_key(publish_rev, &rel);
170                let bytes = store
171                    .get(&key)
172                    .await
173                    .with_context(|| format!("download sim {key}"))?;
174                let got = {
175                    use sha2::{Digest, Sha256};
176                    hex::encode(Sha256::digest(&bytes))
177                };
178                if got != entry.sha256 {
179                    anyhow::bail!("sim pack hash mismatch for {rel}");
180                }
181                let dest = assets_out.join(&rel);
182                if let Some(parent) = dest.parent() {
183                    std::fs::create_dir_all(parent)?;
184                }
185                std::fs::write(&dest, &bytes)?;
186                let n = done.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1;
187                if n % 100 == 0 || n == total {
188                    tracing::info!(n, total, "sim pack download progress");
189                }
190                Ok::<(), anyhow::Error>(())
191            });
192        }
193        if let Some(res) = join_set.join_next().await {
194            res.map_err(|e| anyhow::anyhow!("sim pack download join: {e}"))??;
195        }
196    }
197    Ok(dest_root.to_path_buf())
198}
199
200/// Cache directory for a downloaded sim pack rev.
201pub fn sim_pack_cache_dir(publish_rev: u64) -> anyhow::Result<PathBuf> {
202    let base = dirs::home_dir().ok_or_else(|| anyhow::anyhow!("home directory not found"))?;
203    Ok(base
204        .join(".flatland3")
205        .join("sim-packs")
206        .join(format!("rev-{publish_rev}")))
207}
208
209/// Fetch sim `latest.json`, download pack if needed, return repo-shaped root.
210///
211/// Auto-heal for Oracle clone/rsync: when the checkout's `assets/.content-publish.json`
212/// is ahead of (or equal to, with `FLATLAND_SIM_PACK_PREFER_REPO_ASSETS=1`) the remote
213/// `latest.json`, seed `~/.flatland3/sim-packs/rev-<N>/` from local `assets/` instead of
214/// re-downloading a stale pack that is missing newly cloned item templates.
215pub async fn ensure_sim_pack_cached(target_rev: Option<u64>) -> anyhow::Result<(u64, PathBuf)> {
216    let cfg = AssetBackendConfig::from_env();
217    let store = crate::asset_backend::store_from_config(&cfg)?;
218    let index_bytes = store
219        .get(&cfg.sim_index_object())
220        .await
221        .with_context(|| format!("fetch sim latest.json ({})", cfg.sim_index_object()))?;
222    let index: AssetBundleIndex = serde_json::from_slice(&index_bytes)?;
223    let remote_rev = index.publish_rev;
224
225    if let Some((local_rev, dest)) = maybe_seed_sim_pack_from_repo_assets(remote_rev)? {
226        return Ok((local_rev, dest));
227    }
228
229    let rev = target_rev.unwrap_or(remote_rev);
230    if rev != remote_rev {
231        anyhow::bail!(
232            "requested sim pack rev {rev} but remote latest is {remote_rev}"
233        );
234    }
235    let dest = sim_pack_cache_dir(rev)?;
236    if sim_pack_cache_complete(&dest, &index)? {
237        return Ok((rev, dest));
238    }
239    // Incomplete / corrupt cache — wipe and re-download.
240    let _ = std::fs::remove_dir_all(&dest);
241    download_sim_pack(store.as_ref(), &cfg, &index, &dest).await?;
242    if !sim_pack_cache_complete(&dest, &index)? {
243        anyhow::bail!("sim pack rev {rev} incomplete after download");
244    }
245    Ok((rev, dest))
246}
247
248fn prefer_repo_assets_over_remote() -> bool {
249    std::env::var("FLATLAND_SIM_PACK_PREFER_REPO_ASSETS")
250        .map(|v| matches!(v.as_str(), "1" | "true" | "TRUE" | "yes" | "YES"))
251        .unwrap_or(true)
252}
253
254/// When the repo checkout has a publish marker at/above remote latest, materialize a
255/// sim-pack cache from `assets/` so cloned content (e.g. new harvest templates) wins
256/// over a stale Firebase/GCS pack.
257fn maybe_seed_sim_pack_from_repo_assets(
258    remote_rev: u64,
259) -> anyhow::Result<Option<(u64, PathBuf)>> {
260    if !prefer_repo_assets_over_remote() {
261        return Ok(None);
262    }
263    let Some(repo) = find_repo_root_for_sim_pack() else {
264        return Ok(None);
265    };
266    let marker_path = repo.join("assets/.content-publish.json");
267    let Ok(text) = std::fs::read_to_string(&marker_path) else {
268        return Ok(None);
269    };
270    let marker: serde_json::Value = serde_json::from_str(&text)?;
271    let local_rev = marker
272        .get("rev")
273        .and_then(|v| v.as_u64())
274        .ok_or_else(|| anyhow::anyhow!("assets/.content-publish.json missing rev"))?;
275    // Prefer local when ahead of remote. Equal rev still seeds so rsynced YAML that was
276    // never re-uploaded (marker not bumped) replaces a stale cache built from remote.
277    if local_rev < remote_rev {
278        return Ok(None);
279    }
280    let assets = repo.join("assets");
281    if !assets.join("items").is_dir() {
282        return Ok(None);
283    }
284    let dest = sim_pack_cache_dir(local_rev)?;
285    let dest_assets = dest.join("assets");
286    std::fs::create_dir_all(&dest_assets)?;
287    // Refresh from checkout assets (rsync-like copy of sim pack dirs + marker).
288    for rel_dir in SIM_PACK_ASSET_DIRS {
289        let src = assets.join(rel_dir);
290        if !src.is_dir() {
291            continue;
292        }
293        let dst = dest_assets.join(rel_dir);
294        copy_dir_recursive(&src, &dst)?;
295    }
296    if marker_path.is_file() {
297        std::fs::copy(&marker_path, dest_assets.join(".content-publish.json"))?;
298    }
299    // Stamp so operators can see the pack was seeded locally.
300    let stamp = serde_json::json!({
301        "seeded_from": assets.display().to_string(),
302        "local_rev": local_rev,
303        "remote_latest_seen": remote_rev,
304        "reason": "repo assets at/above remote sim latest — prefer local (clone/rsync auto-heal)",
305    });
306    std::fs::write(
307        dest.join(".seeded-from-repo.json"),
308        serde_json::to_string_pretty(&stamp)?,
309    )?;
310    Ok(Some((local_rev, dest)))
311}
312
313fn find_repo_root_for_sim_pack() -> Option<PathBuf> {
314    if let Ok(root) = std::env::var("FLATLAND_REPO_ROOT") {
315        let p = PathBuf::from(root);
316        if p.join("assets/items").is_dir() {
317            return Some(p);
318        }
319    }
320    // Worker cwd is typically the checkout (Oracle: /home/ubuntu/flatland).
321    if let Ok(cwd) = std::env::current_dir() {
322        let mut dir = cwd;
323        for _ in 0..6 {
324            if dir.join("assets/items").is_dir() && dir.join("assets/world").is_dir() {
325                return Some(dir);
326            }
327            if !dir.pop() {
328                break;
329            }
330        }
331    }
332    None
333}
334
335fn copy_dir_recursive(src: &Path, dst: &Path) -> anyhow::Result<()> {
336    std::fs::create_dir_all(dst)?;
337    for entry in std::fs::read_dir(src).with_context(|| format!("read_dir {}", src.display()))? {
338        let entry = entry?;
339        let name = entry.file_name();
340        let name_str = name.to_string_lossy();
341        if name_str.starts_with('.') || name_str == "node_modules" {
342            continue;
343        }
344        if name_str.ends_with(".map-locks.json") {
345            continue;
346        }
347        let from = entry.path();
348        let to = dst.join(&name);
349        if from.is_dir() {
350            copy_dir_recursive(&from, &to)?;
351        } else if from.is_file() {
352            if let Some(parent) = to.parent() {
353                std::fs::create_dir_all(parent)?;
354            }
355            std::fs::copy(&from, &to)
356                .with_context(|| format!("copy {} → {}", from.display(), to.display()))?;
357        }
358    }
359    Ok(())
360}
361
362fn sim_pack_cache_complete(dest_root: &Path, index: &AssetBundleIndex) -> anyhow::Result<bool> {
363    let assets = dest_root.join("assets");
364    if !assets.is_dir() {
365        return Ok(false);
366    }
367    for (rel, entry) in &index.files {
368        let path = assets.join(rel);
369        if !path.is_file() {
370            return Ok(false);
371        }
372        let meta = std::fs::metadata(&path)?;
373        if meta.len() != entry.size && entry.size > 0 {
374            // Size mismatch — treat as incomplete (hash check is expensive for every boot).
375            return Ok(false);
376        }
377    }
378    Ok(true)
379}
380
381#[cfg(test)]
382mod tests {
383    use super::*;
384    use crate::asset_backend::LocalAssetStore;
385
386    #[tokio::test]
387    async fn local_sim_pack_roundtrip() {
388        let repo = match flatland_repo_fixture() {
389            Some(p) => p,
390            None => return,
391        };
392        let tmp =
393            std::env::temp_dir().join(format!("flatland-sim-pack-test-{}", std::process::id()));
394        let _ = std::fs::remove_dir_all(&tmp);
395        std::fs::create_dir_all(&tmp).unwrap();
396        let store = LocalAssetStore {
397            root: tmp.join("store"),
398        };
399        let cfg = AssetBackendConfig {
400            kind: crate::asset_backend::AssetBackendKind::Local,
401            bucket: "local".into(),
402            client_prefix: "client".into(),
403            sim_prefix: "sim".into(),
404            local_root: tmp.join("store"),
405            s3_region: "us-east-1".into(),
406            s3_endpoint: None,
407            index_url_override: None,
408        };
409        let index = build_sim_pack_index(&repo, 42, "test").expect("index");
410        assert!(!index.files.is_empty());
411        upload_sim_pack(&store, &cfg, &repo, &index)
412            .await
413            .expect("upload");
414        let dest = tmp.join("out");
415        download_sim_pack(&store, &cfg, &index, &dest)
416            .await
417            .expect("download");
418        assert!(dest.join("assets/world").is_dir() || dest.join("assets/items").is_dir());
419        let _ = std::fs::remove_dir_all(&tmp);
420    }
421
422    fn flatland_repo_fixture() -> Option<PathBuf> {
423        let mut dir = std::env::current_dir().ok()?;
424        for _ in 0..8 {
425            if dir.join("assets/world/segments").is_dir() {
426                return Some(dir);
427            }
428            if !dir.pop() {
429                break;
430            }
431        }
432        None
433    }
434}
flatland_client_lib/content_pack.rs

flatland_client_lib/
content_pack.rs
