1use rand::SeedableRng;
38use rand::rngs::StdRng;
39use rusting_brain::mesh::{Bvh, Mesh, QuerySampling};
40use rusting_brain::safetensors::{self, Dtype, SafeTensors};
41use std::collections::BTreeMap;
42use std::path::{Path, PathBuf};
43use std::time::Instant;
44
45struct Args {
46 meshes: PathBuf,
47 out: PathBuf,
48 shard: usize,
49 surface: usize,
50 queries: usize,
51 near_surface: f32,
52 jitter: f32,
53 extent: f32,
54 seed: u64,
55}
56
57fn main() -> Result<(), Box<dyn std::error::Error>> {
58 let args = parse()?;
59 std::fs::create_dir_all(&args.out)?;
60
61 let mut paths: Vec<PathBuf> = std::fs::read_dir(&args.meshes)?
62 .filter_map(|entry| entry.ok().map(|entry| entry.path()))
63 .filter(|path| {
64 path.extension().is_some_and(|extension| {
65 matches!(extension.to_ascii_lowercase().to_str(), Some("obj" | "glb"))
66 })
67 })
68 .collect();
69 paths.sort();
70 if paths.is_empty() {
71 return Err(format!("no .obj or .glb files in {}", args.meshes.display()).into());
72 }
73 println!(
74 "{} meshes, {} per shard, {} surface and {} query points each",
75 paths.len(),
76 args.shard,
77 args.surface,
78 args.queries
79 );
80
81 let sampling = QuerySampling {
82 count: args.queries,
83 near_surface: args.near_surface,
84 jitter: args.jitter,
85 extent: args.extent,
86 };
87
88 let mut index: BTreeMap<String, String> = BTreeMap::new();
89 let mut skipped = 0;
90 let mut colored = 0;
91 let started = Instant::now();
92 let shards = paths.len().div_ceil(args.shard);
93 for (number, chunk) in paths.chunks(args.shard).enumerate() {
94 let shard = format!("samples-{number:05}.safetensors");
95
96 let done = completed(&args.out.join(&shard));
98 if !done.is_empty() {
99 for stem in &done {
100 index.insert(stem.clone(), shard.clone());
101 }
102 write_index(&args.out, &index)?;
103 println!("{shard}: {} meshes, already written", done.len());
104 continue;
105 }
106 let mut sampled: Vec<(String, [usize; 2], Vec<f32>)> = Vec::new();
109 let mut meshes = 0;
110 for path in chunk {
111 let stem = path
112 .file_stem()
113 .and_then(|stem| stem.to_str())
114 .ok_or_else(|| format!("{} has no usable name", path.display()))?
115 .to_string();
116
117 let mesh = match read(path) {
119 Ok(mesh) => mesh,
120 Err(error) => {
121 eprintln!("skipping {}: {error}", path.display());
122 skipped += 1;
123 continue;
124 }
125 };
126 let mut rng = StdRng::seed_from_u64(args.seed ^ hash(&stem));
129 let mut mesh = mesh;
130 let placement = mesh.normalize();
131 let bvh = Bvh::build(&mesh);
132
133 let (points, normals, colors) = bvh.sample_surface_colored(args.surface, &mut rng);
134 let mut surface = Vec::with_capacity(args.surface * 6);
135 for (point, normal) in points.iter().zip(&normals) {
136 surface.extend(point);
137 surface.extend(normal);
138 }
139
140 let queries = bvh.sample_queries(&sampling, &mut rng);
141 let distances = bvh.signed_distance(&queries);
142 let mut samples = Vec::with_capacity(queries.len() * 4);
143 for (query, distance) in queries.iter().zip(&distances) {
144 samples.extend(query);
145 samples.push(*distance);
146 }
147
148 index.insert(stem.clone(), shard.clone());
149 meshes += 1;
150 sampled.push((format!("{stem}.surface"), [points.len(), 6], surface));
151 if !colors.is_empty() {
154 colored += 1;
155 sampled.push((format!("{stem}.colors"), [colors.len(), 3], colors.concat()));
156 }
157 sampled.push((format!("{stem}.queries"), [queries.len(), 4], samples));
158 sampled.push((
159 format!("{stem}.transform"),
160 [1, 4],
161 vec![
162 placement.center[0],
163 placement.center[1],
164 placement.center[2],
165 placement.scale,
166 ],
167 ));
168 }
169 if sampled.is_empty() {
170 continue;
171 }
172
173 let borrowed: Vec<(&str, &[usize], &[f32])> = sampled
174 .iter()
175 .map(|(name, shape, data)| (name.as_str(), &shape[..], &data[..]))
176 .collect();
177 safetensors::write_as(
178 args.out.join(&shard),
179 &borrowed,
180 &BTreeMap::from([
181 ("source".to_string(), "mesh_samples".to_string()),
182 ("surface".to_string(), args.surface.to_string()),
183 ("queries".to_string(), args.queries.to_string()),
184 ]),
185 Dtype::Bf16,
186 )?;
187 write_index(&args.out, &index)?;
188 println!(
189 "{shard}: {meshes} meshes, {}",
190 progress(started, number + 1, shards)
191 );
192 }
193
194 write_index(&args.out, &index)?;
195 println!(
196 "wrote {} entries to index.json, {colored} with colour, skipped {skipped}",
197 index.len()
198 );
199 Ok(())
200}
201
202fn completed(path: &Path) -> Vec<String> {
205 let Ok(shard) = SafeTensors::open(path) else {
206 return Vec::new();
207 };
208 shard
209 .names()
210 .filter_map(|name| name.strip_suffix(".surface").map(str::to_string))
211 .collect()
212}
213
214fn write_index(
217 out: &Path,
218 index: &BTreeMap<String, String>,
219) -> Result<(), Box<dyn std::error::Error>> {
220 std::fs::write(out.join("index.json"), serde_json::to_vec_pretty(index)?)?;
221 Ok(())
222}
223
224fn progress(started: Instant, done: usize, total: usize) -> String {
227 let elapsed = started.elapsed().as_secs_f32();
228 let remaining = elapsed / done.max(1) as f32 * total.saturating_sub(done) as f32;
229 format!(
230 "{done}/{total} shards, {} elapsed, {} left",
231 clock(elapsed),
232 clock(remaining)
233 )
234}
235
236fn clock(seconds: f32) -> String {
237 let seconds = seconds as u64;
238 match seconds / 60 {
239 0 => format!("{seconds}s"),
240 minutes if minutes < 60 => format!("{minutes}m"),
241 minutes => format!("{}h{:02}m", minutes / 60, minutes % 60),
242 }
243}
244
245fn read(path: &std::path::Path) -> Result<Mesh, rusting_brain::NetworkError> {
246 match path.extension().and_then(|e| e.to_str()) {
247 Some("glb") => Mesh::read_glb(path),
248 _ => Mesh::read_obj(path),
249 }
250}
251
252fn hash(name: &str) -> u64 {
255 let mut hash = 0xcbf2_9ce4_8422_2325u64;
256 for byte in name.as_bytes() {
257 hash = (hash ^ u64::from(*byte)).wrapping_mul(0x1000_0000_01b3);
258 }
259 hash
260}
261
262fn parse() -> Result<Args, Box<dyn std::error::Error>> {
263 let mut args = Args {
264 meshes: PathBuf::new(),
265 out: PathBuf::from("samples"),
266 shard: 64,
267 surface: 8192,
268 queries: 200_000,
269 near_surface: 0.7,
270 jitter: 0.02,
271 extent: 1.1,
272 seed: 0,
273 };
274 let mut rest = std::env::args().skip(1);
275 while let Some(flag) = rest.next() {
276 let mut value = || rest.next().ok_or(format!("{flag} needs a value"));
277 match flag.as_str() {
278 "--meshes" => args.meshes = value()?.into(),
279 "--out" => args.out = value()?.into(),
280 "--shard" => args.shard = value()?.parse()?,
281 "--surface" => args.surface = value()?.parse()?,
282 "--queries" => args.queries = value()?.parse()?,
283 "--near-surface" => args.near_surface = value()?.parse()?,
284 "--jitter" => args.jitter = value()?.parse()?,
285 "--extent" => args.extent = value()?.parse()?,
286 "--seed" => args.seed = value()?.parse()?,
287 other => return Err(format!("unknown flag {other}").into()),
288 }
289 }
290 if args.meshes.as_os_str().is_empty() {
291 return Err("usage: --meshes <dir> [--out <dir>] [--shard n] [--surface n] [--queries n] [--near-surface f] [--jitter f] [--extent f] [--seed n]".into());
292 }
293 if args.shard == 0 {
294 return Err("--shard needs at least one mesh".into());
295 }
296 if !(0.0..=1.0).contains(&args.near_surface) {
297 return Err("--near-surface is the fraction of queries near the surface, so 0 to 1".into());
298 }
299 Ok(args)
300}