1use std::fs;
9use std::path::Path;
10use std::process::Command;
11
12use crate::util::prompt;
13
14#[derive(Debug, Clone, Copy, PartialEq, Eq)]
15enum Mode {
16 Mounted,
17 Docker,
18 Native,
19}
20
21pub fn run(
22 name: Option<&str>,
23 docker: bool,
24 native: bool,
25 with_hf: bool,
26) -> Result<(), String> {
27 let name = name.ok_or("usage: fdl init <project-name>")?;
28 validate_name(name)?;
29
30 if Path::new(name).exists() {
31 return Err(format!("'{}' already exists", name));
32 }
33
34 if docker && native {
35 return Err("--docker and --native are mutually exclusive".into());
36 }
37 let flag_driven = docker || native || with_hf;
38 let mode = if docker {
39 Mode::Docker
40 } else if native {
41 Mode::Native
42 } else {
43 pick_mode_interactively()
44 };
45 let include_hf = if flag_driven {
50 with_hf
51 } else {
52 prompt::ask_yn(
53 "Include flodl-hf (HuggingFace: BERT/RoBERTa/DistilBERT, Hub loader, tokenizer)?",
54 false,
55 )
56 };
57
58 let crate_name = name.replace('-', "_");
59 let flodl_dep = resolve_flodl_dep();
60
61 fs::create_dir_all(format!("{}/src", name))
62 .map_err(|e| format!("cannot create directory: {}", e))?;
63
64 match mode {
65 Mode::Mounted => scaffold_mounted(name, &crate_name, &flodl_dep)?,
66 Mode::Docker => scaffold_docker(name, &crate_name, &flodl_dep)?,
67 Mode::Native => scaffold_native(name, &crate_name, &flodl_dep)?,
68 }
69
70 write_file(
72 &format!("{}/src/main.rs", name),
73 &main_rs_template(),
74 )?;
75 write_file(
76 &format!("{}/.gitignore", name),
77 &gitignore_template(mode),
78 )?;
79 write_file(
80 &format!("{}/fdl.yml.example", name),
81 &fdl_yml_example_template(name, mode),
82 )?;
83 write_fdl_bootstrap(name)?;
84
85 if include_hf {
86 let project_dir = Path::new(name);
87 if let Err(e) = crate::add::add_flodl_hf_at(project_dir) {
88 eprintln!("warning: flodl-hf scaffold failed: {e}");
91 eprintln!("You can retry after `cd {}` with `fdl add flodl-hf`.", name);
92 }
93 }
94
95 print_next_steps(name, mode, include_hf);
96 crate::util::install_prompt::offer_global_install();
97 Ok(())
98}
99
100fn pick_mode_interactively() -> Mode {
104 println!();
105 if !prompt::ask_yn("Use Docker for builds?", true) {
106 return Mode::Native;
107 }
108
109 if cfg!(target_os = "macos") {
114 println!();
115 println!(" Heads up: on macOS, Docker runs Linux containers under emulation");
116 println!(" (Rosetta / QEMU). Builds and training will be substantially slower");
117 println!(" than running cargo natively on the host. Native mode keeps");
118 println!(" everything on the Mac and uses macOS libtorch directly.");
119 println!();
120 if !prompt::ask_yn("Continue with Docker?", true) {
121 return Mode::Native;
122 }
123 }
124
125 let choice = prompt::ask_choice(
127 "How should libtorch be provided to the container?",
128 &[
129 "Mount it from the host (recommended: lighter image, swap CUDA variants)",
130 "Bake it into the image at build time (zero host setup)",
131 ],
132 1,
133 );
134 match choice {
135 2 => Mode::Docker,
136 _ => Mode::Mounted,
137 }
138}
139
140fn print_next_steps(name: &str, mode: Mode, include_hf: bool) {
141 println!();
142 println!("Project '{}' created. Next steps:", name);
143 println!();
144 println!(" cd {}", name);
145 match mode {
146 Mode::Mounted => {
147 println!(" ./fdl setup # detect hardware + download libtorch");
148 println!(" ./fdl build # build the project");
149 }
150 Mode::Docker => {
151 println!(" ./fdl build # first build (downloads libtorch, ~5 min)");
152 }
153 Mode::Native => {
154 println!(" ./fdl libtorch download --cpu # or --cuda 12.8");
155 println!(" ./fdl build # cargo build on the host");
156 }
157 }
158 println!(" ./fdl test # run tests");
159 println!(" ./fdl run # train the model");
160 if mode != Mode::Native {
161 println!(" ./fdl shell # interactive shell");
162 }
163 if include_hf {
164 println!();
165 println!(" cd flodl-hf && fdl classify # try the HuggingFace playground");
166 }
167 println!();
168 println!("`./fdl --help` lists every command defined in fdl.yml.");
169 println!("Edit src/main.rs to build your model.");
170 println!();
171 println!("Guides:");
172 println!(" Tutorials: https://flodl.dev/guide/tensors");
173 println!(" Graph Tree: https://flodl.dev/guide/graph-tree");
174 println!(" PyTorch migration: https://flodl.dev/guide/migration");
175 println!(" Troubleshooting: https://flodl.dev/guide/troubleshooting");
176}
177
178fn write_fdl_bootstrap(name: &str) -> Result<(), String> {
179 let fdl_script = include_str!("../assets/fdl");
180 write_file(&format!("{}/fdl", name), fdl_script)?;
181 #[cfg(unix)]
182 {
183 use std::os::unix::fs::PermissionsExt;
184 let _ = fs::set_permissions(
185 format!("{}/fdl", name),
186 fs::Permissions::from_mode(0o755),
187 );
188 }
189 Ok(())
190}
191
192fn validate_name(name: &str) -> Result<(), String> {
193 if name.is_empty() {
194 return Err("project name cannot be empty".into());
195 }
196 if !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') {
197 return Err("project name must contain only letters, digits, hyphens, underscores".into());
198 }
199 Ok(())
200}
201
202fn resolve_flodl_dep() -> String {
203 if let Some(version) = crates_io_version() {
205 format!("flodl = \"{}\"", version)
206 } else {
207 "flodl = { git = \"https://github.com/flodl-labs/flodl.git\" }".into()
208 }
209}
210
211fn crates_io_version() -> Option<String> {
212 let output = Command::new("curl")
213 .args(["-sL", "https://crates.io/api/v1/crates/flodl"])
214 .output()
215 .ok()?;
216 let body = String::from_utf8_lossy(&output.stdout);
217 let marker = "\"max_stable_version\":\"";
219 let start = body.find(marker)? + marker.len();
220 let end = start + body[start..].find('"')?;
221 let version = &body[start..end];
222 if version.is_empty() { None } else { Some(version.to_string()) }
223}
224
225fn scaffold_docker(name: &str, crate_name: &str, flodl_dep: &str) -> Result<(), String> {
230 write_file(
231 &format!("{}/Cargo.toml", name),
232 &cargo_toml_template(crate_name, flodl_dep),
233 )?;
234 write_file(
235 &format!("{}/Dockerfile.cpu", name),
236 DOCKERFILE_CPU,
237 )?;
238 write_file(
239 &format!("{}/Dockerfile.cuda", name),
240 DOCKERFILE_CUDA,
241 )?;
242 write_file(
243 &format!("{}/docker-compose.yml", name),
244 &docker_compose_template(crate_name, true),
245 )?;
246 Ok(())
247}
248
249fn scaffold_mounted(name: &str, crate_name: &str, flodl_dep: &str) -> Result<(), String> {
254 write_file(
255 &format!("{}/Cargo.toml", name),
256 &cargo_toml_template(crate_name, flodl_dep),
257 )?;
258 write_file(
259 &format!("{}/Dockerfile", name),
260 DOCKERFILE_MOUNTED,
261 )?;
262 write_file(
263 &format!("{}/Dockerfile.cuda", name),
264 DOCKERFILE_CUDA_MOUNTED,
265 )?;
266 write_file(
267 &format!("{}/docker-compose.yml", name),
268 &docker_compose_template(crate_name, false),
269 )?;
270 Ok(())
271}
272
273fn scaffold_native(name: &str, crate_name: &str, flodl_dep: &str) -> Result<(), String> {
278 write_file(
279 &format!("{}/Cargo.toml", name),
280 &cargo_toml_template(crate_name, flodl_dep),
281 )?;
282 Ok(())
285}
286
287fn cargo_toml_template(crate_name: &str, flodl_dep: &str) -> String {
292 format!(
293 r#"[package]
294name = "{crate_name}"
295version = "0.1.0"
296edition = "2024"
297
298[dependencies]
299{flodl_dep}
300
301# Optimize floDl in dev builds -- your code stays fast to compile.
302# After the first build, only your graph code recompiles (~2s).
303[profile.dev.package.flodl]
304opt-level = 3
305
306[profile.dev.package.flodl-sys]
307opt-level = 3
308
309# Release: cross-crate optimization for maximum throughput.
310[profile.release]
311lto = "thin"
312codegen-units = 1
313"#
314 )
315}
316
317fn main_rs_template() -> String {
318 r#"//! floDl training template.
319//!
320//! This is a starting point for your model. Edit the architecture,
321//! data loading, and training loop to fit your task.
322//!
323//! New to Rust? Read: https://flodl.dev/guide/rust-primer
324//! Stuck? Read: https://flodl.dev/guide/troubleshooting
325
326use flodl::*;
327use flodl::monitor::Monitor;
328
329fn main() -> Result<()> {
330 // --- Model ---
331 let model = FlowBuilder::from(Linear::new(4, 32)?)
332 .through(GELU)
333 .through(LayerNorm::new(32)?)
334 .also(Linear::new(32, 32)?) // residual connection
335 .through(Linear::new(32, 1)?)
336 .build()?;
337
338 // --- Optimizer ---
339 let params = model.parameters();
340 let mut optimizer = Adam::new(¶ms, 0.001);
341 let scheduler = CosineScheduler::new(0.001, 1e-6, 100);
342 model.train();
343
344 // --- Data ---
345 // Replace this with your data loading.
346 let opts = TensorOptions::default();
347 let batches: Vec<(Tensor, Tensor)> = (0..32)
348 .map(|_| {
349 let x = Tensor::randn(&[16, 4], opts).unwrap();
350 let y = Tensor::randn(&[16, 1], opts).unwrap();
351 (x, y)
352 })
353 .collect();
354
355 // --- Training loop ---
356 let num_epochs = 100usize;
357 let mut monitor = Monitor::new(num_epochs);
358 // monitor.serve(3000)?; // uncomment for live dashboard
359 // monitor.watch(&model); // uncomment to show graph SVG
360 // monitor.save_html("report.html"); // uncomment to save HTML report
361
362 for epoch in 0..num_epochs {
363 let t = std::time::Instant::now();
364 let mut epoch_loss = 0.0;
365
366 for (input_t, target_t) in &batches {
367 let input = Variable::new(input_t.clone(), true);
368 let target = Variable::new(target_t.clone(), false);
369
370 optimizer.zero_grad();
371 let pred = model.forward(&input)?;
372 let loss = mse_loss(&pred, &target)?;
373 loss.backward()?;
374 clip_grad_norm(¶ms, 1.0)?;
375 optimizer.step()?;
376
377 epoch_loss += loss.item()?;
378 }
379
380 let avg_loss = epoch_loss / batches.len() as f64;
381 let lr = scheduler.lr(epoch);
382 optimizer.set_lr(lr);
383 monitor.log(epoch, t.elapsed(), &[("loss", avg_loss), ("lr", lr)]);
384 }
385
386 monitor.finish();
387 Ok(())
388}
389"#
390 .into()
391}
392
393fn gitignore_template(mode: Mode) -> String {
394 let mut s = String::from(
395 "/target
396*.fdl
397*.log
398*.csv
399*.html
400
401# Local fdl config (fdl.yml.example is committed; fdl copies it on first run)
402fdl.yml
403fdl.yaml
404",
405 );
406 match mode {
407 Mode::Docker => {
408 s.push_str(
410 ".cargo-cache/
411.cargo-git/
412.cargo-cache-cuda/
413.cargo-git-cuda/
414",
415 );
416 }
417 Mode::Mounted => {
418 s.push_str(
420 ".cargo-cache/
421.cargo-git/
422.cargo-cache-cuda/
423.cargo-git-cuda/
424libtorch/
425",
426 );
427 }
428 Mode::Native => {
429 s.push_str("libtorch/\n");
432 }
433 }
434 s
435}
436
437fn docker_compose_template(crate_name: &str, baked: bool) -> String {
438 if baked {
439 format!(
440 r#"services:
441 dev:
442 build:
443 context: .
444 dockerfile: Dockerfile.cpu
445 image: {crate_name}-dev
446 user: "${{UID:-1000}}:${{GID:-1000}}"
447 volumes:
448 - .:/workspace
449 - ./.cargo-cache:/usr/local/cargo/registry
450 - ./.cargo-git:/usr/local/cargo/git
451 working_dir: /workspace
452 stdin_open: true
453 tty: true
454
455 cuda:
456 build:
457 context: .
458 dockerfile: Dockerfile.cuda
459 image: {crate_name}-cuda
460 user: "${{UID:-1000}}:${{GID:-1000}}"
461 volumes:
462 - .:/workspace
463 - ./.cargo-cache-cuda:/usr/local/cargo/registry
464 - ./.cargo-git-cuda:/usr/local/cargo/git
465 working_dir: /workspace
466 stdin_open: true
467 tty: true
468 deploy:
469 resources:
470 reservations:
471 devices:
472 - driver: nvidia
473 count: all
474 capabilities: [gpu]
475"#
476 )
477 } else {
478 format!(
479 r#"services:
480 dev:
481 build:
482 context: .
483 dockerfile: Dockerfile
484 image: {crate_name}-dev
485 user: "${{UID:-1000}}:${{GID:-1000}}"
486 volumes:
487 - .:/workspace
488 - ./.cargo-cache:/usr/local/cargo/registry
489 - ./.cargo-git:/usr/local/cargo/git
490 - ${{LIBTORCH_CPU_PATH:-./libtorch/precompiled/cpu}}:/usr/local/libtorch:ro
491 working_dir: /workspace
492 stdin_open: true
493 tty: true
494
495 cuda:
496 build:
497 context: .
498 dockerfile: Dockerfile.cuda
499 args:
500 CUDA_VERSION: ${{CUDA_VERSION:-12.8.0}}
501 image: {crate_name}-cuda:${{CUDA_TAG:-12.8}}
502 user: "${{UID:-1000}}:${{GID:-1000}}"
503 volumes:
504 - .:/workspace
505 - ./.cargo-cache-cuda:/usr/local/cargo/registry
506 - ./.cargo-git-cuda:/usr/local/cargo/git
507 - ${{LIBTORCH_HOST_PATH:-./libtorch/precompiled/cu128}}:/usr/local/libtorch:ro
508 working_dir: /workspace
509 stdin_open: true
510 tty: true
511 deploy:
512 resources:
513 reservations:
514 devices:
515 - driver: nvidia
516 count: all
517 capabilities: [gpu]
518"#
519 )
520 }
521}
522
523const DOCKERFILE_CPU: &str = r#"# CPU-only dev image for floDl projects.
529FROM ubuntu:24.04
530
531ENV DEBIAN_FRONTEND=noninteractive
532
533RUN apt-get update && apt-get install -y --no-install-recommends \
534 wget curl unzip ca-certificates git gcc g++ pkg-config graphviz \
535 && rm -rf /var/lib/apt/lists/*
536
537# Rust
538ENV CARGO_HOME="/usr/local/cargo"
539ENV RUSTUP_HOME="/usr/local/rustup"
540RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable \
541 && chmod -R a+rwx "$CARGO_HOME" "$RUSTUP_HOME"
542ENV PATH="${CARGO_HOME}/bin:${PATH}"
543
544# libtorch (CPU-only, ~200MB)
545ARG LIBTORCH_VERSION=2.10.0
546RUN wget -q https://download.pytorch.org/libtorch/cpu/libtorch-shared-with-deps-${LIBTORCH_VERSION}%2Bcpu.zip \
547 && unzip -q libtorch-shared-with-deps-${LIBTORCH_VERSION}+cpu.zip -d /usr/local \
548 && rm libtorch-shared-with-deps-${LIBTORCH_VERSION}+cpu.zip
549
550ENV LIBTORCH_PATH="/usr/local/libtorch"
551ENV LD_LIBRARY_PATH="${LIBTORCH_PATH}/lib"
552ENV LIBRARY_PATH="${LIBTORCH_PATH}/lib"
553
554WORKDIR /workspace
555"#;
556
557const DOCKERFILE_CUDA: &str = r#"# CUDA dev image for floDl projects.
558# Requires: docker run --gpus all ...
559FROM nvidia/cuda:12.8.0-devel-ubuntu24.04
560
561ENV DEBIAN_FRONTEND=noninteractive
562
563RUN apt-get update && apt-get install -y --no-install-recommends \
564 wget curl unzip ca-certificates git gcc g++ pkg-config graphviz \
565 && rm -rf /var/lib/apt/lists/*
566
567# Rust
568ENV CARGO_HOME="/usr/local/cargo"
569ENV RUSTUP_HOME="/usr/local/rustup"
570RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable \
571 && chmod -R a+rwx "$CARGO_HOME" "$RUSTUP_HOME"
572ENV PATH="${CARGO_HOME}/bin:${PATH}"
573
574# libtorch (CUDA 12.8)
575ARG LIBTORCH_VERSION=2.10.0
576RUN wget -q "https://download.pytorch.org/libtorch/cu128/libtorch-shared-with-deps-${LIBTORCH_VERSION}%2Bcu128.zip" \
577 && unzip -q "libtorch-shared-with-deps-${LIBTORCH_VERSION}+cu128.zip" -d /usr/local \
578 && rm "libtorch-shared-with-deps-${LIBTORCH_VERSION}+cu128.zip"
579
580ENV LIBTORCH_PATH="/usr/local/libtorch"
581ENV LD_LIBRARY_PATH="${LIBTORCH_PATH}/lib:/usr/local/cuda/lib64"
582ENV LIBRARY_PATH="${LIBTORCH_PATH}/lib:/usr/local/cuda/lib64"
583ENV CUDA_HOME="/usr/local/cuda"
584
585WORKDIR /workspace
586"#;
587
588const DOCKERFILE_MOUNTED: &str = r#"# CPU dev image for floDl projects (libtorch mounted at runtime).
590FROM ubuntu:24.04
591
592ENV DEBIAN_FRONTEND=noninteractive
593
594RUN apt-get update && apt-get install -y --no-install-recommends \
595 wget curl unzip ca-certificates git gcc g++ pkg-config graphviz \
596 && rm -rf /var/lib/apt/lists/*
597
598# Rust
599ENV CARGO_HOME="/usr/local/cargo"
600ENV RUSTUP_HOME="/usr/local/rustup"
601RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable \
602 && chmod -R a+rwx "$CARGO_HOME" "$RUSTUP_HOME"
603ENV PATH="${CARGO_HOME}/bin:${PATH}"
604
605ENV LIBTORCH_PATH="/usr/local/libtorch"
606ENV LD_LIBRARY_PATH="${LIBTORCH_PATH}/lib"
607ENV LIBRARY_PATH="${LIBTORCH_PATH}/lib"
608
609WORKDIR /workspace
610"#;
611
612const DOCKERFILE_CUDA_MOUNTED: &str = r#"# CUDA dev image for floDl projects (libtorch mounted at runtime).
613# Requires: docker run --gpus all ...
614ARG CUDA_VERSION=12.8.0
615FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu24.04
616
617ENV DEBIAN_FRONTEND=noninteractive
618
619RUN apt-get update && apt-get install -y --no-install-recommends \
620 wget curl unzip ca-certificates git gcc g++ pkg-config graphviz \
621 && rm -rf /var/lib/apt/lists/*
622
623# Rust
624ENV CARGO_HOME="/usr/local/cargo"
625ENV RUSTUP_HOME="/usr/local/rustup"
626RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable \
627 && chmod -R a+rwx "$CARGO_HOME" "$RUSTUP_HOME"
628ENV PATH="${CARGO_HOME}/bin:${PATH}"
629
630ENV LIBTORCH_PATH="/usr/local/libtorch"
631ENV LD_LIBRARY_PATH="${LIBTORCH_PATH}/lib:/usr/local/cuda/lib64"
632ENV LIBRARY_PATH="${LIBTORCH_PATH}/lib:/usr/local/cuda/lib64"
633ENV CUDA_HOME="/usr/local/cuda"
634
635WORKDIR /workspace
636"#;
637
638fn fdl_yml_example_template(project_name: &str, mode: Mode) -> String {
650 let use_docker = matches!(mode, Mode::Mounted | Mode::Docker);
651 let (cpu_svc, cuda_svc) = if use_docker {
652 ("\n docker: dev", "\n docker: cuda")
653 } else {
654 ("", "")
655 };
656 let cuda_note = if use_docker {
657 "(requires NVIDIA Container Toolkit)"
658 } else {
659 "(requires a matching CUDA toolkit on the host)"
660 };
661 let preamble = if use_docker {
662 "# Run any of these with `./fdl <cmd>` (or `fdl <cmd>` once installed\n\
663 # globally via `./fdl install`). Libtorch env vars are derived from\n\
664 # `libtorch/.active` automatically; missing libtorch surfaces as a\n\
665 # clean linker error, with `./fdl setup` one call away."
666 } else {
667 "# Native mode: commands run on the host. Make sure libtorch is\n\
668 # installed (`./fdl libtorch download --cpu` or `--cuda 12.8`)\n\
669 # and that `$LIBTORCH` / `$LD_LIBRARY_PATH` are exported so\n\
670 # cargo can link. `./fdl libtorch info` prints the commands you\n\
671 # need after a download."
672 };
673
674 let shell_block = if use_docker {
675 format!(
676 r#" shell:
677 description: Interactive shell (CPU container)
678 run: bash{cpu_svc}
679
680"#
681 )
682 } else {
683 String::new()
685 };
686
687 let cuda_shell_block = if use_docker {
688 format!(
689 r#" cuda-shell:
690 description: Interactive shell (CUDA container)
691 run: bash{cuda_svc}
692"#
693 )
694 } else {
695 String::new()
696 };
697
698 format!(
699 r#"description: {project_name}
700
701{preamble}
702
703commands:
704 # --- CPU ---
705 build:
706 description: Build (debug)
707 run: cargo build{cpu_svc}
708 test:
709 description: Run CPU tests
710 run: cargo test -- --nocapture{cpu_svc}
711 run:
712 description: cargo run
713 run: cargo run{cpu_svc}
714 check:
715 description: Type-check without building
716 run: cargo check{cpu_svc}
717 clippy:
718 description: Lint
719 run: cargo clippy -- -W clippy::all{cpu_svc}
720{shell_block} # --- CUDA {cuda_note} ---
721 cuda-build:
722 description: Build with CUDA feature
723 run: cargo build --features cuda{cuda_svc}
724 cuda-test:
725 description: Run CUDA tests
726 run: cargo test --features cuda -- --nocapture{cuda_svc}
727 cuda-run:
728 description: cargo run --features cuda
729 run: cargo run --features cuda{cuda_svc}
730{cuda_shell_block}"#
731 )
732}
733
734fn write_file(path: &str, content: &str) -> Result<(), String> {
739 fs::write(path, content).map_err(|e| format!("cannot write {}: {}", path, e))
740}
741
742#[cfg(test)]
743mod tests {
744 use super::validate_name;
745
746 #[test]
747 fn validate_name_accepts_alnum_hyphen_underscore() {
748 for ok in ["my_project", "my-project", "Proj123", "a", "x_1-2"] {
749 assert!(validate_name(ok).is_ok(), "{ok:?} should be valid");
750 }
751 }
752
753 #[test]
754 fn validate_name_rejects_empty() {
755 let err = validate_name("").unwrap_err();
756 assert!(err.contains("empty"), "unexpected: {err}");
757 }
758
759 #[test]
760 fn validate_name_rejects_disallowed_chars() {
761 for bad in ["my project", "my.project", "a/b", "../evil", "name!"] {
764 let err = validate_name(bad).unwrap_err();
765 assert!(err.contains("only letters"), "{bad:?} -> unexpected: {err}");
766 }
767 }
768}