ffai_core/
lib.rs

1//! # ffai-core
2//!
3//! The spine of FFai: shared media/AI types, one trait per task, and the
4//! engine registry that makes implementations interchangeable.
5//!
6//! The design borrows ffmpeg's load-bearing idea: `AVCodec` is a registry of
7//! interchangeable implementations selected by name (`-c:v libx264`). FFai's
8//! equivalent: [`engine::AsrEngine`], [`engine::TtsEngine`],
9//! [`engine::OcrEngine`], and [`engine::VlmEngine`] are traits with many
10//! competing engines behind them, selected with `--engine <name>`.
11//!
12//! Candle is the tensor spine — re-exported here as [`candle`] so every engine
13//! crate shares one `Tensor`/`Device` and buffers flow between models without
14//! copies.
15
16pub mod engine;
17pub mod error;
18pub mod registry;
19pub mod types;
20
21/// The shared tensor framework (Hugging Face candle).
22pub use candle_core as candle;
23
24pub use error::{Error, Result};
25
26/// Pick the best available compute device.
27///
28/// CPU always works; CUDA/Metal are behind the crate features of the same
29/// name and fall back to CPU when unavailable.
30pub fn best_device() -> candle::Device {
31    #[cfg(feature = "cuda")]
32    if let Ok(dev) = candle::Device::new_cuda(0) {
33        return dev;
34    }
35    #[cfg(feature = "metal")]
36    if let Ok(dev) = candle::Device::new_metal(0) {
37        return dev;
38    }
39    candle::Device::Cpu
40}
41
42/// Release memory the allocator is holding after a large one-off load.
43///
44/// Loading model weights allocates far more than the model keeps: dtype
45/// conversions, quantization scratch, and the safetensors mapping all churn
46/// through the heap and are then freed. Freed is not returned — the allocator
47/// keeps the pages, and they stay counted against the process for the rest of
48/// its life.
49///
50/// Measured on Whisper tiny.en: the process holds **345 MiB** after a run, but
51/// trimming and then repeating the SAME work re-settles at **102 MiB**. So
52/// ~240 MiB of what looked like footprint was never needed by the work — and a
53/// reference implementation that manages its own arena does not carry it,
54/// which is most of why we measured 2.2x its resident memory while needing
55/// half of what it does.
56///
57/// This is a hint, not a free: pages the process still needs fault straight
58/// back in on next use. Call it ONCE, after a known-large load, never in a hot
59/// path — trimming what you are about to touch again just buys page faults.
60///
61/// No-op where the platform offers no equivalent.
62pub fn release_load_arena() {
63    #[cfg(windows)]
64    {
65        unsafe extern "system" {
66            fn SetProcessWorkingSetSizeEx(
67                process: *mut core::ffi::c_void,
68                min: usize,
69                max: usize,
70                flags: u32,
71            ) -> i32;
72            fn GetCurrentProcess() -> *mut core::ffi::c_void;
73        }
74        // SAFETY: pseudo-handle needing no close; (SIZE_T)-1 for both bounds is
75        // the documented request-a-trim form rather than a quota.
76        unsafe {
77            SetProcessWorkingSetSizeEx(GetCurrentProcess(), usize::MAX, usize::MAX, 0);
78        }
79    }
80}
ffai_core/lib.rs

ffai_core/
lib.rs
