ffai_core/
lib.rs

1//! # ffai-core
2//!
3//! The spine of `FFai`: shared media/AI types, one trait per task, and the
4//! engine registry that makes implementations interchangeable.
5//!
6//! The design borrows ffmpeg's load-bearing idea: `AVCodec` is a registry of
7//! interchangeable implementations selected by name (`-c:v libx264`). `FFai`'s
8//! equivalent: [`engine::AsrEngine`], [`engine::TtsEngine`],
9//! [`engine::OcrEngine`], and [`engine::VlmEngine`] are traits with many
10//! competing engines behind them, selected with `--engine <name>`.
11//!
12//! Candle is the tensor spine — re-exported here as [`candle`] so every engine
13//! crate shares one `Tensor`/`Device` and buffers flow between models without
14//! copies.
15
16pub mod engine;
17pub mod error;
18pub mod registry;
19pub mod types;
20
21/// The shared tensor framework (Hugging Face candle).
22pub use candle_core as candle;
23
24pub use error::{Error, Result};
25
26/// Pick the best available compute device.
27///
28/// CPU always works; CUDA/Metal are behind the crate features of the same
29/// name and fall back to CPU when unavailable.
30#[must_use]
31pub const fn best_device() -> candle::Device {
32    #[cfg(feature = "cuda")]
33    if let Ok(dev) = candle::Device::new_cuda(0) {
34        return dev;
35    }
36    #[cfg(feature = "metal")]
37    if let Ok(dev) = candle::Device::new_metal(0) {
38        return dev;
39    }
40    candle::Device::Cpu
41}
42
43/// Release memory the allocator is holding after a large one-off load.
44///
45/// Loading model weights allocates far more than the model keeps: dtype
46/// conversions, quantization scratch, and the safetensors mapping all churn
47/// through the heap and are then freed. Freed is not returned — the allocator
48/// keeps the pages, and they stay counted against the process for the rest of
49/// its life.
50///
51/// Measured on Whisper tiny.en: the process holds **345 MiB** after a run, but
52/// trimming and then repeating the SAME work re-settles at **102 MiB**. So
53/// ~240 MiB of what looked like footprint was never needed by the work — and a
54/// reference implementation that manages its own arena does not carry it,
55/// which is most of why we measured 2.2x its resident memory while needing
56/// half of what it does.
57///
58/// This is a hint, not a free: pages the process still needs fault straight
59/// back in on next use. Call it ONCE, after a known-large load, never in a hot
60/// path — trimming what you are about to touch again just buys page faults.
61///
62/// No-op where the platform offers no equivalent.
63//
64// NOT const, and clippy::nursery is wrong here in a platform-specific way: on
65// non-Windows every branch below is cfg'd out, the body is empty, and
66// `missing_const_for_fn` fires. On Windows the same function calls a Win32
67// entry point and cannot be const. Making it const would therefore compile on
68// Linux and break on Windows - so the lint is allowed rather than obeyed.
69// Caught only when CI first ran on Linux; it never fired on the author's box.
70#[allow(clippy::missing_const_for_fn)]
71pub fn release_load_arena() {
72    #[cfg(windows)]
73    {
74        unsafe extern "system" {
75            fn SetProcessWorkingSetSizeEx(
76                process: *mut core::ffi::c_void,
77                min: usize,
78                max: usize,
79                flags: u32,
80            ) -> i32;
81            fn GetCurrentProcess() -> *mut core::ffi::c_void;
82        }
83        // SAFETY: pseudo-handle needing no close; (SIZE_T)-1 for both bounds is
84        // the documented request-a-trim form rather than a quota.
85        unsafe {
86            SetProcessWorkingSetSizeEx(GetCurrentProcess(), usize::MAX, usize::MAX, 0);
87        }
88    }
89}
ffai_core/lib.rs

ffai_core/
lib.rs
