ffai_core/
lib.rs

1//! # ffai-core
2//!
3//! The spine of `FFai`: shared media/AI types, one trait per task, and the
4//! engine registry that makes implementations interchangeable.
5//!
6//! The design borrows ffmpeg's load-bearing idea: `AVCodec` is a registry of
7//! interchangeable implementations selected by name (`-c:v libx264`). `FFai`'s
8//! equivalent: [`engine::AsrEngine`], [`engine::TtsEngine`],
9//! [`engine::OcrEngine`], and [`engine::VlmEngine`] are traits with many
10//! competing engines behind them, selected with `--engine <name>`.
11//!
12//! Candle is the tensor spine — re-exported here as [`candle`] so every engine
13//! crate shares one `Tensor`/`Device` and buffers flow between models without
14//! copies.
15
16pub mod cost;
17pub mod engine;
18pub mod error;
19pub mod fastmath;
20pub mod fastops;
21pub mod registry;
22pub mod types;
23
24/// The shared tensor framework (Hugging Face candle).
25pub use candle_core as candle;
26
27pub use error::{Error, Result};
28
29/// Pick the best available compute device.
30///
31/// CPU always works; CUDA/Metal are behind the crate features of the same
32/// name and fall back to CPU when unavailable.
33#[must_use]
34pub const fn best_device() -> candle::Device {
35    #[cfg(feature = "cuda")]
36    if let Ok(dev) = candle::Device::new_cuda(0) {
37        return dev;
38    }
39    #[cfg(feature = "metal")]
40    if let Ok(dev) = candle::Device::new_metal(0) {
41        return dev;
42    }
43    candle::Device::Cpu
44}
45
46/// Release memory the allocator is holding after a large one-off load.
47///
48/// Loading model weights allocates far more than the model keeps: dtype
49/// conversions, quantization scratch, and the safetensors mapping all churn
50/// through the heap and are then freed. Freed is not returned — the allocator
51/// keeps the pages, and they stay counted against the process for the rest of
52/// its life.
53///
54/// Measured on Whisper tiny.en: the process holds **345 MiB** after a run, but
55/// trimming and then repeating the SAME work re-settles at **102 MiB**. So
56/// ~240 MiB of what looked like footprint was never needed by the work — and a
57/// reference implementation that manages its own arena does not carry it,
58/// which is most of why we measured 2.2x its resident memory while needing
59/// half of what it does.
60///
61/// This is a hint, not a free: pages the process still needs fault straight
62/// back in on next use. Call it ONCE, after a known-large load, never in a hot
63/// path — trimming what you are about to touch again just buys page faults.
64///
65/// No-op where the platform offers no equivalent.
66//
67// NOT const, and clippy::nursery is wrong here in a platform-specific way: on
68// non-Windows every branch below is cfg'd out, the body is empty, and
69// `missing_const_for_fn` fires. On Windows the same function calls a Win32
70// entry point and cannot be const. Making it const would therefore compile on
71// Linux and break on Windows - so the lint is allowed rather than obeyed.
72// Caught only when CI first ran on Linux; it never fired on the author's box.
73#[allow(clippy::missing_const_for_fn)]
74pub fn release_load_arena() {
75    #[cfg(windows)]
76    {
77        unsafe extern "system" {
78            fn SetProcessWorkingSetSizeEx(
79                process: *mut core::ffi::c_void,
80                min: usize,
81                max: usize,
82                flags: u32,
83            ) -> i32;
84            fn GetCurrentProcess() -> *mut core::ffi::c_void;
85        }
86        // SAFETY: pseudo-handle needing no close; (SIZE_T)-1 for both bounds is
87        // the documented request-a-trim form rather than a quota.
88        unsafe {
89            SetProcessWorkingSetSizeEx(GetCurrentProcess(), usize::MAX, usize::MAX, 0);
90        }
91    }
92}
ffai_core/lib.rs

ffai_core/
lib.rs
