ffai_core/
lib.rs

1//! # ffai-core
2//!
3//! The spine of `FFai`: shared media/AI types, one trait per task, and the
4//! engine registry that makes implementations interchangeable.
5//!
6//! The design borrows ffmpeg's load-bearing idea: `AVCodec` is a registry of
7//! interchangeable implementations selected by name (`-c:v libx264`). `FFai`'s
8//! equivalent: [`engine::AsrEngine`], [`engine::TtsEngine`],
9//! [`engine::OcrEngine`], and [`engine::VlmEngine`] are traits with many
10//! competing engines behind them, selected with `--engine <name>`.
11//!
12//! Candle is the tensor spine — re-exported here as [`candle`] so every engine
13//! crate shares one `Tensor`/`Device` and buffers flow between models without
14//! copies.
15
16pub mod cost;
17pub mod engine;
18pub mod error;
19pub mod fastmath;
20pub mod fastops;
21pub mod registry;
22pub mod threads;
23pub mod types;
24
25/// The shared tensor framework (Hugging Face candle).
26pub use candle_core as candle;
27
28pub use error::{Error, Result};
29
30/// Pick the best available compute device.
31///
32/// CPU always works; CUDA/Metal are behind the crate features of the same
33/// name and fall back to CPU when unavailable.
34#[must_use]
35pub const fn best_device() -> candle::Device {
36    #[cfg(feature = "cuda")]
37    if let Ok(dev) = candle::Device::new_cuda(0) {
38        return dev;
39    }
40    #[cfg(feature = "metal")]
41    if let Ok(dev) = candle::Device::new_metal(0) {
42        return dev;
43    }
44    candle::Device::Cpu
45}
46
47/// Release memory the allocator is holding after a large one-off load.
48///
49/// Loading model weights allocates far more than the model keeps: dtype
50/// conversions, quantization scratch, and the safetensors mapping all churn
51/// through the heap and are then freed. Freed is not returned — the allocator
52/// keeps the pages, and they stay counted against the process for the rest of
53/// its life.
54///
55/// Measured on Whisper tiny.en: the process holds **345 MiB** after a run, but
56/// trimming and then repeating the SAME work re-settles at **102 MiB**. So
57/// ~240 MiB of what looked like footprint was never needed by the work — and a
58/// reference implementation that manages its own arena does not carry it,
59/// which is most of why we measured 2.2x its resident memory while needing
60/// half of what it does.
61///
62/// This is a hint, not a free: pages the process still needs fault straight
63/// back in on next use. Call it ONCE, after a known-large load, never in a hot
64/// path — trimming what you are about to touch again just buys page faults.
65///
66/// No-op where the platform offers no equivalent.
67//
68// NOT const, and clippy::nursery is wrong here in a platform-specific way: on
69// non-Windows every branch below is cfg'd out, the body is empty, and
70// `missing_const_for_fn` fires. On Windows the same function calls a Win32
71// entry point and cannot be const. Making it const would therefore compile on
72// Linux and break on Windows - so the lint is allowed rather than obeyed.
73// Caught only when CI first ran on Linux; it never fired on the author's box.
74#[allow(clippy::missing_const_for_fn)]
75#[allow(unsafe_code)]
76pub fn release_load_arena() {
77    #[cfg(windows)]
78    {
79        unsafe extern "system" {
80            fn SetProcessWorkingSetSizeEx(
81                process: *mut core::ffi::c_void,
82                min: usize,
83                max: usize,
84                flags: u32,
85            ) -> i32;
86            fn GetCurrentProcess() -> *mut core::ffi::c_void;
87        }
88        // SAFETY: pseudo-handle needing no close; (SIZE_T)-1 for both bounds is
89        // the documented request-a-trim form rather than a quota.
90        unsafe {
91            SetProcessWorkingSetSizeEx(GetCurrentProcess(), usize::MAX, usize::MAX, 0);
92        }
93    }
94}
ffai_core/lib.rs

ffai_core/
lib.rs
