pub struct HookManager { /* private fields */ }Expand description
Hook manager for coordinating debugging hooks
Implementations§
Source§impl HookManager
impl HookManager
Sourcepub fn register_hook(&mut self, config: HookConfig) -> Result<Uuid>
pub fn register_hook(&mut self, config: HookConfig) -> Result<Uuid>
Register a new hook
Sourcepub fn register_callback(&mut self, name: String, callback: HookCallback)
pub fn register_callback(&mut self, name: String, callback: HookCallback)
Register a custom callback
Sourcepub fn remove_hook(&mut self, hook_id: Uuid) -> Option<HookConfig>
pub fn remove_hook(&mut self, hook_id: Uuid) -> Option<HookConfig>
Remove a hook
Sourcepub fn set_hook_enabled(&mut self, hook_id: Uuid, enabled: bool) -> Result<()>
pub fn set_hook_enabled(&mut self, hook_id: Uuid, enabled: bool) -> Result<()>
Enable/disable a specific hook
Sourcepub fn set_enabled(&mut self, enabled: bool)
pub fn set_enabled(&mut self, enabled: bool)
Enable/disable all hooks
Sourcepub fn set_loss(&mut self, loss: f64)
pub fn set_loss(&mut self, loss: f64)
Report the current training loss for HookCondition::LossThreshold
evaluation. Call this once per step before Self::execute_hooks;
without it, LossThreshold conditions never fire (see
Self::evaluate_condition).
Sourcepub fn set_gradient_norm(&mut self, grad_norm: f64)
pub fn set_gradient_norm(&mut self, grad_norm: f64)
Report the current gradient norm for
HookCondition::GradientNormThreshold evaluation. See
Self::set_loss.
Sourcepub fn set_memory_mb(&mut self, memory_mb: f64)
pub fn set_memory_mb(&mut self, memory_mb: f64)
Report current memory usage (in MB) for
HookCondition::MemoryThreshold evaluation. See Self::set_loss.
Sourcepub fn tensor_inspector(&self) -> &TensorInspector
pub fn tensor_inspector(&self) -> &TensorInspector
Real statistics recorded by HookAction::InspectTensor and
HookAction::TrackGradients hooks executed so far – shapes,
mean/std/min/max, NaN/Inf counts, per-tensor alerts, etc, computed
from the actual tensor data each hook received.
Sourcepub fn tensor_inspector_mut(&mut self) -> &mut TensorInspector
pub fn tensor_inspector_mut(&mut self) -> &mut TensorInspector
Mutable access to the wired TensorInspector, e.g. to call
TensorInspector::clear between epochs.
Sourcepub fn activation_visualizer(&self) -> &ActivationVisualizer
pub fn activation_visualizer(&self) -> &ActivationVisualizer
Real per-layer activation statistics recorded by
HookAction::RecordActivations hooks executed so far.
Sourcepub fn activation_visualizer_mut(&mut self) -> &mut ActivationVisualizer
pub fn activation_visualizer_mut(&mut self) -> &mut ActivationVisualizer
Mutable access to the wired ActivationVisualizer.
Sourcepub fn pause_flag(&self) -> Arc<AtomicBool> ⓘ
pub fn pause_flag(&self) -> Arc<AtomicBool> ⓘ
Returns a clone of the shared pause flag that
HookAction::PauseTraining sets.
§Contract
The hook system runs inside Self::execute_hooks, called by
whatever training loop is driving it – it has no independent
thread of control and therefore cannot itself halt that loop.
What HookAction::PauseTraining truthfully can do, and does,
is set this flag to true (see Self::execute_action). For
“pause on hook” behaviour, the training loop must cooperate:
- Once, after constructing the
HookManager, clone this flag out withmanager.pause_flag()and keep theArcalongside the loop state. - On each step (or at another convenient point), check
flag.load(Ordering::SeqCst)– equivalentlySelf::is_pausedon the manager, if the loop still has access to it – and iftrue, actually stop advancing (block for operator input, yield to a debug console, etc). - Call
Self::resume_training(orflag.store(false, ...)directly) once ready to continue.
A training loop that never polls this flag is simply not pausable
by hooks; HookAction::PauseTraining does not claim otherwise –
it only guarantees the flag itself is set truthfully when it fires.
Sourcepub fn is_paused(&self) -> bool
pub fn is_paused(&self) -> bool
true if a HookAction::PauseTraining hook has fired and
nothing has called Self::resume_training since. See
Self::pause_flag for the full contract.
Sourcepub fn resume_training(&self)
pub fn resume_training(&self)
Clears the pause flag set by HookAction::PauseTraining. See
Self::pause_flag for the full contract.
Sourcepub fn execute_hooks<T>(
&mut self,
layer_name: &str,
tensor_data: &[T],
tensor_shape: &[usize],
is_forward: bool,
metadata: Option<HashMap<String, String>>,
) -> Vec<(Uuid, HookResult)>
pub fn execute_hooks<T>( &mut self, layer_name: &str, tensor_data: &[T], tensor_shape: &[usize], is_forward: bool, metadata: Option<HashMap<String, String>>, ) -> Vec<(Uuid, HookResult)>
Execute hooks for a tensor operation
Sourcepub fn get_hook(&self, hook_id: Uuid) -> Option<&HookConfig>
pub fn get_hook(&self, hook_id: Uuid) -> Option<&HookConfig>
Get hook configuration
Sourcepub fn get_all_hooks(&self) -> Vec<&HookConfig>
pub fn get_all_hooks(&self) -> Vec<&HookConfig>
Get all hooks
Sourcepub fn get_hook_stats(&self, hook_id: Uuid) -> Option<&HookStats>
pub fn get_hook_stats(&self, hook_id: Uuid) -> Option<&HookStats>
Get hook statistics
Sourcepub fn get_all_stats(&self) -> Vec<&HookStats>
pub fn get_all_stats(&self) -> Vec<&HookStats>
Get all hook statistics
Sourcepub fn clear_hooks(&mut self)
pub fn clear_hooks(&mut self)
Clear all hooks
Sourcepub fn create_tensor_inspection_hook(
&mut self,
layer_patterns: Vec<String>,
) -> Result<Uuid>
pub fn create_tensor_inspection_hook( &mut self, layer_patterns: Vec<String>, ) -> Result<Uuid>
Create a convenient tensor inspection hook
Sourcepub fn create_gradient_tracking_hook(
&mut self,
layer_patterns: Vec<String>,
) -> Result<Uuid>
pub fn create_gradient_tracking_hook( &mut self, layer_patterns: Vec<String>, ) -> Result<Uuid>
Create a gradient tracking hook
Sourcepub fn create_alert_hook(
&mut self,
condition: HookCondition,
message: String,
severity: AlertSeverity,
) -> Result<Uuid>
pub fn create_alert_hook( &mut self, condition: HookCondition, message: String, severity: AlertSeverity, ) -> Result<Uuid>
Create a conditional alert hook
Trait Implementations§
Source§impl Debug for HookManager
impl Debug for HookManager
Auto Trait Implementations§
impl !RefUnwindSafe for HookManager
impl !UnwindSafe for HookManager
impl Freeze for HookManager
impl Send for HookManager
impl Sync for HookManager
impl Unpin for HookManager
impl UnsafeUnpin for HookManager
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
impl<ST, DT> CastableFrom<ST, Initialized, Initialized> for DT
impl<ST, DT> CastableFrom<ST, Uninit, Uninit> for DT
Source§impl<T> Instrument for T
impl<T> Instrument for T
Source§fn instrument(self, span: Span) -> Instrumented<Self> ⓘ
fn instrument(self, span: Span) -> Instrumented<Self> ⓘ
Source§fn in_current_span(self) -> Instrumented<Self> ⓘ
fn in_current_span(self) -> Instrumented<Self> ⓘ
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§impl<T> Pointable for T
impl<T> Pointable for T
impl<T> Read<Exclusive, BecauseExclusive> for Twhere
T: ?Sized,
Source§impl<SS, SP> SupersetOf<SS> for SPwhere
SS: SubsetOf<SP>,
impl<SS, SP> SupersetOf<SS> for SPwhere
SS: SubsetOf<SP>,
Source§fn to_subset(&self) -> Option<SS>
fn to_subset(&self) -> Option<SS>
self from the equivalent element of its
superset. Read moreSource§fn is_in_subset(&self) -> bool
fn is_in_subset(&self) -> bool
self is actually part of its subset T (and can be converted to it).Source§fn to_subset_unchecked(&self) -> SS
fn to_subset_unchecked(&self) -> SS
self.to_subset but without any property checks. Always succeeds.Source§fn from_subset(element: &SS) -> SP
fn from_subset(element: &SS) -> SP
self to the equivalent element of its superset.Source§impl<SS, SP> SupersetOf<SS> for SPwhere
SS: SubsetOf<SP>,
impl<SS, SP> SupersetOf<SS> for SPwhere
SS: SubsetOf<SP>,
Source§fn to_subset(&self) -> Option<SS>
fn to_subset(&self) -> Option<SS>
self from the equivalent element of its
superset. Read moreSource§fn is_in_subset(&self) -> bool
fn is_in_subset(&self) -> bool
self is actually part of its subset T (and can be converted to it).Source§fn to_subset_unchecked(&self) -> SS
fn to_subset_unchecked(&self) -> SS
self.to_subset but without any property checks. Always succeeds.Source§fn from_subset(element: &SS) -> SP
fn from_subset(element: &SS) -> SP
self to the equivalent element of its superset.