use anyhow::Result;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::Path;
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct LayerActivations {
pub layer_idx: usize,
pub mean_magnitudes: Vec<f32>,
pub stddev_magnitudes: Vec<f32>,
pub sample_count: usize,
}
impl LayerActivations {
pub fn new(layer_idx: usize, ffn_dim: usize) -> Self {
LayerActivations {
layer_idx,
mean_magnitudes: vec![0.0f32; ffn_dim],
stddev_magnitudes: vec![0.0f32; ffn_dim],
sample_count: 0,
}
}
pub fn add_sample(&mut self, activations: &[f32]) {
if activations.len() != self.mean_magnitudes.len() {
return;
}
self.sample_count += 1;
let n = self.sample_count as f32;
for (i, &act) in activations.iter().enumerate() {
let delta = act - self.mean_magnitudes[i];
self.mean_magnitudes[i] += delta / n;
if self.sample_count > 1 {
let delta2 = act - self.mean_magnitudes[i];
self.stddev_magnitudes[i] =
(self.stddev_magnitudes[i] * (n - 1.0) + delta * delta2) / n;
}
}
}
pub fn mean_abs(&self) -> Vec<f32> {
self.mean_magnitudes.iter().map(|&m| m.abs()).collect()
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct ActivationMap {
pub layers: HashMap<usize, LayerActivations>,
pub ffn_dim: usize,
}
impl ActivationMap {
pub fn save_to_file(&self, path: &Path) -> Result<()> {
let json = serde_json::to_string_pretty(self)?;
std::fs::write(path, json)?;
Ok(())
}
pub fn load_from_file(path: &Path) -> Result<Self> {
let raw = std::fs::read_to_string(path)?;
let map = serde_json::from_str(&raw)?;
Ok(map)
}
}
pub fn load_domain_maps_from_dir(dir: &Path) -> Result<HashMap<String, ActivationMap>> {
let mut maps = HashMap::new();
for entry in std::fs::read_dir(dir)? {
let entry = entry?;
let path = entry.path();
if path.is_file() && path.extension().map(|e| e == "json").unwrap_or(false) {
let domain = path
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or_default()
.to_string();
if !domain.is_empty() {
if let Ok(map) = ActivationMap::load_from_file(&path) {
maps.insert(domain, map);
}
}
}
}
Ok(maps)
}
pub type ActivationCallback = Box<dyn FnMut(usize, &[f32]) + Send>;
pub struct ActivationCapture {
pub ffn_dim: usize,
pub activations: ActivationMap,
pub domain_activations: HashMap<String, ActivationMap>,
pub callback: Option<ActivationCallback>,
}
impl ActivationCapture {
pub fn new(ffn_dim: usize) -> Self {
ActivationCapture {
ffn_dim,
activations: ActivationMap::default(),
domain_activations: HashMap::new(),
callback: None,
}
}
pub fn with_callback(mut self, callback: ActivationCallback) -> Self {
self.callback = Some(callback);
self
}
pub fn record(&mut self, layer_idx: usize, activations: &[f32]) {
if let Some(ref mut cb) = self.callback {
cb(layer_idx, activations);
}
let layer_acts = self
.activations
.layers
.entry(layer_idx)
.or_insert_with(|| LayerActivations::new(layer_idx, self.ffn_dim));
layer_acts.add_sample(activations);
}
pub fn record_domain(&mut self, domain: &str, layer_idx: usize, activations: &[f32]) {
if let Some(ref mut cb) = self.callback {
cb(layer_idx, activations);
}
let map = self
.domain_activations
.entry(domain.to_string())
.or_default();
map.ffn_dim = self.ffn_dim;
let layer_acts = map
.layers
.entry(layer_idx)
.or_insert_with(|| LayerActivations::new(layer_idx, self.ffn_dim));
layer_acts.add_sample(activations);
}
pub fn finalize(self) -> ActivationMap {
self.activations
}
pub fn finalize_domains(self) -> HashMap<String, ActivationMap> {
self.domain_activations
}
pub fn save_to_file(&self, path: &Path) -> Result<()> {
self.activations.save_to_file(path)
}
pub fn save_domains_to_dir(&self, dir: &Path) -> Result<()> {
std::fs::create_dir_all(dir)?;
for (domain, map) in &self.domain_activations {
map.save_to_file(&dir.join(format!("{domain}.json")))?;
}
Ok(())
}
}
pub trait ActivationCaptureExt {
fn with_activation_capture(ffn_dim: usize, callback: ActivationCallback) -> Self;
fn activation_map(&self) -> &ActivationMap;
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn captures_layer_activations() {
let mut capture = ActivationCapture::new(8);
capture.record(0, &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]);
capture.record(0, &[2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]);
let map = capture.finalize();
assert!(map.layers.contains_key(&0));
let layer0 = &map.layers[&0];
assert_eq!(layer0.sample_count, 2);
}
#[test]
fn callback_is_invoked() {
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
let call_count = Arc::new(AtomicUsize::new(0));
let call_count_clone = call_count.clone();
let mut capture =
ActivationCapture::new(4).with_callback(Box::new(move |_layer, _acts| {
call_count_clone.fetch_add(1, Ordering::SeqCst);
}));
capture.record(0, &[1.0, 2.0, 3.0, 4.0]);
assert_eq!(call_count.load(Ordering::SeqCst), 1);
}
#[test]
fn captures_per_domain_activations() {
let mut capture = ActivationCapture::new(4);
capture.record_domain("sql", 0, &[0.0, 2.0, 0.0, 0.0]);
capture.record_domain("sql", 0, &[0.0, 2.0, 0.0, 0.0]);
capture.record_domain("python", 0, &[0.0, 0.0, 3.0, 0.0]);
let domains = capture.finalize_domains();
assert_eq!(domains.len(), 2);
let sql = &domains["sql"];
let python = &domains["python"];
assert_eq!(sql.layers[&0].sample_count, 2);
assert_eq!(python.layers[&0].sample_count, 1);
assert!(sql.layers[&0].mean_abs()[1] > 0.0);
assert!(python.layers[&0].mean_abs()[2] > 0.0);
}
#[test]
fn domain_maps_round_trip_via_dir() {
let mut capture = ActivationCapture::new(4);
capture.record_domain("sql", 0, &[0.0, 2.0, 0.0, 0.0]);
capture.record_domain("python", 0, &[0.0, 0.0, 3.0, 0.0]);
let dir = std::env::temp_dir().join("tpt_act_capture_dir_test");
let _ = std::fs::remove_dir_all(&dir);
capture.save_domains_to_dir(&dir).unwrap();
let loaded = load_domain_maps_from_dir(&dir).unwrap();
assert_eq!(loaded.len(), 2);
assert!(loaded.contains_key("sql"));
assert!(loaded.contains_key("python"));
assert_eq!(loaded["sql"].layers[&0].mean_abs()[1], 2.0);
let _ = std::fs::remove_dir_all(&dir);
}
}