dreamwell-intelligence 1.0.0

// GoldenRatioConverter — Quantum channel from classical frequency to dephasing rate.
//
// The frequency of a token in a corpus IS the number of times it has been measured.
// More measurements → more decoherence. The converter maps token frequency to an
// initial dephasing rate applied ONCE after embedding, before the causal loop.
//
// The embedding stays pure (learnable, clean gradients). The frequency information
// enters purely through the dephasing channel — which IS the quantum measurement
// operator. No cache. No rebuild. No state mutation. One function.
//
// ε = 1 - (surprise/max_surprise)^(1/φ)
//   Common tokens → high ε → immediate decoherence → thermodynamically gated (BA-34)
//   Rare tokens   → low ε → preserved coherence    → full compute, maximum gradient
//
// Clean Compute: no allocation, no cache, no rebuild. Explicit, measurable, stateless.

const PHI_INV: f32 = 0.618033988; // 1/φ — the compression exponent

/// Pre-computed dephasing rates per token, derived from corpus frequency.
/// Computed once. Never rebuilt. Independent of model parameters.
pub struct GoldenRatioConverter {
    /// Initial dephasing rate per token. ε ∈ [0, 1].
    /// High ε (common) → strong decoherence → low F → thermodynamically gated.
    /// Low ε (rare) → weak decoherence → high F → full compute.
    dephasing_rates: Vec<f32>,
    /// Vocabulary size.
    pub vocab_size: usize,
}

impl GoldenRatioConverter {
    /// Build from corpus token counts. Computed once — no model dependency.
    ///
    /// The dephasing rate for each token is:
    ///   surprise = -ln(count / total)
    ///   surprise_norm = surprise / ln(vocab_size)  ∈ [0, 1]
    ///   ε = 1 - surprise_norm^(1/φ)
    ///
    /// The exponent 1/φ ≈ 0.618 compresses common tokens toward high dephasing
    /// while preserving rare tokens near zero dephasing. φ governs the curve.
    pub fn new(token_counts: &[usize], total_tokens: usize, vocab_size: usize) -> Self {
        let total = total_tokens.max(1) as f32;
        let max_surprise = (vocab_size.max(2) as f32).ln();

        let dephasing_rates: Vec<f32> = (0..vocab_size)
            .map(|token| {
                let count = token_counts.get(token).copied().unwrap_or(0);
                let freq = (count as f32) / total;
                let freq_clamped = freq.max(1.0 / (total + 1.0));
                let surprise = -freq_clamped.ln();
                let surprise_norm = (surprise / max_surprise).clamp(0.0, 1.0);

                // ε = 1 - surprise_norm^(1/φ)
                // surprise_norm = 1 (max rare) → ε = 0 (no dephasing, full coherence)
                // surprise_norm = 0 (max common) → ε = 1 (full dephasing, gated)
                1.0 - surprise_norm.powf(PHI_INV)
            })
            .collect();

        Self {
            dephasing_rates,
            vocab_size,
        }
    }

    /// Get the initial dephasing rate for a token.
    #[inline]
    pub fn dephasing_rate(&self, token: usize) -> f32 {
        self.dephasing_rates.get(token).copied().unwrap_or(0.0)
    }
}

/// Compute token frequency counts from a token sequence.
pub fn compute_token_counts(tokens: &[usize], vocab_size: usize) -> Vec<usize> {
    let mut counts = vec![0usize; vocab_size];
    for &t in tokens {
        if t < vocab_size {
            counts[t] += 1;
        }
    }
    counts
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn common_tokens_higher_dephasing() {
        let counts = vec![100, 50, 10, 1];
        let conv = GoldenRatioConverter::new(&counts, 161, 4);

        // Most common → highest dephasing rate
        assert!(
            conv.dephasing_rate(0) > conv.dephasing_rate(3),
            "common token should have higher dephasing: ε₀={} ε₃={}",
            conv.dephasing_rate(0),
            conv.dephasing_rate(3)
        );

        // Monotonically decreasing with rarity
        for i in 0..3 {
            assert!(
                conv.dephasing_rate(i) >= conv.dephasing_rate(i + 1) - 0.01,
                "dephasing should decrease with rarity"
            );
        }
    }

    #[test]
    fn rare_token_near_zero_dephasing() {
        let counts = vec![100, 1];
        let conv = GoldenRatioConverter::new(&counts, 101, 2);
        let rare_eps = conv.dephasing_rate(1);
        assert!(rare_eps < 0.5, "rare token should have low dephasing: ε={rare_eps}");
    }

    #[test]
    fn dephasing_bounded_zero_one() {
        let counts = vec![1000, 500, 100, 10, 1, 0];
        let conv = GoldenRatioConverter::new(&counts, 1611, 6);
        for i in 0..6 {
            let eps = conv.dephasing_rate(i);
            assert!(eps >= 0.0 && eps <= 1.0, "token {i}: ε={eps} out of [0,1]");
        }
    }

    #[test]
    fn compute_counts_correct() {
        let tokens = vec![0, 1, 1, 2, 2, 2, 3, 3, 3, 3];
        let counts = compute_token_counts(&tokens, 5);
        assert_eq!(counts, vec![1, 2, 3, 4, 0]);
    }
}