eryon-mem 0.0.4

/*
    Appellation: impl_store <module>
    Contrib: @FL03
*/
use crate::features::{FeaturePattern, FeatureRelationship, PersistentFeature, RelationshipType};
use crate::ledger::TopoLedger;

use eryon::{Direction, Head, LearnedRule, State};
use num_traits::{Float, FromPrimitive, NumAssign, ToPrimitive};
use rstmt::PitchMod;
use rstmt::nrt::{LPR, Triad, Triads};
use std::collections::{HashMap, HashSet};

impl<T> TopoLedger<T>
where
    T: Float + FromPrimitive + ToPrimitive,
{
    /// Add a relationship between features
    pub fn add_relationship(
        &mut self,
        from_id: usize,
        to_id: usize,
        relationship_type: RelationshipType,
        strength: T,
    ) {
        // Check if this relationship already exists
        if let Some(rel) = self.relationships_mut().iter_mut().find(|r| {
            r.source_id == from_id && r.to_id == to_id && r.relationship_type == relationship_type
        }) {
            // Update existing relationship
            rel.occurrences += 1;
            rel.strength = rel.strength.max(strength);
            return;
        }

        // Create new relationship
        let relationship = FeatureRelationship {
            source_id: from_id,
            to_id,
            relationship_type,
            strength,
            occurrences: 1,
        };

        self.relationships.push(relationship);
    }
    /// Dynamically adjust feature importance based on usage
    pub fn adjust_feature_importance(&mut self, feature_id: usize, adjustment: T)
    where
        T: core::iter::Sum,
    {
        let scale = T::from_usize(10).unwrap();
        if let Some(feature) = self.features.iter_mut().find(|f| f.id == feature_id) {
            // Remove from current importance bucket
            if let Some(bucket) = self
                .index
                .importance_index_mut()
                .get_mut(&((feature.importance * scale).to_usize().unwrap()))
            {
                bucket.remove(&feature_id);
            }

            // Adjust importance with limits
            feature.importance = (feature.importance + adjustment)
                .max(T::zero())
                .min(T::one());

            // Add to new importance bucket
            let new_bucket = (feature.importance * scale).to_usize().unwrap();
            self.importance_index_mut()
                .entry(new_bucket)
                .or_default()
                .insert(feature_id);
        }
    }
    /// Analyze transformation frequency by chord class
    pub fn analyze_transformation_by_class(&self) -> HashMap<Triads, HashMap<LPR, T>>
    where
        T: core::iter::Sum + NumAssign,
    {
        use strum::IntoEnumIterator;
        // Initialize with all classes and transforms
        let mut results: HashMap<_, _> = Triads::iter()
            .map(|class| {
                let transforms = LPR::iter().map(|cls| (cls, T::zero())).collect();
                (class, transforms)
            })
            .collect();

        // Get only active transformation features (dimension 1)
        let transform_features: Vec<_> = self
            .features
            .iter()
            .filter(|f| f.dimension == 1 && f.death.is_none() && f.content.len() >= 7)
            .collect();

        if transform_features.is_empty() {
            return results;
        }

        // Process transformations in a single pass
        for feature in transform_features {
            // Ensure we have enough data for a valid transformation
            if feature.content.len() < 7 {
                continue;
            }

            // Get source notes and try to determine class directly
            if let Ok(source_class) =
                Triads::try_from_arr([feature.content[0], feature.content[1], feature.content[2]])
            {
                // Get the transformation type
                if let Some(&transform_type) = feature.content.get(6) {
                    if transform_type <= 2 {
                        // Valid LPR type
                        // Convert to LPR type
                        let transform = LPR::from(transform_type);

                        // Increment the count for this class+transform
                        if let Some(transform_map) = results.get_mut(&source_class) {
                            if let Some(count) = transform_map.get_mut(&transform) {
                                *count += feature.importance;
                            }
                        }
                    }
                }
            }
        }

        // Normalize counts to frequencies
        for (_, transforms) in results.iter_mut() {
            let total = transforms.values().copied().sum();
            if total > T::zero() {
                for count in transforms.values_mut() {
                    *count /= total;
                }
            }
        }

        results
    }
    /// Optimize memory usage by compacting
    pub fn compact(&mut self) {
        // Skip if there are no dead features
        if !self.features.iter().any(|f| f.death.is_some()) {
            return;
        }

        // Remove features that have been dead for a while
        let min_death_time = if self.current_epoch() > 100 {
            self.current_epoch() - 100
        } else {
            0
        };
        self.features
            .retain(|f| f.death.is_none() || f.death.unwrap() > min_death_time);

        // Rebuild all indices
        self.rebuild_indices();
    }
    /// Calculate similarity between two content vectors
    pub fn content_similarity(&self, content1: &[usize], content2: &[usize]) -> T {
        if content1.is_empty() || content2.is_empty() {
            return T::zero();
        }

        // Calculate intersection size
        let mut common = 0;
        for &item in content1 {
            if content2.contains(&item) {
                common += 1;
            }
        }

        // Jaccard similarity
        let union = content1.len() + content2.len() - common;
        if union == 0 {
            T::one()
        } else {
            T::from_usize(common).unwrap() / T::from_usize(union).unwrap()
        }
    }
    /// Create a new feature and add it to memory
    pub fn create_feature(&mut self, dimension: usize, content: Vec<usize>) -> usize
    where
        T: ToPrimitive,
    {
        let id = self.next_feature_id();

        // Default importance based on dimension (can be refined later)
        let importance = match dimension {
            0 => 0.3, // Points
            1 => 0.5, // Edges/transformations
            2 => 0.7, // Triads
            _ => 0.4, // Other structures
        };
        let importance = T::from_f32(importance).unwrap();

        let feature = PersistentFeature {
            id,
            birth: self.current_epoch(),
            death: None,
            dimension,
            content: content.clone(),
            importance,
            occurrences: 1,
        };

        // Update indices
        self.dimension_index_mut()
            .entry(dimension)
            .or_default()
            .insert(id);

        self.content_index_mut()
            .entry(content)
            .or_default()
            .insert(id);

        let importance_bucket = (importance * T::from_usize(10).unwrap())
            .to_usize()
            .unwrap();
        self.importance_index_mut()
            .entry(importance_bucket)
            .or_default()
            .insert(id);

        self.features.push(feature);
        id
    }
    /// Batch create multiple features
    pub fn create_features_batch(&mut self, features: Vec<(usize, Vec<usize>)>) -> Vec<usize> {
        let mut ids = Vec::with_capacity(features.len());

        for (dimension, content) in features {
            let id = self.create_feature(dimension, content);
            ids.push(id);
        }

        ids
    }
    /// Create persistent features from temporal patterns
    pub fn create_persistent_features(
        &mut self,
        patterns: &[Vec<usize>],
    ) -> crate::Result<Vec<usize>> {
        let mut feature_ids = Vec::new();

        // Compute filtration of the pattern space
        let mut filtration = Vec::new();
        for (i, pattern) in patterns.iter().enumerate() {
            for window_size in 1..=pattern.len() {
                for window in pattern.windows(window_size) {
                    // Add simplex to filtration with birth time based on position
                    filtration.push((
                        window.to_vec(),
                        T::from_usize(i).unwrap() / T::from_usize(patterns.len()).unwrap(),
                    ));
                }
            }
        }

        // Sort filtration by birth time
        filtration.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(core::cmp::Ordering::Equal));

        // Track persistent features
        let mut active_features = HashMap::new();
        for (simplex, birth_time) in filtration {
            // Calculate boundary
            if simplex.len() > 1 {
                // Record feature with persistence
                let feature_id = self.create_feature(simplex.len() - 1, simplex);
                active_features.insert(feature_id, birth_time);
                feature_ids.push(feature_id);
            }
        }

        Ok(feature_ids)
    }
    /// Efficient feature deletion with object pooling
    pub fn delete_feature(&mut self, id: usize) -> bool {
        // get a copy of the current epoch;
        let epoch = self.current_epoch();
        // Find the feature to mark as deleted
        if let Some(feature) = self.features.iter_mut().find(|f| f.id == id) {
            // Mark as dead instead of removing
            feature.death = Some(epoch);

            // Remove from indices but keep in features array
            if let Some(dim_set) = self.index.dimension_index_mut().get_mut(&feature.dimension) {
                dim_set.remove(&id);
            }

            true
        } else {
            false
        }
    }
    /// Detect patterns in the recorded transformations
    pub fn detect_patterns(&mut self) {
        // Get transformation features
        let transform_features = self.find_by_dimension(1);
        if transform_features.len() < 3 {
            return; // Need at least 3 transforms for meaningful patterns
        }

        // Extract transformation types (LPR values)
        let transforms: Vec<usize> = transform_features
            .iter()
            .filter(|f| f.death.is_none() && f.content.len() > 6)
            .map(|f| f.content[6]) // Last element is transform type
            .collect();

        // Look for repeating sequences (length 2-4)
        for pattern_len in 2..=std::cmp::min(4, transforms.len() / 2) {
            for i in 0..=(transforms.len() - pattern_len * 2) {
                let pattern = &transforms[i..(i + pattern_len)];

                // Look for repetition
                let mut occurrences = 0;
                for j in 0..=(transforms.len() - pattern_len) {
                    if transforms[j..(j + pattern_len)] == pattern[0..pattern_len] {
                        occurrences += 1;
                    }
                }

                // If pattern occurs at least twice
                if occurrences >= 2 {
                    // Check if pattern already exists
                    let pattern_vec = pattern.to_vec();
                    let pattern_exists = self.patterns.iter().any(|p| p.sequence == pattern_vec);

                    if !pattern_exists {
                        // Create new pattern
                        let pattern_obj = FeaturePattern {
                            id: self.next_pattern_id(),
                            sequence: pattern_vec,
                            occurrences,
                            importance: T::from_usize(2).unwrap().recip(), // Initial importance
                        };
                        self.patterns.push(pattern_obj);
                    } else {
                        // Update existing pattern
                        if let Some(existing) =
                            self.patterns.iter_mut().find(|p| p.sequence == pattern_vec)
                        {
                            existing.occurrences = occurrences;
                            existing.update_importance_by_occurrences();
                        }
                    }
                }
            }
        }
    }
    /// Detect patterns using the KMP algorithm for efficient matching
    pub fn detect_patterns_kmp(&mut self) {
        // Get the most recent transformation features (up to 20)
        let transform_features = self
            .features
            .iter()
            .filter(|f| f.dimension == 1 && f.death.is_none())
            .collect::<Vec<_>>();

        if transform_features.len() < 3 {
            return; // Need at least 3 transforms for meaningful patterns
        }

        // Extract actual transformation types (LPR values) from features for more meaningful patterns
        // Safely access the last element
        let transforms = transform_features
            .iter()
            .filter_map(|feature| {
                if feature.content.len() > 6 {
                    feature.content.get(6).copied() // Last element is the transform type (LPR)
                } else {
                    None
                }
            })
            .collect::<Vec<_>>();

        // Look for patterns of different lengths (capped by the data size)
        for pattern_len in 2..=3.min(transforms.len() / 2) {
            // Check each possible pattern starting position
            'outer: for i in 0..=(transforms.len() - pattern_len) {
                if i + pattern_len > transforms.len() {
                    break;
                }

                let pattern = &transforms[i..(i + pattern_len)];

                // Prevent overflow by checking bounds
                if pattern.is_empty() || pattern.len() > transforms.len() {
                    continue 'outer;
                }

                // Use KMP-inspired approach for matches
                let mut count = 0;
                let mut pos = 0;

                while pos <= transforms.len() - pattern.len() {
                    let mut found = true;

                    for j in 0..pattern.len() {
                        if pos + j >= transforms.len() || transforms[pos + j] != pattern[j] {
                            found = false;
                            break;
                        }
                    }

                    if found {
                        count += 1;
                        pos += pattern.len(); // Skip the entire pattern
                    } else {
                        pos += 1; // Move one position
                    }
                }

                // If pattern occurs more than once
                if count >= 2 {
                    // Create or update the pattern
                    self.record_pattern(pattern);
                }
            }
        }
    }
    /// Extract significant patterns as learning targets
    pub fn extract_learning_targets(&self, min_importance: T) -> Vec<Vec<usize>> {
        // Get the most important patterns
        let mut important_patterns = self
            .patterns
            .iter()
            .filter(|p| p.importance >= min_importance)
            .collect::<Vec<_>>();

        // Sort by importance (highest first)
        important_patterns.sort_by(|a, b| {
            b.importance
                .partial_cmp(&a.importance)
                .unwrap_or(std::cmp::Ordering::Equal)
        });

        // Return the sequences
        important_patterns
            .into_iter()
            .map(|p| p.sequence.clone())
            .collect()
    }
    /// Extract patterns with persistent homology analysis
    pub fn extract_persistent_patterns(
        &mut self,
        persistence_threshold: T,
    ) -> Vec<FeaturePattern<T>> {
        // Create a filtration from features
        let mut filtration = Vec::new();

        // Add vertices (0-dim features)
        for feature in self
            .features
            .iter()
            .filter(|f| f.dimension == 0 && f.death.is_none())
        {
            filtration.push((feature.importance, feature.id, 0, Vec::<usize>::new()));
        }

        // Add edges from relationships (1-dim features)
        for rel in &self.relationships {
            // Only consider active features
            if self
                .features
                .iter()
                .any(|f| f.id == rel.source_id && f.death.is_none())
                && self
                    .features
                    .iter()
                    .any(|f| f.id == rel.to_id && f.death.is_none())
            {
                filtration.push((
                    rel.strength,
                    self.state.next_feature_id(),
                    1,
                    vec![rel.source_id, rel.to_id],
                ));
            }
        }

        // Sort by importance/strength (ascending for filtration)
        filtration.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(core::cmp::Ordering::Equal));

        // Compute persistence pairs
        let mut persistent_patterns = Vec::new();
        let mut active_simplices: HashMap<usize, Vec<usize>> = HashMap::new();

        // Simulate persistence computation (simplified)
        for (value, id, dimension, boundary) in filtration.clone() {
            if dimension == 0 {
                // Add vertex
                active_simplices.insert(id, Vec::new());
            } else {
                // Process edge
                // Find connected components that would merge
                let mut components_to_merge = Vec::new();
                for &vertex_id in &boundary {
                    for (&comp_id, vertices) in &active_simplices {
                        if vertices.contains(&vertex_id) {
                            components_to_merge.push(comp_id);
                            break;
                        }
                    }
                }

                // If this edge creates a persistent feature
                if components_to_merge.len() == 2 {
                    let birth_value = filtration
                        .iter()
                        .find(|(_, id, _, _)| *id == components_to_merge[0])
                        .map(|(v, _, _, _)| *v)
                        .unwrap_or(T::zero());

                    let persistence = value - birth_value;

                    if persistence > persistence_threshold {
                        // Extract the edge sequence that forms this persistent feature
                        let pattern_sequence = self.extract_pattern_from_edge(&boundary);

                        if !pattern_sequence.is_empty() {
                            persistent_patterns.push(FeaturePattern {
                                id: self.next_pattern_id(),
                                sequence: pattern_sequence,
                                occurrences: 1,
                                importance: persistence,
                            });
                        }
                    }

                    // Merge components
                    let comp1 = components_to_merge[0];
                    let comp2 = components_to_merge[1];

                    if let Some(mut vertices) = active_simplices.remove(&comp2) {
                        if let Some(comp1_vertices) = active_simplices.get_mut(&comp1) {
                            comp1_vertices.append(&mut vertices);
                        }
                    }
                }
            }
        }

        persistent_patterns
    }
    /// Extract a pattern from edge boundary
    fn extract_pattern_from_edge(&self, boundary: &[usize]) -> Vec<usize> {
        // Map boundary vertices to related transformations

        self.features
            .iter()
            .filter(|f| f.dimension == 1 && f.content.len() > 6)
            .filter(|f| boundary.contains(&f.id))
            .filter_map(|f| f.content.get(6).copied())
            .collect::<Vec<_>>()
    }
    /// Find features with content matching a prefix
    pub fn find_by_content_prefix(&self, prefix: &[usize]) -> Vec<&PersistentFeature<T>> {
        if prefix.is_empty() {
            return Vec::new();
        }

        self.content_index()
            .iter()
            .filter(|(content, _)| {
                content.len() >= prefix.len() && content.iter().take(prefix.len()).eq(prefix.iter())
            })
            .flat_map(|(_, ids)| {
                ids.iter().filter_map(|&id| {
                    self.features
                        .iter()
                        .find(|f| f.id == id && f.death.is_none())
                })
            })
            .collect()
    }
    /// Find features by time range
    pub fn find_by_time_window(
        &self,
        start_time: usize,
        end_time: usize,
    ) -> Vec<&PersistentFeature<T>> {
        self.features
            .iter()
            .filter(|f| f.birth >= start_time && f.birth <= end_time && f.death.is_none())
            .collect()
    }
    /// Find features by dimension
    pub fn find_by_dimension(&self, dimension: usize) -> Vec<&PersistentFeature<T>> {
        match self.dimension_index().get(&dimension) {
            Some(ids) => ids
                .iter()
                .filter_map(|&id| self.features.iter().find(|f| f.id == id))
                .collect(),
            None => Vec::new(),
        }
    }
    /// Find features by minimum importance
    pub fn find_by_importance(&self, min_importance: T) -> Vec<&PersistentFeature<T>> {
        let min_bucket = (min_importance * T::from_usize(10).unwrap())
            .to_usize()
            .unwrap();

        self.importance_index()
            .iter()
            .filter(|&(&bucket, _)| bucket >= min_bucket)
            .flat_map(|(_, ids)| {
                ids.iter()
                    .filter_map(|&id| self.features.iter().find(|f| f.id == id))
            })
            .collect()
    }
    /// Analyze the graph structure to find central features
    pub fn find_central_features(&self, limit: usize) -> Vec<(usize, usize)> {
        // Build adjacency map from relationships
        let mut adjacency = HashMap::<usize, HashSet<usize>>::new();

        for rel in &self.relationships {
            adjacency
                .entry(rel.source_id)
                .or_default()
                .insert(rel.to_id);

            adjacency
                .entry(rel.to_id)
                .or_default()
                .insert(rel.source_id);
        }

        // Calculate degree centrality for each feature
        let mut centrality = adjacency
            .iter()
            .map(|(&id, connections)| {
                // Only count if feature is still active
                if self
                    .features
                    .iter()
                    .any(|f| f.id == id && f.death.is_none())
                {
                    (id, connections.len())
                } else {
                    (id, 0)
                }
            })
            .collect::<Vec<_>>();

        // Sort by centrality (degree), descending
        centrality.sort_by(|a, b| b.1.cmp(&a.1));

        centrality.into_iter().take(limit).collect()
    }
    /// Find events of a specific type
    pub fn find_events(&self, event_type: &str) -> Vec<&PersistentFeature<T>> {
        let event_code = event_type.bytes().fold(0, |acc, b| acc ^ b as usize);

        self.features
            .iter()
            .filter(|f| f.dimension == 0 && f.content.first() == Some(&event_code))
            .collect()
    }
    /// Find a feature by its exact content
    pub fn find_feature_by_content(&self, content: &[usize]) -> Option<usize> {
        if let Some(ids) = self.content_index().get(content) {
            ids.iter().find_map(|&id| {
                self.features
                    .iter()
                    .find(|f| f.id == id && f.death.is_none())
                    .map(|feature| feature.id())
            })
        } else {
            None
        }
    }

    pub fn find_feature_by_id(&self, id: usize) -> Option<&PersistentFeature<T>> {
        self.features.iter().find(|f| f.id == id)
    }
    /// Find patterns that match or contain the given sequence
    pub fn find_matching_patterns(&self, sequence: &[usize]) -> Vec<&FeaturePattern<T>> {
        if sequence.is_empty() || self.patterns.is_empty() {
            return Vec::new();
        }

        self.patterns
            .iter()
            .filter(|pattern| {
                // Check if sequence is a prefix of the pattern
                if sequence.len() <= pattern.sequence.len() {
                    pattern.sequence.starts_with(sequence)
                } else {
                    // Or check if pattern is contained within the sequence
                    for i in 0..=(sequence.len() - pattern.sequence.len()) {
                        if pattern.sequence == sequence[i..(i + pattern.sequence.len())] {
                            return true;
                        }
                    }
                    false
                }
            })
            .collect()
    }
    /// Find harmonic pathways between triads
    pub fn find_harmonic_pathway(
        &self,
        from_notes: &[usize; 3],
        to_notes: &[usize; 3],
        max_length: usize,
    ) -> Option<Vec<LPR>> {
        // Convert note vectors to content format
        let from_content = from_notes.to_vec();
        let to_content = to_notes.to_vec();

        // Find features for these triads
        let from_id = if let Some(ids) = self.content_index().get(&from_content) {
            *ids.iter().next()?
        } else {
            return None;
        };

        let to_id = if let Some(ids) = self.content_index().get(&to_content) {
            *ids.iter().next()?
        } else {
            return None;
        };

        // Find path between the two features using breadth-first search
        let mut queue = std::collections::VecDeque::new();
        let mut visited = HashSet::new();
        let mut predecessors = HashMap::new();

        queue.push_back(from_id);
        visited.insert(from_id);

        while let Some(current) = queue.pop_front() {
            if current == to_id {
                // Path found, reconstruct it
                let mut path = Vec::new();
                let mut current_id = current;

                while current_id != from_id {
                    let (pred_id, transform) = predecessors.get(&current_id)?;
                    path.push(*transform);
                    current_id = *pred_id;
                }

                // Reverse path since we built it backwards
                path.reverse();

                // Convert indices to LPR values
                let lpr_path = path.into_iter().map(LPR::from).collect();

                return Some(lpr_path);
            }

            // Explore neighbors
            for rel in &self.relationships {
                if rel.source_id == current && !visited.contains(&rel.to_id) {
                    // Find the transformation that connects these triads
                    let transform_features = self
                        .features
                        .iter()
                        .filter(|f| f.dimension == 1 && f.content.len() > 6)
                        .filter(|f| {
                            // Check if feature connects our triads
                            let from_match = self.features.iter().any(|triad| {
                                triad.id == current
                                    && triad.content.len() >= 3
                                    && f.content[0] == triad.content[0]
                                    && f.content[1] == triad.content[1]
                                    && f.content[2] == triad.content[2]
                            });

                            let to_match = self.features.iter().any(|triad| {
                                triad.id == rel.to_id
                                    && triad.content.len() >= 3
                                    && f.content[3] == triad.content[0]
                                    && f.content[4] == triad.content[1]
                                    && f.content[5] == triad.content[2]
                            });

                            from_match && to_match
                        })
                        .collect::<Vec<_>>();

                    if let Some(transform) = transform_features.first() {
                        if let Some(&transform_type) = transform.content.get(6) {
                            queue.push_back(rel.to_id);
                            visited.insert(rel.to_id);
                            predecessors.insert(rel.to_id, (current, transform_type));

                            // Stop if path gets too long
                            if visited.len() > max_length {
                                return None;
                            }
                        }
                    }
                }

                // Also check reverse direction
                if rel.to_id == current && !visited.contains(&rel.source_id) {
                    // Similar logic for reverse transforms...
                    let transform_features = self
                        .features
                        .iter()
                        .filter(|f| f.dimension == 1 && f.content.len() > 6)
                        .filter(|f| {
                            let from_match = self.features.iter().any(|triad| {
                                triad.id == rel.source_id
                                    && triad.content.len() >= 3
                                    && f.content[0] == triad.content[0]
                                    && f.content[1] == triad.content[1]
                                    && f.content[2] == triad.content[2]
                            });

                            let to_match = self.features.iter().any(|triad| {
                                triad.id == current
                                    && triad.content.len() >= 3
                                    && f.content[3] == triad.content[0]
                                    && f.content[4] == triad.content[1]
                                    && f.content[5] == triad.content[2]
                            });

                            from_match && to_match
                        })
                        .collect::<Vec<_>>();

                    if let Some(transform) = transform_features.first() {
                        if let Some(&transform_type) = transform.content.get(6) {
                            queue.push_back(rel.source_id);
                            visited.insert(rel.source_id);
                            predecessors.insert(rel.source_id, (current, transform_type));

                            if visited.len() > max_length {
                                return None;
                            }
                        }
                    }
                }
            }
        }

        // No path found
        None
    }
    /// Find structural isomorphisms between different pattern regions
    pub fn find_isomorphisms(&self, min_size: usize) -> Vec<(Vec<usize>, Vec<usize>, T)>
    where
        T: core::iter::Sum,
    {
        // Find regions of connected features
        let regions = self.identify_memory_regions(T::from_usize(2).unwrap().recip());
        let mut isomorphisms = Vec::new();

        // Compare each pair of regions
        for (i, region1) in regions.iter().enumerate() {
            if region1.len() < min_size {
                continue;
            }

            for region2 in regions.iter().skip(i + 1) {
                if region2.len() < min_size {
                    continue;
                }

                // Check for structural similarity
                let similarity = self.region_structural_similarity(region1, region2);

                // If similarity is high enough, record the isomorphism
                if similarity > T::from_f32(0.7).unwrap() {
                    isomorphisms.push((region1.clone(), region2.clone(), similarity));
                }
            }
        }

        // Sort by similarity
        isomorphisms.sort_by(|a, b| b.2.partial_cmp(&a.2).unwrap_or(std::cmp::Ordering::Equal));
        isomorphisms
    }
    /// Find features with related content (partial overlap)
    pub fn find_related_content(
        &self,
        content: &[usize],
        min_overlap: usize,
    ) -> Vec<&PersistentFeature<T>> {
        if content.is_empty() || min_overlap == 0 {
            return Vec::new();
        }

        self.features
            .iter()
            .filter(|f| {
                // Skip dead features
                if f.death.is_some() {
                    return false;
                }

                // Count overlapping elements
                let overlap_count = content
                    .iter()
                    .filter(|&item| f.content.contains(item))
                    .count();

                overlap_count >= min_overlap
            })
            .collect()
    }
    /// Find features by their relationships
    pub fn find_related_features(
        &self,
        feature_id: usize,
        relationship_type: Option<RelationshipType>,
    ) -> Vec<&PersistentFeature<T>> {
        // First find all relationships involving this feature
        let related_ids = self
            .relationships
            .iter()
            .filter(|rel| {
                // Filter by relationship type if specified
                relationship_type.is_none_or(|rt| rel.relationship_type == rt) &&
                // Source or target matches the feature ID
                (rel.source_id == feature_id || rel.to_id == feature_id)
            })
            .map(|rel| {
                if rel.source_id == feature_id {
                    rel.to_id
                } else {
                    rel.source_id
                }
            })
            .collect::<HashSet<_>>();

        // Then find the actual features
        self.features
            .iter()
            .filter(|f| related_ids.contains(&f.id) && f.death.is_none())
            .collect()
    }
    pub fn find_rule(&self, rule: LearnedRule<T, usize, usize>) -> Option<usize> {
        let content = vec![
            3,
            **rule.head().state,
            *rule.head().symbol,
            **rule.tail().state,
            *rule.tail().symbol,
            (rule.tail.direction as usize) + 1,
        ];

        self.find_feature_by_content(&content)
    }
    /// filter the rulespace by the head of the rule
    pub fn find_rule_by_head(&self, head: Head<&usize, &usize>) -> Option<&LearnedRule<T>> {
        self.rulespace.iter().find(|rule| rule.head() == head)
    }
    /// Generate a continuation of a given transformation sequence
    pub fn generate_continuation(&self, seed: &[usize], length: usize) -> Vec<usize> {
        let mut result = seed.to_vec();

        // Generate continuations up to desired length
        while result.len() < length {
            if let Some((next, _)) = self.predict_next_transformation(&result) {
                result.push(next);
            } else {
                break; // No more predictions possible
            }
        }

        result
    }
    /// Generate a compact representation of the pattern network
    pub fn generate_pattern_network(&self) -> String
    where
        T: core::fmt::Debug,
    {
        let mut json = String::from("{\n  \"nodes\": [\n");

        // Add pattern nodes
        for (i, pattern) in self.patterns.iter().enumerate() {
            let seq_str = pattern
                .sequence
                .iter()
                .map(|&x| x.to_string())
                .collect::<Vec<_>>()
                .join(",");

            json.push_str(&format!(
                "    {{\"id\": {}, \"sequence\": [{}], \"importance\": {:?}}}",
                pattern.id, seq_str, pattern.importance
            ));

            if i < self.patterns.len() - 1 {
                json.push_str(",\n");
            } else {
                json.push('\n');
            }
        }

        json.push_str("  ],\n  \"links\": [\n");

        // Find pattern relationships
        let mut links = Vec::new();
        for (i, p1) in self.patterns.iter().enumerate() {
            for (j, p2) in self.patterns.iter().enumerate() {
                if i >= j {
                    continue;
                }

                // Check for shared subsequences
                let similarity = Self::sequence_similarity(&p1.sequence, &p2.sequence);
                if similarity > T::from_usize(2).unwrap().recip() {
                    links.push((p1.id, p2.id, similarity));
                }
            }
        }

        // Output links
        for (i, (source, target, weight)) in links.iter().enumerate() {
            json.push_str(&format!(
                "    {{\"source\": {}, \"target\": {}, \"weight\": {:?}}}",
                source, target, weight
            ));

            if i < links.len() - 1 {
                json.push_str(",\n");
            } else {
                json.push('\n');
            }
        }

        json.push_str("  ]\n}\n");
        json
    }
    /// Generate a training curriculum based on memory patterns
    pub fn generate_training_curriculum(&self, stages: usize) -> Vec<Vec<(usize, T)>>
    where
        T: NumAssign,
    {
        // Early exit if we have no patterns
        if self.patterns.is_empty() {
            return Vec::new();
        }

        // Analyze which transformation types are most common/important
        let mut transform_importance = HashMap::new();

        for pattern in &self.patterns {
            for &transform in &pattern.sequence {
                let entry = transform_importance.entry(transform).or_insert(T::zero());
                *entry += pattern.importance / T::from_usize(pattern.sequence().len()).unwrap();
            }
        }

        // Sort transformations by importance
        let mut transform_vec: Vec<_> = transform_importance.into_iter().collect();
        transform_vec.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));

        // Generate curriculum stages
        let mut curriculum = Vec::new();
        let transforms_per_stage = (transform_vec.len() / stages).max(1);

        for i in 0..stages {
            let start = i * transforms_per_stage;
            let end = if i == stages - 1 {
                transform_vec.len()
            } else {
                (i + 1) * transforms_per_stage
            };

            if start < transform_vec.len() {
                let stage_transforms = transform_vec[start..end.min(transform_vec.len())].to_vec();
                curriculum.push(stage_transforms);
            }
        }

        curriculum
    }
    /// Retrieve the best rule for a given state-symbol pair
    pub fn get_best_rule(
        &self,
        Head { state, symbol }: Head<usize, usize>,
    ) -> Option<(usize, usize, Direction, T)> {
        // Find all rules for this state-symbol pair
        let mut rules = self
            .features
            .iter()
            .filter(|f| {
                f.dimension == 3
                    && f.death.is_none()
                    && f.content.len() >= 6
                    && f.content[1] == *state
                    && f.content[2] == symbol
            })
            .collect::<Vec<_>>();

        // If no rules found, return None
        if rules.is_empty() {
            return None;
        }

        // Sort by confidence (importance)
        rules.sort_by(|a, b| {
            b.importance
                .partial_cmp(&a.importance)
                .unwrap_or(std::cmp::Ordering::Equal)
        });

        // Return the highest confidence rule
        let best = rules[0];

        // Extract the rule components
        let next_state = best.content[3];
        let next_symbol = best.content[4];
        let direction = (best.content[5] as i8) - 1; // Convert back to -1, 0, 1

        Some((next_state, next_symbol, direction.into(), best.importance))
    }
    /// Get all events in chronological order
    pub fn get_recent_events(&self, limit: usize) -> Vec<&PersistentFeature<T>> {
        let mut events: Vec<_> = self.features.iter().filter(|f| f.dimension == 0).collect();

        // Sort by birth time (most recent first)
        events.sort_by(|a, b| b.birth.cmp(&a.birth));

        // Limit results
        events.truncate(limit);
        events
    }
    /// Get a recent pattern of transformations
    pub fn get_recent_pattern(&self, max_length: usize) -> Option<Vec<usize>> {
        let transform_features = self.recent_transformations(max_length);

        if transform_features.is_empty() {
            return None;
        }

        Some(transform_features)
    }
    /// Get statistics about the memory state
    pub fn get_statistics(&self) -> crate::MemoryStatistics<T>
    where
        T: NumAssign,
    {
        let total_features = self.features.len();
        let active_features = self.features.iter().filter(|f| f.death.is_none()).count();

        // Count features by dimension
        let mut dimension_counts = HashMap::new();
        for feature in &self.features {
            if feature.death.is_none() {
                *dimension_counts.entry(feature.dimension).or_insert(0) += 1;
            }
        }

        // Calculate average importance by dimension
        let mut avg_importance_by_dim = HashMap::new();
        for feature in &self.features {
            if feature.death.is_none() {
                let entry = avg_importance_by_dim
                    .entry(feature.dimension)
                    .or_insert((T::zero(), 0));

                entry.0 += feature.importance;
                entry.1 += 1;
            }
        }

        // Finalize averages
        let avg_importance_by_dim = avg_importance_by_dim
            .into_iter()
            .map(|(dim, (sum, count))| (dim, sum / T::from_usize(count).unwrap()))
            .collect();

        // Most common patterns
        let mut pattern_counts = self
            .patterns
            .iter()
            .map(|p| (p.sequence.clone(), p.occurrences))
            .collect::<Vec<_>>();

        pattern_counts.sort_by(|a, b| b.1.cmp(&a.1)); // Sort by frequency, descending
        let most_common_patterns = pattern_counts.into_iter().take(5).collect();

        crate::MemoryStatistics {
            total_features,
            active_features,
            feature_counts: dimension_counts,
            average_importance: avg_importance_by_dim,
            relationship_count: self.relationships().len(),
            pattern_count: self.patterns().len(),
            common_patterns: most_common_patterns,
            memory_time: self.current_epoch(),
        }
    }
    /// Calculate the stability of a triad based on memory patterns and relationships
    pub fn get_stability_for(&self, notes: &[usize; 3]) -> T
    where
        T: NumAssign,
    {
        // Convert to a vector for content lookup
        let content = notes.to_vec();

        // Try to find a feature for this triad
        let triad_feature = if let Some(ids) = self.content_index().get(&content) {
            // Find the first active feature with these notes
            ids.iter()
                .filter_map(|&id| {
                    self.features
                        .iter()
                        .find(|f| f.id == id && f.death.is_none())
                })
                .next()
        } else {
            None
        };

        // If we found a feature, calculate stability based on relationships and patterns
        if let Some(feature) = triad_feature {
            let scale = T::from_f32(0.2).unwrap();
            // Base stability is the feature's importance
            let mut stability = feature.importance;

            // Adjust for feature age (older = more stable)
            let feature_age = self.current_epoch().saturating_sub(feature.birth);
            let age_factor =
                (T::from_usize(feature_age).unwrap() / T::from_usize(10).unwrap()).min(T::one()); // Cap at T::one()
            stability += age_factor * scale; // Age can add up to 0.2

            // Adjust for connectedness (more connections = more stable)
            let connections = self
                .relationships
                .iter()
                .filter(|r| r.source_id == feature.id || r.to_id == feature.id)
                .count();

            let connection_factor =
                (T::from_usize(connections).unwrap() / T::from_usize(5).unwrap()).min(T::one()); // Cap at T::one()
            stability += connection_factor * scale; // Connections can add up to 0.2

            // Adjust for pattern involvement
            let pattern_involvement = self
                .patterns
                .iter()
                .filter(|p| {
                    // Check if any transformation involving this triad is in the pattern
                    self.features
                        .iter()
                        .filter(|f| f.dimension == 1 && f.content.len() > 6)
                        .filter(|f| {
                            // Check if feature represents a transformation involving our triad
                            let from_match = f.content[0] == notes[0]
                                && f.content[1] == notes[1]
                                && f.content[2] == notes[2];

                            let to_match = f.content[3] == notes[0]
                                && f.content[4] == notes[1]
                                && f.content[5] == notes[2];

                            from_match || to_match
                        })
                        .any(|f| {
                            // Check if transformation type is in pattern
                            if let Some(&transform_type) = f.content.get(6) {
                                p.sequence.contains(&transform_type)
                            } else {
                                false
                            }
                        })
                })
                .count();

            let pattern_factor = (T::from_usize(pattern_involvement).unwrap()
                / T::from_usize(3).unwrap())
            .min(T::one()); // Cap at T::one()
            stability += pattern_factor * T::from_f32(0.1).unwrap(); // Patterns can add up to 0.1

            // Cap stability at T::one()
            stability.min(T::one())
        } else {
            // If no feature exists, use default stability (caller will use defaults)
            T::from_usize(2).unwrap().recip()
        }
    }
    /// Get surface parameter at a position
    pub fn get_surface_parameter(&self, position: &[usize]) -> Option<T> {
        // Find features with this position prefix
        let features = self.find_by_content_prefix(&{
            let mut prefix = Vec::with_capacity(position.len() + 1);
            prefix.push(4); // Dimension 4 for surface parameters
            prefix.extend_from_slice(position);
            prefix
        });

        if features.is_empty() {
            return None;
        }

        // Get the most important feature
        let best = features.into_iter().max_by(|a, b| {
            a.importance
                .partial_cmp(&b.importance)
                .unwrap_or(std::cmp::Ordering::Equal)
        })?;

        // Extract value from content
        if best.content.len() > position.len() + 1 {
            let value_hash = best.content[position.len() + 1];
            Some(T::from_usize(value_hash).unwrap() / T::from_usize(10000).unwrap())
        } else {
            None
        }
    }
    /// Identify regions of memory that form coherent structures
    pub fn identify_memory_regions(&self, coherence_threshold: T) -> Vec<Vec<usize>> {
        // Create a graph from relationships
        let mut adjacency = HashMap::<usize, Vec<(usize, T)>>::new();

        for rel in &self.relationships {
            // Only consider active features with sufficient strength
            if rel.strength >= coherence_threshold
                && self
                    .features
                    .iter()
                    .any(|f| f.id == rel.source_id && f.death.is_none())
                && self
                    .features
                    .iter()
                    .any(|f| f.id == rel.to_id && f.death.is_none())
            {
                // Add bidirectional connections
                adjacency
                    .entry(rel.source_id)
                    .or_default()
                    .push((rel.to_id, rel.strength));

                adjacency
                    .entry(rel.to_id)
                    .or_default()
                    .push((rel.source_id, rel.strength));
            }
        }

        // Find connected components using BFS
        let mut visited = HashSet::new();
        let mut regions = Vec::new();

        for feature in self.features.iter().filter(|f| f.death.is_none()) {
            if visited.contains(&feature.id()) {
                continue;
            }

            // Start new region
            let mut region = Vec::new();
            let mut queue = std::collections::VecDeque::new();
            queue.push_back(feature.id());
            visited.insert(feature.id());

            while let Some(current) = queue.pop_front() {
                region.push(current);

                // Add neighbors
                if let Some(neighbors) = adjacency.get(&current) {
                    for &(neighbor, _) in neighbors {
                        if !visited.contains(&neighbor) {
                            queue.push_back(neighbor);
                            visited.insert(neighbor);
                        }
                    }
                }
            }

            if !region.is_empty() {
                regions.push(region);
            }
        }

        // Sort regions by size (largest first)
        regions.sort_by_key(|r| -(r.len() as isize));
        regions
    }
    /// Learn relationships between elements in headspace
    pub fn learn_headspace_relationships(
        &mut self,
        notes: &[usize; 3],
        relationships: &[(usize, usize, T)],
    ) {
        let epoch = self.current_epoch();
        // Create features for each note
        let note_features = notes
            .iter()
            .map(|&note| {
                // Check if feature already exists
                let content = vec![note];
                let feature_ids = self
                    .index
                    .content_index_mut()
                    .entry(content.clone())
                    .or_default();

                match feature_ids.iter().next() {
                    Some(&i) => i,
                    None => {
                        // Create new feature
                        let id = self.state.position.next_feature_id();
                        let feature = PersistentFeature {
                            id,
                            birth: epoch,
                            death: None,
                            dimension: 0, // Notes are 0-dim features
                            content: content.clone(),
                            importance: T::from_usize(2).unwrap().recip(), // Default importance
                            occurrences: 1,
                        };

                        feature_ids.insert(id);
                        self.dimension_index_mut().entry(0).or_default().insert(id);

                        self.features.push(feature);
                        id
                    }
                }
            })
            .collect::<Vec<_>>();

        // Create relationships between notes based on provided relationships
        for &(i, j, strength) in relationships {
            if i < note_features.len() && j < note_features.len() {
                self.create_relationship(
                    note_features[i],
                    note_features[j],
                    RelationshipType::Interval,
                    strength,
                );
            }
        }
    }

    /// Learn surface characteristics from critical points
    pub fn learn_surface(
        &mut self,
        critical_points: &HashMap<String, usize>,
        reference_note: usize,
    ) {
        if critical_points.is_empty() {
            return;
        }

        // Create a feature for the reference note (usually tonic)
        let reference_content = vec![reference_note];
        let reference_id = self.create_or_get_feature(
            0, // Dimension
            reference_content,
            T::from_f32(0.8).unwrap(), // High importance as reference
        );

        // Create features for each critical point
        for (name, &point) in critical_points {
            // Create content with the point value and a name hash
            let name_hash = name
                .bytes()
                .fold(0, |acc: usize, b| acc.wrapping_add(b as usize));
            let content = vec![point, name_hash];

            let point_id = self.create_or_get_feature(
                0, // Dimension
                content,
                T::from_f32(0.7).unwrap(), // Important as critical point
            );

            // Create relationship to reference note
            let distance = ((point as isize - reference_note as isize).pmod()) as usize;
            let strength = T::one() - T::from_usize(distance).unwrap() / T::from_usize(6).unwrap(); // Normalize to T::zero()-T::one()

            self.create_relationship(reference_id, point_id, RelationshipType::Critical, strength);
        }
    }
    /// Learn surface parameters between critical points
    pub fn learn_surface_parameter(&mut self, position: Vec<usize>, value: T) -> usize {
        // Create a content vector for surface parameters
        // Format: [dimension, ...position coordinates, value hash]
        let mut content = Vec::with_capacity(position.len() + 2);
        content.push(4); // Dimension 4 for surface parameters
        content.extend_from_slice(&position);

        // Hash the value to store it in the content
        let value_hash = (value * T::from_usize(10000).unwrap()).to_usize().unwrap();
        content.push(value_hash);

        // Create or update the feature
        self.create_or_get_feature(4, content, value.abs())
    }
    /// Merge another memory system into this one
    ///
    /// This is used for sharing patterns and knowledge between nodes.
    /// Only copies features, patterns and relationships from the source that don't exist in this memory.
    pub fn merge(&mut self, source: &Self) {
        // Skip if source is empty
        if source.features.is_empty() && source.patterns.is_empty() {
            return;
        }

        // Track original and new IDs for feature remapping
        let mut id_mapping: HashMap<usize, usize> = HashMap::new();

        // Copy active features
        for feature in source.features.iter().filter(|f| f.death.is_none()) {
            // Check if we already have this feature by content
            let maybe_existing = if let Some(ids) = self.content_index().get(feature.content()) {
                ids.iter().find_map(|&id| {
                    self.features
                        .iter()
                        .find(|f| f.id == id && f.death.is_none())
                        .map(|f| f.id)
                })
            } else {
                None
            };

            if let Some(existing_id) = maybe_existing {
                // Feature already exists, map source ID to our ID
                id_mapping.insert(feature.id, existing_id);

                // Update importance if source has higher importance
                if let Some(our_feature) = self.features.iter_mut().find(|f| f.id == existing_id) {
                    if feature.importance() > our_feature.importance() {
                        // Remove from current importance bucket
                        if let Some(bucket) = self
                            .index
                            .importance_index_mut()
                            .get_mut(&(feature.importance_to_usize_by_10()))
                        {
                            bucket.remove(&existing_id);
                        }

                        // Update importance
                        our_feature.set_importance(*feature.importance());

                        // Add to new importance bucket
                        let new_bucket = feature.importance_to_usize_by_10();
                        self.importance_index_mut()
                            .entry(new_bucket)
                            .or_default()
                            .insert(existing_id);
                    }
                }
            } else {
                // Create new feature with our own ID
                let new_id = self.next_feature_id();

                // Map source ID to our new ID
                id_mapping.insert(feature.id(), new_id);

                // Create the feature in our memory
                let merged_feature = PersistentFeature {
                    id: new_id,
                    birth: self.current_epoch(), // Use our current epoch
                    death: None,
                    content: feature.content().clone(),
                    dimension: feature.dimension(),
                    importance: feature.importance,
                    occurrences: feature.occurrences(),
                };

                // Update indices
                self.dimension_index_mut()
                    .entry(feature.dimension)
                    .or_default()
                    .insert(new_id);

                self.content_index_mut()
                    .entry(feature.content().clone())
                    .or_default()
                    .insert(new_id);

                let importance_bucket = feature.importance_to_usize_by_10();
                self.importance_index_mut()
                    .entry(importance_bucket)
                    .or_default()
                    .insert(new_id);

                self.features.push(merged_feature);
            }
        }

        // Copy relationships, remapping IDs
        for rel in source.relationships() {
            // Map source IDs to our IDs
            if let (Some(&from_id), Some(&to_id)) =
                (id_mapping.get(&rel.source_id), id_mapping.get(&rel.to_id))
            {
                // Check if this relationship already exists
                let rel_exists = self.relationships.iter().any(|r| {
                    r.source_id == from_id
                        && r.to_id == to_id
                        && r.relationship_type == rel.relationship_type
                });

                if !rel_exists {
                    // Create new relationship
                    let new_rel = FeatureRelationship {
                        source_id: from_id,
                        to_id,
                        ..rel.clone()
                    };

                    self.relationships.push(new_rel);
                } else {
                    // Update existing relationship if needed
                    if let Some(existing) = self.relationships.iter_mut().find(|r| {
                        r.source_id == from_id
                            && r.to_id == to_id
                            && r.relationship_type == rel.relationship_type
                    }) {
                        existing.occurrences += rel.occurrences;
                        existing.strength = existing.strength.max(rel.strength);
                    }
                }
            }
        }
        let point9 = T::from_f32(0.9).unwrap();
        // Copy patterns
        for pattern in &source.patterns {
            // Check if we already have this pattern
            let pattern_exists = self.patterns.iter().any(|p| p.sequence == pattern.sequence);

            if !pattern_exists {
                // Create new pattern with our own ID
                let new_pattern = FeaturePattern {
                    id: self.next_pattern_id(),
                    sequence: pattern.sequence.clone(),
                    occurrences: pattern.occurrences,
                    importance: pattern.importance * point9, // Slightly reduce importance when sharing
                };

                self.patterns.push(new_pattern);
            } else {
                // Update existing pattern if appropriate
                if let Some(existing) = self
                    .patterns
                    .iter_mut()
                    .find(|p| p.sequence == pattern.sequence)
                {
                    // Increment occurrences
                    existing.occurrences += 1;

                    // Update importance as max of current and incoming (with slight discount)
                    existing.importance = existing.importance.max(pattern.importance * point9);
                }
            }
        }
    }
    /// Merge two features, keeping the higher importance one
    pub fn merge_features(&mut self, primary_id: usize, secondary_id: usize) {
        // Get features

        // TODO: make use of the primary feature or drop
        // let primary = match self.find_feature_by_id(primary_id) {
        //     Some(f) if f.death.is_none() => f.clone(),
        //     _ => return,
        // };

        let secondary = match self.find_feature_by_id(secondary_id) {
            Some(f) if f.death.is_none() => f.clone(),
            _ => return,
        };

        // Increase importance of primary feature
        if let Some(feature) = self.features.iter_mut().find(|f| f.id == primary_id) {
            feature.importance =
                feature.importance.max(secondary.importance) + T::from_f32(0.1).unwrap();
            feature.occurrences += secondary.occurrences;
        }
        // get a copy of the current epoch
        let epoch = self.current_epoch();
        // Mark secondary feature as "dead"
        if let Some(feature) = self
            .features_mut()
            .iter_mut()
            .find(|f| f.id == secondary_id)
        {
            feature.death = Some(epoch);
        }

        // Update relationships
        for rel in self.relationships_mut().iter_mut() {
            if rel.source_id == secondary_id {
                rel.source_id = primary_id;
            }
            if rel.to_id == secondary_id {
                rel.to_id = primary_id;
            }
        }
    }
    /// Find most influential transformations in memory
    pub fn most_influential_transformations(&self, limit: usize) -> Vec<(LPR, T)>
    where
        T: NumAssign,
    {
        // Count all transformation occurrences weighted by importance
        let mut transform_influence = HashMap::new();

        // From explicit transformations
        for feature in self
            .features
            .iter()
            .filter(|f| f.dimension == 1 && f.death.is_none())
        {
            if let Some(&transform_type) = feature.content.get(6) {
                let entry = transform_influence
                    .entry(transform_type)
                    .or_insert(T::zero());
                *entry += feature.importance;
            }
        }

        // From patterns
        for pattern in &self.patterns {
            for &transform in &pattern.sequence {
                let entry = transform_influence.entry(transform).or_insert(T::zero());
                *entry += pattern.importance / T::from_usize(pattern.sequence.len()).unwrap();
            }
        }

        // Convert to LPR types and limit results
        let mut results: Vec<_> = transform_influence
            .into_iter()
            .map(|(t, influence)| (t.into(), influence))
            .collect();

        // Sort by influence, descending
        results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
        results.truncate(limit);

        results
    }
    /// Optimize memory by consolidating similar features
    pub fn optimize(&mut self)
    where
        T: NumAssign,
    {
        // First pass: mark similar events for merging
        let mut to_merge = Vec::new();

        // Check dimension 0 features (events) with similar content
        let d0_features = self
            .features
            .iter()
            .filter(|f| f.dimension == 0 && f.death.is_none())
            .collect::<Vec<_>>();

        for (i, f1) in d0_features.iter().enumerate() {
            for f2 in d0_features.iter().skip(i + 1) {
                if f1.content.len() == f2.content.len()
                    && !f1.content.is_empty()
                    && f1.content[0] == f2.content[0]
                    && f1.birth.abs_diff(f2.birth) < 5
                {
                    // Close in time

                    // Check content similarity
                    let mut similarity = T::one();
                    for j in 1..f1.content.len() {
                        if j < f2.content.len() && f1.content[j] != f2.content[j] {
                            similarity -= T::one() / T::from_usize(f1.content.len()).unwrap();
                        }
                    }

                    if similarity > T::from_f32(0.8).unwrap() {
                        // Merge f2 into f1 (keep the older one)
                        if f1.birth <= f2.birth {
                            to_merge.push((f2.id, f1.id));
                        } else {
                            to_merge.push((f1.id, f2.id));
                        }
                    }
                }
            }
        }

        // Perform merges
        for (remove_id, keep_id) in to_merge {
            // get a copy of the current epoch
            let epoch = self.current_epoch();
            // Mark the redundant feature as dead
            if let Some(feature) = self.features.iter_mut().find(|f| f.id == remove_id) {
                feature.death = Some(epoch);
            }

            // Update relationships to point to the kept feature
            for rel in self.relationships.iter_mut() {
                if rel.source_id == remove_id {
                    rel.source_id = keep_id;
                }
                if rel.to_id == remove_id {
                    rel.to_id = keep_id;
                }
            }

            // Boost importance of the kept feature
            if let Some(feature) = self.features.iter_mut().find(|f| f.id == keep_id) {
                feature.importance = (feature.importance + T::from_f32(0.1).unwrap()).min(T::one());
            }
        }

        // Rebuild indices to reflect changes
        self.rebuild_indices();

        // Now deduplicate relationships
        let mut unique_rels = Vec::new();
        let mut seen = HashSet::new();

        for rel in self.relationships.drain(..) {
            let key = (rel.source_id, rel.to_id, rel.relationship_type as usize);
            if !seen.contains(&key) {
                seen.insert(key);
                unique_rels.push(rel);
            }
        }

        self.relationships = unique_rels;
    }
    /// Predict the next likely transformation in a sequence
    pub fn predict_next_transformation(&self, recent_transforms: &[usize]) -> Option<(usize, T)> {
        if recent_transforms.is_empty() {
            return None;
        }

        // Find patterns that match the recent sequence as a prefix
        let mut candidates = Vec::new();

        for pattern in &self.patterns {
            // Skip patterns shorter than the input
            if pattern.sequence.len() <= recent_transforms.len() {
                continue;
            }

            // Check if pattern starts with our recent transforms
            if pattern.sequence.starts_with(recent_transforms) {
                // The next element in the pattern is our prediction
                let predicted = pattern.sequence[recent_transforms.len()];
                let confidence = pattern.importance
                    * (T::from_usize(pattern.occurrences).unwrap() / T::from_usize(10).unwrap())
                        .min(T::one());

                candidates.push((predicted, confidence));
            }
        }

        // If we found direct matches, use the highest confidence one
        if !candidates.is_empty() {
            candidates.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
            return Some(candidates[0]);
        }

        // If no direct matches, try partial matches (fuzzy prediction)
        let mut fuzzy_candidates = Vec::new();
        let min_length = recent_transforms.len().min(3).max(1);
        let recent_suffix = &recent_transforms[recent_transforms.len() - min_length..];

        for pattern in &self.patterns {
            if pattern.sequence.len() <= min_length {
                continue;
            }

            // Look for the suffix in this pattern
            for i in 0..=pattern.sequence.len() - min_length {
                if &pattern.sequence[i..i + min_length] == recent_suffix {
                    // Found a match - predict what comes next in the pattern
                    if i + min_length < pattern.sequence.len() {
                        let predicted = pattern.sequence[i + min_length];
                        // Lower confidence for partial matches
                        let confidence = pattern.importance
                            * T::from_f32(0.7).unwrap()
                            * (T::from_usize(pattern.occurrences).unwrap()
                                / T::from_usize(10).unwrap())
                            .min(T::one());

                        fuzzy_candidates.push((predicted, confidence));
                    }
                }
            }
        }

        // Return best fuzzy match if any
        if !fuzzy_candidates.is_empty() {
            fuzzy_candidates
                .sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(core::cmp::Ordering::Equal));
            return Some(fuzzy_candidates[0]);
        }

        // No predictions found
        None
    }
    /// Process patterns with minimum overhead
    pub fn process_patterns_batch(&mut self, inputs: &[Vec<usize>]) -> Vec<usize> {
        if inputs.is_empty() || self.patterns.is_empty() {
            return Vec::new();
        }

        // Pre-allocate for matched indices
        let mut matched_indices = Vec::new();

        // Skip empty patterns - create references to active patterns
        // This avoids repeated iteration over self.patterns
        let active_patterns: Vec<(usize, &FeaturePattern<T>)> = self
            .patterns
            .iter()
            .enumerate()
            .filter(|(_, p)| !p.sequence.is_empty())
            .collect();

        if active_patterns.is_empty() {
            return Vec::new();
        }

        // Process each input once
        for input in inputs {
            if input.is_empty() {
                continue;
            }

            for &(idx, pattern) in &active_patterns {
                // Quick length check first
                if pattern.sequence.len() != input.len() {
                    continue;
                }

                // Do direct slice comparison
                if pattern.sequence() == input {
                    matched_indices.push(idx);
                }
            }
        }

        // Update pattern occurrences for matches
        for &idx in &matched_indices {
            if let Some(pattern) = self.patterns.get_mut(idx) {
                pattern.occurrences += 1;
            }
        }

        matched_indices
    }
    /// Prune less important features when memory gets too large
    pub fn prune(&mut self, max_features: usize) {
        if self.features.len() <= max_features {
            return;
        }
        // get a copy of the current epoch
        let epoch = self.current_epoch();
        // sort features by importance
        let mut feature_ids: Vec<_> = self
            .features
            .iter()
            .filter(|f| f.death.is_none()) // Only consider active features
            .map(|f| (f.id, f.importance))
            .collect();

        feature_ids.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal));

        // Determine how many features to prune
        let to_prune = self.features.len() - max_features;

        // Mark least important features as dead
        for (id, _) in feature_ids.iter().take(to_prune) {
            if let Some(feature) = self.features.iter_mut().find(|f| f.id == *id) {
                feature.death = Some(epoch);
            }
        }
    }
    /// Record an event in the memory system
    pub fn record_event(&mut self, event_type: &str, metadata: Option<Vec<usize>>) -> usize {
        // Create a feature representing the event
        let dimension = 0; // Events are 0-dimensional features
        let content = if let Some(meta) = metadata {
            // Convert event type to a numeric code and combine with metadata
            let event_code = event_type.bytes().fold(0, |acc, b| acc ^ b as usize);
            let mut content = vec![event_code];
            content.extend(meta);
            content
        } else {
            // Just use the event type code
            let event_code = event_type.bytes().fold(0, |acc, b| acc ^ b as usize);
            vec![event_code]
        };

        // Create the event feature with higher importance
        let feature_id = self.next_feature_id();

        let feature = PersistentFeature {
            id: feature_id,
            dimension,
            birth: self.current_epoch(),
            death: None,
            content,
            importance: T::from_f32(0.8).unwrap(), // Events are important
            occurrences: 1,
        };

        // Add to indices
        self.dimension_index_mut()
            .entry(dimension)
            .or_default()
            .insert(feature_id);

        // Add feature
        self.features.push(feature);

        feature_id
    }
    /// Mark a feature as pruned (dead)
    pub fn prune_feature(&mut self, feature_id: usize) {
        let epoch = self.current_epoch();
        if let Some(feature) = self.features_mut().iter_mut().find(|f| f.id == feature_id) {
            feature.death = Some(epoch);
        }
    }
    /// Rebuild indices after removing features
    pub fn rebuild_indices(&mut self) {
        // Clear existing indices
        self.index_mut().clear();

        // Rebuild from remaining features
        for feature in &self.features {
            if feature.death.is_some() {
                continue;
            }

            self.index
                .dimension_index_mut()
                .entry(feature.dimension)
                .or_default()
                .insert(feature.id);

            self.index
                .content_index_mut()
                .entry(feature.content().to_vec())
                .or_default()
                .insert(feature.id);

            let importance_bucket = feature.importance_to_usize_by_10();
            self.index
                .importance_index_mut()
                .entry(importance_bucket)
                .or_default()
                .insert(feature.id);
        }

        // Clean up relationship references to dead features
        self.relationships.retain(|rel| {
            let from_exists = self
                .features
                .iter()
                .any(|f| f.id == rel.source_id && f.death.is_none());
            let to_exists = self
                .features
                .iter()
                .any(|f| f.id == rel.to_id && f.death.is_none());
            from_exists && to_exists
        });
    }
    /// Records a pattern
    pub fn record_pattern(&mut self, pattern: &[usize]) {
        let pattern_vec = pattern.to_vec();

        // Check if this pattern already exists
        let pattern_exists = self.patterns.iter().any(|p| p.sequence == pattern_vec);

        if !pattern_exists {
            // Create new pattern
            let pattern = FeaturePattern {
                id: self.next_pattern_id(),
                sequence: pattern_vec,
                occurrences: 1,
                importance: T::from_usize(2).unwrap().recip(), // Initial importance
            };
            self.patterns.push(pattern);
        } else {
            // Update existing pattern
            if let Some(pattern) = self.patterns.iter_mut().find(|p| p.sequence == pattern_vec) {
                pattern.occurrences += 1;
                pattern.update_importance_by_occurrences();
            }
        }
    }
    /// records a pattern with the given importance
    pub fn record_pattern_with_importance(&mut self, pattern: &[usize], importance: T) {
        let pattern_vec = pattern.to_vec();

        // Check if this pattern already exists
        let pattern_exists = self.patterns.iter().any(|p| p.sequence == pattern_vec);

        if !pattern_exists {
            // Create new pattern
            let pattern = FeaturePattern {
                id: self.next_pattern_id(),
                sequence: pattern_vec,
                occurrences: 1,
                importance,
            };
            self.patterns.push(pattern);
        } else {
            // Update existing pattern
            if let Some(pattern) = self.patterns.iter_mut().find(|p| p.sequence == pattern_vec) {
                pattern.occurrences += 1;
                pattern.update_importance_by_occurrences();
            }
        }
    }
    /// Record a transformation between triads
    pub fn record_transformation(
        &mut self,
        from_notes: [usize; 3],
        from_class: Triads,
        transform: LPR,
        to_notes: [usize; 3],
        to_class: Triads,
    ) -> usize {
        // Record the transformation as a 1-dimensional feature
        let mut content = Vec::with_capacity(7);
        content.extend_from_slice(&from_notes);
        content.extend_from_slice(&to_notes);
        content.push(transform as usize);

        let id = self.create_feature(1, content);

        // Find the source and target triad features
        let source = Triad::new(from_notes, from_class);
        let source_id = self.find_or_create_triad(&source);

        let target = Triad::new(to_notes, to_class);
        let target_id = self.find_or_create_triad(&target);

        // Create relationship between triads
        self.add_relationship(
            source_id,
            target_id,
            RelationshipType::Transformation,
            T::from_f32(0.7).unwrap(),
        );

        // Look for patterns after recording transformation
        self.detect_patterns();

        id
    }
    /// Get the most recent transformation types in chronological order
    pub fn recent_transformations(&self, count: usize) -> Vec<usize> {
        // Find all transformation features (dimension 1)
        let transform_features = self.find_by_dimension(1);

        // Filter active features and sort by birth time (most recent first)
        let mut recent_transforms = transform_features
            .iter()
            .filter(|f| f.death.is_none() && f.content.len() > 6)
            .collect::<Vec<_>>();

        // Sort by birth time (newest first)
        recent_transforms.sort_by(|a, b| b.birth.cmp(&a.birth));

        // Extract the transformation types (index 6 holds the LPR value)
        recent_transforms
            .into_iter()
            .take(count)
            .map(|f| f.content[6])
            .collect()
    }
    /// Record multiple transformations in batch for efficiency
    pub fn record_transformations_batch(
        &mut self,
        transforms: Vec<([usize; 3], Triads, LPR, [usize; 3], Triads)>,
    ) -> Vec<usize> {
        if transforms.is_empty() {
            return Vec::new();
        }

        let mut transform_ids = Vec::with_capacity(transforms.len());

        for (from_notes, from_class, transform, to_notes, to_class) in transforms {
            let id =
                self.record_transformation(from_notes, from_class, transform, to_notes, to_class);
            transform_ids.push(id);
        }

        transform_ids
    }

    /// Record a successful navigation between two points
    pub fn record_navigation(
        &mut self,
        from_notes: &[usize],
        to_notes: &[usize],
        transforms_used: &[LPR],
    ) {
        // Convert transforms to usize
        let transform_ids: Vec<_> = transforms_used
            .iter()
            .map(|&t| match t {
                LPR::Leading => 0,
                LPR::Parallel => 1,
                LPR::Relative => 2,
            })
            .collect();

        // Create content by combining from_notes, to_notes and a hash of transforms
        let mut content = Vec::with_capacity(from_notes.len() + to_notes.len() + 1);
        content.extend_from_slice(from_notes);
        content.extend_from_slice(to_notes);

        // Add transform path hash
        let path_hash = transform_ids.iter().fold(0, |acc, &t| acc * 3 + t);
        content.push(path_hash);

        // Create or update navigation feature
        let _ = self.create_or_get_feature(
            2, // Navigation features are dimension 2
            content,
            T::from_f32(0.6).unwrap(), // Medium-high importance
        );

        // Record the transformation sequence as a pattern
        if !transform_ids.is_empty() {
            self.record_pattern_with_importance(&transform_ids, T::from_f32(0.7).unwrap());
        }
    }
    /// Reinforce a rule that led to successful outcomes
    pub fn reinforce_rule(
        &mut self,
        State(state): State<usize>,
        symbol: usize,
        State(next_state): State<usize>,
        next_symbol: usize,
        direction: Direction,
        amount: T,
    ) where
        T: core::iter::Sum,
    {
        // Find the rule
        let content = vec![
            3,
            state,
            symbol,
            next_state,
            next_symbol,
            (direction + 1) as usize,
        ];

        if let Some(id) = self.find_feature_by_content(&content) {
            self.adjust_feature_importance(id, amount);
        }
    }
    /// returns the total size of the memory
    pub fn size(&self) -> usize {
        // Base size: fixed overhead
        let mut total_size = 3 * core::mem::size_of::<usize>(); // epoch, next_id, next_pattern_id

        // Features size
        total_size += self
            .features
            .iter()
            .map(|f| {
                // Base feature size + content size
                core::mem::size_of::<PersistentFeature<T>>()
                    + f.content().len() * core::mem::size_of::<usize>()
            })
            .sum::<usize>();

        // Relationships size
        total_size += self.relationships.len() * core::mem::size_of::<FeatureRelationship<T>>();

        // Rules size
        total_size += self
            .rulespace
            .iter()
            .map(|_r| {
                // Base rule size + any dynamic content
                core::mem::size_of::<LearnedRule>() + core::mem::size_of::<(usize, usize)>()
            })
            .sum::<usize>();

        // Patterns size
        total_size += self
            .patterns
            .iter()
            .map(|p| {
                // Base pattern size + sequence size
                core::mem::size_of::<FeaturePattern<T>>()
                    + p.sequence().len() * core::mem::size_of::<usize>()
            })
            .sum::<usize>();

        // Index structures (approximate)
        total_size +=
            self.dimension_index().len() * core::mem::size_of::<(usize, HashSet<usize>)>();
        total_size += self.content_index().len()
            * self
                .content_index()
                .iter()
                .map(|(k, v)| {
                    k.len() * core::mem::size_of::<usize>() + // key size
                v.len() * core::mem::size_of::<usize>() // value set size
                })
                .sum::<usize>();
        total_size +=
            self.importance_index().len() * core::mem::size_of::<(usize, HashSet<usize>)>();

        total_size
    }
    /// Store a learned rule with associated confidence
    pub fn store_learned_rule(
        &mut self,
        State(state): State<usize>,
        symbol: usize,
        State(next_state): State<usize>,
        next_symbol: usize,
        direction: Direction,
        confidence: T,
    ) -> usize {
        // Create a content vector encoding the rule
        // [dimension, state, symbol, next_state, next_symbol, direction+1]
        let content = vec![
            // LearnedRule::from_parts(State(state), symbol, direction, State(next_state), next_symbol, confidence)
            3,
            state,
            symbol,
            next_state,
            next_symbol,
            (direction as usize) + 1,
        ];

        // Check if we already have this rule
        if let Some(existing_id) = self.find_feature_by_content(&content) {
            // Update existing rule's confidence
            if let Some(feature) = self.features.iter_mut().find(|f| f.id == existing_id) {
                let old_confidence = feature.importance;
                let old_bucket = (old_confidence * T::from_usize(10).unwrap())
                    .to_usize()
                    .unwrap();
                feature.importance = (old_confidence * T::from_f32(0.8).unwrap()
                    + confidence * T::from_f32(0.2).unwrap())
                .min(T::one());

                // Update importance index
                let new_bucket = feature.importance_to_usize_by_10();

                if old_bucket != new_bucket {
                    if let Some(bucket) = self.importance_index_mut().get_mut(&old_bucket) {
                        bucket.remove(&existing_id);
                    }

                    self.importance_index_mut()
                        .entry(new_bucket)
                        .or_default()
                        .insert(existing_id);
                }
            }

            return existing_id;
        }

        // Create a new rule feature
        let id = self.create_feature(3, content);
        // Adjust importance to match confidence
        if let Some(feature) = self.features.iter_mut().find(|f| f.id == id) {
            feature.importance = confidence;

            // Update importance index
            let bucket = (confidence * T::from_usize(10).unwrap())
                .to_usize()
                .unwrap();
            if let Some(old_bucket) = self
                .index
                .importance_index_mut()
                .get_mut(&feature.importance_to_usize_by_10())
            {
                old_bucket.remove(&id);
            }

            self.index
                .importance_index_mut()
                .entry(bucket)
                .or_default()
                .insert(id);
        }

        id
    }
    /// Generate a surface parameter prediction model
    pub fn surface_parameter_model(&self) -> HashMap<Vec<usize>, T> {
        let mut model = HashMap::new();

        // Find all surface parameter features (dimension 4)
        let surface_features = self
            .features
            .iter()
            .filter(|f| f.dimension == 4 && f.death.is_none())
            .collect::<Vec<_>>();

        for feature in surface_features {
            if feature.content.len() < 3 {
                continue; // Need at least dimension + position + value
            }

            // Extract position (skip the dimension marker)
            let position = feature.content[1..feature.content.len() - 1].to_vec();

            // Extract value
            let value_hash = feature.content[feature.content.len() - 1];
            let value = T::from_usize(value_hash).unwrap() / T::from_usize(10000).unwrap();

            // Add to model, weighted by importance
            let entry = model.entry(position).or_insert(T::zero());
            *entry = (*entry * T::from_f32(0.7).unwrap())
                + (value * feature.importance * T::from_f32(0.3).unwrap());
        }

        model
    }
    /// Calculate transformation probabilities from memory
    pub fn transformation_probabilities(&self, current_triad: &Triad) -> Vec<(LPR, T)>
    where
        T: NumAssign,
    {
        // Find the triad features
        let content = current_triad.notes().to_vec();
        let _triad_id = if let Some(ids) = self.content_index().get(&content) {
            if let Some(&id) = ids.iter().next() {
                id
            } else {
                return vec![
                    (LPR::Leading, T::from_usize(3).unwrap().recip()),
                    (LPR::Parallel, T::from_usize(3).unwrap().recip()),
                    (LPR::Relative, T::from_usize(3).unwrap().recip()),
                ];
            }
        } else {
            return vec![
                (LPR::Leading, T::from_usize(3).unwrap().recip()),
                (LPR::Parallel, T::from_usize(3).unwrap().recip()),
                (LPR::Relative, T::from_usize(3).unwrap().recip()),
            ];
        };

        // Find all transformations from this triad
        let transform_features = self
            .features
            .iter()
            .filter(|f| f.dimension == 1 && f.death.is_none() && f.content.len() > 6)
            .filter(|f| {
                f.content[0] == current_triad[0]
                    && f.content[1] == current_triad[1]
                    && f.content[2] == current_triad[2]
            })
            .collect::<Vec<_>>();

        // Count occurrences of each transform type
        let mut counts = HashMap::new();
        let mut total = T::zero();

        for feature in transform_features {
            if let Some(&transform_type) = feature.content.get(6) {
                let entry = counts.entry(transform_type).or_insert(T::zero());
                *entry += feature.importance;
                total += feature.importance;
            }
        }

        // Calculate probabilities
        let mut probabilities = Vec::new();
        let default_prop = T::from(0.1).unwrap();
        let denom = total + T::from(0.3).unwrap();
        // Leading transform (0)
        let l_prob = counts.get(&0).copied().unwrap_or(default_prop) / denom;
        probabilities.push((LPR::Leading, l_prob));

        // Parallel transform (1)
        let p_prob = counts.get(&1).copied().unwrap_or(default_prop) / denom;
        probabilities.push((LPR::Parallel, p_prob));

        // Relative transform (2)
        let r_prob = counts.get(&2).copied().unwrap_or(default_prop) / denom;
        probabilities.push((LPR::Relative, r_prob));

        // Sort by probability
        probabilities.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));

        probabilities
    }
    /// Generate a visualization of the memory topology
    pub fn visualize(&self) -> String
    where
        T: core::fmt::Debug,
    {
        // Create a DOT format graph for visualization
        let mut dot = String::from("digraph memory_topology {\n");

        // Add nodes (features)
        for feature in self.features.iter().filter(|f| f.death.is_none()) {
            let color = match feature.dimension {
                0 => "lightblue",
                1 => "lightgreen",
                2 => "orange",
                3 => "pink",
                _ => "gray",
            };

            let label = format!("{}\\nDim:{}", feature.id, feature.dimension);
            dot.push_str(&format!(
                "  node{} [label=\"{}\", style=filled, fillcolor={}];\n",
                feature.id, label, color
            ));
        }

        // Add edges (relationships)
        for rel in &self.relationships {
            // Only include relationships between active features
            if self
                .features
                .iter()
                .any(|f| f.id == rel.source_id && f.death.is_none())
                && self
                    .features
                    .iter()
                    .any(|f| f.id == rel.to_id && f.death.is_none())
            {
                let rel_type = match rel.relationship_type {
                    RelationshipType::Transformation => "transform",
                    RelationshipType::Interval => "interval",
                    RelationshipType::Critical => "critical",
                    _ => "other",
                };

                let width =
                    (rel.strength * T::from_usize(2).unwrap()).max(T::from_f32(0.5).unwrap());

                dot.push_str(&format!(
                    "  node{} -> node{} [label=\"{}\", penwidth={:?}];\n",
                    rel.source_id, rel.to_id, rel_type, width
                ));
            }
        }

        dot.push_str("}\n");
        dot
    }
}

impl<T> TopoLedger<T>
where
    T: Float + FromPrimitive,
{
    /// Create or get a feature by content
    pub(crate) fn create_or_get_feature(
        &mut self,
        dimension: usize,
        content: Vec<usize>,
        importance: T,
    ) -> usize {
        // Check if feature already exists
        if let Some(ids) = self.content_index().get(&content) {
            if let Some(&id) = ids.iter().next() {
                // Feature exists, update importance if necessary
                if let Some(feature) = self.features.iter_mut().find(|f| f.id == id) {
                    if importance > feature.importance {
                        // Update importance
                        feature.importance = importance;
                    }
                }
                return id;
            }
        }

        // Create new feature
        let id = self.next_feature_id();

        let feature = PersistentFeature {
            id,
            birth: self.current_epoch(),
            death: None,
            dimension,
            content: content.clone(),
            importance,
            occurrences: 1,
        };

        // Update indices
        self.content_index_mut()
            .entry(content)
            .or_default()
            .insert(id);

        self.dimension_index_mut()
            .entry(dimension)
            .or_default()
            .insert(id);

        let importance_bucket = feature.importance_to_usize_by_10();
        self.importance_index_mut()
            .entry(importance_bucket)
            .or_default()
            .insert(id);

        self.features.push(feature);
        id
    }
    /// Create a relationship between features
    pub(crate) fn create_relationship(
        &mut self,
        from_id: usize,
        to_id: usize,
        rel_type: RelationshipType,
        strength: T,
    ) {
        // Check if relationship already exists
        let existing = self.relationships.iter_mut().find(|r| {
            r.source_id == from_id && r.to_id == to_id && r.relationship_type == rel_type
        });

        if let Some(rel) = existing {
            // Update existing relationship
            rel.occurrences += 1;
            rel.strength = (rel.strength + strength) / T::from_usize(2).unwrap(); // Average strength
        } else {
            // Create new relationship
            let relationship = FeatureRelationship::new(from_id, to_id, rel_type, strength);

            self.relationships.push(relationship);
        }
    }
    /// Find a triad feature or create it if not found
    pub(crate) fn find_or_create_triad(&mut self, triad: &Triad) -> usize {
        // Try to find an existing triad feature
        let content = triad.notes().to_vec();
        if let Some(ids) = self.content_index().get(&content) {
            if let Some(&id) = ids.iter().next() {
                return id;
            }
        }

        // Create a new triad feature
        self.create_feature(2, content)
    }

    /// Calculate structural similarity between two regions
    pub(crate) fn region_structural_similarity(&self, region1: &[usize], region2: &[usize]) -> T
    where
        T: core::iter::Sum,
    {
        // Early exits for empty regions
        if region1.is_empty() || region2.is_empty() {
            return T::zero();
        }

        // Compute dimension distributions
        let mut dim_dist1 = HashMap::new();
        let mut dim_dist2 = HashMap::new();

        for &id in region1 {
            if let Some(feature) = self
                .features
                .iter()
                .find(|f| f.id == id && f.death.is_none())
            {
                *dim_dist1.entry(feature.dimension).or_insert(0) += 1;
            }
        }

        for &id in region2 {
            if let Some(feature) = self
                .features
                .iter()
                .find(|f| f.id == id && f.death.is_none())
            {
                *dim_dist2.entry(feature.dimension).or_insert(0) += 1;
            }
        }

        // Calculate dimension distribution similarity
        let mut total_dims = 0;
        let mut matching_dims = 0;

        for (&dim, &count1) in &dim_dist1 {
            total_dims += count1;
            if let Some(&count2) = dim_dist2.get(&dim) {
                matching_dims += count1.min(count2);
            }
        }

        for (&_, &count2) in &dim_dist2 {
            total_dims += count2;
        }

        // Avoid double-counting
        total_dims -= matching_dims;

        let dim_similarity = if total_dims > 0 {
            T::from_usize(matching_dims).unwrap() / T::from_usize(total_dims).unwrap()
        } else {
            T::zero()
        };

        // Build adjacency matrices for both regions
        let mut adj1 = HashMap::new();
        let mut adj2 = HashMap::new();

        // Map IDs to positions for each region
        let id_to_pos1: HashMap<_, _> =
            region1.iter().enumerate().map(|(i, &id)| (id, i)).collect();

        let id_to_pos2: HashMap<_, _> =
            region2.iter().enumerate().map(|(i, &id)| (id, i)).collect();

        // Build adjacency matrices from relationships
        for rel in &self.relationships {
            // For region 1
            if let (Some(&pos_from), Some(&pos_to)) =
                (id_to_pos1.get(&rel.source_id), id_to_pos1.get(&rel.to_id))
            {
                adj1.entry(pos_from)
                    .or_insert_with(Vec::new)
                    .push((pos_to, rel.relationship_type as usize));
            }

            // For region 2
            if let (Some(&pos_from), Some(&pos_to)) =
                (id_to_pos2.get(&rel.source_id), id_to_pos2.get(&rel.to_id))
            {
                adj2.entry(pos_from)
                    .or_insert_with(Vec::new)
                    .push((pos_to, rel.relationship_type as usize));
            }
        }

        // Compare structural properties

        // 1. Compare average degree
        let avg_degree1 = if !region1.is_empty() {
            T::from_usize(adj1.values().map(|v| v.len()).sum()).unwrap()
                / T::from_usize(region1.len()).unwrap()
        } else {
            T::zero()
        };

        let avg_degree2 = if !region2.is_empty() {
            T::from_usize(adj2.values().map(|v| v.len()).sum()).unwrap()
                / T::from_usize(region2.len()).unwrap()
        } else {
            T::zero()
        };

        let degree_similarity = if avg_degree1 == T::zero() && avg_degree2 == T::zero() {
            T::one() // Both have no edges
        } else if avg_degree1 == T::zero() || avg_degree2 == T::zero() {
            T::zero() // One has edges, one doesn't
        } else if avg_degree1 > avg_degree2 {
            avg_degree2 / avg_degree1
        } else {
            avg_degree1 / avg_degree2
        };

        // 2. Compare relationship type distributions
        let mut rel_types1 = HashMap::new();
        let mut rel_types2 = HashMap::new();

        for connections in adj1.values() {
            for &(_, rel_type) in connections {
                *rel_types1.entry(rel_type).or_insert(0) += 1;
            }
        }

        for connections in adj2.values() {
            for &(_, rel_type) in connections {
                *rel_types2.entry(rel_type).or_insert(0) += 1;
            }
        }

        let mut total_rels = 0;
        let mut matching_rels = 0;

        for (&rel_type, &count1) in &rel_types1 {
            total_rels += count1;
            if let Some(&count2) = rel_types2.get(&rel_type) {
                matching_rels += count1.min(count2);
            }
        }

        for (&_, &count2) in &rel_types2 {
            total_rels += count2;
        }

        // Avoid double-counting
        total_rels -= matching_rels;

        let rel_type_similarity = if total_rels > 0 {
            T::from_usize(matching_rels).unwrap() / T::from_usize(total_rels).unwrap()
        } else {
            T::zero()
        };

        // 3. Compare connection patterns using Weisfeiler-Lehman test-inspired approach
        // (simplified version that looks at "shapes" of connections)

        // Create signature strings for each node based on its connections
        let mut signatures1 = vec![String::new(); region1.len()];
        let mut signatures2 = vec![String::new(); region2.len()];

        // Generate initial signatures based on dimensions and outgoing edge types
        for (node, connections) in &adj1 {
            let mut sig = Vec::new();

            // Add node dimension to signature
            if let Some(idx) = region1.get(*node) {
                if let Some(feature) = self
                    .features
                    .iter()
                    .find(|f| f.id == *idx && f.death.is_none())
                {
                    sig.push(feature.dimension);
                }
            }

            // Add outgoing connection types (sorted)
            let mut rel_sigs: Vec<_> = connections.iter().map(|&(_, rt)| rt).collect();
            rel_sigs.sort_unstable();
            sig.extend(rel_sigs);

            // Create signature string
            signatures1[*node] = format!("{:?}", sig);
        }

        for (node, connections) in &adj2 {
            let mut sig = Vec::new();

            // Add node dimension to signature
            if let Some(idx) = region2.get(*node) {
                if let Some(feature) = self
                    .features
                    .iter()
                    .find(|f| f.id == *idx && f.death.is_none())
                {
                    sig.push(feature.dimension);
                }
            }

            // Add outgoing connection types (sorted)
            let mut rel_sigs: Vec<_> = connections.iter().map(|&(_, rt)| rt).collect();
            rel_sigs.sort_unstable();
            sig.extend(rel_sigs);

            // Create signature string
            signatures2[*node] = format!("{:?}", sig);
        }

        // Count signature distributions
        let mut sig_counts1 = HashMap::new();
        let mut sig_counts2 = HashMap::new();

        for sig in &signatures1 {
            if !sig.is_empty() {
                *sig_counts1.entry(sig).or_insert(0) += 1;
            }
        }

        for sig in &signatures2 {
            if !sig.is_empty() {
                *sig_counts2.entry(sig).or_insert(0) += 1;
            }
        }

        // Compare signature distributions
        let mut matching_sigs = 0;
        let mut total_sigs = 0;

        for (sig, &count1) in &sig_counts1 {
            total_sigs += count1;
            if let Some(&count2) = sig_counts2.get(sig) {
                matching_sigs += count1.min(count2);
            }
        }

        for &count2 in sig_counts2.values() {
            total_sigs += count2;
        }

        // Avoid double-counting
        total_sigs -= matching_sigs;

        let structure_similarity = if total_sigs > 0 {
            T::from_usize(matching_sigs).unwrap() / T::from_usize(total_sigs).unwrap()
        } else {
            T::zero()
        };

        // Combine the different similarity measures with appropriate weights
        let w1 = T::from_f32(0.2).unwrap();
        let w2 = T::from_f32(0.3).unwrap();
        let w3 = T::from_f32(0.2).unwrap();
        let w4 = T::from_f32(0.3).unwrap();

        w1 * dim_similarity
            + w2 * degree_similarity
            + w3 * rel_type_similarity
            + w4 * structure_similarity
    }
    /// Calculate similarity between two sequences
    pub(crate) fn sequence_similarity(seq1: &[usize], seq2: &[usize]) -> T {
        let max_len = seq1.len().max(seq2.len());
        if max_len == 0 {
            return T::zero();
        }

        // Find longest common subsequence
        let mut dp = vec![vec![0; seq2.len() + 1]; seq1.len() + 1];

        for i in 1..=seq1.len() {
            for j in 1..=seq2.len() {
                if seq1[i - 1] == seq2[j - 1] {
                    dp[i][j] = dp[i - 1][j - 1] + 1;
                } else {
                    dp[i][j] = dp[i - 1][j].max(dp[i][j - 1]);
                }
            }
        }

        let lcs_length = dp[seq1.len()][seq2.len()];
        T::from_usize(lcs_length).unwrap() / T::from_usize(max_len).unwrap()
    }
}