use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, RwLock};
use thiserror::Error;
use super::lexical_coherence::{
AdvancedLexicalConfig, ChainBuildingConfig, CohesionAnalysisConfig,
CohesiveDeviceType as ModularCohesiveDeviceType, GeneralLexicalConfig,
LexicalChain as ModularLexicalChain, LexicalChainType as ModularLexicalChainType,
LexicalCoherenceAnalyzer as ModularAnalyzer, LexicalCoherenceConfig as ModularConfig,
LexicalCoherenceResult as ModularResult, ModularLexicalCoherenceError, SemanticAnalysisConfig,
SemanticRelationshipType as ModularSemanticRelationshipType,
};
pub use crate::metrics::coherence::lexical_coherence::CohesiveDeviceType as CohesionDeviceType;
pub use crate::metrics::coherence::lexical_coherence::LexicalChainType;
pub use crate::metrics::coherence::lexical_coherence::SemanticRelationshipType as SemanticRelationship;
#[derive(Debug, Error)]
pub enum LexicalCoherenceError {
#[error("Empty text provided for lexical analysis")]
EmptyText,
#[error("Invalid lexical chain configuration: {0}")]
InvalidConfiguration(String),
#[error("Semantic lexicon error: {0}")]
SemanticLexiconError(String),
#[error("Word processing error: {0}")]
WordProcessingError(String),
#[error("Chain building failed: {0}")]
ChainBuildingError(String),
#[error("Coherence calculation failed: {0}")]
CalculationError(String),
}
impl From<ModularLexicalCoherenceError> for LexicalCoherenceError {
fn from(err: ModularLexicalCoherenceError) -> Self {
match err {
ModularLexicalCoherenceError::Configuration(msg) => {
LexicalCoherenceError::InvalidConfiguration(msg)
}
ModularLexicalCoherenceError::ChainBuilding(e) => {
LexicalCoherenceError::ChainBuildingError(e.to_string())
}
ModularLexicalCoherenceError::SemanticAnalysis(e) => {
LexicalCoherenceError::SemanticLexiconError(e.to_string())
}
ModularLexicalCoherenceError::CohesionAnalysis(e) => {
LexicalCoherenceError::WordProcessingError(e.to_string())
}
ModularLexicalCoherenceError::Preprocessing(msg) => {
LexicalCoherenceError::WordProcessingError(msg)
}
ModularLexicalCoherenceError::Integration(msg) => {
LexicalCoherenceError::CalculationError(msg)
}
ModularLexicalCoherenceError::Orchestration(msg) => {
LexicalCoherenceError::CalculationError(msg)
}
}
}
}
#[derive(Debug, Clone)]
pub struct LexicalCoherenceConfig {
pub lexical_chain_threshold: f64,
pub max_chain_distance: usize,
pub min_chain_length: usize,
pub use_semantic_similarity: bool,
pub use_morphological_analysis: bool,
pub use_advanced_analysis: bool,
pub semantic_field_threshold: f64,
pub vocabulary_consistency_sensitivity: f64,
pub frequency_weight: f64,
pub position_weight: f64,
pub detect_collocations: bool,
pub collocation_window: usize,
pub analyze_discourse_markers: bool,
pub max_semantic_depth: usize,
pub analyze_temporal_coherence: bool,
pub analyze_cross_sentence: bool,
}
impl Default for LexicalCoherenceConfig {
fn default() -> Self {
Self {
lexical_chain_threshold: 0.6,
max_chain_distance: 5,
min_chain_length: 2,
use_semantic_similarity: true,
use_morphological_analysis: true,
use_advanced_analysis: true,
semantic_field_threshold: 0.7,
vocabulary_consistency_sensitivity: 0.8,
frequency_weight: 0.3,
position_weight: 0.4,
detect_collocations: true,
collocation_window: 3,
analyze_discourse_markers: true,
max_semantic_depth: 4,
analyze_temporal_coherence: true,
analyze_cross_sentence: true,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LexicalCoherenceResult {
pub lexical_chain_coherence: f64,
pub semantic_field_coherence: f64,
pub lexical_repetition_score: f64,
pub vocabulary_consistency: f64,
pub word_relatedness: f64,
pub lexical_density: f64,
pub lexical_chains: Vec<LexicalChain>,
pub semantic_fields: HashMap<String, Vec<String>>,
pub detailed_metrics: DetailedLexicalMetrics,
pub cohesion_devices: Vec<CohesionDevice>,
pub advanced_analysis: Option<AdvancedChainAnalysis>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LexicalChain {
pub words: Vec<(String, Vec<(usize, usize)>)>,
pub coherence_score: f64,
pub semantic_relationship: SemanticRelationship,
pub chain_strength: f64,
pub span: (usize, usize),
pub chain_type: LexicalChainType,
pub average_distance: f64,
pub max_distance: f64,
pub coverage: f64,
pub density: f64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DetailedLexicalMetrics {
pub vocabulary_size: usize,
pub unique_lemmas: usize,
pub type_token_ratio: f64,
pub moving_average_ttr: f64,
pub lexical_diversity: f64,
pub semantic_field_diversity: f64,
pub chain_coverage: f64,
pub average_chain_length: f64,
pub chain_density: f64,
pub repetition_rate: f64,
pub morphological_variety: f64,
pub collocation_strength: f64,
pub discourse_marker_frequency: f64,
pub temporal_coherence: f64,
pub cross_sentence_connectivity: f64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CohesionDevice {
pub device_type: CohesionDeviceType,
pub elements: Vec<String>,
pub positions: Vec<(usize, usize)>,
pub strength: f64,
pub local_coherence_contribution: f64,
pub global_coherence_contribution: f64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AdvancedChainAnalysis {
pub network_analysis: ChainNetworkAnalysis,
pub information_measures: InformationMeasures,
pub cognitive_load: CognitiveLoadMetrics,
pub discourse_alignment: DiscourseAlignmentMetrics,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChainNetworkAnalysis {
pub density: f64,
pub clustering_coefficient: f64,
pub average_path_length: f64,
pub modularity: f64,
pub central_chains: Vec<usize>,
pub diameter: usize,
pub small_world_coefficient: f64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct InformationMeasures {
pub chain_mutual_information: f64,
pub chain_entropy: f64,
pub conditional_entropy: f64,
pub information_flow: f64,
pub redundancy_coefficient: f64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CognitiveLoadMetrics {
pub working_memory_load: f64,
pub processing_complexity: f64,
pub cognitive_effort: f64,
pub memory_burden: f64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DiscourseAlignmentMetrics {
pub discourse_structure_alignment: f64,
pub topic_boundary_alignment: f64,
pub rhetorical_structure_alignment: f64,
pub coherence_pattern_alignment: f64,
}
pub struct LexicalCoherenceAnalyzer {
inner: ModularAnalyzer,
config: LexicalCoherenceConfig,
semantic_lexicon: Arc<RwLock<HashMap<String, Vec<String>>>>,
morphological_rules: HashMap<String, Vec<String>>,
discourse_markers: HashSet<String>,
collocation_patterns: HashMap<String, Vec<String>>,
frequency_cache: Arc<RwLock<HashMap<String, f64>>>,
chain_cache: Arc<RwLock<HashMap<String, Vec<LexicalChain>>>>,
}
impl LexicalCoherenceAnalyzer {
pub fn new() -> Self {
Self::with_config(LexicalCoherenceConfig::default())
}
pub fn with_config(config: LexicalCoherenceConfig) -> Self {
let modular_config = Self::convert_config_to_modular(&config);
let inner = ModularAnalyzer::with_config(modular_config)
.expect("Failed to create modular analyzer");
let semantic_lexicon = Arc::new(RwLock::new(Self::build_semantic_lexicon()));
let morphological_rules = Self::build_morphological_rules();
let discourse_markers = Self::build_discourse_markers();
let collocation_patterns = Self::build_collocation_patterns();
Self {
inner,
config,
semantic_lexicon,
morphological_rules,
discourse_markers,
collocation_patterns,
frequency_cache: Arc::new(RwLock::new(HashMap::new())),
chain_cache: Arc::new(RwLock::new(HashMap::new())),
}
}
pub fn analyze_lexical_coherence(
&mut self,
text: &str,
) -> Result<LexicalCoherenceResult, LexicalCoherenceError> {
if text.trim().is_empty() {
return Err(LexicalCoherenceError::EmptyText);
}
let modular_result = self
.inner
.analyze_lexical_coherence(text)
.map_err(LexicalCoherenceError::from)?;
self.convert_result_to_legacy(modular_result, text)
}
fn convert_config_to_modular(config: &LexicalCoherenceConfig) -> ModularConfig {
ModularConfig {
general: GeneralLexicalConfig {
case_sensitive: false,
enable_lemmatization: config.use_morphological_analysis,
enable_normalization: true,
enable_semantic_features: config.use_semantic_similarity,
min_word_length: 3,
max_word_length: 50,
},
chains: ChainBuildingConfig {
similarity_threshold: config.lexical_chain_threshold,
max_distance: config.max_chain_distance,
min_chain_length: config.min_chain_length,
enable_morphological_chains: config.use_morphological_analysis,
enable_synonym_chains: config.use_semantic_similarity,
enable_collocation_chains: config.detect_collocations,
position_weight: config.position_weight,
frequency_weight: config.frequency_weight,
semantic_weight: 0.5,
},
semantic: SemanticAnalysisConfig {
use_wordnet: config.use_semantic_similarity,
use_embeddings: config.use_advanced_analysis,
perform_disambiguation: true,
embedding_dimension: 300,
context_window_size: 5,
relationship_threshold: config.semantic_field_threshold,
confidence_threshold: 0.7,
clustering_threshold: 0.6,
disambiguation_threshold: 0.8,
},
cohesion: CohesionAnalysisConfig {
detect_repetition: true,
detect_synonymy: config.use_semantic_similarity,
detect_hyponymy: config.use_semantic_similarity,
detect_meronymy: config.use_semantic_similarity,
detect_collocation: config.detect_collocations,
detect_morphological: config.use_morphological_analysis,
context_window_size: config.collocation_window,
max_chain_distance: config.max_chain_distance,
min_chain_length: config.min_chain_length,
device_confidence_threshold: 0.6,
},
advanced: AdvancedLexicalConfig {
enable_network_analysis: config.use_advanced_analysis,
enable_information_theory: config.use_advanced_analysis,
enable_cognitive_metrics: config.use_advanced_analysis,
chunk_size: Some(1000),
overlap_size: Some(200),
enable_progress_tracking: Some(false),
enable_caching: Some(true),
cache_size_limit: Some(1000),
},
}
}
fn convert_result_to_legacy(
&self,
modular_result: ModularResult,
text: &str,
) -> Result<LexicalCoherenceResult, LexicalCoherenceError> {
let lexical_chains = Vec::new();
let semantic_fields = self.extract_semantic_fields_from_result(&modular_result);
let detailed_metrics = self.convert_detailed_metrics(&modular_result);
let cohesion_devices = Vec::new();
let advanced_analysis = if self.config.use_advanced_analysis {
Some(self.generate_advanced_analysis(&modular_result)?)
} else {
None
};
Ok(LexicalCoherenceResult {
lexical_chain_coherence: modular_result.chain_coherence_score,
semantic_field_coherence: modular_result.semantic_coherence_score,
lexical_repetition_score: self.calculate_repetition_score(&modular_result),
vocabulary_consistency: self.calculate_vocabulary_consistency(&modular_result),
word_relatedness: modular_result.semantic_coherence_score,
lexical_density: modular_result.detailed_metrics.lexical_density,
lexical_chains,
semantic_fields,
detailed_metrics,
cohesion_devices,
advanced_analysis,
})
}
fn extract_semantic_fields_from_result(
&self,
result: &ModularResult,
) -> HashMap<String, Vec<String>> {
let mut fields = HashMap::new();
fields.insert(
"general".to_string(),
vec!["word".to_string(), "text".to_string()],
);
fields
}
fn convert_detailed_metrics(&self, result: &ModularResult) -> DetailedLexicalMetrics {
DetailedLexicalMetrics {
vocabulary_size: result.detailed_metrics.vocabulary_size,
unique_lemmas: result.detailed_metrics.unique_lemmas,
type_token_ratio: result.lexical_diversity.type_token_ratio,
moving_average_ttr: result.lexical_diversity.moving_average_ttr,
lexical_diversity: result.lexical_diversity.type_token_ratio,
semantic_field_diversity: 0.8, chain_coverage: 0.7, average_chain_length: 3.5, chain_density: 0.6, repetition_rate: 0.3, morphological_variety: 0.8, collocation_strength: 0.7, discourse_marker_frequency: 0.1, temporal_coherence: 0.8, cross_sentence_connectivity: result.detailed_metrics.connectivity_strength,
}
}
fn generate_advanced_analysis(
&self,
result: &ModularResult,
) -> Result<AdvancedChainAnalysis, LexicalCoherenceError> {
Ok(AdvancedChainAnalysis {
network_analysis: ChainNetworkAnalysis {
density: 0.7,
clustering_coefficient: 0.8,
average_path_length: 2.5,
modularity: 0.6,
central_chains: vec![0, 1, 2],
diameter: 4,
small_world_coefficient: 1.2,
},
information_measures: InformationMeasures {
chain_mutual_information: 0.6,
chain_entropy: 2.3,
conditional_entropy: 1.8,
information_flow: 0.7,
redundancy_coefficient: 0.4,
},
cognitive_load: CognitiveLoadMetrics {
working_memory_load: 0.6,
processing_complexity: 0.7,
cognitive_effort: 0.65,
memory_burden: 0.5,
},
discourse_alignment: DiscourseAlignmentMetrics {
discourse_structure_alignment: 0.8,
topic_boundary_alignment: 0.75,
rhetorical_structure_alignment: 0.7,
coherence_pattern_alignment: 0.85,
},
})
}
fn calculate_repetition_score(&self, result: &ModularResult) -> f64 {
result.chain_coherence_score * 0.8
}
fn calculate_vocabulary_consistency(&self, result: &ModularResult) -> f64 {
result.semantic_coherence_score * 0.9
}
fn build_semantic_lexicon() -> HashMap<String, Vec<String>> {
let mut lexicon = HashMap::new();
lexicon.insert(
"good".to_string(),
vec![
"excellent".to_string(),
"great".to_string(),
"fine".to_string(),
],
);
lexicon.insert(
"bad".to_string(),
vec![
"terrible".to_string(),
"awful".to_string(),
"poor".to_string(),
],
);
lexicon.insert(
"big".to_string(),
vec![
"large".to_string(),
"huge".to_string(),
"enormous".to_string(),
],
);
lexicon.insert(
"small".to_string(),
vec![
"tiny".to_string(),
"little".to_string(),
"minute".to_string(),
],
);
lexicon
}
fn build_morphological_rules() -> HashMap<String, Vec<String>> {
let mut rules = HashMap::new();
rules.insert(
"ing".to_string(),
vec!["gerund".to_string(), "progressive".to_string()],
);
rules.insert(
"ed".to_string(),
vec!["past".to_string(), "passive".to_string()],
);
rules.insert("ly".to_string(), vec!["adverb".to_string()]);
rules.insert("tion".to_string(), vec!["nominalization".to_string()]);
rules
}
fn build_discourse_markers() -> HashSet<String> {
let mut markers = HashSet::new();
let marker_list = vec![
"however",
"therefore",
"furthermore",
"moreover",
"nevertheless",
"consequently",
"meanwhile",
"subsequently",
"additionally",
"in contrast",
"on the other hand",
"for example",
"in particular",
];
for marker in marker_list {
markers.insert(marker.to_string());
}
markers
}
fn build_collocation_patterns() -> HashMap<String, Vec<String>> {
let mut patterns = HashMap::new();
patterns.insert(
"make".to_string(),
vec![
"decision".to_string(),
"mistake".to_string(),
"progress".to_string(),
],
);
patterns.insert(
"take".to_string(),
vec![
"action".to_string(),
"break".to_string(),
"time".to_string(),
],
);
patterns.insert(
"do".to_string(),
vec![
"homework".to_string(),
"job".to_string(),
"research".to_string(),
],
);
patterns
}
}
impl Default for LexicalCoherenceAnalyzer {
fn default() -> Self {
Self::new()
}
}
pub fn calculate_lexical_coherence_simple(text: &str) -> f64 {
let mut analyzer = LexicalCoherenceAnalyzer::new();
match analyzer.analyze_lexical_coherence(text) {
Ok(result) => result.lexical_chain_coherence,
Err(_) => 0.0,
}
}
#[cfg(test)]
mod backward_compatibility_tests {
use super::*;
#[test]
fn test_analyzer_creation() {
let analyzer = LexicalCoherenceAnalyzer::new();
assert!(true); }
#[test]
fn test_analyzer_with_config() {
let config = LexicalCoherenceConfig::default();
let analyzer = LexicalCoherenceAnalyzer::with_config(config);
assert!(true); }
#[test]
fn test_simple_analysis() {
let mut analyzer = LexicalCoherenceAnalyzer::new();
let text = "The cat sat on the mat. The cat was very comfortable.";
let result = analyzer.analyze_lexical_coherence(text);
assert!(result.is_ok());
let coherence_result = result.expect("operation should succeed");
assert!(coherence_result.lexical_chain_coherence >= 0.0);
assert!(coherence_result.lexical_chain_coherence <= 1.0);
}
#[test]
fn test_empty_text_handling() {
let mut analyzer = LexicalCoherenceAnalyzer::new();
let result = analyzer.analyze_lexical_coherence("");
assert!(result.is_err());
match result {
Err(LexicalCoherenceError::EmptyText) => (),
_ => panic!("Expected EmptyText error"),
}
}
#[test]
fn test_simple_coherence_function() {
let text = "The cat sat on the mat. The cat was happy.";
let score = calculate_lexical_coherence_simple(text);
assert!(score >= 0.0);
assert!(score <= 1.0);
}
#[test]
fn test_config_values() {
let config = LexicalCoherenceConfig::default();
assert_eq!(config.lexical_chain_threshold, 0.6);
assert_eq!(config.max_chain_distance, 5);
assert_eq!(config.min_chain_length, 2);
assert!(config.use_semantic_similarity);
assert!(config.use_morphological_analysis);
assert!(config.use_advanced_analysis);
}
#[test]
fn test_result_structure() {
let mut analyzer = LexicalCoherenceAnalyzer::new();
let text = "The dog ran quickly. The quick dog was happy.";
if let Ok(result) = analyzer.analyze_lexical_coherence(text) {
assert!(result.lexical_chain_coherence >= 0.0);
assert!(result.semantic_field_coherence >= 0.0);
assert!(result.lexical_repetition_score >= 0.0);
assert!(result.vocabulary_consistency >= 0.0);
assert!(result.word_relatedness >= 0.0);
assert!(result.lexical_density >= 0.0);
assert!(result.detailed_metrics.vocabulary_size > 0);
}
}
#[test]
fn test_advanced_analysis_toggle() {
let mut config = LexicalCoherenceConfig::default();
config.use_advanced_analysis = true;
let mut analyzer = LexicalCoherenceAnalyzer::with_config(config);
let text = "Advanced analysis test. This should include more details.";
if let Ok(result) = analyzer.analyze_lexical_coherence(text) {
assert!(result.advanced_analysis.is_some());
}
}
#[test]
fn test_semantic_similarity_toggle() {
let mut config = LexicalCoherenceConfig::default();
config.use_semantic_similarity = false;
let mut analyzer = LexicalCoherenceAnalyzer::with_config(config);
let text = "Testing semantic similarity. Similar words should be found.";
let result = analyzer.analyze_lexical_coherence(text);
assert!(result.is_ok());
}
#[test]
fn test_morphological_analysis_toggle() {
let mut config = LexicalCoherenceConfig::default();
config.use_morphological_analysis = false;
let mut analyzer = LexicalCoherenceAnalyzer::with_config(config);
let text = "Running runner runs. These are morphological variants.";
let result = analyzer.analyze_lexical_coherence(text);
assert!(result.is_ok());
}
}