iriq 0.30.2

IRI/URL extraction, normalization, and shape clustering.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

use crate::classifier::{SegmentClassifier, SegmentType};
use std::collections::HashMap;

pub const DEFAULT_MAX_VALUES_PER_POSITION: usize = 5_000;

#[derive(Debug, Clone)]
pub struct PositionStats {
    pub value_counts: HashMap<String, usize>,
    pub type_counts: HashMap<SegmentType, usize>,
    pub total: usize,
    pub max_values: usize,
    pub numeric_count: usize,
    pub numeric_min: f64,
    pub numeric_max: f64,
    pub numeric_sum: f64,
}

impl PositionStats {
    pub fn new(max_values: usize) -> Self {
        let cap = if max_values == 0 {
            DEFAULT_MAX_VALUES_PER_POSITION
        } else {
            max_values
        };
        PositionStats {
            value_counts: HashMap::new(),
            type_counts: HashMap::new(),
            total: 0,
            max_values: cap,
            numeric_count: 0,
            numeric_min: 0.0,
            numeric_max: 0.0,
            numeric_sum: 0.0,
        }
    }

    pub fn observe(&mut self, value: &str, t: SegmentType) {
        self.total += 1;
        *self.type_counts.entry(t).or_insert(0) += 1;
        let present = self.value_counts.contains_key(value);
        if present || self.value_counts.len() < self.max_values {
            *self.value_counts.entry(value.to_string()).or_insert(0) += 1;
        }
        self.record_numeric(value, t);
    }

    fn record_numeric(&mut self, value: &str, t: SegmentType) {
        if t != SegmentType::Integer && t != SegmentType::Float {
            return;
        }
        let Ok(n) = value.parse::<f64>() else {
            return;
        };
        if self.numeric_count == 0 || n < self.numeric_min {
            self.numeric_min = n;
        }
        if self.numeric_count == 0 || n > self.numeric_max {
            self.numeric_max = n;
        }
        self.numeric_count += 1;
        self.numeric_sum += n;
    }

    pub fn numeric_avg(&self) -> f64 {
        if self.numeric_count == 0 {
            0.0
        } else {
            self.numeric_sum / (self.numeric_count as f64)
        }
    }

    pub fn cardinality(&self) -> usize {
        self.value_counts.len()
    }

    pub fn variable_fraction(&self, c: &SegmentClassifier) -> f64 {
        if self.total == 0 {
            return 0.0;
        }
        let v: usize = self
            .type_counts
            .iter()
            .filter(|(t, _)| c.variable(**t))
            .map(|(_, n)| *n)
            .sum();
        (v as f64) / (self.total as f64)
    }

    pub fn value_fraction(&self, value: &str) -> f64 {
        if self.total == 0 {
            return 0.0;
        }
        let n = *self.value_counts.get(value).unwrap_or(&0);
        (n as f64) / (self.total as f64)
    }

    /// SegmentType with the largest count. Ties broken lexicographically
    /// (by the type's display string) for cross-runtime determinism.
    pub fn dominant_type(&self) -> SegmentType {
        let mut best: Option<(SegmentType, usize)> = None;
        for (&t, &n) in &self.type_counts {
            best = match best {
                None => Some((t, n)),
                Some((bt, bn)) => {
                    if n > bn || (n == bn && t.as_str() < bt.as_str()) {
                        Some((t, n))
                    } else {
                        Some((bt, bn))
                    }
                }
            };
        }
        best.map(|(t, _)| t).unwrap_or(SegmentType::Literal)
    }
}