iriq 0.30.2

IRI/URL extraction, normalization, and shape clustering.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

use crate::corpus::Corpus;
use std::collections::HashMap;

#[derive(Debug, Clone)]
pub struct CrossHostShape {
    pub shape: String,
    pub hosts: Vec<String>,
    pub observation_count: usize,
}

impl CrossHostShape {
    pub fn host_count(&self) -> usize {
        self.hosts.len()
    }
}

/// Lists route shapes (the path-only form, stripped of host) that recur
/// across multiple hosts. Mirrors Go's Corpus.CrossHostShapes.
pub fn cross_host_shapes(cp: &Corpus, min_hosts: usize) -> Vec<CrossHostShape> {
    let min = if min_hosts == 0 { 2 } else { min_hosts };
    let mut by_shape: HashMap<String, (Vec<String>, usize)> = HashMap::new();
    for c in cp.clusters() {
        let entry = by_shape
            .entry(c.shape.clone())
            .or_insert_with(|| (Vec::new(), 0));
        if !entry.0.contains(&c.host) {
            entry.0.push(c.host.clone());
        }
        entry.1 += c.count;
    }
    let mut out: Vec<CrossHostShape> = by_shape
        .into_iter()
        .filter_map(|(shape, (mut hosts, count))| {
            if hosts.len() < min {
                None
            } else {
                hosts.sort();
                Some(CrossHostShape {
                    shape,
                    hosts,
                    observation_count: count,
                })
            }
        })
        .collect();
    out.sort_by(|a, b| {
        b.host_count()
            .cmp(&a.host_count())
            .then(b.observation_count.cmp(&a.observation_count))
            .then(a.shape.cmp(&b.shape))
    });
    out
}