Struct SemanticVectorEncoder

Source

pub struct SemanticVectorEncoder;

Expand description

Zero-dependency 384-dimensional dense semantic subword vector encoder.

Implementations§

Source §

impl SemanticVectorEncoder

Source

pub fn new() -> Self

Examples found in repository ?

examples/parity.rs (line 8)

4fn main() {
5    let mut buffer = String::new();
6    io::stdin().read_to_string(&mut buffer).expect("Failed to read stdin");
7
8    let encoder = SemanticVectorEncoder::new();
9    let lines: Vec<&str> = buffer.lines().collect();
10
11    print!("[");
12    for (i, line) in lines.iter().enumerate() {
13        let vec = encoder.encode(line);
14        print!("[");
15        for (j, val) in vec.iter().enumerate() {
16            print!("{:.8}", val);
17            if j + 1 < vec.len() {
18                print!(",");
19            }
20        }
21        print!("]");
22        if i + 1 < lines.len() {
23            print!(",");
24        }
25    }
26    println!("]");
27}

More examples

Hide additional examples

examples/bench.rs (line 11)

5fn main() {
6    let args: Vec<String> = env::args().collect();
7    let iters: usize = args.get(1).and_then(|s| s.parse().ok()).unwrap_or(5000);
8    let sample = "Urgent: Suspicious activity on your account. Click here to cancel wire #8129!";
9
10    let rx = Reflex::new();
11    let encoder = SemanticVectorEncoder::new();
12    let suite = GuardrailSuite::new();
13
14    // 1. Encode
15    let t0 = Instant::now();
16    for _ in 0..iters {
17        let _ = encoder.encode(sample);
18    }
19    let t_enc = t0.elapsed().as_secs_f64();
20
21    // 2. Noul
22    let t0 = Instant::now();
23    for _ in 0..iters {
24        let _ = rx.noul("Is this a security threat or phishing scam?", sample);
25    }
26    let t_noul = t0.elapsed().as_secs_f64();
27
28    // 3. Guardrail
29    let t0 = Instant::now();
30    for _ in 0..iters {
31        let _ = suite.evaluate(sample);
32    }
33    let t_guard = t0.elapsed().as_secs_f64();
34
35    let encode_us = (t_enc / iters as f64) * 1_000_000.0;
36    let noul_us = (t_noul / iters as f64) * 1_000_000.0;
37    let guard_us = (t_guard / iters as f64) * 1_000_000.0;
38    let throughput_ops = iters as f64 / t_noul;
39
40    println!(
41        r#"{{"encode_us": {:.2}, "noul_us": {:.2}, "guard_us": {:.2}, "throughput_ops": {:.2}}}"#,
42        encode_us, noul_us, guard_us, throughput_ops
43    );
44}

Source

pub fn encode(&self, text: &str) -> [f32; 384]

Encodes arbitrary text into an L2-normalized 384-dimensional vector.

Examples found in repository ?

examples/parity.rs (line 13)

4fn main() {
5    let mut buffer = String::new();
6    io::stdin().read_to_string(&mut buffer).expect("Failed to read stdin");
7
8    let encoder = SemanticVectorEncoder::new();
9    let lines: Vec<&str> = buffer.lines().collect();
10
11    print!("[");
12    for (i, line) in lines.iter().enumerate() {
13        let vec = encoder.encode(line);
14        print!("[");
15        for (j, val) in vec.iter().enumerate() {
16            print!("{:.8}", val);
17            if j + 1 < vec.len() {
18                print!(",");
19            }
20        }
21        print!("]");
22        if i + 1 < lines.len() {
23            print!(",");
24        }
25    }
26    println!("]");
27}

More examples

Hide additional examples

examples/bench.rs (line 17)

5fn main() {
6    let args: Vec<String> = env::args().collect();
7    let iters: usize = args.get(1).and_then(|s| s.parse().ok()).unwrap_or(5000);
8    let sample = "Urgent: Suspicious activity on your account. Click here to cancel wire #8129!";
9
10    let rx = Reflex::new();
11    let encoder = SemanticVectorEncoder::new();
12    let suite = GuardrailSuite::new();
13
14    // 1. Encode
15    let t0 = Instant::now();
16    for _ in 0..iters {
17        let _ = encoder.encode(sample);
18    }
19    let t_enc = t0.elapsed().as_secs_f64();
20
21    // 2. Noul
22    let t0 = Instant::now();
23    for _ in 0..iters {
24        let _ = rx.noul("Is this a security threat or phishing scam?", sample);
25    }
26    let t_noul = t0.elapsed().as_secs_f64();
27
28    // 3. Guardrail
29    let t0 = Instant::now();
30    for _ in 0..iters {
31        let _ = suite.evaluate(sample);
32    }
33    let t_guard = t0.elapsed().as_secs_f64();
34
35    let encode_us = (t_enc / iters as f64) * 1_000_000.0;
36    let noul_us = (t_noul / iters as f64) * 1_000_000.0;
37    let guard_us = (t_guard / iters as f64) * 1_000_000.0;
38    let throughput_ops = iters as f64 / t_noul;
39
40    println!(
41        r#"{{"encode_us": {:.2}, "noul_us": {:.2}, "guard_us": {:.2}, "throughput_ops": {:.2}}}"#,
42        encode_us, noul_us, guard_us, throughput_ops
43    );
44}