kweepeer 0.1.2

A generic webservice for interactive query expansion, expansion is provided via various modules
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136

use serde::Deserialize;
use std::fs::File;
use std::io::BufReader;
use std::path::PathBuf;
use tracing::debug;

use crate::lexer::Term;
use crate::modules::Module;
use crate::{Error, QueryParams, TermExpansion, TermExpansions};

use finalfusion::prelude::*;
use finalfusion::similarity::WordSimilarity;

#[derive(Debug, Deserialize, Clone)]
pub struct FinalFusionConfig {
    /// Short identifier
    id: String,

    /// Human readable label
    name: String,

    /// Word-embeddings file (FIFU), as generated by finalfrontier
    file: PathBuf,

    /// Nearest Neighbours, number of results to return
    k: usize,
}

impl FinalFusionConfig {
    pub fn new(id: impl Into<String>, name: impl Into<String>, file: impl Into<PathBuf>) -> Self {
        Self {
            id: id.into(),
            name: name.into(),
            file: file.into(),
            k: 10,
        }
    }

    /// Set nearest Neighbours, number of results to return
    pub fn with_k(mut self, k: usize) -> Self {
        self.k = k;
        self
    }

    pub fn id(&self) -> &str {
        self.id.as_str()
    }

    pub fn name(&self) -> &str {
        self.name.as_str()
    }
}

/// A lexical semantic module using word-embeddings and vector comparison
pub struct FinalFusionModule {
    config: FinalFusionConfig,

    /// the Embedding model from FinalFrontier
    model: Option<Embeddings<VocabWrap, StorageViewWrap>>,
}

impl FinalFusionModule {
    pub fn new(config: FinalFusionConfig) -> Self {
        Self {
            config,
            model: None,
        }
    }
}

impl Module for FinalFusionModule {
    fn id(&self) -> &str {
        self.config.id.as_str()
    }

    fn name(&self) -> &str {
        self.config.name.as_str()
    }

    fn kind(&self) -> &'static str {
        "finalfusion"
    }

    fn load(&mut self) -> Result<(), Error> {
        let mut reader = BufReader::new(File::open(self.config.file.as_path()).map_err(|e| {
            Error::LoadError(format!(
                "FinalFusion Module could not open {}: {}",
                self.config.file.as_path().display(),
                e
            ))
        })?);
        let embeddings = Embeddings::read_embeddings(&mut reader)?;
        self.model = Some(embeddings);
        Ok(())
    }

    fn expand_query(
        &self,
        terms: &Vec<Term>,
        params: &QueryParams,
    ) -> Result<TermExpansions, Error> {
        let k = if let Some(param) = params.get(self.id(), "k") {
            param.as_u64().ok_or_else(|| {
                Error::QueryExpandError("invalid value for k (nearest-neighbours) parameter".into())
            })? as usize
        } else {
            self.config.k
        };
        let mut expansions = TermExpansions::new();
        for term in terms {
            debug!("Looking up {}", term.as_str());
            if let Some(model) = self.model.as_ref() {
                let mut termexpansion = TermExpansion::default().with_source(self);

                if let Some(results) = model.word_similarity(term.as_str(), k, None) {
                    for variant in results {
                        termexpansion.add_variant_with_score(
                            variant.word(),
                            variant.cosine_similarity() as f64,
                        );
                    }
                    expansions.insert(term.as_str().to_string(), vec![termexpansion]);
                }
            } else {
                panic!("expand_query() was called before load()!");
            }
        }
        Ok(expansions)
    }
}

impl From<finalfusion::error::Error> for Error {
    fn from(value: finalfusion::error::Error) -> Self {
        Self::LoadError(format!("{}", value))
    }
}