offline-registry 0.4.0

Registry of Offline Protocol packages, skills, workflows, and templates
Documentation
//! Pre-tokenized registry entries for fast keyword search.

use crate::model::Capability;
use crate::search::{sort_hits, tokenize, SearchHit};
use crate::store::Registry;

#[derive(Debug, Clone)]
struct WeightedTokens {
    tokens: Vec<String>,
    weight: f32,
}

#[derive(Debug, Clone)]
struct IndexedEntry {
    kind: &'static str,
    name: String,
    description: String,
    fields: Vec<WeightedTokens>,
}

#[derive(Debug, Clone, Default)]
pub(crate) struct SearchIndex {
    entries: Vec<IndexedEntry>,
}

impl SearchIndex {
    pub fn build(registry: &Registry, capabilities: &[Capability]) -> Self {
        let mut entries = Vec::new();

        for p in registry.packages() {
            entries.push(IndexedEntry {
                kind: "package",
                name: p.name.clone(),
                description: p.description.clone(),
                fields: vec![
                    weighted(&p.name, 3.0),
                    weighted(&p.capabilities.join(" "), 2.5),
                    weighted(&p.keywords.join(" "), 2.0),
                    weighted(&p.description, 1.0),
                ],
            });
            for ex in &p.examples {
                entries.push(IndexedEntry {
                    kind: "example",
                    name: format!("{} ({})", ex.name, p.name),
                    description: ex.description.clone(),
                    fields: vec![weighted(&ex.name, 3.0), weighted(&ex.description, 1.5)],
                });
            }
        }

        for w in registry.workflows() {
            entries.push(IndexedEntry {
                kind: "workflow",
                name: w.name.clone(),
                description: w.description.clone(),
                fields: vec![
                    weighted(&w.name, 3.0),
                    weighted(&w.title, 3.0),
                    weighted(&w.keywords.join(" "), 2.5),
                    weighted(&w.capabilities.join(" "), 2.0),
                    weighted(&w.description, 1.0),
                ],
            });
        }

        for s in registry.skills() {
            entries.push(IndexedEntry {
                kind: "skill",
                name: s.name.clone(),
                description: s.description.clone(),
                fields: vec![
                    weighted(&s.name, 3.0),
                    weighted(&s.title, 3.0),
                    weighted(&s.use_when.join(" "), 2.5),
                    weighted(&s.description, 1.0),
                    weighted(&s.body, 0.2),
                ],
            });
        }

        for c in capabilities {
            let curated = registry
                .capability_terms
                .get(&c.name)
                .map(|t| t.keywords.join(" "))
                .unwrap_or_default();
            // Keywords of the packages that provide the capability, so a new
            // package's capabilities are findable before they are curated.
            let provider_keywords = c
                .packages
                .iter()
                .filter_map(|p| registry.package(p))
                .flat_map(|p| p.keywords.iter().map(String::as_str))
                .collect::<Vec<_>>()
                .join(" ");
            entries.push(IndexedEntry {
                kind: "capability",
                name: c.name.clone(),
                description: format!(
                    "provided by: {}; workflows: {}",
                    c.packages.join(", "),
                    c.workflows.join(", ")
                ),
                fields: vec![
                    weighted(&c.name, 3.0),
                    weighted(&curated, 2.5),
                    weighted(&provider_keywords, 1.0),
                ],
            });
        }

        Self { entries }
    }

    pub fn search(&self, query: &str) -> Vec<SearchHit> {
        let q: Vec<String> = tokenize(query)
            .into_iter()
            .filter(|t| !is_stopword(t))
            .collect();
        if q.is_empty() {
            return Vec::new();
        }

        let mut hits: Vec<SearchHit> = self
            .entries
            .iter()
            .filter_map(|entry| {
                let score = score_indexed(&q, &entry.fields);
                (score > 0.0).then(|| SearchHit {
                    kind: entry.kind.to_string(),
                    name: entry.name.clone(),
                    description: entry.description.clone(),
                    score,
                })
            })
            .collect();
        sort_hits(&mut hits);
        hits
    }
}

/// Function words dropped from queries. Without this, a sentence such as
/// "phones near each other share data over Bluetooth" scores on "of" or "or"
/// as prefixes of "offline" or "order", and on "in" or "on" as prefixes of
/// "invite" or "online". Only words that carry no product meaning are listed:
/// "no", "not", "who" and "all" stay because registry phrases rely on them.
const STOPWORDS: &[&str] = &[
    "an", "and", "are", "as", "at", "be", "between", "by", "can", "do", "does", "each", "for",
    "from", "in", "into", "is", "it", "its", "me", "my", "of", "on", "or", "other", "our", "over",
    "so", "than", "that", "the", "their", "them", "then", "they", "this", "to", "us", "we", "what",
    "with", "without", "you", "your",
];

fn is_stopword(token: &str) -> bool {
    STOPWORDS.contains(&token)
}

fn weighted(text: &str, weight: f32) -> WeightedTokens {
    WeightedTokens {
        tokens: tokenize(text),
        weight,
    }
}

fn score_indexed(query_tokens: &[String], fields: &[WeightedTokens]) -> f32 {
    let mut score = 0.0;
    for token in query_tokens {
        for field in fields {
            if field.tokens.iter().any(|f| f == token) {
                score += field.weight;
            } else if field.tokens.iter().any(|f| f.starts_with(token.as_str())) {
                score += field.weight * 0.5;
            }
        }
    }
    score
}

pub(crate) fn build_capabilities(registry: &Registry) -> Vec<Capability> {
    use std::collections::BTreeMap;

    let mut map: BTreeMap<String, Capability> = BTreeMap::new();
    let entry = |map: &mut BTreeMap<String, Capability>, name: &str| {
        map.entry(name.to_string()).or_insert_with(|| Capability {
            name: name.to_string(),
            packages: Vec::new(),
            workflows: Vec::new(),
            skills: Vec::new(),
        });
    };

    for p in registry.packages() {
        for c in &p.capabilities {
            entry(&mut map, c);
            map.get_mut(c)
                .expect("just inserted")
                .packages
                .push(p.name.clone());
        }
    }
    for w in registry.workflows() {
        for c in &w.capabilities {
            entry(&mut map, c);
            map.get_mut(c)
                .expect("just inserted")
                .workflows
                .push(w.name.clone());
        }
    }
    for s in registry.skills() {
        if let Some(cap) = map.get_mut(&s.name) {
            cap.skills.push(s.name.clone());
        }
    }
    for (name, terms) in &registry.capability_terms {
        if let Some(cap) = map.get_mut(name) {
            for skill in &terms.skills {
                if !cap.skills.contains(skill) {
                    cap.skills.push(skill.clone());
                }
            }
        }
    }
    map.into_values().collect()
}