patent 0.2.0

A prior-art search for your code ideas — has this dev tool already been shipped?
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

//! PyPI source.
//!
//! PyPI has no public search API (the XML-RPC endpoint was disabled), so this
//! scrapes the `https://pypi.org/search/?q=` results page with CSS selectors.
//! Brittle by nature — if the markup changes the parse yields nothing, which is
//! treated like any empty result (and the run is never blocked on it).

use scraper::{Html, Selector};

use super::SourceAdapter;
use crate::model::{Match, Query, Source};
use crate::{Error, Result};

const DEFAULT_BASE_URL: &str = "https://pypi.org";
const USER_AGENT: &str = concat!("patent/", env!("CARGO_PKG_VERSION"), " (prior-art search)");

/// Searches PyPI (scrape-based; see module note).
#[derive(Debug, Clone)]
pub struct PyPI {
    client: reqwest::Client,
    base_url: String,
}

impl PyPI {
    /// Construct against the live PyPI site.
    pub fn new(client: reqwest::Client) -> Self {
        Self::with_base_url(client, DEFAULT_BASE_URL.to_string())
    }

    /// Construct against an arbitrary base URL (used by tests).
    pub fn with_base_url(client: reqwest::Client, base_url: String) -> Self {
        Self { client, base_url }
    }
}

#[async_trait::async_trait]
impl SourceAdapter for PyPI {
    fn id(&self) -> Source {
        Source::PyPI
    }

    async fn search(&self, query: &Query) -> Result<Vec<Match>> {
        let url = format!("{}/search/", self.base_url);
        let q = query.keywords.join(" ");

        let html = self
            .client
            .get(&url)
            .header(reqwest::header::USER_AGENT, USER_AGENT)
            .query(&[("q", q.as_str())])
            .send()
            .await?
            .error_for_status()?
            .text()
            .await?;

        parse_search_html(&html, &self.base_url)
    }
}

/// Parse a PyPI search results page into matches. A package with no name is
/// skipped; a missing description becomes empty.
fn parse_search_html(html: &str, base_url: &str) -> Result<Vec<Match>> {
    let snippet = Selector::parse("a.package-snippet")
        .map_err(|e| Error::Parse(format!("bad selector: {e}")))?;
    let name = Selector::parse(".package-snippet__name")
        .map_err(|e| Error::Parse(format!("bad selector: {e}")))?;
    let description = Selector::parse(".package-snippet__description")
        .map_err(|e| Error::Parse(format!("bad selector: {e}")))?;

    let document = Html::parse_document(html);
    let mut matches = Vec::new();

    for element in document.select(&snippet) {
        let Some(name_text) = element
            .select(&name)
            .next()
            .map(|n| n.text().collect::<String>().trim().to_string())
            .filter(|s| !s.is_empty())
        else {
            continue;
        };

        let description_text = element
            .select(&description)
            .next()
            .map(|d| d.text().collect::<String>().trim().to_string())
            .unwrap_or_default();

        let href = element.value().attr("href").unwrap_or("");
        let url = if href.starts_with("http") {
            href.to_string()
        } else {
            format!("{base_url}{href}")
        };

        matches.push(Match {
            name: name_text,
            source: Source::PyPI,
            url,
            description: description_text,
            popularity: None,
            similarity: 0.0,
        });
    }

    Ok(matches)
}