paper_gap/
sources.rs

1use crate::Result;
2use serde::{Deserialize, Serialize};
3use std::time::Duration;
4
5#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
6pub struct ProviderInfo {
7    pub id: &'static str,
8    pub display_name: &'static str,
9    pub kind: ProviderKind,
10    pub status: ProviderStatus,
11    pub auth: AuthKind,
12    pub base_url: &'static str,
13    pub notes: &'static str,
14}
15
16#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
17#[serde(rename_all = "snake_case")]
18pub enum ProviderKind {
19    Paper,
20    Code,
21    Repo,
22    Metadata,
23    Both,
24}
25
26#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
27#[serde(rename_all = "snake_case")]
28pub enum ProviderStatus {
29    Enabled,
30    Available,
31    Unavailable,
32    Planned,
33}
34
35#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
36#[serde(rename_all = "snake_case")]
37pub enum AuthKind {
38    None,
39    Optional,
40    Required,
41    Paid,
42    Unknown,
43}
44
45#[derive(Debug, Clone, Copy, PartialEq, Eq)]
46pub enum SourceFilter {
47    All,
48    Papers,
49    Repos,
50}
51
52#[derive(Debug, Clone, Copy, PartialEq, Eq)]
53pub enum DiscoverKind {
54    Papers,
55    Repos,
56    All,
57}
58
59#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
60pub struct DiscoveredSource {
61    #[serde(default = "default_discovery_backend")]
62    pub discovery_backend: String,
63    pub name: String,
64    pub url: String,
65    pub kind: String,
66    pub source_type: String,
67    pub auth: String,
68    pub api_available: String,
69    pub docs_url: Option<String>,
70    pub coverage_notes: String,
71    pub integration_difficulty: String,
72    pub confidence: String,
73    pub evidence_links: Vec<String>,
74    pub evidence_snippets: Vec<String>,
75}
76
77pub fn builtin() -> Vec<ProviderInfo> {
78    vec![
79        ProviderInfo {
80            id: "arxiv",
81            display_name: "arXiv",
82            kind: ProviderKind::Paper,
83            status: ProviderStatus::Enabled,
84            auth: AuthKind::None,
85            base_url: "https://export.arxiv.org/api/query",
86            notes: "MVP paper provider",
87        },
88        ProviderInfo {
89            id: "papers-with-code",
90            display_name: "Papers with Code",
91            kind: ProviderKind::Code,
92            status: ProviderStatus::Enabled,
93            auth: AuthKind::None,
94            base_url: "https://paperswithcode.com/api/v1",
95            notes: "MVP code-link signal",
96        },
97        ProviderInfo {
98            id: "github",
99            display_name: "GitHub",
100            kind: ProviderKind::Repo,
101            status: ProviderStatus::Enabled,
102            auth: AuthKind::Optional,
103            base_url: "https://api.github.com",
104            notes: "MVP repository search",
105        },
106        ProviderInfo {
107            id: "gitlab",
108            display_name: "GitLab",
109            kind: ProviderKind::Repo,
110            status: ProviderStatus::Enabled,
111            auth: AuthKind::Optional,
112            base_url: "https://gitlab.com/api/v4",
113            notes: "MVP repository search",
114        },
115        ProviderInfo {
116            id: "codeberg",
117            display_name: "Codeberg",
118            kind: ProviderKind::Repo,
119            status: ProviderStatus::Enabled,
120            auth: AuthKind::Optional,
121            base_url: "https://codeberg.org/api/v1",
122            notes: "Forgejo/Gitea default instance",
123        },
124    ]
125}
126
127pub fn filter(providers: Vec<ProviderInfo>, filter: SourceFilter) -> Vec<ProviderInfo> {
128    providers
129        .into_iter()
130        .filter(|provider| match filter {
131            SourceFilter::All => true,
132            SourceFilter::Papers => matches!(
133                provider.kind,
134                ProviderKind::Paper | ProviderKind::Metadata | ProviderKind::Both
135            ),
136            SourceFilter::Repos => matches!(provider.kind, ProviderKind::Repo),
137        })
138        .collect()
139}
140
141pub async fn health_check(url: &str) -> &'static str {
142    let client = match reqwest::Client::builder()
143        .timeout(Duration::from_secs(5))
144        .build()
145    {
146        Ok(client) => client,
147        Err(_) => return "unreachable",
148    };
149    match client.get(url).send().await {
150        Ok(response) if response.status().is_success() || response.status().is_redirection() => {
151            "reachable"
152        }
153        Ok(_) => "unreachable",
154        Err(_) => "unreachable",
155    }
156}
157
158pub fn discovery_queries(kind: DiscoverKind) -> Vec<&'static str> {
159    match kind {
160        DiscoverKind::Papers => vec![
161            "scholarly metadata APIs",
162            "research paper APIs",
163            "preprint APIs",
164            "open-access paper indexes",
165            "domain-specific literature databases",
166            "conference open proceedings APIs",
167        ],
168        DiscoverKind::Repos => vec![
169            "public code forges",
170            "Git hosting APIs",
171            "Forgejo Gitea instances",
172            "GitLab instances",
173            "institutional research code hosts",
174            "package registries research software",
175        ],
176        DiscoverKind::All => {
177            let mut queries = discovery_queries(DiscoverKind::Papers);
178            queries.extend(discovery_queries(DiscoverKind::Repos));
179            queries
180        }
181    }
182}
183
184pub async fn discover(kind: DiscoverKind) -> Result<Vec<DiscoveredSource>> {
185    let client = reqwest::Client::builder()
186        .user_agent("paper-gap")
187        .timeout(Duration::from_secs(10))
188        .build()?;
189    let brave_key = std::env::var("BRAVE_SEARCH_API_KEY").ok();
190    let mut sources = Vec::new();
191
192    for query in discovery_queries(kind) {
193        let discovered = if let Some(key) = brave_key.as_deref() {
194            match discover_brave(&client, kind, query, key).await {
195                Ok(found) => found,
196                Err(error) => {
197                    eprintln!(
198                        "paper-gap: Brave discovery failed for {query}: {error}; falling back to DuckDuckGo"
199                    );
200                    discover_duckduckgo(&client, kind, query)
201                        .await
202                        .unwrap_or_else(|error| {
203                            eprintln!(
204                                "paper-gap: DuckDuckGo discovery failed for {query}: {error}"
205                            );
206                            Vec::new()
207                        })
208                }
209            }
210        } else {
211            discover_duckduckgo(&client, kind, query)
212                .await
213                .unwrap_or_else(|error| {
214                    eprintln!("paper-gap: DuckDuckGo discovery failed for {query}: {error}");
215                    Vec::new()
216                })
217        };
218        sources.extend(discovered);
219    }
220
221    Ok(deduplicate_discovered_sources(sources))
222}
223
224async fn discover_duckduckgo(
225    client: &reqwest::Client,
226    kind: DiscoverKind,
227    query: &str,
228) -> Result<Vec<DiscoveredSource>> {
229    let url = format!(
230        "https://duckduckgo.com/html/?q={}",
231        urlencoding::encode(query)
232    );
233    let body = client
234        .get(&url)
235        .send()
236        .await?
237        .error_for_status()?
238        .text()
239        .await?;
240    Ok(parse_duckduckgo_html(kind, query, &url, &body))
241}
242
243async fn discover_brave(
244    client: &reqwest::Client,
245    kind: DiscoverKind,
246    query: &str,
247    key: &str,
248) -> Result<Vec<DiscoveredSource>> {
249    let url = format!(
250        "https://api.search.brave.com/res/v1/web/search?q={}&count=5",
251        urlencoding::encode(query)
252    );
253    let body = client
254        .get(&url)
255        .header("X-Subscription-Token", key)
256        .send()
257        .await?
258        .error_for_status()?
259        .text()
260        .await?;
261    parse_brave_json(kind, query, &url, &body)
262}
263
264pub fn parse_brave_json(
265    kind: DiscoverKind,
266    query: &str,
267    search_url: &str,
268    json: &str,
269) -> Result<Vec<DiscoveredSource>> {
270    #[derive(Deserialize)]
271    struct Response {
272        web: Option<Web>,
273    }
274    #[derive(Deserialize)]
275    struct Web {
276        results: Vec<Item>,
277    }
278    #[derive(Deserialize)]
279    struct Item {
280        title: String,
281        url: String,
282        description: Option<String>,
283    }
284
285    let response: Response = serde_json::from_str(json)?;
286    Ok(response
287        .web
288        .into_iter()
289        .flat_map(|web| web.results)
290        .take(5)
291        .map(|item| {
292            discovered_source(
293                "brave_api",
294                kind,
295                query,
296                search_url,
297                item.title,
298                item.url,
299                item.description.unwrap_or_default(),
300                "medium",
301            )
302        })
303        .collect())
304}
305
306pub fn parse_duckduckgo_html(
307    kind: DiscoverKind,
308    query: &str,
309    search_url: &str,
310    html: &str,
311) -> Vec<DiscoveredSource> {
312    html.lines()
313        .filter(|line| line.contains("result__a"))
314        .take(5)
315        .map(|line| {
316            let name = strip_html(line).trim().to_string();
317            let result_url = attribute(line, "href").unwrap_or_else(|| search_url.to_string());
318            discovered_source(
319                "duckduckgo_html",
320                kind,
321                query,
322                search_url,
323                if name.is_empty() { query.into() } else { name },
324                result_url,
325                strip_html(line),
326                "low",
327            )
328        })
329        .collect()
330}
331
332fn discovered_source(
333    backend: &str,
334    kind: DiscoverKind,
335    query: &str,
336    search_url: &str,
337    name: String,
338    url: String,
339    snippet: String,
340    confidence: &str,
341) -> DiscoveredSource {
342    let lower = format!("{} {}", name, url).to_ascii_lowercase();
343    let docs_url = (lower.contains("docs") || lower.contains("api")).then(|| url.clone());
344    DiscoveredSource {
345        discovery_backend: backend.into(),
346        name,
347        url: url.clone(),
348        kind: match kind {
349            DiscoverKind::Papers => "papers",
350            DiscoverKind::Repos => "repos",
351            DiscoverKind::All => "both",
352        }
353        .into(),
354        source_type: classify(query).into(),
355        auth: "unknown".into(),
356        api_available: if docs_url.is_some() { "yes" } else { "unknown" }.into(),
357        docs_url,
358        coverage_notes: format!("Discovered from query: {query}"),
359        integration_difficulty: "unknown".into(),
360        confidence: confidence.into(),
361        evidence_links: vec![url, search_url.into()],
362        evidence_snippets: vec![snippet],
363    }
364}
365
366pub fn deduplicate_discovered_sources(sources: Vec<DiscoveredSource>) -> Vec<DiscoveredSource> {
367    let mut deduplicated = Vec::new();
368    for source in sources {
369        let domain = normalized_domain(&source.url);
370        if domain.is_empty()
371            || !deduplicated
372                .iter()
373                .any(|existing: &DiscoveredSource| normalized_domain(&existing.url) == domain)
374        {
375            deduplicated.push(source);
376        }
377    }
378    deduplicated
379}
380
381fn normalized_domain(url: &str) -> String {
382    let without_scheme = url.split_once("://").map_or(url, |(_, rest)| rest);
383    without_scheme
384        .split('/')
385        .next()
386        .unwrap_or_default()
387        .trim_start_matches("www.")
388        .to_ascii_lowercase()
389}
390
391fn attribute(input: &str, name: &str) -> Option<String> {
392    let marker = format!("{name}=\"");
393    let start = input.find(&marker)? + marker.len();
394    let end = input[start..].find('"')? + start;
395    Some(input[start..end].replace("&amp;", "&"))
396}
397
398fn default_discovery_backend() -> String {
399    "unknown".into()
400}
401
402fn classify(query: &str) -> &'static str {
403    if query.contains("package") {
404        "package_registry"
405    } else if query.contains("forge") || query.contains("Git") || query.contains("GitLab") {
406        "code_forge"
407    } else if query.contains("preprint") {
408        "preprint_server"
409    } else if query.contains("metadata") {
410        "aggregator_api"
411    } else {
412        "unclear"
413    }
414}
415
416fn strip_html(input: &str) -> String {
417    let mut out = String::new();
418    let mut in_tag = false;
419    for ch in input.chars() {
420        match ch {
421            '<' => in_tag = true,
422            '>' => in_tag = false,
423            _ if !in_tag => out.push(ch),
424            _ => {}
425        }
426    }
427    out.replace("&amp;", "&")
428        .replace("&quot;", "\"")
429        .split_whitespace()
430        .collect::<Vec<_>>()
431        .join(" ")
432}
paper_gap/sources.rs

paper_gap/
sources.rs
