paper_gap/
sources.rs

1use crate::Result;
2use serde::{Deserialize, Serialize};
3use std::time::Duration;
4
5#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
6pub struct ProviderInfo {
7    pub id: &'static str,
8    pub display_name: &'static str,
9    pub kind: ProviderKind,
10    pub status: ProviderStatus,
11    pub auth: AuthKind,
12    pub base_url: &'static str,
13    pub notes: &'static str,
14}
15
16#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
17#[serde(rename_all = "snake_case")]
18pub enum ProviderKind {
19    Paper,
20    Code,
21    Repo,
22    Metadata,
23    Both,
24}
25
26#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
27#[serde(rename_all = "snake_case")]
28pub enum ProviderStatus {
29    Enabled,
30    Available,
31    Unavailable,
32    Planned,
33}
34
35#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
36#[serde(rename_all = "snake_case")]
37pub enum AuthKind {
38    None,
39    Optional,
40    Required,
41    Paid,
42    Unknown,
43}
44
45#[derive(Debug, Clone, Copy, PartialEq, Eq)]
46pub enum SourceFilter {
47    All,
48    Papers,
49    Repos,
50}
51
52#[derive(Debug, Clone, Copy, PartialEq, Eq)]
53pub enum DiscoverKind {
54    Papers,
55    Repos,
56    All,
57}
58
59#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
60pub struct DiscoveredSource {
61    pub name: String,
62    pub url: String,
63    pub kind: String,
64    pub source_type: String,
65    pub auth: String,
66    pub api_available: String,
67    pub docs_url: Option<String>,
68    pub coverage_notes: String,
69    pub integration_difficulty: String,
70    pub confidence: String,
71    pub evidence_links: Vec<String>,
72    pub evidence_snippets: Vec<String>,
73}
74
75pub fn builtin() -> Vec<ProviderInfo> {
76    vec![
77        ProviderInfo {
78            id: "arxiv",
79            display_name: "arXiv",
80            kind: ProviderKind::Paper,
81            status: ProviderStatus::Enabled,
82            auth: AuthKind::None,
83            base_url: "https://export.arxiv.org/api/query",
84            notes: "MVP paper provider",
85        },
86        ProviderInfo {
87            id: "papers-with-code",
88            display_name: "Papers with Code",
89            kind: ProviderKind::Code,
90            status: ProviderStatus::Enabled,
91            auth: AuthKind::None,
92            base_url: "https://paperswithcode.com/api/v1",
93            notes: "MVP code-link signal",
94        },
95        ProviderInfo {
96            id: "github",
97            display_name: "GitHub",
98            kind: ProviderKind::Repo,
99            status: ProviderStatus::Enabled,
100            auth: AuthKind::Optional,
101            base_url: "https://api.github.com",
102            notes: "MVP repository search",
103        },
104        ProviderInfo {
105            id: "gitlab",
106            display_name: "GitLab",
107            kind: ProviderKind::Repo,
108            status: ProviderStatus::Enabled,
109            auth: AuthKind::Optional,
110            base_url: "https://gitlab.com/api/v4",
111            notes: "MVP repository search",
112        },
113        ProviderInfo {
114            id: "codeberg",
115            display_name: "Codeberg",
116            kind: ProviderKind::Repo,
117            status: ProviderStatus::Enabled,
118            auth: AuthKind::Optional,
119            base_url: "https://codeberg.org/api/v1",
120            notes: "Forgejo/Gitea default instance",
121        },
122    ]
123}
124
125pub fn filter(providers: Vec<ProviderInfo>, filter: SourceFilter) -> Vec<ProviderInfo> {
126    providers
127        .into_iter()
128        .filter(|provider| match filter {
129            SourceFilter::All => true,
130            SourceFilter::Papers => matches!(
131                provider.kind,
132                ProviderKind::Paper | ProviderKind::Metadata | ProviderKind::Both
133            ),
134            SourceFilter::Repos => matches!(provider.kind, ProviderKind::Repo),
135        })
136        .collect()
137}
138
139pub async fn health_check(url: &str) -> &'static str {
140    let client = match reqwest::Client::builder()
141        .timeout(Duration::from_secs(5))
142        .build()
143    {
144        Ok(client) => client,
145        Err(_) => return "unreachable",
146    };
147    match client.get(url).send().await {
148        Ok(response) if response.status().is_success() || response.status().is_redirection() => {
149            "reachable"
150        }
151        Ok(_) => "unreachable",
152        Err(_) => "unreachable",
153    }
154}
155
156pub fn discovery_queries(kind: DiscoverKind) -> Vec<&'static str> {
157    match kind {
158        DiscoverKind::Papers => vec![
159            "scholarly metadata APIs",
160            "research paper APIs",
161            "preprint APIs",
162            "open-access paper indexes",
163            "domain-specific literature databases",
164            "conference open proceedings APIs",
165        ],
166        DiscoverKind::Repos => vec![
167            "public code forges",
168            "Git hosting APIs",
169            "Forgejo Gitea instances",
170            "GitLab instances",
171            "institutional research code hosts",
172            "package registries research software",
173        ],
174        DiscoverKind::All => {
175            let mut queries = discovery_queries(DiscoverKind::Papers);
176            queries.extend(discovery_queries(DiscoverKind::Repos));
177            queries
178        }
179    }
180}
181
182pub async fn discover(kind: DiscoverKind) -> Result<Vec<DiscoveredSource>> {
183    // ponytail: DuckDuckGo HTML is brittle; replace with a real search API if discovery becomes product-critical.
184    let client = reqwest::Client::builder()
185        .user_agent("paper-gap")
186        .timeout(Duration::from_secs(10))
187        .build()?;
188    let mut sources = Vec::new();
189    for query in discovery_queries(kind) {
190        let url = format!(
191            "https://duckduckgo.com/html/?q={}",
192            urlencoding::encode(query)
193        );
194        let body = client.get(&url).send().await?.text().await?;
195        sources.extend(parse_duckduckgo_html(kind, query, &url, &body));
196    }
197    Ok(sources)
198}
199
200pub fn parse_duckduckgo_html(
201    kind: DiscoverKind,
202    query: &str,
203    search_url: &str,
204    html: &str,
205) -> Vec<DiscoveredSource> {
206    html.lines()
207        .filter(|line| line.contains("result__a"))
208        .take(5)
209        .map(|line| {
210            let name = strip_html(line).trim().to_string();
211            DiscoveredSource {
212                name: if name.is_empty() { query.into() } else { name },
213                url: search_url.into(),
214                kind: match kind {
215                    DiscoverKind::Papers => "papers",
216                    DiscoverKind::Repos => "repos",
217                    DiscoverKind::All => "both",
218                }
219                .into(),
220                source_type: classify(query).into(),
221                auth: "unknown".into(),
222                api_available: "unknown".into(),
223                docs_url: None,
224                coverage_notes: format!("Discovered from DuckDuckGo query: {query}"),
225                integration_difficulty: "unknown".into(),
226                confidence: "low".into(),
227                evidence_links: vec![search_url.into()],
228                evidence_snippets: vec![strip_html(line)],
229            }
230        })
231        .collect()
232}
233
234fn classify(query: &str) -> &'static str {
235    if query.contains("package") {
236        "package_registry"
237    } else if query.contains("forge") || query.contains("Git") || query.contains("GitLab") {
238        "code_forge"
239    } else if query.contains("preprint") {
240        "preprint_server"
241    } else if query.contains("metadata") {
242        "aggregator_api"
243    } else {
244        "unclear"
245    }
246}
247
248fn strip_html(input: &str) -> String {
249    let mut out = String::new();
250    let mut in_tag = false;
251    for ch in input.chars() {
252        match ch {
253            '<' => in_tag = true,
254            '>' => in_tag = false,
255            _ if !in_tag => out.push(ch),
256            _ => {}
257        }
258    }
259    out.replace("&amp;", "&")
260        .replace("&quot;", "\"")
261        .split_whitespace()
262        .collect::<Vec<_>>()
263        .join(" ")
264}
paper_gap/sources.rs

paper_gap/
sources.rs
