1use crate::Result;
2use serde::{Deserialize, Serialize};
3use std::time::Duration;
4
5#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
6pub struct ProviderInfo {
7 pub id: &'static str,
8 pub display_name: &'static str,
9 pub kind: ProviderKind,
10 pub status: ProviderStatus,
11 pub auth: AuthKind,
12 pub base_url: &'static str,
13 pub notes: &'static str,
14}
15
16#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
17#[serde(rename_all = "snake_case")]
18pub enum ProviderKind {
19 Paper,
20 Code,
21 Repo,
22 Metadata,
23 Both,
24}
25
26#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
27#[serde(rename_all = "snake_case")]
28pub enum ProviderStatus {
29 Enabled,
30 Available,
31 Unavailable,
32 Planned,
33}
34
35#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
36#[serde(rename_all = "snake_case")]
37pub enum AuthKind {
38 None,
39 Optional,
40 Required,
41 Paid,
42 Unknown,
43}
44
45#[derive(Debug, Clone, Copy, PartialEq, Eq)]
46pub enum SourceFilter {
47 All,
48 Papers,
49 Repos,
50}
51
52#[derive(Debug, Clone, Copy, PartialEq, Eq)]
53pub enum DiscoverKind {
54 Papers,
55 Repos,
56 All,
57}
58
59#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
60pub struct DiscoveredSource {
61 #[serde(default = "default_discovery_backend")]
62 pub discovery_backend: String,
63 pub name: String,
64 pub url: String,
65 pub kind: String,
66 pub source_type: String,
67 pub auth: String,
68 pub api_available: String,
69 pub docs_url: Option<String>,
70 pub coverage_notes: String,
71 pub integration_difficulty: String,
72 pub confidence: String,
73 pub evidence_links: Vec<String>,
74 pub evidence_snippets: Vec<String>,
75}
76
77pub fn builtin() -> Vec<ProviderInfo> {
78 vec![
79 ProviderInfo {
80 id: "arxiv",
81 display_name: "arXiv",
82 kind: ProviderKind::Paper,
83 status: ProviderStatus::Enabled,
84 auth: AuthKind::None,
85 base_url: "https://export.arxiv.org/api/query",
86 notes: "MVP paper provider",
87 },
88 ProviderInfo {
89 id: "openalex",
90 display_name: "OpenAlex",
91 kind: ProviderKind::Paper,
92 status: ProviderStatus::Enabled,
93 auth: AuthKind::None,
94 base_url: "https://api.openalex.org/works",
95 notes: "Broad scholarly metadata provider; OPENALEX_EMAIL optional",
96 },
97 ProviderInfo {
98 id: "papers-with-code",
99 display_name: "Papers with Code",
100 kind: ProviderKind::Code,
101 status: ProviderStatus::Enabled,
102 auth: AuthKind::None,
103 base_url: "https://paperswithcode.com/api/v1",
104 notes: "MVP code-link signal",
105 },
106 ProviderInfo {
107 id: "github",
108 display_name: "GitHub",
109 kind: ProviderKind::Repo,
110 status: ProviderStatus::Enabled,
111 auth: AuthKind::Optional,
112 base_url: "https://api.github.com",
113 notes: "MVP repository search",
114 },
115 ProviderInfo {
116 id: "gitlab",
117 display_name: "GitLab",
118 kind: ProviderKind::Repo,
119 status: ProviderStatus::Enabled,
120 auth: AuthKind::Optional,
121 base_url: "https://gitlab.com/api/v4",
122 notes: "MVP repository search",
123 },
124 ProviderInfo {
125 id: "codeberg",
126 display_name: "Codeberg",
127 kind: ProviderKind::Repo,
128 status: ProviderStatus::Enabled,
129 auth: AuthKind::Optional,
130 base_url: "https://codeberg.org/api/v1",
131 notes: "Forgejo/Gitea default instance",
132 },
133 ]
134}
135
136pub fn filter(providers: Vec<ProviderInfo>, filter: SourceFilter) -> Vec<ProviderInfo> {
137 providers
138 .into_iter()
139 .filter(|provider| match filter {
140 SourceFilter::All => true,
141 SourceFilter::Papers => matches!(
142 provider.kind,
143 ProviderKind::Paper | ProviderKind::Metadata | ProviderKind::Both
144 ),
145 SourceFilter::Repos => matches!(provider.kind, ProviderKind::Repo),
146 })
147 .collect()
148}
149
150pub async fn health_check(url: &str) -> &'static str {
151 let client = match reqwest::Client::builder()
152 .timeout(Duration::from_secs(5))
153 .build()
154 {
155 Ok(client) => client,
156 Err(_) => return "unreachable",
157 };
158 match client.get(url).send().await {
159 Ok(response) if response.status().is_success() || response.status().is_redirection() => {
160 "reachable"
161 }
162 Ok(_) => "unreachable",
163 Err(_) => "unreachable",
164 }
165}
166
167pub fn discovery_queries(kind: DiscoverKind) -> Vec<&'static str> {
168 match kind {
169 DiscoverKind::Papers => vec![
170 "scholarly metadata APIs",
171 "research paper APIs",
172 "preprint APIs",
173 "open-access paper indexes",
174 "domain-specific literature databases",
175 "conference open proceedings APIs",
176 ],
177 DiscoverKind::Repos => vec![
178 "public code forges",
179 "Git hosting APIs",
180 "Forgejo Gitea instances",
181 "GitLab instances",
182 "institutional research code hosts",
183 "package registries research software",
184 ],
185 DiscoverKind::All => {
186 let mut queries = discovery_queries(DiscoverKind::Papers);
187 queries.extend(discovery_queries(DiscoverKind::Repos));
188 queries
189 }
190 }
191}
192
193pub async fn discover(kind: DiscoverKind) -> Result<Vec<DiscoveredSource>> {
194 let client = reqwest::Client::builder()
195 .user_agent("paper-gap")
196 .timeout(Duration::from_secs(10))
197 .build()?;
198 let brave_key = std::env::var("BRAVE_SEARCH_API_KEY").ok();
199 let mut sources = Vec::new();
200
201 for query in discovery_queries(kind) {
202 let discovered = if let Some(key) = brave_key.as_deref() {
203 match discover_brave(&client, kind, query, key).await {
204 Ok(found) => found,
205 Err(error) => {
206 eprintln!(
207 "paper-gap: Brave discovery failed for {query}: {error}; falling back to DuckDuckGo"
208 );
209 discover_duckduckgo(&client, kind, query)
210 .await
211 .unwrap_or_else(|error| {
212 eprintln!(
213 "paper-gap: DuckDuckGo discovery failed for {query}: {error}"
214 );
215 Vec::new()
216 })
217 }
218 }
219 } else {
220 discover_duckduckgo(&client, kind, query)
221 .await
222 .unwrap_or_else(|error| {
223 eprintln!("paper-gap: DuckDuckGo discovery failed for {query}: {error}");
224 Vec::new()
225 })
226 };
227 sources.extend(discovered);
228 }
229
230 Ok(deduplicate_discovered_sources(sources))
231}
232
233async fn discover_duckduckgo(
234 client: &reqwest::Client,
235 kind: DiscoverKind,
236 query: &str,
237) -> Result<Vec<DiscoveredSource>> {
238 let url = format!(
239 "https://duckduckgo.com/html/?q={}",
240 urlencoding::encode(query)
241 );
242 let body = client
243 .get(&url)
244 .send()
245 .await?
246 .error_for_status()?
247 .text()
248 .await?;
249 Ok(parse_duckduckgo_html(kind, query, &url, &body))
250}
251
252async fn discover_brave(
253 client: &reqwest::Client,
254 kind: DiscoverKind,
255 query: &str,
256 key: &str,
257) -> Result<Vec<DiscoveredSource>> {
258 let url = format!(
259 "https://api.search.brave.com/res/v1/web/search?q={}&count=5",
260 urlencoding::encode(query)
261 );
262 let body = client
263 .get(&url)
264 .header("X-Subscription-Token", key)
265 .send()
266 .await?
267 .error_for_status()?
268 .text()
269 .await?;
270 parse_brave_json(kind, query, &url, &body)
271}
272
273pub fn parse_brave_json(
274 kind: DiscoverKind,
275 query: &str,
276 search_url: &str,
277 json: &str,
278) -> Result<Vec<DiscoveredSource>> {
279 #[derive(Deserialize)]
280 struct Response {
281 web: Option<Web>,
282 }
283 #[derive(Deserialize)]
284 struct Web {
285 results: Vec<Item>,
286 }
287 #[derive(Deserialize)]
288 struct Item {
289 title: String,
290 url: String,
291 description: Option<String>,
292 }
293
294 let response: Response = serde_json::from_str(json)?;
295 Ok(response
296 .web
297 .into_iter()
298 .flat_map(|web| web.results)
299 .take(5)
300 .map(|item| {
301 discovered_source(
302 "brave_api",
303 kind,
304 query,
305 search_url,
306 item.title,
307 item.url,
308 item.description.unwrap_or_default(),
309 "medium",
310 )
311 })
312 .collect())
313}
314
315pub fn parse_duckduckgo_html(
316 kind: DiscoverKind,
317 query: &str,
318 search_url: &str,
319 html: &str,
320) -> Vec<DiscoveredSource> {
321 html.lines()
322 .filter(|line| line.contains("result__a"))
323 .take(5)
324 .map(|line| {
325 let name = strip_html(line).trim().to_string();
326 let result_url = attribute(line, "href").unwrap_or_else(|| search_url.to_string());
327 discovered_source(
328 "duckduckgo_html",
329 kind,
330 query,
331 search_url,
332 if name.is_empty() { query.into() } else { name },
333 result_url,
334 strip_html(line),
335 "low",
336 )
337 })
338 .collect()
339}
340
341fn discovered_source(
342 backend: &str,
343 kind: DiscoverKind,
344 query: &str,
345 search_url: &str,
346 name: String,
347 url: String,
348 snippet: String,
349 confidence: &str,
350) -> DiscoveredSource {
351 let lower = format!("{} {}", name, url).to_ascii_lowercase();
352 let docs_url = (lower.contains("docs") || lower.contains("api")).then(|| url.clone());
353 DiscoveredSource {
354 discovery_backend: backend.into(),
355 name,
356 url: url.clone(),
357 kind: match kind {
358 DiscoverKind::Papers => "papers",
359 DiscoverKind::Repos => "repos",
360 DiscoverKind::All => "both",
361 }
362 .into(),
363 source_type: classify(query).into(),
364 auth: "unknown".into(),
365 api_available: if docs_url.is_some() { "yes" } else { "unknown" }.into(),
366 docs_url,
367 coverage_notes: format!("Discovered from query: {query}"),
368 integration_difficulty: "unknown".into(),
369 confidence: confidence.into(),
370 evidence_links: vec![url, search_url.into()],
371 evidence_snippets: vec![snippet],
372 }
373}
374
375pub fn deduplicate_discovered_sources(sources: Vec<DiscoveredSource>) -> Vec<DiscoveredSource> {
376 let mut deduplicated = Vec::new();
377 for source in sources {
378 let domain = normalized_domain(&source.url);
379 if domain.is_empty()
380 || !deduplicated
381 .iter()
382 .any(|existing: &DiscoveredSource| normalized_domain(&existing.url) == domain)
383 {
384 deduplicated.push(source);
385 }
386 }
387 deduplicated
388}
389
390fn normalized_domain(url: &str) -> String {
391 let without_scheme = url.split_once("://").map_or(url, |(_, rest)| rest);
392 without_scheme
393 .split('/')
394 .next()
395 .unwrap_or_default()
396 .trim_start_matches("www.")
397 .to_ascii_lowercase()
398}
399
400fn attribute(input: &str, name: &str) -> Option<String> {
401 let marker = format!("{name}=\"");
402 let start = input.find(&marker)? + marker.len();
403 let end = input[start..].find('"')? + start;
404 Some(input[start..end].replace("&", "&"))
405}
406
407fn default_discovery_backend() -> String {
408 "unknown".into()
409}
410
411fn classify(query: &str) -> &'static str {
412 if query.contains("package") {
413 "package_registry"
414 } else if query.contains("forge") || query.contains("Git") || query.contains("GitLab") {
415 "code_forge"
416 } else if query.contains("preprint") {
417 "preprint_server"
418 } else if query.contains("metadata") {
419 "aggregator_api"
420 } else {
421 "unclear"
422 }
423}
424
425fn strip_html(input: &str) -> String {
426 let mut out = String::new();
427 let mut in_tag = false;
428 for ch in input.chars() {
429 match ch {
430 '<' => in_tag = true,
431 '>' => in_tag = false,
432 _ if !in_tag => out.push(ch),
433 _ => {}
434 }
435 }
436 out.replace("&", "&")
437 .replace(""", "\"")
438 .split_whitespace()
439 .collect::<Vec<_>>()
440 .join(" ")
441}