1use crate::arxiv::Paper;
2use crate::{
3 Result,
4 provider::{self, ProviderOutcome},
5};
6use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue, USER_AGENT};
7use serde::{Deserialize, Serialize};
8use std::time::Duration;
9
10#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
11pub struct Repository {
12 pub forge: String,
13 pub url: String,
14 pub owner: String,
15 pub name: String,
16 pub description: Option<String>,
17 pub primary_language: Option<String>,
18 pub stars: Option<u64>,
19 pub forks: Option<u64>,
20 pub license: Option<String>,
21 pub archived: Option<bool>,
22 pub default_branch: Option<String>,
23 pub last_commit_date: Option<String>,
24 pub source_provider: String,
25}
26
27#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
28pub struct RepositoryMatch {
29 pub repository: Repository,
30 pub confidence_score: u32,
31 pub evidence: Vec<MatchEvidence>,
32}
33
34#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
35pub struct MatchEvidence {
36 pub kind: String,
37 pub value: String,
38 pub points: u32,
39}
40
41const MIN_MATCH_CONFIDENCE: u32 = 30;
42
43#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
44pub struct RepositoryRef {
45 pub forge: String,
46 pub owner: String,
47 pub name: String,
48 pub url: String,
49}
50
51pub fn repository_from_url(url: &str) -> Option<Repository> {
52 let normalized = url.trim().trim_end_matches('/').trim_end_matches(".git");
53 let rest = normalized
54 .strip_prefix("https://")
55 .or_else(|| normalized.strip_prefix("http://"))?;
56 let (host, path) = rest.split_once('/')?;
57 let forge = match host.to_ascii_lowercase().as_str() {
58 "github.com" => "github",
59 "gitlab.com" => "gitlab",
60 "codeberg.org" => "codeberg",
61 _ => return None,
62 };
63 let parts: Vec<_> = path.split('/').filter(|part| !part.is_empty()).collect();
64 if parts.len() < 2 || (forge != "gitlab" && parts.len() != 2) {
65 return None;
66 }
67 let name = parts.last()?.to_string();
68 let owner = parts[..parts.len() - 1].join("/");
69 Some(Repository {
70 forge: forge.into(),
71 url: format!("https://{host}/{path}")
72 .trim_end_matches('/')
73 .trim_end_matches(".git")
74 .to_string(),
75 owner,
76 name,
77 description: None,
78 primary_language: None,
79 stars: None,
80 forks: None,
81 license: None,
82 archived: None,
83 default_branch: None,
84 last_commit_date: None,
85 source_provider: "papers-with-code".into(),
86 })
87}
88
89#[derive(Debug, Clone)]
90pub struct RepoSearchQuery {
91 pub terms: Vec<String>,
92 pub limit: usize,
93}
94
95pub fn search_terms(paper: &Paper) -> Vec<String> {
96 let mut terms = vec![format!("\"{}\"", paper.title)];
97 if let Some(arxiv_id) = paper.arxiv_id.as_deref() {
98 terms.push(trim_arxiv_version(arxiv_id).to_string());
99 }
100 if let Some(doi) = paper.doi.as_deref() {
101 terms.push(doi.to_string());
102 }
103 terms.extend(
104 paper
105 .extracted_methods
106 .iter()
107 .filter(|term| is_specific_method_term(term))
108 .cloned(),
109 );
110 dedup_strings(terms)
111}
112
113pub async fn search_for_paper(paper: &Paper, limit: usize) -> Result<Vec<Repository>> {
114 let query = RepoSearchQuery {
115 terms: search_terms(paper).into_iter().take(3).collect(),
117 limit,
118 };
119 let mut repos = Vec::new();
120 repos.extend(github_search(&query).await.unwrap_or_default());
121 repos.extend(gitlab_search(&query).await.unwrap_or_default());
122 repos.extend(
123 forgejo_search("codeberg", "https://codeberg.org/api/v1", &query)
124 .await
125 .unwrap_or_default(),
126 );
127 Ok(filter_repositories(paper, dedup_repositories(repos)))
128}
129
130pub async fn search_for_paper_with_outcome(
131 paper: &Paper,
132 limit: usize,
133) -> (Vec<Repository>, ProviderOutcome) {
134 search_for_paper_with_providers_and_outcome(paper, limit, &["github", "gitlab", "codeberg"])
135 .await
136}
137
138pub async fn search_for_paper_with_providers_and_outcome(
139 paper: &Paper,
140 limit: usize,
141 providers: &[&str],
142) -> (Vec<Repository>, ProviderOutcome) {
143 let query = RepoSearchQuery {
144 terms: search_terms(paper).into_iter().take(3).collect(),
145 limit,
146 };
147 let mut repositories = Vec::new();
148 let mut outcomes = Vec::new();
149
150 for provider in providers {
151 let result = match *provider {
152 "github" => github_search(&query).await,
153 "gitlab" => gitlab_search(&query).await,
154 "codeberg" => forgejo_search("codeberg", "https://codeberg.org/api/v1", &query).await,
155 _ => continue,
156 };
157 match result {
158 Ok(found) => {
159 repositories.extend(found);
160 outcomes.push(ProviderOutcome::success(*provider));
161 }
162 Err(error) => outcomes.push(ProviderOutcome::from_error(*provider, &error)),
163 }
164 }
165
166 (
167 dedup_repositories(repositories),
168 ProviderOutcome::aggregate("repository-search", &outcomes),
169 )
170}
171
172pub fn filter_repositories(paper: &Paper, repositories: Vec<Repository>) -> Vec<Repository> {
173 rank_repositories(paper, repositories, &[])
174 .into_iter()
175 .map(|matched| matched.repository)
176 .collect()
177}
178
179pub fn rank_repositories(
180 paper: &Paper,
181 repositories: Vec<Repository>,
182 official_urls: &[String],
183) -> Vec<RepositoryMatch> {
184 let mut matches: Vec<_> = repositories
185 .into_iter()
186 .filter_map(|repository| {
187 let matched = match_repository(paper, repository, official_urls);
188 (matched.confidence_score >= MIN_MATCH_CONFIDENCE).then_some(matched)
189 })
190 .collect();
191 matches.sort_by_key(|matched| std::cmp::Reverse(matched.confidence_score));
192 matches
193}
194
195pub fn match_repository(
196 paper: &Paper,
197 repository: Repository,
198 official_urls: &[String],
199) -> RepositoryMatch {
200 let mut evidence = Vec::new();
201 let identity_text = format!(
202 "{} {} {}",
203 repository.owner, repository.name, repository.url
204 );
205 let description_text = repository.description.as_deref().unwrap_or_default();
206 let identity = normalize(&identity_text);
207 let description = normalize(description_text);
208 let whole = format!("{identity}{description}");
209
210 if official_urls
211 .iter()
212 .any(|url| normalized_repo_url(url) == normalized_repo_url(&repository.url))
213 {
214 push_evidence(
215 &mut evidence,
216 "official_pwc_link",
217 repository.url.clone(),
218 100,
219 );
220 }
221 if let Some(id) = paper.arxiv_id.as_deref().map(trim_arxiv_version) {
222 if whole.contains(&normalize(id)) {
223 push_evidence(&mut evidence, "exact_arxiv_id", id.into(), 100);
224 }
225 }
226 if let Some(doi) = paper.doi.as_deref() {
227 if whole.contains(&normalize(doi)) {
228 push_evidence(&mut evidence, "exact_doi", doi.into(), 100);
229 }
230 }
231
232 let title = normalize(&paper.title);
233 if !title.is_empty() && whole.contains(&title) {
234 push_evidence(&mut evidence, "exact_title", paper.title.clone(), 60);
235 } else {
236 let title_terms = meaningful_title_terms(&paper.title);
237 let identity_terms: Vec<_> = title_terms
238 .iter()
239 .filter(|term| contains_term(&identity_text, term))
240 .cloned()
241 .collect();
242 let description_terms: Vec<_> = title_terms
243 .iter()
244 .filter(|term| !identity_terms.contains(term) && contains_term(description_text, term))
245 .cloned()
246 .collect();
247 if !identity_terms.is_empty() {
248 push_evidence(
249 &mut evidence,
250 "title_identity_overlap",
251 identity_terms.join(", "),
252 (identity_terms.len() as u32 * 15).min(45),
253 );
254 }
255 if !description_terms.is_empty() {
256 push_evidence(
257 &mut evidence,
258 "description_overlap",
259 description_terms.join(", "),
260 (description_terms.len() as u32 * 10).min(30),
261 );
262 }
263 }
264
265 let methods: Vec<_> = paper
266 .extracted_methods
267 .iter()
268 .filter(|term| {
269 is_specific_method_term(term)
270 && (contains_term(&identity_text, term) || contains_term(description_text, term))
271 })
272 .cloned()
273 .collect();
274 if !methods.is_empty() {
275 push_evidence(
276 &mut evidence,
277 "method_overlap",
278 methods.join(", "),
279 (methods.len() as u32 * 10).min(20),
280 );
281 }
282
283 let confidence_score = evidence
284 .iter()
285 .map(|item| item.points)
286 .sum::<u32>()
287 .min(100);
288 RepositoryMatch {
289 repository,
290 confidence_score,
291 evidence,
292 }
293}
294
295fn push_evidence(evidence: &mut Vec<MatchEvidence>, kind: &str, value: String, points: u32) {
296 evidence.push(MatchEvidence {
297 kind: kind.into(),
298 value,
299 points,
300 });
301}
302
303fn normalized_repo_url(url: &str) -> String {
304 url.trim_end_matches('/')
305 .trim_end_matches(".git")
306 .to_ascii_lowercase()
307}
308
309fn meaningful_title_terms(title: &str) -> Vec<String> {
310 let stop = [
311 "with", "from", "into", "using", "based", "towards", "toward", "paper", "study", "method",
312 "methods", "system", "systems", "program", "programs",
313 ];
314 title
315 .split(|c: char| !c.is_alphanumeric())
316 .map(str::to_ascii_lowercase)
317 .filter(|term| term.len() >= 4 && !stop.contains(&term.as_str()))
318 .collect()
319}
320
321fn is_specific_method_term(term: &str) -> bool {
322 let broad = [
323 "ai", "ml", "rl", "api", "cpu", "gpu", "fft", "pde", "ode", "chc", "nlp", "llm", "cnn",
324 "rnn", "gcn", "sql", "http", "json", "xml",
325 ];
326 let lower = term.to_ascii_lowercase();
327 term.len() > 3 && !broad.contains(&lower.as_str())
328}
329
330fn contains_term(text: &str, term: &str) -> bool {
331 let term: Vec<_> = term
332 .split(|c: char| !c.is_ascii_alphanumeric())
333 .map(normalize)
334 .filter(|part| !part.is_empty())
335 .collect();
336 let text: Vec<_> = text
337 .split(|c: char| !c.is_ascii_alphanumeric())
338 .map(normalize)
339 .filter(|part| !part.is_empty())
340 .collect();
341 !term.is_empty() && text.windows(term.len()).any(|window| window == term)
342}
343
344fn normalize(text: &str) -> String {
345 text.chars()
346 .filter(|c| c.is_ascii_alphanumeric())
347 .flat_map(char::to_lowercase)
348 .collect()
349}
350
351pub fn github_search_url(term: &str, limit: usize) -> String {
352 format!(
353 "https://api.github.com/search/repositories?q={}&per_page={}",
354 urlencoding::encode(term),
355 limit
356 )
357}
358
359pub fn gitlab_search_url(term: &str, limit: usize) -> String {
360 format!(
361 "https://gitlab.com/api/v4/projects?search={}&per_page={}",
362 urlencoding::encode(term),
363 limit
364 )
365}
366
367pub fn forgejo_search_url(base_url: &str, term: &str, limit: usize) -> String {
368 format!(
369 "{}/repos/search?q={}&limit={}",
370 base_url.trim_end_matches('/'),
371 urlencoding::encode(term),
372 limit
373 )
374}
375
376#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
377pub struct RepositoryFile {
378 pub path: String,
379}
380
381pub fn github_tree_url(owner: &str, name: &str, branch: &str) -> String {
382 format!(
383 "https://api.github.com/repos/{}/{}/git/trees/{}?recursive=1",
384 owner,
385 name,
386 urlencoding::encode(branch)
387 )
388}
389
390pub fn gitlab_tree_url(owner: &str, name: &str, branch: &str) -> String {
391 format!(
392 "https://gitlab.com/api/v4/projects/{}/repository/tree?recursive=true&per_page=100&ref={}",
393 urlencoding::encode(&format!("{owner}/{name}")),
394 urlencoding::encode(branch)
395 )
396}
397
398pub fn forgejo_tree_url(base_url: &str, owner: &str, name: &str, branch: &str) -> String {
399 format!(
400 "{}/repos/{}/{}/git/trees/{}?recursive=true",
401 base_url.trim_end_matches('/'),
402 owner,
403 name,
404 urlencoding::encode(branch)
405 )
406}
407
408pub async fn file_paths(repo: &Repository) -> Result<Vec<String>> {
409 let client = client(match repo.forge.as_str() {
410 "github" => Some("GITHUB_TOKEN"),
411 "gitlab" => Some("GITLAB_TOKEN"),
412 _ => None,
413 });
414 let branch = repo.default_branch.as_deref().unwrap_or("main");
415 let url = match repo.forge.as_str() {
416 "github" => github_tree_url(&repo.owner, &repo.name, branch),
417 "gitlab" => gitlab_tree_url(&repo.owner, &repo.name, branch),
418 "codeberg" => forgejo_tree_url(
419 "https://codeberg.org/api/v1",
420 &repo.owner,
421 &repo.name,
422 branch,
423 ),
424 _ => return Ok(Vec::new()),
425 };
426 let response = provider::send_with_retry(client.get(url)).await?;
427 if response.status() == reqwest::StatusCode::NOT_FOUND {
428 return Ok(Vec::new());
429 }
430 let text = response.error_for_status()?.text().await?;
431 match repo.forge.as_str() {
432 "github" => parse_github_tree(&text),
433 "gitlab" => parse_gitlab_tree(&text),
434 _ => parse_forgejo_tree(&text),
435 }
436}
437
438pub fn parse_github_tree(json: &str) -> Result<Vec<String>> {
439 #[derive(Deserialize)]
440 struct Tree {
441 tree: Vec<TreeItem>,
442 }
443 #[derive(Deserialize)]
444 struct TreeItem {
445 path: String,
446 }
447 Ok(serde_json::from_str::<Tree>(json)?
448 .tree
449 .into_iter()
450 .map(|item| item.path)
451 .collect())
452}
453
454pub fn parse_gitlab_tree(json: &str) -> Result<Vec<String>> {
455 #[derive(Deserialize)]
456 struct TreeItem {
457 path: String,
458 }
459 Ok(serde_json::from_str::<Vec<TreeItem>>(json)?
460 .into_iter()
461 .map(|item| item.path)
462 .collect())
463}
464
465pub fn parse_forgejo_tree(json: &str) -> Result<Vec<String>> {
466 parse_github_tree(json)
467}
468
469pub fn github_inspect_url(owner: &str, name: &str) -> String {
470 format!("https://api.github.com/repos/{}/{}", owner, name)
471}
472
473pub fn gitlab_inspect_url(owner: &str, name: &str) -> String {
474 format!(
475 "https://gitlab.com/api/v4/projects/{}",
476 urlencoding::encode(&format!("{owner}/{name}"))
477 )
478}
479
480pub fn forgejo_inspect_url(base_url: &str, owner: &str, name: &str) -> String {
481 format!(
482 "{}/repos/{}/{}",
483 base_url.trim_end_matches('/'),
484 owner,
485 name
486 )
487}
488
489pub async fn github_search(query: &RepoSearchQuery) -> Result<Vec<Repository>> {
490 let client = client(Some("GITHUB_TOKEN"));
491 let mut repos = Vec::new();
492 for term in &query.terms {
493 let response = client.get(github_search_url(term, query.limit));
494 let response = provider::send_with_retry(response).await?;
495 if response.status() == reqwest::StatusCode::FORBIDDEN
496 || response.status() == reqwest::StatusCode::UNAUTHORIZED
497 {
498 continue;
499 }
500 repos.extend(parse_github_search(
501 &response.error_for_status()?.text().await?,
502 )?);
503 }
504 Ok(dedup_repositories(repos))
505}
506
507pub async fn gitlab_search(query: &RepoSearchQuery) -> Result<Vec<Repository>> {
508 let client = client(Some("GITLAB_TOKEN"));
509 let mut repos = Vec::new();
510 for term in &query.terms {
511 let response = client.get(gitlab_search_url(term, query.limit));
512 let response = provider::send_with_retry(response).await?;
513 if response.status() == reqwest::StatusCode::FORBIDDEN
514 || response.status() == reqwest::StatusCode::UNAUTHORIZED
515 {
516 continue;
517 }
518 repos.extend(parse_gitlab_search(
519 &response.error_for_status()?.text().await?,
520 )?);
521 }
522 Ok(dedup_repositories(repos))
523}
524
525pub async fn forgejo_search(
526 provider: &str,
527 base_url: &str,
528 query: &RepoSearchQuery,
529) -> Result<Vec<Repository>> {
530 let client = client(None);
531 let mut repos = Vec::new();
532 for term in &query.terms {
533 let response = client.get(forgejo_search_url(base_url, term, query.limit));
534 let response = provider::send_with_retry(response).await?;
535 if response.status() == reqwest::StatusCode::FORBIDDEN
536 || response.status() == reqwest::StatusCode::UNAUTHORIZED
537 {
538 continue;
539 }
540 repos.extend(parse_forgejo_search(
541 provider,
542 &response.error_for_status()?.text().await?,
543 )?);
544 }
545 Ok(dedup_repositories(repos))
546}
547
548pub async fn inspect(repo: &RepositoryRef) -> Result<Option<Repository>> {
549 let client = client(match repo.forge.as_str() {
550 "github" => Some("GITHUB_TOKEN"),
551 "gitlab" => Some("GITLAB_TOKEN"),
552 _ => None,
553 });
554 let url = match repo.forge.as_str() {
555 "github" => github_inspect_url(&repo.owner, &repo.name),
556 "gitlab" => gitlab_inspect_url(&repo.owner, &repo.name),
557 "codeberg" => forgejo_inspect_url("https://codeberg.org/api/v1", &repo.owner, &repo.name),
558 _ => return Ok(None),
559 };
560 let response = provider::send_with_retry(client.get(url)).await?;
561 if response.status() == reqwest::StatusCode::NOT_FOUND {
562 return Ok(None);
563 }
564 let text = response.error_for_status()?.text().await?;
565 Ok(Some(match repo.forge.as_str() {
566 "github" => parse_github_repo(&text)?,
567 "gitlab" => parse_gitlab_repo(&text)?,
568 _ => parse_forgejo_repo(&repo.forge, &text)?,
569 }))
570}
571
572pub fn parse_github_search(json: &str) -> Result<Vec<Repository>> {
573 #[derive(Deserialize)]
574 struct Search {
575 items: Vec<GithubRepo>,
576 }
577 Ok(serde_json::from_str::<Search>(json)?
578 .items
579 .into_iter()
580 .map(Into::into)
581 .collect())
582}
583
584pub fn parse_github_repo(json: &str) -> Result<Repository> {
585 Ok(serde_json::from_str::<GithubRepo>(json)?.into())
586}
587
588pub fn parse_gitlab_search(json: &str) -> Result<Vec<Repository>> {
589 Ok(serde_json::from_str::<Vec<GitlabRepo>>(json)?
590 .into_iter()
591 .map(Into::into)
592 .collect())
593}
594
595pub fn parse_gitlab_repo(json: &str) -> Result<Repository> {
596 Ok(serde_json::from_str::<GitlabRepo>(json)?.into())
597}
598
599pub fn parse_forgejo_search(provider: &str, json: &str) -> Result<Vec<Repository>> {
600 #[derive(Deserialize)]
601 struct Search {
602 data: Vec<ForgejoRepo>,
603 }
604 Ok(serde_json::from_str::<Search>(json)?
605 .data
606 .into_iter()
607 .map(|r| r.into_repo(provider))
608 .collect())
609}
610
611pub fn parse_forgejo_repo(provider: &str, json: &str) -> Result<Repository> {
612 Ok(serde_json::from_str::<ForgejoRepo>(json)?.into_repo(provider))
613}
614
615fn client(token_env: Option<&str>) -> reqwest::Client {
616 let mut headers = HeaderMap::new();
617 headers.insert(USER_AGENT, HeaderValue::from_static("paper-gap"));
618 if let Some(token_env) = token_env.and_then(|name| std::env::var(name).ok()) {
619 if let Ok(value) = HeaderValue::from_str(&format!("Bearer {token_env}")) {
620 headers.insert(AUTHORIZATION, value);
621 }
622 }
623 reqwest::Client::builder()
624 .default_headers(headers)
625 .timeout(Duration::from_secs(8))
626 .build()
627 .expect("valid reqwest client")
628}
629
630#[derive(Deserialize)]
631struct GithubRepo {
632 html_url: String,
633 name: String,
634 owner: GithubOwner,
635 description: Option<String>,
636 language: Option<String>,
637 stargazers_count: Option<u64>,
638 forks_count: Option<u64>,
639 license: Option<GithubLicense>,
640 archived: Option<bool>,
641 default_branch: Option<String>,
642 pushed_at: Option<String>,
643}
644#[derive(Deserialize)]
645struct GithubOwner {
646 login: String,
647}
648#[derive(Deserialize)]
649struct GithubLicense {
650 spdx_id: Option<String>,
651 name: Option<String>,
652}
653
654impl From<GithubRepo> for Repository {
655 fn from(repo: GithubRepo) -> Self {
656 Repository {
657 forge: "github".into(),
658 url: repo.html_url,
659 owner: repo.owner.login,
660 name: repo.name,
661 description: repo.description,
662 primary_language: repo.language,
663 stars: repo.stargazers_count,
664 forks: repo.forks_count,
665 license: repo.license.and_then(|l| l.spdx_id.or(l.name)),
666 archived: repo.archived,
667 default_branch: repo.default_branch,
668 last_commit_date: repo.pushed_at,
669 source_provider: "github".into(),
670 }
671 }
672}
673
674#[derive(Deserialize)]
675struct GitlabRepo {
676 web_url: String,
677 path: String,
678 path_with_namespace: String,
679 description: Option<String>,
680 star_count: Option<u64>,
681 forks_count: Option<u64>,
682 archived: Option<bool>,
683 default_branch: Option<String>,
684 last_activity_at: Option<String>,
685}
686
687impl From<GitlabRepo> for Repository {
688 fn from(repo: GitlabRepo) -> Self {
689 let owner = repo
690 .path_with_namespace
691 .rsplit_once('/')
692 .map_or("", |(owner, _)| owner)
693 .to_string();
694 Repository {
695 forge: "gitlab".into(),
696 url: repo.web_url,
697 owner,
698 name: repo.path,
699 description: repo.description,
700 primary_language: None,
701 stars: repo.star_count,
702 forks: repo.forks_count,
703 license: None,
704 archived: repo.archived,
705 default_branch: repo.default_branch,
706 last_commit_date: repo.last_activity_at,
707 source_provider: "gitlab".into(),
708 }
709 }
710}
711
712#[derive(Deserialize)]
713struct ForgejoRepo {
714 html_url: String,
715 name: String,
716 owner: ForgejoOwner,
717 description: Option<String>,
718 language: Option<String>,
719 stars_count: Option<u64>,
720 forks_count: Option<u64>,
721 archived: Option<bool>,
722 default_branch: Option<String>,
723 updated_at: Option<String>,
724}
725#[derive(Deserialize)]
726struct ForgejoOwner {
727 login: String,
728}
729
730impl ForgejoRepo {
731 fn into_repo(self, provider: &str) -> Repository {
732 Repository {
733 forge: provider.into(),
734 url: self.html_url,
735 owner: self.owner.login,
736 name: self.name,
737 description: self.description,
738 primary_language: self.language,
739 stars: self.stars_count,
740 forks: self.forks_count,
741 license: None,
742 archived: self.archived,
743 default_branch: self.default_branch,
744 last_commit_date: self.updated_at,
745 source_provider: provider.into(),
746 }
747 }
748}
749
750fn dedup_repositories(repos: Vec<Repository>) -> Vec<Repository> {
751 let mut out = Vec::new();
752 for repo in repos {
753 if !out
754 .iter()
755 .any(|existing: &Repository| existing.url == repo.url)
756 {
757 out.push(repo);
758 }
759 }
760 out
761}
762
763fn dedup_strings(values: Vec<String>) -> Vec<String> {
764 let mut out = Vec::new();
765 for value in values.into_iter().filter(|s| !s.trim().is_empty()) {
766 if !out.iter().any(|existing| existing == &value) {
767 out.push(value);
768 }
769 }
770 out
771}
772
773fn trim_arxiv_version(arxiv_id: &str) -> &str {
774 arxiv_id.rsplit_once('v').map_or(arxiv_id, |(id, version)| {
775 if version.chars().all(|c| c.is_ascii_digit()) {
776 id
777 } else {
778 arxiv_id
779 }
780 })
781}