use chrono::NaiveDate;
use clap::{Args, Parser, Subcommand, ValueEnum};
use paper_gap::{
Result, analyse, arxiv, err, openalex, paper, provider, pwc, report, repos, sources,
};
use std::fs;
use std::io::{self, Read};
use std::path::PathBuf;
#[derive(Parser)]
#[command(
name = "paper-gap",
about = "Find research papers with weak or missing public code"
)]
struct Cli {
#[command(subcommand)]
command: Command,
}
#[derive(Subcommand)]
enum Command {
Scan(ScanArgs),
Inspect(InspectArgs),
Report(ReportArgs),
Sources(SourcesArgs),
}
#[derive(Args)]
struct ScanArgs {
#[arg(long)]
query: Option<String>,
#[arg(long)]
category: Option<String>,
#[arg(long)]
since: Option<String>,
#[arg(long)]
from: Option<NaiveDate>,
#[arg(long)]
to: Option<NaiveDate>,
#[arg(long, value_enum, default_value_t = PaperSource::All)]
paper_source: PaperSource,
#[arg(long, value_enum, default_value_t = RepoSource::All)]
repo_source: RepoSource,
#[arg(long, default_value_t = 25)]
limit: usize,
#[arg(long, value_enum, default_value_t = OutputFormat::Markdown)]
format: OutputFormat,
#[arg(long)]
output: Option<PathBuf>,
}
#[derive(Args)]
struct InspectArgs {
#[arg(long)]
arxiv: String,
#[arg(long, value_enum, default_value_t = RepoSource::All)]
repo_source: RepoSource,
#[arg(long, value_enum, default_value_t = OutputFormat::Markdown)]
format: OutputFormat,
#[arg(long)]
output: Option<PathBuf>,
}
#[derive(Args)]
struct ReportArgs {
#[arg(long, default_value = "-")]
input: String,
#[arg(long, value_enum, default_value_t = OutputFormat::Markdown)]
format: OutputFormat,
#[arg(long)]
output: Option<PathBuf>,
}
#[derive(Args)]
struct SourcesArgs {
#[command(subcommand)]
command: Option<SourcesCommand>,
#[arg(long)]
papers: bool,
#[arg(long)]
repos: bool,
#[arg(long)]
check: bool,
}
#[derive(Subcommand)]
enum SourcesCommand {
Discover(DiscoverArgs),
}
#[derive(Args)]
struct DiscoverArgs {
#[arg(long, value_enum, default_value_t = DiscoverKind::All)]
kind: DiscoverKind,
}
#[derive(Clone, ValueEnum)]
enum DiscoverKind {
Papers,
Repos,
All,
}
#[derive(Clone, Copy, ValueEnum)]
enum PaperSource {
All,
Arxiv,
Openalex,
}
impl PaperSource {
fn as_str(self) -> &'static str {
match self {
Self::All => "all",
Self::Arxiv => "arxiv",
Self::Openalex => "openalex",
}
}
}
#[derive(Clone, Copy, ValueEnum)]
enum RepoSource {
All,
Github,
Gitlab,
Codeberg,
None,
}
impl RepoSource {
fn as_str(self) -> &'static str {
match self {
Self::All => "all",
Self::Github => "github",
Self::Gitlab => "gitlab",
Self::Codeberg => "codeberg",
Self::None => "none",
}
}
fn providers(self) -> &'static [&'static str] {
match self {
Self::All => &["github", "gitlab", "codeberg"],
Self::Github => &["github"],
Self::Gitlab => &["gitlab"],
Self::Codeberg => &["codeberg"],
Self::None => &[],
}
}
}
#[derive(Clone, Copy, ValueEnum)]
enum OutputFormat {
Markdown,
Json,
}
impl From<OutputFormat> for report::OutputFormat {
fn from(value: OutputFormat) -> Self {
match value {
OutputFormat::Markdown => Self::Markdown,
OutputFormat::Json => Self::Json,
}
}
}
#[tokio::main]
async fn main() -> Result<()> {
let cli = Cli::parse();
match cli.command {
Command::Scan(args) => scan(args).await,
Command::Inspect(args) => inspect(args).await,
Command::Report(args) => report(args),
Command::Sources(args) => sources(args).await,
}
}
async fn scan(mut args: ScanArgs) -> Result<()> {
args.query = args.query.and_then(non_empty);
args.category = args.category.and_then(non_empty);
let has_date = args.since.is_some() || args.from.is_some() || args.to.is_some();
if args.query.is_none() && args.category.is_none() && !has_date {
return Err(err(
"scan needs --query, --category, --since, --from, or --to",
));
}
if matches!(args.paper_source, PaperSource::Openalex) && args.category.is_some() {
return Err(err("--category is only supported by arxiv"));
}
if matches!(args.paper_source, PaperSource::Arxiv)
&& args.query.is_none()
&& args.category.is_none()
{
return Err(err("--paper-source arxiv needs --query or --category"));
}
let paper_query = arxiv::PaperQuery {
query: args.query.clone(),
category: args.category.clone(),
since: args.since.clone(),
from: args.from,
to: args.to,
};
let (from, to) = arxiv::date_range(&paper_query)?;
let mut papers = Vec::new();
let mut paper_provider_outcomes = Vec::new();
let use_arxiv = matches!(args.paper_source, PaperSource::All | PaperSource::Arxiv)
&& (args.query.is_some() || args.category.is_some());
if use_arxiv {
let arxiv_query = arxiv::query_string(&paper_query)?;
eprintln!("paper-gap: fetching arXiv papers...");
match arxiv::search(&arxiv_query, args.limit).await {
Ok(found) => {
papers.extend(found);
paper_provider_outcomes.push(provider::ProviderOutcome::success("arxiv"));
}
Err(error) => {
paper_provider_outcomes.push(provider::ProviderOutcome::from_error("arxiv", &error))
}
}
}
let use_openalex = matches!(args.paper_source, PaperSource::All | PaperSource::Openalex)
&& (args.query.is_some() || (args.category.is_none() && has_date));
if use_openalex {
eprintln!("paper-gap: fetching OpenAlex papers...");
match openalex::search(&openalex::SearchQuery {
query: args.query.clone(),
from,
to,
limit: args.limit,
})
.await
{
Ok(found) => {
papers.extend(found);
paper_provider_outcomes.push(provider::ProviderOutcome::success("openalex"));
}
Err(error) => paper_provider_outcomes
.push(provider::ProviderOutcome::from_error("openalex", &error)),
}
}
let mut papers = paper::deduplicate(papers);
papers.sort_by(|left, right| right.published_date.cmp(&left.published_date));
papers.truncate(args.limit);
eprintln!(
"paper-gap: found {} unique paper(s); checking code links and repositories...",
papers.len()
);
let provenance = report::ReportProvenance {
generated_at: Some(chrono::Utc::now()),
operation: Some(report::OperationKind::Scan),
selectors: report::ReportSelectors {
query: args.query,
category: args.category,
since: args.since,
from: args.from,
to: args.to,
arxiv: None,
},
limit: Some(args.limit),
paper_source: Some(args.paper_source.as_str().into()),
repo_source: Some(args.repo_source.as_str().into()),
};
let report = build_report(
papers,
paper_provider_outcomes,
args.repo_source,
provenance,
)
.await?;
write_output(&render_report(&report, args.format)?, args.output)
}
fn non_empty(value: String) -> Option<String> {
let value = value.trim();
(!value.is_empty()).then(|| value.to_owned())
}
async fn inspect(args: InspectArgs) -> Result<()> {
eprintln!("paper-gap: fetching arXiv paper {}...", args.arxiv);
let papers = arxiv::search(&format!("id:{}", args.arxiv), 1).await?;
eprintln!(
"paper-gap: found {} paper(s); checking code links and repositories...",
papers.len()
);
let provenance = report::ReportProvenance {
generated_at: Some(chrono::Utc::now()),
operation: Some(report::OperationKind::Inspect),
selectors: report::ReportSelectors {
arxiv: Some(args.arxiv),
..Default::default()
},
limit: Some(1),
paper_source: Some("arxiv".into()),
repo_source: Some(args.repo_source.as_str().into()),
};
let report = build_report(
papers,
vec![provider::ProviderOutcome::success("arxiv")],
args.repo_source,
provenance,
)
.await?;
write_output(&render_report(&report, args.format)?, args.output)
}
fn report(args: ReportArgs) -> Result<()> {
let input = if args.input == "-" {
let mut s = String::new();
io::stdin().read_to_string(&mut s)?;
s
} else {
fs::read_to_string(&args.input)?
};
let report: report::ScanReport = serde_json::from_str(&input)?;
write_output(&render_report(&report, args.format)?, args.output)
}
async fn sources(args: SourcesArgs) -> Result<()> {
if let Some(SourcesCommand::Discover(args)) = args.command {
let kind = match args.kind {
DiscoverKind::Papers => sources::DiscoverKind::Papers,
DiscoverKind::Repos => sources::DiscoverKind::Repos,
DiscoverKind::All => sources::DiscoverKind::All,
};
println!(
"{}",
serde_json::to_string_pretty(&sources::discover(kind).await?)?
);
return Ok(());
}
let filter = match (args.papers, args.repos) {
(true, false) => sources::SourceFilter::Papers,
(false, true) => sources::SourceFilter::Repos,
_ => sources::SourceFilter::All,
};
let providers = sources::filter(sources::builtin(), filter);
if args.check {
for provider in &providers {
println!(
"{} {} {}",
provider.id,
provider.display_name,
sources::health_check(provider.base_url).await
);
}
} else {
println!("{}", serde_json::to_string_pretty(&providers)?);
}
Ok(())
}
async fn build_report(
papers: Vec<paper::Paper>,
paper_provider_outcomes: Vec<provider::ProviderOutcome>,
repo_source: RepoSource,
provenance: report::ReportProvenance,
) -> Result<report::ScanReport> {
let mut paper_results = Vec::new();
for (index, paper) in papers.into_iter().enumerate() {
eprintln!(
"paper-gap: [{}] checking Papers with Code: {}",
index + 1,
paper.title
);
let (code_links, pwc_outcome) = match pwc::code_links(&paper).await {
Ok(links) => (
links,
provider::ProviderOutcome::success("papers-with-code"),
),
Err(error) => (
Vec::new(),
provider::ProviderOutcome::from_error("papers-with-code", &error),
),
};
eprintln!("paper-gap: [{}] searching repositories...", index + 1);
let repository_search_enabled = !matches!(repo_source, RepoSource::None);
let (mut repositories, repository_search_outcome) =
repos::search_for_paper_with_providers_and_outcome(&paper, 5, repo_source.providers())
.await;
let official_urls: Vec<_> = code_links
.iter()
.map(|link| link.repository_url.clone())
.collect();
repositories.extend(
official_urls
.iter()
.filter_map(|url| repos::repository_from_url(url)),
);
let repository_matches = repos::rank_repositories(
&paper,
analyse::dedup_repositories(repositories),
&official_urls,
);
let repositories: Vec<_> = repository_matches
.iter()
.map(|matched| matched.repository.clone())
.collect();
eprintln!(
"paper-gap: [{}] inspecting repository quality...",
index + 1
);
let mut repository_analyses = Vec::new();
let mut repository_inspection_outcomes = Vec::new();
for repository in &repositories {
let (paths, outcome) = match repos::file_paths(repository).await {
Ok(paths) => (
paths,
provider::ProviderOutcome::success(format!("{}:tree", repository.forge)),
),
Err(error) => (
Vec::new(),
provider::ProviderOutcome::from_error(
format!("{}:tree", repository.forge),
&error,
),
),
};
let succeeded = outcome.succeeded();
repository_inspection_outcomes.push(outcome);
let mut analysis = analyse::analyze_repository_with_paths(repository, &paths);
analysis.inspection_succeeded = succeeded;
repository_analyses.push(analysis);
}
let gap_assessment = analyse::assess_with_evidence(
&code_links,
&repositories,
&repository_analyses,
pwc_outcome.succeeded(),
repository_search_enabled && repository_search_outcome.succeeded(),
);
paper_results.push(report::PaperResult {
paper,
code_links,
repositories,
repository_matches,
repository_analyses,
pwc_outcome,
repository_search_outcome,
repository_inspection_outcomes,
gap_assessment,
});
}
Ok(report::ScanReport {
schema_version: 1,
provenance,
paper_provider_outcomes,
paper_results,
})
}
fn render_report(report: &report::ScanReport, format: OutputFormat) -> Result<String> {
report::render(report, format.into())
}
fn write_output(text: &str, output: Option<PathBuf>) -> Result<()> {
if let Some(path) = output {
fs::write(path, text)?;
} else {
println!("{}", text);
}
Ok(())
}