doiget_cli/commands/
text.rs

1//! `doiget text <ref>` — extract a paper's full text (the #281 "read"
2//! step; ADR-0032).
3//!
4//! Fetches the ar5iv LaTeXML-XHTML rendering of an **arXiv** paper and
5//! emits it as sectioned plain text — the read step of the agent research
6//! loop, without an external pdf-to-text tool. The PDF blob is never
7//! opened (ADR-0032 D1).
8//!
9//! - **arXiv id** → ar5iv extraction via
10//!   [`doiget_core::paper_text::paper_text`].
11//! - **DOI** → a structured `NO_OA_AVAILABLE` ("pass the arXiv id"):
12//!   DOI→arXiv resolution is #281 item 5 (ADR-0032 D5).
13//!
14//! `--max-chars N` caps the returned text (truncation is flagged, never
15//! silent); `--no-cache` bypasses the on-disk text cache. `--mode json`
16//! emits the [`PaperText`] structure; the human mode renders a
17//! Markdown-ish title + section layout. Tier-1 OA metadata, always-on —
18//! ships in the default `oa-only` binary (ADR-0032 D2).
19
20use std::io::Write;
21
22use anyhow::{Context, Result};
23
24use doiget_core::paper_text::{paper_text, PaperText, AR5IV_DEFAULT_BASE};
25use doiget_core::{ArxivId, ErrorCode, Ref};
26
27use super::fetch::{build_resolve_context, cli_exit_code, CliExit};
28use super::output::OutputMode;
29
30/// Stderr sink for `docs/ERRORS.md` §3 human-error lines (mirrors the
31/// `print_err` helper in `commands::search` / `commands::fetch`).
32#[allow(clippy::print_stderr)]
33fn print_err(args: std::fmt::Arguments<'_>) {
34    eprintln!("{args}");
35}
36
37/// Run the `text` subcommand.
38///
39/// # Errors
40///
41/// Surfaces a typed [`ErrorCode`] as a process exit code via
42/// [`CliExit`]: an invalid ref is a usage error; a DOI yields
43/// `NO_OA_AVAILABLE`; an ar5iv render with no extractable prose yields
44/// `TEXT_UNAVAILABLE` (never a silent exit-0 — issue #302) with an
45/// actionable "fetch the PDF" note; other extraction failures map through
46/// [`ErrorCode::from`].
47pub async fn run(
48    ref_: String,
49    max_chars: Option<usize>,
50    no_cache: bool,
51    mode: OutputMode,
52    quiet_was_explicit: bool,
53) -> Result<()> {
54    let parsed = Ref::parse(&ref_).with_context(|| format!("invalid ref {ref_:?}"))?;
55    let id: ArxivId = match parsed {
56        Ref::Arxiv(a) => a,
57        Ref::Doi(_) => {
58            // No full-text HTML source for a bare DOI in PR4; DOI→arXiv
59            // linking is #281 item 5 (ADR-0032 D5). Report honestly rather
60            // than silently failing.
61            let code = ErrorCode::NoOaAvailable;
62            print_err(format_args!(
63                "error[{}]: no full-text source for a DOI — if an arXiv preprint exists, \
64                 pass its id (e.g. `doiget text arxiv:2401.12345`)",
65                code.as_wire()
66            ));
67            return Err(anyhow::Error::new(CliExit(cli_exit_code(code))));
68        }
69    };
70
71    let base = resolve_ar5iv_base()?;
72    // `text` is a read-only resolve command (like `cite` / `verify`), so it
73    // reuses the resolve context — which enables the on-disk cache root
74    // (`docs/CACHE.md`) that `paper_text` consults for the text cache.
75    let mut ctx = build_resolve_context().context("building fetch context")?;
76    if no_cache {
77        ctx.cache_root = None;
78    }
79
80    let text = match paper_text(&base, &id, max_chars, &ctx).await {
81        Ok(t) => t,
82        Err(e) => {
83            let code = ErrorCode::from(&e);
84            print_err(format_args!("error[{}]: {e}", code.as_wire()));
85            // `text unavailable` is the one read-step failure with a
86            // concrete next action: the id is valid and the PDF may well be
87            // fetchable, so spell the exact command out (issue #302) rather
88            // than leave the agent to infer it. Mirrors the `= note:` line
89            // `render_fetch_error` attaches to denial-class failures.
90            if code == ErrorCode::TextUnavailable {
91                print_err(format_args!(
92                    "  = note: the arXiv id is valid — fetch the PDF instead: `doiget fetch arxiv:{}`",
93                    id.as_str()
94                ));
95            }
96            return Err(anyhow::Error::new(CliExit(cli_exit_code(code))));
97        }
98    };
99
100    // Extracted paper prose IS the requested artifact (like `bib` / `info`),
101    // so an *implicit* non-TTY Quiet (e.g. `doiget text arxiv:… > paper.txt`)
102    // must NOT swallow it — only an explicit `--quiet` / `DOIGET_MODE=quiet`
103    // does. This is ADR-0017 Amendment 2 (#301), extended to `text`.
104    if mode == OutputMode::Quiet && quiet_was_explicit {
105        return Ok(());
106    }
107
108    let stdout = std::io::stdout();
109    let mut out = stdout.lock();
110    if mode == OutputMode::Json {
111        let s = serde_json::to_string_pretty(&text).context("serializing paper text JSON")?;
112        writeln!(out, "{s}").context("writing paper text JSON to stdout")?;
113        return Ok(());
114    }
115
116    render_human(&mut out, &text)?;
117    Ok(())
118}
119
120/// Resolve the ar5iv base URL: `DOIGET_AR5IV_BASE` override (tests) or the
121/// production default.
122fn resolve_ar5iv_base() -> Result<url::Url> {
123    let raw = std::env::var("DOIGET_AR5IV_BASE").unwrap_or_else(|_| AR5IV_DEFAULT_BASE.to_string());
124    url::Url::parse(&raw).with_context(|| format!("DOIGET_AR5IV_BASE is not a URL: {raw}"))
125}
126
127/// Render extracted text in human mode: a Markdown-ish title + section
128/// layout. The truncation note (when applicable) goes to stderr so it does
129/// not pollute the piped text body on stdout.
130fn render_human(out: &mut impl Write, text: &PaperText) -> Result<()> {
131    if let Some(t) = &text.title {
132        writeln!(out, "# {t}").context("writing title to stdout")?;
133    }
134    for sec in &text.sections {
135        if let Some(h) = &sec.heading {
136            writeln!(out, "\n## {h}").context("writing section heading to stdout")?;
137        }
138        if !sec.text.is_empty() {
139            writeln!(out, "{}", sec.text).context("writing section body to stdout")?;
140        }
141    }
142    if text.truncated {
143        print_err(format_args!(
144            "note: output truncated to {} chars (raise or drop --max-chars for the full text)",
145            text.char_count
146        ));
147    }
148    Ok(())
149}
150
151#[cfg(test)]
152#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
153mod tests {
154    use super::*;
155    use doiget_core::paper_text::{TextSection, TextSource};
156
157    fn sample() -> PaperText {
158        PaperText {
159            arxiv_id: "2401.12345".into(),
160            source: TextSource::Ar5iv,
161            title: Some("A Title".into()),
162            sections: vec![
163                TextSection {
164                    heading: None,
165                    text: "Lead paragraph.".into(),
166                },
167                TextSection {
168                    heading: Some("1 Introduction".into()),
169                    text: "Body text.".into(),
170                },
171            ],
172            char_count: 25,
173            truncated: false,
174            retrieved_from: "https://ar5iv.labs.arxiv.org/html/2401.12345".into(),
175        }
176    }
177
178    #[test]
179    fn json_envelope_is_the_paper_text_shape() {
180        let v = serde_json::to_value(sample()).expect("serialize");
181        assert_eq!(v["arxiv_id"], "2401.12345");
182        assert_eq!(v["source"], "ar5iv");
183        assert_eq!(v["title"], "A Title");
184        assert_eq!(v["sections"][1]["heading"], "1 Introduction");
185        assert_eq!(v["truncated"], false);
186    }
187
188    #[test]
189    fn human_render_lays_out_title_and_sections() {
190        let mut buf: Vec<u8> = Vec::new();
191        render_human(&mut buf, &sample()).expect("render");
192        let s = String::from_utf8(buf).expect("utf8");
193        assert!(s.contains("# A Title"), "got: {s}");
194        assert!(s.contains("## 1 Introduction"), "got: {s}");
195        assert!(s.contains("Lead paragraph."), "got: {s}");
196        assert!(s.contains("Body text."), "got: {s}");
197    }
198
199    #[test]
200    fn resolve_ar5iv_base_defaults_to_production() {
201        // With no override set, the default base must be the production
202        // ar5iv host. (Serial-free: only asserts the default branch when
203        // the env var is absent.)
204        if std::env::var("DOIGET_AR5IV_BASE").is_err() {
205            let u = resolve_ar5iv_base().expect("base");
206            assert_eq!(u.as_str(), "https://ar5iv.labs.arxiv.org/");
207        }
208    }
209}
doiget_cli/commands/text.rs

doiget_cli/commands/
text.rs
