doiget_cli/commands/
text.rs

1//! `doiget text <ref>` — extract a paper's full text (the #281 "read"
2//! step; ADR-0032).
3//!
4//! Fetches the ar5iv LaTeXML-XHTML rendering of an **arXiv** paper and
5//! emits it as sectioned plain text — the read step of the agent research
6//! loop, without an external pdf-to-text tool. The PDF blob is never
7//! opened (ADR-0032 D1).
8//!
9//! - **arXiv id** → ar5iv extraction via
10//!   [`doiget_core::paper_text::paper_text`].
11//! - **DOI** → a structured `NO_OA_AVAILABLE` ("pass the arXiv id"):
12//!   DOI→arXiv resolution is #281 item 5 (ADR-0032 D5).
13//!
14//! `--max-chars N` caps the returned text (truncation is flagged, never
15//! silent); `--no-cache` bypasses the on-disk text cache. `--mode json`
16//! emits the [`PaperText`] structure; the human mode renders a
17//! Markdown-ish title + section layout. Tier-1 OA metadata, always-on —
18//! ships in the default `oa-only` binary (ADR-0032 D2).
19
20use std::io::Write;
21
22use anyhow::{Context, Result};
23
24use doiget_core::paper_text::{paper_text, PaperText, AR5IV_DEFAULT_BASE};
25use doiget_core::{ArxivId, ErrorCode, Ref};
26
27use super::fetch::{build_resolve_context, cli_exit_code, CliExit};
28use super::output::print_err;
29use super::output::OutputMode;
30
31/// Run the `text` subcommand.
32///
33/// # Errors
34///
35/// Surfaces a typed [`ErrorCode`] as a process exit code via
36/// [`CliExit`]: an invalid ref is a usage error; a DOI yields
37/// `NO_OA_AVAILABLE`; an ar5iv render with no extractable prose yields
38/// `TEXT_UNAVAILABLE` (never a silent exit-0 — issue #302) with an
39/// actionable "fetch the PDF" note; other extraction failures map through
40/// [`ErrorCode::from`].
41pub async fn run(
42    ref_: String,
43    max_chars: Option<usize>,
44    no_cache: bool,
45    mode: OutputMode,
46    quiet_was_explicit: bool,
47) -> Result<()> {
48    let parsed = super::parse_ref_or_exit(&ref_)?;
49    let id: ArxivId = match parsed {
50        Ref::Arxiv(a) => a,
51        Ref::Doi(_) => {
52            // No full-text HTML source for a bare DOI in PR4; DOI→arXiv
53            // linking is #281 item 5 (ADR-0032 D5). Report honestly rather
54            // than silently failing.
55            // `NOT_IMPLEMENTED`, not `NO_OA_AVAILABLE`. The latter carries
56            // disposition `needs_config` -- "a named change makes it" -- and
57            // there is no knob: this command is arXiv-only and DOI-to-arXiv
58            // linking is not built (#281 item 5). Kept identical to the MCP
59            // sibling; changing one surface and not the other is the defect
60            // this release is about, and the first pass at this fix did
61            // exactly that.
62            let code = ErrorCode::NotImplemented;
63            print_err(format_args!(
64                "error[{}]: no full-text source for a DOI — if an arXiv preprint exists, \
65                 pass its id (e.g. `doiget text arxiv:2401.12345`)",
66                code.as_wire()
67            ));
68            return Err(anyhow::Error::new(CliExit(cli_exit_code(code))));
69        }
70    };
71
72    let base = resolve_ar5iv_base()?;
73    // `text` is a read-only resolve command (like `cite` / `verify`), so it
74    // reuses the resolve context — which enables the on-disk cache root
75    // (`docs/CACHE.md`) that `paper_text` consults for the text cache.
76    let mut ctx = build_resolve_context().context("building fetch context")?;
77    if no_cache {
78        ctx.cache_root = None;
79    }
80
81    let text = match paper_text(&base, &id, max_chars, &ctx).await {
82        Ok(t) => t,
83        Err(e) => {
84            let code = ErrorCode::from(&e);
85            print_err(format_args!("error[{}]: {e}", code.as_wire()));
86            // `text unavailable` is the one read-step failure with a
87            // concrete next action: the id is valid and the PDF may well be
88            // fetchable, so spell the exact command out (issue #302) rather
89            // than leave the agent to infer it. Mirrors the `= note:` line
90            // `render_fetch_error` attaches to denial-class failures.
91            if code == ErrorCode::TextUnavailable {
92                print_err(format_args!(
93                    "  = note: the arXiv id is valid — fetch the PDF instead: `doiget fetch arxiv:{}`",
94                    id.as_str()
95                ));
96            }
97            return Err(anyhow::Error::new(CliExit(cli_exit_code(code))));
98        }
99    };
100
101    // Extracted paper prose IS the requested artifact (like `bib` / `info`),
102    // so an *implicit* non-TTY Quiet (e.g. `doiget text arxiv:… > paper.txt`)
103    // must NOT swallow it — only an explicit `--quiet` / `DOIGET_MODE=quiet`
104    // does. This is ADR-0017 Amendment 2 (#301), extended to `text`.
105    if mode == OutputMode::Quiet && quiet_was_explicit {
106        return Ok(());
107    }
108
109    let stdout = std::io::stdout();
110    let mut out = stdout.lock();
111    if mode == OutputMode::Json {
112        let s = serde_json::to_string_pretty(&text).context("serializing paper text JSON")?;
113        writeln!(out, "{s}").context("writing paper text JSON to stdout")?;
114        return Ok(());
115    }
116
117    render_human(&mut out, &text)?;
118    Ok(())
119}
120
121/// Resolve the ar5iv base URL: `DOIGET_AR5IV_BASE` override (tests) or the
122/// production default.
123fn resolve_ar5iv_base() -> Result<url::Url> {
124    let raw = std::env::var("DOIGET_AR5IV_BASE").unwrap_or_else(|_| AR5IV_DEFAULT_BASE.to_string());
125    url::Url::parse(&raw).with_context(|| format!("DOIGET_AR5IV_BASE is not a URL: {raw}"))
126}
127
128/// Render extracted text in human mode: a Markdown-ish title + section
129/// layout. The truncation note (when applicable) goes to stderr so it does
130/// not pollute the piped text body on stdout.
131fn render_human(out: &mut impl Write, text: &PaperText) -> Result<()> {
132    if let Some(t) = &text.title {
133        writeln!(out, "# {t}").context("writing title to stdout")?;
134    }
135    for sec in &text.sections {
136        if let Some(h) = &sec.heading {
137            writeln!(out, "\n## {h}").context("writing section heading to stdout")?;
138        }
139        if !sec.text.is_empty() {
140            writeln!(out, "{}", sec.text).context("writing section body to stdout")?;
141        }
142    }
143    if text.truncated {
144        print_err(format_args!(
145            "note: output truncated to {} chars (raise or drop --max-chars for the full text)",
146            text.char_count
147        ));
148    }
149    Ok(())
150}
151
152#[cfg(test)]
153#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
154mod tests {
155    use super::*;
156    use doiget_core::paper_text::{TextSection, TextSource};
157
158    fn sample() -> PaperText {
159        PaperText {
160            arxiv_id: "2401.12345".into(),
161            source: TextSource::Ar5iv,
162            title: Some("A Title".into()),
163            sections: vec![
164                TextSection {
165                    heading: None,
166                    text: "Lead paragraph.".into(),
167                },
168                TextSection {
169                    heading: Some("1 Introduction".into()),
170                    text: "Body text.".into(),
171                },
172            ],
173            char_count: 25,
174            truncated: false,
175            retrieved_from: "https://ar5iv.labs.arxiv.org/html/2401.12345".into(),
176        }
177    }
178
179    #[test]
180    fn json_envelope_is_the_paper_text_shape() {
181        let v = serde_json::to_value(sample()).expect("serialize");
182        assert_eq!(v["arxiv_id"], "2401.12345");
183        assert_eq!(v["source"], "ar5iv");
184        assert_eq!(v["title"], "A Title");
185        assert_eq!(v["sections"][1]["heading"], "1 Introduction");
186        assert_eq!(v["truncated"], false);
187    }
188
189    #[test]
190    fn human_render_lays_out_title_and_sections() {
191        let mut buf: Vec<u8> = Vec::new();
192        render_human(&mut buf, &sample()).expect("render");
193        let s = String::from_utf8(buf).expect("utf8");
194        assert!(s.contains("# A Title"), "got: {s}");
195        assert!(s.contains("## 1 Introduction"), "got: {s}");
196        assert!(s.contains("Lead paragraph."), "got: {s}");
197        assert!(s.contains("Body text."), "got: {s}");
198    }
199
200    #[test]
201    fn resolve_ar5iv_base_defaults_to_production() {
202        // With no override set, the default base must be the production
203        // ar5iv host. (Serial-free: only asserts the default branch when
204        // the env var is absent.)
205        if std::env::var("DOIGET_AR5IV_BASE").is_err() {
206            let u = resolve_ar5iv_base().expect("base");
207            assert_eq!(u.as_str(), "https://ar5iv.labs.arxiv.org/");
208        }
209    }
210}
doiget_cli/commands/text.rs

doiget_cli/commands/
text.rs
