gthings 0.3.5

CLI binary for gthings — browser-automated web research toolkit
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

use std::path::Path;

use gthings_common::config::GthingsConfig;

/// Shared output helper for PDF extraction results.
fn output_pdf_result(
    text: &str,
    source: &str,
    pages: Option<usize>,
    json: bool,
) -> Result<(), anyhow::Error> {
    if json {
        let value = serde_json::json!({
            "source": source,
            "text": text,
            "length": text.len(),
            "pages": pages,
        });
        let output = serde_json::to_string_pretty(&value)?;
        println!("{}", output);
    } else {
        println!("PDF: {}", source);
        if let Some(p) = pages {
            println!("Pages: {}", p);
        }
        println!("Length: {} chars", text.len());
        println!();
        // Show first 10 lines as preview
        for line in text.lines().take(10) {
            let trimmed = line.trim();
            if !trimmed.is_empty() {
                println!("  │ {}", trimmed);
            }
        }
        if text.lines().count() > 10 {
            println!("  │ … ({} more lines)", text.lines().count() - 10);
        }
    }
    Ok(())
}

/// Handler for `pdf url <url>`
pub(crate) async fn handle_pdf_url(
    config: &GthingsConfig,
    url: &str,
    json: bool,
) -> Result<(), anyhow::Error> {
    let client = reqwest::Client::builder()
        .timeout(std::time::Duration::from_millis(config.request_timeout_ms))
        .build()?;

    let response = client
        .get(url)
        .send()
        .await
        .map_err(|e| anyhow::anyhow!("Failed to fetch PDF URL '{}': {}", url, e))?;

    let status = response.status();
    if !status.is_success() {
        return Err(anyhow::anyhow!(
            "PDF URL '{}' returned HTTP {} {}. Verify the URL is correct and accessible.",
            url,
            status.as_u16(),
            status.canonical_reason().unwrap_or("Unknown"),
        ));
    }

    let content_type = response
        .headers()
        .get(reqwest::header::CONTENT_TYPE)
        .and_then(|v| v.to_str().ok())
        .unwrap_or("unknown")
        .to_string();

    let bytes = response
        .bytes()
        .await
        .map_err(|e| anyhow::anyhow!("Failed to read PDF content from '{}': {}", url, e))?;

    // Quick check: PDF files start with %PDF
    if bytes.len() < 4 || &bytes[..4] != b"%PDF" {
        return Err(anyhow::anyhow!(
            "URL '{}' returned content type '{}' ({} bytes) but does not appear to be a PDF. \
             The URL may point to an HTML page instead of a PDF file. \
             For arXiv papers, use the abstract page URL (https://arxiv.org/abs/XXXX.XXXXX) \
             and the PDF URL will be extracted automatically.",
            url,
            content_type,
            bytes.len(),
        ));
    }

    let text = gthings_extraction::PdfExtractor::extract(&bytes)
        .map_err(|e| anyhow::anyhow!("Failed to extract text from PDF at '{}': {}", url, e))?;
    let pages = gthings_extraction::PdfExtractor::count_pages(&bytes).ok();

    output_pdf_result(&text, url, pages, json)?;
    Ok(())
}

/// Handler for `pdf file <path>`
pub(crate) async fn handle_pdf_file(
    _config: &GthingsConfig,
    path: &Path,
    json: bool,
) -> Result<(), anyhow::Error> {
    let bytes = std::fs::read(path)?;

    let text = gthings_extraction::PdfExtractor::extract(&bytes)?;
    let pages = gthings_extraction::PdfExtractor::count_pages(&bytes).ok();

    let source = path.to_string_lossy();
    output_pdf_result(&text, &source, pages, json)?;
    Ok(())
}