gthings 0.4.9

CLI binary for gthings — browser-automated web research toolkit
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

use gthings_common::config::GthingsConfig;
use gthings_common::trace::TraceWriter;
use gthings_search::types::FollowOpts;

/// Format and print a single [`gthings_search::types::FollowResult`].
fn output_follow_result(
    result: &gthings_search::types::FollowResult,
    json: bool,
) -> Result<(), anyhow::Error> {
    if json {
        let output = serde_json::to_string_pretty(result)?;
        println!("{}", output);
    } else {
        let status = if result.success { "OK" } else { "FAIL" };
        println!("[{}] {}", status, result.url);
        if let Some(ref content) = result.content {
            println!(
                "  Length: {} chars (total page: {}, truncated: {})",
                content.len(),
                result.total_length,
                result.truncated,
            );
            // Print first 3 lines as preview
            for line in content.lines().take(3) {
                let trimmed = line.trim();
                if !trimmed.is_empty() {
                    println!("  │ {}", trimmed);
                }
            }
        }
        if let Some(ref error) = result.error {
            println!("  Error: {}", error);
        }
        if let Some(ref quality) = result.quality {
            println!("  Quality: score={}, ok={}", quality.score, quality.is_ok);
        }
        println!();
    }
    Ok(())
}

/// Handler for `follow url <url> [--selector S] [--offset N] [--max N]`
pub(crate) async fn handle_follow_url(
    config: &GthingsConfig,
    url: &str,
    selector: &str,
    offset: usize,
    max: usize,
    json: bool,
    trace: Option<&mut TraceWriter>,
) -> Result<(), anyhow::Error> {
    let follower = gthings_search::PageFollower::new(config.clone());
    let opts = FollowOpts {
        selector: selector.to_string(),
        offset,
        max_length: max,
        ..FollowOpts::default()
    };
    let result = follower.follow(url, opts, trace).await?;
    output_follow_result(&result, json)?;
    Ok(())
}

/// Handler for `follow batch <urls...> [--selector S] [--offset N] [--max N]`
pub(crate) async fn handle_follow_batch(
    config: &GthingsConfig,
    urls: &[String],
    selector: &str,
    offset: usize,
    max: usize,
    json: bool,
    trace: Option<&mut TraceWriter>,
) -> Result<(), anyhow::Error> {
    let follower = gthings_search::PageFollower::new(config.clone());
    let opts = FollowOpts {
        selector: selector.to_string(),
        offset,
        max_length: max,
        ..FollowOpts::default()
    };
    let results = follower.batch(urls, opts, trace).await?;

    if json {
        let output = serde_json::to_string_pretty(&results)?;
        println!("{}", output);
    } else {
        for result in &results {
            output_follow_result(result, json)?;
        }
    }
    Ok(())
}