urlsieve 0.1.0

Intelligent URL deduplication tool for bug bounty workflows
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

use std::io::Write;

use crate::cli::OutputFormat;
use crate::dedup::DedupResult;

/// Writes deduplication results to a writer in the specified format.
///
/// # Errors
///
/// Returns an error if writing to the output fails.
pub fn write_output<W: Write>(
    mut writer: W,
    result: &DedupResult,
    format: OutputFormat,
) -> Result<(), String> {
    match format {
        OutputFormat::Rep => write_representative(&mut writer, result)?,
        OutputFormat::Counted => write_counted(&mut writer, result)?,
        OutputFormat::Json => write_json(&mut writer, result)?,
        OutputFormat::Jsonl => write_jsonl(&mut writer, result)?,
    }

    writer.flush().map_err(|e| format!("Failed to flush output: {e}"))?;
    Ok(())
}

fn write_representative<W: Write>(
    writer: &mut W,
    result: &DedupResult,
) -> Result<(), String> {
    for group in &result.groups {
        writeln!(writer, "{}", group.representative)
            .map_err(|e| format!("Failed to write output: {e}"))?;
    }
    Ok(())
}

fn write_counted<W: Write>(
    writer: &mut W,
    result: &DedupResult,
) -> Result<(), String> {
    for group in &result.groups {
        if group.count > 1 {
            writeln!(writer, "{}  # {} duplicates", group.representative, group.count)
                .map_err(|e| format!("Failed to write output: {e}"))?;
        } else {
            writeln!(writer, "{}", group.representative)
                .map_err(|e| format!("Failed to write output: {e}"))?;
        }
    }
    Ok(())
}

fn write_json<W: Write>(
    writer: &mut W,
    result: &DedupResult,
) -> Result<(), String> {
    serde_json::to_writer(&mut *writer, result)
        .map_err(|e| format!("Failed to write JSON: {e}"))?;

    writeln!(writer).map_err(|e| format!("Failed to write newline: {e}"))?;

    Ok(())
}

fn write_jsonl<W: Write>(
    writer: &mut W,
    result: &DedupResult,
) -> Result<(), String> {
    for group in &result.groups {
        serde_json::to_writer(&mut *writer, group)
            .map_err(|e| format!("Failed to write JSONL: {e}"))?;
        writeln!(writer).map_err(|e| format!("Failed to write newline: {e}"))?;
    }
    Ok(())
}

/// Writes diff output (new URLs not in baseline) to a writer.
///
/// # Errors
///
/// Returns an error if writing to the output fails.
pub fn write_diff_output<W: Write>(
    mut writer: W,
    urls: &[String],
) -> Result<(), String> {
    for url in urls {
        writeln!(writer, "{url}")
            .map_err(|e| format!("Failed to write output: {e}"))?;
    }

    writer.flush().map_err(|e| format!("Failed to flush output: {e}"))?;
    Ok(())
}