coma 0.2.3

Coma is a lightweight command-line tool designed for crawling websites
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141

use core::fmt;

use clap::{Parser, Subcommand};
use colored::Colorize;
use url::Url;

/// Website scraper
#[derive(Parser, Debug)]
#[command(name = "Coma")]
#[command(author = "Noah")]
#[command(version)]
#[command(help_template = "
{name} - {about}

Author: {author}
Version: {version}

{usage-heading} {usage}
{all-args} {tab}")]
pub struct Cli {
    /// Action to perform with the data
    #[command(subcommand)]
    pub cmd: Display,

    /// Content to scrap
    #[arg(short, long, value_delimiter = ',', default_value = "all")]
    pub content: Vec<Content>,

    /// Url to start the search
    #[arg(short, long)]
    pub url: String,

    /// Depth to search from the given url, 0 for only the current url, < 0 for infinite depth
    #[arg(short, long, default_value_t = 0, allow_negative_numbers = true)]
    pub depth: i32,

    /// Upper bound in the url, any url that doesn't contains this string will be ignored
    // TODO : change default to Option<String>
    #[arg(short, long, default_value = "")]
    pub bound: String,

    /// Max number of concurrent thread
    #[arg(short, long, default_value_t = 5)]
    pub thread: u32,

    // Depth to external website with different domain. Depth have priority to stop the search
    #[arg(short, long, default_value_t = 0)]
    pub external: i32,
}

#[derive(Subcommand, Debug, Clone, PartialEq, Eq, Hash)]
pub enum Display {
    /// Print the extracted content in the terminal
    Print {
        /// Format of the output
        #[arg()]
        format: Format,
    },

    /// Save the extracted content in files
    Save {
        /// Format of the output
        #[arg()]
        format: Format,

        /// Name of the output file
        #[arg(short, long, default_value = "output")]
        name: String,
    },

    /// Create a html topolgy
    Graph,
}

#[derive(clap::ValueEnum, Debug, Clone, Copy, PartialEq, serde::Serialize)]
pub enum Content {
    /// Extract the text in the html
    Texts,

    /// Extract the comments in the html
    Comments,

    /// Extract the links found on the page
    Links,

    /// Extract the images of the page
    Images,

    /// Extract informations about any form
    Inputs,

    /// Extract all information and generate a topology
    All,
}

#[derive(clap::ValueEnum, Debug, Clone, PartialEq, Eq, Hash)]
pub enum Format {
    /// Create a json file with the data
    Json,

    /// Raw data. Link are not raw href but joined with domain
    Raw,
}

pub enum ArgsError {
    InvalidUrl(String),
}

impl ArgsError {
    fn print(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            ArgsError::InvalidUrl(url) => write!(f, "{}: {}", "Invalid URL".red(), url),
        }
    }
}

impl fmt::Display for ArgsError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        self.print(f)
    }
}

impl fmt::Debug for ArgsError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        self.print(f)
    }
}

impl std::error::Error for ArgsError {}

pub fn args() -> Result<Cli, ArgsError> {
    let args = Cli::parse();

    match Url::parse(&args.url) {
        Ok(v) => {
            v.domain().ok_or(ArgsError::InvalidUrl(v.to_string()))?;
            Ok(args)
        }
        Err(e) => Err(ArgsError::InvalidUrl(e.to_string())),
    }
}