webcrawler 0.1.2

A simple Rust web crawler
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

// use reqwest::blocking::get;
// use std::fs::File;
// use std::io::{BufRead, BufReader};
// use std::path::Path;

// fn main() {

//     let mut num_of_valid_routes = 0;

//     // Create a new String to hold the user input
//     let mut input_url = String::new();

//     // Print a message to the user
//     println!("Enter a URL:");

//     // Read the user input
//     std::io::stdin()
//         .read_line(&mut input_url)
//         .expect("Failed to read URL");

//     // Trim the input URL
//     input_url = input_url.trim().to_string();


//     // Define the path to the file
//     let path = Path::new("src/routes.txt");

//     // Open the file
//     let file = File::open(&path).expect("Could not open file");

//     // Create a buffered reader
//     let reader = BufReader::new(file);

//     // Read the file line by line
//     for line in reader.lines() {
//         match line {
//             Ok(line) => {
//                 if input_url.ends_with("/") {
//                     input_url.pop();
//                 }
                
//                 // Concatenate the input URL with the route without moving input_url
//                 let full_url = format!("{}/{}", input_url, line.trim());

//                 // Make an HTTP GET request to the concatenated URL
//                 match get(&full_url) {
//                     Ok(response) => {
//                         println!("URL: {} - Status Code: {}", full_url, response.status());
//                         if response.status().is_success() {
//                             num_of_valid_routes += 1;
//                         }
//                     }
//                     Err(_) => {
//                         println!("URL: {} - Failed to get a response", full_url);
//                     }
//                 }
//             }
//             Err(e) => println!("Error reading line: {}", e),
//         }
//     }
//     println!("Number of valid routes: {}", num_of_valid_routes);
// }

use webcrawler::WebCrawler;

fn main() {
    let base_url = "https://youtube.com";
    let file_path = "src/routes.txt";

    let mut crawler = WebCrawler::new();

    match crawler.check_valid_routes(base_url, file_path) {
        Ok(valid_routes) => {
            println!("Number of valid routes: {}", valid_routes);
        }
        Err(e) => {
            println!("Error: {}", e);
        }
    }

    // crawler.print_visited_urls();

    let visited_urls = crawler.get_visited_urls();

    for url in visited_urls {
        println!("{}", url);
    }
    
}