1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166

use futures::future::join_all;
use regex::Regex;
use std::env;

const GOOGLE_RESULT_SIZE: usize = 10;
const GOOGLE_ENDPOINT: &'static str = "https://www.googleapis.com/customsearch/v1";

fn get_creds() -> (String, String) {
    let api_key =
        env::var("CUSTOM_SEARCH_API_KEY").expect("variable `CUSTOM_SEARCH_API_KEY` not set");
    let search_engine_id =
        env::var("SEARCH_ENGINE_ID").expect("variable `SEARCH_ENGINE_ID` not set");

    (api_key, search_engine_id)
}

enum SearchType {
    Image,
}

#[allow(dead_code)]
pub struct Params {
    cx: String,
    key: String,
    search_type: SearchType,
    q: String,
    num: usize,
    start: usize,
}

impl Params {
    pub fn new() -> Self {
        let (key, cx) = get_creds();
        Params {
            cx,
            key,
            search_type: SearchType::Image,
            q: String::new(),
            num: 10, //max num
            start: 0,
        }
    }

    pub fn to_list(self) -> Vec<(String, String)> {
        vec![
            ("cx".to_string(), self.cx),
            ("key".to_string(), self.key),
            ("searchType".to_string(), "image".to_string()), //bleh
            ("q".to_string(), self.q),
            ("num".to_string(), self.num.to_string()),
            ("start".to_string(), self.start.to_string()),
        ]
    }
}

pub struct Fetcher {
    // tx: Arc<mpsc::Sender<Vec<String>>>,
}

impl Fetcher {
    pub async fn query_api(query: &str, n: usize) -> Vec<String> {
        //guesstimate how many api calls we need to make
        let maybe_one = if n % GOOGLE_RESULT_SIZE > 0 { 1 } else { 0 };
        let nqueries = n / GOOGLE_RESULT_SIZE + maybe_one + 1; //+1 guesstimate
        let nqueries = usize::min(nqueries, 10);

        let mut futures = vec![];
        for i in 0..nqueries {
            //from specs
            let mut offset = 0;
            if i > 0 {
                offset = 10 * i + 1;
            }

            //build params
            let mut params = Params::new();
            params.start = offset;
            params.q = format_query(query);

            futures.push(tokio::spawn(async move {
                Self::google_search(params).await.unwrap()
            }));
        }

        let urls: Vec<Vec<String>> = join_all(futures)
            .await
            .into_iter()
            .map(|x| x.unwrap())
            .collect();

        let urls = urls.into_iter().flatten().collect();
        urls
    }

    // could make this a streaming function
    async fn google_search(params: Params) -> Result<Vec<String>, reqwest::Error> {
        let params = params.to_list();

        let client = reqwest::Client::new();
        let res = client.get(GOOGLE_ENDPOINT).query(&params).send().await?;
        let body = res.text().await?;

        let urls = extract_urls(&body);
        Ok(urls)
    }
}

fn format_query(topic: &str) -> String {
    //remove underscores
    let new_topic: Vec<String> = topic.split('_').map(String::from).collect();
    new_topic.join(" ")
}

// from chat
fn extract_urls(text: &str) -> Vec<String> {
    // Define the regex pattern to match the URLs
    let re = Regex::new(r#""link":\s*"(https?://[^"]*)""#).unwrap();

    // Initialize a vector to store the extracted URLs
    let mut urls = Vec::new();

    // Iterate over all matches in the text
    for cap in re.captures_iter(text) {
        // Capture group 1 contains the URL
        if let Some(url) = cap.get(1) {
            urls.push(url.as_str().to_string());
        }
    }

    urls
}

//dummy
// fn get_urls() -> Vec<String> {
//     let urls = fs::read_to_string("urls.txt").expect("couldn't read urls");
//     let urls: Vec<String> = urls.lines().map(|s| s.to_string()).collect();
//     urls
// }
// pub async fn stream_batched(&self) {
//     let urls = get_urls();
//     let urls = urls.as_slice();
//
//     sleep(Duration::from_millis(500)).await;
//
//     let mut buffer = Vec::new();
//     for batch in urls.windows(10).step_by(10) {
//         buffer.extend_from_slice(batch);
//
//         if buffer.len() >= 150 {
//             // println!("sent: {}", buffer.len());
//             match self.tx.send(buffer.clone()).await {
//                 Ok(_) => (),
//                 _ => return (),
//             }
//             buffer.clear();
//         }
//     }
//     if !buffer.is_empty() {
//         // println!("sent: {}", buffer.len());
//         match self.tx.send(buffer.clone()).await {
//             Ok(_) => (),
//             _ => return (),
//         }
//         buffer.clear();
//     }
// }