searchfox_lib/
file_reader.rs1use crate::client::SearchfoxClient;
2use crate::utils::searchfox_url_repo;
3use anyhow::Result;
4use log::debug;
5use scraper::{Html, Selector};
6
7impl SearchfoxClient {
8 pub async fn get_file(&self, path: &str) -> Result<String> {
9 let url = format!(
10 "{}/{}/source/{}",
11 self.base_url,
12 searchfox_url_repo(&self.repo),
13 path
14 );
15
16 let cache_key = format!("source:{url}");
17
18 if let Some(entry) = self.cache_get(&cache_key) {
19 if entry.is_fresh() {
20 debug!("Cache hit (fresh) for: {}", url);
21 return Ok(entry.content);
22 }
23
24 debug!("Cache stale, revalidating: {}", url);
25 match self
26 .get_html_with_meta(&url, entry.etag.as_deref(), entry.last_modified.as_deref())
27 .await?
28 {
29 None => {
30 debug!("Cache revalidated (304) for: {}", url);
31 self.cache_set(
32 &cache_key,
33 &entry.content,
34 entry.etag.as_deref(),
35 entry.last_modified.as_deref(),
36 );
37 return Ok(entry.content);
38 }
39 Some((html, etag, last_modified)) => {
40 let content = parse_source_lines(&html, &url)?;
41 self.cache_set(
42 &cache_key,
43 &content,
44 etag.as_deref(),
45 last_modified.as_deref(),
46 );
47 return Ok(content);
48 }
49 }
50 }
51
52 if self.force_refetch() {
53 debug!("Force refetch for: {}", url);
54 }
55
56 let (html, etag, last_modified) = self
57 .get_html_with_meta(&url, None, None)
58 .await?
59 .ok_or_else(|| anyhow::anyhow!("Unexpected 304 on cold cache miss"))?;
60
61 let content = parse_source_lines(&html, &url)?;
62 self.cache_set(
63 &cache_key,
64 &content,
65 etag.as_deref(),
66 last_modified.as_deref(),
67 );
68
69 Ok(content)
70 }
71}
72
73fn parse_source_lines(html: &str, url: &str) -> Result<String> {
74 let document = Html::parse_document(html);
75 let selector = Selector::parse("code.source-line").expect("valid selector");
76 let lines: Vec<String> = document
77 .select(&selector)
78 .map(|el| el.text().collect::<String>())
79 .collect();
80 if lines.is_empty() {
81 anyhow::bail!("Could not find file content at {}", url);
82 }
83 Ok(lines.join(""))
84}
85
86#[cfg(test)]
87mod tests {
88 use super::*;
89 use wiremock::matchers::{header, method, path};
90 use wiremock::{Mock, MockServer, ResponseTemplate};
91
92 const HTML_V1: &str = r#"<html><body>
93 <code class="source-line">fn foo() {</code>
94 <code class="source-line"> 42</code>
95 <code class="source-line">}</code>
96 </body></html>"#;
97
98 const HTML_V2: &str = r#"<html><body>
99 <code class="source-line">fn foo() {</code>
100 <code class="source-line"> 99</code>
101 <code class="source-line">}</code>
102 </body></html>"#;
103
104 #[tokio::test]
105 async fn cold_miss_fetches_and_returns_parsed_content() {
106 let server = MockServer::start().await;
107 Mock::given(method("GET"))
108 .and(path("/firefox-main/source/some/file.js"))
109 .respond_with(
110 ResponseTemplate::new(200)
111 .set_body_string(HTML_V1)
112 .insert_header("etag", "\"v1\""),
113 )
114 .expect(1)
115 .mount(&server)
116 .await;
117
118 let client = SearchfoxClient::new_for_test("mozilla-central".into(), server.uri()).unwrap();
119 let content = client.get_file("some/file.js").await.unwrap();
120
121 assert!(content.contains("fn foo()"));
122 assert!(!content.contains("<code"));
123 }
124
125 #[tokio::test]
126 async fn fresh_hit_skips_network() {
127 let server = MockServer::start().await;
128 Mock::given(method("GET"))
129 .and(path("/firefox-main/source/some/file.js"))
130 .respond_with(
131 ResponseTemplate::new(200)
132 .set_body_string(HTML_V1)
133 .insert_header("etag", "\"v1\""),
134 )
135 .expect(1)
136 .mount(&server)
137 .await;
138
139 let client = SearchfoxClient::new_for_test("mozilla-central".into(), server.uri()).unwrap();
140 let first = client.get_file("some/file.js").await.unwrap();
141 let second = client.get_file("some/file.js").await.unwrap();
142
143 assert_eq!(first, second);
144 }
145
146 #[tokio::test]
147 async fn stale_hit_304_refreshes_without_reparse() {
148 let server = MockServer::start().await;
149 Mock::given(method("GET"))
150 .and(path("/firefox-main/source/some/file.js"))
151 .and(header("If-None-Match", "\"v1\""))
152 .respond_with(ResponseTemplate::new(304))
153 .expect(1)
154 .mount(&server)
155 .await;
156
157 let client = SearchfoxClient::new_for_test("mozilla-central".into(), server.uri()).unwrap();
158 let url = format!("{}/firefox-main/source/some/file.js", server.uri());
159 let cache_key = format!("source:{url}");
160 client.cache_set_stale(&cache_key, "fn foo() {\n 42\n}\n", Some("\"v1\""), None);
161
162 let content = client.get_file("some/file.js").await.unwrap();
163 assert!(content.contains("fn foo()"));
164 }
165
166 #[tokio::test]
167 async fn server_error_returns_error() {
168 let server = MockServer::start().await;
169 Mock::given(method("GET"))
170 .and(path("/firefox-main/source/some/file.js"))
171 .respond_with(ResponseTemplate::new(404))
172 .mount(&server)
173 .await;
174
175 let client = SearchfoxClient::new_for_test("mozilla-central".into(), server.uri()).unwrap();
176 assert!(client.get_file("some/file.js").await.is_err());
177 }
178
179 #[tokio::test]
180 async fn html_without_source_lines_returns_error() {
181 let server = MockServer::start().await;
182 Mock::given(method("GET"))
183 .and(path("/firefox-main/source/some/file.js"))
184 .respond_with(
185 ResponseTemplate::new(200)
186 .set_body_string("<html><body>no lines here</body></html>"),
187 )
188 .mount(&server)
189 .await;
190
191 let client = SearchfoxClient::new_for_test("mozilla-central".into(), server.uri()).unwrap();
192 assert!(client.get_file("some/file.js").await.is_err());
193 }
194
195 #[tokio::test]
196 async fn stale_without_etag_does_unconditional_get() {
197 let server = MockServer::start().await;
198 Mock::given(method("GET"))
199 .and(path("/firefox-main/source/some/file.js"))
200 .respond_with(
201 ResponseTemplate::new(200)
202 .set_body_string(HTML_V2)
203 .insert_header("etag", "\"v2\""),
204 )
205 .expect(1)
206 .mount(&server)
207 .await;
208
209 let client = SearchfoxClient::new_for_test("mozilla-central".into(), server.uri()).unwrap();
210 let url = format!("{}/firefox-main/source/some/file.js", server.uri());
211 let cache_key = format!("source:{url}");
212 client.cache_set_stale(&cache_key, "fn foo() {\n 42\n}\n", None, None);
213
214 let content = client.get_file("some/file.js").await.unwrap();
215 assert!(content.contains("99"));
216 }
217
218 #[tokio::test]
219 async fn stale_hit_200_fetches_and_reparses() {
220 let server = MockServer::start().await;
221 Mock::given(method("GET"))
222 .and(path("/firefox-main/source/some/file.js"))
223 .and(header("If-None-Match", "\"v1\""))
224 .respond_with(
225 ResponseTemplate::new(200)
226 .set_body_string(HTML_V2)
227 .insert_header("etag", "\"v2\""),
228 )
229 .expect(1)
230 .mount(&server)
231 .await;
232
233 let client = SearchfoxClient::new_for_test("mozilla-central".into(), server.uri()).unwrap();
234 let url = format!("{}/firefox-main/source/some/file.js", server.uri());
235 let cache_key = format!("source:{url}");
236 client.cache_set_stale(&cache_key, "fn foo() {\n 42\n}\n", Some("\"v1\""), None);
237
238 let content = client.get_file("some/file.js").await.unwrap();
239 assert!(content.contains("99"));
240 assert!(!content.contains("42"));
241 }
242}