searchfox_lib/
file_reader.rs1use crate::client::SearchfoxClient;
2use crate::utils::searchfox_url_repo;
3use anyhow::Result;
4use log::debug;
5use scraper::{Html, Selector};
6
7impl SearchfoxClient {
8 pub async fn get_file(&self, path: &str) -> Result<String> {
9 let url = format!(
10 "{}/{}/source/{}",
11 self.base_url,
12 searchfox_url_repo(&self.repo),
13 path
14 );
15
16 let cache_key = format!("source:{url}");
17
18 if let Some(entry) = self.cache_get(&cache_key) {
19 if entry.is_fresh() {
20 debug!("Cache hit (fresh) for: {}", url);
21 return Ok(entry.content);
22 }
23
24 debug!("Cache stale, revalidating: {}", url);
25 match self
26 .get_html_with_meta(&url, entry.etag.as_deref(), entry.last_modified.as_deref())
27 .await?
28 {
29 None => {
30 debug!("Cache revalidated (304) for: {}", url);
31 self.cache_set(
32 &cache_key,
33 &entry.content,
34 entry.etag.as_deref(),
35 entry.last_modified.as_deref(),
36 );
37 return Ok(entry.content);
38 }
39 Some((html, etag, last_modified)) => {
40 let content = parse_source_lines(&html, &url)?;
41 self.cache_set(
42 &cache_key,
43 &content,
44 etag.as_deref(),
45 last_modified.as_deref(),
46 );
47 return Ok(content);
48 }
49 }
50 }
51
52 if self.force_refetch() {
53 debug!("Force refetch for: {}", url);
54 }
55
56 let (html, etag, last_modified) = self
57 .get_html_with_meta(&url, None, None)
58 .await?
59 .ok_or_else(|| anyhow::anyhow!("Unexpected 304 on cold cache miss"))?;
60
61 let content = parse_source_lines(&html, &url)?;
62 self.cache_set(
63 &cache_key,
64 &content,
65 etag.as_deref(),
66 last_modified.as_deref(),
67 );
68
69 Ok(content)
70 }
71
72 pub async fn get_file_at_revision(&self, path: &str, revision: &str) -> Result<String> {
73 let url = format!(
74 "{}/{}/rev/{}/{}",
75 self.base_url,
76 searchfox_url_repo(&self.repo),
77 revision,
78 path
79 );
80
81 let cache_key = format!("source-rev:{revision}:{url}");
82
83 if let Some(entry) = self.cache_get(&cache_key) {
84 if entry.is_fresh() {
85 debug!("Cache hit (fresh) for: {}", url);
86 return Ok(entry.content);
87 }
88 }
89
90 let response = self.get(url.parse()?).await?;
91 let html = response.text().await?;
92 let content = parse_source_lines(&html, &url)?;
93 self.cache_set(&cache_key, &content, None, None);
95
96 Ok(content)
97 }
98}
99
100fn parse_source_lines(html: &str, url: &str) -> Result<String> {
101 let document = Html::parse_document(html);
102 let selector = Selector::parse("code.source-line").expect("valid selector");
103 let lines: Vec<String> = document
104 .select(&selector)
105 .map(|el| el.text().collect::<String>())
106 .collect();
107 if lines.is_empty() {
108 anyhow::bail!("Could not find file content at {}", url);
109 }
110 Ok(lines.join(""))
111}
112
113#[cfg(test)]
114mod tests {
115 use super::*;
116 use wiremock::matchers::{header, method, path};
117 use wiremock::{Mock, MockServer, ResponseTemplate};
118
119 const HTML_V1: &str = r#"<html><body>
120 <code class="source-line">fn foo() {</code>
121 <code class="source-line"> 42</code>
122 <code class="source-line">}</code>
123 </body></html>"#;
124
125 const HTML_V2: &str = r#"<html><body>
126 <code class="source-line">fn foo() {</code>
127 <code class="source-line"> 99</code>
128 <code class="source-line">}</code>
129 </body></html>"#;
130
131 #[tokio::test]
132 async fn cold_miss_fetches_and_returns_parsed_content() {
133 let server = MockServer::start().await;
134 Mock::given(method("GET"))
135 .and(path("/firefox-main/source/some/file.js"))
136 .respond_with(
137 ResponseTemplate::new(200)
138 .set_body_string(HTML_V1)
139 .insert_header("etag", "\"v1\""),
140 )
141 .expect(1)
142 .mount(&server)
143 .await;
144
145 let client = SearchfoxClient::new_for_test("mozilla-central".into(), server.uri()).unwrap();
146 let content = client.get_file("some/file.js").await.unwrap();
147
148 assert!(content.contains("fn foo()"));
149 assert!(!content.contains("<code"));
150 }
151
152 #[tokio::test]
153 async fn fresh_hit_skips_network() {
154 let server = MockServer::start().await;
155 Mock::given(method("GET"))
156 .and(path("/firefox-main/source/some/file.js"))
157 .respond_with(
158 ResponseTemplate::new(200)
159 .set_body_string(HTML_V1)
160 .insert_header("etag", "\"v1\""),
161 )
162 .expect(1)
163 .mount(&server)
164 .await;
165
166 let client = SearchfoxClient::new_for_test("mozilla-central".into(), server.uri()).unwrap();
167 let first = client.get_file("some/file.js").await.unwrap();
168 let second = client.get_file("some/file.js").await.unwrap();
169
170 assert_eq!(first, second);
171 }
172
173 #[tokio::test]
174 async fn stale_hit_304_refreshes_without_reparse() {
175 let server = MockServer::start().await;
176 Mock::given(method("GET"))
177 .and(path("/firefox-main/source/some/file.js"))
178 .and(header("If-None-Match", "\"v1\""))
179 .respond_with(ResponseTemplate::new(304))
180 .expect(1)
181 .mount(&server)
182 .await;
183
184 let client = SearchfoxClient::new_for_test("mozilla-central".into(), server.uri()).unwrap();
185 let url = format!("{}/firefox-main/source/some/file.js", server.uri());
186 let cache_key = format!("source:{url}");
187 client.cache_set_stale(&cache_key, "fn foo() {\n 42\n}\n", Some("\"v1\""), None);
188
189 let content = client.get_file("some/file.js").await.unwrap();
190 assert!(content.contains("fn foo()"));
191 }
192
193 #[tokio::test]
194 async fn server_error_returns_error() {
195 let server = MockServer::start().await;
196 Mock::given(method("GET"))
197 .and(path("/firefox-main/source/some/file.js"))
198 .respond_with(ResponseTemplate::new(404))
199 .mount(&server)
200 .await;
201
202 let client = SearchfoxClient::new_for_test("mozilla-central".into(), server.uri()).unwrap();
203 assert!(client.get_file("some/file.js").await.is_err());
204 }
205
206 #[tokio::test]
207 async fn html_without_source_lines_returns_error() {
208 let server = MockServer::start().await;
209 Mock::given(method("GET"))
210 .and(path("/firefox-main/source/some/file.js"))
211 .respond_with(
212 ResponseTemplate::new(200)
213 .set_body_string("<html><body>no lines here</body></html>"),
214 )
215 .mount(&server)
216 .await;
217
218 let client = SearchfoxClient::new_for_test("mozilla-central".into(), server.uri()).unwrap();
219 assert!(client.get_file("some/file.js").await.is_err());
220 }
221
222 #[tokio::test]
223 async fn stale_without_etag_does_unconditional_get() {
224 let server = MockServer::start().await;
225 Mock::given(method("GET"))
226 .and(path("/firefox-main/source/some/file.js"))
227 .respond_with(
228 ResponseTemplate::new(200)
229 .set_body_string(HTML_V2)
230 .insert_header("etag", "\"v2\""),
231 )
232 .expect(1)
233 .mount(&server)
234 .await;
235
236 let client = SearchfoxClient::new_for_test("mozilla-central".into(), server.uri()).unwrap();
237 let url = format!("{}/firefox-main/source/some/file.js", server.uri());
238 let cache_key = format!("source:{url}");
239 client.cache_set_stale(&cache_key, "fn foo() {\n 42\n}\n", None, None);
240
241 let content = client.get_file("some/file.js").await.unwrap();
242 assert!(content.contains("99"));
243 }
244
245 #[tokio::test]
246 async fn stale_hit_200_fetches_and_reparses() {
247 let server = MockServer::start().await;
248 Mock::given(method("GET"))
249 .and(path("/firefox-main/source/some/file.js"))
250 .and(header("If-None-Match", "\"v1\""))
251 .respond_with(
252 ResponseTemplate::new(200)
253 .set_body_string(HTML_V2)
254 .insert_header("etag", "\"v2\""),
255 )
256 .expect(1)
257 .mount(&server)
258 .await;
259
260 let client = SearchfoxClient::new_for_test("mozilla-central".into(), server.uri()).unwrap();
261 let url = format!("{}/firefox-main/source/some/file.js", server.uri());
262 let cache_key = format!("source:{url}");
263 client.cache_set_stale(&cache_key, "fn foo() {\n 42\n}\n", Some("\"v1\""), None);
264
265 let content = client.get_file("some/file.js").await.unwrap();
266 assert!(content.contains("99"));
267 assert!(!content.contains("42"));
268 }
269}