1use std::fs;
2use std::io::Read;
3use std::path::{Path, PathBuf};
4use std::process::Command;
5
6use chrono::Utc;
7use regex::Regex;
8
9use crate::error::{CodeSynapseError, Result};
10
11const MAX_TEXT_BYTES: usize = 10 * 1024 * 1024;
12const MAX_BINARY_BYTES: usize = 50 * 1024 * 1024;
13
14#[derive(Debug, Clone, PartialEq)]
15pub enum UrlType {
16 Tweet,
17 Arxiv,
18 Github,
19 Youtube,
20 Pdf,
21 Image,
22 Webpage,
23}
24
25#[derive(Debug, Clone)]
26pub struct IngestResult {
27 pub source_url: String,
28 pub file_path: PathBuf,
29 pub url_type: UrlType,
30}
31
32pub fn detect_url_type(url: &str) -> UrlType {
33 let lower = url.to_lowercase();
34 if lower.contains("twitter.com") || lower.contains("x.com") {
35 return UrlType::Tweet;
36 }
37 if lower.contains("arxiv.org") {
38 return UrlType::Arxiv;
39 }
40 if lower.contains("github.com") {
41 return UrlType::Github;
42 }
43 if lower.contains("youtube.com") || lower.contains("youtu.be") {
44 return UrlType::Youtube;
45 }
46 let path = url_path(url).to_lowercase();
47 if path.ends_with(".pdf") {
48 return UrlType::Pdf;
49 }
50 for ext in [".png", ".jpg", ".jpeg", ".webp", ".gif"] {
51 if path.ends_with(ext) {
52 return UrlType::Image;
53 }
54 }
55 UrlType::Webpage
56}
57
58fn url_path(url: &str) -> &str {
59 let s = url.find("://").map_or(url, |i| &url[i + 3..]);
60 let path_start = s.find('/').unwrap_or(s.len());
61 let path = &s[path_start..];
62 let path = path.split('?').next().unwrap_or(path);
63 path.split('#').next().unwrap_or(path)
64}
65
66pub fn safe_filename(url: &str, suffix: &str) -> String {
67 let without_scheme = url.find("://").map_or(url, |i| &url[i + 3..]);
68 let clean = without_scheme.split('?').next().unwrap_or(without_scheme);
69 let clean = clean.split('#').next().unwrap_or(clean);
70 let re_nonword = Regex::new(r"[^\w\-]").unwrap();
71 let name = re_nonword.replace_all(clean, "_").to_string();
72 let name = name.trim_matches('_').to_string();
73 let re_multi = Regex::new(r"_+").unwrap();
74 let name = re_multi.replace_all(&name, "_").to_string();
75 let name: String = name.chars().take(80).collect();
76 format!("{name}{suffix}")
77}
78
79pub fn yaml_str(s: &str) -> String {
80 let mut out = String::with_capacity(s.len());
81 for ch in s.chars() {
82 let cp = ch as u32;
83 match ch {
84 '\\' => out.push_str("\\\\"),
85 '"' => out.push_str("\\\""),
86 '\n' => out.push_str("\\n"),
87 '\r' => out.push_str("\\r"),
88 '\t' => out.push_str("\\t"),
89 '\0' => out.push_str("\\0"),
90 _ if cp == 0x2028 => out.push_str("\\L"),
91 _ if cp == 0x2029 => out.push_str("\\P"),
92 _ if cp < 0x20 || cp == 0x7f => out.push_str(&format!("\\x{cp:02x}")),
93 _ => out.push(ch),
94 }
95 }
96 out
97}
98
99pub fn html_to_text(html: &str) -> String {
100 let re_script = Regex::new(r"(?is)<script[^>]*>.*?</script>").unwrap();
101 let re_style = Regex::new(r"(?is)<style[^>]*>.*?</style>").unwrap();
102 let re_tags = Regex::new(r"<[^>]+>").unwrap();
103 let re_ws = Regex::new(r"\s+").unwrap();
104 let s = re_script.replace_all(html, " ").into_owned();
105 let s = re_style.replace_all(&s, " ").into_owned();
106 let s = re_tags.replace_all(&s, " ").into_owned();
107 let s = re_ws.replace_all(&s, " ").into_owned();
108 let s = s.trim().to_string();
109 if s.chars().count() > 8000 {
110 s.chars().take(8000).collect()
111 } else {
112 s
113 }
114}
115
116pub fn extract_arxiv_id(url: &str) -> Option<String> {
117 Regex::new(r"(\d{4}\.\d{4,5})")
118 .unwrap()
119 .captures(url)
120 .map(|c| c[1].to_string())
121}
122
123fn fetch_text(url: &str) -> Result<String> {
124 let resp = ureq::get(url)
125 .set("User-Agent", "Mozilla/5.0 codesynapse/1.0")
126 .call()
127 .map_err(|e| CodeSynapseError::Other(format!("fetch failed for {url:?}: {e}")))?;
128 let mut buf = Vec::new();
129 resp.into_reader()
130 .take((MAX_TEXT_BYTES + 1) as u64)
131 .read_to_end(&mut buf)
132 .map_err(CodeSynapseError::Io)?;
133 if buf.len() > MAX_TEXT_BYTES {
134 return Err(CodeSynapseError::Other(format!(
135 "response from {url:?} exceeds {} MiB limit",
136 MAX_TEXT_BYTES / 1_048_576
137 )));
138 }
139 Ok(String::from_utf8_lossy(&buf).into_owned())
140}
141
142fn fetch_bytes(url: &str) -> Result<Vec<u8>> {
143 let resp = ureq::get(url)
144 .set("User-Agent", "Mozilla/5.0 codesynapse/1.0")
145 .call()
146 .map_err(|e| CodeSynapseError::Other(format!("fetch failed for {url:?}: {e}")))?;
147 let mut buf = Vec::new();
148 resp.into_reader()
149 .take((MAX_BINARY_BYTES + 1) as u64)
150 .read_to_end(&mut buf)
151 .map_err(CodeSynapseError::Io)?;
152 if buf.len() > MAX_BINARY_BYTES {
153 return Err(CodeSynapseError::Other(format!(
154 "response from {url:?} exceeds {} MiB limit",
155 MAX_BINARY_BYTES / 1_048_576
156 )));
157 }
158 Ok(buf)
159}
160
161fn html_title(html: &str) -> String {
162 Regex::new(r"(?is)<title[^>]*>(.*?)</title>")
163 .unwrap()
164 .captures(html)
165 .map(|c| {
166 Regex::new(r"\s+")
167 .unwrap()
168 .replace_all(&c[1], " ")
169 .trim()
170 .to_string()
171 })
172 .unwrap_or_default()
173}
174
175fn fetch_webpage_content(url: &str) -> Result<(String, String)> {
176 let html = fetch_text(url)?;
177 let title = html_title(&html);
178 let title = if title.is_empty() {
179 url.to_string()
180 } else {
181 title
182 };
183 let text = html_to_text(&html);
184 Ok((title, text))
185}
186
187fn fetch_arxiv_content(url: &str) -> Result<(String, String)> {
188 let Some(arxiv_id) = extract_arxiv_id(url) else {
189 let (title, text) = fetch_webpage_content(url)?;
190 let now = Utc::now().to_rfc3339();
191 let content = format!(
192 "---\nsource_url: \"{}\"\ntype: webpage\ntitle: \"{}\"\ncaptured_at: {now}\n---\n\n# {title}\n\nSource: {url}\n\n---\n\n{text}\n",
193 yaml_str(url),
194 yaml_str(&title),
195 );
196 return Ok((content, safe_filename(url, ".md")));
197 };
198
199 let api_url = format!("https://export.arxiv.org/abs/{arxiv_id}");
200 let (title, abstract_text, paper_authors) = match fetch_text(&api_url) {
201 Ok(html) => {
202 let re_tags = Regex::new(r"<[^>]+>").unwrap();
203 let abstract_text = Regex::new(r#"(?is)class="abstract[^"]*"[^>]*>(.*?)</blockquote>"#)
204 .unwrap()
205 .captures(&html)
206 .map(|c| re_tags.replace_all(&c[1], "").trim().to_string())
207 .unwrap_or_default();
208 let title = Regex::new(r#"(?is)class="title[^"]*"[^>]*>(.*?)</h1>"#)
209 .unwrap()
210 .captures(&html)
211 .map(|c| re_tags.replace_all(&c[1], " ").trim().to_string())
212 .unwrap_or_else(|| arxiv_id.clone());
213 let paper_authors = Regex::new(r#"(?is)class="authors"[^>]*>(.*?)</div>"#)
214 .unwrap()
215 .captures(&html)
216 .map(|c| re_tags.replace_all(&c[1], "").trim().to_string())
217 .unwrap_or_default();
218 (title, abstract_text, paper_authors)
219 }
220 Err(_) => (arxiv_id.clone(), String::new(), String::new()),
221 };
222
223 let now = Utc::now().to_rfc3339();
224 let content = format!(
225 "---\nsource_url: \"{}\"\narxiv_id: \"{}\"\ntype: paper\ntitle: \"{}\"\npaper_authors: \"{}\"\ncaptured_at: {now}\n---\n\n# {title}\n\n**Authors:** {paper_authors}\n**arXiv:** {arxiv_id}\n\n## Abstract\n\n{abstract_text}\n\nSource: {url}\n",
226 yaml_str(url),
227 yaml_str(&arxiv_id),
228 yaml_str(&title),
229 yaml_str(&paper_authors),
230 );
231 let filename = format!("arxiv_{}.md", arxiv_id.replace('.', "_"));
232 Ok((content, filename))
233}
234
235fn fetch_tweet_content(url: &str) -> Result<(String, String)> {
236 let oembed_url = url.replace("x.com", "twitter.com");
237 let (tweet_text, tweet_author) = match ureq::get("https://publish.twitter.com/oembed")
238 .query("url", &oembed_url)
239 .query("omit_script", "true")
240 .set("User-Agent", "Mozilla/5.0 codesynapse/1.0")
241 .call()
242 {
243 Ok(resp) => {
244 let mut buf = Vec::new();
245 let _ = resp
246 .into_reader()
247 .take((MAX_TEXT_BYTES + 1) as u64)
248 .read_to_end(&mut buf);
249 match serde_json::from_slice::<serde_json::Value>(&buf) {
250 Ok(data) => {
251 let html = data.get("html").and_then(|v| v.as_str()).unwrap_or("");
252 let text = Regex::new(r"<[^>]+>")
253 .unwrap()
254 .replace_all(html, "")
255 .trim()
256 .to_string();
257 let author = data
258 .get("author_name")
259 .and_then(|v| v.as_str())
260 .unwrap_or("unknown")
261 .to_string();
262 (text, author)
263 }
264 Err(_) => (
265 format!("Tweet at {url} (could not fetch content)"),
266 "unknown".to_string(),
267 ),
268 }
269 }
270 Err(_) => (
271 format!("Tweet at {url} (could not fetch content)"),
272 "unknown".to_string(),
273 ),
274 };
275
276 let now = Utc::now().to_rfc3339();
277 let content = format!(
278 "---\nsource_url: \"{}\"\ntype: tweet\nauthor: \"{}\"\ncaptured_at: {now}\n---\n\n# Tweet by @{tweet_author}\n\n{tweet_text}\n\nSource: {url}\n",
279 yaml_str(url),
280 yaml_str(&tweet_author),
281 );
282 Ok((content, safe_filename(url, ".md")))
283}
284
285fn clone_github(url: &str, target_dir: &Path) -> Result<PathBuf> {
286 let repo_name = url.split('/').next_back().unwrap_or("repo");
287 let repo_name = repo_name.trim_end_matches(".git");
288 let clone_dir = target_dir.join(repo_name);
289 let status = Command::new("git")
290 .args([
291 "clone",
292 "--depth",
293 "1",
294 url,
295 clone_dir.to_str().unwrap_or("."),
296 ])
297 .status()
298 .map_err(CodeSynapseError::Io)?;
299 if !status.success() {
300 return Err(CodeSynapseError::Other(format!(
301 "git clone failed for {url:?}"
302 )));
303 }
304 Ok(clone_dir)
305}
306
307fn unique_path(target_dir: &Path, filename: &str) -> PathBuf {
308 let out = target_dir.join(filename);
309 if !out.exists() {
310 return out;
311 }
312 let p = Path::new(filename);
313 let stem = p.file_stem().unwrap_or_default().to_string_lossy();
314 let ext = p.extension().unwrap_or_default().to_string_lossy();
315 let ext_str = if ext.is_empty() {
316 String::new()
317 } else {
318 format!(".{ext}")
319 };
320 for i in 1..1000 {
321 let candidate = target_dir.join(format!("{stem}_{i}{ext_str}"));
322 if !candidate.exists() {
323 return candidate;
324 }
325 }
326 target_dir.join(filename)
327}
328
329pub fn ingest(url: &str, target_dir: &Path) -> Result<IngestResult> {
330 fs::create_dir_all(target_dir).map_err(CodeSynapseError::Io)?;
331 match detect_url_type(url) {
332 UrlType::Github => {
333 let file_path = clone_github(url, target_dir)?;
334 Ok(IngestResult {
335 source_url: url.to_string(),
336 file_path,
337 url_type: UrlType::Github,
338 })
339 }
340 UrlType::Pdf => {
341 let bytes = fetch_bytes(url)?;
342 let filename = safe_filename(url, ".pdf");
343 let out_path = unique_path(target_dir, &filename);
344 fs::write(&out_path, bytes).map_err(CodeSynapseError::Io)?;
345 Ok(IngestResult {
346 source_url: url.to_string(),
347 file_path: out_path,
348 url_type: UrlType::Pdf,
349 })
350 }
351 UrlType::Image => {
352 let suffix = url_path(url)
353 .rsplit('.')
354 .next()
355 .map(|e| format!(".{e}"))
356 .unwrap_or_else(|| ".jpg".to_string());
357 let bytes = fetch_bytes(url)?;
358 let filename = safe_filename(url, &suffix);
359 let out_path = unique_path(target_dir, &filename);
360 fs::write(&out_path, bytes).map_err(CodeSynapseError::Io)?;
361 Ok(IngestResult {
362 source_url: url.to_string(),
363 file_path: out_path,
364 url_type: UrlType::Image,
365 })
366 }
367 UrlType::Arxiv => {
368 let (content, filename) = fetch_arxiv_content(url)?;
369 let out_path = unique_path(target_dir, &filename);
370 fs::write(&out_path, content).map_err(CodeSynapseError::Io)?;
371 Ok(IngestResult {
372 source_url: url.to_string(),
373 file_path: out_path,
374 url_type: UrlType::Arxiv,
375 })
376 }
377 UrlType::Tweet => {
378 let (content, filename) = fetch_tweet_content(url)?;
379 let out_path = unique_path(target_dir, &filename);
380 fs::write(&out_path, content).map_err(CodeSynapseError::Io)?;
381 Ok(IngestResult {
382 source_url: url.to_string(),
383 file_path: out_path,
384 url_type: UrlType::Tweet,
385 })
386 }
387 url_type @ (UrlType::Youtube | UrlType::Webpage) => {
388 let (title, text) = fetch_webpage_content(url)?;
389 let now = Utc::now().to_rfc3339();
390 let content = format!(
391 "---\nsource_url: \"{}\"\ntype: webpage\ntitle: \"{}\"\ncaptured_at: {now}\n---\n\n# {title}\n\nSource: {url}\n\n---\n\n{text}\n",
392 yaml_str(url),
393 yaml_str(&title),
394 );
395 let filename = safe_filename(url, ".md");
396 let out_path = unique_path(target_dir, &filename);
397 fs::write(&out_path, content).map_err(CodeSynapseError::Io)?;
398 Ok(IngestResult {
399 source_url: url.to_string(),
400 file_path: out_path,
401 url_type,
402 })
403 }
404 }
405}
406
407pub fn save_query_result(
408 question: &str,
409 answer: &str,
410 memory_dir: &Path,
411 source_nodes: Option<&[&str]>,
412) -> Result<PathBuf> {
413 fs::create_dir_all(memory_dir).map_err(CodeSynapseError::Io)?;
414 let now = Utc::now();
415 let slug: String = question
416 .to_lowercase()
417 .chars()
418 .map(|c| if c.is_alphanumeric() { c } else { '_' })
419 .take(50)
420 .collect::<String>()
421 .trim_end_matches('_')
422 .to_string();
423 let filename = format!("query_{}_{slug}.md", now.format("%Y%m%d_%H%M%S"));
424 let mut lines: Vec<String> = vec![
425 "---".into(),
426 "type: \"query\"".into(),
427 format!("date: \"{}\"", now.to_rfc3339()),
428 format!("question: \"{}\"", yaml_str(question)),
429 "contributor: \"codesynapse\"".into(),
430 ];
431 if let Some(nodes) = source_nodes {
432 let nodes_str: Vec<String> = nodes.iter().take(10).map(|n| format!("\"{n}\"")).collect();
433 lines.push(format!("source_nodes: [{}]", nodes_str.join(", ")));
434 }
435 lines.push("---".into());
436 lines.push(String::new());
437 lines.push(format!("# Q: {question}"));
438 lines.push(String::new());
439 lines.push("## Answer".into());
440 lines.push(String::new());
441 lines.push(answer.into());
442 if let Some(nodes) = source_nodes {
443 lines.push(String::new());
444 lines.push("## Source Nodes".into());
445 lines.push(String::new());
446 for node in nodes {
447 lines.push(format!("- {node}"));
448 }
449 }
450 let content = lines.join("\n");
451 let out_path = memory_dir.join(&filename);
452 fs::write(&out_path, content).map_err(CodeSynapseError::Io)?;
453 Ok(out_path)
454}
455
456#[cfg(test)]
457mod tests {
458 use super::*;
459 use std::fs;
460
461 #[test]
462 fn test_detect_url_type_arxiv() {
463 assert_eq!(
464 detect_url_type("https://arxiv.org/abs/2301.00001"),
465 UrlType::Arxiv
466 );
467 assert_eq!(
468 detect_url_type("https://arxiv.org/pdf/2301.00001"),
469 UrlType::Arxiv
470 );
471 }
472
473 #[test]
474 fn test_detect_url_type_github() {
475 assert_eq!(
476 detect_url_type("https://github.com/rust-lang/rust"),
477 UrlType::Github
478 );
479 }
480
481 #[test]
482 fn test_detect_url_type_pdf() {
483 assert_eq!(
484 detect_url_type("https://example.com/paper.pdf"),
485 UrlType::Pdf
486 );
487 }
488
489 #[test]
490 fn test_detect_url_type_image() {
491 assert_eq!(
492 detect_url_type("https://example.com/photo.jpg"),
493 UrlType::Image
494 );
495 assert_eq!(
496 detect_url_type("https://example.com/photo.png"),
497 UrlType::Image
498 );
499 assert_eq!(
500 detect_url_type("https://example.com/img.webp"),
501 UrlType::Image
502 );
503 }
504
505 #[test]
506 fn test_detect_url_type_tweet() {
507 assert_eq!(
508 detect_url_type("https://twitter.com/user/status/123"),
509 UrlType::Tweet
510 );
511 assert_eq!(
512 detect_url_type("https://x.com/user/status/123"),
513 UrlType::Tweet
514 );
515 }
516
517 #[test]
518 fn test_detect_url_type_youtube() {
519 assert_eq!(
520 detect_url_type("https://youtube.com/watch?v=abc"),
521 UrlType::Youtube
522 );
523 assert_eq!(detect_url_type("https://youtu.be/abc"), UrlType::Youtube);
524 }
525
526 #[test]
527 fn test_detect_url_type_webpage() {
528 assert_eq!(
529 detect_url_type("https://example.com/page"),
530 UrlType::Webpage
531 );
532 assert_eq!(detect_url_type("https://example.com/"), UrlType::Webpage);
533 assert_eq!(
534 detect_url_type("https://docs.rs/crate/1.0/"),
535 UrlType::Webpage
536 );
537 }
538
539 #[test]
540 fn test_safe_filename_basic() {
541 let name = safe_filename("https://example.com/path/to/page", ".md");
542 assert!(name.ends_with(".md"), "name: {name}");
543 assert!(!name.contains('/'), "name: {name}");
544 assert!(!name.contains(':'), "name: {name}");
545 }
546
547 #[test]
548 fn test_safe_filename_strips_query() {
549 let a = safe_filename("https://example.com/page?foo=bar", ".md");
550 let b = safe_filename("https://example.com/page", ".md");
551 assert_eq!(a, b);
552 }
553
554 #[test]
555 fn test_safe_filename_max_length() {
556 let long_url = format!("https://example.com/{}", "a".repeat(200));
557 let name = safe_filename(&long_url, ".md");
558 let stem = name.strip_suffix(".md").unwrap_or(&name);
559 assert!(
560 stem.chars().count() <= 80,
561 "stem len {} > 80",
562 stem.chars().count()
563 );
564 }
565
566 #[test]
567 fn test_yaml_str_backslash() {
568 assert_eq!(yaml_str("a\\b"), "a\\\\b");
569 }
570
571 #[test]
572 fn test_yaml_str_quote() {
573 assert_eq!(yaml_str(r#"say "hi""#), r#"say \"hi\""#);
574 }
575
576 #[test]
577 fn test_yaml_str_newline() {
578 assert_eq!(yaml_str("line1\nline2"), "line1\\nline2");
579 }
580
581 #[test]
582 fn test_yaml_str_tab() {
583 assert_eq!(yaml_str("col1\tcol2"), "col1\\tcol2");
584 }
585
586 #[test]
587 fn test_yaml_str_control_char() {
588 assert_eq!(yaml_str("\x01"), "\\x01");
589 assert_eq!(yaml_str("\x7f"), "\\x7f");
590 }
591
592 #[test]
593 fn test_yaml_str_unicode_separators() {
594 assert_eq!(yaml_str("\u{2028}"), "\\L");
595 assert_eq!(yaml_str("\u{2029}"), "\\P");
596 }
597
598 #[test]
599 fn test_yaml_str_null() {
600 assert_eq!(yaml_str("\0"), "\\0");
601 }
602
603 #[test]
604 fn test_yaml_str_carriage_return() {
605 assert_eq!(yaml_str("\r"), "\\r");
606 }
607
608 #[test]
609 fn test_html_to_text_basic() {
610 let text = html_to_text("<p>Hello <b>world</b></p>");
611 assert!(text.contains("Hello"), "text: {text}");
612 assert!(text.contains("world"), "text: {text}");
613 assert!(!text.contains('<'), "text: {text}");
614 }
615
616 #[test]
617 fn test_html_to_text_strips_script() {
618 let html = "<p>Hello</p><script>alert('xss')</script><p>World</p>";
619 let text = html_to_text(html);
620 assert!(text.contains("Hello"), "text: {text}");
621 assert!(text.contains("World"), "text: {text}");
622 assert!(!text.contains("alert"), "text: {text}");
623 assert!(!text.contains("xss"), "text: {text}");
624 }
625
626 #[test]
627 fn test_html_to_text_strips_style() {
628 let html = "<style>.foo { color: red }</style><p>Content</p>";
629 let text = html_to_text(html);
630 assert!(text.contains("Content"), "text: {text}");
631 assert!(!text.contains("color"), "text: {text}");
632 }
633
634 #[test]
635 fn test_html_to_text_truncates_at_8000_chars() {
636 let big_text = "word ".repeat(2000);
637 let html = format!("<p>{big_text}</p>");
638 let text = html_to_text(&html);
639 assert!(
640 text.chars().count() <= 8000,
641 "char count {} > 8000",
642 text.chars().count()
643 );
644 }
645
646 #[test]
647 fn test_extract_arxiv_id_abs_url() {
648 assert_eq!(
649 extract_arxiv_id("https://arxiv.org/abs/2301.00001"),
650 Some("2301.00001".to_string())
651 );
652 }
653
654 #[test]
655 fn test_extract_arxiv_id_pdf_url() {
656 assert_eq!(
657 extract_arxiv_id("https://arxiv.org/pdf/2305.12345"),
658 Some("2305.12345".to_string())
659 );
660 }
661
662 #[test]
663 fn test_extract_arxiv_id_five_digits() {
664 assert_eq!(
665 extract_arxiv_id("https://arxiv.org/abs/2301.12345"),
666 Some("2301.12345".to_string())
667 );
668 }
669
670 #[test]
671 fn test_extract_arxiv_id_missing() {
672 assert_eq!(extract_arxiv_id("https://example.com/paper"), None);
673 }
674
675 #[test]
676 fn test_ingest_result_fields() {
677 let r = IngestResult {
678 source_url: "https://example.com".to_string(),
679 file_path: PathBuf::from("/tmp/test.md"),
680 url_type: UrlType::Webpage,
681 };
682 assert_eq!(r.source_url, "https://example.com");
683 assert_eq!(r.url_type, UrlType::Webpage);
684 assert_eq!(r.file_path, PathBuf::from("/tmp/test.md"));
685 }
686
687 #[test]
688 fn test_unique_path_no_conflict() {
689 let dir = std::env::temp_dir();
690 let path = unique_path(&dir, "codesynapse_no_such_file_xyz_99999.md");
691 assert_eq!(path, dir.join("codesynapse_no_such_file_xyz_99999.md"));
692 }
693
694 #[test]
695 fn test_unique_path_with_conflict() {
696 let dir = std::env::temp_dir();
697 let filename = "codesynapse_ingest_test_conflict.md";
698 let p = dir.join(filename);
699 fs::write(&p, "x").unwrap();
700 let result = unique_path(&dir, filename);
701 assert_ne!(result, p, "should not overwrite existing file");
702 assert!(
703 result
704 .to_string_lossy()
705 .contains("codesynapse_ingest_test_conflict_1"),
706 "result: {}",
707 result.display()
708 );
709 fs::remove_file(p).ok();
710 }
711
712 #[test]
713 fn test_save_query_result_creates_file() {
714 let dir = std::env::temp_dir().join("codesynapse_test_save_query_basic");
715 fs::create_dir_all(&dir).unwrap();
716 let path =
717 save_query_result("What is a graph?", "A set of nodes and edges.", &dir, None).unwrap();
718 assert!(
719 path.exists(),
720 "file should be created at {}",
721 path.display()
722 );
723 let content = fs::read_to_string(&path).unwrap();
724 assert!(content.contains("What is a graph?"), "content: {content}");
725 assert!(content.contains("nodes and edges"), "content: {content}");
726 assert!(content.contains("type: \"query\""), "content: {content}");
727 fs::remove_dir_all(&dir).ok();
728 }
729
730 #[test]
731 fn test_save_query_result_with_nodes() {
732 let dir = std::env::temp_dir().join("codesynapse_test_save_query_nodes");
733 fs::create_dir_all(&dir).unwrap();
734 let nodes = ["node_a", "node_b"];
735 let path = save_query_result("Query?", "Answer.", &dir, Some(&nodes)).unwrap();
736 let content = fs::read_to_string(&path).unwrap();
737 assert!(content.contains("node_a"), "content: {content}");
738 assert!(content.contains("## Source Nodes"), "content: {content}");
739 fs::remove_dir_all(&dir).ok();
740 }
741
742 #[test]
743 #[ignore]
744 fn test_ingest_arxiv_live() {
745 let dir = std::env::temp_dir().join("codesynapse_test_arxiv_live");
746 fs::create_dir_all(&dir).unwrap();
747 let result = ingest("https://arxiv.org/abs/2301.00001", &dir).unwrap();
748 assert_eq!(result.url_type, UrlType::Arxiv);
749 assert!(result.file_path.exists());
750 let content = fs::read_to_string(&result.file_path).unwrap();
751 assert!(content.contains("arxiv_id"), "content: {content}");
752 fs::remove_dir_all(&dir).ok();
753 }
754
755 #[test]
756 #[ignore]
757 fn test_ingest_webpage_live() {
758 let dir = std::env::temp_dir().join("codesynapse_test_webpage_live");
759 fs::create_dir_all(&dir).unwrap();
760 let result = ingest("https://example.com", &dir).unwrap();
761 assert_eq!(result.url_type, UrlType::Webpage);
762 assert!(result.file_path.exists());
763 let content = fs::read_to_string(&result.file_path).unwrap();
764 assert!(content.contains("source_url"), "content: {content}");
765 fs::remove_dir_all(&dir).ok();
766 }
767}