use anyhow::{Context, Result};
use scraper::{Html, Selector};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Chunk {
pub id: usize,
pub text: String,
pub dom_path: String,
#[serde(skip)]
pub embedding: Option<Vec<f32>>,
}
#[derive(Debug, Clone)]
pub struct ChunkerConfig {
pub target_chunk_size: usize,
pub min_chunk_size: usize,
pub max_chunk_size: usize,
pub boundary_tags: Vec<&'static str>,
}
impl Default for ChunkerConfig {
fn default() -> Self {
Self {
target_chunk_size: 350, min_chunk_size: 200, max_chunk_size: 500, boundary_tags: vec![
"article",
"section",
"div",
"main",
"aside",
"table",
"ul",
"ol",
"p",
"blockquote",
],
}
}
}
pub struct SemanticChunker {
config: ChunkerConfig,
}
#[derive(Default)]
struct ChunkAccumulator {
chunks: Vec<Chunk>,
chunk_id: usize,
pending_text: String,
pending_path: String,
}
impl ChunkAccumulator {
fn push_pending_chunk(&mut self) {
if self.pending_text.is_empty() {
return;
}
self.chunks.push(Chunk {
id: self.chunk_id,
text: std::mem::take(&mut self.pending_text),
dom_path: std::mem::take(&mut self.pending_path),
embedding: None,
});
self.chunk_id += 1;
}
fn push_structural(&mut self, text: String, dom_path: String) {
self.push_pending_chunk();
self.chunks.push(Chunk {
id: self.chunk_id,
text,
dom_path,
embedding: None,
});
self.chunk_id += 1;
}
fn append_text(&mut self, text: String, dom_path: String) {
if self.pending_text.is_empty() {
self.pending_path = dom_path;
}
if !self.pending_text.is_empty() {
self.pending_text.push('\n');
}
self.pending_text.push_str(&text);
}
}
impl SemanticChunker {
pub fn new(config: ChunkerConfig) -> Self {
Self { config }
}
pub fn chunk_html(&self, html: &str) -> Result<Vec<Chunk>> {
let document = Html::parse_document(html);
let boundary_nodes = self.collect_boundary_nodes(&document);
if boundary_nodes.is_empty() {
let mut chunk_id = 0;
return self.fallback_chunk(html, &mut chunk_id);
}
let boundary_node_count = boundary_nodes.len();
let mut acc = ChunkAccumulator::default();
for (element, dom_path) in boundary_nodes {
self.accumulate_boundary_node(&element, dom_path, &mut acc);
}
self.flush_tail(&mut acc);
self.merge_small_chunks(&mut acc.chunks);
log::info!(
"Semantic chunking: {} chunks from HTML ({} boundary nodes)",
acc.chunks.len(),
boundary_node_count
);
Ok(acc.chunks)
}
fn accumulate_boundary_node(
&self,
element: &scraper::ElementRef,
dom_path: String,
acc: &mut ChunkAccumulator,
) {
let text = self.extract_visible_text(element);
let text = text.trim().to_string();
if text.is_empty() {
return;
}
if self.is_structural_element(element) {
acc.push_structural(text, dom_path);
return;
}
acc.append_text(text, dom_path);
self.flush_accumulated(acc);
}
fn flush_accumulated(&self, acc: &mut ChunkAccumulator) {
let char_estimate = self.token_to_chars(self.config.target_chunk_size);
if acc.pending_text.len() < char_estimate {
return;
}
if acc.pending_text.len() > self.token_to_chars(self.config.max_chunk_size) {
let sub_chunks =
self.split_large_text(&acc.pending_text, &acc.pending_path, acc.chunk_id);
let sub_len = sub_chunks.len();
acc.chunks.extend(sub_chunks);
acc.chunk_id += sub_len;
} else {
acc.push_pending_chunk();
}
}
fn flush_tail(&self, acc: &mut ChunkAccumulator) {
if acc.pending_text.is_empty() {
return;
}
let char_min = self.token_to_chars(self.config.min_chunk_size);
if acc.pending_text.len() < char_min && acc.chunks.len() > 1 {
if let Some(last) = acc.chunks.last_mut() {
last.text.push('\n');
last.text.push_str(&acc.pending_text);
}
} else {
acc.chunks.push(Chunk {
id: acc.chunk_id,
text: std::mem::take(&mut acc.pending_text),
dom_path: std::mem::take(&mut acc.pending_path),
embedding: None,
});
}
}
fn collect_boundary_nodes<'a>(
&self,
document: &'a Html,
) -> Vec<(scraper::ElementRef<'a>, String)> {
let mut results = Vec::new();
let mut seen_ids = std::collections::HashSet::new();
for tag in &self.config.boundary_tags {
let Ok(selector) = Selector::parse(tag) else {
continue;
};
for element in document.select(&selector) {
let node_id = element.id();
if seen_ids.contains(&node_id) {
continue;
}
seen_ids.insert(node_id);
let dom_path = self.build_dom_path(element);
results.push((element, dom_path));
}
}
results
}
fn build_dom_path(&self, element: scraper::ElementRef) -> String {
let mut path_parts = Vec::new();
let mut current = Some(element);
while let Some(el) = current {
let tag = el.value().name();
if tag == "html" || tag == "body" {
break;
}
path_parts.push(tag.to_string());
current = el.parent().and_then(scraper::ElementRef::wrap);
if path_parts.len() > 10 {
break;
}
}
path_parts.reverse();
path_parts.join(" > ")
}
fn extract_visible_text(&self, element: &scraper::ElementRef) -> String {
let texts: Vec<&str> = element.text().collect();
let joined = texts.join(" ");
joined.split_whitespace().collect::<Vec<_>>().join(" ")
}
fn is_structural_element(&self, element: &scraper::ElementRef) -> bool {
let tag = element.value().name();
matches!(tag, "table" | "ul" | "ol" | "dl" | "figure" | "pre")
}
fn fallback_chunk(&self, html: &str, chunk_id: &mut usize) -> Result<Vec<Chunk>> {
let document = Html::parse_document(html);
let mut chunks = self.paragraph_chunks(&document, chunk_id);
if chunks.is_empty() {
if let Some(chunk) = self.whole_document_chunk(html, chunk_id) {
chunks.push(chunk);
}
}
Ok(chunks)
}
fn paragraph_chunks(&self, document: &Html, chunk_id: &mut usize) -> Vec<Chunk> {
let mut chunks = Vec::new();
if let Ok(selector) = Selector::parse("p") {
let mut pending = String::new();
for element in document.select(&selector) {
let text = element.text().collect::<Vec<_>>().join(" ");
let text = text.trim().to_string();
if text.is_empty() {
continue;
}
if !pending.is_empty() {
pending.push('\n');
}
pending.push_str(&text);
let char_target = self.token_to_chars(self.config.target_chunk_size);
if pending.len() >= char_target {
chunks.push(Chunk {
id: *chunk_id,
text: std::mem::take(&mut pending),
dom_path: "p".to_string(),
embedding: None,
});
*chunk_id += 1;
}
}
if !pending.is_empty() {
chunks.push(Chunk {
id: *chunk_id,
text: pending,
dom_path: "p".to_string(),
embedding: None,
});
*chunk_id += 1;
}
}
chunks
}
fn whole_document_chunk(&self, html: &str, chunk_id: &mut usize) -> Option<Chunk> {
let clean_text = self.strip_html_tags(html);
if clean_text.trim().is_empty() {
return None;
}
let chunk = Chunk {
id: *chunk_id,
text: clean_text,
dom_path: "body".to_string(),
embedding: None,
};
*chunk_id += 1;
Some(chunk)
}
fn split_large_text(&self, text: &str, dom_path: &str, start_id: usize) -> Vec<Chunk> {
let max_chars = self.token_to_chars(self.config.max_chunk_size);
let mut chunks = Vec::new();
let mut remaining = text;
let mut id = start_id;
while !remaining.is_empty() {
let (chunk_text, rest) = if remaining.len() <= max_chars {
(remaining, "")
} else {
#[allow(clippy::manual_pattern_char_comparison)]
let split_at = remaining[..max_chars]
.rfind(|c: char| matches!(c, '.' | '。' | '\n'))
.map(|i| i + 1)
.unwrap_or(max_chars);
(&remaining[..split_at], &remaining[split_at..])
};
chunks.push(Chunk {
id,
text: chunk_text.trim().to_string(),
dom_path: dom_path.to_string(),
embedding: None,
});
id += 1;
remaining = rest;
}
chunks
}
fn merge_small_chunks(&self, chunks: &mut Vec<Chunk>) {
let char_min = self.token_to_chars(self.config.min_chunk_size);
let mut i = 0;
while i < chunks.len().saturating_sub(1) {
if chunks[i].text.len() < char_min {
let small = chunks.remove(i);
if let Some(next) = chunks.get_mut(i) {
next.text = format!("{}\n{}", small.text, next.text);
}
} else {
i += 1;
}
}
}
fn token_to_chars(&self, tokens: usize) -> usize {
tokens * 4
}
fn strip_html_tags(&self, html: &str) -> String {
let document = Html::parse_document(html);
document
.root_element()
.text()
.collect::<Vec<_>>()
.join(" ")
.trim()
.to_string()
}
}
#[async_trait::async_trait]
pub trait EmbeddingProvider: Send + Sync {
async fn embed(&self, texts: &[String]) -> Result<Vec<Vec<f32>>>;
fn dimensions(&self) -> usize;
}
pub struct FallbackEmbeddingProvider {
primary: Arc<dyn EmbeddingProvider>,
secondary: Arc<dyn EmbeddingProvider>,
}
impl FallbackEmbeddingProvider {
pub fn new(primary: Arc<dyn EmbeddingProvider>, secondary: Arc<dyn EmbeddingProvider>) -> Self {
Self { primary, secondary }
}
}
#[async_trait::async_trait]
impl EmbeddingProvider for FallbackEmbeddingProvider {
async fn embed(&self, texts: &[String]) -> Result<Vec<Vec<f32>>> {
match self.primary.embed(texts).await {
Ok(vectors) => Ok(vectors),
Err(primary_err) => {
log::warn!(
"primary embedding provider failed, falling back to secondary: {primary_err:#}"
);
self.secondary.embed(texts).await.map_err(|secondary_err| {
anyhow::anyhow!(
"both embedding providers failed; primary: {primary_err:#}; \
secondary: {secondary_err:#}"
)
})
}
}
}
fn dimensions(&self) -> usize {
self.primary.dimensions()
}
}
pub struct VectorStore {
embeddings: HashMap<usize, Vec<f32>>,
chunks: HashMap<usize, Chunk>,
}
impl VectorStore {
pub fn new() -> Self {
Self {
embeddings: HashMap::new(),
chunks: HashMap::new(),
}
}
pub fn add(&mut self, chunk: Chunk) {
if let Some(embedding) = &chunk.embedding {
self.embeddings.insert(chunk.id, embedding.clone());
}
self.chunks.insert(chunk.id, chunk);
}
pub fn search(&self, query_embedding: &[f32], top_k: usize) -> Vec<(Chunk, f32)> {
let mut scored: Vec<(usize, f32)> = self
.embeddings
.iter()
.map(|(id, emb)| (*id, cosine_similarity(query_embedding, emb)))
.collect();
scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
scored
.into_iter()
.take(top_k)
.filter_map(|(id, score)| self.chunks.get(&id).map(|chunk| (chunk.clone(), score)))
.collect()
}
pub fn len(&self) -> usize {
self.chunks.len()
}
pub fn is_empty(&self) -> bool {
self.chunks.is_empty()
}
}
impl Default for VectorStore {
fn default() -> Self {
Self::new()
}
}
fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
if a.len() != b.len() || a.is_empty() {
return 0.0;
}
let dot: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
let norm_a: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
let norm_b: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
if norm_a == 0.0 || norm_b == 0.0 {
return 0.0;
}
dot / (norm_a * norm_b)
}
pub struct RagExtractionStrategy {
chunker: SemanticChunker,
embedding_provider: Arc<dyn EmbeddingProvider>,
vector_store: VectorStore,
top_k: usize,
}
impl RagExtractionStrategy {
pub fn new(
config: ChunkerConfig,
embedding_provider: Arc<dyn EmbeddingProvider>,
top_k: usize,
) -> Self {
Self {
chunker: SemanticChunker::new(config),
embedding_provider,
vector_store: VectorStore::new(),
top_k,
}
}
pub async fn index_document(&mut self, html: &str, doc_id: &str) -> Result<usize> {
let chunks = self
.chunker
.chunk_html(html)
.context("Failed to chunk HTML")?;
if chunks.is_empty() {
log::warn!("No chunks produced for document {}", doc_id);
return Ok(0);
}
let texts: Vec<String> = chunks.iter().map(|c| c.text.clone()).collect();
let embeddings = self
.embedding_provider
.embed(&texts)
.await
.context("Failed to generate embeddings")?;
if embeddings.len() != chunks.len() {
anyhow::bail!(
"Embedding count ({}) != chunk count ({})",
embeddings.len(),
chunks.len()
);
}
let chunk_count = chunks.len();
for (mut chunk, embedding) in chunks.into_iter().zip(embeddings) {
chunk.embedding = Some(embedding);
self.vector_store.add(chunk);
}
log::info!(
"Indexed document {}: {} chunks with {}-dim embeddings",
doc_id,
chunk_count,
self.embedding_provider.dimensions()
);
Ok(chunk_count)
}
pub async fn retrieve(&self, query: &str) -> Result<Vec<(Chunk, f32)>> {
let query_embedding = self
.embedding_provider
.embed(&[query.to_string()])
.await
.context("Failed to embed query")?;
if query_embedding.is_empty() {
anyhow::bail!("Empty query embedding");
}
Ok(self.vector_store.search(&query_embedding[0], self.top_k))
}
pub async fn build_context(&self, query: &str) -> Result<String> {
let results = self.retrieve(query).await?;
if results.is_empty() {
return Ok(String::new());
}
let context: String = results
.iter()
.enumerate()
.map(|(i, (chunk, score))| {
format!(
"[Chunk {} (score: {:.3}, path: {})]\n{}\n",
i + 1,
score,
chunk.dom_path,
chunk.text
)
})
.collect::<Vec<_>>()
.join("\n---\n\n");
Ok(context)
}
pub fn vector_store(&self) -> &VectorStore {
&self.vector_store
}
}
pub struct RagExtractionRuntime {
chunker_config: ChunkerConfig,
embedding_provider: Arc<dyn EmbeddingProvider>,
top_k: usize,
}
impl RagExtractionRuntime {
pub fn new(
chunker_config: ChunkerConfig,
embedding_provider: Arc<dyn EmbeddingProvider>,
top_k: usize,
) -> Self {
Self {
chunker_config,
embedding_provider,
top_k,
}
}
pub fn create_strategy(&self) -> RagExtractionStrategy {
RagExtractionStrategy::new(
self.chunker_config.clone(),
self.embedding_provider.clone(),
self.top_k,
)
}
pub fn embedding_provider(&self) -> &Arc<dyn EmbeddingProvider> {
&self.embedding_provider
}
}
#[cfg(test)]
mod tests {
use super::*;
struct MockEmbeddingProvider {
dims: usize,
}
#[async_trait::async_trait]
impl EmbeddingProvider for MockEmbeddingProvider {
async fn embed(&self, texts: &[String]) -> Result<Vec<Vec<f32>>> {
Ok(texts
.iter()
.map(|text| {
let mut emb = vec![0.0f32; self.dims];
for (i, byte) in text.bytes().enumerate() {
emb[i % self.dims] += byte as f32 / 255.0;
}
let norm: f32 = emb.iter().map(|x| x * x).sum::<f32>().sqrt();
if norm > 0.0 {
emb.iter_mut().for_each(|x| *x /= norm);
}
emb
})
.collect())
}
fn dimensions(&self) -> usize {
self.dims
}
}
struct FallbackMockEmbeddingProvider {
fail: bool,
calls: std::sync::Arc<std::sync::atomic::AtomicUsize>,
}
#[async_trait::async_trait]
impl EmbeddingProvider for FallbackMockEmbeddingProvider {
async fn embed(&self, texts: &[String]) -> Result<Vec<Vec<f32>>> {
self.calls
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
if self.fail {
return Err(anyhow::anyhow!("mock down"));
}
Ok(texts.iter().map(|_| vec![0.5f32; 2]).collect())
}
fn dimensions(&self) -> usize {
2
}
}
#[tokio::test]
async fn fallback_embedding_primary_success_skips_secondary() {
let primary_calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let secondary_calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let fallback = FallbackEmbeddingProvider::new(
Arc::new(FallbackMockEmbeddingProvider {
fail: false,
calls: primary_calls.clone(),
}),
Arc::new(FallbackMockEmbeddingProvider {
fail: true,
calls: secondary_calls.clone(),
}),
);
let out = fallback
.embed(&["a".to_string(), "b".to_string()])
.await
.expect("primary ok");
assert_eq!(out.len(), 2);
assert_eq!(primary_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
assert_eq!(
secondary_calls.load(std::sync::atomic::Ordering::Relaxed),
0,
"secondary must not be called when primary succeeds"
);
}
#[tokio::test]
async fn fallback_embedding_primary_failure_switches_once() {
let primary_calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let secondary_calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let fallback = FallbackEmbeddingProvider::new(
Arc::new(FallbackMockEmbeddingProvider {
fail: true,
calls: primary_calls.clone(),
}),
Arc::new(FallbackMockEmbeddingProvider {
fail: false,
calls: secondary_calls.clone(),
}),
);
let out = fallback
.embed(&["a".to_string()])
.await
.expect("secondary ok");
assert_eq!(out, vec![vec![0.5, 0.5]]);
assert_eq!(primary_calls.load(std::sync::atomic::Ordering::Relaxed), 1);
assert_eq!(
secondary_calls.load(std::sync::atomic::Ordering::Relaxed),
1
);
}
#[tokio::test]
async fn fallback_embedding_both_fail_reports_both_contexts() {
let fallback = FallbackEmbeddingProvider::new(
Arc::new(FallbackMockEmbeddingProvider {
fail: true,
calls: std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)),
}),
Arc::new(FallbackMockEmbeddingProvider {
fail: true,
calls: std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)),
}),
);
let err = fallback
.embed(&["a".to_string()])
.await
.expect_err("must error");
let rendered = format!("{err:#}");
assert!(
rendered.contains("both embedding providers failed"),
"{rendered}"
);
assert!(
rendered.contains("mock down"),
"must carry both contexts: {rendered}"
);
}
#[test]
fn test_chunk_html_article_boundary() {
let html = r#"
<html><body>
<article>
<h1>Article 1</h1>
<p>First article content with enough text to meet minimum chunk size requirements for testing.</p>
</article>
<article>
<h1>Article 2</h1>
<p>Second article content with enough text to meet minimum chunk size requirements for testing.</p>
</article>
</body></html>
"#;
let chunker = SemanticChunker::new(ChunkerConfig::default());
let chunks = chunker.chunk_html(html).unwrap();
assert!(!chunks.is_empty(), "should produce at least one chunk");
let all_text: String = chunks
.iter()
.map(|c| c.text.as_str())
.collect::<Vec<_>>()
.join(" ");
assert!(all_text.contains("Article 1"));
assert!(all_text.contains("Article 2"));
}
#[test]
fn test_chunk_html_table_not_truncated() {
let html = r#"
<html><body>
<table>
<tr><td>Row 1 Col 1</td><td>Row 1 Col 2</td></tr>
<tr><td>Row 2 Col 1</td><td>Row 2 Col 2</td></tr>
<tr><td>Row 3 Col 1</td><td>Row 3 Col 2</td></tr>
</table>
</body></html>
"#;
let chunker = SemanticChunker::new(ChunkerConfig::default());
let chunks = chunker.chunk_html(html).unwrap();
let table_chunk = chunks.iter().find(|c| c.text.contains("Row 1"));
assert!(table_chunk.is_some(), "table should be in a chunk");
let table_text = &table_chunk.unwrap().text;
assert!(table_text.contains("Row 2"));
assert!(table_text.contains("Row 3"));
}
#[test]
fn test_chunk_html_list_not_truncated() {
let html = r#"
<html><body>
<ul>
<li>Item 1</li>
<li>Item 2</li>
<li>Item 3</li>
<li>Item 4</li>
</ul>
</body></html>
"#;
let chunker = SemanticChunker::new(ChunkerConfig::default());
let chunks = chunker.chunk_html(html).unwrap();
let list_chunk = chunks.iter().find(|c| c.text.contains("Item 1"));
assert!(list_chunk.is_some(), "list should be in a chunk");
let list_text = &list_chunk.unwrap().text;
assert!(list_text.contains("Item 4"), "list should not be truncated");
}
#[test]
fn test_chunk_html_empty_html() {
let html = "<html><body></body></html>";
let chunker = SemanticChunker::new(ChunkerConfig::default());
let chunks = chunker.chunk_html(html).unwrap();
assert!(chunks.is_empty(), "empty HTML should produce no chunks");
}
#[test]
fn test_chunk_html_script_style_excluded() {
let html = r#"
<html><body>
<script>var x = 1;</script>
<style>.foo { color: red; }</style>
<p>Visible content that should be extracted properly for testing.</p>
</body></html>
"#;
let chunker = SemanticChunker::new(ChunkerConfig::default());
let chunks = chunker.chunk_html(html).unwrap();
let all_text: String = chunks
.iter()
.map(|c| c.text.as_str())
.collect::<Vec<_>>()
.join(" ");
assert!(all_text.contains("Visible content"));
}
#[test]
fn test_cosine_similarity_identical() {
let a = vec![1.0, 0.0, 0.0];
let b = vec![1.0, 0.0, 0.0];
let sim = cosine_similarity(&a, &b);
assert!(
(sim - 1.0).abs() < 1e-6,
"identical vectors should have similarity 1.0"
);
}
#[test]
fn test_cosine_similarity_orthogonal() {
let a = vec![1.0, 0.0];
let b = vec![0.0, 1.0];
let sim = cosine_similarity(&a, &b);
assert!(
sim.abs() < 1e-6,
"orthogonal vectors should have similarity 0.0"
);
}
#[test]
fn test_cosine_similarity_opposite() {
let a = vec![1.0, 0.0];
let b = vec![-1.0, 0.0];
let sim = cosine_similarity(&a, &b);
assert!(
(sim + 1.0).abs() < 1e-6,
"opposite vectors should have similarity -1.0"
);
}
#[test]
fn test_cosine_similarity_empty() {
let a: Vec<f32> = vec![];
let b: Vec<f32> = vec![];
assert_eq!(cosine_similarity(&a, &b), 0.0);
}
#[test]
fn test_vector_store_search() {
let mut store = VectorStore::new();
store.add(Chunk {
id: 0,
text: "Rust programming".to_string(),
dom_path: "p".to_string(),
embedding: Some(vec![1.0, 0.0, 0.0]),
});
store.add(Chunk {
id: 1,
text: "Python programming".to_string(),
dom_path: "p".to_string(),
embedding: Some(vec![0.9, 0.1, 0.0]),
});
store.add(Chunk {
id: 2,
text: "Cooking recipe".to_string(),
dom_path: "p".to_string(),
embedding: Some(vec![0.0, 0.0, 1.0]),
});
let results = store.search(&[1.0, 0.0, 0.0], 2);
assert_eq!(results.len(), 2);
assert_eq!(results[0].0.id, 0);
assert!(
results[0].1 > results[1].1,
"should be sorted by similarity desc"
);
}
#[tokio::test]
async fn test_rag_index_and_retrieve() {
let provider = Arc::new(MockEmbeddingProvider { dims: 8 });
let mut strategy = RagExtractionStrategy::new(ChunkerConfig::default(), provider, 2);
let html = r#"
<html><body>
<article>
<h1>Rust Guide</h1>
<p>Rust is a systems programming language focused on safety and performance.</p>
</article>
<article>
<h1>Python Guide</h1>
<p>Python is a high-level programming language known for readability.</p>
</article>
</body></html>
"#;
let chunk_count = strategy.index_document(html, "doc1").await.unwrap();
assert!(chunk_count > 0, "should index at least one chunk");
let results = strategy
.retrieve("Rust programming language")
.await
.unwrap();
assert!(!results.is_empty(), "should find relevant chunks");
}
#[tokio::test]
async fn test_rag_build_context() {
let provider = Arc::new(MockEmbeddingProvider { dims: 4 });
let mut strategy = RagExtractionStrategy::new(ChunkerConfig::default(), provider, 3);
let html = r#"
<html><body>
<p>First paragraph about web scraping techniques.</p>
<p>Second paragraph about data extraction methods.</p>
</body></html>
"#;
strategy.index_document(html, "doc1").await.unwrap();
let context = strategy.build_context("web scraping").await.unwrap();
assert!(!context.is_empty(), "context should not be empty");
assert!(
context.contains("[Chunk"),
"context should contain chunk markers"
);
}
}