embedcache 0.1.1

High-performance text embedding service with caching capabilities
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

//! Content fetching utilities

use anyhow::{Context, Result};
use readability::extractor;
use tokio::task;

/// Fetch content from a URL
///
/// This function extracts text content from a URL using the readability library.
pub async fn fetch_content(url: String) -> Result<String> {
    task::spawn_blocking(move || {
        extractor::scrape(&url)
            .map(|product| product.content)
            .unwrap_or_else(|_| String::from("Failed to scrape content"))
    })
    .await
    .context("Failed to fetch content")
}