kalosm-language 0.2.1

A set of pretrained language models
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

use super::document::Document;
use url::Url;

mod browse;
pub use browse::*;
mod crawl;
pub use crawl::*;
mod node;
pub use node::*;
#[allow(clippy::module_inception)]
mod page;
pub use page::*;

pub(crate) async fn get_article(url: Url) -> Result<Document, anyhow::Error> {
    let html = reqwest::get(url.clone()).await?.text().await?;
    extract_article(&html)
}

pub(crate) fn extract_article(html: &str) -> anyhow::Result<Document> {
    let cleaned =
        readability::extractor::extract(&mut html.as_bytes(), &Url::parse("https://example.com")?)
            .unwrap();
    Ok(Document::from_parts(cleaned.title, cleaned.text))
}