kproc 0.7.0

Knowledge Processing library.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

//! Module for the processing of HTML pages.

use kproc_pmacros::Processor;
use std::future::Future;

use crate::processor::{InputStreams, OutputStreams};
use crate::{processor, Result};

/// Inputs for html to text
#[derive(Clone, InputStreams)]
pub struct Inputs
{
  /// Source URL
  pub url: String,
  /// Page data
  pub data: String,
}

impl From<crate::processors::http::Outputs> for Inputs
{
  fn from(value: crate::processors::http::Outputs) -> Self
  {
    Self {
      url: value.url,
      data: value.data,
    }
  }
}

/// Outputs for html to text
#[derive(Clone, OutputStreams)]
pub struct Outputs
{
  /// Source URL
  pub url: String,
  /// Html data
  pub data: String,
}

/// Processor for html to text
#[derive(Processor)]
#[streams(Inputs, Outputs)]
pub struct Html2Text {}

impl processor::ValueProcessor for Html2Text
{
  #[allow(clippy::manual_async_fn)]
  fn process_one(&self, inputs: Self::Inputs)
    -> impl Future<Output = Result<Self::Outputs>> + Send
  {
    async move {
      let r = html2text::from_read(inputs.data.as_bytes(), 120)?;
      Ok(Outputs {
        url: inputs.url,
        data: r,
      })
    }
  }
}