affine_importer 0.1.2

AFFiNE import planning and batching core.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

use crate::ImportedIconData;

pub(super) fn html_to_markdown(html: &str) -> String {
  let body = html
    .split_once("page-body")
    .and_then(|(_, rest)| rest.split_once('>').map(|(_, rest)| rest))
    .unwrap_or(html);
  let mut text = html_images_to_markdown(body)
    .replace("<br>", "\n")
    .replace("<br/>", "\n")
    .replace("<br />", "\n")
    .replace("</p>", "\n\n")
    .replace("</div>", "\n")
    .replace("</li>", "\n");
  text = strip_html_tags(&text);
  html_unescape(&text).trim().to_string()
}

fn html_images_to_markdown(html: &str) -> String {
  let mut output = String::with_capacity(html.len());
  let mut rest = html;
  loop {
    let Some(img_start) = rest.find("<img") else {
      output.push_str(rest);
      break;
    };
    output.push_str(&rest[..img_start]);
    let img_rest = &rest[img_start..];
    let Some(tag_end) = img_rest.find('>') else {
      output.push_str(img_rest);
      break;
    };
    let tag = &img_rest[..=tag_end];
    if let Some(src) = html_attr(tag, "src") {
      output.push_str(&format!("\n![]({src})\n"));
    }
    rest = &img_rest[tag_end + 1..];
  }
  output
}

fn html_attr(tag: &str, name: &str) -> Option<String> {
  let pattern = format!("{name}=\"");
  let (_, rest) = tag.split_once(&pattern)?;
  let (value, _) = rest.split_once('"')?;
  Some(html_unescape(value))
}

pub(super) fn extract_html_title(html: &str) -> Option<String> {
  extract_html_class_text(html, "page-title")
    .or_else(|| extract_html_tag_text(html, "h1"))
    .map(|title| title.trim().to_string())
    .filter(|title| !title.is_empty())
}

fn extract_html_class_text(html: &str, class_name: &str) -> Option<String> {
  let marker = format!("class=\"{class_name}\"");
  let (_, rest) = html.split_once(&marker)?;
  let (_, rest) = rest.split_once('>')?;
  let (content, _) = rest.split_once('<')?;
  Some(html_unescape(content))
}

fn extract_html_tag_text(html: &str, tag_name: &str) -> Option<String> {
  let marker = format!("<{tag_name}");
  let (_, rest) = html.split_once(&marker)?;
  let (_, rest) = rest.split_once('>')?;
  let (content, _) = rest.split_once(&format!("</{tag_name}>"))?;
  Some(html_unescape(&strip_html_tags(content)))
}

fn strip_html_tags(value: &str) -> String {
  let mut output = String::with_capacity(value.len());
  let mut in_tag = false;
  for ch in value.chars() {
    match ch {
      '<' => in_tag = true,
      '>' => in_tag = false,
      _ if !in_tag => output.push(ch),
      _ => {}
    }
  }
  output
}

fn html_unescape(value: &str) -> String {
  value
    .replace("&nbsp;", " ")
    .replace("&amp;", "&")
    .replace("&lt;", "<")
    .replace("&gt;", ">")
    .replace("&quot;", "\"")
    .replace("&#39;", "'")
}

pub(super) fn extract_html_icon(html: &str) -> Option<ImportedIconData> {
  let marker = "class=\"icon\"";
  let (_, rest) = html.split_once(marker)?;
  let (_, rest) = rest.split_once('>')?;
  let (content, _) = rest.split_once('<')?;
  let content = html_unescape(content).trim().to_string();
  (!content.is_empty()).then_some(ImportedIconData {
    kind: "emoji".to_string(),
    unicode: content,
  })
}

pub(super) fn split_leading_emoji(title: &str) -> (String, Option<String>) {
  let trimmed = title.trim();
  let Some(first) = trimmed.chars().next() else {
    return (String::new(), None);
  };
  if is_emoji(first) {
    let rest = trimmed[first.len_utf8()..].trim().to_string();
    (rest, Some(first.to_string()))
  } else {
    (trimmed.to_string(), None)
  }
}

fn is_emoji(ch: char) -> bool {
  matches!(
    ch as u32,
    0x1F300..=0x1FAFF | 0x2600..=0x27BF
  )
}