cleft 0.1.0

Unicode character properties, and grapheme, word and line segmentation with dictionaries, case mapping and CSS classes, for no_std.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84

[package]
name = "cleft"
version = "0.1.0"
edition.workspace = true
rust-version.workspace = true
license = "(MIT OR Apache-2.0) AND Unicode-3.0"
repository.workspace = true
documentation = "https://docs.rs/cleft"
readme = "README.md"
description = "Unicode character properties, and grapheme, word and line segmentation with dictionaries, case mapping and CSS classes, for no_std."
keywords = ["unicode", "segmentation", "line-breaking", "word-break", "grapheme"]
categories = ["text-processing", "internationalization", "no-std"]
# The sources, the generated tables and dictionaries, and their licenses.
# The tests and benches read conformance files and corpora from the
# repository's `support/testing`, which is not published.
include = [
    "/src/**/*.rs",
    "!/src/**/tests.rs",
    "!/src/**/tests/**",
    "/src/dictionary/data/*.dafsa",
    "/README.md",
    "/CHANGELOG.md",
    "/LICENSE-APACHE",
    "/LICENSE-MIT",
    "/LICENSE-UNICODE",
    "/THIRD-PARTY",
]

[package.metadata.docs.rs]
all-features = true

[lints.clippy]
# A module with children is `foo/mod.rs`, never `foo.rs` beside a `foo/`
# folder, as in winkin.
self_named_module_files = "deny"

[features]
default = []

# The classes CSS defines and Unicode does not: East Asian spacing
# (`text-autospace`), the text-spacing class (`text-spacing-trim`), the
# emphasis class (`text-emphasis-skip`) and the first-letter class
# (`::first-letter`). On, they are bits of the rare word, read by the same
# one lookup; off, a smaller table without them is compiled instead.
css = []

# Full case mapping with the language tailorings Chrome applies, and its
# tables (about 5 KB).
case = []

# The word dictionaries of Thai, Lao, Khmer, Myanmar, and Chinese and
# Japanese, compiled in (about 2.7 MB): `Dictionaries::with_compiled`, which
# line and word segmentation read. Without it a host can still load any of
# them at run time.
dictionaries = []

# Nothing at runtime: the crate reads its own generated tables.
[dependencies]

[dev-dependencies]
# The oracles the sweep tests hold the generated tables to, for every
# character: icu_properties for the properties, icu_casemap (with
# icu_locale_core's language ids) for case mapping. Their data is Unicode
# 17.0. Nothing at runtime reads them.
icu_casemap = { version = "2.3.0", default-features = false, features = ["compiled_data"] }
icu_locale_core = { version = "2.3.0", default-features = false }
icu_properties = { version = "2.3.0", default-features = false, features = ["compiled_data"] }

# The oracle the differential tests hold the crate's line and word
# segmenters to, and the baseline they must beat: ICU4X's line, grapheme and
# word segmenters with their dictionaries, timed by `benches/segmentation.rs`
# over the corpora `testing` builds, whose allocator counts what each scan
# allocates.
icu_segmenter = { version = "2.3.0", default-features = false, features = ["compiled_data"] }
testing = { path = "../../support/testing" }

# ICU4X's dictionary tries, read through its data provider, which the
# dictionary tests hold the generated payloads to, word for word.
icu_collections = { version = "2.3.0", default-features = false }
icu_provider = { version = "2.3.1", default-features = false }

[[bench]]
name = "segmentation"
harness = false