rust_tokenizers 8.1.1

High performance tokenizers for Rust
Documentation

Build #3465651 2026-06-03 03:35:39

# rustc version rustc 1.98.0-nightly (d595fce01 2026-06-02)

# docs.rs version docsrs 0.0.0 (f16e33f3e6ac9fc9e4413728c5dd1b2ba1c6b799 2026-05-23 )

# build log [INFO] running `Command { std: "docker" "exec" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "DOCS_RS=1" "-w" "/opt/rustwide/workdir" "--user" "1001:1001" "ab2cf61d67897aaaaaf112bf584820c29a1ec14dd828eddd22656c76cd4e0d0c" "/opt/rustwide/cargo-home/bin/cargo" "+nightly" "rustdoc" "--lib" "-Zrustdoc-map" "--config" "build.rustdocflags=[\"--cfg\", \"docsrs\", \"-Z\", \"unstable-options\", \"--emit=invocation-specific\", \"--resource-suffix\", \"-20260602-1.98.0-nightly-d595fce01\", \"--static-root-path\", \"/-/rustdoc.static/\", \"--cap-lints\", \"warn\", \"--extern-html-root-takes-precedence\"]" "--offline" "-Zunstable-options" "--config=doc.extern-map.registries.crates-io=\"https://docs.rs/{pkg_name}/{version}/x86_64-unknown-linux-gnu\"" "-Zrustdoc-scrape-examples" "-j6" "--target" "x86_64-unknown-linux-gnu", kill_on_drop: false }` [INFO] [stderr] warning: target filter specified, but no targets matched; this is a no-op [INFO] [stderr] Documenting rust_tokenizers v8.1.1 (/opt/rustwide/workdir) [INFO] [stderr] warning: lint `box_pointers` has been removed: it does not detect other kinds of allocations, and existed only for historical reasons [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:11:10 [INFO] [stderr] | [INFO] [stderr] 11 | #![allow(box_pointers)] [INFO] [stderr] | ^^^^^^^^^^^^ [INFO] [stderr] | [INFO] [stderr] = note: `#[warn(renamed_and_removed_lints)]` on by default [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:1517:30 [INFO] [stderr] | [INFO] [stderr] 1517 | fn as_any(&self) -> &dyn (::std::any::Any) { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] = note: `#[warn(unused_parens)]` (part of `#[warn(unused)]`) on by default [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 1517 - fn as_any(&self) -> &dyn (::std::any::Any) { [INFO] [stderr] 1517 + fn as_any(&self) -> &dyn ::std::any::Any { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:1518:22 [INFO] [stderr] | [INFO] [stderr] 1518 | self as &dyn (::std::any::Any) [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 1518 - self as &dyn (::std::any::Any) [INFO] [stderr] 1518 + self as &dyn ::std::any::Any [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:1520:42 [INFO] [stderr] | [INFO] [stderr] 1520 | fn as_any_mut(&mut self) -> &mut dyn (::std::any::Any) { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 1520 - fn as_any_mut(&mut self) -> &mut dyn (::std::any::Any) { [INFO] [stderr] 1520 + fn as_any_mut(&mut self) -> &mut dyn ::std::any::Any { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:1521:26 [INFO] [stderr] | [INFO] [stderr] 1521 | self as &mut dyn (::std::any::Any) [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 1521 - self as &mut dyn (::std::any::Any) [INFO] [stderr] 1521 + self as &mut dyn ::std::any::Any [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:1523:73 [INFO] [stderr] | [INFO] [stderr] 1523 | fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 1523 - fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> { [INFO] [stderr] 1523 + fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn ::std::any::Any> { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2168:30 [INFO] [stderr] | [INFO] [stderr] 2168 | fn as_any(&self) -> &dyn (::std::any::Any) { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2168 - fn as_any(&self) -> &dyn (::std::any::Any) { [INFO] [stderr] 2168 + fn as_any(&self) -> &dyn ::std::any::Any { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2169:22 [INFO] [stderr] | [INFO] [stderr] 2169 | self as &dyn (::std::any::Any) [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2169 - self as &dyn (::std::any::Any) [INFO] [stderr] 2169 + self as &dyn ::std::any::Any [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2171:42 [INFO] [stderr] | [INFO] [stderr] 2171 | fn as_any_mut(&mut self) -> &mut dyn (::std::any::Any) { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2171 - fn as_any_mut(&mut self) -> &mut dyn (::std::any::Any) { [INFO] [stderr] 2171 + fn as_any_mut(&mut self) -> &mut dyn ::std::any::Any { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2172:26 [INFO] [stderr] | [INFO] [stderr] 2172 | self as &mut dyn (::std::any::Any) [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2172 - self as &mut dyn (::std::any::Any) [INFO] [stderr] 2172 + self as &mut dyn ::std::any::Any [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2174:73 [INFO] [stderr] | [INFO] [stderr] 2174 | fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2174 - fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> { [INFO] [stderr] 2174 + fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn ::std::any::Any> { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2364:30 [INFO] [stderr] | [INFO] [stderr] 2364 | fn as_any(&self) -> &dyn (::std::any::Any) { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2364 - fn as_any(&self) -> &dyn (::std::any::Any) { [INFO] [stderr] 2364 + fn as_any(&self) -> &dyn ::std::any::Any { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2365:22 [INFO] [stderr] | [INFO] [stderr] 2365 | self as &dyn (::std::any::Any) [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2365 - self as &dyn (::std::any::Any) [INFO] [stderr] 2365 + self as &dyn ::std::any::Any [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2367:42 [INFO] [stderr] | [INFO] [stderr] 2367 | fn as_any_mut(&mut self) -> &mut dyn (::std::any::Any) { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2367 - fn as_any_mut(&mut self) -> &mut dyn (::std::any::Any) { [INFO] [stderr] 2367 + fn as_any_mut(&mut self) -> &mut dyn ::std::any::Any { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2368:26 [INFO] [stderr] | [INFO] [stderr] 2368 | self as &mut dyn (::std::any::Any) [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2368 - self as &mut dyn (::std::any::Any) [INFO] [stderr] 2368 + self as &mut dyn ::std::any::Any [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2370:73 [INFO] [stderr] | [INFO] [stderr] 2370 | fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2370 - fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> { [INFO] [stderr] 2370 + fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn ::std::any::Any> { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2579:30 [INFO] [stderr] | [INFO] [stderr] 2579 | fn as_any(&self) -> &dyn (::std::any::Any) { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2579 - fn as_any(&self) -> &dyn (::std::any::Any) { [INFO] [stderr] 2579 + fn as_any(&self) -> &dyn ::std::any::Any { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2580:22 [INFO] [stderr] | [INFO] [stderr] 2580 | self as &dyn (::std::any::Any) [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2580 - self as &dyn (::std::any::Any) [INFO] [stderr] 2580 + self as &dyn ::std::any::Any [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2582:42 [INFO] [stderr] | [INFO] [stderr] 2582 | fn as_any_mut(&mut self) -> &mut dyn (::std::any::Any) { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2582 - fn as_any_mut(&mut self) -> &mut dyn (::std::any::Any) { [INFO] [stderr] 2582 + fn as_any_mut(&mut self) -> &mut dyn ::std::any::Any { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2583:26 [INFO] [stderr] | [INFO] [stderr] 2583 | self as &mut dyn (::std::any::Any) [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2583 - self as &mut dyn (::std::any::Any) [INFO] [stderr] 2583 + self as &mut dyn ::std::any::Any [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2585:73 [INFO] [stderr] | [INFO] [stderr] 2585 | fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2585 - fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> { [INFO] [stderr] 2585 + fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn ::std::any::Any> { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2955:30 [INFO] [stderr] | [INFO] [stderr] 2955 | fn as_any(&self) -> &dyn (::std::any::Any) { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2955 - fn as_any(&self) -> &dyn (::std::any::Any) { [INFO] [stderr] 2955 + fn as_any(&self) -> &dyn ::std::any::Any { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2956:22 [INFO] [stderr] | [INFO] [stderr] 2956 | self as &dyn (::std::any::Any) [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2956 - self as &dyn (::std::any::Any) [INFO] [stderr] 2956 + self as &dyn ::std::any::Any [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2958:42 [INFO] [stderr] | [INFO] [stderr] 2958 | fn as_any_mut(&mut self) -> &mut dyn (::std::any::Any) { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2958 - fn as_any_mut(&mut self) -> &mut dyn (::std::any::Any) { [INFO] [stderr] 2958 + fn as_any_mut(&mut self) -> &mut dyn ::std::any::Any { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2959:26 [INFO] [stderr] | [INFO] [stderr] 2959 | self as &mut dyn (::std::any::Any) [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2959 - self as &mut dyn (::std::any::Any) [INFO] [stderr] 2959 + self as &mut dyn ::std::any::Any [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:2961:73 [INFO] [stderr] | [INFO] [stderr] 2961 | fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 2961 - fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> { [INFO] [stderr] 2961 + fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn ::std::any::Any> { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:3210:30 [INFO] [stderr] | [INFO] [stderr] 3210 | fn as_any(&self) -> &dyn (::std::any::Any) { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 3210 - fn as_any(&self) -> &dyn (::std::any::Any) { [INFO] [stderr] 3210 + fn as_any(&self) -> &dyn ::std::any::Any { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:3211:22 [INFO] [stderr] | [INFO] [stderr] 3211 | self as &dyn (::std::any::Any) [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 3211 - self as &dyn (::std::any::Any) [INFO] [stderr] 3211 + self as &dyn ::std::any::Any [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:3213:42 [INFO] [stderr] | [INFO] [stderr] 3213 | fn as_any_mut(&mut self) -> &mut dyn (::std::any::Any) { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 3213 - fn as_any_mut(&mut self) -> &mut dyn (::std::any::Any) { [INFO] [stderr] 3213 + fn as_any_mut(&mut self) -> &mut dyn ::std::any::Any { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:3214:26 [INFO] [stderr] | [INFO] [stderr] 3214 | self as &mut dyn (::std::any::Any) [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 3214 - self as &mut dyn (::std::any::Any) [INFO] [stderr] 3214 + self as &mut dyn ::std::any::Any [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unnecessary parentheses around type [INFO] [stderr] --> src/vocab/sentencepiece_proto/sentencepiece_model.rs:3216:73 [INFO] [stderr] | [INFO] [stderr] 3216 | fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> { [INFO] [stderr] | ^ ^ [INFO] [stderr] | [INFO] [stderr] help: remove these parentheses [INFO] [stderr] | [INFO] [stderr] 3216 - fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> { [INFO] [stderr] 3216 + fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn ::std::any::Any> { [INFO] [stderr] | [INFO] [stderr] [INFO] [stderr] warning: unresolved link to `SEP` [INFO] [stderr] --> src/tokenizer/base_tokenizer.rs:374:82 [INFO] [stderr] | [INFO] [stderr] 374 | /// Vector segments ids (for example for BERT segments are separated with a [SEP] marker, each incrementing the segment ID). [INFO] [stderr] | ^^^ no item named `SEP` in scope [INFO] [stderr] | [INFO] [stderr] = help: to escape `[` and `]` characters, add '\' before them like `\[` or `\]` [INFO] [stderr] = note: `#[warn(rustdoc::broken_intra_doc_links)]` on by default [INFO] [stderr] [INFO] [stderr] warning: unresolved link to `SEP` [INFO] [stderr] --> src/tokenizer/base_tokenizer.rs:406:82 [INFO] [stderr] | [INFO] [stderr] 406 | /// Vector segments ids (for example for BERT segments are separated with a [SEP] marker, each incrementing the segment ID). [INFO] [stderr] | ^^^ no item named `SEP` in scope [INFO] [stderr] | [INFO] [stderr] = help: to escape `[` and `]` characters, add '\' before them like `\[` or `\]` [INFO] [stderr] [INFO] [stderr] warning: unresolved link to `UNK` [INFO] [stderr] --> src/vocab/base_vocab.rs:411:27 [INFO] [stderr] | [INFO] [stderr] 411 | /// Base vocabulary with [UNK] unknown token used as a pre-tokenization step for BERT-class tokenizers. [INFO] [stderr] | ^^^ no item named `UNK` in scope [INFO] [stderr] | [INFO] [stderr] = help: to escape `[` and `]` characters, add '\' before them like `\[` or `\]` [INFO] [stderr] [INFO] [stderr] warning: unopened HTML tag `s` [INFO] [stderr] --> src/tokenizer/base_tokenizer.rs:1122:40 [INFO] [stderr] | [INFO] [stderr] 1122 | /// - pair of sequences: <s> A </s></s> B </s> [INFO] [stderr] | ^^^^ [INFO] [stderr] | [INFO] [stderr] = note: `#[warn(rustdoc::invalid_html_tags)]` on by default [INFO] [stderr] [INFO] [stderr] warning: unopened HTML tag `s` [INFO] [stderr] --> src/tokenizer/base_tokenizer.rs:1122:47 [INFO] [stderr] | [INFO] [stderr] 1122 | /// - pair of sequences: <s> A </s></s> B </s> [INFO] [stderr] | ^^^^ [INFO] [stderr] [INFO] [stderr] warning: `rust_tokenizers` (lib doc) generated 36 warnings (run `cargo fix --lib -p rust_tokenizers` to apply 30 suggestions) [INFO] [stderr] Finished `dev` profile [unoptimized + debuginfo] target(s) in 1.38s [INFO] [stderr] Generated /opt/rustwide/target/x86_64-unknown-linux-gnu/doc/rust_tokenizers/index.html [INFO] running `Command { std: "docker" "inspect" "ab2cf61d67897aaaaaf112bf584820c29a1ec14dd828eddd22656c76cd4e0d0c", kill_on_drop: false }`