xutf 1.3.0

Permissive UTF-8/16/32 transcoding, comparison and BOM detection with SIMD ASCII fast paths
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

//! ANSI-aware visible width benchmark against allocating strip-and-measure
//! baselines.
//!
//! Run default ratio bench: `cargo bench --bench width_ansi`
//! Run Divan bench: `DIVAN=1 cargo bench --bench width_ansi`

use divan::{Bencher, counter::BytesCount};
use rayon::prelude::*;
use strip_ansi::strip_ansi as reference_strip_ansi;
use unicode_width::UnicodeWidthStr;
use xutf::{Text, ToAnsiStripped, width_ansi_str};

mod common;
use common::{build_input, measure, print_ratio_table};

common::bench_main!();

const TARGET: usize = 1 << 20;

fn inputs() -> Vec<(&'static str, String)> {
	vec![
		(
			"sgr-heavy",
			build_input(
				"\x1b[38;5;244mINFO\x1b[0m \x1b[1mrequest\x1b[0m completed in \x1b[32m12ms\x1b[0m\n",
				TARGET,
			),
		),
		("ascii", build_input("The quick brown fox jumps over 13 lazy dogs. ", TARGET)),
		("mixed", build_input("状态 café 漢字 🦀 👩‍🚀 e\u{301}\n", TARGET)),
		(
			"hyperlinks",
			build_input(
				"\x1b]8;;https://example.com/docs\x07documentation\x1b]8;;\x1b\\ \x1b]8;;https://example.com/status\x07status\x1b]8;;\x1b\\\n",
				TARGET,
			),
		),
	]
}

fn reference_width(input: &str) -> usize {
	let stripped = reference_strip_ansi(input);
	UnicodeWidthStr::width(stripped.as_str())
}

fn run_ratio_table() {
	let rows: Vec<_> = inputs()
		.par_iter()
		.map(|(name, input)| {
			let expected = width_ansi_str(input);
			let stripped = input.to_ansi_stripped();
			assert_eq!(stripped.visible_width(), expected);
			// A contender that computes a different result did different work;
			// its throughput is not comparable. Gate on equality and print N/A
			// otherwise (strip-ansi 0.1 leaves OSC-8 payloads behind and
			// unicode-width diverges on emoji presentation).
			let reference_matches = reference_width(input) == expected;

			(*name, vec![
				measure(input.len(), || width_ansi_str(input)),
				measure(input.len(), || {
					let stripped = input.to_ansi_stripped();
					stripped.visible_width()
				}),
				if reference_matches {
					measure(input.len(), || reference_width(input))
				} else {
					f64::NAN
				},
			])
		})
		.collect();

	print_ratio_table(
		"ANSI-aware visible width",
		&["xutf one-pass", "xutf two-pass", "strip-ansi + unicode-width"],
		&rows,
	);
}

fn divan_input() -> String {
	build_input(
		"\x1b[38;5;244mINFO\x1b[0m \x1b[1mrequest\x1b[0m completed in \x1b[32m12ms\x1b[0m\n",
		TARGET,
	)
}

#[divan::bench]
fn xutf_one_pass(bencher: Bencher) {
	let input = divan_input();
	bencher
		.counter(BytesCount::of_slice(input.as_bytes()))
		.bench_local(|| width_ansi_str(&input));
}

#[divan::bench]
fn xutf_two_pass(bencher: Bencher) {
	let input = divan_input();
	bencher
		.counter(BytesCount::of_slice(input.as_bytes()))
		.bench_local(|| {
			let stripped = input.to_ansi_stripped();
			stripped.visible_width()
		});
}

#[divan::bench]
fn strip_ansi_unicode_width(bencher: Bencher) {
	let input = divan_input();
	bencher
		.counter(BytesCount::of_slice(input.as_bytes()))
		.bench_local(|| reference_width(&input));
}