1use encoding_rs::{EncoderResult, Encoding, UTF_16BE, UTF_16LE, UTF_8};
4use runmat_builtins::{
5 BuiltinCompletionPolicy, BuiltinDescriptor, BuiltinErrorDescriptor, BuiltinOutputMode,
6 BuiltinParamArity, BuiltinParamDescriptor, BuiltinParamType, BuiltinSignatureDescriptor,
7 CharArray, NumericDType, StringArray, Tensor, Value,
8};
9use runmat_macros::runtime_builtin;
10
11use crate::builtins::common::map_control_flow_with_builtin;
12use crate::builtins::common::spec::{
13 BroadcastSemantics, BuiltinFusionSpec, BuiltinGpuSpec, ConstantStrategy, GpuOpKind,
14 ReductionNaN, ResidencyPolicy, ShapeRequirements,
15};
16use crate::builtins::strings::common::char_row_to_string;
17use crate::builtins::strings::type_resolvers::text_search_indices_type;
18use crate::{build_runtime_error, gather_if_needed_async, BuiltinResult, RuntimeError};
19
20const BUILTIN_NAME: &str = "unicode2native";
21
22const UNICODE2NATIVE_OUTPUT: [BuiltinParamDescriptor; 1] = [BuiltinParamDescriptor {
23 name: "bytes",
24 ty: BuiltinParamType::NumericArray,
25 arity: BuiltinParamArity::Required,
26 default: None,
27 description: "uint8 row vector containing encoded bytes.",
28}];
29
30const UNICODE2NATIVE_INPUT_TEXT: [BuiltinParamDescriptor; 1] = [BuiltinParamDescriptor {
31 name: "unicodestr",
32 ty: BuiltinParamType::StringScalar,
33 arity: BuiltinParamArity::Required,
34 default: None,
35 description: "Unicode text, provided as a string scalar or character vector.",
36}];
37
38const UNICODE2NATIVE_INPUT_TEXT_ENCODING: [BuiltinParamDescriptor; 2] = [
39 BuiltinParamDescriptor {
40 name: "unicodestr",
41 ty: BuiltinParamType::StringScalar,
42 arity: BuiltinParamArity::Required,
43 default: None,
44 description: "Unicode text, provided as a string scalar or character vector.",
45 },
46 BuiltinParamDescriptor {
47 name: "encoding",
48 ty: BuiltinParamType::StringScalar,
49 arity: BuiltinParamArity::Optional,
50 default: Some("\"UTF-8\""),
51 description: "Character encoding name.",
52 },
53];
54
55const UNICODE2NATIVE_SIGNATURES: [BuiltinSignatureDescriptor; 2] = [
56 BuiltinSignatureDescriptor {
57 label: "bytes = unicode2native(unicodestr)",
58 inputs: &UNICODE2NATIVE_INPUT_TEXT,
59 outputs: &UNICODE2NATIVE_OUTPUT,
60 },
61 BuiltinSignatureDescriptor {
62 label: "bytes = unicode2native(unicodestr, encoding)",
63 inputs: &UNICODE2NATIVE_INPUT_TEXT_ENCODING,
64 outputs: &UNICODE2NATIVE_OUTPUT,
65 },
66];
67
68const UNICODE2NATIVE_ERROR_INVALID_INPUT: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
69 code: "RM.UNICODE2NATIVE.INVALID_INPUT",
70 identifier: Some("RunMat:unicode2native:InvalidInput"),
71 when: "Input text is not a string scalar or character vector.",
72 message: "unicode2native: input must be a string scalar or character vector",
73};
74
75const UNICODE2NATIVE_ERROR_INVALID_ENCODING: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
76 code: "RM.UNICODE2NATIVE.INVALID_ENCODING",
77 identifier: Some("RunMat:unicode2native:InvalidEncoding"),
78 when: "Encoding name is unsupported or malformed.",
79 message: "unicode2native: unsupported character encoding",
80};
81
82const UNICODE2NATIVE_ERROR_INVALID_ARGUMENT: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
83 code: "RM.UNICODE2NATIVE.INVALID_ARGUMENT",
84 identifier: Some("RunMat:unicode2native:InvalidArgument"),
85 when: "Too many arguments were supplied.",
86 message: "unicode2native: invalid argument",
87};
88
89const UNICODE2NATIVE_ERROR_INTERNAL: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
90 code: "RM.UNICODE2NATIVE.INTERNAL",
91 identifier: Some("RunMat:unicode2native:InternalError"),
92 when: "Internal byte-vector tensor construction failed.",
93 message: "unicode2native: internal error",
94};
95
96const UNICODE2NATIVE_ERRORS: [BuiltinErrorDescriptor; 4] = [
97 UNICODE2NATIVE_ERROR_INVALID_INPUT,
98 UNICODE2NATIVE_ERROR_INVALID_ENCODING,
99 UNICODE2NATIVE_ERROR_INVALID_ARGUMENT,
100 UNICODE2NATIVE_ERROR_INTERNAL,
101];
102
103pub const UNICODE2NATIVE_DESCRIPTOR: BuiltinDescriptor = BuiltinDescriptor {
104 signatures: &UNICODE2NATIVE_SIGNATURES,
105 output_mode: BuiltinOutputMode::Fixed,
106 completion_policy: BuiltinCompletionPolicy::Public,
107 errors: &UNICODE2NATIVE_ERRORS,
108};
109
110#[runmat_macros::register_gpu_spec(builtin_path = "crate::builtins::strings::core::unicode2native")]
111pub const GPU_SPEC: BuiltinGpuSpec = BuiltinGpuSpec {
112 name: "unicode2native",
113 op_kind: GpuOpKind::Custom("text-encoding"),
114 supported_precisions: &[],
115 broadcast: BroadcastSemantics::None,
116 provider_hooks: &[],
117 constant_strategy: ConstantStrategy::InlineLiteral,
118 residency: ResidencyPolicy::GatherImmediately,
119 nan_mode: ReductionNaN::Include,
120 two_pass_threshold: None,
121 workgroup_size: None,
122 accepts_nan_mode: false,
123 notes: "Text encoding runs on the CPU; GPU-resident inputs are gathered before validation.",
124};
125
126#[runmat_macros::register_fusion_spec(
127 builtin_path = "crate::builtins::strings::core::unicode2native"
128)]
129pub const FUSION_SPEC: BuiltinFusionSpec = BuiltinFusionSpec {
130 name: "unicode2native",
131 shape: ShapeRequirements::Any,
132 constant_strategy: ConstantStrategy::InlineLiteral,
133 elementwise: None,
134 reduction: None,
135 emits_nan: false,
136 notes: "Conversion builtin; not eligible for fusion and always materialises host uint8 bytes.",
137};
138
139fn unicode2native_error(error: &'static BuiltinErrorDescriptor) -> RuntimeError {
140 unicode2native_error_with_message(error.message, error)
141}
142
143fn unicode2native_error_with_message(
144 message: impl Into<String>,
145 error: &'static BuiltinErrorDescriptor,
146) -> RuntimeError {
147 let mut builder = build_runtime_error(message).with_builtin(BUILTIN_NAME);
148 if let Some(identifier) = error.identifier {
149 builder = builder.with_identifier(identifier);
150 }
151 builder.build()
152}
153
154fn remap_unicode2native_flow(err: RuntimeError) -> RuntimeError {
155 map_control_flow_with_builtin(err, BUILTIN_NAME)
156}
157
158#[runtime_builtin(
159 name = "unicode2native",
160 category = "strings/core",
161 summary = "Convert Unicode text into encoded uint8 bytes.",
162 keywords = "unicode2native,encoding,utf-8,latin1,uint8,text",
163 examples = "bytes = unicode2native(\"hello\", \"UTF-8\");",
164 accel = "sink",
165 type_resolver(text_search_indices_type),
166 descriptor(crate::builtins::strings::core::unicode2native::UNICODE2NATIVE_DESCRIPTOR),
167 builtin_path = "crate::builtins::strings::core::unicode2native"
168)]
169async fn unicode2native_builtin(value: Value, rest: Vec<Value>) -> BuiltinResult<Value> {
170 if rest.len() > 1 {
171 return Err(unicode2native_error_with_message(
172 "unicode2native: too many input arguments",
173 &UNICODE2NATIVE_ERROR_INVALID_ARGUMENT,
174 ));
175 }
176
177 let gathered = gather_if_needed_async(&value)
178 .await
179 .map_err(remap_unicode2native_flow)?;
180 let text = value_to_text_scalar(&gathered)?;
181 let encoding = match rest.into_iter().next() {
182 Some(value) => {
183 let gathered = gather_if_needed_async(&value)
184 .await
185 .map_err(remap_unicode2native_flow)?;
186 EncodingSpec::parse(&value_to_encoding_name(&gathered)?)?
187 }
188 None => EncodingSpec::Encoding(UTF_8),
189 };
190
191 bytes_to_uint8_row(encoding.encode(&text)?)
192}
193
194#[derive(Clone, Copy)]
195enum EncodingSpec {
196 Encoding(&'static Encoding),
197 Ascii,
198 Latin1,
199 Utf32 { little_endian: bool, bom: bool },
200 Utf16WithBom,
201}
202
203impl EncodingSpec {
204 fn parse(name: &str) -> BuiltinResult<Self> {
205 let trimmed = name.trim();
206 if trimmed.is_empty() {
207 return Err(unicode2native_error_with_message(
208 "unicode2native: encoding name cannot be empty",
209 &UNICODE2NATIVE_ERROR_INVALID_ENCODING,
210 ));
211 }
212
213 let lowered = normalize_encoding_name(trimmed);
214 match lowered.as_str() {
215 "ascii" | "us-ascii" | "ansi_x3.4-1968" | "iso-ir-6" | "iso646-us" | "us" => {
216 return Ok(Self::Ascii);
217 }
218 "latin1" | "latin-1" | "iso-8859-1" | "iso8859-1" | "ibm819" | "cp819"
219 | "csisolatin1" | "l1" => return Ok(Self::Latin1),
220 "utf-16" | "utf16" => return Ok(Self::Utf16WithBom),
221 "utf-32" | "utf32" => {
222 return Ok(Self::Utf32 {
223 little_endian: true,
224 bom: true,
225 });
226 }
227 "utf-32le" | "utf32le" => {
228 return Ok(Self::Utf32 {
229 little_endian: true,
230 bom: false,
231 });
232 }
233 "utf-32be" | "utf32be" => {
234 return Ok(Self::Utf32 {
235 little_endian: false,
236 bom: false,
237 });
238 }
239 "unicode" | "system" => return Ok(Self::Encoding(UTF_8)),
240 _ => {}
241 }
242
243 Encoding::for_label(lowered.as_bytes())
244 .map(Self::Encoding)
245 .ok_or_else(|| {
246 unicode2native_error_with_message(
247 format!("unicode2native: unsupported character encoding '{trimmed}'"),
248 &UNICODE2NATIVE_ERROR_INVALID_ENCODING,
249 )
250 })
251 }
252
253 fn encode(self, text: &str) -> BuiltinResult<Vec<u8>> {
254 match self {
255 Self::Ascii => {
256 encode_single_byte(text, |code| if code <= 0x7F { code as u8 } else { b'?' })
257 }
258 Self::Latin1 => {
259 encode_single_byte(text, |code| if code <= 0xFF { code as u8 } else { b'?' })
260 }
261 Self::Encoding(encoding) if encoding == UTF_16LE => {
262 encode_utf16_without_bom(text, true)
263 }
264 Self::Encoding(encoding) if encoding == UTF_16BE => {
265 encode_utf16_without_bom(text, false)
266 }
267 Self::Encoding(encoding) => encode_with_question_replacement(encoding, text),
268 Self::Utf32 { little_endian, bom } => encode_utf32(text, little_endian, bom),
269 Self::Utf16WithBom => {
270 let capacity = text
271 .len()
272 .checked_mul(2)
273 .and_then(|len| len.checked_add(2))
274 .ok_or_else(|| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
275 let mut bytes = Vec::new();
276 bytes
277 .try_reserve_exact(capacity)
278 .map_err(|_| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
279 bytes.extend_from_slice(&[0xFF, 0xFE]);
280 bytes.extend(encode_utf16_without_bom(text, true)?);
281 Ok(bytes)
282 }
283 }
284 }
285}
286
287fn normalize_encoding_name(raw: &str) -> String {
288 let lowered = raw.to_ascii_lowercase();
289 match lowered.as_str() {
290 "utf_16" => "utf-16".to_string(),
291 "utf_16le" => "utf-16le".to_string(),
292 "utf_16be" => "utf-16be".to_string(),
293 "utf_32" => "utf-32".to_string(),
294 "utf_32le" => "utf-32le".to_string(),
295 "utf_32be" => "utf-32be".to_string(),
296 "windows_1252" => "windows-1252".to_string(),
297 "shift_jis" => "shift_jis".to_string(),
298 _ => lowered,
299 }
300}
301
302fn encode_single_byte<F>(text: &str, mut encode: F) -> BuiltinResult<Vec<u8>>
303where
304 F: FnMut(u32) -> u8,
305{
306 let mut bytes = Vec::new();
307 bytes
308 .try_reserve_exact(text.len())
309 .map_err(|_| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
310 bytes.extend(text.chars().map(|ch| encode(ch as u32)));
311 Ok(bytes)
312}
313
314fn encode_with_question_replacement(
315 encoding: &'static Encoding,
316 text: &str,
317) -> BuiltinResult<Vec<u8>> {
318 let mut encoder = encoding.new_encoder();
319 let capacity = encoder
320 .max_buffer_length_from_utf8_without_replacement(text.len())
321 .ok_or_else(|| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
322 let mut bytes = Vec::new();
323 bytes
324 .try_reserve_exact(capacity)
325 .map_err(|_| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
326
327 let mut remaining = text;
328 loop {
329 let chunk_capacity = encoder
330 .max_buffer_length_from_utf8_without_replacement(remaining.len())
331 .ok_or_else(|| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
332 let start = bytes.len();
333 bytes
334 .try_reserve_exact(chunk_capacity)
335 .map_err(|_| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
336 bytes.resize(start + chunk_capacity, 0);
337
338 let (result, read, written) =
339 encoder.encode_from_utf8_without_replacement(remaining, &mut bytes[start..], true);
340 bytes.truncate(start + written);
341 remaining = &remaining[read..];
342
343 match result {
344 EncoderResult::InputEmpty => return Ok(bytes),
345 EncoderResult::OutputFull => continue,
346 EncoderResult::Unmappable(ch) => {
347 bytes
348 .try_reserve_exact(1)
349 .map_err(|_| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
350 bytes.push(b'?');
351 if remaining.starts_with(ch) {
352 let skip = ch.len_utf8();
353 remaining = remaining.get(skip..).ok_or_else(|| {
354 unicode2native_error_with_message(
355 "unicode2native: internal encoding cursor error",
356 &UNICODE2NATIVE_ERROR_INTERNAL,
357 )
358 })?;
359 }
360 }
361 }
362 }
363}
364
365fn encode_utf16_without_bom(text: &str, little_endian: bool) -> BuiltinResult<Vec<u8>> {
366 let capacity = text
367 .len()
368 .checked_mul(2)
369 .ok_or_else(|| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
370 let mut bytes = Vec::new();
371 bytes
372 .try_reserve_exact(capacity)
373 .map_err(|_| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
374 for unit in text.encode_utf16() {
375 let encoded = if little_endian {
376 unit.to_le_bytes()
377 } else {
378 unit.to_be_bytes()
379 };
380 bytes.extend_from_slice(&encoded);
381 }
382 Ok(bytes)
383}
384
385fn encode_utf32(text: &str, little_endian: bool, bom: bool) -> BuiltinResult<Vec<u8>> {
386 let char_count = text.chars().count();
387 let capacity = char_count
388 .checked_add(usize::from(bom))
389 .and_then(|units| units.checked_mul(4))
390 .ok_or_else(|| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
391 let mut bytes = Vec::new();
392 bytes
393 .try_reserve_exact(capacity)
394 .map_err(|_| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
395 if bom {
396 if little_endian {
397 bytes.extend_from_slice(&[0xFF, 0xFE, 0x00, 0x00]);
398 } else {
399 bytes.extend_from_slice(&[0x00, 0x00, 0xFE, 0xFF]);
400 }
401 }
402 for ch in text.chars() {
403 let encoded = if little_endian {
404 (ch as u32).to_le_bytes()
405 } else {
406 (ch as u32).to_be_bytes()
407 };
408 bytes.extend_from_slice(&encoded);
409 }
410 Ok(bytes)
411}
412
413fn value_to_text_scalar(value: &Value) -> BuiltinResult<String> {
414 match value {
415 Value::String(text) => Ok(text.clone()),
416 Value::StringArray(array) => string_array_scalar(array),
417 Value::CharArray(array) if array.rows <= 1 => Ok(char_row_or_empty(array)),
418 Value::CharArray(_) => Err(unicode2native_error_with_message(
419 "unicode2native: character input must be a row vector",
420 &UNICODE2NATIVE_ERROR_INVALID_INPUT,
421 )),
422 _ => Err(unicode2native_error(&UNICODE2NATIVE_ERROR_INVALID_INPUT)),
423 }
424}
425
426fn value_to_encoding_name(value: &Value) -> BuiltinResult<String> {
427 match value {
428 Value::String(text) => Ok(text.clone()),
429 Value::StringArray(array) => string_array_scalar(array),
430 Value::CharArray(array) if array.rows <= 1 => Ok(char_row_or_empty(array)),
431 Value::CharArray(_) => Err(unicode2native_error_with_message(
432 "unicode2native: encoding must be a character row vector",
433 &UNICODE2NATIVE_ERROR_INVALID_ENCODING,
434 )),
435 _ => Err(unicode2native_error_with_message(
436 "unicode2native: encoding must be a string scalar or character vector",
437 &UNICODE2NATIVE_ERROR_INVALID_ENCODING,
438 )),
439 }
440}
441
442fn string_array_scalar(array: &StringArray) -> BuiltinResult<String> {
443 if array.data.len() == 1 {
444 Ok(array.data[0].clone())
445 } else {
446 Err(unicode2native_error_with_message(
447 "unicode2native: string input must be scalar",
448 &UNICODE2NATIVE_ERROR_INVALID_INPUT,
449 ))
450 }
451}
452
453fn char_row_or_empty(array: &CharArray) -> String {
454 if array.rows == 0 {
455 String::new()
456 } else {
457 char_row_to_string(array, 0)
458 }
459}
460
461fn bytes_to_uint8_row(bytes: Vec<u8>) -> BuiltinResult<Value> {
462 let len = bytes.len();
463 let mut data = Vec::new();
464 data.try_reserve_exact(len)
465 .map_err(|_| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
466 data.extend(bytes.into_iter().map(f64::from));
467 let tensor = Tensor::new_with_dtype(data, vec![1, len], NumericDType::U8)
468 .map_err(|_| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
469 Ok(Value::Tensor(tensor))
470}
471
472#[cfg(test)]
473pub(crate) mod tests {
474 use super::*;
475 use futures::executor::block_on;
476 use runmat_builtins::{ResolveContext, Type};
477
478 fn unicode2native_builtin(value: Value, rest: Vec<Value>) -> BuiltinResult<Value> {
479 block_on(super::unicode2native_builtin(value, rest))
480 }
481
482 fn tensor_bytes(value: Value) -> (Vec<u8>, Vec<usize>, NumericDType) {
483 match value {
484 Value::Tensor(tensor) => {
485 let bytes = tensor.data.iter().map(|value| *value as u8).collect();
486 (bytes, tensor.shape, tensor.dtype)
487 }
488 other => panic!("expected uint8 tensor, got {other:?}"),
489 }
490 }
491
492 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
493 #[test]
494 fn unicode2native_defaults_to_utf8_row_vector() {
495 let (bytes, shape, dtype) =
496 tensor_bytes(unicode2native_builtin(Value::String("hé".into()), vec![]).unwrap());
497 assert_eq!(bytes, vec![104, 195, 169]);
498 assert_eq!(shape, vec![1, 3]);
499 assert_eq!(dtype, NumericDType::U8);
500 }
501
502 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
503 #[test]
504 fn unicode2native_accepts_char_vector_and_latin1_alias() {
505 let text = Value::CharArray(CharArray::new_row("café"));
506 let encoding = Value::CharArray(CharArray::new_row("latin1"));
507 let (bytes, shape, dtype) =
508 tensor_bytes(unicode2native_builtin(text, vec![encoding]).unwrap());
509 assert_eq!(bytes, vec![99, 97, 102, 233]);
510 assert_eq!(shape, vec![1, 4]);
511 assert_eq!(dtype, NumericDType::U8);
512 }
513
514 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
515 #[test]
516 fn unicode2native_accepts_shift_jis_aliases() {
517 let encoding = Value::String("Shift_JIS".into());
518 let (bytes, shape, dtype) = tensor_bytes(
519 unicode2native_builtin(Value::String("漢".into()), vec![encoding]).unwrap(),
520 );
521 assert_eq!(bytes, vec![138, 191]);
522 assert_eq!(shape, vec![1, 2]);
523 assert_eq!(dtype, NumericDType::U8);
524 }
525
526 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
527 #[test]
528 fn unicode2native_utf16_emits_little_endian_bom() {
529 let (bytes, shape, dtype) = tensor_bytes(
530 unicode2native_builtin(
531 Value::String("hi".into()),
532 vec![Value::String("UTF-16".into())],
533 )
534 .unwrap(),
535 );
536 assert_eq!(bytes, vec![255, 254, 104, 0, 105, 0]);
537 assert_eq!(shape, vec![1, 6]);
538 assert_eq!(dtype, NumericDType::U8);
539 }
540
541 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
542 #[test]
543 fn unicode2native_utf16le_and_utf16be_skip_bom() {
544 let (le, le_shape, le_dtype) = tensor_bytes(
545 unicode2native_builtin(
546 Value::String("A".into()),
547 vec![Value::String("UTF-16LE".into())],
548 )
549 .unwrap(),
550 );
551 assert_eq!(le, vec![65, 0]);
552 assert_eq!(le_shape, vec![1, 2]);
553 assert_eq!(le_dtype, NumericDType::U8);
554
555 let (be, be_shape, be_dtype) = tensor_bytes(
556 unicode2native_builtin(
557 Value::String("A".into()),
558 vec![Value::String("UTF-16BE".into())],
559 )
560 .unwrap(),
561 );
562 assert_eq!(be, vec![0, 65]);
563 assert_eq!(be_shape, vec![1, 2]);
564 assert_eq!(be_dtype, NumericDType::U8);
565 }
566
567 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
568 #[test]
569 fn unicode2native_ascii_replaces_unencodable_text() {
570 let (bytes, shape, dtype) = tensor_bytes(
571 unicode2native_builtin(
572 Value::String("é".into()),
573 vec![Value::String("US-ASCII".into())],
574 )
575 .unwrap(),
576 );
577 assert_eq!(bytes, vec![63]);
578 assert_eq!(shape, vec![1, 1]);
579 assert_eq!(dtype, NumericDType::U8);
580 }
581
582 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
583 #[test]
584 fn unicode2native_windows1252_replaces_unencodable_text_with_question_mark() {
585 let (bytes, shape, dtype) = tensor_bytes(
586 unicode2native_builtin(
587 Value::String("€☃".into()),
588 vec![Value::String("windows_1252".into())],
589 )
590 .unwrap(),
591 );
592 assert_eq!(bytes, vec![128, 63]);
593 assert_eq!(shape, vec![1, 2]);
594 assert_eq!(dtype, NumericDType::U8);
595 }
596
597 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
598 #[test]
599 fn unicode2native_utf32_aliases_encode_unicode_code_points() {
600 let (bytes, shape, dtype) = tensor_bytes(
601 unicode2native_builtin(
602 Value::String("A".into()),
603 vec![Value::String("UTF_32".into())],
604 )
605 .unwrap(),
606 );
607 assert_eq!(bytes, vec![255, 254, 0, 0, 65, 0, 0, 0]);
608 assert_eq!(shape, vec![1, 8]);
609 assert_eq!(dtype, NumericDType::U8);
610
611 let (be, be_shape, be_dtype) = tensor_bytes(
612 unicode2native_builtin(
613 Value::String("A".into()),
614 vec![Value::String("UTF-32BE".into())],
615 )
616 .unwrap(),
617 );
618 assert_eq!(be, vec![0, 0, 0, 65]);
619 assert_eq!(be_shape, vec![1, 4]);
620 assert_eq!(be_dtype, NumericDType::U8);
621 }
622
623 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
624 #[test]
625 fn unicode2native_empty_text_preserves_uint8_empty_row() {
626 let (bytes, shape, dtype) =
627 tensor_bytes(unicode2native_builtin(Value::String(String::new()), vec![]).unwrap());
628 assert!(bytes.is_empty());
629 assert_eq!(shape, vec![1, 0]);
630 assert_eq!(dtype, NumericDType::U8);
631 }
632
633 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
634 #[test]
635 fn unicode2native_rejects_string_arrays_that_are_not_scalar() {
636 let array = StringArray::new(vec!["a".into(), "b".into()], vec![1, 2]).unwrap();
637 let err = unicode2native_builtin(Value::StringArray(array), vec![]).unwrap_err();
638 assert_eq!(err.identifier(), Some("RunMat:unicode2native:InvalidInput"));
639 }
640
641 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
642 #[test]
643 fn unicode2native_rejects_char_matrices() {
644 let chars = CharArray::new(vec!['a', 'b', 'c', 'd'], 2, 2).unwrap();
645 let err = unicode2native_builtin(Value::CharArray(chars), vec![]).unwrap_err();
646 assert_eq!(err.identifier(), Some("RunMat:unicode2native:InvalidInput"));
647 }
648
649 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
650 #[test]
651 fn unicode2native_rejects_unknown_encoding() {
652 let err = unicode2native_builtin(
653 Value::String("x".into()),
654 vec![Value::String("not-real".into())],
655 )
656 .unwrap_err();
657 assert_eq!(
658 err.identifier(),
659 Some("RunMat:unicode2native:InvalidEncoding")
660 );
661 }
662
663 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
664 #[test]
665 fn unicode2native_rejects_extra_arguments() {
666 let err = unicode2native_builtin(
667 Value::String("x".into()),
668 vec![Value::String("UTF-8".into()), Value::String("extra".into())],
669 )
670 .unwrap_err();
671 assert_eq!(
672 err.identifier(),
673 Some("RunMat:unicode2native:InvalidArgument")
674 );
675 }
676
677 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
678 #[test]
679 fn unicode2native_descriptor_covers_documented_forms() {
680 let labels: Vec<&str> = UNICODE2NATIVE_DESCRIPTOR
681 .signatures
682 .iter()
683 .map(|sig| sig.label)
684 .collect();
685 assert_eq!(
686 labels,
687 vec![
688 "bytes = unicode2native(unicodestr)",
689 "bytes = unicode2native(unicodestr, encoding)"
690 ]
691 );
692 }
693
694 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
695 #[test]
696 fn unicode2native_type_resolver_returns_tensor_for_text() {
697 let out = text_search_indices_type(&[Type::String], &ResolveContext::new(Vec::new()));
698 assert_eq!(out, Type::tensor());
699 }
700}