1use encoding_rs::{EncoderResult, Encoding, UTF_16BE, UTF_16LE, UTF_8};
4use runmat_builtins::{
5 BuiltinCompletionPolicy, BuiltinDescriptor, BuiltinErrorDescriptor, BuiltinOutputMode,
6 BuiltinParamArity, BuiltinParamDescriptor, BuiltinParamType, BuiltinSignatureDescriptor,
7};
8use runmat_macros::runtime_builtin;
9use runmat_value::{CharArray, IntegerStorage, StringArray, Tensor, Value};
10
11use crate::builtins::common::map_control_flow_with_builtin;
12use crate::builtins::common::spec::{
13 BroadcastSemantics, BuiltinFusionSpec, BuiltinGpuSpec, ConstantStrategy, GpuOpKind,
14 ReductionNaN, ResidencyPolicy, ShapeRequirements,
15};
16use crate::builtins::strings::common::char_row_to_string;
17use crate::builtins::strings::type_resolvers::text_search_indices_type;
18use crate::{build_runtime_error, gather_if_needed_async, BuiltinResult, RuntimeError};
19
20const BUILTIN_NAME: &str = "unicode2native";
21
22const UNICODE2NATIVE_OUTPUT: [BuiltinParamDescriptor; 1] = [BuiltinParamDescriptor {
23 name: "bytes",
24 ty: BuiltinParamType::NumericArray,
25 arity: BuiltinParamArity::Required,
26 default: None,
27 description: "uint8 row vector containing encoded bytes.",
28}];
29
30const UNICODE2NATIVE_INPUT_TEXT: [BuiltinParamDescriptor; 1] = [BuiltinParamDescriptor {
31 name: "unicodestr",
32 ty: BuiltinParamType::StringScalar,
33 arity: BuiltinParamArity::Required,
34 default: None,
35 description: "Unicode text, provided as a string scalar or character vector.",
36}];
37
38const UNICODE2NATIVE_INPUT_TEXT_ENCODING: [BuiltinParamDescriptor; 2] = [
39 BuiltinParamDescriptor {
40 name: "unicodestr",
41 ty: BuiltinParamType::StringScalar,
42 arity: BuiltinParamArity::Required,
43 default: None,
44 description: "Unicode text, provided as a string scalar or character vector.",
45 },
46 BuiltinParamDescriptor {
47 name: "encoding",
48 ty: BuiltinParamType::StringScalar,
49 arity: BuiltinParamArity::Optional,
50 default: Some("\"UTF-8\""),
51 description: "Character encoding name.",
52 },
53];
54
55const UNICODE2NATIVE_SIGNATURES: [BuiltinSignatureDescriptor; 2] = [
56 BuiltinSignatureDescriptor {
57 label: "bytes = unicode2native(unicodestr)",
58 inputs: &UNICODE2NATIVE_INPUT_TEXT,
59 outputs: &UNICODE2NATIVE_OUTPUT,
60 },
61 BuiltinSignatureDescriptor {
62 label: "bytes = unicode2native(unicodestr, encoding)",
63 inputs: &UNICODE2NATIVE_INPUT_TEXT_ENCODING,
64 outputs: &UNICODE2NATIVE_OUTPUT,
65 },
66];
67
68const UNICODE2NATIVE_ERROR_INVALID_INPUT: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
69 code: "RM.UNICODE2NATIVE.INVALID_INPUT",
70 identifier: Some("RunMat:unicode2native:InvalidInput"),
71 when: "Input text is not a string scalar or character vector.",
72 message: "unicode2native: input must be a string scalar or character vector",
73};
74
75const UNICODE2NATIVE_ERROR_INVALID_ENCODING: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
76 code: "RM.UNICODE2NATIVE.INVALID_ENCODING",
77 identifier: Some("RunMat:unicode2native:InvalidEncoding"),
78 when: "Encoding name is unsupported or malformed.",
79 message: "unicode2native: unsupported character encoding",
80};
81
82const UNICODE2NATIVE_ERROR_INVALID_ARGUMENT: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
83 code: "RM.UNICODE2NATIVE.INVALID_ARGUMENT",
84 identifier: Some("RunMat:unicode2native:InvalidArgument"),
85 when: "Too many arguments were supplied.",
86 message: "unicode2native: invalid argument",
87};
88
89const UNICODE2NATIVE_ERROR_INTERNAL: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
90 code: "RM.UNICODE2NATIVE.INTERNAL",
91 identifier: Some("RunMat:unicode2native:InternalError"),
92 when: "Internal byte-vector tensor construction failed.",
93 message: "unicode2native: internal error",
94};
95
96const UNICODE2NATIVE_ERRORS: [BuiltinErrorDescriptor; 4] = [
97 UNICODE2NATIVE_ERROR_INVALID_INPUT,
98 UNICODE2NATIVE_ERROR_INVALID_ENCODING,
99 UNICODE2NATIVE_ERROR_INVALID_ARGUMENT,
100 UNICODE2NATIVE_ERROR_INTERNAL,
101];
102
103pub const UNICODE2NATIVE_DESCRIPTOR: BuiltinDescriptor = BuiltinDescriptor {
104 signatures: &UNICODE2NATIVE_SIGNATURES,
105 output_mode: BuiltinOutputMode::Fixed,
106 completion_policy: BuiltinCompletionPolicy::Public,
107 errors: &UNICODE2NATIVE_ERRORS,
108};
109
110#[runmat_macros::register_gpu_spec(builtin_path = "crate::builtins::strings::core::unicode2native")]
111pub const GPU_SPEC: BuiltinGpuSpec = BuiltinGpuSpec {
112 name: "unicode2native",
113 op_kind: GpuOpKind::Custom("text-encoding"),
114 supported_precisions: &[],
115 broadcast: BroadcastSemantics::None,
116 provider_hooks: &[],
117 constant_strategy: ConstantStrategy::InlineLiteral,
118 residency: ResidencyPolicy::GatherImmediately,
119 nan_mode: ReductionNaN::Include,
120 two_pass_threshold: None,
121 workgroup_size: None,
122 accepts_nan_mode: false,
123 notes: "Text encoding runs on the CPU; GPU-resident inputs are gathered before validation.",
124};
125
126#[runmat_macros::register_fusion_spec(
127 builtin_path = "crate::builtins::strings::core::unicode2native"
128)]
129pub const FUSION_SPEC: BuiltinFusionSpec = BuiltinFusionSpec {
130 name: "unicode2native",
131 shape: ShapeRequirements::Any,
132 constant_strategy: ConstantStrategy::InlineLiteral,
133 elementwise: None,
134 reduction: None,
135 emits_nan: false,
136 notes: "Conversion builtin; not eligible for fusion and always materialises host uint8 bytes.",
137};
138
139fn unicode2native_error(error: &'static BuiltinErrorDescriptor) -> RuntimeError {
140 unicode2native_error_with_message(error.message, error)
141}
142
143fn unicode2native_error_with_message(
144 message: impl Into<String>,
145 error: &'static BuiltinErrorDescriptor,
146) -> RuntimeError {
147 let mut builder = build_runtime_error(message).with_builtin(BUILTIN_NAME);
148 if let Some(identifier) = error.identifier {
149 builder = builder.with_identifier(identifier);
150 }
151 builder.build()
152}
153
154fn remap_unicode2native_flow(err: RuntimeError) -> RuntimeError {
155 map_control_flow_with_builtin(err, BUILTIN_NAME)
156}
157
158#[runtime_builtin(
159 name = "unicode2native",
160 category = "strings/core",
161 summary = "Convert Unicode text into encoded uint8 bytes.",
162 keywords = "unicode2native,encoding,utf-8,latin1,uint8,text",
163 examples = "bytes = unicode2native(\"hello\", \"UTF-8\");",
164 accel = "sink",
165 type_resolver(text_search_indices_type),
166 descriptor(crate::builtins::strings::core::unicode2native::UNICODE2NATIVE_DESCRIPTOR),
167 builtin_path = "crate::builtins::strings::core::unicode2native"
168)]
169async fn unicode2native_builtin(value: Value, rest: Vec<Value>) -> BuiltinResult<Value> {
170 if rest.len() > 1 {
171 return Err(unicode2native_error_with_message(
172 "unicode2native: too many input arguments",
173 &UNICODE2NATIVE_ERROR_INVALID_ARGUMENT,
174 ));
175 }
176
177 let gathered = gather_if_needed_async(&value)
178 .await
179 .map_err(remap_unicode2native_flow)?;
180 let text = value_to_text_scalar(&gathered)?;
181 let encoding = match rest.into_iter().next() {
182 Some(value) => {
183 let gathered = gather_if_needed_async(&value)
184 .await
185 .map_err(remap_unicode2native_flow)?;
186 EncodingSpec::parse(&value_to_encoding_name(&gathered)?)?
187 }
188 None => EncodingSpec::Encoding(UTF_8),
189 };
190
191 bytes_to_uint8_row(encoding.encode(&text)?)
192}
193
194#[derive(Clone, Copy)]
195enum EncodingSpec {
196 Encoding(&'static Encoding),
197 Ascii,
198 Latin1,
199 Utf32 { little_endian: bool, bom: bool },
200 Utf16WithBom,
201}
202
203impl EncodingSpec {
204 fn parse(name: &str) -> BuiltinResult<Self> {
205 let trimmed = name.trim();
206 if trimmed.is_empty() {
207 return Err(unicode2native_error_with_message(
208 "unicode2native: encoding name cannot be empty",
209 &UNICODE2NATIVE_ERROR_INVALID_ENCODING,
210 ));
211 }
212
213 let lowered = normalize_encoding_name(trimmed);
214 match lowered.as_str() {
215 "ascii" | "us-ascii" | "ansi_x3.4-1968" | "iso-ir-6" | "iso646-us" | "us" => {
216 return Ok(Self::Ascii);
217 }
218 "latin1" | "latin-1" | "iso-8859-1" | "iso8859-1" | "ibm819" | "cp819"
219 | "csisolatin1" | "l1" => return Ok(Self::Latin1),
220 "utf-16" | "utf16" => return Ok(Self::Utf16WithBom),
221 "utf-32" | "utf32" => {
222 return Ok(Self::Utf32 {
223 little_endian: true,
224 bom: true,
225 });
226 }
227 "utf-32le" | "utf32le" => {
228 return Ok(Self::Utf32 {
229 little_endian: true,
230 bom: false,
231 });
232 }
233 "utf-32be" | "utf32be" => {
234 return Ok(Self::Utf32 {
235 little_endian: false,
236 bom: false,
237 });
238 }
239 "unicode" | "system" => return Ok(Self::Encoding(UTF_8)),
240 _ => {}
241 }
242
243 Encoding::for_label(lowered.as_bytes())
244 .map(Self::Encoding)
245 .ok_or_else(|| {
246 unicode2native_error_with_message(
247 format!("unicode2native: unsupported character encoding '{trimmed}'"),
248 &UNICODE2NATIVE_ERROR_INVALID_ENCODING,
249 )
250 })
251 }
252
253 fn encode(self, text: &str) -> BuiltinResult<Vec<u8>> {
254 match self {
255 Self::Ascii => {
256 encode_single_byte(text, |code| if code <= 0x7F { code as u8 } else { b'?' })
257 }
258 Self::Latin1 => {
259 encode_single_byte(text, |code| if code <= 0xFF { code as u8 } else { b'?' })
260 }
261 Self::Encoding(encoding) if encoding == UTF_16LE => {
262 encode_utf16_without_bom(text, true)
263 }
264 Self::Encoding(encoding) if encoding == UTF_16BE => {
265 encode_utf16_without_bom(text, false)
266 }
267 Self::Encoding(encoding) => encode_with_question_replacement(encoding, text),
268 Self::Utf32 { little_endian, bom } => encode_utf32(text, little_endian, bom),
269 Self::Utf16WithBom => {
270 let capacity = text
271 .len()
272 .checked_mul(2)
273 .and_then(|len| len.checked_add(2))
274 .ok_or_else(|| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
275 let mut bytes = Vec::new();
276 bytes
277 .try_reserve_exact(capacity)
278 .map_err(|_| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
279 bytes.extend_from_slice(&[0xFF, 0xFE]);
280 bytes.extend(encode_utf16_without_bom(text, true)?);
281 Ok(bytes)
282 }
283 }
284 }
285}
286
287fn normalize_encoding_name(raw: &str) -> String {
288 let lowered = raw.to_ascii_lowercase();
289 match lowered.as_str() {
290 "utf_16" => "utf-16".to_string(),
291 "utf_16le" => "utf-16le".to_string(),
292 "utf_16be" => "utf-16be".to_string(),
293 "utf_32" => "utf-32".to_string(),
294 "utf_32le" => "utf-32le".to_string(),
295 "utf_32be" => "utf-32be".to_string(),
296 "windows_1252" => "windows-1252".to_string(),
297 "shift_jis" => "shift_jis".to_string(),
298 _ => lowered,
299 }
300}
301
302fn encode_single_byte<F>(text: &str, mut encode: F) -> BuiltinResult<Vec<u8>>
303where
304 F: FnMut(u32) -> u8,
305{
306 let mut bytes = Vec::new();
307 bytes
308 .try_reserve_exact(text.len())
309 .map_err(|_| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
310 bytes.extend(text.chars().map(|ch| encode(ch as u32)));
311 Ok(bytes)
312}
313
314fn encode_with_question_replacement(
315 encoding: &'static Encoding,
316 text: &str,
317) -> BuiltinResult<Vec<u8>> {
318 let mut encoder = encoding.new_encoder();
319 let capacity = encoder
320 .max_buffer_length_from_utf8_without_replacement(text.len())
321 .ok_or_else(|| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
322 let mut bytes = Vec::new();
323 bytes
324 .try_reserve_exact(capacity)
325 .map_err(|_| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
326
327 let mut remaining = text;
328 loop {
329 let chunk_capacity = encoder
330 .max_buffer_length_from_utf8_without_replacement(remaining.len())
331 .ok_or_else(|| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
332 let start = bytes.len();
333 bytes
334 .try_reserve_exact(chunk_capacity)
335 .map_err(|_| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
336 bytes.resize(start + chunk_capacity, 0);
337
338 let (result, read, written) =
339 encoder.encode_from_utf8_without_replacement(remaining, &mut bytes[start..], true);
340 bytes.truncate(start + written);
341 remaining = &remaining[read..];
342
343 match result {
344 EncoderResult::InputEmpty => return Ok(bytes),
345 EncoderResult::OutputFull => continue,
346 EncoderResult::Unmappable(ch) => {
347 bytes
348 .try_reserve_exact(1)
349 .map_err(|_| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
350 bytes.push(b'?');
351 if remaining.starts_with(ch) {
352 let skip = ch.len_utf8();
353 remaining = remaining.get(skip..).ok_or_else(|| {
354 unicode2native_error_with_message(
355 "unicode2native: internal encoding cursor error",
356 &UNICODE2NATIVE_ERROR_INTERNAL,
357 )
358 })?;
359 }
360 }
361 }
362 }
363}
364
365fn encode_utf16_without_bom(text: &str, little_endian: bool) -> BuiltinResult<Vec<u8>> {
366 let capacity = text
367 .len()
368 .checked_mul(2)
369 .ok_or_else(|| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
370 let mut bytes = Vec::new();
371 bytes
372 .try_reserve_exact(capacity)
373 .map_err(|_| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
374 for unit in text.encode_utf16() {
375 let encoded = if little_endian {
376 unit.to_le_bytes()
377 } else {
378 unit.to_be_bytes()
379 };
380 bytes.extend_from_slice(&encoded);
381 }
382 Ok(bytes)
383}
384
385fn encode_utf32(text: &str, little_endian: bool, bom: bool) -> BuiltinResult<Vec<u8>> {
386 let char_count = text.chars().count();
387 let capacity = char_count
388 .checked_add(usize::from(bom))
389 .and_then(|units| units.checked_mul(4))
390 .ok_or_else(|| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
391 let mut bytes = Vec::new();
392 bytes
393 .try_reserve_exact(capacity)
394 .map_err(|_| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
395 if bom {
396 if little_endian {
397 bytes.extend_from_slice(&[0xFF, 0xFE, 0x00, 0x00]);
398 } else {
399 bytes.extend_from_slice(&[0x00, 0x00, 0xFE, 0xFF]);
400 }
401 }
402 for ch in text.chars() {
403 let encoded = if little_endian {
404 (ch as u32).to_le_bytes()
405 } else {
406 (ch as u32).to_be_bytes()
407 };
408 bytes.extend_from_slice(&encoded);
409 }
410 Ok(bytes)
411}
412
413fn value_to_text_scalar(value: &Value) -> BuiltinResult<String> {
414 match value {
415 Value::String(text) => Ok(text.clone()),
416 Value::StringArray(array) => string_array_scalar(array),
417 Value::CharArray(array) if array.rows <= 1 => Ok(char_row_or_empty(array)),
418 Value::CharArray(_) => Err(unicode2native_error_with_message(
419 "unicode2native: character input must be a row vector",
420 &UNICODE2NATIVE_ERROR_INVALID_INPUT,
421 )),
422 _ => Err(unicode2native_error(&UNICODE2NATIVE_ERROR_INVALID_INPUT)),
423 }
424}
425
426fn value_to_encoding_name(value: &Value) -> BuiltinResult<String> {
427 match value {
428 Value::String(text) => Ok(text.clone()),
429 Value::StringArray(array) => string_array_scalar(array),
430 Value::CharArray(array) if array.rows <= 1 => Ok(char_row_or_empty(array)),
431 Value::CharArray(_) => Err(unicode2native_error_with_message(
432 "unicode2native: encoding must be a character row vector",
433 &UNICODE2NATIVE_ERROR_INVALID_ENCODING,
434 )),
435 _ => Err(unicode2native_error_with_message(
436 "unicode2native: encoding must be a string scalar or character vector",
437 &UNICODE2NATIVE_ERROR_INVALID_ENCODING,
438 )),
439 }
440}
441
442fn string_array_scalar(array: &StringArray) -> BuiltinResult<String> {
443 if array.data.len() == 1 {
444 Ok(array.data[0].clone())
445 } else {
446 Err(unicode2native_error_with_message(
447 "unicode2native: string input must be scalar",
448 &UNICODE2NATIVE_ERROR_INVALID_INPUT,
449 ))
450 }
451}
452
453fn char_row_or_empty(array: &CharArray) -> String {
454 if array.rows == 0 {
455 String::new()
456 } else {
457 char_row_to_string(array, 0)
458 }
459}
460
461fn bytes_to_uint8_row(bytes: Vec<u8>) -> BuiltinResult<Value> {
462 let len = bytes.len();
463 let tensor = Tensor::new_integer(IntegerStorage::U8(bytes), vec![1, len])
464 .map_err(|_| unicode2native_error(&UNICODE2NATIVE_ERROR_INTERNAL))?;
465 Ok(Value::Tensor(tensor))
466}
467
468#[cfg(test)]
469pub(crate) mod tests {
470 use super::*;
471 use futures::executor::block_on;
472 use runmat_builtins::{ResolveContext, Type};
473 use runmat_value::NumericDType;
474
475 fn unicode2native_builtin(value: Value, rest: Vec<Value>) -> BuiltinResult<Value> {
476 block_on(super::unicode2native_builtin(value, rest))
477 }
478
479 fn tensor_bytes(value: Value) -> (Vec<u8>, Vec<usize>, NumericDType) {
480 match value {
481 Value::Tensor(tensor) => {
482 let bytes = match tensor.integer_storage() {
483 Some(IntegerStorage::U8(bytes)) => bytes.clone(),
484 other => panic!("expected native uint8 storage, got {other:?}"),
485 };
486 let dtype = tensor.numeric_dtype();
487 (bytes, tensor.shape, dtype)
488 }
489 other => panic!("expected uint8 tensor, got {other:?}"),
490 }
491 }
492
493 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
494 #[test]
495 fn unicode2native_defaults_to_utf8_row_vector() {
496 let (bytes, shape, dtype) =
497 tensor_bytes(unicode2native_builtin(Value::String("hé".into()), vec![]).unwrap());
498 assert_eq!(bytes, vec![104, 195, 169]);
499 assert_eq!(shape, vec![1, 3]);
500 assert_eq!(dtype, NumericDType::U8);
501 }
502
503 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
504 #[test]
505 fn unicode2native_accepts_char_vector_and_latin1_alias() {
506 let text = Value::CharArray(CharArray::new_row("café"));
507 let encoding = Value::CharArray(CharArray::new_row("latin1"));
508 let (bytes, shape, dtype) =
509 tensor_bytes(unicode2native_builtin(text, vec![encoding]).unwrap());
510 assert_eq!(bytes, vec![99, 97, 102, 233]);
511 assert_eq!(shape, vec![1, 4]);
512 assert_eq!(dtype, NumericDType::U8);
513 }
514
515 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
516 #[test]
517 fn unicode2native_accepts_shift_jis_aliases() {
518 let encoding = Value::String("Shift_JIS".into());
519 let (bytes, shape, dtype) = tensor_bytes(
520 unicode2native_builtin(Value::String("漢".into()), vec![encoding]).unwrap(),
521 );
522 assert_eq!(bytes, vec![138, 191]);
523 assert_eq!(shape, vec![1, 2]);
524 assert_eq!(dtype, NumericDType::U8);
525 }
526
527 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
528 #[test]
529 fn unicode2native_utf16_emits_little_endian_bom() {
530 let (bytes, shape, dtype) = tensor_bytes(
531 unicode2native_builtin(
532 Value::String("hi".into()),
533 vec![Value::String("UTF-16".into())],
534 )
535 .unwrap(),
536 );
537 assert_eq!(bytes, vec![255, 254, 104, 0, 105, 0]);
538 assert_eq!(shape, vec![1, 6]);
539 assert_eq!(dtype, NumericDType::U8);
540 }
541
542 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
543 #[test]
544 fn unicode2native_utf16le_and_utf16be_skip_bom() {
545 let (le, le_shape, le_dtype) = tensor_bytes(
546 unicode2native_builtin(
547 Value::String("A".into()),
548 vec![Value::String("UTF-16LE".into())],
549 )
550 .unwrap(),
551 );
552 assert_eq!(le, vec![65, 0]);
553 assert_eq!(le_shape, vec![1, 2]);
554 assert_eq!(le_dtype, NumericDType::U8);
555
556 let (be, be_shape, be_dtype) = tensor_bytes(
557 unicode2native_builtin(
558 Value::String("A".into()),
559 vec![Value::String("UTF-16BE".into())],
560 )
561 .unwrap(),
562 );
563 assert_eq!(be, vec![0, 65]);
564 assert_eq!(be_shape, vec![1, 2]);
565 assert_eq!(be_dtype, NumericDType::U8);
566 }
567
568 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
569 #[test]
570 fn unicode2native_ascii_replaces_unencodable_text() {
571 let (bytes, shape, dtype) = tensor_bytes(
572 unicode2native_builtin(
573 Value::String("é".into()),
574 vec![Value::String("US-ASCII".into())],
575 )
576 .unwrap(),
577 );
578 assert_eq!(bytes, vec![63]);
579 assert_eq!(shape, vec![1, 1]);
580 assert_eq!(dtype, NumericDType::U8);
581 }
582
583 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
584 #[test]
585 fn unicode2native_windows1252_replaces_unencodable_text_with_question_mark() {
586 let (bytes, shape, dtype) = tensor_bytes(
587 unicode2native_builtin(
588 Value::String("€☃".into()),
589 vec![Value::String("windows_1252".into())],
590 )
591 .unwrap(),
592 );
593 assert_eq!(bytes, vec![128, 63]);
594 assert_eq!(shape, vec![1, 2]);
595 assert_eq!(dtype, NumericDType::U8);
596 }
597
598 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
599 #[test]
600 fn unicode2native_utf32_aliases_encode_unicode_code_points() {
601 let (bytes, shape, dtype) = tensor_bytes(
602 unicode2native_builtin(
603 Value::String("A".into()),
604 vec![Value::String("UTF_32".into())],
605 )
606 .unwrap(),
607 );
608 assert_eq!(bytes, vec![255, 254, 0, 0, 65, 0, 0, 0]);
609 assert_eq!(shape, vec![1, 8]);
610 assert_eq!(dtype, NumericDType::U8);
611
612 let (be, be_shape, be_dtype) = tensor_bytes(
613 unicode2native_builtin(
614 Value::String("A".into()),
615 vec![Value::String("UTF-32BE".into())],
616 )
617 .unwrap(),
618 );
619 assert_eq!(be, vec![0, 0, 0, 65]);
620 assert_eq!(be_shape, vec![1, 4]);
621 assert_eq!(be_dtype, NumericDType::U8);
622 }
623
624 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
625 #[test]
626 fn unicode2native_empty_text_preserves_uint8_empty_row() {
627 let (bytes, shape, dtype) =
628 tensor_bytes(unicode2native_builtin(Value::String(String::new()), vec![]).unwrap());
629 assert!(bytes.is_empty());
630 assert_eq!(shape, vec![1, 0]);
631 assert_eq!(dtype, NumericDType::U8);
632 }
633
634 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
635 #[test]
636 fn unicode2native_rejects_string_arrays_that_are_not_scalar() {
637 let array = StringArray::new(vec!["a".into(), "b".into()], vec![1, 2]).unwrap();
638 let err = unicode2native_builtin(Value::StringArray(array), vec![]).unwrap_err();
639 assert_eq!(err.identifier(), Some("RunMat:unicode2native:InvalidInput"));
640 }
641
642 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
643 #[test]
644 fn unicode2native_rejects_char_matrices() {
645 let chars = CharArray::new(vec!['a', 'b', 'c', 'd'], 2, 2).unwrap();
646 let err = unicode2native_builtin(Value::CharArray(chars), vec![]).unwrap_err();
647 assert_eq!(err.identifier(), Some("RunMat:unicode2native:InvalidInput"));
648 }
649
650 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
651 #[test]
652 fn unicode2native_rejects_unknown_encoding() {
653 let err = unicode2native_builtin(
654 Value::String("x".into()),
655 vec![Value::String("not-real".into())],
656 )
657 .unwrap_err();
658 assert_eq!(
659 err.identifier(),
660 Some("RunMat:unicode2native:InvalidEncoding")
661 );
662 }
663
664 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
665 #[test]
666 fn unicode2native_rejects_extra_arguments() {
667 let err = unicode2native_builtin(
668 Value::String("x".into()),
669 vec![Value::String("UTF-8".into()), Value::String("extra".into())],
670 )
671 .unwrap_err();
672 assert_eq!(
673 err.identifier(),
674 Some("RunMat:unicode2native:InvalidArgument")
675 );
676 }
677
678 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
679 #[test]
680 fn unicode2native_descriptor_covers_documented_forms() {
681 let labels: Vec<&str> = UNICODE2NATIVE_DESCRIPTOR
682 .signatures
683 .iter()
684 .map(|sig| sig.label)
685 .collect();
686 assert_eq!(
687 labels,
688 vec![
689 "bytes = unicode2native(unicodestr)",
690 "bytes = unicode2native(unicodestr, encoding)"
691 ]
692 );
693 }
694
695 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
696 #[test]
697 fn unicode2native_type_resolver_returns_tensor_for_text() {
698 let out = text_search_indices_type(&[Type::String], &ResolveContext::new(Vec::new()));
699 assert_eq!(out, Type::tensor());
700 }
701}