1#[cfg(test)]
4use runmat_accelerate_api::HostTensorView;
5use runmat_builtins::{
6 BuiltinExtensionDescriptor, BuiltinExtensionMode, BuiltinIntegerAuditDescriptor,
7 BuiltinIntegerAuditKind, BuiltinIntegerBackendRule, BuiltinIntegerCapabilityDescriptor,
8 BuiltinIntegerComputationDomain, BuiltinIntegerInputAvailability,
9 BuiltinIntegerInputCapability, BuiltinIntegerOutputClassRule, BuiltinIntegerOverflowRule,
10 BuiltinIntegerOverloadKind, BuiltinIntegerScalarDoubleRule,
11};
12use std::collections::HashSet;
13
14use regex::RegexBuilder;
15use runmat_builtins::{
16 BuiltinCompletionPolicy, BuiltinDescriptor, BuiltinErrorDescriptor, BuiltinOutputMode,
17 BuiltinParamArity, BuiltinParamDescriptor, BuiltinParamType, BuiltinSignatureDescriptor,
18};
19use runmat_macros::runtime_builtin;
20use runmat_value::{CellArray, CharArray, StringArray, Value};
21
22use crate::builtins::common::map_control_flow_with_builtin;
23use crate::builtins::common::spec::{
24 BroadcastSemantics, BuiltinFusionSpec, BuiltinGpuSpec, ConstantStrategy, GpuOpKind,
25 ReductionNaN, ResidencyPolicy, ShapeRequirements,
26};
27use crate::builtins::common::tensor;
28use crate::builtins::strings::common::{char_row_to_string_slice, is_missing_string};
29use crate::builtins::strings::type_resolvers::{string_array_type, unknown_type};
30use crate::{build_runtime_error, gather_if_needed_async, make_cell, BuiltinResult, RuntimeError};
31
32#[runmat_macros::register_gpu_spec(builtin_path = "crate::builtins::strings::transform::split")]
33pub const GPU_SPEC: BuiltinGpuSpec = BuiltinGpuSpec {
34 name: "split",
35 op_kind: GpuOpKind::Custom("string-transform"),
36 supported_precisions: &[],
37 broadcast: BroadcastSemantics::None,
38 provider_hooks: &[],
39 constant_strategy: ConstantStrategy::InlineLiteral,
40 residency: ResidencyPolicy::GatherImmediately,
41 nan_mode: ReductionNaN::Include,
42 two_pass_threshold: None,
43 workgroup_size: None,
44 accepts_nan_mode: false,
45 notes: "Executes on the CPU; GPU-resident inputs are gathered to host memory before splitting.",
46};
47
48#[runmat_macros::register_fusion_spec(builtin_path = "crate::builtins::strings::transform::split")]
49pub const FUSION_SPEC: BuiltinFusionSpec = BuiltinFusionSpec {
50 name: "split",
51 shape: ShapeRequirements::Any,
52 constant_strategy: ConstantStrategy::InlineLiteral,
53 elementwise: None,
54 reduction: None,
55 emits_nan: false,
56 notes: "String transformation builtin; not eligible for fusion planning and always gathers GPU inputs.",
57};
58
59const BUILTIN_NAME: &str = "split";
60const STRSPLIT_BUILTIN_NAME: &str = "strsplit";
61const MAX_SPLIT_DIMENSION: usize = 1024;
62
63const SPLIT_OUTPUT: [BuiltinParamDescriptor; 2] = [
64 BuiltinParamDescriptor {
65 name: "newStr",
66 ty: BuiltinParamType::Any,
67 arity: BuiltinParamArity::Required,
68 default: None,
69 description: "String or cell array containing split tokens.",
70 },
71 BuiltinParamDescriptor {
72 name: "match",
73 ty: BuiltinParamType::Any,
74 arity: BuiltinParamArity::Optional,
75 default: None,
76 description: "String or cell array containing the delimiters at which splitting occurred.",
77 },
78];
79
80const SPLIT_INPUTS_BASE: [BuiltinParamDescriptor; 1] = [BuiltinParamDescriptor {
81 name: "str",
82 ty: BuiltinParamType::Any,
83 arity: BuiltinParamArity::Required,
84 default: None,
85 description: "Input text scalar/array/cell to split.",
86}];
87
88const SPLIT_INPUTS_DELIMITER: [BuiltinParamDescriptor; 2] = [
89 BuiltinParamDescriptor {
90 name: "str",
91 ty: BuiltinParamType::Any,
92 arity: BuiltinParamArity::Required,
93 default: None,
94 description: "Input text scalar/array/cell to split.",
95 },
96 BuiltinParamDescriptor {
97 name: "delimiter",
98 ty: BuiltinParamType::Any,
99 arity: BuiltinParamArity::Required,
100 default: None,
101 description: "Delimiter scalar/array/cell.",
102 },
103];
104
105const SPLIT_INPUTS_DELIMITER_DIM: [BuiltinParamDescriptor; 3] = [
106 BuiltinParamDescriptor {
107 name: "str",
108 ty: BuiltinParamType::Any,
109 arity: BuiltinParamArity::Required,
110 default: None,
111 description: "Input text scalar/array/cell to split.",
112 },
113 BuiltinParamDescriptor {
114 name: "delimiter",
115 ty: BuiltinParamType::Any,
116 arity: BuiltinParamArity::Required,
117 default: None,
118 description: "Delimiter scalar/array/cell.",
119 },
120 BuiltinParamDescriptor {
121 name: "dim",
122 ty: BuiltinParamType::IntegerScalar,
123 arity: BuiltinParamArity::Required,
124 default: None,
125 description: "Positive dimension along which output substrings are oriented.",
126 },
127];
128
129const SPLIT_INPUTS_DELIMITER_NAMEVALUE: [BuiltinParamDescriptor; 4] = [
130 BuiltinParamDescriptor {
131 name: "str",
132 ty: BuiltinParamType::Any,
133 arity: BuiltinParamArity::Required,
134 default: None,
135 description: "Input text scalar/array/cell to split.",
136 },
137 BuiltinParamDescriptor {
138 name: "delimiter",
139 ty: BuiltinParamType::Any,
140 arity: BuiltinParamArity::Required,
141 default: None,
142 description: "Delimiter scalar/array/cell.",
143 },
144 BuiltinParamDescriptor {
145 name: "Name",
146 ty: BuiltinParamType::StringScalar,
147 arity: BuiltinParamArity::Required,
148 default: None,
149 description: "Option name (`CollapseDelimiters` or `IncludeDelimiters`).",
150 },
151 BuiltinParamDescriptor {
152 name: "Value",
153 ty: BuiltinParamType::Any,
154 arity: BuiltinParamArity::Variadic,
155 default: None,
156 description: "Option values and additional Name/Value pairs.",
157 },
158];
159
160const SPLIT_INPUTS_NAMEVALUE: [BuiltinParamDescriptor; 3] = [
161 BuiltinParamDescriptor {
162 name: "str",
163 ty: BuiltinParamType::Any,
164 arity: BuiltinParamArity::Required,
165 default: None,
166 description: "Input text scalar/array/cell to split.",
167 },
168 BuiltinParamDescriptor {
169 name: "Name",
170 ty: BuiltinParamType::StringScalar,
171 arity: BuiltinParamArity::Required,
172 default: None,
173 description: "Option name (`CollapseDelimiters` or `IncludeDelimiters`).",
174 },
175 BuiltinParamDescriptor {
176 name: "Value",
177 ty: BuiltinParamType::Any,
178 arity: BuiltinParamArity::Variadic,
179 default: None,
180 description: "Option values and additional Name/Value pairs.",
181 },
182];
183
184const SPLIT_SIGNATURES: [BuiltinSignatureDescriptor; 5] = [
185 BuiltinSignatureDescriptor {
186 label: "newStr = split(str)",
187 inputs: &SPLIT_INPUTS_BASE,
188 outputs: &SPLIT_OUTPUT,
189 },
190 BuiltinSignatureDescriptor {
191 label: "newStr = split(str, delimiter)",
192 inputs: &SPLIT_INPUTS_DELIMITER,
193 outputs: &SPLIT_OUTPUT,
194 },
195 BuiltinSignatureDescriptor {
196 label: "[newStr, match] = split(str, delimiter, dim)",
197 inputs: &SPLIT_INPUTS_DELIMITER_DIM,
198 outputs: &SPLIT_OUTPUT,
199 },
200 BuiltinSignatureDescriptor {
201 label: "newStr = split(str, delimiter, Name, Value, ...)",
202 inputs: &SPLIT_INPUTS_DELIMITER_NAMEVALUE,
203 outputs: &SPLIT_OUTPUT,
204 },
205 BuiltinSignatureDescriptor {
206 label: "newStr = split(str, Name, Value, ...)",
207 inputs: &SPLIT_INPUTS_NAMEVALUE,
208 outputs: &SPLIT_OUTPUT,
209 },
210];
211
212const SPLIT_ERROR_INVALID_INPUT: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
213 code: "RM.SPLIT.INVALID_INPUT",
214 identifier: Some("RunMat:split:InvalidInput"),
215 when: "First argument is not a string scalar/array, char array, or cell array of text scalars.",
216 message:
217 "split: first argument must be a string scalar, string array, character array, or cell array of character vectors",
218};
219
220const SPLIT_ERROR_DELIMITER_TYPE: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
221 code: "RM.SPLIT.DELIMITER_TYPE",
222 identifier: Some("RunMat:split:DelimiterType"),
223 when: "Delimiter input is not a supported text scalar/array/cell.",
224 message:
225 "split: delimiter input must be a string scalar, string array, character array, or cell array of character vectors",
226};
227
228const SPLIT_ERROR_NAME_VALUE_PAIR: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
229 code: "RM.SPLIT.NAME_VALUE_PAIR",
230 identifier: Some("RunMat:split:NameValuePair"),
231 when: "Name-value options are not supplied in complete pairs.",
232 message: "split: name-value arguments must be supplied in pairs",
233};
234
235const SPLIT_ERROR_UNKNOWN_NAME: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
236 code: "RM.SPLIT.UNKNOWN_NAME",
237 identifier: Some("RunMat:split:UnknownName"),
238 when: "An option name is not recognized.",
239 message:
240 "split: unrecognized name-value argument; supported names are 'CollapseDelimiters' and 'IncludeDelimiters'",
241};
242
243const SPLIT_ERROR_EMPTY_DELIMITER: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
244 code: "RM.SPLIT.EMPTY_DELIMITER",
245 identifier: Some("RunMat:split:EmptyDelimiter"),
246 when: "Delimiter list is empty or contains empty delimiter entries.",
247 message: "split: delimiters must contain at least one character",
248};
249
250const SPLIT_ERROR_CELL_ELEMENT: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
251 code: "RM.SPLIT.CELL_ELEMENT",
252 identifier: Some("RunMat:split:CellElement"),
253 when: "Cell arrays contain non-text elements or non-row char arrays.",
254 message: "split: cell array elements must be string scalars or character vectors",
255};
256
257const SPLIT_ERROR_OPTION_VALUE: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
258 code: "RM.SPLIT.OPTION_VALUE",
259 identifier: Some("RunMat:split:OptionValue"),
260 when: "Option values are not logical true/false values.",
261 message: "split: option values must be logical true or false",
262};
263
264const SPLIT_ERROR_INTERNAL: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
265 code: "RM.SPLIT.INTERNAL",
266 identifier: Some("RunMat:split:InternalError"),
267 when: "Internal output container construction failed.",
268 message: "split: internal error",
269};
270
271const SPLIT_ERROR_DIMENSION: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
272 code: "RM.SPLIT.DIMENSION",
273 identifier: Some("RunMat:split:Dimension"),
274 when: "The dimension argument is not a positive integer scalar in RunMat's supported rank range of 1 through 1024.",
275 message: "split: dimension must be a positive integer scalar no greater than 1024",
276};
277
278const SPLIT_ERRORS: [BuiltinErrorDescriptor; 9] = [
279 SPLIT_ERROR_INVALID_INPUT,
280 SPLIT_ERROR_DELIMITER_TYPE,
281 SPLIT_ERROR_NAME_VALUE_PAIR,
282 SPLIT_ERROR_UNKNOWN_NAME,
283 SPLIT_ERROR_EMPTY_DELIMITER,
284 SPLIT_ERROR_CELL_ELEMENT,
285 SPLIT_ERROR_OPTION_VALUE,
286 SPLIT_ERROR_DIMENSION,
287 SPLIT_ERROR_INTERNAL,
288];
289
290pub const SPLIT_DESCRIPTOR: BuiltinDescriptor = BuiltinDescriptor {
291 signatures: &SPLIT_SIGNATURES,
292 output_mode: BuiltinOutputMode::ByRequestedOutputCount,
293 completion_policy: BuiltinCompletionPolicy::Public,
294 errors: &SPLIT_ERRORS,
295};
296
297const SPLIT_TYPED_DIMENSION_EXTENSION: BuiltinExtensionDescriptor = BuiltinExtensionDescriptor {
298 id: "split-typed-dimension",
299 mode: BuiltinExtensionMode::RunMatOnly,
300 description: "split with a typed-integer dimension",
301 error_identifier: Some("RunMat:compatibility:SplitTypedDimensionExtension"),
302};
303
304const SPLIT_RESIDENT_DIMENSION_EXTENSION: BuiltinExtensionDescriptor = BuiltinExtensionDescriptor {
305 id: "split-resident-dimension",
306 mode: BuiltinExtensionMode::RunMatOnly,
307 description: "split with an explicitly GPU-resident dimension is a RunMat extension",
308 error_identifier: Some("RunMat:compatibility:SplitResidentDimensionExtension"),
309};
310
311const SPLIT_ADVANCED_OPTIONS_EXTENSION: BuiltinExtensionDescriptor = BuiltinExtensionDescriptor {
312 id: "split-advanced-options",
313 mode: BuiltinExtensionMode::RunMatOnly,
314 description: "split CollapseDelimiters and IncludeDelimiters options are RunMat extensions",
315 error_identifier: Some("RunMat:compatibility:SplitAdvancedOptionsExtension"),
316};
317
318pub const SPLIT_EXTENSIONS: [BuiltinExtensionDescriptor; 3] = [
319 SPLIT_TYPED_DIMENSION_EXTENSION,
320 SPLIT_RESIDENT_DIMENSION_EXTENSION,
321 SPLIT_ADVANCED_OPTIONS_EXTENSION,
322];
323
324const SPLIT_INTEGER_DIMENSION_INPUTS: [BuiltinIntegerInputCapability; 1] =
325 [BuiltinIntegerInputCapability {
326 name: "dim",
327 classes: &crate::builtins::common::integer_capability::ALL_INTEGER_CLASSES,
328 availability: BuiltinIntegerInputAvailability::RunMatOnly,
329 scalar_double: BuiltinIntegerScalarDoubleRule::Allowed,
330 notes: "The compatibility target specifies a positive integer dimension but does not enumerate typed storage classes. RunMat accepts every exact integer class behind a compatibility gate; ordinary host double integer dimensions remain documented behavior.",
331 }];
332
333pub const SPLIT_INTEGER_CAPABILITIES: [BuiltinIntegerCapabilityDescriptor; 1] =
334 [BuiltinIntegerCapabilityDescriptor {
335 form: "[newStr, match] = split(str, delimiter, integer_dim)",
336 inputs: &SPLIT_INTEGER_DIMENSION_INPUTS,
337 computation_domain: BuiltinIntegerComputationDomain::Structural,
338 output_class: BuiltinIntegerOutputClassRule::NotApplicable,
339 overflow: BuiltinIntegerOverflowRule::Error,
340 backend: BuiltinIntegerBackendRule::GatherFallback,
341 overload: BuiltinIntegerOverloadKind::StructuralParameter,
342 notes: "The typed dimension is a gated RunMat extension whose exact one-based value controls only output orientation. Explicit resident dimensions are separately gated before gather; automatic residency may gather transparently.",
343 }];
344
345const STRSPLIT_OUTPUT: [BuiltinParamDescriptor; 2] = [
346 BuiltinParamDescriptor {
347 name: "parts",
348 ty: BuiltinParamType::Any,
349 arity: BuiltinParamArity::Required,
350 default: None,
351 description: "Split tokens.",
352 },
353 BuiltinParamDescriptor {
354 name: "matches",
355 ty: BuiltinParamType::Any,
356 arity: BuiltinParamArity::Optional,
357 default: None,
358 description: "Matched delimiters when requested as second output.",
359 },
360];
361
362const STRSPLIT_INPUTS_BASE: [BuiltinParamDescriptor; 1] = [BuiltinParamDescriptor {
363 name: "str",
364 ty: BuiltinParamType::Any,
365 arity: BuiltinParamArity::Required,
366 default: None,
367 description: "String scalar or character vector input.",
368}];
369
370const STRSPLIT_INPUTS_DELIMITER: [BuiltinParamDescriptor; 2] = [
371 BuiltinParamDescriptor {
372 name: "str",
373 ty: BuiltinParamType::Any,
374 arity: BuiltinParamArity::Required,
375 default: None,
376 description: "String scalar or character vector input.",
377 },
378 BuiltinParamDescriptor {
379 name: "delimiter",
380 ty: BuiltinParamType::Any,
381 arity: BuiltinParamArity::Required,
382 default: None,
383 description: "Delimiter scalar/array/cell.",
384 },
385];
386
387const STRSPLIT_INPUTS_DELIMITER_NAMEVALUE: [BuiltinParamDescriptor; 4] = [
388 BuiltinParamDescriptor {
389 name: "str",
390 ty: BuiltinParamType::Any,
391 arity: BuiltinParamArity::Required,
392 default: None,
393 description: "String scalar or character vector input.",
394 },
395 BuiltinParamDescriptor {
396 name: "delimiter",
397 ty: BuiltinParamType::Any,
398 arity: BuiltinParamArity::Required,
399 default: None,
400 description: "Delimiter scalar/array/cell.",
401 },
402 BuiltinParamDescriptor {
403 name: "Name",
404 ty: BuiltinParamType::StringScalar,
405 arity: BuiltinParamArity::Required,
406 default: None,
407 description: "Option name (`CollapseDelimiters` or `DelimiterType`).",
408 },
409 BuiltinParamDescriptor {
410 name: "Value",
411 ty: BuiltinParamType::Any,
412 arity: BuiltinParamArity::Variadic,
413 default: None,
414 description: "Option values and additional Name/Value pairs.",
415 },
416];
417
418const STRSPLIT_INPUTS_NAMEVALUE: [BuiltinParamDescriptor; 3] = [
419 BuiltinParamDescriptor {
420 name: "str",
421 ty: BuiltinParamType::Any,
422 arity: BuiltinParamArity::Required,
423 default: None,
424 description: "String scalar or character vector input.",
425 },
426 BuiltinParamDescriptor {
427 name: "Name",
428 ty: BuiltinParamType::StringScalar,
429 arity: BuiltinParamArity::Required,
430 default: None,
431 description: "Option name (`CollapseDelimiters` or `DelimiterType`).",
432 },
433 BuiltinParamDescriptor {
434 name: "Value",
435 ty: BuiltinParamType::Any,
436 arity: BuiltinParamArity::Variadic,
437 default: None,
438 description: "Option values and additional Name/Value pairs.",
439 },
440];
441
442const STRSPLIT_SIGNATURES: [BuiltinSignatureDescriptor; 4] = [
443 BuiltinSignatureDescriptor {
444 label: "[parts, matches] = strsplit(str)",
445 inputs: &STRSPLIT_INPUTS_BASE,
446 outputs: &STRSPLIT_OUTPUT,
447 },
448 BuiltinSignatureDescriptor {
449 label: "[parts, matches] = strsplit(str, delimiter)",
450 inputs: &STRSPLIT_INPUTS_DELIMITER,
451 outputs: &STRSPLIT_OUTPUT,
452 },
453 BuiltinSignatureDescriptor {
454 label: "[parts, matches] = strsplit(str, delimiter, Name, Value, ...)",
455 inputs: &STRSPLIT_INPUTS_DELIMITER_NAMEVALUE,
456 outputs: &STRSPLIT_OUTPUT,
457 },
458 BuiltinSignatureDescriptor {
459 label: "[parts, matches] = strsplit(str, Name, Value, ...)",
460 inputs: &STRSPLIT_INPUTS_NAMEVALUE,
461 outputs: &STRSPLIT_OUTPUT,
462 },
463];
464
465const STRSPLIT_ERROR_INVALID_INPUT: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
466 code: "RM.STRSPLIT.INVALID_INPUT",
467 identifier: Some("RunMat:strsplit:InvalidInput"),
468 when: "First argument is not a string scalar or character vector.",
469 message: "strsplit: first argument must be a string scalar or character vector",
470};
471
472const STRSPLIT_ERROR_DELIMITER_TYPE: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
473 code: "RM.STRSPLIT.DELIMITER_TYPE",
474 identifier: Some("RunMat:strsplit:DelimiterType"),
475 when: "Delimiter input is not a supported text scalar/array/cell.",
476 message:
477 "strsplit: delimiter must be a character vector, string scalar, string array, or cell array of character vectors",
478};
479
480const STRSPLIT_ERROR_NAME_VALUE_PAIR: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
481 code: "RM.STRSPLIT.NAME_VALUE_PAIR",
482 identifier: Some("RunMat:strsplit:NameValuePair"),
483 when: "Name-value options are not supplied in complete pairs.",
484 message: "strsplit: name-value arguments must be supplied in pairs",
485};
486
487const STRSPLIT_ERROR_UNKNOWN_NAME: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
488 code: "RM.STRSPLIT.UNKNOWN_NAME",
489 identifier: Some("RunMat:strsplit:UnknownName"),
490 when: "An option name is not recognized.",
491 message:
492 "strsplit: unrecognized name-value argument; supported names are 'CollapseDelimiters' and 'DelimiterType'",
493};
494
495const STRSPLIT_ERROR_EMPTY_DELIMITER: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
496 code: "RM.STRSPLIT.EMPTY_DELIMITER",
497 identifier: Some("RunMat:strsplit:EmptyDelimiter"),
498 when: "Delimiter list is empty or contains empty delimiter entries.",
499 message: "strsplit: delimiters must contain at least one character",
500};
501
502const STRSPLIT_ERROR_DELIMITER_MODE: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
503 code: "RM.STRSPLIT.DELIMITER_MODE",
504 identifier: Some("RunMat:strsplit:DelimiterMode"),
505 when: "DelimiterType option is not `Simple` or `RegularExpression`.",
506 message: "strsplit: value for 'DelimiterType' must be 'Simple' or 'RegularExpression'",
507};
508
509const STRSPLIT_ERROR_OPTION_VALUE: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
510 code: "RM.STRSPLIT.OPTION_VALUE",
511 identifier: Some("RunMat:strsplit:OptionValue"),
512 when: "Option values are not logical true/false values.",
513 message: "strsplit: option values must be logical true or false",
514};
515
516const STRSPLIT_ERROR_REGEX_INVALID: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
517 code: "RM.STRSPLIT.REGEX_INVALID",
518 identifier: Some("RunMat:strsplit:RegexInvalid"),
519 when: "Regular expression delimiter pattern fails to compile.",
520 message: "strsplit: invalid delimiter regular expression",
521};
522
523const STRSPLIT_ERROR_INTERNAL: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
524 code: "RM.STRSPLIT.INTERNAL",
525 identifier: Some("RunMat:strsplit:InternalError"),
526 when: "Internal output container construction failed.",
527 message: "strsplit: internal error",
528};
529
530const STRSPLIT_ERRORS: [BuiltinErrorDescriptor; 9] = [
531 STRSPLIT_ERROR_INVALID_INPUT,
532 STRSPLIT_ERROR_DELIMITER_TYPE,
533 STRSPLIT_ERROR_NAME_VALUE_PAIR,
534 STRSPLIT_ERROR_UNKNOWN_NAME,
535 STRSPLIT_ERROR_EMPTY_DELIMITER,
536 STRSPLIT_ERROR_DELIMITER_MODE,
537 STRSPLIT_ERROR_OPTION_VALUE,
538 STRSPLIT_ERROR_REGEX_INVALID,
539 STRSPLIT_ERROR_INTERNAL,
540];
541
542pub const STRSPLIT_DESCRIPTOR: BuiltinDescriptor = BuiltinDescriptor {
543 signatures: &STRSPLIT_SIGNATURES,
544 output_mode: BuiltinOutputMode::ByRequestedOutputCount,
545 completion_policy: BuiltinCompletionPolicy::Public,
546 errors: &STRSPLIT_ERRORS,
547};
548
549pub const STRSPLIT_INTEGER_AUDIT: BuiltinIntegerAuditDescriptor = BuiltinIntegerAuditDescriptor {
550 kind: BuiltinIntegerAuditKind::NotApplicable,
551 canonical_builtin: None,
552 notes: "strsplit accepts scalar text, text delimiters, and textual or logical options. Integer and provider-resident numeric values have no documented role and reject before provider access without implicit character conversion.",
553};
554
555fn map_flow(err: RuntimeError) -> RuntimeError {
556 map_control_flow_with_builtin(err, BUILTIN_NAME)
557}
558
559fn split_error_with_message(
560 message: impl Into<String>,
561 error: &'static BuiltinErrorDescriptor,
562) -> RuntimeError {
563 let mut builder = build_runtime_error(message).with_builtin(BUILTIN_NAME);
564 if let Some(identifier) = error.identifier {
565 builder = builder.with_identifier(identifier);
566 }
567 builder.build()
568}
569
570fn split_error(error: &'static BuiltinErrorDescriptor) -> RuntimeError {
571 split_error_with_message(error.message, error)
572}
573
574fn strsplit_error_with_message(
575 message: impl Into<String>,
576 error: &'static BuiltinErrorDescriptor,
577) -> RuntimeError {
578 let mut builder = build_runtime_error(message).with_builtin(STRSPLIT_BUILTIN_NAME);
579 if let Some(identifier) = error.identifier {
580 builder = builder.with_identifier(identifier);
581 }
582 builder.build()
583}
584
585fn strsplit_error(error: &'static BuiltinErrorDescriptor) -> RuntimeError {
586 strsplit_error_with_message(error.message, error)
587}
588
589#[runtime_builtin(
590 name = "split",
591 category = "strings/transform",
592 summary = "Split text inputs into substrings using delimiter rules.",
593 keywords = "split,strsplit,delimiter,CollapseDelimiters,IncludeDelimiters",
594 accel = "sink",
595 type_resolver(string_array_type),
596 descriptor(crate::builtins::strings::transform::split::SPLIT_DESCRIPTOR),
597 extensions(crate::builtins::strings::transform::split::SPLIT_EXTENSIONS),
598 integer_capabilities(crate::builtins::strings::transform::split::SPLIT_INTEGER_CAPABILITIES),
599 builtin_path = "crate::builtins::strings::transform::split"
600)]
601async fn split_builtin(text: Value, rest: Vec<Value>) -> BuiltinResult<Value> {
602 if crate::dispatcher::value_contains_gpu(&text) {
603 return Err(split_error(&SPLIT_ERROR_INVALID_INPUT));
604 }
605 let text = gather_if_needed_async(&text).await.map_err(map_flow)?;
606 let (args, dimension) = prepare_split_arguments(rest).await?;
607
608 let options = SplitOptions::parse(&args)?;
609 let matrix = TextMatrix::from_value(text)?;
610 matrix.into_split_result(&options, dimension)
611}
612
613async fn prepare_split_arguments(
614 mut rest: Vec<Value>,
615) -> BuiltinResult<(Vec<Value>, Option<usize>)> {
616 let dimension_index =
617 if rest.len() >= 2 && !is_name_key(&rest[0]) && is_dimension_candidate(&rest[1]) {
618 Some(1usize)
619 } else {
620 None
621 };
622 if rest.last().is_some_and(is_name_key) {
623 return Err(split_error(&SPLIT_ERROR_NAME_VALUE_PAIR));
624 }
625 if rest.iter().any(is_name_key) {
626 crate::compatibility::ensure_builtin_extension_enabled(
627 &SPLIT_ADVANCED_OPTIONS_EXTENSION,
628 BUILTIN_NAME,
629 )?;
630 }
631 if let Some(index) = dimension_index {
632 let value = &rest[index];
633 if is_typed_integer_dimension(value) {
634 crate::compatibility::ensure_builtin_extension_enabled(
635 &SPLIT_TYPED_DIMENSION_EXTENSION,
636 BUILTIN_NAME,
637 )?;
638 }
639 if matches!(value, Value::GpuTensor(handle) if runmat_accelerate_api::handle_is_explicit(handle))
640 {
641 crate::compatibility::ensure_builtin_extension_enabled(
642 &SPLIT_RESIDENT_DIMENSION_EXTENSION,
643 BUILTIN_NAME,
644 )?;
645 }
646 }
647 let mut host = Vec::with_capacity(rest.len());
648 for value in rest.drain(..) {
649 host.push(gather_if_needed_async(&value).await.map_err(map_flow)?);
650 }
651 let dimension = if let Some(index) = dimension_index {
652 let value = host.remove(index);
653 Some(parse_split_dimension(&value)?)
654 } else {
655 None
656 };
657 Ok((host, dimension))
658}
659
660fn is_dimension_candidate(value: &Value) -> bool {
661 matches!(
662 value,
663 Value::Num(_) | Value::Int(_) | Value::Tensor(_) | Value::GpuTensor(_)
664 )
665}
666
667fn is_typed_integer_dimension(value: &Value) -> bool {
668 match value {
669 Value::Int(_) => true,
670 Value::Tensor(value) => value.integer_storage().is_some(),
671 Value::GpuTensor(handle) => runmat_accelerate_api::handle_integer_type(handle).is_some(),
672 _ => false,
673 }
674}
675
676fn parse_split_dimension(value: &Value) -> BuiltinResult<usize> {
677 let dimension = match value {
678 Value::Num(value) => positive_usize(*value),
679 Value::Int(value) => value.try_to_usize().filter(|value| *value > 0),
680 Value::Tensor(value) if tensor::is_scalar_tensor(value) => {
681 if let Some(integer) = value
682 .integer_storage()
683 .and_then(|storage| storage.value_at(0))
684 {
685 integer.try_to_usize().filter(|value| *value > 0)
686 } else {
687 positive_usize(tensor::tensor_value_f64(value, 0))
688 }
689 }
690 _ => None,
691 };
692 dimension
693 .filter(|dimension| *dimension <= MAX_SPLIT_DIMENSION)
694 .ok_or_else(|| split_error(&SPLIT_ERROR_DIMENSION))
695}
696
697fn positive_usize(value: f64) -> Option<usize> {
698 if value.is_finite()
699 && value >= 1.0
700 && value.fract() == 0.0
701 && (value < usize::MAX as f64 || (usize::BITS < 64 && value == usize::MAX as f64))
702 {
703 Some(value as usize)
704 } else {
705 None
706 }
707}
708
709#[runtime_builtin(
710 name = "strsplit",
711 category = "strings/transform",
712 summary = "Split scalar text into substrings using simple or regex delimiters.",
713 keywords = "strsplit,split,delimiter,CollapseDelimiters,DelimiterType,matches",
714 accel = "sink",
715 type_resolver(unknown_type),
716 descriptor(crate::builtins::strings::transform::split::STRSPLIT_DESCRIPTOR),
717 integer_audit(crate::builtins::strings::transform::split::STRSPLIT_INTEGER_AUDIT),
718 builtin_path = "crate::builtins::strings::transform::split"
719)]
720async fn strsplit_builtin(text: Value, rest: Vec<Value>) -> BuiltinResult<Value> {
721 if crate::builtins::strings::common::contains_numeric_or_resident_text_input(&text)
722 || rest.iter().any(|value| {
723 matches!(
724 value,
725 Value::Num(_)
726 | Value::Int(_)
727 | Value::Tensor(_)
728 | Value::SparseTensor(_)
729 | Value::Complex(_, _)
730 | Value::ComplexTensor(_)
731 | Value::Symbolic(_)
732 | Value::GpuTensor(_)
733 )
734 })
735 {
736 return Err(split_error(&SPLIT_ERROR_INVALID_INPUT));
737 }
738 let text = gather_if_needed_async(&text)
739 .await
740 .map_err(|err| map_control_flow_with_builtin(err, STRSPLIT_BUILTIN_NAME))?;
741 let mut args = Vec::with_capacity(rest.len());
742 for arg in rest {
743 args.push(
744 gather_if_needed_async(&arg)
745 .await
746 .map_err(|err| map_control_flow_with_builtin(err, STRSPLIT_BUILTIN_NAME))?,
747 );
748 }
749
750 let (input_kind, subject) = extract_strsplit_subject(text)?;
751 let options = StrsplitOptions::parse(&args)?;
752 let (parts, matches) = strsplit_text(&subject, &options)?;
753 let parts_value = make_strsplit_output(parts, input_kind)?;
754
755 if let Some(out_count) = crate::output_count::current_output_count() {
756 if out_count == 0 {
757 return Ok(Value::OutputList(Vec::new()));
758 }
759 let matches_value = make_strsplit_output(matches, input_kind)?;
760 return Ok(crate::output_count::output_list_with_padding(
761 out_count,
762 vec![parts_value, matches_value],
763 ));
764 }
765
766 Ok(parts_value)
767}
768
769#[derive(Clone)]
770enum DelimiterSpec {
771 Whitespace,
772 Patterns(Vec<String>),
773}
774
775#[derive(Clone)]
776struct SplitOptions {
777 delimiters: DelimiterSpec,
778 collapse_delimiters: bool,
779 include_delimiters: bool,
780}
781
782impl SplitOptions {
783 fn parse(args: &[Value]) -> BuiltinResult<Self> {
784 let mut index = 0usize;
785 let mut delimiters = DelimiterSpec::Whitespace;
786
787 if index < args.len() && !is_name_key(&args[index]) {
788 let list = extract_delimiters(&args[index])?;
789 if list.is_empty() {
790 return Err(split_error(&SPLIT_ERROR_EMPTY_DELIMITER));
791 }
792 let mut seen = HashSet::new();
793 let mut patterns: Vec<String> = Vec::new();
794 for pattern in list {
795 if pattern.is_empty() {
796 return Err(split_error(&SPLIT_ERROR_EMPTY_DELIMITER));
797 }
798 if seen.insert(pattern.clone()) {
799 patterns.push(pattern);
800 }
801 }
802 patterns.sort_by_key(|pat| std::cmp::Reverse(pat.len()));
803 delimiters = DelimiterSpec::Patterns(patterns);
804 index += 1;
805 }
806
807 let mut collapse = match delimiters {
808 DelimiterSpec::Whitespace => true,
809 DelimiterSpec::Patterns(_) => false,
810 };
811 let mut include = false;
812
813 while index < args.len() {
814 let name = match name_key(&args[index]) {
815 Some(NameKey::CollapseDelimiters) => NameKey::CollapseDelimiters,
816 Some(NameKey::IncludeDelimiters) => NameKey::IncludeDelimiters,
817 None => return Err(split_error(&SPLIT_ERROR_UNKNOWN_NAME)),
818 };
819 index += 1;
820 if index >= args.len() {
821 return Err(split_error(&SPLIT_ERROR_NAME_VALUE_PAIR));
822 }
823 let value = &args[index];
824 index += 1;
825
826 match name {
827 NameKey::CollapseDelimiters => {
828 collapse = parse_bool(value, "CollapseDelimiters")?;
829 }
830 NameKey::IncludeDelimiters => {
831 include = parse_bool(value, "IncludeDelimiters")?;
832 }
833 }
834 }
835
836 Ok(Self {
837 delimiters,
838 collapse_delimiters: collapse,
839 include_delimiters: include,
840 })
841 }
842}
843
844struct TextMatrix {
845 data: Vec<String>,
846 rows: usize,
847 cols: usize,
848}
849
850impl TextMatrix {
851 fn from_value(value: Value) -> BuiltinResult<Self> {
852 match value {
853 Value::String(text) => Ok(Self {
854 data: vec![text],
855 rows: 1,
856 cols: 1,
857 }),
858 Value::StringArray(array) => Ok(Self {
859 data: array.data,
860 rows: array.rows,
861 cols: array.cols,
862 }),
863 Value::CharArray(array) => Self::from_char_array(array),
864 Value::Cell(cell) => Self::from_cell_array(cell),
865 _ => Err(split_error(&SPLIT_ERROR_INVALID_INPUT)),
866 }
867 }
868
869 fn from_char_array(array: CharArray) -> BuiltinResult<Self> {
870 let CharArray {
871 data, rows, cols, ..
872 } = array;
873 if rows == 0 {
874 return Ok(Self {
875 data: Vec::new(),
876 rows: 0,
877 cols: 1,
878 });
879 }
880 let mut strings = Vec::with_capacity(rows);
881 for row in 0..rows {
882 strings.push(char_row_to_string_slice(&data, cols, row));
883 }
884 Ok(Self {
885 data: strings,
886 rows,
887 cols: 1,
888 })
889 }
890
891 fn from_cell_array(cell: CellArray) -> BuiltinResult<Self> {
892 let CellArray {
893 data, rows, cols, ..
894 } = cell;
895 let mut strings = Vec::with_capacity(data.len());
896 for col in 0..cols {
897 for row in 0..rows {
898 let idx = row * cols + col;
899 let value_ref: &Value = &data[idx];
900 strings.push(
901 cell_element_to_string(value_ref)
902 .ok_or_else(|| split_error(&SPLIT_ERROR_CELL_ELEMENT))?,
903 );
904 }
905 }
906 Ok(Self {
907 data: strings,
908 rows,
909 cols,
910 })
911 }
912
913 fn into_split_result(
914 self,
915 options: &SplitOptions,
916 dimension: Option<usize>,
917 ) -> BuiltinResult<Value> {
918 let TextMatrix { data, rows, cols } = self;
919
920 if data.is_empty() {
921 let shape = vec![rows, cols];
922 let parts = StringArray::new(Vec::new(), shape.clone()).map_err(|e| {
923 split_error_with_message(format!("{BUILTIN_NAME}: {e}"), &SPLIT_ERROR_INTERNAL)
924 })?;
925 let parts = Value::StringArray(parts);
926 if let Some(output_count) = crate::output_count::current_output_count() {
927 if output_count == 0 {
928 return Ok(Value::OutputList(Vec::new()));
929 }
930 let matches = StringArray::new(Vec::new(), shape).map_err(|e| {
931 split_error_with_message(format!("{BUILTIN_NAME}: {e}"), &SPLIT_ERROR_INTERNAL)
932 })?;
933 return Ok(crate::output_count::output_list_with_padding(
934 output_count,
935 vec![parts, Value::StringArray(matches)],
936 ));
937 }
938 return Ok(parts);
939 }
940
941 let mut per_element: Vec<Vec<String>> = Vec::with_capacity(data.len());
942 let mut per_element_matches: Vec<Vec<String>> = Vec::with_capacity(data.len());
943 let mut max_tokens = 0usize;
944 let mut max_matches = 0usize;
945 for text in &data {
946 let tokens = split_text(text, options);
947 let matches = split_delimiter_matches(text, options);
948 max_tokens = max_tokens.max(tokens.len());
949 max_matches = max_matches.max(matches.len());
950 per_element.push(tokens);
951 per_element_matches.push(matches);
952 }
953 if max_tokens == 0 {
954 max_tokens = 1;
955 }
956 let dimension = dimension.unwrap_or_else(|| default_split_dimension(rows, cols));
957 let (output, shape) = orient_split_values(&per_element, rows, cols, dimension, max_tokens)?;
958 let array = StringArray::new(output, shape).map_err(|e| {
959 split_error_with_message(format!("{BUILTIN_NAME}: {e}"), &SPLIT_ERROR_INTERNAL)
960 })?;
961 let parts = Value::StringArray(array);
962 if let Some(output_count) = crate::output_count::current_output_count() {
963 if output_count == 0 {
964 return Ok(Value::OutputList(Vec::new()));
965 }
966 let (matches, match_shape) =
967 orient_split_values(&per_element_matches, rows, cols, dimension, max_matches)?;
968 let matches = StringArray::new(matches, match_shape).map_err(|error| {
969 split_error_with_message(format!("{BUILTIN_NAME}: {error}"), &SPLIT_ERROR_INTERNAL)
970 })?;
971 return Ok(crate::output_count::output_list_with_padding(
972 output_count,
973 vec![parts, Value::StringArray(matches)],
974 ));
975 }
976 Ok(parts)
977 }
978}
979
980fn default_split_dimension(rows: usize, cols: usize) -> usize {
981 if rows <= 1 && cols <= 1 {
982 1
983 } else if cols <= 1 {
984 2
985 } else {
986 3
987 }
988}
989
990fn orient_split_values(
991 per_element: &[Vec<String>],
992 rows: usize,
993 cols: usize,
994 dimension: usize,
995 value_count: usize,
996) -> BuiltinResult<(Vec<String>, Vec<usize>)> {
997 let mut input_shape = vec![rows, cols.max(1)];
998 while input_shape.len() < dimension {
999 input_shape.push(1);
1000 }
1001 let dimension_index = dimension - 1;
1002 let last_non_singleton = input_shape.iter().rposition(|size| *size != 1);
1003 let replace_trailing_singleton = input_shape[dimension_index] == 1
1004 && last_non_singleton.is_none_or(|index| dimension_index > index);
1005 let mut output_shape = input_shape.clone();
1006 if replace_trailing_singleton {
1007 output_shape[dimension_index] = value_count;
1008 } else {
1009 output_shape.insert(dimension_index, value_count);
1010 }
1011 let total = output_shape
1012 .iter()
1013 .try_fold(1usize, |product, size| product.checked_mul(*size));
1014 let total = total.ok_or_else(|| split_error(&SPLIT_ERROR_INTERNAL))?;
1015 let mut output = Vec::new();
1016 output
1017 .try_reserve_exact(total)
1018 .map_err(|_| split_error(&SPLIT_ERROR_INTERNAL))?;
1019 output.resize(total, "<missing>".to_string());
1020 for (input_linear, values) in per_element.iter().enumerate() {
1021 let input_coords = linear_to_subscripts(input_linear, &input_shape);
1022 for value_index in 0..value_count {
1023 let mut output_coords = input_coords.clone();
1024 if replace_trailing_singleton {
1025 output_coords[dimension_index] = value_index;
1026 } else {
1027 output_coords.insert(dimension_index, value_index);
1028 }
1029 if let Some(value) = values.get(value_index) {
1030 let output_linear = subscripts_to_linear(&output_coords, &output_shape);
1031 output[output_linear] = value.clone();
1032 }
1033 }
1034 }
1035 while output_shape.len() > 2 && output_shape.last() == Some(&1) {
1036 output_shape.pop();
1037 }
1038 Ok((output, output_shape))
1039}
1040
1041fn linear_to_subscripts(mut linear: usize, shape: &[usize]) -> Vec<usize> {
1042 let mut subscripts = Vec::with_capacity(shape.len());
1043 for size in shape {
1044 if *size == 0 {
1045 subscripts.push(0);
1046 } else {
1047 subscripts.push(linear % size);
1048 linear /= size;
1049 }
1050 }
1051 subscripts
1052}
1053
1054fn subscripts_to_linear(subscripts: &[usize], shape: &[usize]) -> usize {
1055 let mut stride = 1usize;
1056 let mut linear = 0usize;
1057 for (subscript, size) in subscripts.iter().zip(shape) {
1058 linear += subscript * stride;
1059 stride *= size;
1060 }
1061 linear
1062}
1063
1064fn split_text(text: &str, options: &SplitOptions) -> Vec<String> {
1065 if is_missing_string(text) {
1066 return vec![text.to_string()];
1067 }
1068 match &options.delimiters {
1069 DelimiterSpec::Whitespace => split_whitespace(text, options),
1070 DelimiterSpec::Patterns(patterns) => split_by_patterns(text, patterns, options),
1071 }
1072}
1073
1074fn split_delimiter_matches(text: &str, options: &SplitOptions) -> Vec<String> {
1075 if text.is_empty() || is_missing_string(text) {
1076 return Vec::new();
1077 }
1078 match &options.delimiters {
1079 DelimiterSpec::Whitespace => {
1080 let mut matches = Vec::new();
1081 let mut index = 0usize;
1082 while index < text.len() {
1083 let character = text[index..]
1084 .chars()
1085 .next()
1086 .expect("valid character boundary");
1087 if !character.is_whitespace() {
1088 index += character.len_utf8();
1089 continue;
1090 }
1091 if options.collapse_delimiters {
1092 let end = advance_whitespace(text, index);
1093 matches.push(text[index..end].to_string());
1094 index = end;
1095 } else {
1096 let end = index + character.len_utf8();
1097 matches.push(text[index..end].to_string());
1098 index = end;
1099 }
1100 }
1101 matches
1102 }
1103 DelimiterSpec::Patterns(patterns) => {
1104 let mut matches = Vec::new();
1105 let mut index = 0usize;
1106 while index < text.len() {
1107 let Some(pattern) = patterns
1108 .iter()
1109 .find(|candidate| text[index..].starts_with(candidate.as_str()))
1110 else {
1111 index += text[index..]
1112 .chars()
1113 .next()
1114 .expect("valid character boundary")
1115 .len_utf8();
1116 continue;
1117 };
1118 let mut end = index + pattern.len();
1119 if options.collapse_delimiters {
1120 while end < text.len() {
1121 let Some(next) = patterns
1122 .iter()
1123 .find(|candidate| text[end..].starts_with(candidate.as_str()))
1124 else {
1125 break;
1126 };
1127 end += next.len();
1128 }
1129 }
1130 matches.push(text[index..end].to_string());
1131 index = end;
1132 }
1133 matches
1134 }
1135 }
1136}
1137
1138fn split_whitespace(text: &str, options: &SplitOptions) -> Vec<String> {
1139 if text.is_empty() {
1140 return vec![String::new()];
1141 }
1142
1143 let mut parts: Vec<String> = Vec::new();
1144 let mut idx = 0usize;
1145 let mut last = 0usize;
1146 let len = text.len();
1147
1148 while idx < len {
1149 let ch = text[idx..].chars().next().unwrap();
1150 let width = ch.len_utf8();
1151 if !ch.is_whitespace() {
1152 idx += width;
1153 continue;
1154 }
1155
1156 let token = &text[last..idx];
1157 if !token.is_empty() || !options.collapse_delimiters {
1158 parts.push(token.to_string());
1159 }
1160
1161 let run_end = advance_whitespace(text, idx);
1162 if options.include_delimiters {
1163 if options.collapse_delimiters {
1164 parts.push(text[idx..run_end].to_string());
1165 } else {
1166 parts.push(text[idx..idx + width].to_string());
1167 }
1168 }
1169
1170 if options.collapse_delimiters {
1171 idx = run_end;
1172 last = run_end;
1173 } else {
1174 idx += width;
1175 last = idx;
1176 }
1177 }
1178
1179 let tail = &text[last..];
1180 if !tail.is_empty() || !options.collapse_delimiters {
1181 parts.push(tail.to_string());
1182 }
1183 if parts.is_empty() {
1184 parts.push(String::new());
1185 }
1186 parts
1187}
1188
1189fn split_by_patterns(text: &str, patterns: &[String], options: &SplitOptions) -> Vec<String> {
1190 if patterns.is_empty() {
1191 return vec![text.to_string()];
1192 }
1193
1194 let mut parts: Vec<String> = Vec::new();
1195 let mut idx = 0usize;
1196 let mut last = 0usize;
1197 while idx < text.len() {
1198 if let Some(pattern) = patterns
1199 .iter()
1200 .find(|candidate| text[idx..].starts_with(candidate.as_str()))
1201 {
1202 let token = &text[last..idx];
1203 if !token.is_empty() || !options.collapse_delimiters {
1204 parts.push(token.to_string());
1205 }
1206
1207 let pat_len = pattern.len();
1208 if options.collapse_delimiters {
1209 let mut run_end = idx + pat_len;
1210 while run_end < text.len() {
1211 if let Some(next) = patterns
1212 .iter()
1213 .find(|candidate| text[run_end..].starts_with(candidate.as_str()))
1214 {
1215 let len = next.len();
1216 if len == 0 {
1217 break;
1218 }
1219 run_end += len;
1220 } else {
1221 break;
1222 }
1223 }
1224 if options.include_delimiters {
1225 parts.push(text[idx..run_end].to_string());
1226 }
1227 idx = run_end;
1228 last = run_end;
1229 } else {
1230 if options.include_delimiters {
1231 parts.push(text[idx..idx + pat_len].to_string());
1232 }
1233 idx += pat_len;
1234 last = idx;
1235 }
1236
1237 continue;
1238 }
1239 let ch = text[idx..].chars().next().unwrap();
1240 idx += ch.len_utf8();
1241 }
1242 let tail = &text[last..];
1243 if !tail.is_empty() || !options.collapse_delimiters {
1244 parts.push(tail.to_string());
1245 }
1246 if parts.is_empty() {
1247 parts.push(String::new());
1248 }
1249 parts
1250}
1251
1252fn advance_whitespace(text: &str, mut start: usize) -> usize {
1253 while start < text.len() {
1254 let ch = text[start..].chars().next().unwrap();
1255 if !ch.is_whitespace() {
1256 break;
1257 }
1258 start += ch.len_utf8();
1259 }
1260 start
1261}
1262
1263fn extract_delimiters(value: &Value) -> BuiltinResult<Vec<String>> {
1264 match value {
1265 Value::String(text) => Ok(vec![text.clone()]),
1266 Value::StringArray(array) => Ok(array.data.clone()),
1267 Value::CharArray(array) => {
1268 if array.rows == 0 {
1269 return Ok(Vec::new());
1270 }
1271 let mut entries = Vec::with_capacity(array.rows);
1272 for row in 0..array.rows {
1273 entries.push(char_row_to_string_slice(&array.data, array.cols, row));
1274 }
1275 Ok(entries)
1276 }
1277 Value::Cell(cell) => {
1278 let mut entries = Vec::with_capacity(cell.data.len());
1279 for element in &cell.data {
1280 entries.push(
1281 cell_element_to_string(element)
1282 .ok_or_else(|| split_error(&SPLIT_ERROR_CELL_ELEMENT))?,
1283 );
1284 }
1285 Ok(entries)
1286 }
1287 _ => Err(split_error(&SPLIT_ERROR_DELIMITER_TYPE)),
1288 }
1289}
1290
1291fn cell_element_to_string(value: &Value) -> Option<String> {
1292 match value {
1293 Value::String(text) => Some(text.clone()),
1294 Value::StringArray(array) if array.data.len() == 1 => Some(array.data[0].clone()),
1295 Value::CharArray(array) if array.rows <= 1 => {
1296 if array.rows == 0 {
1297 Some(String::new())
1298 } else {
1299 Some(char_row_to_string_slice(&array.data, array.cols, 0))
1300 }
1301 }
1302 _ => None,
1303 }
1304}
1305
1306fn value_to_scalar_string(value: &Value) -> Option<String> {
1307 match value {
1308 Value::String(text) => Some(text.clone()),
1309 Value::StringArray(array) if array.data.len() == 1 => Some(array.data[0].clone()),
1310 Value::CharArray(array) if array.rows <= 1 => {
1311 if array.rows == 0 {
1312 Some(String::new())
1313 } else {
1314 Some(char_row_to_string_slice(&array.data, array.cols, 0))
1315 }
1316 }
1317 Value::Cell(cell) if cell.data.len() == 1 => cell_element_to_string(&cell.data[0]),
1318 _ => None,
1319 }
1320}
1321
1322fn parse_bool(value: &Value, name: &str) -> BuiltinResult<bool> {
1323 parse_bool_for_builtin(value, name, BUILTIN_NAME, &SPLIT_ERROR_OPTION_VALUE)
1324}
1325
1326fn parse_bool_for_builtin(
1327 value: &Value,
1328 name: &str,
1329 builtin_name: &'static str,
1330 error: &'static BuiltinErrorDescriptor,
1331) -> BuiltinResult<bool> {
1332 match value {
1333 Value::Bool(b) => Ok(*b),
1334 Value::Int(i) => Ok(!i.is_zero()),
1335 Value::Num(n) => Ok(*n != 0.0),
1336 Value::LogicalArray(array) => {
1337 if array.data.len() == 1 {
1338 Ok(array.data[0] != 0)
1339 } else {
1340 Err(builtin_error_with_descriptor(
1341 builtin_name,
1342 format!(
1343 "{builtin_name}: value for '{}' must be logical true or false",
1344 name
1345 ),
1346 error,
1347 ))
1348 }
1349 }
1350 Value::Tensor(tensor) => {
1351 if tensor::is_scalar_tensor(tensor) {
1352 if let Some(value) = tensor
1353 .integer_storage()
1354 .and_then(|storage| storage.value_at(0))
1355 {
1356 return Ok(!value.is_zero());
1357 }
1358 Ok(tensor::tensor_value_f64(tensor, 0) != 0.0)
1359 } else {
1360 Err(builtin_error_with_descriptor(
1361 builtin_name,
1362 format!(
1363 "{builtin_name}: value for '{}' must be logical true or false",
1364 name
1365 ),
1366 error,
1367 ))
1368 }
1369 }
1370 _ => {
1371 if let Some(text) = value_to_scalar_string(value) {
1372 let lowered = text.trim().to_ascii_lowercase();
1373 match lowered.as_str() {
1374 "true" | "on" | "yes" => Ok(true),
1375 "false" | "off" | "no" => Ok(false),
1376 _ => Err(builtin_error_with_descriptor(
1377 builtin_name,
1378 format!(
1379 "{builtin_name}: value for '{}' must be logical true or false",
1380 name
1381 ),
1382 error,
1383 )),
1384 }
1385 } else {
1386 Err(builtin_error_with_descriptor(
1387 builtin_name,
1388 format!(
1389 "{builtin_name}: value for '{}' must be logical true or false",
1390 name
1391 ),
1392 error,
1393 ))
1394 }
1395 }
1396 }
1397}
1398
1399fn builtin_error_with_descriptor(
1400 builtin_name: &'static str,
1401 message: impl Into<String>,
1402 error: &'static BuiltinErrorDescriptor,
1403) -> RuntimeError {
1404 let mut builder = build_runtime_error(message).with_builtin(builtin_name);
1405 if let Some(identifier) = error.identifier {
1406 builder = builder.with_identifier(identifier);
1407 }
1408 builder.build()
1409}
1410
1411fn extract_strsplit_subject(value: Value) -> BuiltinResult<(StrsplitInputKind, String)> {
1412 match value {
1413 Value::String(text) => Ok((StrsplitInputKind::String, text)),
1414 Value::StringArray(array) if array.data.len() == 1 => {
1415 Ok((StrsplitInputKind::String, array.data[0].clone()))
1416 }
1417 Value::CharArray(array) if array.rows <= 1 => {
1418 if array.rows == 0 {
1419 Ok((StrsplitInputKind::Char, String::new()))
1420 } else {
1421 Ok((
1422 StrsplitInputKind::Char,
1423 char_row_to_string_slice(&array.data, array.cols, 0),
1424 ))
1425 }
1426 }
1427 _ => Err(strsplit_error(&STRSPLIT_ERROR_INVALID_INPUT)),
1428 }
1429}
1430
1431fn strsplit_text(
1432 text: &str,
1433 options: &StrsplitOptions,
1434) -> BuiltinResult<(Vec<String>, Vec<String>)> {
1435 let regex = compile_strsplit_regex(options)?;
1436 let mut parts = Vec::new();
1437 let mut matches = Vec::new();
1438 let mut last = 0usize;
1439
1440 for found in regex.find_iter(text) {
1441 parts.push(text[last..found.start()].to_string());
1442 matches.push(found.as_str().to_string());
1443 last = found.end();
1444 }
1445
1446 parts.push(text[last..].to_string());
1447 Ok((parts, matches))
1448}
1449
1450fn compile_strsplit_regex(options: &StrsplitOptions) -> BuiltinResult<regex::Regex> {
1451 let pattern = match (&options.delimiters, options.delimiter_type) {
1452 (None, _) => {
1453 if options.collapse_delimiters {
1454 "[\\x20\\x0C\\n\\r\\t\\x0B]+".to_string()
1455 } else {
1456 "[\\x20\\x0C\\n\\r\\t\\x0B]".to_string()
1457 }
1458 }
1459 (Some(delimiters), StrsplitDelimiterType::Simple) => {
1460 let alternation = delimiters
1461 .iter()
1462 .map(|pattern| regex::escape(pattern))
1463 .collect::<Vec<_>>()
1464 .join("|");
1465 if options.collapse_delimiters {
1466 format!("(?:{alternation})+")
1467 } else {
1468 format!("(?:{alternation})")
1469 }
1470 }
1471 (Some(delimiters), StrsplitDelimiterType::RegularExpression) => {
1472 let alternation = delimiters.join("|");
1473 if options.collapse_delimiters {
1474 format!("(?:{alternation})+")
1475 } else {
1476 format!("(?:{alternation})")
1477 }
1478 }
1479 };
1480
1481 RegexBuilder::new(&pattern).build().map_err(|err| {
1482 strsplit_error_with_message(format!("strsplit: {err}"), &STRSPLIT_ERROR_REGEX_INVALID)
1483 })
1484}
1485
1486fn make_strsplit_output(tokens: Vec<String>, kind: StrsplitInputKind) -> BuiltinResult<Value> {
1487 match kind {
1488 StrsplitInputKind::String => {
1489 let len = tokens.len();
1490 let array = StringArray::new(tokens, vec![1, len]).map_err(|err| {
1491 strsplit_error_with_message(format!("strsplit: {err}"), &STRSPLIT_ERROR_INTERNAL)
1492 })?;
1493 Ok(Value::StringArray(array))
1494 }
1495 StrsplitInputKind::Char => {
1496 let values: Vec<Value> = tokens.into_iter().map(Value::String).collect();
1497 let len = values.len();
1498 make_cell(values, 1, len).map_err(|err| {
1499 strsplit_error_with_message(format!("strsplit: {err}"), &STRSPLIT_ERROR_INTERNAL)
1500 })
1501 }
1502 }
1503}
1504
1505#[derive(PartialEq, Eq)]
1506enum NameKey {
1507 CollapseDelimiters,
1508 IncludeDelimiters,
1509}
1510
1511#[derive(Clone, Copy)]
1512enum StrsplitInputKind {
1513 Char,
1514 String,
1515}
1516
1517#[derive(Clone, Copy)]
1518enum StrsplitDelimiterType {
1519 Simple,
1520 RegularExpression,
1521}
1522
1523#[derive(Clone)]
1524struct StrsplitOptions {
1525 delimiters: Option<Vec<String>>,
1526 collapse_delimiters: bool,
1527 delimiter_type: StrsplitDelimiterType,
1528}
1529
1530impl StrsplitOptions {
1531 fn parse(args: &[Value]) -> BuiltinResult<Self> {
1532 let mut index = 0usize;
1533 let mut delimiters = None;
1534
1535 if index < args.len() && !is_strsplit_name_key(&args[index]) {
1536 let list = extract_delimiters(&args[index])
1537 .map_err(|_| strsplit_error(&STRSPLIT_ERROR_DELIMITER_TYPE))?;
1538 delimiters = Some(list);
1539 index += 1;
1540 }
1541
1542 let mut collapse_delimiters = true;
1543 let mut delimiter_type = StrsplitDelimiterType::Simple;
1544
1545 while index < args.len() {
1546 let name = match strsplit_name_key(&args[index]) {
1547 Some(name) => name,
1548 None => return Err(strsplit_error(&STRSPLIT_ERROR_UNKNOWN_NAME)),
1549 };
1550 index += 1;
1551 if index >= args.len() {
1552 return Err(strsplit_error(&STRSPLIT_ERROR_NAME_VALUE_PAIR));
1553 }
1554 let value = &args[index];
1555 index += 1;
1556
1557 match name {
1558 StrsplitNameKey::CollapseDelimiters => {
1559 collapse_delimiters = parse_bool_for_builtin(
1560 value,
1561 "CollapseDelimiters",
1562 STRSPLIT_BUILTIN_NAME,
1563 &STRSPLIT_ERROR_OPTION_VALUE,
1564 )?;
1565 }
1566 StrsplitNameKey::DelimiterType => {
1567 let text = value_to_scalar_string(value)
1568 .ok_or_else(|| strsplit_error(&STRSPLIT_ERROR_DELIMITER_MODE))?;
1569 delimiter_type = match text.trim().to_ascii_lowercase().as_str() {
1570 "simple" => StrsplitDelimiterType::Simple,
1571 "regularexpression" => StrsplitDelimiterType::RegularExpression,
1572 _ => return Err(strsplit_error(&STRSPLIT_ERROR_DELIMITER_MODE)),
1573 };
1574 }
1575 }
1576 }
1577
1578 if let Some(patterns) = &delimiters {
1579 if patterns.is_empty() {
1580 return Err(strsplit_error(&STRSPLIT_ERROR_EMPTY_DELIMITER));
1581 }
1582 if matches!(delimiter_type, StrsplitDelimiterType::Simple)
1583 && patterns.iter().any(|pattern| pattern.is_empty())
1584 {
1585 return Err(strsplit_error(&STRSPLIT_ERROR_EMPTY_DELIMITER));
1586 }
1587 }
1588
1589 Ok(Self {
1590 delimiters,
1591 collapse_delimiters,
1592 delimiter_type,
1593 })
1594 }
1595}
1596
1597#[derive(PartialEq, Eq)]
1598enum StrsplitNameKey {
1599 CollapseDelimiters,
1600 DelimiterType,
1601}
1602
1603fn is_name_key(value: &Value) -> bool {
1604 name_key(value).is_some()
1605}
1606
1607fn is_strsplit_name_key(value: &Value) -> bool {
1608 strsplit_name_key(value).is_some()
1609}
1610
1611fn name_key(value: &Value) -> Option<NameKey> {
1612 value_to_scalar_string(value).and_then(|text| {
1613 let lowered = text.trim().to_ascii_lowercase();
1614 match lowered.as_str() {
1615 "collapsedelimiters" => Some(NameKey::CollapseDelimiters),
1616 "includedelimiters" => Some(NameKey::IncludeDelimiters),
1617 _ => None,
1618 }
1619 })
1620}
1621
1622fn strsplit_name_key(value: &Value) -> Option<StrsplitNameKey> {
1623 value_to_scalar_string(value).and_then(|text| {
1624 let lowered = text.trim().to_ascii_lowercase();
1625 match lowered.as_str() {
1626 "collapsedelimiters" => Some(StrsplitNameKey::CollapseDelimiters),
1627 "delimitertype" => Some(StrsplitNameKey::DelimiterType),
1628 _ => None,
1629 }
1630 })
1631}
1632
1633#[cfg(test)]
1634pub(crate) mod tests {
1635 use super::*;
1636 use crate::builtins::common::test_support;
1637 use runmat_builtins::{ResolveContext, Type};
1638 use runmat_value::{CellArray, IntValue, IntegerStorage, LogicalArray, Tensor};
1639
1640 fn split_builtin(text: Value, rest: Vec<Value>) -> BuiltinResult<Value> {
1641 futures::executor::block_on(super::split_builtin(text, rest))
1642 }
1643
1644 fn strsplit_builtin(text: Value, rest: Vec<Value>) -> BuiltinResult<Value> {
1645 futures::executor::block_on(super::strsplit_builtin(text, rest))
1646 }
1647
1648 #[test]
1649 fn split_bool_options_read_wide_uint64_truth_exactly() {
1650 assert!(parse_bool(&Value::Int(IntValue::U64(u64::MAX)), "IncludeDelimiters").unwrap());
1651
1652 for storage in [
1653 IntegerStorage::I8(vec![1]),
1654 IntegerStorage::I16(vec![1]),
1655 IntegerStorage::I32(vec![1]),
1656 IntegerStorage::I64(vec![1]),
1657 IntegerStorage::U8(vec![1]),
1658 IntegerStorage::U16(vec![1]),
1659 IntegerStorage::U32(vec![1]),
1660 IntegerStorage::U64(vec![u64::MAX]),
1661 ] {
1662 let enabled = Tensor::new_integer(storage, vec![1, 1]).expect("enabled");
1663 assert!(parse_bool(&Value::Tensor(enabled), "IncludeDelimiters").unwrap());
1664 }
1665
1666 let disabled =
1667 Tensor::new_integer(IntegerStorage::I16(vec![0]), vec![1, 1]).expect("disabled");
1668 assert!(!parse_bool(&Value::Tensor(disabled), "IncludeDelimiters").unwrap());
1669 }
1670
1671 #[test]
1672 fn split_gathers_automatic_double_dimension_but_gates_explicit_dimension() {
1673 test_support::with_test_provider(|provider| {
1674 let handle = provider
1675 .upload(&HostTensorView {
1676 data: &[1.0],
1677 shape: &[1, 1],
1678 })
1679 .expect("resident dimension");
1680 let handle =
1681 handle.with_provenance(runmat_accelerate_api::GpuHandleProvenance::Automatic);
1682 let result = split_builtin(
1683 Value::String("Mary Butler".into()),
1684 vec![Value::String(" ".into()), Value::GpuTensor(handle.clone())],
1685 )
1686 .expect("automatic residency is transparent");
1687 let Value::StringArray(result) = result else {
1688 panic!("expected string array")
1689 };
1690 assert_eq!(result.shape, vec![2, 1]);
1691
1692 let _strict = crate::compatibility::push_runmat_extensions_enabled(false);
1693 let handle =
1694 handle.with_provenance(runmat_accelerate_api::GpuHandleProvenance::Explicit);
1695 let error = split_builtin(
1696 Value::String("Mary Butler".into()),
1697 vec![Value::String(" ".into()), Value::GpuTensor(handle.clone())],
1698 )
1699 .expect_err("explicit resident dimension is gated");
1700 assert_eq!(
1701 error.identifier(),
1702 SPLIT_RESIDENT_DIMENSION_EXTENSION.error_identifier
1703 );
1704 provider.free(&handle).expect("free dimension");
1705 });
1706 }
1707
1708 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
1709 #[test]
1710 fn split_string_whitespace_default() {
1711 let input = Value::String("RunMat Accelerate Planner".to_string());
1712 let result = split_builtin(input, Vec::new()).expect("split");
1713 match result {
1714 Value::StringArray(array) => {
1715 assert_eq!(array.shape, vec![3, 1]);
1716 assert_eq!(
1717 array.data,
1718 vec![
1719 "RunMat".to_string(),
1720 "Accelerate".to_string(),
1721 "Planner".to_string()
1722 ]
1723 );
1724 }
1725 other => panic!("expected string array, got {other:?}"),
1726 }
1727 }
1728
1729 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
1730 #[test]
1731 fn split_string_custom_delimiter() {
1732 let input = Value::String("alpha,beta,gamma".to_string());
1733 let args = vec![Value::String(",".to_string())];
1734 let result = split_builtin(input, args).expect("split");
1735 match result {
1736 Value::StringArray(array) => {
1737 assert_eq!(array.shape, vec![3, 1]);
1738 assert_eq!(
1739 array.data,
1740 vec!["alpha".to_string(), "beta".to_string(), "gamma".to_string()]
1741 );
1742 }
1743 other => panic!("expected string array, got {other:?}"),
1744 }
1745 }
1746
1747 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
1748 #[test]
1749 fn split_include_delimiters_true() {
1750 let _runmat = crate::compatibility::push_runmat_extensions_enabled(true);
1751 let input = Value::String("A+B-C".to_string());
1752 let args = vec![
1753 Value::StringArray(
1754 StringArray::new(vec!["+".to_string(), "-".to_string()], vec![1, 2]).unwrap(),
1755 ),
1756 Value::String("IncludeDelimiters".to_string()),
1757 Value::Bool(true),
1758 ];
1759 let result = split_builtin(input, args).expect("split");
1760 match result {
1761 Value::StringArray(array) => {
1762 assert_eq!(array.shape, vec![5, 1]);
1763 assert_eq!(
1764 array.data,
1765 vec![
1766 "A".to_string(),
1767 "+".to_string(),
1768 "B".to_string(),
1769 "-".to_string(),
1770 "C".to_string()
1771 ]
1772 );
1773 }
1774 other => panic!("expected string array, got {other:?}"),
1775 }
1776 }
1777
1778 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
1779 #[test]
1780 fn split_include_delimiters_whitespace_collapse_default() {
1781 let _runmat = crate::compatibility::push_runmat_extensions_enabled(true);
1782 let input = Value::String("A B".to_string());
1783 let args = vec![
1784 Value::String("IncludeDelimiters".to_string()),
1785 Value::Bool(true),
1786 ];
1787 let result = split_builtin(input, args).expect("split");
1788 match result {
1789 Value::StringArray(array) => {
1790 assert_eq!(array.shape, vec![3, 1]);
1791 assert_eq!(
1792 array.data,
1793 vec!["A".to_string(), " ".to_string(), "B".to_string()]
1794 );
1795 }
1796 other => panic!("expected string array, got {other:?}"),
1797 }
1798 }
1799
1800 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
1801 #[test]
1802 fn split_patterns_include_delimiters_collapse_true() {
1803 let _runmat = crate::compatibility::push_runmat_extensions_enabled(true);
1804 let input = Value::String("a,,b".to_string());
1805 let args = vec![
1806 Value::String(",".to_string()),
1807 Value::String("IncludeDelimiters".to_string()),
1808 Value::Bool(true),
1809 Value::String("CollapseDelimiters".to_string()),
1810 Value::Bool(true),
1811 ];
1812 let result = split_builtin(input, args).expect("split");
1813 match result {
1814 Value::StringArray(array) => {
1815 assert_eq!(array.shape, vec![3, 1]);
1816 assert_eq!(
1817 array.data,
1818 vec!["a".to_string(), ",,".to_string(), "b".to_string()]
1819 );
1820 }
1821 other => panic!("expected string array, got {other:?}"),
1822 }
1823 }
1824
1825 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
1826 #[test]
1827 fn split_collapse_false_preserves_empty_segments() {
1828 let _runmat = crate::compatibility::push_runmat_extensions_enabled(true);
1829 let input = Value::String("one,,three,".to_string());
1830 let args = vec![
1831 Value::String(",".to_string()),
1832 Value::String("CollapseDelimiters".to_string()),
1833 Value::Bool(false),
1834 ];
1835 let result = split_builtin(input, args).expect("split");
1836 match result {
1837 Value::StringArray(array) => {
1838 assert_eq!(array.shape, vec![4, 1]);
1839 assert_eq!(
1840 array.data,
1841 vec![
1842 "one".to_string(),
1843 "".to_string(),
1844 "three".to_string(),
1845 "".to_string()
1846 ]
1847 );
1848 }
1849 other => panic!("expected string array, got {other:?}"),
1850 }
1851 }
1852
1853 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
1854 #[test]
1855 fn split_character_array_rows() {
1856 let mut row1: Vec<char> = "GPU Accelerate".chars().collect();
1857 let mut row2: Vec<char> = "VM Engine".chars().collect();
1858 let width = row1.len().max(row2.len());
1859 row1.resize(width, ' ');
1860 row2.resize(width, ' ');
1861 let mut data = row1;
1862 data.extend(row2);
1863 let char_array = CharArray::new(data, 2, width).unwrap();
1864 let input = Value::CharArray(char_array);
1865 let result = split_builtin(input, Vec::new()).expect("split");
1866 match result {
1867 Value::StringArray(array) => {
1868 assert_eq!(array.shape, vec![2, 2]);
1869 assert_eq!(
1870 array.data,
1871 vec![
1872 "GPU".to_string(),
1873 "VM".to_string(),
1874 "Accelerate".to_string(),
1875 "Engine".to_string()
1876 ]
1877 );
1878 }
1879 other => panic!("expected string array, got {other:?}"),
1880 }
1881 }
1882
1883 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
1884 #[test]
1885 fn split_string_array_multiple_columns() {
1886 let data = vec![
1887 "RunMat Core".to_string(),
1888 "VM Interpreter".to_string(),
1889 "Accelerate Engine".to_string(),
1890 "<missing>".to_string(),
1891 ];
1892 let array = StringArray::new(data, vec![2, 2]).unwrap();
1893 let input = Value::StringArray(array);
1894 let result = split_builtin(input, Vec::new()).expect("split");
1895 match result {
1896 Value::StringArray(array) => {
1897 assert_eq!(array.shape, vec![2, 2, 2]);
1898 assert_eq!(
1899 array.data,
1900 vec![
1901 "RunMat".to_string(),
1902 "VM".to_string(),
1903 "Accelerate".to_string(),
1904 "<missing>".to_string(),
1905 "Core".to_string(),
1906 "Interpreter".to_string(),
1907 "Engine".to_string(),
1908 "<missing>".to_string()
1909 ]
1910 );
1911 }
1912 other => panic!("expected string array, got {other:?}"),
1913 }
1914 }
1915
1916 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
1917 #[test]
1918 fn split_cell_array_outputs_string_array() {
1919 let values = vec![
1920 Value::String("RunMat Snapshot".to_string()),
1921 Value::String("Fusion Planner".to_string()),
1922 ];
1923 let cell = crate::make_cell(values, 2, 1).expect("cell");
1924 let result = split_builtin(cell, vec![Value::String(" ".to_string())]).expect("split");
1925 match result {
1926 Value::StringArray(array) => {
1927 assert_eq!(array.shape, vec![2, 2]);
1928 assert_eq!(
1929 array.data,
1930 vec![
1931 "RunMat".to_string(),
1932 "Fusion".to_string(),
1933 "Snapshot".to_string(),
1934 "Planner".to_string()
1935 ]
1936 );
1937 }
1938 other => panic!("expected string array, got {other:?}"),
1939 }
1940 }
1941
1942 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
1943 #[test]
1944 fn split_cell_array_multiple_columns() {
1945 let values = vec![
1946 Value::String("alpha beta".to_string()),
1947 Value::String("gamma".to_string()),
1948 Value::String("delta epsilon".to_string()),
1949 Value::String("<missing>".to_string()),
1950 ];
1951 let cell = crate::make_cell(values, 2, 2).expect("cell");
1952 let result = split_builtin(cell, Vec::new()).expect("split");
1953 match result {
1954 Value::StringArray(array) => {
1955 assert_eq!(array.shape, vec![2, 2, 2]);
1956 assert_eq!(
1957 array.data,
1958 vec![
1959 "alpha".to_string(),
1960 "delta".to_string(),
1961 "gamma".to_string(),
1962 "<missing>".to_string(),
1963 "beta".to_string(),
1964 "epsilon".to_string(),
1965 "<missing>".to_string(),
1966 "<missing>".to_string()
1967 ]
1968 );
1969 }
1970 other => panic!("expected string array, got {other:?}"),
1971 }
1972 }
1973
1974 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
1975 #[test]
1976 fn split_missing_string_propagates() {
1977 let input = Value::String("<missing>".to_string());
1978 let result = split_builtin(input, Vec::new()).expect("split");
1979 match result {
1980 Value::StringArray(array) => {
1981 assert_eq!(array.shape, vec![1, 1]);
1982 assert_eq!(array.data, vec!["<missing>".to_string()]);
1983 }
1984 other => panic!("expected string array, got {other:?}"),
1985 }
1986 }
1987
1988 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
1989 #[test]
1990 fn split_invalid_name_value_pair_errors() {
1991 let input = Value::String("abc".to_string());
1992 let args = vec![Value::String("CollapseDelimiters".to_string())];
1993 let err = split_builtin(input, args).unwrap_err();
1994 assert!(err.to_string().contains("name-value"));
1995 }
1996
1997 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
1998 #[test]
1999 fn split_invalid_text_argument_errors() {
2000 let err = split_builtin(Value::Num(1.0), Vec::new()).unwrap_err();
2001 assert!(err.to_string().contains("first argument"));
2002 }
2003
2004 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
2005 #[test]
2006 fn split_invalid_delimiter_type_errors() {
2007 let err =
2008 split_builtin(Value::String("abc".to_string()), vec![Value::Num(1.0)]).unwrap_err();
2009 assert!(err.to_string().contains("delimiter input"));
2010 }
2011
2012 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
2013 #[test]
2014 fn split_empty_delimiter_errors() {
2015 let err = split_builtin(
2016 Value::String("abc".to_string()),
2017 vec![Value::String(String::new())],
2018 )
2019 .unwrap_err();
2020 assert!(err.to_string().contains("at least one character"));
2021 }
2022
2023 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
2024 #[test]
2025 fn split_unknown_name_argument_errors() {
2026 let err = split_builtin(
2027 Value::String("abc".to_string()),
2028 vec![
2029 Value::String("UnknownOption".to_string()),
2030 Value::Bool(true),
2031 ],
2032 )
2033 .unwrap_err();
2034 assert!(err.to_string().contains("unrecognized"));
2035 }
2036
2037 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
2038 #[test]
2039 fn split_collapse_delimiters_accepts_logical_array() {
2040 let _runmat = crate::compatibility::push_runmat_extensions_enabled(true);
2041 let logical = LogicalArray::new(vec![1u8], vec![1]).unwrap();
2042 let args = vec![
2043 Value::String(",".to_string()),
2044 Value::String("CollapseDelimiters".to_string()),
2045 Value::LogicalArray(logical),
2046 ];
2047 let result = split_builtin(Value::String("a,,b".to_string()), args).expect("split");
2048 match result {
2049 Value::StringArray(array) => {
2050 assert_eq!(array.shape, vec![2, 1]);
2051 assert_eq!(array.data, vec!["a".to_string(), "b".to_string()]);
2052 }
2053 other => panic!("expected string array, got {other:?}"),
2054 }
2055 }
2056
2057 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
2058 #[test]
2059 fn split_include_delimiters_accepts_tensor_scalar() {
2060 let _runmat = crate::compatibility::push_runmat_extensions_enabled(true);
2061 let tensor = Tensor::new(vec![1.0], vec![1, 1]).unwrap();
2062 let args = vec![
2063 Value::String(",".to_string()),
2064 Value::String("IncludeDelimiters".to_string()),
2065 Value::Tensor(tensor),
2066 ];
2067 let result = split_builtin(Value::String("a,b".to_string()), args).expect("split");
2068 match result {
2069 Value::StringArray(array) => {
2070 assert_eq!(array.shape, vec![3, 1]);
2071 assert_eq!(
2072 array.data,
2073 vec!["a".to_string(), ",".to_string(), "b".to_string()]
2074 );
2075 }
2076 other => panic!("expected string array, got {other:?}"),
2077 }
2078 }
2079
2080 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
2081 #[test]
2082 fn split_cell_array_mixed_inputs() {
2083 let values = vec![
2084 Value::String("alpha beta".to_string()),
2085 Value::CharArray(CharArray::new("gamma".chars().collect(), 1, 5).unwrap()),
2086 ];
2087 let cell = Value::Cell(CellArray::new(values, 1, 2).expect("cell array construction"));
2088 let result = split_builtin(cell, Vec::new()).expect("split");
2089 match result {
2090 Value::StringArray(array) => {
2091 assert_eq!(array.shape, vec![1, 2, 2]);
2092 assert_eq!(
2093 array.data,
2094 vec![
2095 "alpha".to_string(),
2096 "gamma".to_string(),
2097 "beta".to_string(),
2098 "<missing>".to_string()
2099 ]
2100 );
2101 }
2102 other => panic!("expected string array, got {other:?}"),
2103 }
2104 }
2105
2106 #[test]
2107 fn split_typed_integer_dimensions_cover_all_classes_exactly() {
2108 let _runmat = crate::compatibility::push_runmat_extensions_enabled(true);
2109 for storage in [
2110 IntegerStorage::I8(vec![1]),
2111 IntegerStorage::I16(vec![1]),
2112 IntegerStorage::I32(vec![1]),
2113 IntegerStorage::I64(vec![1]),
2114 IntegerStorage::U8(vec![1]),
2115 IntegerStorage::U16(vec![1]),
2116 IntegerStorage::U32(vec![1]),
2117 IntegerStorage::U64(vec![1]),
2118 ] {
2119 let dimension = Tensor::new_integer(storage, vec![1, 1]).expect("dimension");
2120 let input = StringArray::new(
2121 vec![
2122 "Mary Butler".into(),
2123 "Diana Lee".into(),
2124 "James King".into(),
2125 ],
2126 vec![3, 1],
2127 )
2128 .expect("input");
2129 let result = split_builtin(
2130 Value::StringArray(input),
2131 vec![Value::String(" ".into()), Value::Tensor(dimension)],
2132 )
2133 .expect("typed dimension");
2134 let Value::StringArray(result) = result else {
2135 panic!("expected string array")
2136 };
2137 assert_eq!(result.shape, vec![2, 3]);
2138 assert_eq!(
2139 result.data,
2140 vec!["Mary", "Butler", "Diana", "Lee", "James", "King"]
2141 );
2142 }
2143 }
2144
2145 #[test]
2146 fn split_typed_dimension_is_gated_but_documented_double_dimension_remains_available() {
2147 let input = || Value::String("alpha beta".into());
2148 let _strict = crate::compatibility::push_runmat_extensions_enabled(false);
2149 let integer_dimension = Tensor::new_integer(IntegerStorage::U8(vec![1]), vec![1, 1])
2150 .expect("integer dimension");
2151 let error = split_builtin(
2152 input(),
2153 vec![Value::String(" ".into()), Value::Tensor(integer_dimension)],
2154 )
2155 .expect_err("typed dimension extension");
2156 assert_eq!(
2157 error.identifier(),
2158 SPLIT_TYPED_DIMENSION_EXTENSION.error_identifier
2159 );
2160
2161 let Value::StringArray(result) =
2162 split_builtin(input(), vec![Value::String(" ".into()), Value::Num(1.0)])
2163 .expect("documented double dimension")
2164 else {
2165 panic!("expected string array")
2166 };
2167 assert_eq!(result.shape, vec![2, 1]);
2168 }
2169
2170 #[test]
2171 fn split_rejects_dimensions_above_the_supported_rank_without_allocating() {
2172 let error = split_builtin(
2173 Value::String("alpha beta".into()),
2174 vec![
2175 Value::String(" ".into()),
2176 Value::Num((MAX_SPLIT_DIMENSION + 1) as f64),
2177 ],
2178 )
2179 .expect_err("oversized double dimension");
2180 assert_eq!(error.identifier(), SPLIT_ERROR_DIMENSION.identifier);
2181
2182 let _runmat = crate::compatibility::push_runmat_extensions_enabled(true);
2183 let dimension = Tensor::new_integer(
2184 IntegerStorage::U64(vec![(MAX_SPLIT_DIMENSION + 1) as u64]),
2185 vec![1, 1],
2186 )
2187 .expect("typed dimension");
2188 let error = split_builtin(
2189 Value::String("alpha beta".into()),
2190 vec![Value::String(" ".into()), Value::Tensor(dimension)],
2191 )
2192 .expect_err("oversized typed dimension");
2193 assert_eq!(error.identifier(), SPLIT_ERROR_DIMENSION.identifier);
2194 }
2195
2196 #[test]
2197 fn split_empty_input_honors_requested_output_count() {
2198 let _outputs = crate::output_count::push_output_count(Some(2));
2199 let input = StringArray::new(Vec::new(), vec![0, 1]).expect("empty input");
2200 let result = split_builtin(Value::StringArray(input), Vec::new()).expect("empty split");
2201 let Value::OutputList(outputs) = result else {
2202 panic!("expected output list")
2203 };
2204 assert_eq!(outputs.len(), 2);
2205 for output in outputs {
2206 let Value::StringArray(array) = output else {
2207 panic!("expected empty string array")
2208 };
2209 assert!(array.data.is_empty());
2210 assert_eq!(array.shape, vec![0, 1]);
2211 }
2212 }
2213
2214 #[test]
2215 fn split_second_output_contains_matched_delimiters_in_requested_orientation() {
2216 let _outputs = crate::output_count::push_output_count(Some(2));
2217 let result = split_builtin(
2218 Value::String("a,b,c".into()),
2219 vec![Value::String(",".into()), Value::Num(2.0)],
2220 )
2221 .expect("two-output split");
2222 let Value::OutputList(outputs) = result else {
2223 panic!("expected output list")
2224 };
2225 let Value::StringArray(parts) = &outputs[0] else {
2226 panic!("expected string parts")
2227 };
2228 let Value::StringArray(matches) = &outputs[1] else {
2229 panic!("expected string matches")
2230 };
2231 assert_eq!(parts.shape, vec![1, 3]);
2232 assert_eq!(parts.data, vec!["a", "b", "c"]);
2233 assert_eq!(matches.shape, vec![1, 2]);
2234 assert_eq!(matches.data, vec![",", ","]);
2235 }
2236
2237 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
2238 #[test]
2239 fn strsplit_string_scalar_returns_string_array() {
2240 let result =
2241 strsplit_builtin(Value::String("one two three".into()), Vec::new()).expect("strsplit");
2242 match result {
2243 Value::StringArray(array) => {
2244 assert_eq!(array.shape, vec![1, 3]);
2245 assert_eq!(
2246 array.data,
2247 vec!["one".to_string(), "two".to_string(), "three".to_string()]
2248 );
2249 }
2250 other => panic!("expected string array, got {other:?}"),
2251 }
2252 }
2253
2254 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
2255 #[test]
2256 fn strsplit_char_vector_returns_cell() {
2257 let input = Value::CharArray(CharArray::new("a,b".chars().collect(), 1, 3).unwrap());
2258 let result = strsplit_builtin(input, vec![Value::String(",".into())]).expect("strsplit");
2259 match result {
2260 Value::Cell(cell) => {
2261 assert_eq!(cell.rows, 1);
2262 assert_eq!(cell.cols, 2);
2263 assert_eq!(&cell.data[0], &Value::String("a".into()));
2264 assert_eq!(&cell.data[1], &Value::String("b".into()));
2265 }
2266 other => panic!("expected cell output, got {other:?}"),
2267 }
2268 }
2269
2270 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
2271 #[test]
2272 fn strsplit_multi_output_returns_matches() {
2273 let _guard = crate::output_count::push_output_count(Some(2));
2274 let result = strsplit_builtin(
2275 Value::String("a,,b,".into()),
2276 vec![Value::String(",".into())],
2277 )
2278 .expect("strsplit");
2279 match result {
2280 Value::OutputList(values) => {
2281 assert_eq!(values.len(), 2);
2282 match &values[0] {
2283 Value::StringArray(array) => {
2284 assert_eq!(
2285 array.data,
2286 vec!["a".to_string(), "b".to_string(), "".to_string()]
2287 );
2288 }
2289 other => panic!("expected first output string array, got {other:?}"),
2290 }
2291 match &values[1] {
2292 Value::StringArray(array) => {
2293 assert_eq!(array.data, vec![",,".to_string(), ",".to_string()]);
2294 }
2295 other => panic!("expected second output string array, got {other:?}"),
2296 }
2297 }
2298 other => panic!("expected output list, got {other:?}"),
2299 }
2300 }
2301
2302 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
2303 #[test]
2304 fn strsplit_regular_expression_mode() {
2305 let _guard = crate::output_count::push_output_count(Some(2));
2306 let result = strsplit_builtin(
2307 Value::String("1.21m/s 1.985 m/s".into()),
2308 vec![
2309 Value::String("\\s*m/s\\s*".into()),
2310 Value::String("DelimiterType".into()),
2311 Value::String("RegularExpression".into()),
2312 ],
2313 )
2314 .expect("strsplit");
2315 match result {
2316 Value::OutputList(values) => {
2317 match &values[0] {
2318 Value::StringArray(array) => {
2319 assert_eq!(
2320 array.data,
2321 vec!["1.21".to_string(), "1.985".to_string(), "".to_string()]
2322 );
2323 }
2324 other => panic!("expected split output string array, got {other:?}"),
2325 }
2326 match &values[1] {
2327 Value::StringArray(array) => {
2328 assert_eq!(array.data, vec!["m/s ".to_string(), " m/s".to_string()]);
2329 }
2330 other => panic!("expected matches output string array, got {other:?}"),
2331 }
2332 }
2333 other => panic!("expected output list, got {other:?}"),
2334 }
2335 }
2336
2337 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
2338 #[test]
2339 fn strsplit_collapse_false_preserves_empty_segments() {
2340 let result = strsplit_builtin(
2341 Value::String("a,,b".into()),
2342 vec![
2343 Value::String(",".into()),
2344 Value::String("CollapseDelimiters".into()),
2345 Value::Bool(false),
2346 ],
2347 )
2348 .expect("strsplit");
2349 match result {
2350 Value::StringArray(array) => {
2351 assert_eq!(
2352 array.data,
2353 vec!["a".to_string(), "".to_string(), "b".to_string()]
2354 );
2355 }
2356 other => panic!("expected string array, got {other:?}"),
2357 }
2358 }
2359
2360 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
2361 #[test]
2362 fn strsplit_rejects_nonscalar_text_inputs() {
2363 let input = Value::StringArray(
2364 StringArray::new(vec!["a b".into(), "c d".into()], vec![2, 1]).unwrap(),
2365 );
2366 let err = strsplit_builtin(input, Vec::new()).unwrap_err();
2367 assert!(err.to_string().contains("first argument"));
2368 }
2369
2370 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
2371 #[test]
2372 fn strsplit_invalid_delimiter_type_option_errors() {
2373 let err = strsplit_builtin(
2374 Value::String("a,b".into()),
2375 vec![
2376 Value::String(",".into()),
2377 Value::String("DelimiterType".into()),
2378 Value::String("BadMode".into()),
2379 ],
2380 )
2381 .unwrap_err();
2382 assert!(err.to_string().contains("DelimiterType"));
2383 }
2384
2385 #[test]
2386 fn split_type_is_string_array() {
2387 assert_eq!(
2388 string_array_type(&[Type::String], &ResolveContext::new(Vec::new())),
2389 Type::cell_of(Type::String)
2390 );
2391 }
2392}