1use runmat_builtins::{
4 BuiltinCompletionPolicy, BuiltinDescriptor, BuiltinErrorDescriptor, BuiltinOutputMode,
5 BuiltinParamArity, BuiltinParamDescriptor, BuiltinParamType, BuiltinSignatureDescriptor,
6 CellArray, ObjectInstance, ResolveContext, StringArray, Tensor, Type, Value,
7};
8use runmat_macros::runtime_builtin;
9
10use crate::builtins::strings::text_analytics::details::add_sentence_details_builtin;
11use crate::builtins::strings::text_analytics::documents::{
12 document_token_type_with_options, documents_from_object, options_from_document_object,
13 text_analytics_error, tokenized_document_language, DocumentTokenType, TOKENIZED_DOCUMENT_CLASS,
14};
15use crate::builtins::strings::text_analytics::pos::{part_of_speech_for_token, PosLanguage};
16use crate::{gather_if_needed_async, BuiltinResult};
17
18pub(in crate::builtins::strings::text_analytics) const HEAD_DETAILS_PROPERTY: &str = "HeadDetails";
19pub(in crate::builtins::strings::text_analytics) const DEPENDENCY_DETAILS_PROPERTY: &str =
20 "DependencyDetails";
21
22const OUT_DOCUMENTS: [BuiltinParamDescriptor; 1] = [BuiltinParamDescriptor {
23 name: "updatedDocuments",
24 ty: BuiltinParamType::Any,
25 arity: BuiltinParamArity::Required,
26 default: None,
27 description: "Updated tokenized document object.",
28}];
29
30const IN_DOCUMENTS: [BuiltinParamDescriptor; 1] = [BuiltinParamDescriptor {
31 name: "documents",
32 ty: BuiltinParamType::Any,
33 arity: BuiltinParamArity::Required,
34 default: None,
35 description: "tokenizedDocument object.",
36}];
37
38const ERROR_INVALID_INPUT: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
39 code: "RM.TEXT_ANALYTICS.ADD_DEPENDENCY_DETAILS.INVALID_INPUT",
40 identifier: Some("RunMat:addDependencyDetails:InvalidInput"),
41 when: "Input is not a supported tokenizedDocument object.",
42 message: "addDependencyDetails: invalid input",
43};
44
45const ERRORS: [BuiltinErrorDescriptor; 1] = [ERROR_INVALID_INPUT];
46
47pub const ADD_DEPENDENCY_DETAILS_DESCRIPTOR: BuiltinDescriptor = BuiltinDescriptor {
48 signatures: &[BuiltinSignatureDescriptor {
49 label: "updatedDocuments = addDependencyDetails(documents)",
50 inputs: &IN_DOCUMENTS,
51 outputs: &OUT_DOCUMENTS,
52 }],
53 output_mode: BuiltinOutputMode::Fixed,
54 completion_policy: BuiltinCompletionPolicy::Public,
55 errors: &ERRORS,
56};
57
58fn any_type(_args: &[Type], _ctx: &ResolveContext) -> Type {
59 Type::Unknown
60}
61
62#[runtime_builtin(
63 name = "addDependencyDetails",
64 category = "strings/text_analytics",
65 summary = "Add dependency parse details to tokenizedDocument objects.",
66 keywords = "addDependencyDetails,text analytics,tokenizedDocument,dependency,head,universal dependencies",
67 accel = "sink",
68 type_resolver(any_type),
69 descriptor(
70 crate::builtins::strings::text_analytics::dependencies::ADD_DEPENDENCY_DETAILS_DESCRIPTOR
71 ),
72 builtin_path = "crate::builtins::strings::text_analytics::dependencies"
73)]
74async fn add_dependency_details_builtin(args: Vec<Value>) -> BuiltinResult<Value> {
75 let gathered = gather_args(args).await?;
76 let documents = parse_args(gathered)?;
77 let mut object = tokenized_document_object(documents)?;
78 let language = PosLanguage::from_document_for(
79 &tokenized_document_language(&object),
80 "addDependencyDetails",
81 )?;
82
83 if !object.properties.contains_key("SentenceNumbers") {
84 object = add_sentence_details_for_dependencies(object).await?;
85 }
86
87 let documents = documents_from_object(&object, "addDependencyDetails")?;
88 let sentence_numbers = sentence_numbers_from_object(&object, "addDependencyDetails")?;
89 let document_options = options_from_document_object(&object);
90 let (heads, dependencies) = dependency_details_for_documents(
91 &documents,
92 sentence_numbers.as_deref(),
93 language,
94 &document_options,
95 )?;
96 object
97 .properties
98 .insert(HEAD_DETAILS_PROPERTY.to_string(), heads);
99 object
100 .properties
101 .insert(DEPENDENCY_DETAILS_PROPERTY.to_string(), dependencies);
102 Ok(Value::Object(object))
103}
104
105async fn gather_args(args: Vec<Value>) -> BuiltinResult<Vec<Value>> {
106 let mut out = Vec::with_capacity(args.len());
107 for arg in args {
108 out.push(gather_if_needed_async(&arg).await.map_err(|err| {
109 text_analytics_error(
110 "addDependencyDetails",
111 format!("addDependencyDetails: failed to gather input: {err}"),
112 )
113 })?);
114 }
115 Ok(out)
116}
117
118fn parse_args(args: Vec<Value>) -> BuiltinResult<Value> {
119 if args.is_empty() {
120 return Err(text_analytics_error(
121 "addDependencyDetails",
122 "addDependencyDetails: expected tokenizedDocument input",
123 ));
124 }
125 if args.len() != 1 {
126 return Err(text_analytics_error(
127 "addDependencyDetails",
128 "addDependencyDetails: name-value options are not supported",
129 ));
130 }
131 Ok(args[0].clone())
132}
133
134fn tokenized_document_object(value: Value) -> BuiltinResult<ObjectInstance> {
135 match value {
136 Value::Object(object) if object.is_class(TOKENIZED_DOCUMENT_CLASS) => Ok(object),
137 Value::Object(object) => Err(text_analytics_error(
138 "addDependencyDetails",
139 format!(
140 "addDependencyDetails: expected tokenizedDocument object, got {}",
141 object.class_name
142 ),
143 )),
144 other => Err(text_analytics_error(
145 "addDependencyDetails",
146 format!("addDependencyDetails: expected tokenizedDocument object, got {other:?}"),
147 )),
148 }
149}
150
151async fn add_sentence_details_for_dependencies(
152 object: ObjectInstance,
153) -> BuiltinResult<ObjectInstance> {
154 let Value::Object(object) = add_sentence_details_builtin(vec![Value::Object(object)]).await?
155 else {
156 return Err(text_analytics_error(
157 "addDependencyDetails",
158 "addDependencyDetails: addSentenceDetails did not return tokenizedDocument",
159 ));
160 };
161 Ok(object)
162}
163
164fn dependency_details_for_documents(
165 documents: &[Vec<String>],
166 sentence_numbers: Option<&[Vec<f64>]>,
167 language: PosLanguage,
168 options: &crate::builtins::strings::text_analytics::documents::DocumentOptions,
169) -> BuiltinResult<(Value, Value)> {
170 let mut head_cells = Vec::with_capacity(documents.len());
171 let mut dependency_cells = Vec::with_capacity(documents.len());
172 if let Some(numbers) = sentence_numbers {
173 validate_sentence_numbers(documents, numbers)?;
174 }
175 for (doc_idx, doc) in documents.iter().enumerate() {
176 let default_numbers;
177 let numbers = if let Some(numbers) = sentence_numbers.and_then(|all| all.get(doc_idx)) {
178 numbers.as_slice()
179 } else {
180 default_numbers = vec![1.0; doc.len()];
181 default_numbers.as_slice()
182 };
183 let (heads, dependencies) =
184 dependency_details_for_document(doc, numbers, language, options);
185 head_cells.push(Value::Tensor(
186 Tensor::new(heads, vec![1, doc.len()])
187 .map_err(|err| text_analytics_error("addDependencyDetails", err))?,
188 ));
189 dependency_cells.push(Value::StringArray(
190 StringArray::new(dependencies, vec![1, doc.len()])
191 .map_err(|err| text_analytics_error("addDependencyDetails", err))?,
192 ));
193 }
194 Ok((
195 Value::Cell(
196 CellArray::new(head_cells, documents.len(), 1)
197 .map_err(|err| text_analytics_error("addDependencyDetails", err))?,
198 ),
199 Value::Cell(
200 CellArray::new(dependency_cells, documents.len(), 1)
201 .map_err(|err| text_analytics_error("addDependencyDetails", err))?,
202 ),
203 ))
204}
205
206fn validate_sentence_numbers(documents: &[Vec<String>], numbers: &[Vec<f64>]) -> BuiltinResult<()> {
207 if numbers.len() != documents.len() {
208 return Err(text_analytics_error(
209 "addDependencyDetails",
210 format!(
211 "addDependencyDetails: SentenceNumbers has {} documents but Documents has {}",
212 numbers.len(),
213 documents.len()
214 ),
215 ));
216 }
217 for (idx, (doc_numbers, doc)) in numbers.iter().zip(documents).enumerate() {
218 if doc_numbers.len() != doc.len() {
219 return Err(text_analytics_error(
220 "addDependencyDetails",
221 format!(
222 "addDependencyDetails: SentenceNumbers entry {} has {} values but document has {} tokens",
223 idx + 1,
224 doc_numbers.len(),
225 doc.len()
226 ),
227 ));
228 }
229 if doc_numbers
230 .iter()
231 .any(|value| !value.is_finite() || *value < 1.0 || value.fract() != 0.0)
232 {
233 return Err(text_analytics_error(
234 "addDependencyDetails",
235 format!(
236 "addDependencyDetails: SentenceNumbers entry {} contains invalid sentence numbers",
237 idx + 1
238 ),
239 ));
240 }
241 }
242 Ok(())
243}
244
245fn dependency_details_for_document(
246 doc: &[String],
247 sentence_numbers: &[f64],
248 language: PosLanguage,
249 options: &crate::builtins::strings::text_analytics::documents::DocumentOptions,
250) -> (Vec<f64>, Vec<String>) {
251 let mut heads = vec![0.0; doc.len()];
252 let mut dependencies = vec!["dep".to_string(); doc.len()];
253 let mut start = 0usize;
254 while start < doc.len() {
255 let sentence = sentence_numbers.get(start).copied().unwrap_or(1.0);
256 let mut end = start + 1;
257 while end < doc.len() && sentence_numbers.get(end).copied().unwrap_or(1.0) == sentence {
258 end += 1;
259 }
260 fill_sentence_dependencies(
261 doc,
262 start..end,
263 language,
264 options,
265 &mut heads,
266 &mut dependencies,
267 );
268 start = end;
269 }
270 (heads, dependencies)
271}
272
273fn fill_sentence_dependencies(
274 doc: &[String],
275 range: std::ops::Range<usize>,
276 language: PosLanguage,
277 options: &crate::builtins::strings::text_analytics::documents::DocumentOptions,
278 heads: &mut [f64],
279 dependencies: &mut [String],
280) {
281 if range.is_empty() {
282 return;
283 }
284 let root = sentence_root(doc, range.clone(), language, options);
285 for idx in range.clone() {
286 if idx == root {
287 heads[idx] = 0.0;
288 dependencies[idx] = "root".to_string();
289 continue;
290 }
291 let tag = part_of_speech_for_token(&doc[idx], language, options);
292 let (head, dependency) = dependency_for_token(doc, range.clone(), idx, root, tag, options);
293 heads[idx] = (head + 1) as f64;
294 dependencies[idx] = dependency.to_string();
295 }
296}
297
298fn sentence_root(
299 doc: &[String],
300 range: std::ops::Range<usize>,
301 language: PosLanguage,
302 options: &crate::builtins::strings::text_analytics::documents::DocumentOptions,
303) -> usize {
304 for preferred in ["verb", "auxiliary-verb"] {
305 if let Some(idx) = range
306 .clone()
307 .find(|idx| part_of_speech_for_token(&doc[*idx], language, options) == preferred)
308 {
309 return idx;
310 }
311 }
312 range
313 .clone()
314 .find(|idx| {
315 document_token_type_with_options(&doc[*idx], options) != DocumentTokenType::Punctuation
316 })
317 .unwrap_or(range.start)
318}
319
320fn dependency_for_token(
321 doc: &[String],
322 range: std::ops::Range<usize>,
323 idx: usize,
324 root: usize,
325 tag: &str,
326 options: &crate::builtins::strings::text_analytics::documents::DocumentOptions,
327) -> (usize, &'static str) {
328 match tag {
329 "punctuation" => (root, "punct"),
330 "determiner" => (
331 nearest_content_to_right(doc, idx, range.clone(), options).unwrap_or(root),
332 "det",
333 ),
334 "adjective" => (
335 nearest_content_to_right(doc, idx, range.clone(), options).unwrap_or(root),
336 "amod",
337 ),
338 "numeral" => (
339 nearest_content_to_right(doc, idx, range.clone(), options).unwrap_or(root),
340 "nummod",
341 ),
342 "adverb" => (root, "advmod"),
343 "coord-conjunction" => (
344 nearest_content_to_left(doc, idx, range.clone(), options).unwrap_or(root),
345 "cc",
346 ),
347 "subord-conjunction" => (root, "mark"),
348 "adposition" | "particle" => (
349 nearest_content_to_right(doc, idx, range.clone(), options).unwrap_or(root),
350 "case",
351 ),
352 "auxiliary-verb" => (root, "aux"),
353 "pronoun" | "noun" => {
354 if idx < root {
355 (root, "nsubj")
356 } else if has_near_left_adposition(doc, idx, range, options) {
357 (root, "obl")
358 } else {
359 (root, "obj")
360 }
361 }
362 _ => (root, "dep"),
363 }
364}
365
366fn nearest_content_to_right(
367 doc: &[String],
368 idx: usize,
369 range: std::ops::Range<usize>,
370 options: &crate::builtins::strings::text_analytics::documents::DocumentOptions,
371) -> Option<usize> {
372 ((idx + 1)..range.end).find(|candidate| is_content_token(&doc[*candidate], options))
373}
374
375fn nearest_content_to_left(
376 doc: &[String],
377 idx: usize,
378 range: std::ops::Range<usize>,
379 options: &crate::builtins::strings::text_analytics::documents::DocumentOptions,
380) -> Option<usize> {
381 (range.start..idx)
382 .rev()
383 .find(|candidate| is_content_token(&doc[*candidate], options))
384}
385
386fn has_near_left_adposition(
387 doc: &[String],
388 idx: usize,
389 range: std::ops::Range<usize>,
390 options: &crate::builtins::strings::text_analytics::documents::DocumentOptions,
391) -> bool {
392 let start = idx.saturating_sub(3).max(range.start);
393 (start..idx).any(|candidate| {
394 let lower = doc[candidate].to_ascii_lowercase();
395 matches!(
396 lower.as_str(),
397 "in" | "on" | "at" | "by" | "from" | "with" | "under" | "over" | "to" | "of"
398 ) && is_content_token(&doc[idx], options)
399 })
400}
401
402fn is_content_token(
403 token: &str,
404 options: &crate::builtins::strings::text_analytics::documents::DocumentOptions,
405) -> bool {
406 !matches!(
407 document_token_type_with_options(token, options),
408 DocumentTokenType::Punctuation
409 | DocumentTokenType::WebAddress
410 | DocumentTokenType::EmailAddress
411 | DocumentTokenType::Hashtag
412 | DocumentTokenType::AtMention
413 | DocumentTokenType::Emoticon
414 | DocumentTokenType::Emoji
415 | DocumentTokenType::Other
416 )
417}
418
419pub(in crate::builtins::strings::text_analytics) fn dependency_heads_from_object(
420 object: &ObjectInstance,
421 fn_name: &str,
422) -> BuiltinResult<Option<Vec<Vec<f64>>>> {
423 let Some(value) = object.properties.get(HEAD_DETAILS_PROPERTY) else {
424 return Ok(None);
425 };
426 let Value::Cell(cell) = value else {
427 return Err(text_analytics_error(
428 fn_name,
429 format!("{fn_name}: tokenizedDocument object has invalid HeadDetails property"),
430 ));
431 };
432 if cell.cols != 1 {
433 return Err(text_analytics_error(
434 fn_name,
435 format!("{fn_name}: tokenizedDocument object has invalid HeadDetails shape"),
436 ));
437 }
438 let mut out = Vec::with_capacity(cell.data.len());
439 for item in &cell.data {
440 let Value::Tensor(tensor) = item else {
441 return Err(text_analytics_error(
442 fn_name,
443 format!("{fn_name}: tokenizedDocument object has invalid HeadDetails entry"),
444 ));
445 };
446 if tensor.rows != 1 {
447 return Err(text_analytics_error(
448 fn_name,
449 format!("{fn_name}: tokenizedDocument object has invalid HeadDetails entry shape"),
450 ));
451 }
452 out.push(tensor.data.clone());
453 }
454 Ok(Some(out))
455}
456
457pub(in crate::builtins::strings::text_analytics) fn dependency_details_from_object(
458 object: &ObjectInstance,
459 fn_name: &str,
460) -> BuiltinResult<Option<Vec<Vec<String>>>> {
461 let Some(value) = object.properties.get(DEPENDENCY_DETAILS_PROPERTY) else {
462 return Ok(None);
463 };
464 let Value::Cell(cell) = value else {
465 return Err(text_analytics_error(
466 fn_name,
467 format!("{fn_name}: tokenizedDocument object has invalid DependencyDetails property"),
468 ));
469 };
470 if cell.cols != 1 {
471 return Err(text_analytics_error(
472 fn_name,
473 format!("{fn_name}: tokenizedDocument object has invalid DependencyDetails shape"),
474 ));
475 }
476 let mut out = Vec::with_capacity(cell.data.len());
477 for item in &cell.data {
478 let Value::StringArray(array) = item else {
479 return Err(text_analytics_error(
480 fn_name,
481 format!("{fn_name}: tokenizedDocument object has invalid DependencyDetails entry"),
482 ));
483 };
484 if array.rows != 1 {
485 return Err(text_analytics_error(
486 fn_name,
487 format!(
488 "{fn_name}: tokenizedDocument object has invalid DependencyDetails entry shape"
489 ),
490 ));
491 }
492 out.push(array.data.clone());
493 }
494 Ok(Some(out))
495}
496
497fn sentence_numbers_from_object(
498 object: &ObjectInstance,
499 fn_name: &str,
500) -> BuiltinResult<Option<Vec<Vec<f64>>>> {
501 let Some(value) = object.properties.get("SentenceNumbers") else {
502 return Ok(None);
503 };
504 let Value::Cell(cell) = value else {
505 return Err(text_analytics_error(
506 fn_name,
507 format!("{fn_name}: tokenizedDocument object has invalid SentenceNumbers property"),
508 ));
509 };
510 if cell.cols != 1 {
511 return Err(text_analytics_error(
512 fn_name,
513 format!("{fn_name}: tokenizedDocument object has invalid SentenceNumbers shape"),
514 ));
515 }
516 let mut out = Vec::with_capacity(cell.data.len());
517 for item in &cell.data {
518 let Value::Tensor(tensor) = item else {
519 return Err(text_analytics_error(
520 fn_name,
521 format!("{fn_name}: tokenizedDocument object has invalid SentenceNumbers entry"),
522 ));
523 };
524 if tensor.rows != 1 {
525 return Err(text_analytics_error(
526 fn_name,
527 format!(
528 "{fn_name}: tokenizedDocument object has invalid SentenceNumbers entry shape"
529 ),
530 ));
531 }
532 out.push(tensor.data.clone());
533 }
534 Ok(Some(out))
535}
536
537#[cfg(test)]
538mod tests {
539 use super::*;
540 use crate::builtins::strings::text_analytics::details::token_details_builtin;
541 use crate::builtins::strings::text_analytics::documents::tokenized_document_builtin;
542 use crate::builtins::table::{table_variable_names_from_object, table_variables};
543
544 fn run_tokenized(args: Vec<Value>) -> BuiltinResult<Value> {
545 futures::executor::block_on(tokenized_document_builtin(args))
546 }
547
548 fn run_add_dependency(args: Vec<Value>) -> BuiltinResult<Value> {
549 futures::executor::block_on(add_dependency_details_builtin(args))
550 }
551
552 fn run_token_details(value: Value) -> BuiltinResult<Value> {
553 futures::executor::block_on(token_details_builtin(value))
554 }
555
556 fn object(value: Value) -> ObjectInstance {
557 let Value::Object(object) = value else {
558 panic!("expected object");
559 };
560 object
561 }
562
563 fn table_column(table: &ObjectInstance, name: &str) -> Value {
564 table_variables(table)
565 .expect("table variables")
566 .fields
567 .get(name)
568 .cloned()
569 .unwrap_or_else(|| panic!("missing table column {name}"))
570 }
571
572 fn string_column(table: &ObjectInstance, name: &str) -> Vec<String> {
573 match table_column(table, name) {
574 Value::StringArray(array) => array.data,
575 other => panic!("expected string column {name}, got {other:?}"),
576 }
577 }
578
579 fn numeric_column(table: &ObjectInstance, name: &str) -> Vec<f64> {
580 match table_column(table, name) {
581 Value::Tensor(tensor) => tensor.data,
582 other => panic!("expected numeric column {name}, got {other:?}"),
583 }
584 }
585
586 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
587 #[test]
588 fn add_dependency_details_adds_head_dependency_and_sentence_columns() {
589 let docs =
590 run_tokenized(vec![Value::String("The dogs chase cats.".into())]).expect("tokenized");
591 let updated = run_add_dependency(vec![docs]).expect("dependencies");
592 let table = object(run_token_details(updated).expect("details"));
593
594 assert_eq!(
595 table_variable_names_from_object(&table).expect("names"),
596 vec![
597 "Token",
598 "DocumentNumber",
599 "SentenceNumber",
600 "LineNumber",
601 "Type",
602 "Language",
603 "Head",
604 "Dependency"
605 ]
606 );
607 assert_eq!(
608 string_column(&table, "Token"),
609 vec!["The", "dogs", "chase", "cats", "."]
610 );
611 assert_eq!(
612 numeric_column(&table, "Head"),
613 vec![2.0, 3.0, 0.0, 3.0, 3.0]
614 );
615 assert_eq!(
616 string_column(&table, "Dependency"),
617 vec!["det", "nsubj", "root", "obj", "punct"]
618 );
619 }
620
621 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
622 #[test]
623 fn add_dependency_details_respects_sentence_boundaries() {
624 let docs =
625 run_tokenized(vec![Value::String("Dogs run. Cats sleep.".into())]).expect("tokenized");
626 let updated = run_add_dependency(vec![docs]).expect("dependencies");
627 let table = object(run_token_details(updated).expect("details"));
628 assert_eq!(
629 numeric_column(&table, "Head"),
630 vec![2.0, 0.0, 2.0, 5.0, 0.0, 5.0]
631 );
632 assert_eq!(
633 string_column(&table, "Dependency"),
634 vec!["nsubj", "root", "punct", "nsubj", "root", "punct"]
635 );
636 }
637
638 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
639 #[test]
640 fn add_dependency_details_rejects_options_and_bad_languages() {
641 let docs = run_tokenized(vec![Value::String("dogs run".into())]).expect("tokenized");
642 let err = run_add_dependency(vec![
643 docs.clone(),
644 Value::String("DiscardKnownValues".into()),
645 Value::Bool(true),
646 ])
647 .expect_err("expected bad option");
648 assert!(err
649 .to_string()
650 .contains("name-value options are not supported"));
651
652 let mut unsupported = object(docs);
653 unsupported
654 .properties
655 .insert("Language".into(), Value::String("fr".into()));
656 let err =
657 run_add_dependency(vec![Value::Object(unsupported)]).expect_err("unsupported language");
658 assert!(err.to_string().contains("unsupported document language"));
659 }
660
661 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
662 #[test]
663 fn add_dependency_details_rejects_malformed_sentence_numbers() {
664 let docs = object(run_tokenized(vec![Value::String("dogs run".into())]).unwrap());
665 let mut malformed = docs;
666 malformed.properties.insert(
667 "SentenceNumbers".to_string(),
668 Value::Cell(
669 CellArray::new(
670 vec![Value::Tensor(Tensor::new(vec![1.0], vec![1, 1]).unwrap())],
671 1,
672 1,
673 )
674 .unwrap(),
675 ),
676 );
677 let err =
678 run_add_dependency(vec![Value::Object(malformed)]).expect_err("expected bad numbers");
679 assert!(err.to_string().contains("SentenceNumbers entry"));
680 }
681
682 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
683 #[test]
684 fn token_details_rejects_malformed_dependency_details() {
685 let docs = object(run_tokenized(vec![Value::String("dogs run".into())]).unwrap());
686 let mut malformed = docs.clone();
687 malformed.properties.insert(
688 HEAD_DETAILS_PROPERTY.to_string(),
689 Value::Cell(
690 CellArray::new(
691 vec![Value::Tensor(Tensor::new(vec![0.0], vec![1, 1]).unwrap())],
692 1,
693 1,
694 )
695 .unwrap(),
696 ),
697 );
698 malformed.properties.insert(
699 DEPENDENCY_DETAILS_PROPERTY.to_string(),
700 Value::Cell(
701 CellArray::new(
702 vec![Value::StringArray(
703 StringArray::new(vec!["root".into(), "dep".into()], vec![1, 2]).unwrap(),
704 )],
705 1,
706 1,
707 )
708 .unwrap(),
709 ),
710 );
711 let err = run_token_details(Value::Object(malformed)).expect_err("expected error");
712 assert!(err.to_string().contains("HeadDetails entry"));
713 }
714
715 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
716 #[test]
717 fn add_dependency_details_accepts_german_language_metadata() {
718 let docs = object(run_tokenized(vec![Value::String("Hunde laufen.".into())]).unwrap());
719 let mut german = docs;
720 german
721 .properties
722 .insert("Language".into(), Value::String("de".into()));
723 let updated = run_add_dependency(vec![Value::Object(german)]).expect("german deps");
724 let table = object(run_token_details(updated).expect("details"));
725 assert!(string_column(&table, "Dependency").contains(&"root".to_string()));
726 }
727}