1pub(crate) const RANGE_CHAR: char = '~';
15
16const MAN_YEN: u64 = 10_000;
18const SEN_YEN: u64 = 1_000;
20const OKU_YEN: u64 = 100_000_000;
22const MAX_MAN_FRACTION_DIGITS: usize = 4;
24const MAX_OKU_FRACTION_DIGITS: usize = 8;
26const MAX_SEN_DIGITS: usize = 1;
29const STRUCTURAL_CHARS: [char; 4] = [':', RANGE_CHAR, '?', '.'];
34
35#[derive(Clone)]
37enum Tok {
38 Num(String),
40 Ch(char),
42}
43
44#[must_use]
72pub fn normalize(text: &str) -> String {
73 let mapped: String = text.chars().filter_map(map_char).collect();
74 let mapped = mapped.replace("以上", "~");
76 let has_context = mapped
81 .chars()
82 .any(|c| !c.is_ascii_digit() && !STRUCTURAL_CHARS.contains(&c));
83 let toks = tokenize(&mapped);
84 let toks = strip_colons(&toks);
85 let toks = merge_comma_runs(toks);
86 let toks = strip_bonus_noise(&toks);
87 let toks = rewrite_yen_symbol(&toks);
88 let toks = distribute_range_man(&toks);
89 let toks = expand_oku(&toks);
90 let toks = expand_man(&toks);
91 let toks = rewrite_mojibake_range(toks);
92 let toks = align_units(&toks, has_context);
93 render(&toks)
94}
95
96fn map_char(c: char) -> Option<char> {
98 if c.is_whitespace() {
99 return None;
100 }
101 match c {
102 '\u{FF10}' => Some('0'),
104 '\u{FF11}' => Some('1'),
105 '\u{FF12}' => Some('2'),
106 '\u{FF13}' => Some('3'),
107 '\u{FF14}' => Some('4'),
108 '\u{FF15}' => Some('5'),
109 '\u{FF16}' => Some('6'),
110 '\u{FF17}' => Some('7'),
111 '\u{FF18}' => Some('8'),
112 '\u{FF19}' => Some('9'),
113 ',' | '、' | '、' | '\u{FF0C}' => Some(','),
115 '.' | '\u{FF0E}' => Some('.'),
117 '~' | '~' | '〜' | '-' | '‐' | '‑' | '–' | '—' | '−' | '\u{FF0D}' => {
119 Some(RANGE_CHAR)
120 }
121 '?' | '\u{FF1F}' => Some('?'),
123 '¥' | '¥' => Some('¥'),
125 '☆' | '★' | '【' | '】' | '[' | ']' | '(' | ')' | '\u{FF08}' | '\u{FF09}' | '≪' | '≫'
128 | '《' | '》' | '〈' | '〉' | '「' | '」' | '『' | '』' | ';' | '\u{FF1B}' | '・'
129 | '\\' | '※' => None,
130 ':' | '\u{FF1A}' => Some(':'),
134 _ => Some(c),
135 }
136}
137
138fn tokenize(s: &str) -> Vec<Tok> {
140 let mut toks: Vec<Tok> = Vec::new();
141 for c in s.chars() {
142 if c.is_ascii_digit() {
143 if let Some(Tok::Num(run)) = toks.last_mut() {
144 run.push(c);
145 } else {
146 toks.push(Tok::Num(c.to_string()));
147 }
148 } else {
149 toks.push(Tok::Ch(c));
150 }
151 }
152 toks
153}
154
155fn strip_colons(toks: &[Tok]) -> Vec<Tok> {
162 let mut out: Vec<Tok> = Vec::new();
163 for (i, tok) in toks.iter().enumerate() {
164 if matches!(tok, Tok::Ch(':')) {
165 let between_digits = matches!(out.last(), Some(Tok::Num(_)))
166 && matches!(toks.get(i + 1), Some(Tok::Num(_)));
167 if !between_digits {
168 continue;
169 }
170 }
171 out.push(tok.clone());
172 }
173 out
174}
175
176fn merge_comma_runs(toks: Vec<Tok>) -> Vec<Tok> {
179 let mut out: Vec<Tok> = Vec::new();
180 let mut it = toks.into_iter().peekable();
181 while let Some(tok) = it.next() {
182 match tok {
183 Tok::Ch(',') => {
184 if let (Some(Tok::Num(run)), Some(Tok::Num(_))) = (out.last_mut(), it.peek()) {
185 if let Some(Tok::Num(next)) = it.next() {
186 run.push_str(&next);
187 }
188 } else {
189 out.push(Tok::Ch(','));
190 }
191 }
192 other => out.push(other),
193 }
194 }
195 out
196}
197
198fn strip_bonus_noise(toks: &[Tok]) -> Vec<Tok> {
201 let mut out: Vec<Tok> = Vec::new();
202 let mut i = 0;
203 while i < toks.len() {
204 if let (Some(Tok::Ch('年')), Some(Tok::Num(_)), Some(Tok::Ch('回'))) =
206 (toks.get(i), toks.get(i + 1), toks.get(i + 2))
207 {
208 i += 3;
209 continue;
210 }
211 if let Some(Tok::Num(_)) = toks.get(i) {
213 let mut j = i + 1;
214 if let (Some(Tok::Ch('.')), Some(Tok::Num(_))) = (toks.get(j), toks.get(j + 1)) {
215 j += 2;
216 }
217 if let (Some(Tok::Ch(counter)), Some(Tok::Ch('月'))) = (toks.get(j), toks.get(j + 1))
218 && matches!(counter, 'ヶ' | 'ヵ' | 'カ' | 'か' | 'ケ')
219 {
220 j += 2;
221 if let Some(Tok::Ch('分')) = toks.get(j) {
222 j += 1;
223 }
224 i = j;
225 continue;
226 }
227 }
228 if let Some(tok) = toks.get(i) {
229 out.push(tok.clone());
230 }
231 i += 1;
232 }
233 out
234}
235
236fn rewrite_yen_symbol(toks: &[Tok]) -> Vec<Tok> {
241 let mut out: Vec<Tok> = Vec::new();
242 let mut i = 0;
243 while i < toks.len() {
244 if let (Some(Tok::Ch('¥')), Some(Tok::Num(run))) = (toks.get(i), toks.get(i + 1)) {
245 out.push(Tok::Num(run.clone()));
246 let needs_yen = !matches!(toks.get(i + 2), Some(Tok::Ch('円' | '万' | '千' | '.')));
247 if needs_yen {
248 out.push(Tok::Ch('円'));
249 }
250 i += 2;
251 continue;
252 }
253 if let Some(Tok::Ch('¥')) = toks.get(i) {
254 i += 1;
256 continue;
257 }
258 if let Some(tok) = toks.get(i) {
259 out.push(tok.clone());
260 }
261 i += 1;
262 }
263 out
264}
265
266fn distribute_range_man(toks: &[Tok]) -> Vec<Tok> {
274 let mut out: Vec<Tok> = Vec::new();
275 for (i, tok) in toks.iter().enumerate() {
276 out.push(tok.clone());
277 if !matches!(tok, Tok::Num(_)) {
278 continue;
279 }
280 let Some(Tok::Ch(sep)) = toks.get(i + 1) else {
281 continue;
282 };
283 if !matches!(*sep, RANGE_CHAR | '?') {
284 continue;
285 }
286 let Some(Tok::Num(_)) = toks.get(i + 2) else {
287 continue;
288 };
289 let mut j = i + 3;
290 if let (Some(Tok::Ch('.')), Some(Tok::Num(_))) = (toks.get(j), toks.get(j + 1)) {
291 j += 2;
292 }
293 if matches!(toks.get(j), Some(Tok::Ch('万'))) {
294 out.push(Tok::Ch('万'));
295 }
296 }
297 out
298}
299
300fn push_expanded(out: &mut Vec<Tok>, value: u64, next: Option<&Tok>) {
306 out.push(Tok::Num(value.to_string()));
307 if !matches!(next, Some(Tok::Ch('円'))) {
308 out.push(Tok::Ch('円'));
309 }
310}
311
312fn expand_oku(toks: &[Tok]) -> Vec<Tok> {
322 let mut out: Vec<Tok> = Vec::new();
323 let mut i = 0;
324 while i < toks.len() {
325 if let Some(Tok::Num(whole)) = toks.get(i) {
326 if let (Some(Tok::Ch('.')), Some(Tok::Num(frac)), Some(Tok::Ch('億'))) =
328 (toks.get(i + 1), toks.get(i + 2), toks.get(i + 3))
329 && let Some(value) =
330 unit_with_fraction(whole, frac, OKU_YEN, MAX_OKU_FRACTION_DIGITS)
331 {
332 push_expanded(&mut out, value, toks.get(i + 4));
333 i += 4;
334 continue;
335 }
336 if let Some(Tok::Ch('億')) = toks.get(i + 1) {
337 if let (Some(Tok::Num(man)), Some(Tok::Ch('万'))) =
339 (toks.get(i + 2), toks.get(i + 3))
340 && let Some(value) = oku_with_man(whole, man)
341 {
342 push_expanded(&mut out, value, toks.get(i + 4));
343 i += 4;
344 continue;
345 }
346 if !matches!(toks.get(i + 2), Some(Tok::Num(_)))
349 && let Some(value) = parse_u64(whole).and_then(|w| w.checked_mul(OKU_YEN))
350 {
351 push_expanded(&mut out, value, toks.get(i + 2));
352 i += 2;
353 continue;
354 }
355 }
356 }
357 if let Some(tok) = toks.get(i) {
358 out.push(tok.clone());
359 }
360 i += 1;
361 }
362 out
363}
364
365fn expand_man(toks: &[Tok]) -> Vec<Tok> {
375 let mut out: Vec<Tok> = Vec::new();
376 let mut i = 0;
377 while i < toks.len() {
378 if let Some(Tok::Num(whole)) = toks.get(i) {
379 if let (Some(Tok::Ch('.')), Some(Tok::Num(frac)), Some(Tok::Ch('万'))) =
381 (toks.get(i + 1), toks.get(i + 2), toks.get(i + 3))
382 && let Some(value) =
383 unit_with_fraction(whole, frac, MAN_YEN, MAX_MAN_FRACTION_DIGITS)
384 {
385 push_expanded(&mut out, value, toks.get(i + 4));
386 i += 4;
387 continue;
388 }
389 if let Some(Tok::Ch('万')) = toks.get(i + 1) {
390 if let (Some(Tok::Num(sen)), Some(Tok::Ch('千'))) =
392 (toks.get(i + 2), toks.get(i + 3))
393 && let Some(value) = man_with_sen(whole, sen)
394 {
395 push_expanded(&mut out, value, toks.get(i + 4));
396 i += 4;
397 continue;
398 }
399 if let Some(Tok::Num(rest)) = toks.get(i + 2) {
401 let boundary =
402 matches!(toks.get(i + 3), None | Some(Tok::Ch('円' | RANGE_CHAR)));
403 if boundary
404 && rest.len() <= MAX_MAN_FRACTION_DIGITS
405 && let Some(value) = man_with_rest(whole, rest)
406 {
407 out.push(Tok::Num(value.to_string()));
408 i += 3;
409 continue;
410 }
411 }
412 if !matches!(toks.get(i + 2), Some(Tok::Num(_)))
417 && let Some(value) = parse_u64(whole).and_then(|w| w.checked_mul(MAN_YEN))
418 {
419 push_expanded(&mut out, value, toks.get(i + 2));
420 i += 2;
421 continue;
422 }
423 }
424 }
425 if let Some(tok) = toks.get(i) {
426 out.push(tok.clone());
427 }
428 i += 1;
429 }
430 out
431}
432
433fn rewrite_mojibake_range(toks: Vec<Tok>) -> Vec<Tok> {
437 let mut out: Vec<Tok> = Vec::new();
438 let mut it = toks.into_iter().peekable();
439 while let Some(tok) = it.next() {
440 if matches!(tok, Tok::Ch('?'))
443 && matches!(out.last(), Some(Tok::Num(_) | Tok::Ch('円')))
444 && matches!(it.peek(), Some(Tok::Num(_)))
445 {
446 out.push(Tok::Ch(RANGE_CHAR));
447 } else {
448 out.push(tok);
449 }
450 }
451 out
452}
453
454fn align_units(toks: &[Tok], has_context: bool) -> Vec<Tok> {
461 let mut merged: Vec<Tok> = Vec::new();
463 let mut i = 0;
464 while i < toks.len() {
465 if let (Some(Tok::Num(a)), Some(Tok::Ch('.')), Some(Tok::Num(b)), Some(Tok::Ch('円'))) = (
466 toks.get(i),
467 toks.get(i + 1),
468 toks.get(i + 2),
469 toks.get(i + 3),
470 ) {
471 let mut joined = a.clone();
472 joined.push_str(b);
473 merged.push(Tok::Num(joined));
474 i += 3; continue;
476 }
477 if let Some(tok) = toks.get(i) {
478 merged.push(tok.clone());
479 }
480 i += 1;
481 }
482 let mut out: Vec<Tok> = Vec::new();
484 for (i, tok) in merged.iter().enumerate() {
485 out.push(tok.clone());
486 if matches!(tok, Tok::Num(_))
487 && matches!(merged.get(i + 1), Some(Tok::Ch(RANGE_CHAR)))
488 && range_right_is_amount(&merged, i + 2)
489 {
490 out.push(Tok::Ch('円'));
491 }
492 }
493 if has_context && matches!(out.last(), Some(Tok::Num(_))) && !ends_with_bare_range(&out) {
495 out.push(Tok::Ch('円'));
496 }
497 out
498}
499
500fn ends_with_bare_range(toks: &[Tok]) -> bool {
507 let len = toks.len();
508 len >= 3
509 && matches!(toks.get(len - 2), Some(Tok::Ch(RANGE_CHAR)))
510 && matches!(toks.get(len - 3), Some(Tok::Num(_)))
511}
512
513fn range_right_is_amount(toks: &[Tok], start: usize) -> bool {
522 match toks.get(start) {
523 None => true,
524 Some(Tok::Num(_)) => matches!(toks.get(start + 1), Some(Tok::Ch('円'))),
525 Some(Tok::Ch(_)) => false,
526 }
527}
528
529fn render(toks: &[Tok]) -> String {
531 let mut s = String::new();
532 for tok in toks {
533 match tok {
534 Tok::Num(run) => s.push_str(run),
535 Tok::Ch(c) => s.push(*c),
536 }
537 }
538 s
539}
540
541fn parse_u64(digits: &str) -> Option<u64> {
543 digits.parse::<u64>().ok()
544}
545
546fn unit_with_fraction(whole: &str, frac: &str, unit_yen: u64, max_digits: usize) -> Option<u64> {
552 if frac.len() > max_digits {
553 return None;
554 }
555 let scale = 10_u64.pow(u32::try_from(max_digits - frac.len()).ok()?);
556 let base = parse_u64(whole)?.checked_mul(unit_yen)?;
557 let fraction = parse_u64(frac)?.checked_mul(scale)?;
558 base.checked_add(fraction)
559}
560
561fn oku_with_man(whole: &str, man: &str) -> Option<u64> {
563 let base = parse_u64(whole)?.checked_mul(OKU_YEN)?;
564 let man_part = parse_u64(man)?.checked_mul(MAN_YEN)?;
565 base.checked_add(man_part)
566}
567
568fn man_with_sen(whole: &str, sen: &str) -> Option<u64> {
570 if sen.len() != MAX_SEN_DIGITS {
571 return None;
572 }
573 let base = parse_u64(whole)?.checked_mul(MAN_YEN)?;
574 let sen_part = parse_u64(sen)?.checked_mul(SEN_YEN)?;
575 base.checked_add(sen_part)
576}
577
578fn man_with_rest(whole: &str, rest: &str) -> Option<u64> {
580 let base = parse_u64(whole)?.checked_mul(MAN_YEN)?;
581 base.checked_add(parse_u64(rest)?)
582}
583
584#[cfg(test)]
585mod tests {
586 use super::normalize;
587
588 #[test]
589 fn 全角数字とカンマ連鎖を半角へ揃える() {
590 assert_eq!(normalize("1,000,000円"), "1000000円");
591 assert_eq!(normalize("210000円"), "210000円");
592 assert_eq!(normalize("1234567890円"), "1234567890円");
593 assert_eq!(normalize("1、000円"), "1000円");
594 assert_eq!(normalize("応相談、経験による"), "応相談,経験による");
596 }
597
598 #[test]
599 fn 万と千の表記を桁へ展開する() {
600 assert_eq!(normalize("21万円"), "210000円");
601 assert_eq!(normalize("21.5万円"), "215000円");
602 assert_eq!(normalize("21.05万円"), "210500円");
603 assert_eq!(normalize("21万5千円"), "215000円");
604 assert_eq!(normalize("21万5000円"), "215000円");
605 assert_eq!(normalize("月給20万"), "月給200000円");
606 }
607
608 #[test]
609 fn 範囲区切りのゆれを正準形へ揃える() {
610 assert_eq!(normalize("20万〜25万円"), "200000円~250000円");
611 assert_eq!(normalize("20万-25万円"), "200000円~250000円");
612 assert_eq!(normalize("20万?25万円"), "200000円~250000円");
613 assert_eq!(normalize("1200円以上"), "1200円~");
614 assert_eq!(normalize("1200以上"), "1200円~");
616 }
617
618 #[test]
619 fn 範囲の右側だけの万を左へ分配する() {
620 assert_eq!(normalize("20~25万円"), "200000円~250000円");
621 assert_eq!(normalize("300〜400万円"), "3000000円~4000000円");
622 assert_eq!(normalize("20.5~25万円"), "205000円~250000円");
623 assert_eq!(normalize("20~25.5万円"), "200000円~255000円");
624 assert_eq!(normalize("1200~1500円"), "1200円~1500円");
626 assert_eq!(normalize("20~応相談"), "20~応相談");
629 }
630
631 #[test]
632 fn 円記号を円表記へ揃える() {
633 assert_eq!(normalize("¥1200"), "1200円");
634 assert_eq!(normalize("¥1200円"), "1200円");
635 assert_eq!(normalize("¥20万"), "200000円");
636 assert_eq!(normalize("値段は¥です"), "値段はです");
637 }
638
639 #[test]
640 fn 賞与などの給与外数値を落とす() {
641 assert_eq!(normalize("月給20万円(賞与年2回)"), "月給200000円賞与");
642 assert_eq!(normalize("月給20万円 賞与4.45ヶ月分"), "月給200000円賞与");
643 assert_eq!(normalize("賞与3カ月"), "賞与");
644 }
645
646 #[test]
647 fn 装飾記号と空白を取り除く() {
648 assert_eq!(normalize("【月給】 210,000円"), "月給210000円");
649 assert_eq!(normalize("☆時給:1200円☆"), "時給1200円");
650 }
651
652 #[test]
653 fn 桁区切り誤記の小数点を救済する() {
654 assert_eq!(normalize("1.200円"), "1200円");
655 }
656
657 #[test]
658 fn 桁あふれする破損データは展開しない() {
659 let huge = "99999999999999999999万円";
661 assert_eq!(normalize(huge), huge);
662 let huge_rest = "99999999999999999999万5000円";
664 assert_eq!(normalize(huge_rest), huge_rest);
665 }
666
667 #[test]
668 fn 位取りに収まらない万表記は展開しない() {
669 assert_eq!(normalize("1.55555万円"), "1555550000円");
672 assert_eq!(normalize("2万55千円"), "2万55千円");
676 }
677
678 #[test]
679 fn 単位展開で桁が連結する形は展開しない() {
680 assert_eq!(normalize("月給5万12345円"), "月給5万12345円");
683 assert_eq!(normalize("1億23456789円"), "1億23456789円");
684 }
685
686 #[test]
687 fn 億を桁へ展開する() {
688 assert_eq!(normalize("年収1億円"), "年収100000000円");
689 assert_eq!(normalize("年収1億2000万円"), "年収120000000円");
690 assert_eq!(normalize("年収1.5億円"), "年収150000000円");
691 let huge = "99999999999999999999億円";
693 assert_eq!(normalize(huge), huge);
694 }
695
696 #[test]
697 fn 金額でない範囲の右端へは円を補わない() {
698 assert_eq!(normalize("〒150-0001"), "〒150~0001");
701 assert_eq!(normalize("東京都渋谷区1-2-3"), "東京都渋谷区1~2~3");
702 assert_eq!(normalize("03-1234-5678"), "03~1234~5678");
703 assert_eq!(normalize("月給20~25万"), "月給200000円~250000円");
705 }
706
707 #[test]
708 fn 数字に挟まれたコロンだけ残す() {
709 assert_eq!(normalize("時給:1200円"), "時給1200円");
711 assert_eq!(normalize("9:00~18:00"), "9:00~18:00");
713 assert_eq!(normalize("9:00~18:00 月給25万円"), "9:00~18:00月給250000円");
714 }
715
716 #[test]
717 fn 数値のみのテキストへは円を補わない() {
718 assert_eq!(normalize("210000"), "210000");
719 assert_eq!(normalize("210,000"), "210000円");
721 }
722}