univert 0.4.0

Universal file converter (library and CLI)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

//! JSON Lines (a.k.a. NDJSON) — one JSON value per line.

use std::fs;
use std::io::{BufRead, BufReader, Write};

use crate::{Error, Job, Result};

pub fn jsonl_to_json(job: &Job) -> Result<()> {
  let values = read_values(job)?;
  let json = serde_json::to_string(&serde_json::Value::Array(values))
    .map_err(|e| Error::other(format!("json serialize: {e}")))?;
  fs::write(job.output, json + "\n")?;
  Ok(())
}

pub fn json_to_jsonl(job: &Job) -> Result<()> {
  let raw = fs::read_to_string(job.input)?;
  let value: serde_json::Value = serde_json::from_str(&raw)
    .map_err(|e| Error::other(format!("invalid JSON: {e}")))?;
  let items = match value {
    serde_json::Value::Array(items) => items,
    other => vec![other],
  };
  let mut file = fs::File::create(job.output)?;
  for item in items {
    let line = serde_json::to_string(&item)
      .map_err(|e| Error::other(format!("json serialize: {e}")))?;
    writeln!(file, "{line}")?;
  }
  Ok(())
}

pub fn jsonl_to_csv(job: &Job) -> Result<()> {
  jsonl_to_sv(job, b',')
}

pub fn jsonl_to_tsv(job: &Job) -> Result<()> {
  jsonl_to_sv(job, b'\t')
}

pub fn csv_to_jsonl(job: &Job) -> Result<()> {
  sv_to_jsonl(job, b',')
}

pub fn tsv_to_jsonl(job: &Job) -> Result<()> {
  sv_to_jsonl(job, b'\t')
}

fn read_values(job: &Job) -> Result<Vec<serde_json::Value>> {
  let file = fs::File::open(job.input)?;
  let reader = BufReader::new(file);
  let mut out = Vec::new();
  for (i, line) in reader.lines().enumerate() {
    let line = line?;
    let trimmed = line.trim();
    if trimmed.is_empty() {
      continue;
    }
    let v: serde_json::Value = serde_json::from_str(trimmed).map_err(|e| {
      Error::other(format!("invalid JSON on line {}: {e}", i + 1))
    })?;
    out.push(v);
  }
  Ok(out)
}

fn jsonl_to_sv(job: &Job, delimiter: u8) -> Result<()> {
  let values = read_values(job)?;
  let mut headers: Vec<String> = Vec::new();
  let mut seen = std::collections::BTreeSet::new();
  for v in &values {
    if let serde_json::Value::Object(map) = v {
      for k in map.keys() {
        if seen.insert(k.clone()) {
          headers.push(k.clone());
        }
      }
    }
  }

  let mut writer = csv::WriterBuilder::new()
    .delimiter(delimiter)
    .from_path(job.output)?;
  writer.write_record(&headers)?;
  for v in &values {
    let obj = match v {
      serde_json::Value::Object(map) => map,
      _ => {
        return Err(Error::other(
          "jsonl: rows must be JSON objects to convert to csv/tsv",
        ));
      }
    };
    let row: Vec<String> = headers
      .iter()
      .map(|h| match obj.get(h) {
        None | Some(serde_json::Value::Null) => String::new(),
        Some(serde_json::Value::String(s)) => s.clone(),
        Some(v) => v.to_string(),
      })
      .collect();
    writer.write_record(&row)?;
  }
  writer.flush()?;
  Ok(())
}

fn sv_to_jsonl(job: &Job, delimiter: u8) -> Result<()> {
  let mut reader = csv::ReaderBuilder::new()
    .delimiter(delimiter)
    .has_headers(true)
    .from_path(job.input)?;
  let headers = reader.headers()?.clone();
  let mut file = fs::File::create(job.output)?;
  for record in reader.records() {
    let record = record?;
    let mut obj = serde_json::Map::with_capacity(headers.len());
    for (h, v) in headers.iter().zip(record.iter()) {
      obj.insert(h.to_string(), serde_json::Value::String(v.to_string()));
    }
    let line = serde_json::to_string(&serde_json::Value::Object(obj))
      .map_err(|e| Error::other(format!("json serialize: {e}")))?;
    writeln!(file, "{line}")?;
  }
  Ok(())
}