cie_csv_to_json/
cie_csv_to_json.rs

1//! Convert CIE data-table CSV files to the spectral-io JSON format.
2//!
3//! # Usage
4//!
5//! ```text
6//! cargo run --example cie_csv_to_json --features csv [-- --input DIR --output DIR]
7//! ```
8//!
9//! Reads raw CIE CSV files from `--input` (default `data/cie-raw/`) and writes
10//! JSON files to `--output` (default `data/spectral-io/cie/`).
11//! The generated files are published in the
12//! [spectral-data](https://github.com/harbik/spectral-data) repository under
13//! `spectra/cie/`; they are not committed to this repo.
14//!
15//! # Obtaining the source files
16//!
17//! Download the required CSVs from <https://cie.co.at/data-tables> (CC BY-SA 4.0):
18//!
19//! ```sh
20//! mkdir -p data/cie-raw && cd data/cie-raw
21//! curl -O https://files.cie.co.at/CIE_std_illum_A_1nm.csv
22//! curl -O https://files.cie.co.at/CIE_std_illum_D50.csv
23//! curl -O https://files.cie.co.at/CIE_std_illum_D65.csv
24//! curl -O https://files.cie.co.at/CIE_illum_C.csv
25//! curl -O https://files.cie.co.at/CIE_illum_D55.csv
26//! curl -O https://files.cie.co.at/CIE_illum_D75.csv
27//! curl -O https://files.cie.co.at/CIE_illum_HPs.csv
28//! curl -O https://files.cie.co.at/CIE_illum_FLs.csv
29//! curl -O https://files.cie.co.at/CIE_illum_FLs_1nm.csv
30//! curl -O https://files.cie.co.at/CIE_illum_LEDs.csv
31//! curl -O https://files.cie.co.at/CIE_illum_LEDs_1nm.csv
32//! curl -O https://files.cie.co.at/CIE_illum_ID50.csv
33//! curl -O https://files.cie.co.at/CIE_illum_ID65.csv
34//! curl -O https://files.cie.co.at/CIE_illum_Dxx_comp.csv
35//! curl -O https://files.cie.co.at/CIE_srf_cri.csv
36//! curl -O https://files.cie.co.at/CIE_srf_cfi.csv
37//! curl -O https://files.cie.co.at/CIE_srf_cfi_1nm.csv
38//! curl -O https://files.cie.co.at/CIE_srf_CQS_5nm.csv
39//! curl -O https://files.cie.co.at/CIE_srf_FCI_5nm.csv
40//! curl -O https://files.cie.co.at/CIE_srf_PS_5nm.csv
41//! # Spectral sensitivity / response functions
42//! curl -O https://files.cie.co.at/CIE_xyz_1931_2deg.csv
43//! curl -O https://files.cie.co.at/CIE_xyz_1964_10deg.csv
44//! curl -O https://files.cie.co.at/CIE_lms_cf_2deg.csv
45//! curl -O https://files.cie.co.at/CIE_lms_cf_10deg.csv
46//! curl -O "https://files.cie.co.at/CIE_a-opic_action_spectra.csv"
47//! curl -O https://files.cie.co.at/CIE_cfb_stv_2deg.csv
48//! curl -O https://files.cie.co.at/CIE_cfb_stv_10deg.csv
49//! curl -O https://files.cie.co.at/CIE_cfb_sle_2deg.csv
50//! curl -O https://files.cie.co.at/CIE_cfb_sle_10deg.csv
51//! curl -O https://files.cie.co.at/CIE_sle_photopic.csv
52//! curl -O https://files.cie.co.at/CIE_sle_scotopic.csv
53//! curl -O https://files.cie.co.at/CIE_RefSpectrum_L41.csv
54//! curl -O https://files.cie.co.at/CIE_1st_deriv_meta_ind.csv
55//! ```
56
57use spectral_io::{
58    BatchMetadata, MeasurementType, Provenance, SpectralData, SpectrumFile, SpectrumMetadata,
59    SpectrumRecord, WavelengthAxis, WavelengthRange,
60};
61use std::{
62    env, fs,
63    path::{Path, PathBuf},
64    process,
65};
66
67// ─────────────────────────────────────────────────────────────────────────────
68// Dataset catalogue
69// ─────────────────────────────────────────────────────────────────────────────
70
71struct Dataset {
72    csv_file: &'static str,
73    subdir: &'static str,
74    json_file: &'static str,
75    title: &'static str,
76    mtype: &'static str,
77    date: &'static str,
78    source: &'static str,
79    doi: &'static str,
80    columns: Vec<String>,
81}
82
83fn datasets() -> Vec<Dataset> {
84    let fl_cols: Vec<String> = (1u8..=12)
85        .map(|i| format!("FL{i}"))
86        .chain((1u8..=15).map(|i| format!("FL3.{i}")))
87        .collect();
88    let led_cols: Vec<String> = [
89        "LED-B1", "LED-B2", "LED-B3", "LED-B4", "LED-B5", "LED-BH1", "LED-RGB1", "LED-V1", "LED-V2",
90    ]
91    .iter()
92    .map(|s| s.to_string())
93    .collect();
94    let cri_cols: Vec<String> = (1u8..=14).map(|i| format!("TCS{i:02}")).collect();
95    // CFI specifies 99 samples; the published CSV may contain 99 or 100 columns.
96    // We provide 99 names; any extra column gets the csv_parse fallback id.
97    let cfi_cols: Vec<String> = (1u8..=99).map(|i| format!("CS{i:02}")).collect();
98    let cqs_cols: Vec<String> = (1u8..=15).map(|i| format!("VS{i}")).collect();
99
100    vec![
101        // ── Standard illuminants ─────────────────────────────────────────────
102        Dataset {
103            csv_file: "CIE_std_illum_A_1nm.csv",
104            subdir: "illuminants",
105            json_file: "cie_std_illum_a.json",
106            title: "CIE Standard Illuminant A",
107            mtype: "irradiance",
108            date: "2018-01-01",
109            source: "CIE 015:2018 Colorimetry, 4th Edition, Equation 4.1",
110            doi: "10.25039/CIE.DS.8jsxjrsn",
111            columns: vec!["A".into()],
112        },
113        Dataset {
114            csv_file: "CIE_std_illum_D50.csv",
115            subdir: "illuminants",
116            json_file: "cie_std_illum_d50.json",
117            title: "CIE Standard Illuminant D50",
118            mtype: "irradiance",
119            date: "2022-01-01",
120            source: "ISO/CIE 11664-2:2022 Colorimetry — Part 2: CIE Standard Illuminants, Table B.1",
121            doi: "10.25039/CIE.DS.etgmuqt5",
122            columns: vec!["D50".into()],
123        },
124        Dataset {
125            csv_file: "CIE_std_illum_D65.csv",
126            subdir: "illuminants",
127            json_file: "cie_std_illum_d65.json",
128            title: "CIE Standard Illuminant D65",
129            mtype: "irradiance",
130            date: "2022-01-01",
131            source: "ISO/CIE 11664-2:2022 Colorimetry — Part 2: CIE Standard Illuminants, Table B.1",
132            doi: "10.25039/CIE.DS.hjfjmt59",
133            columns: vec!["D65".into()],
134        },
135        Dataset {
136            csv_file: "CIE_illum_C.csv",
137            subdir: "illuminants",
138            json_file: "cie_std_illum_c.json",
139            title: "CIE Standard Illuminant C",
140            mtype: "irradiance",
141            date: "2018-01-01",
142            source: "CIE 015:2018 Colorimetry, 4th Edition, Table 5",
143            doi: "10.25039/CIE.DS.mjdd2enu",
144            columns: vec!["C".into()],
145        },
146        Dataset {
147            csv_file: "CIE_illum_D55.csv",
148            subdir: "illuminants",
149            json_file: "cie_std_illum_d55.json",
150            title: "CIE Standard Illuminant D55",
151            mtype: "irradiance",
152            date: "2018-01-01",
153            source: "CIE 015:2018 Colorimetry, 4th Edition, Table 5",
154            doi: "10.25039/CIE.DS.qewfb3kp",
155            columns: vec!["D55".into()],
156        },
157        Dataset {
158            csv_file: "CIE_illum_D75.csv",
159            subdir: "illuminants",
160            json_file: "cie_std_illum_d75.json",
161            title: "CIE Standard Illuminant D75",
162            mtype: "irradiance",
163            date: "2018-01-01",
164            source: "CIE 015:2018 Colorimetry, 4th Edition, Table 5",
165            doi: "10.25039/CIE.DS.9fvcmrk4",
166            columns: vec!["D75".into()],
167        },
168        // ── Discharge and gas-discharge lamps ────────────────────────────────
169        Dataset {
170            csv_file: "CIE_illum_HPs.csv",
171            subdir: "illuminants",
172            json_file: "cie_illum_hp_lamps.json",
173            title: "CIE High-Pressure Discharge Lamp Illuminants HP1–HP5",
174            mtype: "irradiance",
175            date: "2018-01-01",
176            source: "CIE 015:2018 Colorimetry, 4th Edition, Table 11",
177            doi: "10.25039/CIE.DS.f6rvvnev",
178            columns: vec!["HP1".into(), "HP2".into(), "HP3".into(), "HP4".into(), "HP5".into()],
179        },
180        // ── Fluorescent lamp illuminants ─────────────────────────────────────
181        Dataset {
182            csv_file: "CIE_illum_FLs.csv",
183            subdir: "illuminants",
184            json_file: "cie_illum_fl_lamps_5nm.json",
185            title: "CIE Fluorescent Lamp Illuminants FL1–FL12 and FL3.1–FL3.15 (5 nm)",
186            mtype: "irradiance",
187            date: "2018-01-01",
188            source: "CIE 015:2018 Colorimetry, 4th Edition, Tables 10.1–10.3",
189            doi: "10.25039/CIE.DS.vgssnyfg",
190            columns: fl_cols.clone(),
191        },
192        Dataset {
193            csv_file: "CIE_illum_FLs_1nm.csv",
194            subdir: "illuminants",
195            json_file: "cie_illum_fl_lamps_1nm.json",
196            title: "CIE Fluorescent Lamp Illuminants FL1–FL12 and FL3.1–FL3.15 (1 nm)",
197            mtype: "irradiance",
198            date: "2018-01-01",
199            source: "CIE 015:2018 Colorimetry, 4th Edition, Tables 10.1–10.3",
200            doi: "10.25039/CIE.DS.54hy6srn",
201            columns: fl_cols,
202        },
203        // ── LED illuminants ──────────────────────────────────────────────────
204        Dataset {
205            csv_file: "CIE_illum_LEDs.csv",
206            subdir: "illuminants",
207            json_file: "cie_illum_led_lamps_5nm.json",
208            title: "CIE LED Illuminants LED-B1–B5, LED-BH1, LED-RGB1, LED-V1–V2 (5 nm)",
209            mtype: "irradiance",
210            date: "2018-01-01",
211            source: "CIE 015:2018 Colorimetry, 4th Edition, Table 12",
212            doi: "10.25039/CIE.DS.vgssnyfg",
213            columns: led_cols.clone(),
214        },
215        Dataset {
216            csv_file: "CIE_illum_LEDs_1nm.csv",
217            subdir: "illuminants",
218            json_file: "cie_illum_led_lamps_1nm.json",
219            title: "CIE LED Illuminants LED-B1–B5, LED-BH1, LED-RGB1, LED-V1–V2 (1 nm)",
220            mtype: "irradiance",
221            date: "2018-01-01",
222            source: "CIE 015:2018 Colorimetry, 4th Edition, Table 12",
223            doi: "10.25039/CIE.DS.dhcw57sd",
224            columns: led_cols,
225        },
226        // ── Indoor daylight illuminants ──────────────────────────────────────
227        Dataset {
228            csv_file: "CIE_illum_ID50.csv",
229            subdir: "illuminants",
230            json_file: "cie_illum_id50.json",
231            title: "CIE Indoor Daylight Illuminant ID50",
232            mtype: "irradiance",
233            date: "2009-01-01",
234            source: "CIE 184:2009 Indoor Daylight Illuminants",
235            doi: "10.25039/CIE.DS.r4gcnrzc",
236            columns: vec!["ID50".into()],
237        },
238        Dataset {
239            csv_file: "CIE_illum_ID65.csv",
240            subdir: "illuminants",
241            json_file: "cie_illum_id65.json",
242            title: "CIE Indoor Daylight Illuminant ID65",
243            mtype: "irradiance",
244            date: "2009-01-01",
245            source: "CIE 184:2009 Indoor Daylight Illuminants",
246            doi: "10.25039/CIE.DS.bd53qdqk",
247            columns: vec!["ID65".into()],
248        },
249        // ── Daylight spectral components ─────────────────────────────────────
250        Dataset {
251            csv_file: "CIE_illum_Dxx_comp.csv",
252            subdir: "illuminants",
253            json_file: "cie_illum_daylight_components.json",
254            title: "CIE Daylight Spectral Components S0, S1, S2",
255            mtype: "irradiance",
256            date: "2018-01-01",
257            source: "CIE 015:2018 Colorimetry, 4th Edition",
258            doi: "10.25039/CIE.DS.w7zunnny",
259            columns: vec!["S0".into(), "S1".into(), "S2".into()],
260        },
261        // ── Colour rendering test samples ────────────────────────────────────
262        Dataset {
263            csv_file: "CIE_srf_cri.csv",
264            subdir: "color_rendering",
265            json_file: "cie_cri_14_test_samples.json",
266            title: "CIE Colour Rendering Index — Spectral Radiance Factors of 14 Test Colour Samples",
267            mtype: "reflectance",
268            date: "1995-01-01",
269            source: "CIE 13.3:1995 Method of Measuring and Specifying Colour Rendering Properties of Light Sources, Table A7.1",
270            doi: "10.25039/CIE.DS.wuiuu9cz",
271            columns: cri_cols,
272        },
273        Dataset {
274            csv_file: "CIE_srf_cfi.csv",
275            subdir: "color_rendering",
276            json_file: "cie_cfi_99_test_samples_5nm.json",
277            title: "CIE Colour Fidelity Index — Spectral Radiance Factors of 99 Test Colour Samples (5 nm)",
278            mtype: "reflectance",
279            date: "2017-01-01",
280            source: "CIE 224:2017 Colour Fidelity Index for Accurate Scientific Use, Table A.1",
281            doi: "10.25039/CIE.DS.wi5idbqu",
282            columns: cfi_cols.clone(),
283        },
284        Dataset {
285            csv_file: "CIE_srf_cfi_1nm.csv",
286            subdir: "color_rendering",
287            json_file: "cie_cfi_99_test_samples_1nm.json",
288            title: "CIE Colour Fidelity Index — Spectral Radiance Factors of 99 Test Colour Samples (1 nm)",
289            mtype: "reflectance",
290            date: "2017-01-01",
291            source: "CIE 224:2017 Colour Fidelity Index for Accurate Scientific Use, CFI Calculator Toolbox",
292            doi: "10.25039/CIE.DS.8svs5rqd",
293            columns: cfi_cols,
294        },
295        Dataset {
296            csv_file: "CIE_srf_CQS_5nm.csv",
297            subdir: "color_rendering",
298            json_file: "cie_cqs_15_test_samples.json",
299            title: "CIE Colour Quality Scale — Spectral Radiance Factors of 15 Test Colour Samples (5 nm)",
300            mtype: "reflectance",
301            date: "2024-01-01",
302            source: "CIE 253:2024 Colour Quality Scale, Table A.1",
303            doi: "10.25039/CIE.DS.yzfhz3cm",
304            columns: cqs_cols,
305        },
306        Dataset {
307            csv_file: "CIE_srf_FCI_5nm.csv",
308            subdir: "color_rendering",
309            json_file: "cie_four_colour_rygb.json",
310            title: "CIE Four-Colour Combination — Spectral Radiance Factors of Red, Yellow, Green, Blue (5 nm)",
311            mtype: "reflectance",
312            date: "2018-01-01",
313            source: "CIE 015:2018 Colorimetry, 4th Edition",
314            doi: "10.25039/CIE.DS.vkss79ef",
315            columns: vec!["Red".into(), "Yellow".into(), "Green".into(), "Blue".into()],
316        },
317        Dataset {
318            csv_file: "CIE_srf_PS_5nm.csv",
319            subdir: "color_rendering",
320            json_file: "cie_japanese_skin_complexion.json",
321            title: "CIE Test Colour Sample 15 — Japanese Skin Complexion (5 nm)",
322            mtype: "reflectance",
323            date: "2018-01-01",
324            source: "CIE 015:2018 Colorimetry, 4th Edition",
325            doi: "10.25039/CIE.DS.7chm7z5h",
326            columns: vec!["beta_15".into()],
327        },
328        // ── Colour matching functions ─────────────────────────────────────────
329        Dataset {
330            csv_file: "CIE_xyz_1931_2deg.csv",
331            subdir: "sensitivity",
332            json_file: "cie_cmf_1931_2deg.json",
333            title: "CIE 1931 Colour-Matching Functions, 2° Observer",
334            mtype: "sensitivity",
335            date: "2019-01-01",
336            source: "ISO/CIE 11664-1:2019 Colorimetry — Part 1: CIE Standard Colorimetric Observers, Annex A",
337            doi: "10.25039/CIE.DS.xvudnb9b",
338            columns: vec!["x_bar".into(), "y_bar".into(), "z_bar".into()],
339        },
340        Dataset {
341            csv_file: "CIE_xyz_1964_10deg.csv",
342            subdir: "sensitivity",
343            json_file: "cie_cmf_1964_10deg.json",
344            title: "CIE 1964 Colour-Matching Functions, 10° Observer",
345            mtype: "sensitivity",
346            date: "2019-01-01",
347            source: "ISO/CIE 11664-1:2019 Colorimetry — Part 1: CIE Standard Colorimetric Observers, Annex B",
348            doi: "10.25039/CIE.DS.sqksu2n5",
349            columns: vec!["x_bar_10".into(), "y_bar_10".into(), "z_bar_10".into()],
350        },
351        // ── LMS cone fundamentals ─────────────────────────────────────────────
352        Dataset {
353            csv_file: "CIE_lms_cf_2deg.csv",
354            subdir: "sensitivity",
355            json_file: "cie_lms_cone_fundamentals_2deg.json",
356            title: "CIE 2006 LMS Cone Fundamentals, 2° Field Size",
357            mtype: "sensitivity",
358            date: "2006-01-01",
359            source: "CIE 170-1:2006 Fundamental Chromaticity Diagram with Physiological Axes — Part 1, Table 2",
360            doi: "10.25039/CIE.DS.tijidesg",
361            columns: vec!["l_bar".into(), "m_bar".into(), "s_bar".into()],
362        },
363        Dataset {
364            csv_file: "CIE_lms_cf_10deg.csv",
365            subdir: "sensitivity",
366            json_file: "cie_lms_cone_fundamentals_10deg.json",
367            title: "CIE 2006 LMS Cone Fundamentals, 10° Field Size",
368            mtype: "sensitivity",
369            date: "2006-01-01",
370            source: "CIE 170-1:2006 Fundamental Chromaticity Diagram with Physiological Axes — Part 1, Table 3",
371            doi: "10.25039/CIE.DS.nxsqeri8",
372            columns: vec!["l_bar_10".into(), "m_bar_10".into(), "s_bar_10".into()],
373        },
374        // Alpha-opic action spectra handled separately (each photoreceptor has
375        // its own valid wavelength range; see convert_alpha_opic below).
376        // ── Cone-fundamental-based functions ─────────────────────────────────
377        Dataset {
378            csv_file: "CIE_cfb_stv_2deg.csv",
379            subdir: "sensitivity",
380            json_file: "cie_cfb_xyz_2deg.json",
381            title: "CIE Cone-Fundamental-Based Spectral Tristimulus Values, 2° Field Size",
382            mtype: "sensitivity",
383            date: "2015-01-01",
384            source: "CIE 170-2:2015 Fundamental Chromaticity Diagram with Physiological Axes — Part 2, Table 1",
385            doi: "10.25039/CIE.DS.548rw69q",
386            columns: vec!["x_F".into(), "y_F".into(), "z_F".into()],
387        },
388        Dataset {
389            csv_file: "CIE_cfb_stv_10deg.csv",
390            subdir: "sensitivity",
391            json_file: "cie_cfb_xyz_10deg.json",
392            title: "CIE Cone-Fundamental-Based Spectral Tristimulus Values, 10° Field Size",
393            mtype: "sensitivity",
394            date: "2015-01-01",
395            source: "CIE 170-2:2015 Fundamental Chromaticity Diagram with Physiological Axes — Part 2, Table 3",
396            doi: "10.25039/CIE.DS.dm6qiig7",
397            columns: vec!["x_F10".into(), "y_F10".into(), "z_F10".into()],
398        },
399        Dataset {
400            csv_file: "CIE_cfb_sle_2deg.csv",
401            subdir: "sensitivity",
402            json_file: "cie_cfb_luminous_efficiency_2deg.json",
403            title: "CIE Cone-Fundamental-Based Spectral Luminous Efficiency, 2° Field Size",
404            mtype: "sensitivity",
405            date: "2015-01-01",
406            source: "CIE 170-2:2015 Fundamental Chromaticity Diagram with Physiological Axes — Part 2, Table 2",
407            doi: "10.25039/CIE.DS.pyqkh5rw",
408            columns: vec!["V_F".into()],
409        },
410        Dataset {
411            csv_file: "CIE_cfb_sle_10deg.csv",
412            subdir: "sensitivity",
413            json_file: "cie_cfb_luminous_efficiency_10deg.json",
414            title: "CIE Cone-Fundamental-Based Spectral Luminous Efficiency, 10° Field Size",
415            mtype: "sensitivity",
416            date: "2015-01-01",
417            source: "CIE 170-2:2015 Fundamental Chromaticity Diagram with Physiological Axes — Part 2, Table 4",
418            doi: "10.25039/CIE.DS.8mrru44q",
419            columns: vec!["V_F10".into()],
420        },
421        // ── Luminous efficiency functions ─────────────────────────────────────
422        Dataset {
423            csv_file: "CIE_sle_photopic.csv",
424            subdir: "sensitivity",
425            json_file: "cie_luminous_efficiency_photopic.json",
426            title: "CIE Spectral Luminous Efficiency for Photopic Vision V(λ)",
427            mtype: "sensitivity",
428            date: "2019-01-01",
429            source: "ISO/CIE 11664-1:2019 Colorimetry — Part 1: CIE Standard Colorimetric Observers, Table 1",
430            doi: "10.25039/CIE.DS.dktna2s3",
431            columns: vec!["V".into()],
432        },
433        Dataset {
434            csv_file: "CIE_sle_scotopic.csv",
435            subdir: "sensitivity",
436            json_file: "cie_luminous_efficiency_scotopic.json",
437            title: "CIE Spectral Luminous Efficiency for Scotopic Vision V′(λ)",
438            mtype: "sensitivity",
439            date: "2019-01-01",
440            source: "ISO/CIE 11664-1:2019 Colorimetry — Part 1: CIE Standard Colorimetric Observers, Table 2",
441            doi: "10.25039/CIE.DS.gr6w4b5g",
442            columns: vec!["V_prime".into()],
443        },
444        // ── Reference spectrum ────────────────────────────────────────────────
445        Dataset {
446            csv_file: "CIE_RefSpectrum_L41.csv",
447            subdir: "illuminants",
448            json_file: "cie_reference_spectrum_l41.json",
449            title: "CIE Reference Spectrum L41",
450            mtype: "irradiance",
451            date: "2023-01-01",
452            source: "CIE 251:2023 CIE L41 Spectral Power Distribution, Table 1",
453            doi: "10.25039/CIE.DS.van56dfj",
454            columns: vec!["L41".into()],
455        },
456        // ── Metamerism index deviation function ───────────────────────────────
457        Dataset {
458            csv_file: "CIE_1st_deriv_meta_ind.csv",
459            subdir: "sensitivity",
460            json_file: "cie_metamerism_index_1st_deviation.json",
461            title: "CIE First Deviation Function for Special Metamerism Index (Change in Observer)",
462            mtype: "sensitivity",
463            date: "2018-01-01",
464            source: "CIE 015:2018 Colorimetry, 4th Edition, Table A.4",
465            doi: "10.25039/CIE.DS.caky9gj2",
466            columns: vec!["f1".into(), "f2".into(), "f3".into()],
467        },
468    ]
469}
470
471// ─────────────────────────────────────────────────────────────────────────────
472// Alpha-opic special case
473// ─────────────────────────────────────────────────────────────────────────────
474
475// Each CIE alpha-opic photoreceptor action spectrum is defined over its own
476// valid wavelength range (s_sc only up to ~615 nm; the others extend to 780 nm).
477// Storing all five in one batch with a shared wavelength axis would force a
478// common range and lose data, so we parse the raw CSV column-by-column and
479// build each SpectrumRecord with its own WavelengthAxis.
480fn convert_alpha_opic(input_dir: &Path, output_dir: &Path) -> bool {
481    let csv_path = input_dir.join("CIE_a-opic_action_spectra.csv");
482    if !csv_path.exists() {
483        eprintln!("  SKIP  CIE_a-opic_action_spectra.csv (file not found)");
484        return false;
485    }
486    let raw = match fs::read_to_string(&csv_path) {
487        Ok(s) => s,
488        Err(e) => {
489            eprintln!("  ERROR reading CIE_a-opic_action_spectra.csv: {e}");
490            return false;
491        }
492    };
493
494    let ids = ["s_sc", "s_mc", "s_lc", "s_rh", "s_mel"];
495    let mut col_points: [Vec<(f64, f64)>; 5] = Default::default();
496
497    for line in raw.lines() {
498        let line = line.trim_end_matches('\r');
499        let fields: Vec<&str> = line.split(',').collect();
500        if fields.len() < 6 {
501            continue;
502        }
503        let Ok(wl) = fields[0]
504            .trim()
505            .trim_start_matches('\u{FEFF}')
506            .parse::<f64>()
507        else {
508            continue;
509        };
510        for (i, col) in col_points.iter_mut().enumerate() {
511            let v_str = fields[i + 1].trim();
512            if v_str.eq_ignore_ascii_case("NaN") {
513                continue;
514            }
515            if let Ok(v) = v_str.parse::<f64>() {
516                col.push((wl, v));
517            }
518        }
519    }
520
521    let copyright = "© International Commission on Illumination (CIE). \
522                     Licensed CC BY-SA 4.0. https://cie.co.at/data-tables";
523    let notes = "Source: CIE S026/E:2018 CIE System for Metrology of Optical Radiation \
524                 for ipRGC-Influenced Responses to Light, Table 2. \
525                 DOI: https://doi.org/10.25039/CIE.DS.vqqhzp5a";
526
527    let spectra: Vec<SpectrumRecord> = ids
528        .iter()
529        .zip(col_points.iter())
530        .map(|(id, pts)| {
531            let wavelengths: Vec<f64> = pts.iter().map(|(w, _)| *w).collect();
532            let values: Vec<f64> = pts.iter().map(|(_, v)| *v).collect();
533            let wavelength_axis = if wavelengths.len() >= 2 {
534                let start = wavelengths[0];
535                let end = *wavelengths.last().unwrap();
536                let interval = wavelengths[1] - wavelengths[0];
537                let uniform = wavelengths
538                    .windows(2)
539                    .all(|w| (w[1] - w[0] - interval).abs() < 1e-6);
540                if uniform {
541                    WavelengthAxis {
542                        values_nm: None,
543                        range_nm: Some(WavelengthRange {
544                            start,
545                            end,
546                            interval,
547                        }),
548                    }
549                } else {
550                    WavelengthAxis {
551                        values_nm: Some(wavelengths),
552                        range_nm: None,
553                    }
554                }
555            } else {
556                WavelengthAxis {
557                    values_nm: Some(wavelengths),
558                    range_nm: None,
559                }
560            };
561            SpectrumRecord {
562                id: id.to_string(),
563                metadata: SpectrumMetadata {
564                    measurement_type: MeasurementType::Sensitivity,
565                    date: "2018-01-01".to_string(),
566                    title: Some(
567                        "CIE Alpha-Opic Action Spectra \
568                             (S-cone, M-cone, L-cone, Rod, Melanopsin)"
569                            .to_string(),
570                    ),
571                    copyright: Some(copyright.to_string()),
572                    description: None,
573                    sample_id: None,
574                    time: None,
575                    operator: None,
576                    instrument: None,
577                    measurement_conditions: None,
578                    surface: None,
579                    sample_backing: None,
580                    tags: None,
581                    custom: None,
582                },
583                wavelength_axis,
584                spectral_data: SpectralData {
585                    values,
586                    uncertainty: None,
587                    scale: None,
588                },
589                provenance: Some(Provenance {
590                    source_file: Some(
591                        "https://files.cie.co.at/CIE_a-opic_action_spectra.csv".to_string(),
592                    ),
593                    source_format: Some("CIE CSV".to_string()),
594                    notes: Some(notes.to_string()),
595                    software: None,
596                    software_version: None,
597                    processing_steps: None,
598                }),
599                color_science: None,
600            }
601        })
602        .collect();
603
604    let file = SpectrumFile::Batch {
605        schema_version: "1.0.0".to_string(),
606        batch_metadata: Some(Box::new(BatchMetadata {
607            title: Some(
608                "CIE Alpha-Opic Action Spectra (S-cone, M-cone, L-cone, Rod, Melanopsin)"
609                    .to_string(),
610            ),
611            description: None,
612            operator: None,
613            date: None,
614            instrument: None,
615            measurement_conditions: None,
616        })),
617        spectra,
618    };
619
620    let out_dir = output_dir.join("sensitivity");
621    if let Err(e) = fs::create_dir_all(&out_dir) {
622        eprintln!("  ERROR creating {}: {e}", out_dir.display());
623        return false;
624    }
625    let out_path = out_dir.join("cie_alpha_opic_action_spectra.json");
626    let json = match serde_json::to_string_pretty(&file) {
627        Ok(j) => j,
628        Err(e) => {
629            eprintln!("  ERROR serialising cie_alpha_opic_action_spectra.json: {e}");
630            return false;
631        }
632    };
633    match fs::write(&out_path, json) {
634        Ok(()) => {
635            eprintln!(
636                "  OK    CIE_a-opic_action_spectra.csv → {} (5 spectra)",
637                out_path.display()
638            );
639            true
640        }
641        Err(e) => {
642            eprintln!("  ERROR writing {}: {e}", out_path.display());
643            false
644        }
645    }
646}
647
648// ─────────────────────────────────────────────────────────────────────────────
649// Conversion helpers
650// ─────────────────────────────────────────────────────────────────────────────
651
652/// Remove NaN entries from every spectrum in `file`, rebuilding each affected
653/// spectrum's WavelengthAxis to cover only the valid (non-NaN) positions.
654/// CIE CSVs sometimes use NaN to indicate that a function is not defined over
655/// part of the wavelength range (e.g. z̄₁₀ above 559 nm, s̄ above 619 nm).
656fn strip_nan_entries(file: &mut SpectrumFile) {
657    let strip = |sp: &mut SpectrumRecord| {
658        if !sp.spectral_data.values.iter().any(|v: &f64| v.is_nan()) {
659            return;
660        }
661        let wls = sp.wavelength_axis.wavelengths_nm();
662        let pairs: Vec<(f64, f64)> = wls
663            .iter()
664            .cloned()
665            .zip(sp.spectral_data.values.iter().cloned())
666            .filter(|(_, v)| !v.is_nan())
667            .collect();
668        if pairs.len() < 2 {
669            return;
670        }
671        let wavelengths: Vec<f64> = pairs.iter().map(|(w, _)| *w).collect();
672        let values: Vec<f64> = pairs.iter().map(|(_, v)| *v).collect();
673        let interval = wavelengths[1] - wavelengths[0];
674        let uniform = wavelengths
675            .windows(2)
676            .all(|w| (w[1] - w[0] - interval).abs() < 1e-6);
677        sp.wavelength_axis = if uniform {
678            WavelengthAxis {
679                range_nm: Some(WavelengthRange {
680                    start: wavelengths[0],
681                    end: *wavelengths.last().unwrap(),
682                    interval,
683                }),
684                values_nm: None,
685            }
686        } else {
687            WavelengthAxis {
688                values_nm: Some(wavelengths),
689                range_nm: None,
690            }
691        };
692        sp.spectral_data.values = values;
693    };
694    match file {
695        SpectrumFile::Single { spectrum, .. } => strip(spectrum),
696        SpectrumFile::Batch { spectra, .. } => {
697            for sp in spectra.iter_mut() {
698                strip(sp);
699            }
700        }
701    }
702}
703
704/// Count how many data columns (after the wavelength column) are in the first
705/// data row of a raw CIE CSV. Used to trim the column-name list to the actual
706/// width so no extra names are injected.
707fn count_data_cols(csv_content: &str) -> usize {
708    for line in csv_content.lines() {
709        let t = line.trim();
710        if t.is_empty() || t.starts_with('#') {
711            continue;
712        }
713        if t.split(',').next().unwrap_or("").parse::<f64>().is_ok() {
714            return t.split(',').count().saturating_sub(1);
715        }
716    }
717    0
718}
719
720/// Build the synthetic metadata header block that the CSV reader understands.
721fn build_header(ds: &Dataset, n_data_cols: usize) -> String {
722    let col_names: Vec<&str> = ds
723        .columns
724        .iter()
725        .map(String::as_str)
726        .take(n_data_cols)
727        .collect();
728    if col_names.len() < n_data_cols {
729        eprintln!(
730            "  WARN  {}: CSV has {n_data_cols} data columns but only {} names defined \
731             — extra columns will receive auto-generated ids",
732            ds.csv_file,
733            col_names.len()
734        );
735    }
736    let col_row = format!("wavelength_nm,{}", col_names.join(","));
737
738    format!(
739        "Title: {title}\n\
740         Measurement_Type: {mtype}\n\
741         Date: {date}\n\
742         Copyright: \u{00a9} International Commission on Illumination (CIE). \
743         Licensed CC BY-SA 4.0. https://cie.co.at/data-tables\n\
744         Notes: Source: {source}. DOI: https://doi.org/{doi}\n\
745         \n\
746         {col_row}\n",
747        title = ds.title,
748        mtype = ds.mtype,
749        date = ds.date,
750        source = ds.source,
751        doi = ds.doi,
752        col_row = col_row,
753    )
754}
755
756/// Update provenance on every spectrum to record the original CIE CSV URL.
757fn set_provenance(file: &mut SpectrumFile, csv_file: &str) {
758    let url = format!("https://files.cie.co.at/{csv_file}");
759    let fix = |prov: &mut Option<spectral_io::Provenance>| {
760        if let Some(p) = prov {
761            p.source_file = Some(url.clone());
762            p.source_format = Some("CIE CSV".into());
763        }
764    };
765    match file {
766        SpectrumFile::Single { spectrum, .. } => fix(&mut spectrum.provenance),
767        SpectrumFile::Batch { spectra, .. } => {
768            for sp in spectra.iter_mut() {
769                fix(&mut sp.provenance);
770            }
771        }
772    }
773}
774
775// ─────────────────────────────────────────────────────────────────────────────
776// Main
777// ─────────────────────────────────────────────────────────────────────────────
778
779fn main() {
780    let args: Vec<String> = env::args().collect();
781    let mut input_dir = PathBuf::from("data/cie-raw");
782    let mut output_dir = PathBuf::from("data/spectral-io/cie");
783
784    let mut i = 1usize;
785    while i < args.len() {
786        match args[i].as_str() {
787            "--input" => {
788                i += 1;
789                if i < args.len() {
790                    input_dir = PathBuf::from(&args[i]);
791                }
792            }
793            "--output" => {
794                i += 1;
795                if i < args.len() {
796                    output_dir = PathBuf::from(&args[i]);
797                }
798            }
799            arg => {
800                eprintln!("Unknown argument: {arg}");
801                process::exit(1);
802            }
803        }
804        i += 1;
805    }
806
807    let datasets = datasets();
808    let mut ok = 0usize;
809    let mut skipped = 0usize;
810    let mut failed = 0usize;
811
812    for ds in &datasets {
813        let csv_path = input_dir.join(ds.csv_file);
814        if !csv_path.exists() {
815            eprintln!("  SKIP  {} (file not found)", ds.csv_file);
816            skipped += 1;
817            continue;
818        }
819
820        let raw_owned = match fs::read_to_string(&csv_path) {
821            Ok(s) => s,
822            Err(e) => {
823                eprintln!("  ERROR reading {}: {e}", ds.csv_file);
824                failed += 1;
825                continue;
826            }
827        };
828        // Strip a UTF-8 BOM that some CIE files carry; if left in place it
829        // ends up in the middle of the synthetic header + raw concatenation
830        // where csv_parse's own BOM strip (which only fires at position 0)
831        // cannot reach it.
832        let raw = raw_owned.strip_prefix('\u{FEFF}').unwrap_or(&raw_owned);
833
834        let n_cols = count_data_cols(raw);
835        if n_cols == 0 {
836            eprintln!("  ERROR {}: no data rows found", ds.csv_file);
837            failed += 1;
838            continue;
839        }
840
841        let synthetic = format!("{}{}", build_header(ds, n_cols), raw);
842
843        let mut file = match SpectrumFile::from_csv_str(&synthetic) {
844            Ok(f) => f,
845            Err(e) => {
846                eprintln!("  ERROR parsing {}: {e}", ds.csv_file);
847                failed += 1;
848                continue;
849            }
850        };
851
852        strip_nan_entries(&mut file);
853        set_provenance(&mut file, ds.csv_file);
854
855        let out_dir = output_dir.join(ds.subdir);
856        if let Err(e) = fs::create_dir_all(&out_dir) {
857            eprintln!("  ERROR creating {}: {e}", out_dir.display());
858            failed += 1;
859            continue;
860        }
861
862        let json = match serde_json::to_string_pretty(&file) {
863            Ok(j) => j,
864            Err(e) => {
865                eprintln!("  ERROR serialising {}: {e}", ds.json_file);
866                failed += 1;
867                continue;
868            }
869        };
870
871        let out_path = out_dir.join(ds.json_file);
872        if let Err(e) = fs::write(&out_path, &json) {
873            eprintln!("  ERROR writing {}: {e}", out_path.display());
874            failed += 1;
875            continue;
876        }
877
878        let n_spectra = file.spectra().len();
879        eprintln!(
880            "  OK    {} → {} ({} {})",
881            ds.csv_file,
882            out_path.display(),
883            n_spectra,
884            if n_spectra == 1 {
885                "spectrum"
886            } else {
887                "spectra"
888            }
889        );
890        ok += 1;
891    }
892
893    // Alpha-opic handled separately (per-column wavelength ranges)
894    if convert_alpha_opic(&input_dir, &output_dir) {
895        ok += 1;
896    } else if !input_dir.join("CIE_a-opic_action_spectra.csv").exists() {
897        skipped += 1;
898    } else {
899        failed += 1;
900    }
901
902    eprintln!();
903    eprintln!("{ok} converted, {skipped} skipped, {failed} failed");
904    if failed > 0 {
905        process::exit(1);
906    }
907}
cie_csv_to_json/cie_csv_to_json.rs

cie_csv_to_json/
cie_csv_to_json.rs
