1use crate::types::{AdminError, AdminResult};
7use csv::ReaderBuilder;
8use rayon::prelude::*;
9use serde::{Deserialize, Serialize};
10use std::collections::HashMap;
11use std::io::Cursor;
12
13#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
15pub enum ImportFormat {
16 CSV,
18 JSON,
20 TSV,
22}
23
24impl ImportFormat {
25 pub fn extensions(&self) -> &[&'static str] {
27 match self {
28 ImportFormat::CSV => &["csv"],
29 ImportFormat::JSON => &["json"],
30 ImportFormat::TSV => &["tsv", "tab"],
31 }
32 }
33
34 pub fn from_filename(filename: &str) -> Option<Self> {
36 let ext = filename.split('.').next_back()?.to_lowercase();
37 match ext.as_str() {
38 "csv" => Some(ImportFormat::CSV),
39 "json" => Some(ImportFormat::JSON),
40 "tsv" | "tab" => Some(ImportFormat::TSV),
41 _ => None,
42 }
43 }
44
45 pub fn from_content_type(content_type: &str) -> Option<Self> {
49 let mime_type = content_type.split(';').next()?.trim().to_lowercase();
51
52 match mime_type.as_str() {
53 "application/json" | "text/json" => Some(ImportFormat::JSON),
55 "text/csv" | "application/csv" => Some(ImportFormat::CSV),
57 "text/tab-separated-values" | "text/tsv" => Some(ImportFormat::TSV),
59 _ => None,
60 }
61 }
62}
63
64#[derive(Debug, Clone)]
79pub struct ImportConfig {
80 model_name: String,
82 format: ImportFormat,
84 field_mappings: HashMap<String, String>,
86 skip_fields: Vec<String>,
88 skip_duplicates: bool,
90 update_existing: bool,
92 key_field: Option<String>,
94 max_records: Option<usize>,
96 skip_header: bool,
98 validate_first: bool,
100}
101
102impl ImportConfig {
103 pub fn new(model_name: impl Into<String>, format: ImportFormat) -> Self {
105 Self {
106 model_name: model_name.into(),
107 format,
108 field_mappings: HashMap::new(),
109 skip_fields: Vec::new(),
110 skip_duplicates: false,
111 update_existing: false,
112 key_field: None,
113 max_records: None,
114 skip_header: true,
115 validate_first: true,
116 }
117 }
118
119 pub fn model_name(&self) -> &str {
121 &self.model_name
122 }
123
124 pub fn format(&self) -> ImportFormat {
126 self.format
127 }
128
129 pub fn with_field_mapping(
131 mut self,
132 import_field: impl Into<String>,
133 model_field: impl Into<String>,
134 ) -> Self {
135 self.field_mappings
136 .insert(import_field.into(), model_field.into());
137 self
138 }
139
140 pub fn field_mappings(&self) -> &HashMap<String, String> {
142 &self.field_mappings
143 }
144
145 pub fn map_field<'a>(&'a self, import_field: &'a str) -> &'a str {
147 self.field_mappings
148 .get(import_field)
149 .map(|s| s.as_str())
150 .unwrap_or(import_field)
151 }
152
153 pub fn skip_field(mut self, field: impl Into<String>) -> Self {
155 self.skip_fields.push(field.into());
156 self
157 }
158
159 pub fn skip_fields(&self) -> &[String] {
161 &self.skip_fields
162 }
163
164 pub fn skip_duplicates(mut self, skip: bool) -> Self {
166 self.skip_duplicates = skip;
167 self
168 }
169
170 pub fn should_skip_duplicates(&self) -> bool {
172 self.skip_duplicates
173 }
174
175 pub fn update_existing(mut self, update: bool) -> Self {
177 self.update_existing = update;
178 self
179 }
180
181 pub fn should_update_existing(&self) -> bool {
183 self.update_existing
184 }
185
186 pub fn with_key_field(mut self, field: impl Into<String>) -> Self {
188 self.key_field = Some(field.into());
189 self
190 }
191
192 pub fn key_field(&self) -> Option<&String> {
194 self.key_field.as_ref()
195 }
196
197 pub fn with_max_records(mut self, max: usize) -> Self {
199 self.max_records = Some(max);
200 self
201 }
202
203 pub fn max_records(&self) -> Option<usize> {
205 self.max_records
206 }
207
208 pub fn with_skip_header(mut self, skip: bool) -> Self {
210 self.skip_header = skip;
211 self
212 }
213
214 pub fn should_skip_header(&self) -> bool {
216 self.skip_header
217 }
218
219 pub fn with_validation(mut self, validate: bool) -> Self {
221 self.validate_first = validate;
222 self
223 }
224
225 pub fn should_validate(&self) -> bool {
227 self.validate_first
228 }
229}
230
231#[derive(Debug, Clone, Serialize, Deserialize)]
233pub struct ImportResult {
234 pub imported_count: usize,
236 pub updated_count: usize,
238 pub skipped_count: usize,
240 pub failed_count: usize,
242 pub errors: Vec<ImportError>,
244}
245
246impl ImportResult {
247 pub fn new() -> Self {
249 Self {
250 imported_count: 0,
251 updated_count: 0,
252 skipped_count: 0,
253 failed_count: 0,
254 errors: Vec::new(),
255 }
256 }
257
258 pub fn total_processed(&self) -> usize {
260 self.imported_count + self.updated_count + self.skipped_count + self.failed_count
261 }
262
263 pub fn is_successful(&self) -> bool {
265 self.failed_count == 0
266 }
267
268 pub fn add_imported(&mut self) {
270 self.imported_count += 1;
271 }
272
273 pub fn add_updated(&mut self) {
275 self.updated_count += 1;
276 }
277
278 pub fn add_skipped(&mut self) {
280 self.skipped_count += 1;
281 }
282
283 pub fn add_failed(&mut self, error: ImportError) {
285 self.failed_count += 1;
286 self.errors.push(error);
287 }
288}
289
290impl Default for ImportResult {
291 fn default() -> Self {
292 Self::new()
293 }
294}
295
296#[derive(Debug, Clone, Serialize, Deserialize)]
298pub struct ImportError {
299 pub row_number: usize,
301 pub message: String,
303 pub data: Option<HashMap<String, String>>,
305}
306
307impl ImportError {
308 pub fn new(row_number: usize, message: impl Into<String>) -> Self {
310 Self {
311 row_number,
312 message: message.into(),
313 data: None,
314 }
315 }
316
317 pub fn with_data(
319 row_number: usize,
320 message: impl Into<String>,
321 data: HashMap<String, String>,
322 ) -> Self {
323 Self {
324 row_number,
325 message: message.into(),
326 data: Some(data),
327 }
328 }
329}
330
331pub struct CsvImporter;
333
334impl CsvImporter {
335 pub fn import(data: &[u8], skip_header: bool) -> AdminResult<Vec<HashMap<String, String>>> {
348 let mut reader = ReaderBuilder::new()
353 .has_headers(skip_header)
354 .flexible(false) .trim(csv::Trim::All) .from_reader(Cursor::new(data));
357
358 let headers = if skip_header {
360 let hdrs = reader
362 .headers()
363 .map_err(|e| {
364 AdminError::ValidationError(format!("Failed to read CSV headers: {}", e))
365 })?
366 .iter()
367 .map(|h| h.to_string())
368 .collect::<Vec<_>>();
369
370 if hdrs.is_empty() {
371 return Err(AdminError::ValidationError(
372 "CSV header is empty".to_string(),
373 ));
374 }
375 hdrs
376 } else {
377 let mut peek_reader = ReaderBuilder::new()
380 .has_headers(false)
381 .flexible(false)
382 .trim(csv::Trim::All)
383 .from_reader(Cursor::new(data));
384
385 let first_record = peek_reader.records().next();
386 let col_count = match first_record {
387 Some(Ok(ref rec)) => rec.len(),
388 Some(Err(e)) => {
389 return Err(AdminError::ValidationError(format!(
390 "Row 1: CSV parse error: {}",
391 e
392 )));
393 }
394 None => return Ok(Vec::new()),
395 };
396
397 reader = ReaderBuilder::new()
399 .has_headers(false)
400 .flexible(false)
401 .trim(csv::Trim::All)
402 .from_reader(Cursor::new(data));
403
404 (0..col_count)
405 .map(|i| format!("column_{}", i))
406 .collect::<Vec<_>>()
407 };
408
409 let mut records = Vec::new();
411 let mut row_num = if skip_header { 1 } else { 0 };
412
413 for result in reader.records() {
414 row_num += 1;
415
416 let record = result.map_err(|e| {
417 AdminError::ValidationError(format!("Row {}: CSV parse error: {}", row_num, e))
418 })?;
419
420 if record.len() != headers.len() {
422 return Err(AdminError::ValidationError(format!(
423 "Row {}: Expected {} columns, got {}",
424 row_num,
425 headers.len(),
426 record.len()
427 )));
428 }
429
430 let mut map = HashMap::new();
432 for (header, value) in headers.iter().zip(record.iter()) {
433 map.insert(header.clone(), value.to_string());
434 }
435
436 records.push(map);
437 }
438
439 Ok(records)
440 }
441}
442
443pub struct JsonImporter;
445
446impl JsonImporter {
447 pub fn import(data: &[u8]) -> AdminResult<Vec<HashMap<String, String>>> {
460 let value: serde_json::Value = serde_json::from_slice(data)
461 .map_err(|e| AdminError::ValidationError(format!("Invalid JSON: {}", e)))?;
462
463 let array = value
464 .as_array()
465 .ok_or_else(|| AdminError::ValidationError("JSON must be an array".to_string()))?;
466
467 for (idx, item) in array.iter().enumerate() {
470 if !item.is_object() {
471 return Err(AdminError::ValidationError(format!(
472 "Item {} is not an object",
473 idx
474 )));
475 }
476 }
477
478 let records: Vec<HashMap<String, String>> = if array.len() > 1000 {
480 array
482 .par_iter()
483 .map(|item| {
484 let obj = item.as_object().expect("validated as object above");
486
487 let mut record = HashMap::new();
488 for (key, value) in obj {
489 let value_str = match value {
490 serde_json::Value::String(s) => s.clone(),
491 serde_json::Value::Number(n) => n.to_string(),
492 serde_json::Value::Bool(b) => b.to_string(),
493 serde_json::Value::Null => String::new(),
494 _ => value.to_string(),
495 };
496 record.insert(key.clone(), value_str);
497 }
498
499 record
500 })
501 .collect()
502 } else {
503 let mut records = Vec::new();
505
506 for item in array.iter() {
507 let obj = item.as_object().expect("validated as object above");
509
510 let mut record = HashMap::new();
511 for (key, value) in obj {
512 let value_str = match value {
513 serde_json::Value::String(s) => s.clone(),
514 serde_json::Value::Number(n) => n.to_string(),
515 serde_json::Value::Bool(b) => b.to_string(),
516 serde_json::Value::Null => String::new(),
517 _ => value.to_string(),
518 };
519 record.insert(key.clone(), value_str);
520 }
521
522 records.push(record);
523 }
524
525 records
526 };
527
528 Ok(records)
529 }
530}
531
532pub struct TsvImporter;
534
535impl TsvImporter {
536 pub fn import(data: &[u8], skip_header: bool) -> AdminResult<Vec<HashMap<String, String>>> {
538 let content = String::from_utf8(data.to_vec())
539 .map_err(|e| AdminError::ValidationError(format!("Invalid UTF-8: {}", e)))?;
540
541 let lines: Vec<&str> = content.lines().collect();
542
543 if lines.is_empty() {
544 return Ok(Vec::new());
545 }
546
547 let headers: Vec<String> = lines[0].split('\t').map(|s| s.to_string()).collect();
549
550 if headers.iter().all(|h| h.is_empty()) {
553 return Err(AdminError::ValidationError(
554 "TSV header is empty".to_string(),
555 ));
556 }
557
558 let start_row = if skip_header { 1 } else { 0 };
559 let mut records = Vec::new();
560
561 for (idx, line) in lines.iter().enumerate().skip(start_row) {
562 if line.trim().is_empty() {
563 continue;
564 }
565
566 let values: Vec<String> = line.split('\t').map(|s| s.to_string()).collect();
567
568 if values.len() != headers.len() {
569 return Err(AdminError::ValidationError(format!(
570 "Row {}: Expected {} columns, got {}",
571 idx + 1,
572 headers.len(),
573 values.len()
574 )));
575 }
576
577 let mut record = HashMap::new();
578 for (header, value) in headers.iter().zip(values.iter()) {
579 record.insert(header.clone(), value.clone());
580 }
581
582 records.push(record);
583 }
584
585 Ok(records)
586 }
587}
588
589pub struct ImportBuilder {
606 config: ImportConfig,
607 data: Vec<u8>,
608}
609
610impl ImportBuilder {
611 pub fn new(model_name: impl Into<String>, format: ImportFormat) -> Self {
613 Self {
614 config: ImportConfig::new(model_name, format),
615 data: Vec::new(),
616 }
617 }
618
619 pub fn data(mut self, data: Vec<u8>) -> Self {
621 self.data = data;
622 self
623 }
624
625 pub fn field_mapping(
627 mut self,
628 import_field: impl Into<String>,
629 model_field: impl Into<String>,
630 ) -> Self {
631 self.config = self.config.with_field_mapping(import_field, model_field);
632 self
633 }
634
635 pub fn skip_duplicates(mut self, skip: bool) -> Self {
637 self.config = self.config.skip_duplicates(skip);
638 self
639 }
640
641 pub fn update_existing(mut self, update: bool) -> Self {
643 self.config = self.config.update_existing(update);
644 self
645 }
646
647 pub fn key_field(mut self, field: impl Into<String>) -> Self {
649 self.config = self.config.with_key_field(field);
650 self
651 }
652
653 pub fn max_records(mut self, max: usize) -> Self {
655 self.config = self.config.with_max_records(max);
656 self
657 }
658
659 pub fn parse(self) -> AdminResult<Vec<HashMap<String, String>>> {
661 let mut records = match self.config.format() {
662 ImportFormat::CSV => CsvImporter::import(&self.data, self.config.should_skip_header())?,
663 ImportFormat::JSON => JsonImporter::import(&self.data)?,
664 ImportFormat::TSV => TsvImporter::import(&self.data, self.config.should_skip_header())?,
665 };
666
667 if !self.config.field_mappings().is_empty() {
669 records = records
670 .into_iter()
671 .map(|mut record| {
672 let mut mapped_record = HashMap::new();
673 for (key, value) in record.drain() {
674 let mapped_key = self.config.map_field(&key).to_string();
675 mapped_record.insert(mapped_key, value);
676 }
677 mapped_record
678 })
679 .collect();
680 }
681
682 if let Some(max) = self.config.max_records() {
684 records.truncate(max);
685 }
686
687 Ok(records)
688 }
689}
690
691#[cfg(all(test, server))]
692mod tests {
693 use super::*;
694 use rstest::rstest;
695
696 #[test]
697 fn test_import_format_from_filename() {
698 assert_eq!(
699 ImportFormat::from_filename("data.csv"),
700 Some(ImportFormat::CSV)
701 );
702 assert_eq!(
703 ImportFormat::from_filename("data.json"),
704 Some(ImportFormat::JSON)
705 );
706 assert_eq!(
707 ImportFormat::from_filename("data.tsv"),
708 Some(ImportFormat::TSV)
709 );
710 assert_eq!(ImportFormat::from_filename("data.txt"), None);
711 }
712
713 #[test]
714 fn test_import_config_new() {
715 let config = ImportConfig::new("User", ImportFormat::CSV);
716 assert_eq!(config.model_name(), "User");
717 assert_eq!(config.format(), ImportFormat::CSV);
718 assert!(config.should_skip_header());
719 assert!(config.should_validate());
720 }
721
722 #[test]
723 fn test_import_config_field_mapping() {
724 let config =
725 ImportConfig::new("User", ImportFormat::CSV).with_field_mapping("username", "login");
726
727 assert_eq!(config.map_field("username"), "login");
728 assert_eq!(config.map_field("email"), "email");
729 }
730
731 #[test]
732 fn test_csv_importer_basic() {
733 let csv_data = b"id,name\n1,Alice\n2,Bob";
734 let result = CsvImporter::import(csv_data, true);
735
736 let records = result.unwrap();
737 assert_eq!(records.len(), 2);
738 assert_eq!(records[0].get("id"), Some(&"1".to_string()));
739 assert_eq!(records[0].get("name"), Some(&"Alice".to_string()));
740 }
741
742 #[test]
743 fn test_csv_importer_quoted() {
744 let csv_data = b"id,name\n1,\"Smith, John\"\n2,\"Doe, Jane\"";
745 let result = CsvImporter::import(csv_data, true);
746
747 let records = result.unwrap();
748 assert_eq!(records.len(), 2);
749 assert_eq!(records[0].get("name"), Some(&"Smith, John".to_string()));
750 }
751
752 #[test]
753 fn test_json_importer() {
754 let json_data = br#"[{"id":"1","name":"Alice"},{"id":"2","name":"Bob"}]"#;
755 let result = JsonImporter::import(json_data);
756
757 let records = result.unwrap();
758 assert_eq!(records.len(), 2);
759 assert_eq!(records[0].get("id"), Some(&"1".to_string()));
760 assert_eq!(records[0].get("name"), Some(&"Alice".to_string()));
761 }
762
763 #[test]
764 fn test_tsv_importer() {
765 let tsv_data = b"id\tname\n1\tAlice\n2\tBob";
766 let result = TsvImporter::import(tsv_data, true);
767
768 let records = result.unwrap();
769 assert_eq!(records.len(), 2);
770 assert_eq!(records[0].get("id"), Some(&"1".to_string()));
771 assert_eq!(records[0].get("name"), Some(&"Alice".to_string()));
772 }
773
774 #[test]
775 fn test_import_builder() {
776 let csv_data = b"id,name\n1,Alice\n2,Bob";
777
778 let result = ImportBuilder::new("User", ImportFormat::CSV)
779 .data(csv_data.to_vec())
780 .parse();
781
782 let records = result.unwrap();
783 assert_eq!(records.len(), 2);
784 }
785
786 #[test]
787 fn test_import_builder_with_mapping() {
788 let csv_data = b"id,username\n1,alice\n2,bob";
789
790 let result = ImportBuilder::new("User", ImportFormat::CSV)
791 .data(csv_data.to_vec())
792 .field_mapping("username", "login")
793 .parse();
794
795 let records = result.unwrap();
796 assert_eq!(records[0].get("login"), Some(&"alice".to_string()));
797 assert_eq!(records[0].get("username"), None);
798 }
799
800 #[test]
801 fn test_import_builder_max_records() {
802 let csv_data = b"id,name\n1,Alice\n2,Bob\n3,Charlie";
803
804 let result = ImportBuilder::new("User", ImportFormat::CSV)
805 .data(csv_data.to_vec())
806 .max_records(2)
807 .parse();
808
809 let records = result.unwrap();
810 assert_eq!(records.len(), 2);
811 }
812
813 #[test]
814 fn test_import_result() {
815 let mut result = ImportResult::new();
816 assert_eq!(result.total_processed(), 0);
817 assert!(result.is_successful());
818
819 result.add_imported();
820 result.add_updated();
821 assert_eq!(result.imported_count, 1);
822 assert_eq!(result.updated_count, 1);
823 assert_eq!(result.total_processed(), 2);
824
825 result.add_failed(ImportError::new(1, "Test error".to_string()));
826 assert!(!result.is_successful());
827 assert_eq!(result.failed_count, 1);
828 }
829
830 #[test]
833 fn test_from_content_type_json() {
834 assert_eq!(
835 ImportFormat::from_content_type("application/json"),
836 Some(ImportFormat::JSON)
837 );
838 }
839
840 #[test]
841 fn test_from_content_type_csv() {
842 assert_eq!(
843 ImportFormat::from_content_type("text/csv"),
844 Some(ImportFormat::CSV)
845 );
846 }
847
848 #[test]
849 fn test_from_content_type_tsv() {
850 assert_eq!(
851 ImportFormat::from_content_type("text/tab-separated-values"),
852 Some(ImportFormat::TSV)
853 );
854 }
855
856 #[test]
857 fn test_from_content_type_with_charset() {
858 assert_eq!(
860 ImportFormat::from_content_type("application/json; charset=utf-8"),
861 Some(ImportFormat::JSON)
862 );
863 assert_eq!(
864 ImportFormat::from_content_type("text/csv; charset=utf-8"),
865 Some(ImportFormat::CSV)
866 );
867 assert_eq!(
868 ImportFormat::from_content_type("text/tab-separated-values; charset=utf-8"),
869 Some(ImportFormat::TSV)
870 );
871 }
872
873 #[test]
874 fn test_from_content_type_unknown() {
875 assert_eq!(ImportFormat::from_content_type("text/html"), None);
876 assert_eq!(ImportFormat::from_content_type("application/xml"), None);
877 assert_eq!(ImportFormat::from_content_type("image/png"), None);
878 }
879
880 #[test]
881 fn test_from_content_type_empty() {
882 assert_eq!(ImportFormat::from_content_type(""), None);
883 }
884
885 #[test]
886 fn test_from_content_type_case_insensitive() {
887 assert_eq!(
889 ImportFormat::from_content_type("Application/JSON"),
890 Some(ImportFormat::JSON)
891 );
892 assert_eq!(
893 ImportFormat::from_content_type("TEXT/CSV"),
894 Some(ImportFormat::CSV)
895 );
896 assert_eq!(
897 ImportFormat::from_content_type("Text/Tab-Separated-Values"),
898 Some(ImportFormat::TSV)
899 );
900 }
901
902 #[test]
903 fn test_from_content_type_text_json() {
904 assert_eq!(
906 ImportFormat::from_content_type("text/json"),
907 Some(ImportFormat::JSON)
908 );
909 }
910
911 #[test]
912 fn test_from_content_type_application_csv() {
913 assert_eq!(
915 ImportFormat::from_content_type("application/csv"),
916 Some(ImportFormat::CSV)
917 );
918 }
919
920 #[test]
921 fn test_from_content_type_text_tsv() {
922 assert_eq!(
924 ImportFormat::from_content_type("text/tsv"),
925 Some(ImportFormat::TSV)
926 );
927 }
928
929 #[test]
930 fn test_from_content_type_with_extra_parameters() {
931 assert_eq!(
933 ImportFormat::from_content_type("application/json; charset=utf-8; boundary=something"),
934 Some(ImportFormat::JSON)
935 );
936 }
937
938 #[test]
939 fn test_from_content_type_whitespace() {
940 assert_eq!(
942 ImportFormat::from_content_type(" application/json "),
943 Some(ImportFormat::JSON)
944 );
945 assert_eq!(
946 ImportFormat::from_content_type("application/json ;charset=utf-8"),
947 Some(ImportFormat::JSON)
948 );
949 }
950
951 #[rstest]
952 fn test_csv_import_without_skip_header() {
953 let csv_data = b"1,Alice\n2,Bob\n3,Charlie";
956
957 let result = CsvImporter::import(csv_data, false);
959
960 let records = result.unwrap();
962 assert_eq!(records.len(), 3);
963 assert_eq!(records[0].get("column_0"), Some(&"1".to_string()));
964 assert_eq!(records[0].get("column_1"), Some(&"Alice".to_string()));
965 assert_eq!(records[1].get("column_0"), Some(&"2".to_string()));
966 assert_eq!(records[1].get("column_1"), Some(&"Bob".to_string()));
967 assert_eq!(records[2].get("column_0"), Some(&"3".to_string()));
968 assert_eq!(records[2].get("column_1"), Some(&"Charlie".to_string()));
969 }
970
971 #[rstest]
972 fn test_tsv_import_rejects_empty_headers() {
973 let tsv_data = b"\t\t\n1\tAlice\teng";
976
977 let result = TsvImporter::import(tsv_data, true);
979
980 assert!(result.is_err());
982 let err = result.unwrap_err();
983 assert!(
984 matches!(err, AdminError::ValidationError(ref msg) if msg == "TSV header is empty")
985 );
986 }
987}