pub mod report;
pub use report::{
ConversionCounts, ConversionIssue, ConversionIssueCode, ConversionReport, ConversionSeverity,
ConversionStage,
};
use crate::ir::Dataset;
use std::collections::HashSet;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Format {
IrJson,
Coco,
IbmCloudAnnotations,
Cvat,
LabelStudio,
Labelbox,
ScaleAi,
UnityPerception,
Tfod,
Tfrecord,
VottCsv,
VottJson,
Yolo,
YoloKeras,
YoloV4Pytorch,
Voc,
HfImagefolder,
SageMaker,
LabelMe,
SuperAnnotate,
Supervisely,
Cityscapes,
Marmot,
CreateMl,
Kitti,
Via,
Retinanet,
OpenImages,
Datumaro,
WiderFace,
Oidv4,
Bdd100k,
V7Darwin,
EdgeImpulse,
OpenLabel,
ViaCsv,
KaggleWheat,
AutoMlVision,
Udacity,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum IrLossiness {
Lossless,
Conditional,
Lossy,
}
impl Format {
pub fn name(&self) -> &'static str {
match self {
Format::IrJson => "ir-json",
Format::Coco => "coco",
Format::IbmCloudAnnotations => "ibm-cloud-annotations",
Format::Cvat => "cvat",
Format::LabelStudio => "label-studio",
Format::Labelbox => "labelbox",
Format::ScaleAi => "scale-ai",
Format::UnityPerception => "unity-perception",
Format::Tfod => "tfod",
Format::Tfrecord => "tfrecord",
Format::VottCsv => "vott-csv",
Format::VottJson => "vott-json",
Format::Yolo => "yolo",
Format::YoloKeras => "yolo-keras",
Format::YoloV4Pytorch => "yolov4-pytorch",
Format::Voc => "voc",
Format::HfImagefolder => "hf",
Format::SageMaker => "sagemaker",
Format::LabelMe => "labelme",
Format::SuperAnnotate => "superannotate",
Format::Supervisely => "supervisely",
Format::Cityscapes => "cityscapes",
Format::Marmot => "marmot",
Format::CreateMl => "create-ml",
Format::Kitti => "kitti",
Format::Via => "via",
Format::Retinanet => "retinanet",
Format::OpenImages => "openimages",
Format::Datumaro => "datumaro",
Format::WiderFace => "wider-face",
Format::Oidv4 => "oidv4",
Format::Bdd100k => "bdd100k",
Format::V7Darwin => "v7-darwin",
Format::EdgeImpulse => "edge-impulse",
Format::OpenLabel => "openlabel",
Format::ViaCsv => "via-csv",
Format::KaggleWheat => "kaggle-wheat",
Format::AutoMlVision => "automl-vision",
Format::Udacity => "udacity",
}
}
pub fn lossiness_relative_to_ir(&self) -> IrLossiness {
match self {
Format::IrJson => IrLossiness::Lossless,
Format::Coco => IrLossiness::Conditional,
Format::IbmCloudAnnotations => IrLossiness::Lossy,
Format::Cvat => IrLossiness::Lossy,
Format::LabelStudio => IrLossiness::Lossy,
Format::Labelbox => IrLossiness::Lossy,
Format::ScaleAi => IrLossiness::Lossy,
Format::UnityPerception => IrLossiness::Lossy,
Format::Tfod => IrLossiness::Lossy,
Format::Tfrecord => IrLossiness::Lossy,
Format::VottCsv => IrLossiness::Lossy,
Format::VottJson => IrLossiness::Lossy,
Format::Yolo => IrLossiness::Lossy,
Format::YoloKeras => IrLossiness::Lossy,
Format::YoloV4Pytorch => IrLossiness::Lossy,
Format::Voc => IrLossiness::Lossy,
Format::HfImagefolder => IrLossiness::Lossy,
Format::SageMaker => IrLossiness::Lossy,
Format::LabelMe => IrLossiness::Lossy,
Format::SuperAnnotate => IrLossiness::Lossy,
Format::Supervisely => IrLossiness::Lossy,
Format::Cityscapes => IrLossiness::Lossy,
Format::Marmot => IrLossiness::Lossy,
Format::CreateMl => IrLossiness::Lossy,
Format::Kitti => IrLossiness::Lossy,
Format::Via => IrLossiness::Lossy,
Format::Retinanet => IrLossiness::Lossy,
Format::OpenImages => IrLossiness::Lossy,
Format::Datumaro => IrLossiness::Lossy,
Format::WiderFace => IrLossiness::Lossy,
Format::Oidv4 => IrLossiness::Lossy,
Format::Bdd100k => IrLossiness::Lossy,
Format::V7Darwin => IrLossiness::Lossy,
Format::EdgeImpulse => IrLossiness::Lossy,
Format::OpenLabel => IrLossiness::Lossy,
Format::ViaCsv => IrLossiness::Lossy,
Format::KaggleWheat => IrLossiness::Lossy,
Format::AutoMlVision => IrLossiness::Lossy,
Format::Udacity => IrLossiness::Lossy,
}
}
}
pub fn build_conversion_report(dataset: &Dataset, from: Format, to: Format) -> ConversionReport {
let mut report = ConversionReport::new(from.name(), to.name());
report.input = ConversionCounts {
images: dataset.images.len(),
categories: dataset.categories.len(),
annotations: dataset.annotations.len(),
};
match to {
Format::Tfod => analyze_to_tfod(dataset, &mut report),
Format::Tfrecord => analyze_to_tfrecord(dataset, &mut report),
Format::VottCsv => analyze_to_vott_csv(dataset, &mut report),
Format::VottJson => analyze_to_vott_json(dataset, &mut report),
Format::Yolo => analyze_to_yolo(dataset, &mut report),
Format::YoloKeras | Format::YoloV4Pytorch => {
analyze_to_yolo_keras_txt(dataset, &mut report)
}
Format::Voc => analyze_to_voc(dataset, &mut report),
Format::LabelStudio => analyze_to_label_studio(dataset, &mut report),
Format::Labelbox => analyze_to_labelbox(dataset, &mut report),
Format::ScaleAi => analyze_to_scale_ai(dataset, &mut report),
Format::UnityPerception => analyze_to_unity_perception(dataset, &mut report),
Format::Coco => analyze_to_coco(dataset, &mut report),
Format::IbmCloudAnnotations => analyze_to_cloud_annotations(dataset, &mut report),
Format::Cvat => analyze_to_cvat(dataset, &mut report),
Format::IrJson => analyze_to_ir_json(dataset, &mut report),
Format::HfImagefolder => analyze_to_hf(dataset, &mut report),
Format::SageMaker => analyze_to_sagemaker(dataset, &mut report),
Format::LabelMe => analyze_to_labelme(dataset, &mut report),
Format::SuperAnnotate => analyze_to_superannotate(dataset, &mut report),
Format::Supervisely => analyze_to_supervisely(dataset, &mut report),
Format::Cityscapes => analyze_to_cityscapes(dataset, &mut report),
Format::Marmot => analyze_to_marmot(dataset, &mut report),
Format::CreateMl => analyze_to_createml(dataset, &mut report),
Format::Kitti => analyze_to_kitti(dataset, &mut report),
Format::Via => analyze_to_via(dataset, &mut report),
Format::Retinanet => analyze_to_retinanet(dataset, &mut report),
Format::OpenImages => analyze_to_openimages(dataset, &mut report),
Format::Datumaro => analyze_to_basic_bbox_preserving(dataset, &mut report, true),
Format::WiderFace => analyze_to_wider_face(dataset, &mut report),
Format::Oidv4 => analyze_to_basic_bbox_preserving(dataset, &mut report, false),
Format::Bdd100k => analyze_to_basic_bbox_preserving(dataset, &mut report, true),
Format::V7Darwin => analyze_to_basic_bbox_preserving(dataset, &mut report, false),
Format::EdgeImpulse => analyze_to_basic_bbox_preserving(dataset, &mut report, false),
Format::OpenLabel => analyze_to_basic_bbox_preserving(dataset, &mut report, true),
Format::ViaCsv => analyze_to_via_csv(dataset, &mut report),
Format::KaggleWheat => analyze_to_kaggle_wheat(dataset, &mut report),
Format::AutoMlVision => analyze_to_automl_vision(dataset, &mut report),
Format::Udacity => analyze_to_udacity(dataset, &mut report),
}
match from {
Format::Tfod => add_tfod_reader_policy(&mut report),
Format::Tfrecord => add_tfrecord_reader_policy(&mut report),
Format::VottCsv => add_vott_csv_reader_policy(&mut report),
Format::VottJson => add_vott_json_reader_policy(dataset, &mut report),
Format::Yolo => add_yolo_reader_policy(dataset, &mut report),
Format::YoloKeras | Format::YoloV4Pytorch => add_yolo_keras_txt_reader_policy(&mut report),
Format::Voc => add_voc_reader_policy(dataset, &mut report),
Format::LabelStudio => add_label_studio_reader_policy(dataset, &mut report),
Format::Labelbox => add_labelbox_reader_policy(dataset, &mut report),
Format::ScaleAi => add_scale_ai_reader_policy(dataset, &mut report),
Format::UnityPerception => add_unity_perception_reader_policy(dataset, &mut report),
Format::Cvat => add_cvat_reader_policy(dataset, &mut report),
Format::Coco => add_coco_reader_policy(&mut report),
Format::IbmCloudAnnotations => add_cloud_annotations_reader_policy(&mut report),
Format::HfImagefolder => add_hf_reader_policy(&mut report),
Format::SageMaker => add_sagemaker_reader_policy(&mut report),
Format::LabelMe => add_labelme_reader_policy(dataset, &mut report),
Format::SuperAnnotate => add_superannotate_reader_policy(dataset, &mut report),
Format::Supervisely => add_supervisely_reader_policy(dataset, &mut report),
Format::Cityscapes => add_cityscapes_reader_policy(dataset, &mut report),
Format::Marmot => add_marmot_reader_policy(&mut report),
Format::CreateMl => add_createml_reader_policy(&mut report),
Format::Kitti => add_kitti_reader_policy(&mut report),
Format::Via => add_via_reader_policy(&mut report),
Format::Retinanet => add_retinanet_reader_policy(&mut report),
Format::OpenImages => add_openimages_reader_policy(&mut report),
Format::Datumaro => add_simple_reader_policy(&mut report, ConversionIssueCode::DatumaroReaderIdAssignment, "Datumaro reader assigns IDs deterministically and reads bbox annotations only"),
Format::WiderFace => add_simple_reader_policy(&mut report, ConversionIssueCode::WiderFaceReaderIdAssignment, "WIDER Face reader assigns IDs deterministically and maps all boxes to face"),
Format::Oidv4 => add_simple_reader_policy(&mut report, ConversionIssueCode::Oidv4ReaderIdAssignment, "OIDv4 reader assigns IDs deterministically from Label/ files"),
Format::Bdd100k => add_simple_reader_policy(&mut report, ConversionIssueCode::Bdd100kReaderIdAssignment, "BDD100K reader assigns IDs deterministically and reads box2d labels only"),
Format::V7Darwin => add_simple_reader_policy(&mut report, ConversionIssueCode::V7DarwinReaderIdAssignment, "V7 Darwin reader assigns IDs deterministically and reads bounding_box annotations only"),
Format::EdgeImpulse => add_simple_reader_policy(&mut report, ConversionIssueCode::EdgeImpulseReaderIdAssignment, "Edge Impulse reader assigns IDs deterministically from bounding_boxes.labels"),
Format::OpenLabel => add_simple_reader_policy(&mut report, ConversionIssueCode::OpenlabelReaderIdAssignment, "OpenLABEL reader treats frames as static images and reads 2D bbox values only"),
Format::ViaCsv => add_simple_reader_policy(&mut report, ConversionIssueCode::ViaCsvReaderIdAssignment, "VIA CSV reader assigns IDs deterministically and reads rect regions only"),
Format::KaggleWheat => add_kaggle_wheat_reader_policy(&mut report),
Format::AutoMlVision => add_automl_vision_reader_policy(&mut report),
Format::Udacity => add_udacity_reader_policy(&mut report),
Format::IrJson => {}
}
match to {
Format::Tfod => add_tfod_writer_policy(&mut report),
Format::Tfrecord => add_tfrecord_writer_policy(&mut report),
Format::VottCsv => add_vott_csv_writer_policy(&mut report),
Format::VottJson => add_vott_json_writer_policy(&mut report),
Format::Yolo => add_yolo_writer_policy(&mut report),
Format::YoloKeras | Format::YoloV4Pytorch => add_yolo_keras_txt_writer_policy(&mut report),
Format::Voc => add_voc_writer_policy(&mut report),
Format::LabelStudio => add_label_studio_writer_policy(dataset, &mut report),
Format::Labelbox => add_labelbox_writer_policy(&mut report),
Format::ScaleAi => add_scale_ai_writer_policy(&mut report),
Format::UnityPerception => add_unity_perception_writer_policy(&mut report),
Format::Cvat => add_cvat_writer_policy(&mut report),
Format::Coco => add_coco_writer_policy(&mut report),
Format::IbmCloudAnnotations => add_cloud_annotations_writer_policy(&mut report),
Format::HfImagefolder => add_hf_writer_policy(&mut report),
Format::SageMaker => add_sagemaker_writer_policy(&mut report),
Format::LabelMe => add_labelme_writer_policy(&mut report),
Format::SuperAnnotate => add_superannotate_writer_policy(&mut report),
Format::Supervisely => add_supervisely_writer_policy(&mut report),
Format::Cityscapes => add_cityscapes_writer_policy(&mut report),
Format::Marmot => add_marmot_writer_policy(&mut report),
Format::CreateMl => add_createml_writer_policy(&mut report),
Format::Kitti => add_kitti_writer_policy(&mut report),
Format::Via => add_via_writer_policy(&mut report),
Format::Retinanet => add_retinanet_writer_policy(&mut report),
Format::OpenImages => add_openimages_writer_policy(&mut report),
Format::Datumaro => add_simple_writer_policy(&mut report, ConversionIssueCode::DatumaroWriterDeterministicOrder, "Datumaro writer emits deterministic bbox-only JSON"),
Format::WiderFace => add_simple_writer_policy(&mut report, ConversionIssueCode::WiderFaceWriterFileLayout, "WIDER Face writer emits aggregate TXT with default face attributes"),
Format::Oidv4 => add_simple_writer_policy(&mut report, ConversionIssueCode::Oidv4WriterFileLayout, "OIDv4 writer emits Label/ .txt files and does not copy images"),
Format::Bdd100k => add_simple_writer_policy(&mut report, ConversionIssueCode::Bdd100kWriterDeterministicOrder, "BDD100K writer emits deterministic box2d JSON"),
Format::V7Darwin => add_simple_writer_policy(&mut report, ConversionIssueCode::V7DarwinWriterDeterministicOrder, "V7 Darwin writer emits deterministic bbox-only JSON"),
Format::EdgeImpulse => add_simple_writer_policy(&mut report, ConversionIssueCode::EdgeImpulseWriterDeterministicOrder, "Edge Impulse writer emits deterministic bounding_boxes.labels and does not copy images"),
Format::OpenLabel => add_simple_writer_policy(&mut report, ConversionIssueCode::OpenlabelWriterFrameLayout, "OpenLABEL writer emits one static-image frame per IR image"),
Format::ViaCsv => add_simple_writer_policy(&mut report, ConversionIssueCode::ViaCsvWriterDeterministicOrder, "VIA CSV writer emits deterministic rect rows and does not copy images"),
Format::KaggleWheat => add_kaggle_wheat_writer_policy(&mut report),
Format::AutoMlVision => add_automl_vision_writer_policy(&mut report),
Format::Udacity => add_udacity_writer_policy(&mut report),
Format::IrJson => {}
}
report
}
fn analyze_to_tfod(dataset: &Dataset, report: &mut ConversionReport) {
add_common_csv_lossiness_warnings(dataset, report);
add_annotation_drop_warnings_and_output_counts(dataset, report);
}
fn analyze_to_tfrecord(dataset: &Dataset, report: &mut ConversionReport) {
if !dataset.info.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropDatasetInfo,
"dataset info/metadata will be dropped".to_string(),
));
}
if !dataset.licenses.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropLicenses,
format!("{} license(s) will be dropped", dataset.licenses.len()),
));
}
let images_with_unrepresentable_metadata = dataset
.images
.iter()
.filter(|img| {
img.license_id.is_some()
|| img.date_captured.is_some()
|| img.attributes.keys().any(|key| {
!matches!(
key.as_str(),
crate::ir::io_tfrecord::ATTR_SOURCE_ID
| crate::ir::io_tfrecord::ATTR_KEY_SHA256
| crate::ir::io_tfrecord::ATTR_FORMAT
| crate::ir::io_tfrecord::ATTR_HAD_ENCODED_IMAGE
)
})
})
.count();
if images_with_unrepresentable_metadata > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropImageMetadata,
format!(
"{} image(s) have metadata outside TFRecord's preserved image feature set",
images_with_unrepresentable_metadata
),
));
}
let cats_with_supercategory = dataset
.categories
.iter()
.filter(|cat| cat.supercategory.is_some())
.count();
if cats_with_supercategory > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropCategorySupercategory,
format!(
"{} category(s) have supercategory that will be dropped",
cats_with_supercategory
),
));
}
let preserved_sparse_attrs = [
crate::ir::io_tfrecord::ATTR_AREA,
crate::ir::io_tfrecord::ATTR_IS_CROWD,
crate::ir::io_tfrecord::ATTR_DIFFICULT,
crate::ir::io_tfrecord::ATTR_GROUP_OF,
crate::ir::io_tfrecord::ATTR_WEIGHT,
];
let mut anns_with_confidence = 0usize;
let mut anns_with_unrepresentable_attrs = 0usize;
let mut used_category_ids = HashSet::new();
let mut sparse_attr_counts_by_image = std::collections::HashMap::new();
for ann in &dataset.annotations {
if ann.confidence.is_some() {
anns_with_confidence += 1;
}
if ann.attributes.keys().any(|key| {
!matches!(
key.as_str(),
crate::ir::io_tfrecord::ATTR_CLASS_LABEL
| crate::ir::io_tfrecord::ATTR_AREA
| crate::ir::io_tfrecord::ATTR_IS_CROWD
| crate::ir::io_tfrecord::ATTR_DIFFICULT
| crate::ir::io_tfrecord::ATTR_GROUP_OF
| crate::ir::io_tfrecord::ATTR_WEIGHT
)
}) {
anns_with_unrepresentable_attrs += 1;
}
used_category_ids.insert(ann.category_id);
let (ann_count, attr_counts) = sparse_attr_counts_by_image
.entry(ann.image_id)
.or_insert((0usize, [0usize; 5]));
*ann_count += 1;
for (idx, attr) in preserved_sparse_attrs.iter().enumerate() {
if ann.attributes.contains_key(*attr) {
attr_counts[idx] += 1;
}
}
}
if anns_with_confidence > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationConfidence,
format!(
"{} annotation(s) have confidence scores that will be dropped",
anns_with_confidence
),
));
}
if anns_with_unrepresentable_attrs > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationAttributes,
format!(
"{} annotation(s) have attributes outside TFRecord's preserved set (class label/area/iscrowd/difficult/group_of/weight)",
anns_with_unrepresentable_attrs
),
));
}
let image_groups_with_sparse_attrs = sparse_attr_counts_by_image
.values()
.filter(|(ann_count, attr_counts)| {
attr_counts
.iter()
.any(|&present| present > 0 && present < *ann_count)
})
.count();
if image_groups_with_sparse_attrs > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationAttributes,
format!(
"{} image record(s) have sparse per-object TFRecord attributes; those partial attribute lists will be dropped to avoid misaligning object features",
image_groups_with_sparse_attrs
),
));
}
let unused_categories = dataset
.categories
.iter()
.filter(|cat| !used_category_ids.contains(&cat.id))
.count();
if unused_categories > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropUnusedCategories,
format!(
"{} category(s) not referenced by any annotation will be dropped",
unused_categories
),
));
}
report.output = ConversionCounts {
images: dataset.images.len(),
categories: used_category_ids.len(),
annotations: dataset.annotations.len(),
};
}
fn analyze_to_vott_csv(dataset: &Dataset, report: &mut ConversionReport) {
add_common_csv_lossiness_warnings(dataset, report);
add_annotation_drop_warnings_and_output_counts(dataset, report);
}
fn analyze_to_vott_json(dataset: &Dataset, report: &mut ConversionReport) {
add_common_csv_lossiness_warnings(dataset, report);
add_annotation_drop_warnings(dataset, report);
report.output = report.input.clone();
}
fn analyze_to_cloud_annotations(dataset: &Dataset, report: &mut ConversionReport) {
add_common_csv_lossiness_warnings(dataset, report);
report.output = ConversionCounts {
images: dataset.images.len(),
categories: dataset.categories.len(),
annotations: dataset.annotations.len(),
};
}
fn analyze_to_yolo(dataset: &Dataset, report: &mut ConversionReport) {
if !dataset.info.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropDatasetInfo,
"dataset info/metadata will be dropped".to_string(),
));
}
if !dataset.licenses.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropLicenses,
format!("{} license(s) will be dropped", dataset.licenses.len()),
));
}
let images_with_metadata = dataset
.images
.iter()
.filter(|img| img.license_id.is_some() || img.date_captured.is_some())
.count();
if images_with_metadata > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropImageMetadata,
format!(
"{} image(s) have license_id/date_captured that will be dropped",
images_with_metadata
),
));
}
let cats_with_supercategory = dataset
.categories
.iter()
.filter(|cat| cat.supercategory.is_some())
.count();
if cats_with_supercategory > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropCategorySupercategory,
format!(
"{} category(s) have supercategory that will be dropped",
cats_with_supercategory
),
));
}
let anns_with_attributes = dataset
.annotations
.iter()
.filter(|ann| !ann.attributes.is_empty())
.count();
if anns_with_attributes > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationAttributes,
format!(
"{} annotation(s) have attributes that will be dropped",
anns_with_attributes
),
));
}
report.output = report.input.clone();
}
fn analyze_to_yolo_keras_txt(dataset: &Dataset, report: &mut ConversionReport) {
add_common_csv_lossiness_warnings(dataset, report);
add_annotation_drop_warnings(dataset, report);
report.output = report.input.clone();
}
fn analyze_to_voc(dataset: &Dataset, report: &mut ConversionReport) {
if !dataset.info.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropDatasetInfo,
"dataset info/metadata will be dropped".to_string(),
));
}
if !dataset.licenses.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropLicenses,
format!("{} license(s) will be dropped", dataset.licenses.len()),
));
}
let images_with_metadata = dataset
.images
.iter()
.filter(|img| {
img.license_id.is_some()
|| img.date_captured.is_some()
|| img
.attributes
.iter()
.any(|(key, value)| key != "depth" || value.trim().parse::<u32>().is_err())
})
.count();
if images_with_metadata > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropImageMetadata,
format!(
"{} image(s) have metadata that VOC cannot represent (license/date or non-depth image attributes)",
images_with_metadata
),
));
}
let cats_with_supercategory = dataset
.categories
.iter()
.filter(|cat| cat.supercategory.is_some())
.count();
if cats_with_supercategory > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropCategorySupercategory,
format!(
"{} category(s) have supercategory that will be dropped",
cats_with_supercategory
),
));
}
let anns_with_confidence = dataset
.annotations
.iter()
.filter(|ann| ann.confidence.is_some())
.count();
if anns_with_confidence > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationConfidence,
format!(
"{} annotation(s) have confidence scores that will be dropped",
anns_with_confidence
),
));
}
let anns_with_unrepresentable_attrs = dataset
.annotations
.iter()
.filter(|ann| {
ann.attributes.keys().any(|key| {
!matches!(
key.as_str(),
"pose" | "truncated" | "difficult" | "occluded"
)
})
})
.count();
if anns_with_unrepresentable_attrs > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationAttributes,
format!(
"{} annotation(s) have attributes outside VOC's preserved set (pose/truncated/difficult/occluded)",
anns_with_unrepresentable_attrs
),
));
}
report.output = report.input.clone();
}
fn analyze_to_label_studio(dataset: &Dataset, report: &mut ConversionReport) {
if !dataset.info.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropDatasetInfo,
"dataset info/metadata will be dropped".to_string(),
));
}
if !dataset.licenses.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropLicenses,
format!("{} license(s) will be dropped", dataset.licenses.len()),
));
}
let images_with_metadata = dataset
.images
.iter()
.filter(|img| img.license_id.is_some() || img.date_captured.is_some())
.count();
if images_with_metadata > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropImageMetadata,
format!(
"{} image(s) have license_id/date_captured that will be dropped",
images_with_metadata
),
));
}
let cats_with_supercategory = dataset
.categories
.iter()
.filter(|cat| cat.supercategory.is_some())
.count();
if cats_with_supercategory > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropCategorySupercategory,
format!(
"{} category(s) have supercategory that will be dropped",
cats_with_supercategory
),
));
}
let anns_with_unrepresentable_attrs = dataset
.annotations
.iter()
.filter(|ann| {
ann.attributes
.keys()
.any(|key| key.as_str() != "ls_rotation_deg")
})
.count();
if anns_with_unrepresentable_attrs > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationAttributes,
format!(
"{} annotation(s) have attributes outside Label Studio's preserved set",
anns_with_unrepresentable_attrs
),
));
}
report.output = report.input.clone();
}
fn analyze_to_coco(dataset: &Dataset, report: &mut ConversionReport) {
if dataset.info.name.is_some() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropDatasetInfoName,
"dataset info.name has no COCO equivalent".to_string(),
));
}
let anns_with_other_attributes = dataset
.annotations
.iter()
.filter(|ann| ann.attributes.keys().any(|k| k != "area" && k != "iscrowd"))
.count();
if anns_with_other_attributes > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::CocoAttributesMayNotBePreserved,
format!(
"{} annotation(s) have attributes (other than area/iscrowd) that may not be preserved by COCO tools",
anns_with_other_attributes
),
));
}
report.output = report.input.clone();
}
fn analyze_to_ir_json(_dataset: &Dataset, report: &mut ConversionReport) {
report.output = report.input.clone();
}
fn analyze_to_hf(dataset: &Dataset, report: &mut ConversionReport) {
if !dataset.info.is_empty() || !dataset.licenses.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::HfMetadataLost,
"HF ImageFolder metadata.jsonl cannot represent full IR dataset metadata/licenses"
.to_string(),
));
}
let images_with_unrepresentable_attrs = dataset
.images
.iter()
.filter(|img| {
img.license_id.is_some() || img.date_captured.is_some() || !img.attributes.is_empty()
})
.count();
if images_with_unrepresentable_attrs > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::HfAttributesLost,
format!(
"{} image(s) have metadata/attributes that HF metadata.jsonl cannot represent",
images_with_unrepresentable_attrs
),
));
}
let categories_with_supercategory = dataset
.categories
.iter()
.filter(|category| category.supercategory.is_some())
.count();
if categories_with_supercategory > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::HfMetadataLost,
format!(
"{} category(s) have supercategory that HF metadata.jsonl cannot represent",
categories_with_supercategory
),
));
}
let anns_with_confidence = dataset
.annotations
.iter()
.filter(|ann| ann.confidence.is_some())
.count();
if anns_with_confidence > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::HfConfidenceLost,
format!(
"{} annotation(s) have confidence scores that will be dropped",
anns_with_confidence
),
));
}
let anns_with_attributes = dataset
.annotations
.iter()
.filter(|ann| !ann.attributes.is_empty())
.count();
if anns_with_attributes > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::HfAttributesLost,
format!(
"{} annotation(s) have attributes that will be dropped",
anns_with_attributes
),
));
}
report.output = report.input.clone();
}
fn analyze_to_sagemaker(dataset: &Dataset, report: &mut ConversionReport) {
let info_has_unrepresentable_fields = dataset.info.name.is_some()
|| dataset.info.version.is_some()
|| dataset.info.description.is_some()
|| dataset.info.url.is_some()
|| dataset.info.year.is_some()
|| dataset.info.contributor.is_some()
|| dataset.info.date_created.is_some()
|| dataset.info.attributes.keys().any(|key| {
!matches!(
key.as_str(),
"sagemaker_label_attribute_name" | "sagemaker_creation_date"
)
});
if info_has_unrepresentable_fields {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropDatasetInfo,
"dataset info/metadata outside SageMaker label-attribute/date hints will be dropped"
.to_string(),
));
}
if !dataset.licenses.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropLicenses,
format!("{} license(s) will be dropped", dataset.licenses.len()),
));
}
let images_with_unrepresentable_metadata = dataset
.images
.iter()
.filter(|img| {
img.license_id.is_some()
|| img.date_captured.is_some()
|| img.attributes.keys().any(|key| {
!matches!(
key.as_str(),
"sagemaker_source_ref"
| "sagemaker_image_depth"
| "sagemaker_creation_date"
| "sagemaker_job_name"
)
})
})
.count();
if images_with_unrepresentable_metadata > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropImageMetadata,
format!(
"{} image(s) have metadata/attributes outside SageMaker's preserved source-ref/depth/date/job-name set",
images_with_unrepresentable_metadata
),
));
}
let cats_with_supercategory = dataset
.categories
.iter()
.filter(|cat| cat.supercategory.is_some())
.count();
if cats_with_supercategory > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropCategorySupercategory,
format!(
"{} category(s) have supercategory that will be dropped",
cats_with_supercategory
),
));
}
let anns_with_attributes = dataset
.annotations
.iter()
.filter(|ann| !ann.attributes.is_empty())
.count();
if anns_with_attributes > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationAttributes,
format!(
"{} annotation(s) have attributes that will be dropped; SageMaker class IDs are assigned from category order on write",
anns_with_attributes
),
));
}
report.output = report.input.clone();
}
fn analyze_to_cvat(dataset: &Dataset, report: &mut ConversionReport) {
if !dataset.info.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropDatasetInfo,
"dataset info/metadata will be dropped".to_string(),
));
}
if !dataset.licenses.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropLicenses,
format!("{} license(s) will be dropped", dataset.licenses.len()),
));
}
let images_with_metadata = dataset
.images
.iter()
.filter(|img| {
img.license_id.is_some() || img.date_captured.is_some() || !img.attributes.is_empty()
})
.count();
if images_with_metadata > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropImageMetadata,
format!(
"{} image(s) have metadata that CVAT cannot represent (license/date/image attributes)",
images_with_metadata
),
));
}
let cats_with_supercategory = dataset
.categories
.iter()
.filter(|cat| cat.supercategory.is_some())
.count();
if cats_with_supercategory > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropCategorySupercategory,
format!(
"{} category(s) have supercategory that will be dropped",
cats_with_supercategory
),
));
}
let anns_with_confidence = dataset
.annotations
.iter()
.filter(|ann| ann.confidence.is_some())
.count();
if anns_with_confidence > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationConfidence,
format!(
"{} annotation(s) have confidence scores that will be dropped",
anns_with_confidence
),
));
}
let anns_with_unrepresentable_attrs = dataset
.annotations
.iter()
.filter(|ann| {
ann.attributes.keys().any(|key| {
!(key == "occluded"
|| key == "z_order"
|| key == "source"
|| key.starts_with("cvat_attr_"))
})
})
.count();
if anns_with_unrepresentable_attrs > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationAttributes,
format!(
"{} annotation(s) have attributes outside CVAT's preserved set (occluded/z_order/source/cvat_attr_*)",
anns_with_unrepresentable_attrs
),
));
}
let used_category_ids: HashSet<_> = dataset.annotations.iter().map(|a| a.category_id).collect();
let unused_count = dataset
.categories
.iter()
.filter(|cat| !used_category_ids.contains(&cat.id))
.count();
report.output = report.input.clone();
if unused_count > 0 {
report.output.categories -= unused_count;
report.add(ConversionIssue::warning(
ConversionIssueCode::CvatWriterDropUnusedCategories,
format!(
"{} category(s) not referenced by any annotation will be dropped from CVAT <meta><labels>",
unused_count
),
));
}
}
fn add_tfod_reader_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::TfodReaderIdAssignment,
"TFOD reader assigns IDs deterministically: images by filename (lexicographic), \
categories by class name (lexicographic), annotations by CSV row order"
.to_string(),
));
}
fn add_tfod_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::TfodWriterRowOrder,
"TFOD writer orders rows by annotation ID for deterministic output".to_string(),
));
}
fn add_tfrecord_reader_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::TfrecordReaderIdAssignment,
"TFRecord reader assigns IDs deterministically: images by filename, categories by class name, annotations by record/object order".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::TfrecordReaderPayloadPolicy,
"TFRecord reader supports uncompressed TFOD-style tf.train.Example records, maps normalized XYXY boxes to pixel XYXY, and records that image/encoded bytes were present without storing image bytes in IR".to_string(),
));
}
fn add_tfrecord_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::TfrecordWriterExampleOrder,
"TFRecord writer emits one Example per image, ordered by filename then image ID; objects are ordered by annotation ID".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::TfrecordWriterPayloadPolicy,
"TFRecord writer emits uncompressed TFOD-style tf.train.Example records with normalized XYXY boxes, class text, numeric class labels, and no embedded image bytes".to_string(),
));
}
fn add_yolo_reader_policy(dataset: &Dataset, report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::YoloReaderIdAssignment,
"YOLO reader assigns IDs deterministically: images by relative path (lexicographic), categories by class index, annotations by label-file order then line number".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::YoloReaderClassMapSource,
"YOLO reader class map source precedence: data.yaml, then classes.txt, then inferred from label files".to_string(),
));
if let Some(mode) = dataset.info.attributes.get("yolo_layout_mode") {
if mode == "split_aware" {
let found = dataset
.info
.attributes
.get("yolo_splits_found")
.map(|s| s.as_str())
.unwrap_or("?");
let read = dataset
.info
.attributes
.get("yolo_splits_read")
.map(|s| s.as_str())
.unwrap_or("?");
let message = if found == read {
format!(
"YOLO reader discovered splits [{}] and merged them into one dataset",
found
)
} else {
format!(
"YOLO reader discovered splits [{}]; selected split(s): [{}]",
found, read
)
};
report.add(ConversionIssue::reader_info(
ConversionIssueCode::YoloReaderSplitHandling,
message,
));
}
}
}
fn add_yolo_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::YoloWriterClassOrder,
"YOLO writer assigns class indices by CategoryId order (sorted ascending)".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::YoloWriterEmptyLabelFiles,
"YOLO writer creates empty .txt files for images without annotations".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::YoloWriterFloatPrecision,
"YOLO writer outputs normalized coordinates (and confidence when present) with 6 decimal places".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::YoloWriterDeterministicOrder,
"YOLO writer orders images and label files by file_name (lexicographic) for deterministic output".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::YoloWriterNoImageCopy,
"YOLO writer creates only label files and data.yaml; image binaries are not copied to the output directory".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::YoloWriterDataYamlPolicy,
"YOLO writer emits data.yaml with a names: mapping (sorted by class index); does not emit train/val paths or nc".to_string(),
));
}
fn add_yolo_keras_txt_reader_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::YoloKerasTxtReaderIdAssignment,
"YOLO Keras-style TXT reader assigns IDs deterministically: images by image_ref, categories by zero-based class_id, annotations by row order".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::YoloKerasTxtReaderClassMapSource,
"YOLO Keras-style TXT reader resolves class names from classes.txt/class_names.txt/classes.names/obj.names, falling back to class_<id> names".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::YoloKerasTxtReaderImageResolution,
"YOLO Keras-style TXT reader resolves image dimensions from image_ref relative to the annotation directory, then images/image_ref".to_string(),
));
}
fn add_yolo_keras_txt_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::YoloKerasTxtWriterClassOrder,
"YOLO Keras-style TXT writer assigns zero-based class IDs by CategoryId order".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::YoloKerasTxtWriterDeterministicOrder,
"YOLO Keras-style TXT writer orders rows by image file_name and boxes by annotation ID"
.to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::YoloKerasTxtWriterEmptyRows,
"YOLO Keras-style TXT writer emits image-only rows for images without annotations"
.to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::YoloKerasTxtWriterNoImageCopy,
"YOLO Keras-style TXT writer creates only annotation and class files; image binaries are not copied".to_string(),
));
}
fn add_voc_reader_policy(dataset: &Dataset, report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::VocReaderIdAssignment,
"VOC reader assigns IDs deterministically: images by <filename> (lexicographic), categories by class name (lexicographic), annotations by XML file order then <object> order".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::VocReaderAttributeMapping,
"VOC reader maps pose/truncated/difficult/occluded into annotation attributes".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::VocReaderCoordinatePolicy,
"VOC reader keeps bndbox coordinates exactly as provided (no 0/1-based adjustment)"
.to_string(),
));
let has_non_rgb_depth = dataset
.images
.iter()
.filter_map(|image| image.attributes.get("depth"))
.filter_map(|depth| depth.parse::<u32>().ok())
.any(|depth| depth != 3);
if has_non_rgb_depth {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::VocReaderDepthHandling,
"VOC reader preserves <depth> as image attribute 'depth'; non-3 depth values may indicate non-RGB imagery"
.to_string(),
));
}
}
fn add_voc_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::VocWriterFileLayout,
"VOC writer emits one XML per image under Annotations/, preserving image subdirectory structure"
.to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::VocWriterNoImageCopy,
"VOC writer creates JPEGImages/README.txt but does not copy image binaries".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::VocWriterBoolNormalization,
"VOC writer normalizes truncated/difficult/occluded attributes: true/yes/1 -> 1 and false/no/0 -> 0"
.to_string(),
));
}
fn add_label_studio_reader_policy(dataset: &Dataset, report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::LabelStudioReaderIdAssignment,
"Label Studio reader assigns IDs deterministically: images by derived file_name (lexicographic), categories by label (lexicographic), annotations by image order then result order".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::LabelStudioReaderImageRefPolicy,
"Label Studio reader derives Image.file_name from data.image basename and preserves full source reference in image attribute ls_image_ref".to_string(),
));
let has_rotation = dataset
.annotations
.iter()
.any(|ann| ann.attributes.contains_key("ls_rotation_deg"));
if has_rotation {
report.add(ConversionIssue::warning(
ConversionIssueCode::LabelStudioRotationDropped,
"Label Studio rotated bbox converted to axis-aligned envelope (original angle stored in annotation attribute ls_rotation_deg)".to_string(),
));
}
}
fn add_label_studio_writer_policy(dataset: &Dataset, report: &mut ConversionReport) {
let used_defaults = dataset.images.iter().any(|image| {
!image.attributes.contains_key("ls_from_name")
|| !image.attributes.contains_key("ls_to_name")
});
if used_defaults {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::LabelStudioWriterFromToDefaults,
"Label Studio writer uses from_name='label' and to_name='image' when ls_from_name/ls_to_name attributes are absent".to_string(),
));
}
let has_confidence = dataset
.annotations
.iter()
.any(|ann| ann.confidence.is_some());
if has_confidence {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::LabelStudioWriterConfidenceRouting,
"Label Studio writer routes annotations with confidence scores to the 'predictions' block instead of 'annotations'".to_string(),
));
}
}
fn analyze_to_labelbox(dataset: &Dataset, report: &mut ConversionReport) {
add_dataset_metadata_and_license_drop_warnings(dataset, report);
let images_with_unrepresentable_metadata = dataset
.images
.iter()
.filter(|image| {
image.license_id.is_some()
|| image.date_captured.is_some()
|| image.attributes.keys().any(|key| {
!matches!(
key.as_str(),
"labelbox_data_row_id" | "labelbox_row_data" | "labelbox_global_key"
)
})
})
.count();
if images_with_unrepresentable_metadata > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropImageMetadata,
format!(
"{} image(s) have metadata/attributes outside Labelbox's preserved data_row hints",
images_with_unrepresentable_metadata
),
));
}
add_category_supercategory_drop_warning(dataset, report);
add_annotation_confidence_drop_warning(dataset, report);
add_annotation_attributes_drop_warnings(dataset, report);
report.output = report.input.clone();
}
fn analyze_to_scale_ai(dataset: &Dataset, report: &mut ConversionReport) {
add_dataset_metadata_and_license_drop_warnings(dataset, report);
let images_with_unrepresentable_metadata = dataset
.images
.iter()
.filter(|image| {
image.license_id.is_some()
|| image.date_captured.is_some()
|| image
.attributes
.keys()
.any(|key| !matches!(key.as_str(), "scale_ai_task_id" | "scale_ai_attachment"))
})
.count();
if images_with_unrepresentable_metadata > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropImageMetadata,
format!(
"{} image(s) have metadata/attributes outside Scale AI's preserved task/attachment hints",
images_with_unrepresentable_metadata
),
));
}
add_category_supercategory_drop_warning(dataset, report);
add_annotation_confidence_drop_warning(dataset, report);
add_annotation_attributes_drop_warnings(dataset, report);
report.output = report.input.clone();
}
fn add_labelbox_reader_policy(dataset: &Dataset, report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::LabelboxReaderIdAssignment,
"Labelbox reader assigns IDs deterministically: images by file_name, categories by label name, annotations by sorted image/project/label/object order".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::LabelboxReaderImageMetadata,
"Labelbox reader derives Image.file_name from data_row.external_id, falling back to row_data basename or global_key; media_attributes provide dimensions".to_string(),
));
if dataset
.info
.attributes
.get("labelbox_polygon_envelopes")
.and_then(|value| value.parse::<usize>().ok())
.unwrap_or(0)
> 0
{
report.add(ConversionIssue::warning(
ConversionIssueCode::LabelboxPolygonEnvelopeApplied,
"Labelbox reader converted polygon objects to axis-aligned bounding box envelopes"
.to_string(),
));
}
if let Some(skipped) = dataset.info.attributes.get("labelbox_skipped_objects") {
report.add(ConversionIssue::warning(
ConversionIssueCode::LabelboxUnsupportedObjectsSkipped,
format!(
"Labelbox reader skipped {skipped} unsupported non-detection object(s) while preserving their image rows"
),
));
}
}
fn add_labelbox_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::LabelboxWriterFormatPolicy,
"Labelbox writer emits NDJSON for .ndjson/.jsonl outputs and a JSON array for other paths"
.to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::LabelboxWriterRectanglePolicy,
"Labelbox writer emits all annotations as ImageBoundingBox objects with bounding_box geometry".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::LabelboxWriterNoImageCopy,
"Labelbox writer creates only annotation rows; image binaries are not copied".to_string(),
));
}
fn add_scale_ai_reader_policy(dataset: &Dataset, report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::ScaleAiReaderIdAssignment,
"Scale AI reader assigns IDs deterministically: images by derived file_name, categories by label name, annotations by sorted image/source annotation order".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::ScaleAiReaderImageMetadata,
"Scale AI reader derives Image.file_name from metadata.file_name or attachment basename and resolves dimensions from metadata, local image files, or annotation extents".to_string(),
));
let polygon_envelopes = dataset
.info
.attributes
.get("scale_ai_polygon_envelopes")
.and_then(|value| value.parse::<usize>().ok())
.unwrap_or(0);
let rotated_envelopes = dataset
.info
.attributes
.get("scale_ai_rotated_box_envelopes")
.and_then(|value| value.parse::<usize>().ok())
.unwrap_or(0);
if polygon_envelopes > 0 || rotated_envelopes > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::ScaleAiGeometryEnvelopeApplied,
format!(
"Scale AI reader converted {polygon_envelopes} polygon annotation(s) and {rotated_envelopes} rotated box annotation(s) to axis-aligned bounding box envelopes"
),
));
}
}
fn add_scale_ai_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::ScaleAiWriterDeterministicOrder,
"Scale AI writer emits deterministic task objects ordered by image filename with annotations ordered by annotation ID".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::ScaleAiWriterRectanglePolicy,
"Scale AI writer emits all IR annotations as type='box' response annotations using left/top/width/height".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::ScaleAiWriterNoImageCopy,
"Scale AI writer creates annotation JSON and an images/README.txt placeholder for directory outputs; image binaries are not copied".to_string(),
));
}
fn analyze_to_unity_perception(dataset: &Dataset, report: &mut ConversionReport) {
add_dataset_metadata_and_license_drop_warnings(dataset, report);
let images_with_unrepresentable_metadata = dataset
.images
.iter()
.filter(|image| {
image.license_id.is_some()
|| image.date_captured.is_some()
|| image.attributes.keys().any(|key| {
!matches!(
key.as_str(),
"unity_perception_capture_id"
| "unity_perception_sequence_id"
| "unity_perception_sequence"
| "unity_perception_step"
| "unity_perception_frame"
| "unity_perception_timestamp"
| "unity_perception_sensor_id"
)
})
})
.count();
if images_with_unrepresentable_metadata > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropImageMetadata,
format!(
"{} image(s) have metadata/attributes outside Unity Perception's preserved capture hints",
images_with_unrepresentable_metadata
),
));
}
add_category_supercategory_drop_warning(dataset, report);
add_annotation_confidence_drop_warning(dataset, report);
add_annotation_attributes_drop_warnings(dataset, report);
report.output = report.input.clone();
}
fn add_unity_perception_reader_policy(dataset: &Dataset, report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::UnityPerceptionReaderIdAssignment,
"Unity Perception reader assigns IDs deterministically: images by filename, categories by source label order plus label name, annotations by sorted image/frame annotation order".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::UnityPerceptionReaderImageMetadata,
"Unity Perception reader preserves captures as images and resolves dimensions from capture dimension, local image files, or bbox extents".to_string(),
));
if let Some(skipped) = dataset
.info
.attributes
.get("unity_perception_skipped_annotations")
{
report.add(ConversionIssue::warning(
ConversionIssueCode::UnityPerceptionUnsupportedAnnotationsSkipped,
format!(
"Unity Perception reader skipped {skipped} unsupported non-bbox annotation block(s) while preserving their captures/images"
),
));
}
}
fn add_unity_perception_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::UnityPerceptionWriterDirectoryLayout,
"Unity Perception writer emits a minimal SOLO-like directory with annotation_definitions.json and sequence.0/step*.frame_data.json files".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::UnityPerceptionWriterRectanglePolicy,
"Unity Perception writer emits all IR annotations as BoundingBox2D values using x/y/width/height".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::UnityPerceptionWriterNoImageCopy,
"Unity Perception writer creates an images/README.txt placeholder; image binaries are not copied".to_string(),
));
}
fn add_cvat_reader_policy(_dataset: &Dataset, report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::CvatReaderIdAssignment,
"CVAT reader assigns IDs deterministically: images by <image name> (lexicographic), categories by label name (lexicographic), annotations by image order then <box> order".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::CvatReaderAttributePolicy,
"CVAT reader maps xtl/ytl/xbr/ybr to IR pixel XYXY 1:1; custom <attribute> children are stored as annotation attributes with 'cvat_attr_' prefix".to_string(),
));
}
fn add_cvat_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::CvatWriterMetaDefaults,
"CVAT writer emits a minimal <meta><task> block with name='panlabel export', mode='annotation', and size equal to image count".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::CvatWriterDeterministicOrder,
"CVAT writer orders images by file_name (lexicographic) and boxes within each image by annotation ID".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::CvatWriterImageIdReassignment,
"CVAT writer assigns sequential image IDs (0, 1, 2, ...) by sorted order; original cvat_image_id attributes are not preserved in output".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::CvatWriterSourceDefault,
"CVAT writer defaults missing or empty source attribute to 'manual'".to_string(),
));
}
fn add_hf_reader_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::HfReaderCategoryResolution,
"HF reader resolves category names with precedence: ClassLabel/preflight map, then --hf-category-map, then integer fallback"
.to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::HfReaderObjectContainerPrecedence,
"HF reader selects the object container with precedence: --hf-objects-column override, then 'objects' column, then 'faces' column"
.to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::HfReaderBboxFormatDependence,
"HF reader interprets bounding boxes according to --hf-bbox-format (default: xywh); incorrect setting will produce wrong coordinates"
.to_string(),
));
}
fn add_hf_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::HfWriterDeterministicOrder,
"HF writer orders metadata rows by image file_name and annotation lists by annotation ID"
.to_string(),
));
}
fn add_sagemaker_reader_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::SagemakerReaderIdAssignment,
"SageMaker reader assigns IDs deterministically: images by derived file_name, categories by source class_id, annotations by image/source order"
.to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::SagemakerReaderLabelAttributeDetection,
"SageMaker reader accepts one object-detection label attribute per manifest and rejects mixed or ambiguous label attributes"
.to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::SagemakerReaderClassMapResolution,
"SageMaker reader resolves category names from metadata class-map, falling back to the numeric class_id string"
.to_string(),
));
}
fn add_sagemaker_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::SagemakerWriterDeterministicOrder,
"SageMaker writer orders JSONL rows by image file_name and annotations by annotation ID"
.to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::SagemakerWriterClassMapPolicy,
"SageMaker writer assigns class_id values by CategoryId order and emits class-map entries for all categories"
.to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::SagemakerWriterMetadataDefaults,
"SageMaker writer defaults the label attribute to 'bounding-box', preserves stored job-name/creation-date hints when present, and emits deterministic object-detection metadata defaults"
.to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::SagemakerWriterNoImageCopy,
"SageMaker writer creates only the manifest file; image binaries are not copied"
.to_string(),
));
}
fn add_coco_reader_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::CocoReaderAttributeMapping,
"COCO reader maps score to IR confidence and stores area/iscrowd as annotation attributes"
.to_string(),
));
}
fn add_coco_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::CocoWriterDeterministicOrder,
"COCO writer sorts licenses, images, categories, and annotations by ID for deterministic output"
.to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::CocoWriterScoreMapping,
"COCO writer maps IR confidence to the COCO score field".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::CocoWriterAreaIscrowdMapping,
"COCO writer reads area/iscrowd from annotation attributes; defaults to bbox-computed area and iscrowd=0 when absent"
.to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::CocoWriterEmptySegmentation,
"COCO writer emits an empty segmentation array for detection-only output".to_string(),
));
}
const LABELME_ATTR_SHAPE_TYPE: &str = "labelme_shape_type";
fn analyze_to_labelme(dataset: &Dataset, report: &mut ConversionReport) {
if !dataset.info.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropDatasetInfo,
"dataset info/metadata will be dropped".to_string(),
));
}
if !dataset.licenses.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropLicenses,
format!("{} license(s) will be dropped", dataset.licenses.len()),
));
}
let has_license_or_date = dataset
.images
.iter()
.any(|img| img.license_id.is_some() || img.date_captured.is_some());
if has_license_or_date {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropImageMetadata,
"image license_id and/or date_captured will be dropped".to_string(),
));
}
let has_supercategory = dataset.categories.iter().any(|c| c.supercategory.is_some());
if has_supercategory {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropCategorySupercategory,
"category supercategory will be dropped".to_string(),
));
}
let has_confidence = dataset.annotations.iter().any(|a| a.confidence.is_some());
if has_confidence {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationConfidence,
"annotation confidence values will be dropped".to_string(),
));
}
let has_non_labelme_attrs = dataset
.annotations
.iter()
.any(|a| a.attributes.keys().any(|k| k != LABELME_ATTR_SHAPE_TYPE));
if has_non_labelme_attrs {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationAttributes,
"annotation attributes (other than labelme_shape_type) will be dropped".to_string(),
));
}
let referenced_cats: HashSet<_> = dataset.annotations.iter().map(|a| a.category_id).collect();
let output_categories = referenced_cats.len();
report.output = ConversionCounts {
images: dataset.images.len(),
categories: output_categories,
annotations: dataset.annotations.len(),
};
}
fn add_labelme_reader_policy(dataset: &Dataset, report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::LabelmeReaderIdAssignment,
"LabelMe reader assigns image IDs by sorted file_name, category IDs by sorted label, \
and annotation IDs sequentially by image then shape order"
.to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::LabelmeReaderPathPolicy,
"LabelMe reader derives IR file_name from annotation file path and imagePath extension; \
raw imagePath is stored in image attributes as labelme_image_path"
.to_string(),
));
let has_polygons = dataset.annotations.iter().any(|a| {
a.attributes
.get(LABELME_ATTR_SHAPE_TYPE)
.map(|v| v.as_str())
== Some("polygon")
});
if has_polygons {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::LabelmePolygonEnvelopeApplied,
"LabelMe reader converted polygon shapes to axis-aligned bounding box envelopes; \
original shape type stored as labelme_shape_type=polygon attribute"
.to_string(),
));
}
}
fn add_labelme_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::LabelmeWriterFileLayout,
"LabelMe writer emits annotations/<stem>.json files in a canonical directory layout"
.to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::LabelmeWriterRectanglePolicy,
"LabelMe writer emits all annotations as rectangle shapes with 2 corner points".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::LabelmeWriterNoImageCopy,
"LabelMe writer creates only annotation files; images are not copied".to_string(),
));
}
fn analyze_to_superannotate(dataset: &Dataset, report: &mut ConversionReport) {
add_common_csv_lossiness_warnings(dataset, report);
add_image_attributes_drop_warning(dataset, report);
add_annotation_attributes_drop_warnings(dataset, report);
report.output = report.input.clone();
}
fn analyze_to_supervisely(dataset: &Dataset, report: &mut ConversionReport) {
add_common_csv_lossiness_warnings(dataset, report);
add_image_attributes_drop_warning(dataset, report);
add_annotation_drop_warnings(dataset, report);
report.output = report.input.clone();
}
fn add_superannotate_reader_policy(dataset: &Dataset, report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::SuperannotateReaderIdAssignment,
"SuperAnnotate reader assigns image IDs by sorted file_name, category IDs by sorted label, and annotation IDs sequentially by image then instance order"
.to_string(),
));
let has_enveloped_geometry = dataset.annotations.iter().any(|ann| {
ann.attributes
.get("superannotate_geometry_type")
.map(|value| value != "bbox")
.unwrap_or(false)
});
if has_enveloped_geometry {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::SuperannotatePolygonEnvelopeApplied,
"SuperAnnotate reader converted polygon/rotated geometries to axis-aligned bounding box envelopes; original geometry type is stored as superannotate_geometry_type"
.to_string(),
));
}
}
fn add_superannotate_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::SuperannotateWriterFileLayout,
"SuperAnnotate writer emits annotations/<stem>.json files plus classes/classes.json in a canonical directory layout"
.to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::SuperannotateWriterRectanglePolicy,
"SuperAnnotate writer emits all annotations as bbox instances".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::SuperannotateWriterNoImageCopy,
"SuperAnnotate writer creates only annotation/class files; images are not copied"
.to_string(),
));
}
fn add_supervisely_reader_policy(dataset: &Dataset, report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::SuperviselyReaderIdAssignment,
"Supervisely reader assigns image IDs by sorted file_name, category IDs by sorted label, and annotation IDs sequentially by image then object order"
.to_string(),
));
let has_polygons = dataset.annotations.iter().any(|ann| {
ann.attributes
.get("supervisely_geometry_type")
.map(|value| value == "polygon")
.unwrap_or(false)
});
if has_polygons {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::SuperviselyPolygonEnvelopeApplied,
"Supervisely reader converted polygon geometries to axis-aligned bounding box envelopes; original geometry type is stored as supervisely_geometry_type"
.to_string(),
));
}
}
fn add_supervisely_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::SuperviselyWriterProjectLayout,
"Supervisely writer emits a canonical project layout with meta.json and dataset/ann/*.json"
.to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::SuperviselyWriterRectanglePolicy,
"Supervisely writer emits all annotations as rectangle objects".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::SuperviselyWriterNoImageCopy,
"Supervisely writer creates only annotation/meta files; images are not copied".to_string(),
));
}
fn analyze_to_cityscapes(dataset: &Dataset, report: &mut ConversionReport) {
add_common_csv_lossiness_warnings(dataset, report);
add_image_attributes_drop_warning(dataset, report);
add_annotation_drop_warnings(dataset, report);
report.output = report.input.clone();
}
fn analyze_to_marmot(dataset: &Dataset, report: &mut ConversionReport) {
add_common_csv_lossiness_warnings(dataset, report);
add_image_attributes_drop_warning(dataset, report);
add_annotation_drop_warnings(dataset, report);
report.output = report.input.clone();
}
fn add_cityscapes_reader_policy(dataset: &Dataset, report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::CityscapesReaderIdAssignment,
"Cityscapes reader assigns image IDs by sorted file_name, category IDs by sorted label, and annotation IDs sequentially by image then object order"
.to_string(),
));
let has_envelopes = dataset.annotations.iter().any(|ann| {
ann.attributes
.get("cityscapes_bbox_source")
.map(|value| value == "polygon_envelope")
.unwrap_or(false)
});
if has_envelopes {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::CityscapesPolygonEnvelopeApplied,
"Cityscapes reader converted polygons to axis-aligned bounding box envelopes; original labels are stored as cityscapes_original_label"
.to_string(),
));
}
report.add(ConversionIssue::reader_info(
ConversionIssueCode::CityscapesSkippedLabels,
"Cityscapes reader skips deleted objects plus ignored/stuff labels, and marks unknown kept labels with cityscapes_label_status=unknown"
.to_string(),
));
}
fn add_cityscapes_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::CityscapesWriterGtFineLayout,
"Cityscapes writer emits gtFine/<split>/<city>/*_gtFine_polygons.json files using image attributes when available"
.to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::CityscapesWriterRectanglePolygonPolicy,
"Cityscapes writer emits each bbox as a deterministic four-point rectangle polygon"
.to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::CityscapesWriterNoImageCopy,
"Cityscapes writer creates only annotation files and a leftImg8bit placeholder; images are not copied"
.to_string(),
));
}
fn add_marmot_reader_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::MarmotReaderIdAssignment,
"Marmot reader assigns image IDs by sorted companion image name, category IDs by sorted Composite label, and annotation IDs by XML/object order".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::MarmotReaderHexCoordinateTransform,
"Marmot reader decodes Page@CropBox and Composite@BBox as four big-endian f64 hex tokens, scales through the CropBox, and flips the Y axis into pixel-space XYXY".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::MarmotReaderCompanionImageRequired,
"Marmot reader requires a same-stem companion image to get pixel dimensions; CropBox alone is not treated as image dimensions".to_string(),
));
}
fn add_marmot_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::MarmotWriterMinimalXml,
"Marmot writer emits deterministic minimal Page/Composites/Composite XML with CropBox/BBox encoded as big-endian f64 hex tokens".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::MarmotWriterNoImageCopy,
"Marmot writer creates XML annotation files only; image binaries are not copied"
.to_string(),
));
}
fn analyze_to_createml(dataset: &Dataset, report: &mut ConversionReport) {
if !dataset.info.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropDatasetInfo,
"dataset info/metadata will be dropped".to_string(),
));
}
if !dataset.licenses.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropLicenses,
format!("{} license(s) will be dropped", dataset.licenses.len()),
));
}
let has_license_or_date = dataset
.images
.iter()
.any(|img| img.license_id.is_some() || img.date_captured.is_some());
if has_license_or_date {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropImageMetadata,
"image license_id and/or date_captured will be dropped".to_string(),
));
}
let has_supercategory = dataset.categories.iter().any(|c| c.supercategory.is_some());
if has_supercategory {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropCategorySupercategory,
"category supercategory will be dropped".to_string(),
));
}
let has_confidence = dataset.annotations.iter().any(|a| a.confidence.is_some());
if has_confidence {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationConfidence,
"annotation confidence values will be dropped".to_string(),
));
}
let has_attributes = dataset.annotations.iter().any(|a| !a.attributes.is_empty());
if has_attributes {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationAttributes,
"annotation attributes will be dropped".to_string(),
));
}
let referenced_cats: HashSet<_> = dataset.annotations.iter().map(|a| a.category_id).collect();
let output_categories = referenced_cats.len();
report.output = ConversionCounts {
images: dataset.images.len(),
categories: output_categories,
annotations: dataset.annotations.len(),
};
}
fn add_createml_reader_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::CreatemlReaderIdAssignment,
"CreateML reader assigns image IDs by sorted filename, category IDs by sorted label, \
and annotation IDs sequentially by image then annotation order"
.to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::CreatemlReaderImageResolution,
"CreateML reader resolves image dimensions from local files: \
tries <json_dir>/<image>, then <json_dir>/images/<image>"
.to_string(),
));
}
fn add_createml_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::CreatemlWriterDeterministicOrder,
"CreateML writer orders image rows by filename and annotations by ID".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::CreatemlWriterCoordinateMapping,
"CreateML writer converts IR pixel XYXY to center-based absolute coordinates (x, y, width, height)"
.to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::CreatemlWriterNoImageCopy,
"CreateML writer creates only the JSON file; images are not copied".to_string(),
));
}
const KITTI_PRESERVED_ATTRS: &[&str] = &[
"kitti_truncated",
"kitti_occluded",
"kitti_alpha",
"kitti_dim_height",
"kitti_dim_width",
"kitti_dim_length",
"kitti_loc_x",
"kitti_loc_y",
"kitti_loc_z",
"kitti_rotation_y",
];
fn analyze_to_kitti(dataset: &Dataset, report: &mut ConversionReport) {
if !dataset.info.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropDatasetInfo,
"dataset info/metadata will be dropped".to_string(),
));
}
if !dataset.licenses.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropLicenses,
format!("{} license(s) will be dropped", dataset.licenses.len()),
));
}
let images_with_metadata = dataset
.images
.iter()
.filter(|img| img.license_id.is_some() || img.date_captured.is_some())
.count();
if images_with_metadata > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropImageMetadata,
format!(
"{} image(s) have license_id/date_captured that will be dropped",
images_with_metadata
),
));
}
let cats_with_supercategory = dataset
.categories
.iter()
.filter(|cat| cat.supercategory.is_some())
.count();
if cats_with_supercategory > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropCategorySupercategory,
format!(
"{} category(s) have supercategory that will be dropped",
cats_with_supercategory
),
));
}
let anns_with_unrepresentable_attrs = dataset
.annotations
.iter()
.filter(|ann| {
ann.attributes
.keys()
.any(|key| !KITTI_PRESERVED_ATTRS.contains(&key.as_str()))
})
.count();
if anns_with_unrepresentable_attrs > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationAttributes,
format!(
"{} annotation(s) have attributes outside KITTI's preserved set (kitti_*)",
anns_with_unrepresentable_attrs
),
));
}
report.output = report.input.clone();
}
fn analyze_to_via(dataset: &Dataset, report: &mut ConversionReport) {
if !dataset.info.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropDatasetInfo,
"dataset info/metadata will be dropped".to_string(),
));
}
if !dataset.licenses.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropLicenses,
format!("{} license(s) will be dropped", dataset.licenses.len()),
));
}
let images_with_metadata = dataset
.images
.iter()
.filter(|img| {
img.license_id.is_some()
|| img.date_captured.is_some()
|| img
.attributes
.keys()
.any(|key| key != "via_size_bytes" && !key.starts_with("via_file_attr_"))
})
.count();
if images_with_metadata > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropImageMetadata,
format!(
"{} image(s) have metadata that VIA cannot represent",
images_with_metadata
),
));
}
let cats_with_supercategory = dataset
.categories
.iter()
.filter(|cat| cat.supercategory.is_some())
.count();
if cats_with_supercategory > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropCategorySupercategory,
format!(
"{} category(s) have supercategory that will be dropped",
cats_with_supercategory
),
));
}
let anns_with_confidence = dataset
.annotations
.iter()
.filter(|ann| ann.confidence.is_some())
.count();
if anns_with_confidence > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationConfidence,
format!(
"{} annotation(s) have confidence scores that will be dropped",
anns_with_confidence
),
));
}
let anns_with_unrepresentable_attrs = dataset
.annotations
.iter()
.filter(|ann| {
ann.attributes
.keys()
.any(|key| !key.starts_with("via_region_attr_"))
})
.count();
if anns_with_unrepresentable_attrs > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationAttributes,
format!(
"{} annotation(s) have attributes outside VIA's preserved set (via_region_attr_*)",
anns_with_unrepresentable_attrs
),
));
}
report.output = report.input.clone();
}
fn analyze_to_retinanet(dataset: &Dataset, report: &mut ConversionReport) {
add_common_csv_lossiness_warnings(dataset, report);
add_annotation_drop_warnings(dataset, report);
report.output = report.input.clone();
}
fn add_kitti_reader_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::KittiReaderIdAssignment,
"KITTI reader assigns image IDs by filename order, category IDs by class name order, annotation IDs by file/line order".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::KittiReaderFieldMapping,
"KITTI non-bbox fields (truncated, occluded, alpha, dimensions, location, rotation) are stored as kitti_* annotation attributes".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::KittiReaderImageResolution,
"KITTI reader resolves image dimensions from image_2/ with extension precedence: .png, .jpg, .jpeg, .bmp, .webp".to_string(),
));
}
fn add_kitti_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::KittiWriterFileLayout,
"KITTI writer creates label_2/ with one .txt per image and image_2/README.txt".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::KittiWriterDefaultFieldValues,
"KITTI writer uses default values for missing kitti_* attributes (truncated=0, occluded=0, alpha=-10, dims=-1, loc=-1000, rotation=-10)".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::KittiWriterDeterministicOrder,
"KITTI writer sorts images by filename and annotations within each image by ID".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::KittiWriterNoImageCopy,
"KITTI writer creates only label files; images are not copied".to_string(),
));
}
fn add_via_reader_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::ViaReaderIdAssignment,
"VIA reader assigns image IDs by filename order, category IDs by label order, annotation IDs by image/region order".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::ViaReaderLabelResolution,
"VIA reader resolves category labels from region_attributes with precedence: 'label', 'class', then sole scalar attribute".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::ViaReaderImageResolution,
"VIA reader resolves image dimensions from disk: <json_dir>/<filename> then <json_dir>/images/<filename>".to_string(),
));
}
fn add_via_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::ViaWriterDeterministicOrder,
"VIA writer orders entries by filename and regions by annotation ID".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::ViaWriterLabelAttributeKey,
"VIA writer uses canonical 'label' key in region_attributes for category names".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::ViaWriterNoImageCopy,
"VIA writer creates only the JSON file; images are not copied".to_string(),
));
}
fn add_retinanet_reader_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::RetinanetReaderIdAssignment,
"RetinaNet reader assigns image IDs by path order, category IDs by class name order, annotation IDs by row order".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::RetinanetReaderImageResolution,
"RetinaNet reader resolves image dimensions from disk relative to CSV parent directory"
.to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::RetinanetReaderEmptyRowHandling,
"RetinaNet reader treats rows with empty bbox/class fields as unannotated image entries"
.to_string(),
));
}
fn add_retinanet_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::RetinanetWriterDeterministicOrder,
"RetinaNet writer groups rows by image (sorted by filename) with annotations sorted by ID"
.to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::RetinanetWriterEmptyRows,
"RetinaNet writer emits path,,,,, rows for images without annotations".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::RetinanetWriterNoImageCopy,
"RetinaNet writer creates only the CSV file; images are not copied".to_string(),
));
}
fn add_vott_csv_reader_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::VottCsvReaderIdAssignment,
"VoTT CSV reader assigns image IDs by image path, category IDs by label, and annotation IDs by sorted row content".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::VottCsvReaderImageResolution,
"VoTT CSV reader resolves image dimensions from disk: <csv_dir>/<image> then <csv_dir>/images/<image>".to_string(),
));
}
fn add_vott_csv_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::VottCsvWriterRowOrder,
"VoTT CSV writer emits the header image,xmin,ymin,xmax,ymax,label and orders rows by image filename then annotation ID".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::VottCsvWriterNoImageCopy,
"VoTT CSV writer creates only the CSV file; images are not copied".to_string(),
));
}
fn add_vott_json_reader_policy(dataset: &Dataset, report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::VottJsonReaderIdAssignment,
"VoTT JSON reader assigns image IDs by filename, category IDs by project tag order plus extra labels, and annotation IDs by sorted image/region/tag order".to_string(),
));
let has_enveloped_geometry = dataset.annotations.iter().any(|ann| {
ann.attributes
.get("vott_geometry_enveloped")
.map(|value| value == "true")
.unwrap_or(false)
});
if has_enveloped_geometry {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::VottJsonPolygonEnvelopeApplied,
"VoTT JSON reader converted polygon-like point regions to axis-aligned bounding box envelopes".to_string(),
));
}
}
fn add_vott_json_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::VottJsonWriterDeterministicOrder,
"VoTT JSON writer emits deterministic aggregate assets ordered by image filename with regions ordered by annotation ID".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::VottJsonWriterRectanglePolicy,
"VoTT JSON writer emits all annotations as RECTANGLE regions with boundingBox and corner points".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::VottJsonWriterNoImageCopy,
"VoTT JSON writer creates annotation JSON and an images/README.txt placeholder for directory outputs; image binaries are not copied".to_string(),
));
}
fn add_cloud_annotations_reader_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::CloudAnnotationsReaderIdAssignment,
"IBM Cloud Annotations reader assigns image IDs by image key order, category IDs by labels array order plus extra labels, and annotation IDs by image/object order".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::CloudAnnotationsReaderImageResolution,
"IBM Cloud Annotations reader resolves image dimensions from disk: <json_dir>/<image> then <json_dir>/images/<image>".to_string(),
));
}
fn add_cloud_annotations_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::CloudAnnotationsWriterDeterministicOrder,
"IBM Cloud Annotations writer orders labels by category ID, image keys by filename, and objects by annotation ID".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::CloudAnnotationsWriterNormalizedCoordinates,
"IBM Cloud Annotations writer converts pixel XYXY boxes to normalized x/y/x2/y2 coordinates".to_string(),
));
report.add(ConversionIssue::writer_info(
ConversionIssueCode::CloudAnnotationsWriterNoImageCopy,
"IBM Cloud Annotations writer creates _annotations.json and an images/README.txt placeholder for directory outputs; image binaries are not copied".to_string(),
));
}
fn analyze_to_openimages(dataset: &Dataset, report: &mut ConversionReport) {
add_common_csv_lossiness_warnings(dataset, report);
let anns_with_non_openimages_attrs = dataset
.annotations
.iter()
.filter(|ann| ann.attributes.keys().any(|k| !k.starts_with("openimages_")))
.count();
if anns_with_non_openimages_attrs > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationAttributes,
format!(
"{} annotation(s) have non-OpenImages attributes that will be dropped",
anns_with_non_openimages_attrs
),
));
}
add_images_without_annotations_warning_and_output_counts(dataset, report);
}
fn add_openimages_reader_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::OpenimagesReaderIdAssignment,
"OpenImages reader assigns image IDs by ImageID order, category IDs by LabelName order, annotation IDs by row order".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::OpenimagesReaderImageResolution,
"OpenImages reader resolves image dimensions from local files (base_dir and base_dir/images with extension probing)".to_string(),
));
}
fn add_openimages_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::OpenimagesWriterDeterministicOrder,
"OpenImages writer orders rows by annotation ID".to_string(),
));
}
fn analyze_to_basic_bbox_preserving(
dataset: &Dataset,
report: &mut ConversionReport,
preserves_confidence: bool,
) {
add_dataset_metadata_and_license_drop_warnings(dataset, report);
add_image_attributes_drop_warning(dataset, report);
add_category_supercategory_drop_warning(dataset, report);
if !preserves_confidence {
add_annotation_confidence_drop_warning(dataset, report);
}
add_annotation_attributes_drop_warnings(dataset, report);
report.output = report.input.clone();
}
fn analyze_to_wider_face(dataset: &Dataset, report: &mut ConversionReport) {
analyze_to_basic_bbox_preserving(dataset, report, false);
if dataset.categories.len() > 1 {
report.add(ConversionIssue::warning(
ConversionIssueCode::CollapseMultipleCategoriesToSingleClass,
format!(
"{} categories will be collapsed to WIDER Face's single face class",
dataset.categories.len()
),
));
}
report.output.categories = if dataset.annotations.is_empty() { 0 } else { 1 };
}
fn analyze_to_via_csv(dataset: &Dataset, report: &mut ConversionReport) {
analyze_to_basic_bbox_preserving(dataset, report, false);
}
fn add_simple_reader_policy(
report: &mut ConversionReport,
code: ConversionIssueCode,
message: &str,
) {
report.add(ConversionIssue::reader_info(code, message.to_string()));
}
fn add_simple_writer_policy(
report: &mut ConversionReport,
code: ConversionIssueCode,
message: &str,
) {
report.add(ConversionIssue::writer_info(code, message.to_string()));
}
fn analyze_to_kaggle_wheat(dataset: &Dataset, report: &mut ConversionReport) {
add_common_csv_lossiness_warnings(dataset, report);
add_annotation_drop_warnings(dataset, report);
if dataset.categories.len() > 1 {
report.add(ConversionIssue::warning(
ConversionIssueCode::CollapseMultipleCategoriesToSingleClass,
format!(
"{} categories will be collapsed to single class 'wheat_head'",
dataset.categories.len()
),
));
}
let distinct_image_ids = add_images_without_annotations_warning(dataset, report);
report.output = ConversionCounts {
images: distinct_image_ids.len(),
categories: if dataset.annotations.is_empty() { 0 } else { 1 },
annotations: dataset.annotations.len(),
};
}
fn add_kaggle_wheat_reader_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::KaggleWheatReaderIdAssignment,
"Kaggle Wheat reader assigns image IDs by image_id order, single category 'wheat_head', annotation IDs by row order; source stored as kaggle_wheat_source image attribute".to_string(),
));
}
fn add_kaggle_wheat_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::KaggleWheatWriterDeterministicOrder,
"Kaggle Wheat writer orders rows by annotation ID and emits bbox as [x, y, width, height]"
.to_string(),
));
}
fn analyze_to_automl_vision(dataset: &Dataset, report: &mut ConversionReport) {
add_common_csv_lossiness_warnings(dataset, report);
add_annotation_drop_warnings_and_output_counts(dataset, report);
}
fn add_automl_vision_reader_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::AutomlVisionReaderIdAssignment,
"AutoML Vision reader assigns image IDs by URI order, category IDs by label order, annotation IDs by row order".to_string(),
));
report.add(ConversionIssue::reader_info(
ConversionIssueCode::AutomlVisionReaderImageResolution,
"AutoML Vision reader resolves image dimensions from local files; GCS URIs resolved by path suffix then basename".to_string(),
));
}
fn add_automl_vision_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::AutomlVisionWriterDeterministicOrder,
"AutoML Vision writer emits headerless 11-column sparse rows ordered by annotation ID"
.to_string(),
));
}
fn analyze_to_udacity(dataset: &Dataset, report: &mut ConversionReport) {
add_common_csv_lossiness_warnings(dataset, report);
add_annotation_drop_warnings_and_output_counts(dataset, report);
}
fn add_udacity_reader_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::reader_info(
ConversionIssueCode::UdacityReaderIdAssignment,
"Udacity reader assigns image IDs by filename order, category IDs by class name order, annotation IDs by row order".to_string(),
));
}
fn add_udacity_writer_policy(report: &mut ConversionReport) {
report.add(ConversionIssue::writer_info(
ConversionIssueCode::UdacityWriterRowOrder,
"Udacity writer orders rows by annotation ID".to_string(),
));
}
fn add_dataset_metadata_and_license_drop_warnings(
dataset: &Dataset,
report: &mut ConversionReport,
) {
if !dataset.info.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropDatasetInfo,
"dataset info/metadata will be dropped".to_string(),
));
}
if !dataset.licenses.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropLicenses,
format!("{} license(s) will be dropped", dataset.licenses.len()),
));
}
}
fn add_category_supercategory_drop_warning(dataset: &Dataset, report: &mut ConversionReport) {
let cats_with_supercategory = dataset
.categories
.iter()
.filter(|category| category.supercategory.is_some())
.count();
if cats_with_supercategory > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropCategorySupercategory,
format!(
"{} category(s) have supercategory that will be dropped",
cats_with_supercategory
),
));
}
}
fn add_annotation_confidence_drop_warning(dataset: &Dataset, report: &mut ConversionReport) {
let anns_with_confidence = dataset
.annotations
.iter()
.filter(|ann| ann.confidence.is_some())
.count();
if anns_with_confidence > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationConfidence,
format!(
"{} annotation(s) have confidence scores that will be dropped",
anns_with_confidence
),
));
}
}
fn add_common_csv_lossiness_warnings(dataset: &Dataset, report: &mut ConversionReport) {
if !dataset.info.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropDatasetInfo,
"dataset info/metadata will be dropped".to_string(),
));
}
if !dataset.licenses.is_empty() {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropLicenses,
format!("{} license(s) will be dropped", dataset.licenses.len()),
));
}
let images_with_metadata = dataset
.images
.iter()
.filter(|img| img.license_id.is_some() || img.date_captured.is_some())
.count();
if images_with_metadata > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropImageMetadata,
format!(
"{} image(s) have license_id/date_captured that will be dropped",
images_with_metadata
),
));
}
let cats_with_supercategory = dataset
.categories
.iter()
.filter(|cat| cat.supercategory.is_some())
.count();
if cats_with_supercategory > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropCategorySupercategory,
format!(
"{} category(s) have supercategory that will be dropped",
cats_with_supercategory
),
));
}
}
fn add_annotation_drop_warnings(dataset: &Dataset, report: &mut ConversionReport) {
let anns_with_confidence = dataset
.annotations
.iter()
.filter(|ann| ann.confidence.is_some())
.count();
if anns_with_confidence > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationConfidence,
format!(
"{} annotation(s) have confidence scores that will be dropped",
anns_with_confidence
),
));
}
add_annotation_attributes_drop_warnings(dataset, report);
}
fn add_annotation_attributes_drop_warnings(dataset: &Dataset, report: &mut ConversionReport) {
let anns_with_attributes = dataset
.annotations
.iter()
.filter(|ann| !ann.attributes.is_empty())
.count();
if anns_with_attributes > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropAnnotationAttributes,
format!(
"{} annotation(s) have attributes that will be dropped",
anns_with_attributes
),
));
}
}
fn add_image_attributes_drop_warning(dataset: &Dataset, report: &mut ConversionReport) {
let images_with_attributes = dataset
.images
.iter()
.filter(|img| !img.attributes.is_empty())
.count();
if images_with_attributes > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropImageMetadata,
format!(
"{} image(s) have attributes that will be dropped",
images_with_attributes
),
));
}
}
fn add_images_without_annotations_warning(
dataset: &Dataset,
report: &mut ConversionReport,
) -> HashSet<crate::ir::ImageId> {
let distinct_image_ids: HashSet<_> = dataset.annotations.iter().map(|a| a.image_id).collect();
let images_without = dataset
.images
.iter()
.filter(|img| !distinct_image_ids.contains(&img.id))
.count();
if images_without > 0 {
report.add(ConversionIssue::warning(
ConversionIssueCode::DropImagesWithoutAnnotations,
format!(
"{} image(s) have no annotations and will not appear in output",
images_without
),
));
}
distinct_image_ids
}
fn add_images_without_annotations_warning_and_output_counts(
dataset: &Dataset,
report: &mut ConversionReport,
) {
let distinct_image_ids = add_images_without_annotations_warning(dataset, report);
let distinct_category_ids: HashSet<_> =
dataset.annotations.iter().map(|a| a.category_id).collect();
report.output = ConversionCounts {
images: distinct_image_ids.len(),
categories: distinct_category_ids.len(),
annotations: dataset.annotations.len(),
};
}
fn add_annotation_drop_warnings_and_output_counts(
dataset: &Dataset,
report: &mut ConversionReport,
) {
add_annotation_drop_warnings(dataset, report);
add_images_without_annotations_warning_and_output_counts(dataset, report);
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ir::{
Annotation, AnnotationId, BBoxXYXY, Category, CategoryId, Coord, DatasetInfo, Image,
ImageId, License, LicenseId, Pixel,
};
fn sample_dataset() -> Dataset {
Dataset {
info: DatasetInfo {
name: Some("Test Dataset".to_string()),
..Default::default()
},
licenses: vec![License {
id: LicenseId(1),
name: "CC0".to_string(),
url: None,
}],
images: vec![
Image {
id: ImageId(1),
file_name: "img1.jpg".to_string(),
width: 100,
height: 100,
license_id: Some(LicenseId(1)),
date_captured: None,
attributes: std::collections::BTreeMap::new(),
},
Image {
id: ImageId(2),
file_name: "img2.jpg".to_string(),
width: 100,
height: 100,
license_id: None,
date_captured: None,
attributes: std::collections::BTreeMap::new(),
},
],
categories: vec![Category {
id: CategoryId(1),
name: "cat".to_string(),
supercategory: Some("animal".to_string()),
}],
annotations: vec![Annotation {
id: AnnotationId(1),
image_id: ImageId(1),
category_id: CategoryId(1),
bbox: BBoxXYXY::<Pixel>::new(Coord::new(10.0, 10.0), Coord::new(50.0, 50.0)),
confidence: Some(0.95),
attributes: [("custom".to_string(), "value".to_string())]
.into_iter()
.collect(),
}],
}
}
#[test]
fn to_tfod_detects_all_lossiness() {
let dataset = sample_dataset();
let report = build_conversion_report(&dataset, Format::Coco, Format::Tfod);
assert!(report.is_lossy());
assert!(report.warning_count() >= 6);
}
#[test]
fn to_ir_json_is_not_lossy() {
let dataset = sample_dataset();
let report = build_conversion_report(&dataset, Format::Coco, Format::IrJson);
assert!(!report.is_lossy());
assert_eq!(report.warning_count(), 0);
}
#[test]
fn to_coco_detects_name_lossiness() {
let dataset = sample_dataset();
let report = build_conversion_report(&dataset, Format::IrJson, Format::Coco);
assert!(report.is_lossy());
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::DropDatasetInfoName));
}
#[test]
fn tfod_source_adds_policy_note() {
let dataset = Dataset::default();
let report = build_conversion_report(&dataset, Format::Tfod, Format::Coco);
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::TfodReaderIdAssignment));
}
#[test]
fn tfod_target_adds_policy_note() {
let dataset = Dataset::default();
let report = build_conversion_report(&dataset, Format::Coco, Format::Tfod);
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::TfodWriterRowOrder));
}
#[test]
fn output_counts_differ_for_tfod() {
let dataset = sample_dataset(); let report = build_conversion_report(&dataset, Format::Coco, Format::Tfod);
assert_eq!(report.input.images, 2);
assert_eq!(report.output.images, 1); }
#[test]
fn yolo_target_keeps_images_without_annotations() {
let dataset = sample_dataset();
let report = build_conversion_report(&dataset, Format::IrJson, Format::Yolo);
assert!(report
.issues
.iter()
.all(|issue| issue.code != ConversionIssueCode::DropImagesWithoutAnnotations));
assert_eq!(report.output.images, report.input.images);
}
#[test]
fn yolo_source_adds_policy_notes() {
let dataset = Dataset::default();
let report = build_conversion_report(&dataset, Format::Yolo, Format::Coco);
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::YoloReaderIdAssignment));
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::YoloReaderClassMapSource));
}
#[test]
fn yolo_target_adds_policy_notes() {
let dataset = Dataset::default();
let report = build_conversion_report(&dataset, Format::Coco, Format::Yolo);
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::YoloWriterClassOrder));
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::YoloWriterEmptyLabelFiles));
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::YoloWriterFloatPrecision));
}
#[test]
fn to_voc_detects_lossiness() {
let dataset = sample_dataset();
let report = build_conversion_report(&dataset, Format::IrJson, Format::Voc);
assert!(report.is_lossy());
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::DropAnnotationAttributes));
}
#[test]
fn voc_source_adds_policy_notes_and_depth_note() {
let mut dataset = Dataset::default();
let mut image = Image::new(1u64, "img1.jpg", 100, 100);
image
.attributes
.insert("depth".to_string(), "1".to_string());
dataset.images.push(image);
let report = build_conversion_report(&dataset, Format::Voc, Format::Coco);
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::VocReaderIdAssignment));
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::VocReaderAttributeMapping));
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::VocReaderCoordinatePolicy));
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::VocReaderDepthHandling));
}
#[test]
fn voc_target_adds_policy_notes() {
let dataset = Dataset::default();
let report = build_conversion_report(&dataset, Format::Coco, Format::Voc);
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::VocWriterFileLayout));
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::VocWriterNoImageCopy));
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::VocWriterBoolNormalization));
}
#[test]
fn to_label_studio_detects_lossiness() {
let dataset = sample_dataset();
let report = build_conversion_report(&dataset, Format::IrJson, Format::LabelStudio);
assert!(report.is_lossy());
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::DropDatasetInfo));
}
#[test]
fn label_studio_source_adds_policy_notes_and_rotation_warning() {
let mut dataset = Dataset::default();
dataset.images.push(Image::new(1u64, "img.jpg", 100, 100));
dataset.categories.push(Category::new(1u64, "cat"));
let mut ann = Annotation::new(
1u64,
1u64,
1u64,
BBoxXYXY::<Pixel>::new(Coord::new(10.0, 10.0), Coord::new(20.0, 20.0)),
);
ann.attributes
.insert("ls_rotation_deg".to_string(), "15".to_string());
dataset.annotations.push(ann);
let report = build_conversion_report(&dataset, Format::LabelStudio, Format::Coco);
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::LabelStudioReaderIdAssignment));
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::LabelStudioReaderImageRefPolicy));
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::LabelStudioRotationDropped));
}
#[test]
fn label_studio_target_adds_default_name_policy_note() {
let mut dataset = Dataset::default();
dataset.images.push(Image::new(1u64, "img.jpg", 100, 100));
let report = build_conversion_report(&dataset, Format::Coco, Format::LabelStudio);
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::LabelStudioWriterFromToDefaults));
}
#[test]
fn to_cvat_detects_lossiness() {
let dataset = sample_dataset();
let report = build_conversion_report(&dataset, Format::IrJson, Format::Cvat);
assert!(report.is_lossy());
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::DropImageMetadata));
}
#[test]
fn cvat_source_and_target_add_policy_notes() {
let dataset = Dataset::default();
let from_report = build_conversion_report(&dataset, Format::Cvat, Format::Coco);
assert!(from_report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::CvatReaderIdAssignment));
assert!(from_report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::CvatReaderAttributePolicy));
let to_report = build_conversion_report(&dataset, Format::Coco, Format::Cvat);
assert!(to_report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::CvatWriterMetaDefaults));
assert!(to_report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::CvatWriterDeterministicOrder));
assert!(to_report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::CvatWriterImageIdReassignment));
assert!(to_report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::CvatWriterSourceDefault));
}
#[test]
fn cvat_output_counts_reflect_unused_category_drop() {
let mut dataset = sample_dataset();
dataset
.categories
.push(Category::new(99u64, "unused_label"));
let report = build_conversion_report(&dataset, Format::IrJson, Format::Cvat);
assert_eq!(report.input.categories, 2);
assert_eq!(report.output.categories, 1);
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::CvatWriterDropUnusedCategories));
}
#[test]
fn cvat_no_unused_category_warning_when_all_used() {
let dataset = sample_dataset();
let report = build_conversion_report(&dataset, Format::IrJson, Format::Cvat);
assert_eq!(report.input.categories, report.output.categories);
assert!(!report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::CvatWriterDropUnusedCategories));
}
#[test]
fn to_superannotate_warns_on_dropped_image_attributes() {
let mut dataset = Dataset::default();
let mut image = Image::new(1u64, "img.jpg", 100, 100);
image
.attributes
.insert("source".to_string(), "camera-a".to_string());
dataset.images.push(image);
let report = build_conversion_report(&dataset, Format::IrJson, Format::SuperAnnotate);
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::DropImageMetadata));
}
#[test]
fn to_supervisely_warns_on_dropped_image_attributes() {
let mut dataset = Dataset::default();
let mut image = Image::new(1u64, "img.jpg", 100, 100);
image
.attributes
.insert("source".to_string(), "camera-a".to_string());
dataset.images.push(image);
let report = build_conversion_report(&dataset, Format::IrJson, Format::Supervisely);
assert!(report
.issues
.iter()
.any(|i| i.code == ConversionIssueCode::DropImageMetadata));
}
}