use arrow_schema::Schema as ArrowSchema;
use datafusion::execution::SendableRecordBatchStream;
use futures::StreamExt;
use lance_core::Error;
use lance_core::datatypes::Schema;
use lance_datafusion::chunker::{break_stream, chunk_stream};
use lance_datafusion::utils::StreamingWriteSource;
#[cfg(test)]
use lance_file::version::LanceFileVersion;
use lance_file::version::{ConcreteFileVersion, stable_file_version};
use lance_file::versions::v1::writer::FileWriter as V1FileWriter;
use lance_io::object_store::ObjectStore;
use lance_table::format::{DataFile, Fragment};
use lance_table::io::manifest::ManifestDescribing;
use std::borrow::Cow;
use uuid::Uuid;
use crate::Result;
use crate::dataset::builder::DatasetBuilder;
use crate::dataset::write::{
WriterOptions, blob_v2_external_base_resolver, promote_legacy_blob_schema,
validate_and_resolve_target_bases_with_primary,
};
use crate::dataset::{DATA_DIR, Dataset, ReadParams, WriteMode, WriteParams};
pub(crate) fn generate_random_filename() -> String {
let uuid = Uuid::new_v4();
let bytes = uuid.as_bytes();
let mut out = String::with_capacity(50);
for &b in &bytes[..3] {
for i in (0..8).rev() {
out.push(if (b >> i) & 1 == 1 { '1' } else { '0' });
}
}
const HEX: &[u8; 16] = b"0123456789abcdef";
for &b in &bytes[3..] {
out.push(HEX[(b >> 4) as usize] as char);
out.push(HEX[(b & 0xf) as usize] as char);
}
out
}
fn promote_legacy_blobs(version: ConcreteFileVersion, schema: Schema) -> Result<Schema> {
if crate::dataset::versions::writes_blob_v2(version) {
promote_legacy_blob_schema(&schema)
} else {
Ok(schema)
}
}
pub struct FragmentCreateBuilder<'a> {
dataset_uri: &'a str,
schema: Option<&'a Schema>,
write_params: Option<&'a WriteParams>,
}
impl<'a> FragmentCreateBuilder<'a> {
pub fn new(dataset_uri: &'a str) -> Self {
Self {
dataset_uri,
schema: None,
write_params: None,
}
}
pub fn schema(mut self, schema: &'a Schema) -> Self {
self.schema = Some(schema);
self
}
pub fn write_params(mut self, params: &'a WriteParams) -> Self {
self.write_params = Some(params);
self
}
pub async fn write(
&self,
source: impl StreamingWriteSource,
id: Option<u64>,
) -> Result<Fragment> {
let (stream, schema) = self.get_stream_and_schema(Box::new(source)).await?;
let version = self
.write_params
.map(|params| params.storage_version_or_default())
.unwrap_or_else(stable_file_version);
crate::dataset::versions::write_fragment(
version,
self,
stream,
schema,
id.unwrap_or_default(),
)
.await
}
pub async fn write_fragments(
&self,
source: impl StreamingWriteSource,
) -> Result<Vec<Fragment>> {
let (stream, schema) = self.get_stream_and_schema(Box::new(source)).await?;
self.write_fragments_v2_impl(stream, schema).await
}
pub(crate) async fn write_current_impl(
&self,
version: ConcreteFileVersion,
stream: SendableRecordBatchStream,
schema: Schema,
id: u64,
) -> Result<Fragment> {
let params = self.write_params.map(Cow::Borrowed).unwrap_or_default();
let progress = params.progress.as_ref();
Self::validate_schema(&schema, stream.schema().as_ref())?;
let schema = promote_legacy_blobs(version, schema)?;
let (object_store, base_path) = ObjectStore::from_uri_and_params(
params.store_registry(),
self.dataset_uri,
¶ms.store_params.clone().unwrap_or_default(),
)
.await?;
let external_base_resolver = if crate::dataset::versions::writes_blob_v2(version) {
blob_v2_external_base_resolver(None, ¶ms, &schema).await?
} else {
None
};
let mut writer = crate::dataset::versions::open_writer(
version,
&object_store,
&schema,
&base_path,
WriterOptions::from_params(¶ms, external_base_resolver),
)
.await?;
let mut fragment = Fragment::new(id);
let (filename, _) = writer.data_file_path();
fragment
.files
.push(DataFile::new_unstarted(filename.to_string(), version));
progress.begin(&fragment).await?;
let break_limit = (128 * 1024).min(params.max_rows_per_file);
let mut broken_stream = break_stream(stream, break_limit);
while let Some(batch) = broken_stream.next().await {
writer.write(&[batch?]).await?;
}
let (num_rows, data_file) = writer.finish().await?;
fragment.physical_rows = Some(num_rows as usize);
if matches!(fragment.physical_rows, Some(0)) {
return Err(Error::invalid_input("Input data was empty."));
}
fragment.files[0] = data_file;
progress.complete(&fragment).await?;
Ok(fragment)
}
async fn write_fragments_v2_impl(
&self,
stream: SendableRecordBatchStream,
schema: Schema,
) -> Result<Vec<Fragment>> {
let mut params = self.write_params.cloned().unwrap_or_default();
Self::validate_schema(&schema, stream.schema().as_ref())?;
let version = params.storage_version_or_default();
let schema = promote_legacy_blobs(version, schema)?;
let needs_existing_dataset = params.target_base_names_or_paths.is_some()
|| params.target_bases.is_some()
|| params.target_all_bases.is_some()
|| params.initial_bases.is_some();
let existing_dataset = if needs_existing_dataset {
self.existing_dataset(¶ms).await?
} else {
None
};
let existing_base_paths = existing_dataset
.as_ref()
.map(|dataset| &dataset.manifest.base_paths);
let (object_store, base_path) = ObjectStore::from_uri_and_params(
params.store_registry(),
self.dataset_uri,
¶ms.store_params.clone().unwrap_or_default(),
)
.await?;
let target_bases_info = if needs_existing_dataset {
validate_and_resolve_target_bases_with_primary(
&mut params,
existing_base_paths,
&object_store,
&base_path,
self.dataset_uri,
)
.await?
} else {
None
};
crate::dataset::versions::write_fragments_direct(
version,
existing_dataset.as_ref(),
object_store,
&base_path,
&schema,
stream,
params,
target_bases_info,
crate::dataset::write::seeds::SeedCollector::disabled(),
None,
None,
)
.await
}
pub(crate) async fn write_v1_impl(
&self,
stream: SendableRecordBatchStream,
schema: Schema,
id: u64,
) -> Result<Fragment> {
let params = self.write_params.map(Cow::Borrowed).unwrap_or_default();
let progress = params.progress.as_ref();
Self::validate_schema(&schema, stream.schema().as_ref())?;
let (object_store, base_path) = ObjectStore::from_uri_and_params(
params.store_registry(),
self.dataset_uri,
¶ms.store_params.clone().unwrap_or_default(),
)
.await?;
let filename = format!("{}.lance", generate_random_filename());
let mut fragment = Fragment::with_file_legacy(id, &filename, &schema, None);
let full_path = base_path.clone().join(DATA_DIR).join(filename.clone());
let mut writer = V1FileWriter::<ManifestDescribing>::try_new(
&object_store,
&full_path,
schema,
&Default::default(),
)
.await?;
progress.begin(&fragment).await?;
let mut buffered_reader = chunk_stream(stream, params.max_rows_per_group);
while let Some(batched_chunk) = buffered_reader.next().await {
let batch = batched_chunk?;
writer.write(&batch).await?;
}
if writer.is_empty() {
return Err(Error::invalid_input("Input data was empty."));
}
fragment.physical_rows = Some(writer.finish().await?.num_rows as usize);
progress.complete(&fragment).await?;
Ok(fragment)
}
async fn get_stream_and_schema(
&self,
source: impl StreamingWriteSource,
) -> Result<(SendableRecordBatchStream, Schema)> {
if let Some(schema) = self.schema {
return Ok((source.into_stream(), schema.clone()));
} else if matches!(self.write_params.map(|p| p.mode), Some(WriteMode::Append))
&& let Some(schema) = self.existing_dataset_schema().await?
{
return Ok((source.into_stream(), schema));
}
source.into_stream_and_schema().await
}
async fn existing_dataset_schema(&self) -> Result<Option<Schema>> {
let params = self.write_params.map(Cow::Borrowed).unwrap_or_default();
match self.load_existing_dataset(¶ms).await {
Ok(dataset) => Ok(Some(dataset.schema().clone())),
Err(Error::DatasetNotFound { .. }) => Ok(None),
Err(e) => Err(e),
}
}
async fn existing_dataset(&self, params: &WriteParams) -> Result<Option<Dataset>> {
match self.load_existing_dataset(params).await {
Ok(dataset) => Ok(Some(dataset)),
Err(Error::DatasetNotFound { .. } | Error::NotFound { .. }) => Ok(None),
Err(e) => Err(e),
}
}
async fn load_existing_dataset(&self, params: &WriteParams) -> Result<Dataset> {
let mut builder = DatasetBuilder::from_uri(self.dataset_uri).with_read_params(ReadParams {
store_options: params.store_params.clone(),
commit_handler: params.commit_handler.clone(),
session: params.session.clone(),
..Default::default()
});
if let Some(base_store_params) = ¶ms.base_store_params {
for (base_path, store_params) in base_store_params {
builder = builder.with_base_store_params(base_path, store_params.clone());
}
}
builder.load().await
}
fn validate_schema(expected: &Schema, actual: &ArrowSchema) -> Result<()> {
if actual.fields().is_empty() {
return Err(Error::invalid_input("Cannot write with an empty schema."));
}
let actual_lance = Schema::try_from(actual)?;
actual_lance.check_compatible(expected, &Default::default())?;
Ok(())
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use arrow_array::{
Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray, record_batch,
};
use arrow_schema::{DataType, Field as ArrowField};
use lance_arrow::SchemaExt;
use lance_core::utils::tempfile::{TempDir, TempStrDir};
use lance_table::format::BasePath;
use rstest::rstest;
use super::*;
use crate::dataset::transaction::{Operation, Transaction};
use crate::dataset::{CommitBuilder, InsertBuilder};
fn test_data() -> Box<dyn RecordBatchReader + Send> {
let schema = Arc::new(ArrowSchema::new(vec![
ArrowField::new("a", DataType::Int64, false),
ArrowField::new("b", DataType::Utf8, false),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int64Array::from(vec![1, 2, 3])),
Arc::new(StringArray::from(vec!["a", "b", "c"])),
],
);
Box::new(RecordBatchIterator::new(vec![batch], schema))
}
#[tokio::test]
async fn test_fragment_write_validation() {
let empty_schema = Arc::new(ArrowSchema::empty());
let empty_reader = Box::new(RecordBatchIterator::new(vec![], empty_schema));
let tmp_dir = TempDir::default();
let result = FragmentCreateBuilder::new(&tmp_dir.path_str())
.write(empty_reader, None)
.await;
assert!(result.is_err());
assert!(
matches!(result.as_ref().unwrap_err(), Error::InvalidInput { source, .. }
if source.to_string().contains("Cannot write with an empty schema.")),
"{:?}",
result
);
let arrow_schema = test_data().schema();
let empty_reader = Box::new(RecordBatchIterator::new(vec![], arrow_schema.clone()));
let result = FragmentCreateBuilder::new(tmp_dir.std_path().to_str().unwrap())
.write(empty_reader, None)
.await;
assert!(result.is_err());
assert!(
matches!(result.as_ref().unwrap_err(), Error::InvalidInput { source, .. }
if source.to_string().contains("Input data was empty.")),
"{:?}",
result
);
let wrong_schema = arrow_schema
.as_ref()
.try_with_column(ArrowField::new("c", DataType::Utf8, false))
.unwrap();
let wrong_schema = Schema::try_from(&wrong_schema).unwrap();
let result = FragmentCreateBuilder::new(tmp_dir.std_path().to_str().unwrap())
.schema(&wrong_schema)
.write(test_data(), None)
.await;
assert!(result.is_err());
assert!(
matches!(result.as_ref().unwrap_err(), Error::SchemaMismatch { difference, .. }
if difference.contains("fields did not match")),
"{:?}",
result
);
}
#[tokio::test]
async fn test_fragment_write_default_schema() {
let data = test_data();
let tmp_dir = TempStrDir::default();
let fragment = FragmentCreateBuilder::new(&tmp_dir)
.write(data, None)
.await
.unwrap();
assert_eq!(fragment.id, 0);
assert_eq!(fragment.deletion_file, None);
assert_eq!(fragment.files.len(), 1);
assert_eq!(fragment.files[0].fields.as_ref(), &[0, 1]);
}
#[rstest]
#[case::single_fragment(false)]
#[case::multiple_fragments(true)]
#[tokio::test]
async fn test_legacy_blob_input_is_promoted_at_v2_2(#[case] multi: bool) {
use arrow_array::{LargeBinaryArray, UInt32Array};
let schema = Arc::new(ArrowSchema::new(vec![
ArrowField::new("id", DataType::UInt32, false),
ArrowField::new("blob", DataType::LargeBinary, true).with_metadata(
std::collections::HashMap::from([(
lance_arrow::BLOB_META_KEY.to_string(),
"true".to_string(),
)]),
),
]));
let reader = |ids: Vec<u32>, blobs: Vec<Option<&[u8]>>| {
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(UInt32Array::from(ids)),
Arc::new(LargeBinaryArray::from(blobs)),
],
)
.unwrap();
Box::new(RecordBatchIterator::new([Ok(batch)], schema.clone()))
as Box<dyn RecordBatchReader + Send>
};
let test_dir = TempStrDir::default();
let dataset = Dataset::write(
reader(vec![0, 1], vec![Some(b"legacy".as_slice()), None]),
&test_dir,
Some(WriteParams {
data_storage_version: Some(LanceFileVersion::V2_1),
..Default::default()
}),
)
.await
.unwrap();
let params = WriteParams {
data_storage_version: Some(LanceFileVersion::V2_2),
..Default::default()
};
let builder = FragmentCreateBuilder::new(&test_dir).write_params(¶ms);
let source = reader(vec![2, 3], vec![Some(b"fragment".as_slice()), Some(b"")]);
let fragments = if multi {
builder.write_fragments(source).await.unwrap()
} else {
vec![builder.write(source, Some(1)).await.unwrap()]
};
let data_file = &fragments[0].files[0];
assert_eq!(data_file.file_version().unwrap(), ConcreteFileVersion::V2_2);
let file_schema = crate::utils::test::data_file_schema(&dataset, &data_file.path).await;
assert!(
matches!(
file_schema.field("blob").unwrap().data_type(),
DataType::Struct(_)
),
"{file_schema:?}"
);
let read_version = dataset.manifest.version;
let dataset = CommitBuilder::new(Arc::new(dataset))
.execute(Transaction::new(
read_version,
Operation::Append { fragments },
None,
))
.await
.unwrap();
let dataset = Arc::new(dataset);
assert!(dataset.schema().field("blob").unwrap().is_blob_v2());
let blobs = dataset
.take_blobs_by_indices(&[0, 1, 2, 3], "blob")
.await
.unwrap();
assert_eq!(
blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(),
b"legacy"
);
assert!(blobs[1].is_none());
assert_eq!(
blobs[2].as_ref().unwrap().read().await.unwrap().as_ref(),
b"fragment"
);
assert_eq!(blobs[3].as_ref().unwrap().size(), 0);
}
#[tokio::test]
async fn test_fragment_write_with_schema() {
let data = test_data();
let arrow_schema = data.schema();
let mut custom_schema = Schema::try_from(arrow_schema.as_ref()).unwrap();
custom_schema.mut_field_by_id(0).unwrap().id = 3;
custom_schema.mut_field_by_id(1).unwrap().id = 1;
let tmp_dir = TempStrDir::default();
let fragment = FragmentCreateBuilder::new(&tmp_dir)
.schema(&custom_schema)
.write(data, Some(42))
.await
.unwrap();
assert_eq!(fragment.id, 42);
assert_eq!(fragment.deletion_file, None);
assert_eq!(fragment.files.len(), 1);
assert_eq!(fragment.files[0].fields.as_ref(), &[3, 1]);
assert_eq!(fragment.files[0].column_indices.as_ref(), &[0, 1]);
}
#[tokio::test]
async fn test_fragment_create_with_session() {
let session = Arc::new(crate::session::Session::new(
0,
1024 * 1024,
Default::default(),
));
let write_params = WriteParams {
session: Some(session.clone()),
..Default::default()
};
let initial_batch =
record_batch!(("a", Int64, [1, 2, 3]), ("b", Utf8, ["a", "b", "c"])).unwrap();
let mut dataset = InsertBuilder::new("memory://")
.with_params(&write_params)
.execute(vec![initial_batch])
.await
.unwrap();
dataset.drop_columns(&["a"]).await.unwrap();
let field_id = dataset.schema().field("b").unwrap().id;
assert_ne!(field_id, 0);
let append_batch = record_batch!(("b", Utf8, ["d", "e"])).unwrap();
let append_data =
RecordBatchIterator::new([Ok(append_batch.clone())], append_batch.schema());
let append_params = WriteParams {
session: Some(session.clone()),
mode: WriteMode::Append,
..Default::default()
};
let fragment = FragmentCreateBuilder::new(dataset.uri())
.write_params(&append_params)
.write(append_data, None)
.await
.unwrap();
assert_eq!(fragment.files[0].fields.as_ref(), &[field_id]);
assert!(session.metadata_cache_stats().await.num_entries > 0);
}
#[tokio::test]
async fn test_write_fragments_validation() {
let empty_schema = Arc::new(ArrowSchema::empty());
let empty_reader = Box::new(RecordBatchIterator::new(vec![], empty_schema));
let tmp_dir = TempDir::default();
let result = FragmentCreateBuilder::new(&tmp_dir.path_str())
.write_fragments(empty_reader)
.await;
assert!(result.is_err());
assert!(
matches!(result.as_ref().unwrap_err(), Error::InvalidInput { source, .. }
if source.to_string().contains("Cannot write with an empty schema.")),
"{:?}",
result
);
let arrow_schema = test_data().schema();
let empty_reader = Box::new(RecordBatchIterator::new(vec![], arrow_schema.clone()));
let result = FragmentCreateBuilder::new(tmp_dir.std_path().to_str().unwrap())
.write_fragments(empty_reader)
.await;
assert!(result.is_ok());
assert_eq!(result.unwrap().len(), 0);
let wrong_schema = arrow_schema
.as_ref()
.try_with_column(ArrowField::new("c", DataType::Utf8, false))
.unwrap();
let wrong_schema = Schema::try_from(&wrong_schema).unwrap();
let result = FragmentCreateBuilder::new(tmp_dir.std_path().to_str().unwrap())
.schema(&wrong_schema)
.write_fragments(test_data())
.await;
assert!(result.is_err());
assert!(
matches!(result.as_ref().unwrap_err(), Error::SchemaMismatch { difference, .. }
if difference.contains("fields did not match")),
"{:?}",
result
);
}
#[tokio::test]
async fn test_write_fragments_default_schema() {
let data = test_data();
let tmp_dir = TempStrDir::default();
let fragments = FragmentCreateBuilder::new(&tmp_dir)
.write_fragments(data)
.await
.unwrap();
assert_eq!(fragments.len(), 1);
assert_eq!(fragments[0].deletion_file, None);
assert_eq!(fragments[0].files.len(), 1);
assert_eq!(fragments[0].files[0].fields.as_ref(), &[0, 1]);
}
#[tokio::test]
async fn test_write_fragments_with_options() {
let data = test_data();
let tmp_dir = TempStrDir::default();
let writer_params = WriteParams {
max_rows_per_file: 1,
..Default::default()
};
let fragments = FragmentCreateBuilder::new(&tmp_dir)
.write_params(&writer_params)
.write_fragments(data)
.await
.unwrap();
assert_eq!(fragments.len(), 3);
assert_eq!(fragments[0].deletion_file, None);
assert_eq!(fragments[0].files.len(), 1);
assert_eq!(fragments[0].files[0].column_indices.as_ref(), &[0, 1]);
assert_eq!(fragments[1].deletion_file, None);
assert_eq!(fragments[1].files.len(), 1);
assert_eq!(fragments[1].files[0].column_indices.as_ref(), &[0, 1]);
assert_eq!(fragments[2].deletion_file, None);
assert_eq!(fragments[2].files.len(), 1);
assert_eq!(fragments[2].files[0].column_indices.as_ref(), &[0, 1]);
}
#[tokio::test]
async fn test_write_fragments_with_target_base() {
let primary = TempStrDir::default();
let base1 = TempStrDir::default();
let base2 = TempStrDir::default();
let create_params = WriteParams::default()
.with_initial_bases(vec![
BasePath::new(0, base1.to_string(), Some("base1".to_string()), false),
BasePath::new(0, base2.to_string(), Some("base2".to_string()), false),
])
.with_target_base_names_or_paths(vec!["base1".to_string()]);
let dataset = InsertBuilder::new(primary.as_str())
.with_params(&create_params)
.execute_stream(test_data())
.await
.unwrap();
let append_params = WriteParams {
mode: WriteMode::Append,
..Default::default()
}
.with_target_base_names_or_paths(vec!["base2".to_string()]);
let fragments = FragmentCreateBuilder::new(dataset.uri.as_str())
.write_params(&append_params)
.write_fragments(test_data())
.await
.unwrap();
assert_eq!(fragments.len(), 1);
assert_eq!(fragments[0].files[0].base_id, Some(2));
}
#[tokio::test]
async fn test_write_fragments_with_target_all_bases() {
let primary = TempStrDir::default();
let base1 = TempStrDir::default();
let base2 = TempStrDir::default();
let create_params = WriteParams::default().with_initial_bases(vec![
BasePath::new(0, base1.to_string(), Some("base1".to_string()), false),
BasePath::new(0, base2.to_string(), Some("base2".to_string()), false),
]);
let dataset = InsertBuilder::new(primary.as_str())
.with_params(&create_params)
.execute_stream(test_data())
.await
.unwrap();
let append_params = WriteParams {
mode: WriteMode::Append,
..Default::default()
}
.with_target_all_bases(false);
let fragments = FragmentCreateBuilder::new(dataset.uri.as_str())
.write_params(&append_params)
.write_fragments(test_data())
.await
.unwrap();
assert_eq!(fragments.len(), 1);
assert_eq!(fragments[0].files[0].base_id, Some(1));
let append_params = WriteParams {
mode: WriteMode::Append,
..Default::default()
}
.with_target_all_bases(true);
let fragments = FragmentCreateBuilder::new(dataset.uri.as_str())
.write_params(&append_params)
.write_fragments(test_data())
.await
.unwrap();
assert_eq!(fragments.len(), 1);
assert_eq!(fragments[0].files[0].base_id, None);
}
#[rstest]
#[tokio::test]
async fn test_write_with_format_version(
#[values(
LanceFileVersion::V2_0,
LanceFileVersion::V2_1,
LanceFileVersion::V2_2,
LanceFileVersion::Legacy,
LanceFileVersion::Stable
)]
file_version: LanceFileVersion,
) {
let data = test_data();
let tmp_dir = TempStrDir::default();
let writer_params = WriteParams {
data_storage_version: Some(file_version),
..Default::default()
};
let fragment = FragmentCreateBuilder::new(&tmp_dir)
.write_params(&writer_params)
.write(data, None)
.await
.unwrap();
assert!(!fragment.files.is_empty());
fragment.files.iter().for_each(|f| {
let (major_version, minor_version) = file_version.resolve().to_data_file_numbers();
assert_eq!(f.file_major_version, major_version);
assert_eq!(f.file_minor_version, minor_version);
})
}
#[rstest]
#[tokio::test]
async fn test_write_fragments_with_format_version(
#[values(
LanceFileVersion::V2_0,
LanceFileVersion::V2_1,
LanceFileVersion::V2_2,
LanceFileVersion::Legacy,
LanceFileVersion::Stable
)]
file_version: LanceFileVersion,
) {
let data = test_data();
let tmp_dir = TempStrDir::default();
let writer_params = WriteParams {
data_storage_version: Some(file_version),
..Default::default()
};
let fragment = FragmentCreateBuilder::new(&tmp_dir)
.write_params(&writer_params)
.write_fragments(data)
.await
.unwrap();
assert!(!fragment.is_empty());
fragment[0].files.iter().for_each(|f| {
let (major_version, minor_version) = file_version.resolve().to_data_file_numbers();
assert_eq!(f.file_major_version, major_version);
assert_eq!(f.file_minor_version, minor_version);
})
}
#[test]
fn test_binary_filename_generation() {
use std::collections::HashSet;
let mut filenames = HashSet::new();
for _ in 0..100 {
let filename = generate_random_filename();
assert_eq!(filename.len(), 50, "Filename should be 50 characters");
let binary_part = &filename[0..24];
assert!(
binary_part.chars().all(|c| c == '0' || c == '1'),
"First 24 chars should be binary: {}",
binary_part
);
let hex_part = &filename[24..];
assert_eq!(hex_part.len(), 26, "Hex part should be 26 characters");
assert!(
hex_part.chars().all(|c| c.is_ascii_hexdigit()),
"Last 26 chars should be hex: {}",
hex_part
);
assert!(filenames.insert(filename.clone()));
}
}
}