pub const FINEWEB_EDU_100B_SHUFFLE: &str = "karpathy/fineweb-edu-100b-shuffle";
pub const FINEWEB_EDU_100B_SHUFFLE_REVISION: &str = "4c8f30d6756da75362432a4d5569e1b229263b71";
#[cfg(test)]
mod tests {
use super::*;
use crate::data::datasets::{
ShardId,
ShardSetDescriptor,
};
#[test]
fn test_fineweb_edu() {
let d = ShardSetDescriptor::from_builtin(FINEWEB_EDU_100B_SHUFFLE).unwrap();
assert_eq!(d.count, 1823);
assert!(d.is_pinned());
assert_eq!(d.file_name(ShardId(0)), "shard_00000.parquet");
assert_eq!(d.file_name(ShardId(1822)), "shard_01822.parquet");
assert_eq!(
d.urls(ShardId(312)),
vec![format!(
"https://huggingface.co/datasets/karpathy/fineweb-edu-100b-shuffle/resolve/{FINEWEB_EDU_100B_SHUFFLE_REVISION}/shard_00312.parquet"
)]
);
assert_eq!(d.parse_file_name("shard_00312.parquet"), Some(ShardId(312)));
assert_eq!(d.parse_file_name("shard_01823.parquet"), None);
assert_eq!(
d.digest(ShardId(0)),
Some("6a3450ea745ee6b6830c09819c0f2c72682eba443022de595bcb50ceb3092c22")
);
assert_eq!(
d.digest(ShardId(1822)),
Some("a97e7f2f4680561c1fdf0461295b0cfc38d85f497ba03851ab267ac1b8165990")
);
assert_eq!(d.digest(ShardId(1823)), None);
}
}