mod fineweb_edu_100b_shuffle_digests;
pub use fineweb_edu_100b_shuffle_digests::FINEWEB_EDU_100B_SHUFFLE_SHA256;
use crate::data::shards::{
StaticShardDigests,
StaticShardSetDescriptor,
StaticShardSetMap,
};
pub const FINEWEB_EDU_100B_SHUFFLE_REVISION: &str = "4c8f30d6756da75362432a4d5569e1b229263b71";
pub static FINEWEB_EDU_100B_SHUFFLE: StaticShardSetDescriptor<'static> = StaticShardSetDescriptor {
name: "fineweb-edu-100b-shuffle",
description: "karpathy's shuffled 100B-token sample of fineweb-edu: 1823 parquet shards of ~90 MB, \
the nanochat training corpus",
license: Some("ODC-By 1.0"),
origin: Some("https://huggingface.co/datasets/karpathy/fineweb-edu-100b-shuffle"),
base_urls: &[
"https://huggingface.co/datasets/karpathy/fineweb-edu-100b-shuffle/resolve/4c8f30d6756da75362432a4d5569e1b229263b71",
],
template: "shard_{index}.parquet",
index_width: 5,
count: 1823,
format: "parquet",
digests: StaticShardDigests::Table(FINEWEB_EDU_100B_SHUFFLE_SHA256),
};
pub static NANOCHAT_SHARD_SETS: StaticShardSetMap = StaticShardSetMap {
name: "nanochat",
description: "The datasets NanoChat trains on.",
items: &[&FINEWEB_EDU_100B_SHUFFLE],
};
#[cfg(test)]
mod tests {
use super::*;
use crate::data::shards::ShardId;
#[test]
fn test_fineweb_edu_table() {
let d = NANOCHAT_SHARD_SETS.expect_lookup("fineweb-edu-100b-shuffle");
d.validate().unwrap();
assert_eq!(d.count, 1823);
assert_eq!(FINEWEB_EDU_100B_SHUFFLE_SHA256.len(), 1823);
assert!(d.is_pinned());
assert_eq!(d.file_name(ShardId(0)), "shard_00000.parquet");
assert_eq!(d.file_name(ShardId(1822)), "shard_01822.parquet");
assert_eq!(
d.urls(ShardId(312)),
vec![format!(
"https://huggingface.co/datasets/karpathy/fineweb-edu-100b-shuffle/resolve/{FINEWEB_EDU_100B_SHUFFLE_REVISION}/shard_00312.parquet"
)]
);
assert!(d.base_urls[0].ends_with(FINEWEB_EDU_100B_SHUFFLE_REVISION));
assert_eq!(d.parse_file_name("shard_00312.parquet"), Some(ShardId(312)));
assert_eq!(d.parse_file_name("shard_01823.parquet"), None);
assert_eq!(
NANOCHAT_SHARD_SETS.names(),
vec!["fineweb-edu-100b-shuffle"]
);
}
#[test]
fn test_fineweb_edu_digests_match_the_listing() {
let d = FINEWEB_EDU_100B_SHUFFLE.to_descriptor();
assert_eq!(
d.digest(ShardId(0)),
Some("6a3450ea745ee6b6830c09819c0f2c72682eba443022de595bcb50ceb3092c22")
);
assert_eq!(
d.digest(ShardId(1822)),
Some("a97e7f2f4680561c1fdf0461295b0cfc38d85f497ba03851ab267ac1b8165990")
);
assert_eq!(d.digest(ShardId(1823)), None);
}
}