use std::{
collections::{BTreeMap, HashSet},
fs,
path::Path,
thread,
time::Duration,
};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use toko_feed::{
PROVIDER_CACHE_MAX_BYTES, PROVIDER_CACHE_MAX_RECORDS, Provider, ProviderCacheKind,
ProviderCachePage, ProviderCachePageArgs, ProviderCardsIngestArgs, ProviderCardsIngestReceipt,
ProviderSetIngestArgs, ProviderSetIngestReceipt, ReplayProviderCacheArgs,
ReplayProviderCacheReceipt, ReplayProviderCacheRecord,
};
use super::{CliError, DEFAULT_MAX_PAGES, MAX_PROVIDER_SET_CARDS, Target, call_one};
const MAX_PROVIDER_DATA_FILE_BYTES: usize = 256 * 1024 * 1024;
const PROVIDER_DATA_FORMAT_VERSION: u32 = 1;
const PROVIDER_DATA_REPLAY_BATCH_RECORDS: usize = 5;
const PROVIDER_CARD_BATCH_LIMIT: u16 = 10;
const BUSY_RETRY_ATTEMPTS: usize = 60;
const BUSY_RETRY_DELAY: Duration = Duration::from_millis(500);
const PROVIDER_RETRY_ATTEMPTS: u32 = 5;
const PROVIDER_RETRY_MAX_DELAY: Duration = Duration::from_secs(8);
const TCGDEX_REQUEST_DELAY: Duration = Duration::from_millis(500);
const SCRYDEX_REQUEST_DELAY: Duration = Duration::from_secs(1);
const COLLECTIONS_FILE: &str = "collections.json";
const SETS_FILE: &str = "sets.json";
const CARDS_DIRECTORY: &str = "cards";
pub const DEFAULT_PROVIDER_SOURCE_PATH: &str = "source/providers/just-tcg/v1";
pub const DEFAULT_MAGIC_SOURCE_PATH: &str = "source/providers/just-tcg/v1/magic-the-gathering";
pub const DEFAULT_TCGDEX_SOURCE_PATH: &str = "source/providers/tcgdex/v2/en";
pub const DEFAULT_SCRYDEX_SOURCE_PATH: &str = "source/providers/scrydex/pokemon-v1/en";
pub const DEFAULT_SCRYDEX_MAGIC_SOURCE_PATH: &str = "source/providers/scrydex/magic-v1";
const JUST_TCG_PROVIDER: &str = "just-tcg";
const JUST_TCG_API_VERSION: &str = "v1";
const POKEMON_COLLECTION: &str = "pokemon";
const TCGDEX_PROVIDER: &str = "tcgdex";
const TCGDEX_API_VERSION: &str = "v2";
const SCRYDEX_PROVIDER: &str = "scrydex";
const SCRYDEX_API_VERSION: &str = "pokemon-v1";
const SCRYDEX_MAGIC_API_VERSION: &str = "magic-v1";
const AQUAPOLIS_PROVIDER_SET_ID: &str = "ecard2";
const THE_DARK_PROVIDER_SET_ID: &str = "the-dark-magic-the-gathering";
const SCRYDEX_THE_DARK_PROVIDER_SET_ID: &str = "DRK";
#[derive(Debug, Deserialize, Serialize)]
struct ProviderDataArtifact {
format_version: u32,
provider: String,
api_version: String,
#[serde(default = "default_artifact_collection")]
collection: String,
records: Vec<ProviderDataRecord>,
}
fn default_artifact_collection() -> String {
POKEMON_COLLECTION.to_owned()
}
#[derive(Clone, Debug, Deserialize, Serialize)]
struct ProviderDataRecord {
kind: ProviderCacheKind,
payload: String,
observed_at_ns: u64,
}
#[derive(Debug)]
struct ExportedProviderDataRecord {
provider_set_id: Option<String>,
record: ProviderDataRecord,
}
#[derive(Debug, Eq, PartialEq)]
struct MissingTcgDexCard {
offset: u64,
provider_card_id: String,
name: String,
}
#[derive(Debug, Eq, PartialEq)]
struct TcgDexResumePlan {
missing: Vec<MissingTcgDexCard>,
known_unavailable: Vec<MissingTcgDexCard>,
}
#[derive(Debug)]
struct ProviderDataSet {
collections: ProviderDataArtifact,
sets: ProviderDataArtifact,
cards: Vec<(String, ProviderDataArtifact)>,
}
#[derive(Debug)]
struct ReplayedProviderData {
data: ProviderDataSet,
receipts: Vec<ReplayProviderCacheReceipt>,
}
#[derive(Clone, Copy)]
struct ProviderIdentity<'a> {
provider: &'a str,
api_version: &'a str,
collection: &'a str,
}
impl ProviderDataSet {
fn record_count(&self) -> usize {
self.collections
.records
.len()
.saturating_add(self.sets.records.len())
.saturating_add(
self.cards
.iter()
.map(|(_, artifact)| artifact.records.len())
.sum::<usize>(),
)
}
const fn file_count(&self) -> usize {
2usize.saturating_add(self.cards.len())
}
}
pub fn execute_bootstrap(target: &Target) -> Result<Value, CliError> {
let retained = [
(DEFAULT_PROVIDER_SOURCE_PATH, None, true),
(
DEFAULT_TCGDEX_SOURCE_PATH,
Some(AQUAPOLIS_PROVIDER_SET_ID),
true,
),
(
DEFAULT_SCRYDEX_SOURCE_PATH,
Some(AQUAPOLIS_PROVIDER_SET_ID),
true,
),
(
DEFAULT_MAGIC_SOURCE_PATH,
Some(THE_DARK_PROVIDER_SET_ID),
false,
),
(
DEFAULT_SCRYDEX_MAGIC_SOURCE_PATH,
Some(SCRYDEX_THE_DARK_PROVIDER_SET_ID),
false,
),
];
let mut providers = Vec::with_capacity(retained.len());
let mut files = 0usize;
let mut records = 0usize;
for (source_path, required_set_id, required) in retained {
let path = Path::new(source_path);
let Some(replayed) = replay_provider_data_path(target, path, required_set_id)? else {
if required {
return Err(CliError::InvalidCache(format!(
"retained provider data `{source_path}` is missing"
)));
}
continue;
};
let provider_files = replayed.data.file_count();
let provider_records = replayed.data.record_count();
let batch_count = replayed.receipts.len();
let (records_replayed, inserted, updated, observations) = replayed.receipts.iter().fold(
(0_u64, 0_u64, 0_u64, 0_u64),
|(records, inserted, updated, observations), receipt| {
(
records.saturating_add(u64::from(receipt.records_replayed)),
inserted.saturating_add(u64::from(receipt.inserted)),
updated.saturating_add(u64::from(receipt.updated)),
observations.saturating_add(u64::from(receipt.observations)),
)
},
);
files = files.saturating_add(provider_files);
records = records.saturating_add(provider_records);
providers.push(json!({
"path": source_path,
"files": provider_files,
"records": provider_records,
"batches": batch_count,
"records_replayed": records_replayed,
"inserted": inserted,
"updated": updated,
"observations": observations,
}));
}
Ok(json!({
"source": "repository",
"providers": providers,
"files": files,
"records": records,
}))
}
pub fn execute_provider_set(
target: &Target,
provider: Provider,
collection: &str,
provider_set_id: &str,
source_path: &Path,
refresh: bool,
cards: Option<u16>,
) -> Result<Value, CliError> {
if cards.is_some_and(|cards| cards > MAX_PROVIDER_SET_CARDS) {
return Err(CliError::Usage(format!(
"provider `{}` accepts at most {} cards per acquisition",
provider.name(),
MAX_PROVIDER_SET_CARDS
)));
}
if provider == Provider::TcgDex && collection != POKEMON_COLLECTION {
return Err(CliError::Usage(format!(
"provider `{}` does not support collection `{collection}`",
provider.name()
)));
}
match provider {
Provider::JustTcg => execute_just_tcg_set(
target,
collection,
provider_set_id,
source_path,
refresh,
cards,
),
Provider::TcgDex => {
execute_tcgdex_set(target, provider_set_id, source_path, refresh, cards)
}
Provider::Scrydex => execute_scrydex_set(
target,
collection,
provider_set_id,
source_path,
refresh,
cards,
),
}
}
fn execute_just_tcg_set(
target: &Target,
collection: &str,
provider_set_id: &str,
source_path: &Path,
refresh: bool,
records: Option<u16>,
) -> Result<Value, CliError> {
if let Some(replay) = replay_provider_set_cache(target, provider_set_id, source_path, refresh)?
{
return Ok(replay);
}
let set = retry_ingestion(|| {
call_one::<_, ProviderSetIngestReceipt>(
target,
"toko_feed_ingest_provider_set",
ProviderSetIngestArgs {
provider: Provider::JustTcg.name().to_owned(),
collection: collection.to_owned(),
provider_set_id: provider_set_id.to_owned(),
},
false,
)
})?;
let records_available = set.records_available.ok_or_else(|| {
CliError::InvalidCache("JustTCG set omitted its set-feed record count".to_owned())
})?;
if records_available > u64::from(MAX_PROVIDER_SET_CARDS) {
return Err(CliError::Usage(format!(
"provider set reports more than {MAX_PROVIDER_SET_CARDS} records"
)));
}
let records_to_fetch = records
.map_or(records_available, u64::from)
.min(records_available);
let resolved_set_id = set.provider_set_id.clone();
let existing =
export_provider_data(target, JUST_TCG_PROVIDER, JUST_TCG_API_VERSION, collection)?;
let (records_present, _) = just_tcg_cached_progress(&existing, &resolved_set_id)?;
let records_already_present = records_present.min(records_to_fetch);
let mut offset = records_present;
let mut batches = Vec::<ProviderCardsIngestReceipt>::new();
for _ in 0..DEFAULT_MAX_PAGES {
if offset >= records_to_fetch {
break;
}
let remaining = records_to_fetch.saturating_sub(offset);
let limit = u16::try_from(remaining.min(u64::from(PROVIDER_CARD_BATCH_LIMIT)))
.map_err(|_| CliError::Usage("provider card limit overflowed".to_owned()))?;
let receipt = retry_ingestion(|| {
call_one::<_, ProviderCardsIngestReceipt>(
target,
"toko_feed_ingest_provider_cards",
ProviderCardsIngestArgs {
provider: Provider::JustTcg.name().to_owned(),
collection: collection.to_owned(),
provider_set_id: resolved_set_id.clone(),
offset,
limit,
},
false,
)
})?;
let provider_complete = receipt.provider_complete;
let next_offset = receipt.next_offset;
batches.push(receipt);
checkpoint_provider_data(
target,
source_path,
JUST_TCG_PROVIDER,
JUST_TCG_API_VERSION,
collection,
)?;
if provider_complete {
break;
}
let next_offset = next_offset.ok_or_else(|| {
CliError::InvalidCache("JustTCG card page omitted its continuation".to_owned())
})?;
if next_offset <= offset {
return Err(CliError::PaginationStalled(next_offset.to_string()));
}
offset = next_offset;
}
if offset < records_to_fetch
&& !batches
.last()
.is_some_and(|receipt| receipt.provider_complete)
{
return Err(CliError::PaginationLimit(DEFAULT_MAX_PAGES));
}
let data = export_provider_data(target, JUST_TCG_PROVIDER, JUST_TCG_API_VERSION, collection)?;
write_provider_data(source_path, &data)?;
Ok(json!({
"source": "provider",
"provider_set_id": resolved_set_id,
"path": source_path,
"files": data.file_count(),
"records": data.record_count(),
"set": set,
"batches": batches,
"records_requested": records_to_fetch,
"records_already_present": records_already_present,
}))
}
#[expect(
clippy::too_many_lines,
reason = "the bounded acquisition workflow keeps checkpoint and continuation handling linear"
)]
fn execute_tcgdex_set(
target: &Target,
provider_set_id: &str,
source_path: &Path,
refresh: bool,
cards: Option<u16>,
) -> Result<Value, CliError> {
if let Some(replay) = replay_provider_set_cache(target, provider_set_id, source_path, refresh)?
{
return Ok(replay);
}
let (set, existing, cards_available) = tcgdex_set_for_acquisition(target, provider_set_id)?;
let cards_to_fetch = cards.unwrap_or(cards_available).min(cards_available);
let resume = tcgdex_resume_plan(&existing, provider_set_id, cards_to_fetch)?;
let cards_already_present = usize::from(cards_to_fetch)
.saturating_sub(resume.missing.len())
.saturating_sub(resume.known_unavailable.len());
let mut card_receipts = Vec::<ProviderCardsIngestReceipt>::new();
let mut unavailable_cards = resume
.known_unavailable
.into_iter()
.map(|missing| {
json!({
"offset": missing.offset,
"provider_card_id": missing.provider_card_id,
"name": missing.name,
"error": "PreviouslyUnavailable",
})
})
.collect::<Vec<_>>();
for (missing_index, missing) in resume.missing.into_iter().enumerate() {
if missing_index != 0 {
thread::sleep(TCGDEX_REQUEST_DELAY);
}
let offset = missing.offset;
let receipt = match retry_ingestion(|| {
call_one::<_, ProviderCardsIngestReceipt>(
target,
"toko_feed_ingest_provider_cards",
ProviderCardsIngestArgs {
provider: Provider::TcgDex.name().to_owned(),
collection: POKEMON_COLLECTION.to_owned(),
provider_set_id: provider_set_id.to_owned(),
offset,
limit: 1,
},
false,
)
}) {
Ok(receipt) => receipt,
Err(CliError::Canister { error, .. })
if matches!(error.as_str(), "ProviderNotFound" | "ProviderData") =>
{
unavailable_cards.push(json!({
"offset": offset,
"provider_card_id": missing.provider_card_id,
"name": missing.name,
"error": error,
}));
continue;
}
Err(source) => {
checkpoint_pokemon_provider_data(
target,
source_path,
TCGDEX_PROVIDER,
TCGDEX_API_VERSION,
)?;
return Err(CliError::ProviderCard {
provider: TCGDEX_PROVIDER,
set_id: provider_set_id.to_owned(),
offset,
source: Box::new(source),
});
}
};
let expected_next = offset.checked_add(1).ok_or_else(|| {
CliError::InvalidCache("TCGdex card continuation overflowed".to_owned())
})?;
if !receipt.provider_complete && receipt.next_offset != Some(expected_next) {
return Err(CliError::PaginationStalled(
receipt
.next_offset
.map_or_else(|| "missing".to_owned(), |next| next.to_string()),
));
}
let provider_complete = receipt.provider_complete;
card_receipts.push(receipt);
if card_receipts.len().is_multiple_of(10) {
checkpoint_pokemon_provider_data(
target,
source_path,
TCGDEX_PROVIDER,
TCGDEX_API_VERSION,
)?;
}
if provider_complete {
break;
}
}
let data = export_pokemon_provider_data(target, TCGDEX_PROVIDER, TCGDEX_API_VERSION)?;
write_provider_data(source_path, &data)?;
Ok(json!({
"source": "provider",
"provider_set_id": provider_set_id,
"path": source_path,
"files": data.file_count(),
"records": data.record_count(),
"set": set,
"cards": card_receipts,
"cards_already_present": cards_already_present,
"cards_requested": cards_to_fetch,
"unavailable_cards": unavailable_cards,
}))
}
fn tcgdex_set_for_acquisition(
target: &Target,
provider_set_id: &str,
) -> Result<(Value, ProviderDataSet, u16), CliError> {
let set_result = retry_ingestion(|| {
call_one::<_, ProviderSetIngestReceipt>(
target,
"toko_feed_ingest_provider_set",
ProviderSetIngestArgs {
provider: Provider::TcgDex.name().to_owned(),
collection: POKEMON_COLLECTION.to_owned(),
provider_set_id: provider_set_id.to_owned(),
},
false,
)
});
match set_result {
Ok(set) => {
let existing =
export_pokemon_provider_data(target, TCGDEX_PROVIDER, TCGDEX_API_VERSION)?;
let cards_available = set.records_available.ok_or_else(|| {
CliError::InvalidCache("TCGdex set omitted its card count".to_owned())
})?;
let cards_available = u16::try_from(cards_available).map_err(|_| {
CliError::Usage(format!(
"provider set reports more than {MAX_PROVIDER_SET_CARDS} cards"
))
})?;
Ok((serde_json::to_value(set)?, existing, cards_available))
}
Err(source) if is_transient_provider_error(&source) => {
let Ok(existing) =
export_pokemon_provider_data(target, TCGDEX_PROVIDER, TCGDEX_API_VERSION)
else {
return Err(source);
};
let (name, cards_available) = tcgdex_set_summary(&existing, provider_set_id)?;
Ok((
json!({
"provider_set_id": provider_set_id,
"name": name,
"cards_available": cards_available,
"refreshed": false,
}),
existing,
cards_available,
))
}
Err(source) => Err(source),
}
}
fn checkpoint_provider_data(
target: &Target,
source_path: &Path,
provider: &str,
api_version: &str,
collection: &str,
) -> Result<(), CliError> {
let data = export_provider_data(target, provider, api_version, collection)?;
write_provider_data(source_path, &data)
}
fn checkpoint_pokemon_provider_data(
target: &Target,
source_path: &Path,
provider: &str,
api_version: &str,
) -> Result<(), CliError> {
checkpoint_provider_data(
target,
source_path,
provider,
api_version,
POKEMON_COLLECTION,
)
}
fn tcgdex_resume_plan(
data: &ProviderDataSet,
provider_set_id: &str,
cards: u16,
) -> Result<TcgDexResumePlan, CliError> {
let set_cards = data
.sets
.records
.iter()
.filter_map(|record| serde_json::from_str::<Value>(&record.payload).ok())
.find(|payload| payload.get("id").and_then(Value::as_str) == Some(provider_set_id))
.and_then(|payload| payload.get("cards").and_then(Value::as_array).cloned())
.ok_or_else(|| {
CliError::InvalidCache(format!(
"TCGdex set `{provider_set_id}` omitted its ordered card index"
))
})?;
if set_cards.len() < usize::from(cards) {
return Err(CliError::InvalidCache(format!(
"TCGdex set `{provider_set_id}` exposed fewer cards than requested"
)));
}
let existing_ids = data
.cards
.iter()
.filter(|(set_id, _)| set_id == provider_set_id)
.flat_map(|(_, artifact)| &artifact.records)
.filter(|record| record.kind == ProviderCacheKind::Cards)
.filter_map(|record| serde_json::from_str::<Value>(&record.payload).ok())
.filter_map(|payload| payload.get("id").and_then(Value::as_str).map(str::to_owned))
.collect::<HashSet<_>>();
let acquisition_frontier = set_cards
.iter()
.take(usize::from(cards))
.enumerate()
.filter(|(_, card)| {
card.get("id")
.and_then(Value::as_str)
.is_some_and(|id| existing_ids.contains(id))
})
.map(|(offset, _)| offset)
.max();
let mut missing = Vec::new();
let mut known_unavailable = Vec::new();
for (offset, card) in set_cards.iter().take(usize::from(cards)).enumerate() {
let provider_card_id = card
.get("id")
.and_then(Value::as_str)
.ok_or_else(|| CliError::InvalidCache("TCGdex card index omitted an ID".to_owned()))?;
if existing_ids.contains(provider_card_id) {
continue;
}
let name = card
.get("name")
.and_then(Value::as_str)
.ok_or_else(|| CliError::InvalidCache("TCGdex card index omitted a name".to_owned()))?;
let missing_card = MissingTcgDexCard {
offset: u64::try_from(offset)
.map_err(|_| CliError::InvalidCache("TCGdex card offset overflowed".to_owned()))?,
provider_card_id: provider_card_id.to_owned(),
name: name.to_owned(),
};
if acquisition_frontier.is_some_and(|frontier| offset < frontier) {
known_unavailable.push(missing_card);
} else {
missing.push(missing_card);
}
}
Ok(TcgDexResumePlan {
missing,
known_unavailable,
})
}
fn tcgdex_set_summary(
data: &ProviderDataSet,
provider_set_id: &str,
) -> Result<(String, u16), CliError> {
let payload = data
.sets
.records
.iter()
.filter_map(|record| serde_json::from_str::<Value>(&record.payload).ok())
.find(|payload| payload.get("id").and_then(Value::as_str) == Some(provider_set_id))
.ok_or_else(|| {
CliError::InvalidCache(format!(
"TCGdex set `{provider_set_id}` is absent from retained evidence"
))
})?;
let name = payload
.get("name")
.and_then(Value::as_str)
.ok_or_else(|| CliError::InvalidCache("TCGdex set omitted its name".to_owned()))?;
let cards = payload
.get("cards")
.and_then(Value::as_array)
.ok_or_else(|| CliError::InvalidCache("TCGdex set omitted its card index".to_owned()))?;
let cards = u16::try_from(cards.len()).map_err(|_| {
CliError::Usage(format!(
"provider set reports more than {MAX_PROVIDER_SET_CARDS} cards"
))
})?;
Ok((name.to_owned(), cards))
}
#[expect(
clippy::too_many_lines,
reason = "the bounded acquisition workflow keeps checkpoint and continuation handling linear"
)]
fn execute_scrydex_set(
target: &Target,
collection: &str,
provider_set_id: &str,
source_path: &Path,
refresh: bool,
cards: Option<u16>,
) -> Result<Value, CliError> {
if let Some(replay) = replay_provider_set_cache(target, provider_set_id, source_path, refresh)?
{
return Ok(replay);
}
let set = retry_ingestion(|| {
call_one::<_, ProviderSetIngestReceipt>(
target,
"toko_feed_ingest_provider_set",
ProviderSetIngestArgs {
provider: Provider::Scrydex.name().to_owned(),
collection: collection.to_owned(),
provider_set_id: provider_set_id.to_owned(),
},
false,
)
})?;
let cards_available = set
.records_available
.ok_or_else(|| CliError::InvalidCache("Scrydex set omitted its card count".to_owned()))?;
if cards_available > u64::from(MAX_PROVIDER_SET_CARDS) {
return Err(CliError::Usage(format!(
"provider set reports more than {MAX_PROVIDER_SET_CARDS} cards"
)));
}
let cards_to_fetch = cards
.map_or(cards_available, u64::from)
.min(cards_available);
let api_version = scrydex_api_version(collection)?;
let existing = export_provider_data(target, SCRYDEX_PROVIDER, api_version, collection)?;
let cards_already_present = scrydex_cached_prefix_len(&existing, provider_set_id)?;
let mut card_pages = Vec::<ProviderCardsIngestReceipt>::new();
let (_, page_size) = scrydex_resume_position(cards_already_present, cards_to_fetch)?;
let mut fetched = cards_already_present;
for page_index in 0..DEFAULT_MAX_PAGES {
if fetched >= cards_to_fetch {
break;
}
if page_index != 0 {
thread::sleep(SCRYDEX_REQUEST_DELAY);
}
let receipt = match retry_ingestion(|| {
call_one::<_, ProviderCardsIngestReceipt>(
target,
"toko_feed_ingest_provider_cards",
ProviderCardsIngestArgs {
provider: Provider::Scrydex.name().to_owned(),
collection: collection.to_owned(),
provider_set_id: provider_set_id.to_owned(),
offset: fetched,
limit: page_size,
},
false,
)
}) {
Ok(receipt) => receipt,
Err(source) => {
checkpoint_provider_data(
target,
source_path,
SCRYDEX_PROVIDER,
api_version,
collection,
)?;
return Err(source);
}
};
fetched = fetched.saturating_add(u64::from(receipt.cards_fetched));
let provider_complete = receipt.provider_complete;
let next_offset = receipt.next_offset;
card_pages.push(receipt);
checkpoint_provider_data(
target,
source_path,
SCRYDEX_PROVIDER,
api_version,
collection,
)?;
if provider_complete || fetched >= cards_to_fetch {
break;
}
let next_offset = next_offset.ok_or_else(|| {
CliError::InvalidCache("Scrydex card page omitted its continuation".to_owned())
})?;
if next_offset != fetched {
return Err(CliError::PaginationStalled(next_offset.to_string()));
}
}
if fetched < cards_to_fetch
&& !card_pages
.last()
.is_some_and(|receipt| receipt.provider_complete)
{
return Err(CliError::PaginationLimit(DEFAULT_MAX_PAGES));
}
let data = export_provider_data(target, SCRYDEX_PROVIDER, api_version, collection)?;
write_provider_data(source_path, &data)?;
Ok(json!({
"source": "provider",
"provider_set_id": provider_set_id,
"path": source_path,
"files": data.file_count(),
"records": data.record_count(),
"set": set,
"cards_requested": cards_to_fetch,
"cards_already_present": cards_already_present,
"card_pages": card_pages,
}))
}
fn scrydex_api_version(collection: &str) -> Result<&'static str, CliError> {
match collection {
POKEMON_COLLECTION => Ok(SCRYDEX_API_VERSION),
"magic-the-gathering" => Ok(SCRYDEX_MAGIC_API_VERSION),
_ => Err(CliError::Usage(format!(
"Scrydex does not support collection `{collection}`"
))),
}
}
fn replay_provider_set_cache(
target: &Target,
provider_set_id: &str,
source_path: &Path,
refresh: bool,
) -> Result<Option<Value>, CliError> {
let Some(replayed) = replay_provider_data_path(target, source_path, Some(provider_set_id))?
else {
return Ok(None);
};
if refresh {
return Ok(None);
}
Ok(Some(json!({
"source": "repository",
"provider_set_id": provider_set_id,
"path": source_path,
"files": replayed.data.file_count(),
"records": replayed.data.record_count(),
"batches": replayed.receipts,
})))
}
fn replay_provider_data_path(
target: &Target,
source_path: &Path,
required_set_id: Option<&str>,
) -> Result<Option<ReplayedProviderData>, CliError> {
if !source_path.exists() {
return Ok(None);
}
let data = read_provider_data(source_path)?;
if let Some(provider_set_id) = required_set_id {
ensure_set_present(&data, provider_set_id)?;
}
let receipts = replay_provider_data(target, &data)?;
Ok(Some(ReplayedProviderData { data, receipts }))
}
fn scrydex_cached_prefix_len(
data: &ProviderDataSet,
provider_set_id: &str,
) -> Result<u64, CliError> {
let Some((_, artifact)) = data
.cards
.iter()
.find(|(set_id, _)| set_id == provider_set_id)
else {
return Ok(0);
};
let mut positions = artifact
.records
.iter()
.map(|record| {
serde_json::from_str::<Value>(&record.payload)?
.get("expansion_sort_order")
.and_then(Value::as_u64)
.ok_or_else(|| {
CliError::InvalidCache(
"Scrydex card omitted its expansion sort position".to_owned(),
)
})
})
.collect::<Result<Vec<_>, CliError>>()?;
positions.sort_unstable();
for (index, position) in positions.iter().enumerate() {
let expected = u64::try_from(index)
.map_err(|_| CliError::InvalidCache("Scrydex cache was too large".to_owned()))?
.saturating_add(1);
if *position != expected {
return Err(CliError::InvalidCache(
"Scrydex card cache is not a contiguous provider prefix".to_owned(),
));
}
}
u64::try_from(positions.len())
.map_err(|_| CliError::InvalidCache("Scrydex cache was too large".to_owned()))
}
fn scrydex_resume_position(existing: u64, requested: u64) -> Result<(u32, u16), CliError> {
let maximum_page_size = u64::from(PROVIDER_CARD_BATCH_LIMIT);
let page_size = if existing == 0 {
requested.min(maximum_page_size).max(1)
} else {
(1..=existing.min(maximum_page_size))
.rev()
.find(|candidate| existing.is_multiple_of(*candidate))
.unwrap_or(1)
};
let page = existing
.checked_div(page_size)
.and_then(|completed_pages| completed_pages.checked_add(1))
.and_then(|page| u32::try_from(page).ok())
.ok_or_else(|| CliError::InvalidCache("Scrydex page position overflowed".to_owned()))?;
let page_size = u16::try_from(page_size)
.map_err(|_| CliError::Usage("Scrydex page size exceeded its bound".to_owned()))?;
Ok((page, page_size))
}
fn ensure_set_present(data: &ProviderDataSet, provider_set_id: &str) -> Result<(), CliError> {
let present = data.sets.records.iter().any(|record| {
serde_json::from_str::<Value>(&record.payload)
.ok()
.is_some_and(|payload| {
payload.get("id").and_then(Value::as_str) == Some(provider_set_id)
|| payload
.get("name")
.and_then(Value::as_str)
.is_some_and(|name| name.trim().eq_ignore_ascii_case(provider_set_id))
})
});
if present {
Ok(())
} else {
Err(CliError::InvalidCache(format!(
"provider set `{provider_set_id}` is not present in the tracked data; use --refresh"
)))
}
}
fn just_tcg_cached_progress(
data: &ProviderDataSet,
provider_set_id: &str,
) -> Result<(u64, u64), CliError> {
let present = data
.cards
.iter()
.find(|(set_id, _)| set_id == provider_set_id)
.map_or(0, |(_, artifact)| artifact.records.len());
let set = data
.sets
.records
.iter()
.filter_map(|record| serde_json::from_str::<Value>(&record.payload).ok())
.find(|payload| payload.get("id").and_then(Value::as_str) == Some(provider_set_id))
.ok_or_else(|| {
CliError::InvalidCache(format!(
"JustTCG set `{provider_set_id}` is absent from retained evidence"
))
})?;
let cards = set
.get("cards_count")
.and_then(Value::as_u64)
.ok_or_else(|| CliError::InvalidCache("JustTCG set omitted cards_count".to_owned()))?;
let sealed = set.get("sealed_count").and_then(Value::as_u64).unwrap_or(0);
let expected = cards
.checked_add(sealed)
.ok_or_else(|| CliError::InvalidCache("JustTCG set count overflowed".to_owned()))?;
let present = u64::try_from(present)
.map_err(|_| CliError::InvalidCache("JustTCG cache was too large".to_owned()))?;
if present > expected {
return Err(CliError::InvalidCache(format!(
"JustTCG set `{provider_set_id}` cache exceeds its retained set count"
)));
}
Ok((present, expected))
}
fn export_provider_data(
target: &Target,
provider: &str,
api_version: &str,
collection: &str,
) -> Result<ProviderDataSet, CliError> {
let mut collections = Vec::new();
let mut sets = Vec::new();
let mut cards = BTreeMap::<String, Vec<ProviderDataRecord>>::new();
for kind in provider_cache_kinds() {
for exported in export_provider_kind(target, provider, api_version, collection, kind)? {
match kind {
ProviderCacheKind::Collections => {
require_catalog_partition(&exported)?;
collections.push(exported.record);
}
ProviderCacheKind::Sets => {
require_catalog_partition(&exported)?;
sets.push(exported.record);
}
ProviderCacheKind::Cards | ProviderCacheKind::Sealed => {
let provider_set_id = exported.provider_set_id.ok_or_else(|| {
CliError::InvalidCache(
"canister omitted a card record's provider set partition".to_owned(),
)
})?;
validate_provider_set_id(&provider_set_id)?;
cards
.entry(provider_set_id)
.or_default()
.push(exported.record);
}
}
}
}
let data = ProviderDataSet {
collections: artifact(provider, api_version, collection, collections),
sets: artifact(provider, api_version, collection, sets),
cards: cards
.into_iter()
.map(|(provider_set_id, records)| {
(
provider_set_id,
artifact(provider, api_version, collection, records),
)
})
.collect(),
};
validate_provider_data_layers(&data, false)?;
Ok(data)
}
fn export_pokemon_provider_data(
target: &Target,
provider: &str,
api_version: &str,
) -> Result<ProviderDataSet, CliError> {
export_provider_data(target, provider, api_version, POKEMON_COLLECTION)
}
fn export_provider_kind(
target: &Target,
provider: &str,
api_version: &str,
collection: &str,
kind: ProviderCacheKind,
) -> Result<Vec<ExportedProviderDataRecord>, CliError> {
let mut records = Vec::new();
let mut after = None;
let mut seen = HashSet::new();
for _ in 0..DEFAULT_MAX_PAGES {
let page = call_one::<_, ProviderCachePage>(
target,
"toko_feed_provider_cache_page",
ProviderCachePageArgs {
provider: provider.to_owned(),
collection: collection.to_owned(),
kind,
after: after.clone(),
limit: PROVIDER_CACHE_MAX_RECORDS,
},
true,
)?;
if page.provider != provider
|| page.collection != collection
|| page.api_version != api_version
|| page.kind != kind
{
return Err(CliError::InvalidCache(
"canister returned a different provider source class".to_owned(),
));
}
for record in page.records {
let payload = String::from_utf8(record.payload).map_err(|_| {
CliError::InvalidCache("provider payload was not UTF-8 JSON".to_owned())
})?;
records.push(ExportedProviderDataRecord {
provider_set_id: record.provider_set_id,
record: ProviderDataRecord {
kind,
payload,
observed_at_ns: record.observed_at_ns,
},
});
}
let Some(next) = page.next_after else {
return Ok(records);
};
if !seen.insert(next.clone()) {
return Err(CliError::PaginationStalled(next));
}
after = Some(next);
}
Err(CliError::PaginationLimit(DEFAULT_MAX_PAGES))
}
fn require_catalog_partition(record: &ExportedProviderDataRecord) -> Result<(), CliError> {
if record.provider_set_id.is_some() {
return Err(CliError::InvalidCache(
"canister attached a card partition to catalog data".to_owned(),
));
}
Ok(())
}
fn replay_provider_data(
target: &Target,
data: &ProviderDataSet,
) -> Result<Vec<ReplayProviderCacheReceipt>, CliError> {
validate_provider_data(data)?;
let mut receipts = Vec::new();
replay_records(
target,
ProviderIdentity {
provider: &data.collections.provider,
api_version: &data.collections.api_version,
collection: &data.collections.collection,
},
ProviderCacheKind::Collections,
None,
&data.collections.records,
&mut receipts,
)?;
replay_records(
target,
ProviderIdentity {
provider: &data.sets.provider,
api_version: &data.sets.api_version,
collection: &data.sets.collection,
},
ProviderCacheKind::Sets,
None,
&data.sets.records,
&mut receipts,
)?;
for (provider_set_id, artifact) in &data.cards {
replay_records(
target,
ProviderIdentity {
provider: &artifact.provider,
api_version: &artifact.api_version,
collection: &artifact.collection,
},
ProviderCacheKind::Cards,
Some(provider_set_id.as_str()),
&artifact.records,
&mut receipts,
)?;
replay_records(
target,
ProviderIdentity {
provider: &artifact.provider,
api_version: &artifact.api_version,
collection: &artifact.collection,
},
ProviderCacheKind::Sealed,
Some(provider_set_id.as_str()),
&artifact.records,
&mut receipts,
)?;
}
Ok(receipts)
}
fn replay_records(
target: &Target,
identity: ProviderIdentity<'_>,
kind: ProviderCacheKind,
provider_set_id: Option<&str>,
source: &[ProviderDataRecord],
receipts: &mut Vec<ReplayProviderCacheReceipt>,
) -> Result<(), CliError> {
let mut records = Vec::new();
let mut payload_bytes = 0usize;
let mut batch_start = 1usize;
let mut record_position = 0usize;
for record in source.iter().filter(|record| record.kind == kind) {
record_position = record_position.saturating_add(1);
let payload = record.payload.as_bytes();
if !records.is_empty()
&& (records.len() == PROVIDER_DATA_REPLAY_BATCH_RECORDS
|| payload_bytes.saturating_add(payload.len()) > PROVIDER_CACHE_MAX_BYTES)
{
let receipt = replay_provider_data_batch(
target,
identity,
kind,
provider_set_id,
std::mem::take(&mut records),
)
.map_err(|source| CliError::ProviderReplay {
provider: identity.provider.to_owned(),
kind: provider_cache_kind_name(kind),
set: provider_set_context(provider_set_id),
first_record: batch_start,
last_record: record_position.saturating_sub(1),
source: Box::new(source),
})?;
receipts.push(receipt);
batch_start = record_position;
payload_bytes = 0;
}
payload_bytes = payload_bytes.saturating_add(payload.len());
records.push(ReplayProviderCacheRecord {
payload: payload.to_vec(),
observed_at_ns: record.observed_at_ns,
});
}
if !records.is_empty() {
let receipt = replay_provider_data_batch(target, identity, kind, provider_set_id, records)
.map_err(|source| CliError::ProviderReplay {
provider: identity.provider.to_owned(),
kind: provider_cache_kind_name(kind),
set: provider_set_context(provider_set_id),
first_record: batch_start,
last_record: record_position,
source: Box::new(source),
})?;
receipts.push(receipt);
}
Ok(())
}
const fn provider_cache_kind_name(kind: ProviderCacheKind) -> &'static str {
match kind {
ProviderCacheKind::Collections => "collection",
ProviderCacheKind::Sets => "set",
ProviderCacheKind::Cards => "card",
ProviderCacheKind::Sealed => "sealed",
}
}
fn provider_set_context(provider_set_id: Option<&str>) -> String {
provider_set_id.map_or_else(String::new, |set_id| format!(" in set `{set_id}`"))
}
fn replay_provider_data_batch(
target: &Target,
identity: ProviderIdentity<'_>,
kind: ProviderCacheKind,
provider_set_id: Option<&str>,
records: Vec<ReplayProviderCacheRecord>,
) -> Result<ReplayProviderCacheReceipt, CliError> {
let args = ReplayProviderCacheArgs {
provider: identity.provider.to_owned(),
collection: identity.collection.to_owned(),
api_version: identity.api_version.to_owned(),
kind,
provider_set_id: provider_set_id.map(str::to_owned),
records,
};
retry_ingestion(|| {
call_one(
target,
"toko_feed_replay_provider_cache",
args.clone(),
false,
)
})
}
fn retry_ingestion<T>(mut operation: impl FnMut() -> Result<T, CliError>) -> Result<T, CliError> {
let mut busy_attempts = 0usize;
let mut provider_attempts = 0u32;
loop {
match operation() {
Err(CliError::Canister { error, .. })
if error == "Busy" && busy_attempts.saturating_add(1) < BUSY_RETRY_ATTEMPTS =>
{
busy_attempts = busy_attempts.saturating_add(1);
thread::sleep(BUSY_RETRY_DELAY);
}
Err(CliError::Canister { error, .. })
if is_retryable_provider_error(&error)
&& provider_attempts.saturating_add(1) < PROVIDER_RETRY_ATTEMPTS =>
{
provider_attempts = provider_attempts.saturating_add(1);
let delay = Duration::from_secs(1u64 << provider_attempts.saturating_sub(1));
thread::sleep(delay.min(PROVIDER_RETRY_MAX_DELAY));
}
result => return result,
}
}
}
fn is_retryable_provider_error(error: &str) -> bool {
matches!(error, "ProviderUnavailable" | "ProviderRateLimited")
}
fn is_transient_provider_error(error: &CliError) -> bool {
matches!(
error,
CliError::Canister { error, .. } if is_retryable_provider_error(error)
)
}
fn read_provider_data(path: &Path) -> Result<ProviderDataSet, CliError> {
if !path.is_dir() {
return Err(CliError::InvalidCache(format!(
"provider data path '{}' was not a directory",
path.display()
)));
}
let collections = read_artifact(&path.join(COLLECTIONS_FILE))?;
let sets = read_artifact(&path.join(SETS_FILE))?;
let cards_path = path.join(CARDS_DIRECTORY);
let entries = fs::read_dir(&cards_path).map_err(|source| CliError::CacheIo {
action: "list",
path: cards_path.clone(),
source,
})?;
let mut files = entries
.map(|entry| {
entry
.map(|entry| entry.path())
.map_err(|source| CliError::CacheIo {
action: "list",
path: cards_path.clone(),
source,
})
})
.collect::<Result<Vec<_>, _>>()?;
files.retain(|file| file.extension().and_then(|value| value.to_str()) == Some("json"));
files.sort();
let cards = files
.into_iter()
.map(|file| {
let provider_set_id = file
.file_stem()
.and_then(|value| value.to_str())
.ok_or_else(|| {
CliError::InvalidCache(format!(
"provider card data file '{}' had no UTF-8 stem",
file.display()
))
})?
.to_owned();
let artifact = read_artifact(&file)?;
Ok((provider_set_id, artifact))
})
.collect::<Result<Vec<_>, CliError>>()?;
let data = ProviderDataSet {
collections,
sets,
cards,
};
validate_provider_data(&data)?;
Ok(data)
}
fn read_artifact(path: &Path) -> Result<ProviderDataArtifact, CliError> {
let bytes = fs::read(path).map_err(|source| CliError::CacheIo {
action: "read",
path: path.to_owned(),
source,
})?;
if bytes.len() > MAX_PROVIDER_DATA_FILE_BYTES {
return Err(CliError::InvalidCache(format!(
"provider data file '{}' exceeded {MAX_PROVIDER_DATA_FILE_BYTES} bytes",
path.display()
)));
}
serde_json::from_slice(&bytes).map_err(Into::into)
}
fn write_provider_data(path: &Path, data: &ProviderDataSet) -> Result<(), CliError> {
validate_provider_data(data)?;
fs::create_dir_all(path).map_err(|source| CliError::CacheIo {
action: "create",
path: path.to_owned(),
source,
})?;
write_artifact(&path.join(COLLECTIONS_FILE), &data.collections)?;
write_artifact(&path.join(SETS_FILE), &data.sets)?;
let cards_path = path.join(CARDS_DIRECTORY);
fs::create_dir_all(&cards_path).map_err(|source| CliError::CacheIo {
action: "create",
path: cards_path.clone(),
source,
})?;
for (provider_set_id, artifact) in &data.cards {
write_artifact(
&cards_path.join(format!("{provider_set_id}.json")),
artifact,
)?;
}
Ok(())
}
fn write_artifact(path: &Path, artifact: &ProviderDataArtifact) -> Result<(), CliError> {
let bytes = serde_json::to_vec_pretty(artifact)?;
if bytes.len() > MAX_PROVIDER_DATA_FILE_BYTES {
return Err(CliError::InvalidCache(format!(
"provider data file '{}' exceeded {MAX_PROVIDER_DATA_FILE_BYTES} bytes",
path.display()
)));
}
let file_name = path
.file_name()
.and_then(|name| name.to_str())
.ok_or_else(|| CliError::InvalidCache("data path had no UTF-8 file name".to_owned()))?;
let temporary = path.with_file_name(format!(".{file_name}.{}.tmp", std::process::id()));
fs::write(&temporary, bytes).map_err(|source| CliError::CacheIo {
action: "write",
path: temporary.clone(),
source,
})?;
if let Err(source) = fs::rename(&temporary, path) {
let _ = fs::remove_file(&temporary);
return Err(CliError::CacheIo {
action: "replace",
path: path.to_owned(),
source,
});
}
Ok(())
}
fn validate_provider_data(data: &ProviderDataSet) -> Result<(), CliError> {
validate_provider_data_layers(data, true)
}
fn validate_provider_data_layers(
data: &ProviderDataSet,
require_cards: bool,
) -> Result<(), CliError> {
validate_artifact(&data.collections)?;
validate_artifact(&data.sets)?;
let expected_provider = data.collections.provider.as_str();
let expected_api_version = data.collections.api_version.as_str();
let expected_collection = data.collections.collection.as_str();
if data.sets.provider != expected_provider
|| data.sets.api_version != expected_api_version
|| data.sets.collection != expected_collection
{
return Err(CliError::InvalidCache(
"provider data mixed provider identities, API versions, or collections".to_owned(),
));
}
if data.collections.records.is_empty()
|| data
.collections
.records
.iter()
.any(|record| record.kind != ProviderCacheKind::Collections)
{
return Err(CliError::InvalidCache(
"collections.json must contain collection records".to_owned(),
));
}
if data.sets.records.is_empty()
|| data
.sets
.records
.iter()
.any(|record| record.kind != ProviderCacheKind::Sets)
{
return Err(CliError::InvalidCache(
"sets.json must contain set records".to_owned(),
));
}
if require_cards && data.cards.is_empty() {
return Err(CliError::InvalidCache(
"provider data contained no card-set files".to_owned(),
));
}
let mut seen = HashSet::new();
for (provider_set_id, artifact) in &data.cards {
validate_provider_set_id(provider_set_id)?;
validate_artifact(artifact)?;
if artifact.provider != expected_provider
|| artifact.api_version != expected_api_version
|| artifact.collection != expected_collection
{
return Err(CliError::InvalidCache(
"provider data mixed provider identities, API versions, or collections".to_owned(),
));
}
if artifact.records.is_empty() || !seen.insert(provider_set_id) {
return Err(CliError::InvalidCache(
"card-set data files must be nonempty and unique".to_owned(),
));
}
for record in &artifact.records {
if !matches!(
record.kind,
ProviderCacheKind::Cards | ProviderCacheKind::Sealed
) {
return Err(CliError::InvalidCache(
"card-set data contained a non-card record".to_owned(),
));
}
}
}
Ok(())
}
fn validate_artifact(artifact: &ProviderDataArtifact) -> Result<(), CliError> {
if artifact.format_version != PROVIDER_DATA_FORMAT_VERSION {
return Err(CliError::InvalidCache(format!(
"unsupported format version {}",
artifact.format_version
)));
}
if !matches!(
(artifact.provider.as_str(), artifact.api_version.as_str()),
(JUST_TCG_PROVIDER, JUST_TCG_API_VERSION)
| (TCGDEX_PROVIDER, TCGDEX_API_VERSION)
| (
SCRYDEX_PROVIDER,
SCRYDEX_API_VERSION | SCRYDEX_MAGIC_API_VERSION
)
) {
return Err(CliError::InvalidCache(
"artifact provider or API version is unsupported".to_owned(),
));
}
if !matches!(
artifact.collection.as_str(),
"pokemon" | "magic-the-gathering"
) || (artifact.collection != POKEMON_COLLECTION
&& !matches!(
artifact.provider.as_str(),
JUST_TCG_PROVIDER | SCRYDEX_PROVIDER
))
{
return Err(CliError::InvalidCache(
"artifact collection is unsupported by its provider".to_owned(),
));
}
for record in &artifact.records {
if record.payload.is_empty() || record.payload.len() > PROVIDER_CACHE_MAX_BYTES {
return Err(CliError::InvalidCache(
"one provider payload was empty or exceeded the replay bound".to_owned(),
));
}
serde_json::from_str::<Value>(&record.payload).map_err(|_| {
CliError::InvalidCache("one provider payload was not valid JSON".to_owned())
})?;
}
Ok(())
}
fn validate_provider_set_id(provider_set_id: &str) -> Result<(), CliError> {
let safe = !provider_set_id.is_empty()
&& provider_set_id
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_'));
if !safe {
return Err(CliError::InvalidCache(
"card data file had an unsafe provider set identifier".to_owned(),
));
}
Ok(())
}
fn artifact(
provider: &str,
api_version: &str,
collection: &str,
records: Vec<ProviderDataRecord>,
) -> ProviderDataArtifact {
ProviderDataArtifact {
format_version: PROVIDER_DATA_FORMAT_VERSION,
provider: provider.to_owned(),
api_version: api_version.to_owned(),
collection: collection.to_owned(),
records,
}
}
const fn provider_cache_kinds() -> [ProviderCacheKind; 4] {
[
ProviderCacheKind::Collections,
ProviderCacheKind::Sets,
ProviderCacheKind::Cards,
ProviderCacheKind::Sealed,
]
}
#[cfg(test)]
mod tests {
use super::*;
fn record(kind: ProviderCacheKind, payload: &str) -> ProviderDataRecord {
ProviderDataRecord {
kind,
payload: payload.to_owned(),
observed_at_ns: 1,
}
}
fn valid_data() -> ProviderDataSet {
ProviderDataSet {
collections: artifact(
JUST_TCG_PROVIDER,
JUST_TCG_API_VERSION,
POKEMON_COLLECTION,
vec![record(
ProviderCacheKind::Collections,
r#"{"id":"pokemon","name":"Pokemon"}"#,
)],
),
sets: artifact(
JUST_TCG_PROVIDER,
JUST_TCG_API_VERSION,
POKEMON_COLLECTION,
vec![record(
ProviderCacheKind::Sets,
r#"{"id":"aquapolis-pokemon","name":"Aquapolis","game":"Pokemon"}"#,
)],
),
cards: vec![(
"aquapolis-pokemon".to_owned(),
artifact(
JUST_TCG_PROVIDER,
JUST_TCG_API_VERSION,
POKEMON_COLLECTION,
vec![record(
ProviderCacheKind::Cards,
r#"{"id":"pokemon-aquapolis-test","name":"Test","set":"aquapolis-pokemon","number":"1","variants":[]}"#,
)],
),
)],
}
}
#[test]
fn provider_data_requires_separate_complete_layers() {
let mut data = valid_data();
assert!(validate_provider_data(&data).is_ok());
data.sets.records.clear();
assert!(validate_provider_data(&data).is_err());
}
#[test]
fn provider_data_selects_sets_by_exact_provider_id() {
let data = valid_data();
assert!(ensure_set_present(&data, "aquapolis-pokemon").is_ok());
assert!(ensure_set_present(&data, "skyridge-pokemon").is_err());
}
#[test]
fn tcgdex_resume_plan_fetches_only_uncached_tail_cards() {
let data = ProviderDataSet {
collections: artifact(
TCGDEX_PROVIDER,
TCGDEX_API_VERSION,
POKEMON_COLLECTION,
vec![record(
ProviderCacheKind::Collections,
r#"{"id":"pokemon","name":"Pokémon"}"#,
)],
),
sets: artifact(
TCGDEX_PROVIDER,
TCGDEX_API_VERSION,
POKEMON_COLLECTION,
vec![record(
ProviderCacheKind::Sets,
r#"{"id":"ecard2","cards":[{"id":"ecard2-1","name":"One"},{"id":"ecard2-2","name":"Two"},{"id":"ecard2-3","name":"Three"}]}"#,
)],
),
cards: vec![(
"ecard2".to_owned(),
artifact(
TCGDEX_PROVIDER,
TCGDEX_API_VERSION,
POKEMON_COLLECTION,
vec![record(ProviderCacheKind::Cards, r#"{"id":"ecard2-1"}"#)],
),
)],
};
let plan = tcgdex_resume_plan(&data, "ecard2", 3).expect("resume plan should build");
assert_eq!(
plan.missing
.into_iter()
.map(|card| card.offset)
.collect::<Vec<_>>(),
vec![1, 2]
);
assert!(plan.known_unavailable.is_empty());
}
#[test]
fn tcgdex_resume_plan_does_not_retry_gaps_behind_the_frontier() {
let data = ProviderDataSet {
collections: artifact(
TCGDEX_PROVIDER,
TCGDEX_API_VERSION,
POKEMON_COLLECTION,
vec![record(
ProviderCacheKind::Collections,
r#"{"id":"pokemon","name":"Pokémon"}"#,
)],
),
sets: artifact(
TCGDEX_PROVIDER,
TCGDEX_API_VERSION,
POKEMON_COLLECTION,
vec![record(
ProviderCacheKind::Sets,
r#"{"id":"ecard2","cards":[{"id":"ecard2-1","name":"One"},{"id":"ecard2-2","name":"Two"},{"id":"ecard2-3","name":"Three"}]}"#,
)],
),
cards: vec![(
"ecard2".to_owned(),
artifact(
TCGDEX_PROVIDER,
TCGDEX_API_VERSION,
POKEMON_COLLECTION,
vec![
record(ProviderCacheKind::Cards, r#"{"id":"ecard2-1"}"#),
record(ProviderCacheKind::Cards, r#"{"id":"ecard2-3"}"#),
],
),
)],
};
let plan = tcgdex_resume_plan(&data, "ecard2", 3).expect("resume plan should build");
assert!(plan.missing.is_empty());
assert_eq!(
plan.known_unavailable
.into_iter()
.map(|card| card.offset)
.collect::<Vec<_>>(),
vec![1]
);
}
#[test]
fn just_tcg_resume_progress_counts_cards_and_sealed_products() {
let mut data = valid_data();
data.sets.records[0] = record(
ProviderCacheKind::Sets,
r#"{"id":"aquapolis-pokemon","name":"Aquapolis","game":"Pokemon","cards_count":1,"sealed_count":1}"#,
);
data.cards[0].1.records.push(record(
ProviderCacheKind::Sealed,
r#"{"id":"pokemon-aquapolis-box","name":"Box","set":"aquapolis-pokemon","number":"BOX","variants":[]}"#,
));
assert_eq!(
just_tcg_cached_progress(&data, "aquapolis-pokemon")
.expect("complete cache should be recognized"),
(2, 2)
);
}
#[test]
fn scrydex_resume_requires_a_contiguous_prefix_without_refetching() {
let mut data = ProviderDataSet {
collections: artifact(
SCRYDEX_PROVIDER,
SCRYDEX_API_VERSION,
POKEMON_COLLECTION,
vec![record(
ProviderCacheKind::Collections,
r#"{"id":"pokemon","name":"Pokémon"}"#,
)],
),
sets: artifact(
SCRYDEX_PROVIDER,
SCRYDEX_API_VERSION,
POKEMON_COLLECTION,
vec![record(
ProviderCacheKind::Sets,
r#"{"id":"ecard2","name":"Aquapolis"}"#,
)],
),
cards: vec![(
"ecard2".to_owned(),
artifact(
SCRYDEX_PROVIDER,
SCRYDEX_API_VERSION,
POKEMON_COLLECTION,
vec![
record(
ProviderCacheKind::Cards,
r#"{"id":"ecard2-1","expansion_sort_order":1}"#,
),
record(
ProviderCacheKind::Cards,
r#"{"id":"ecard2-2","expansion_sort_order":2}"#,
),
record(
ProviderCacheKind::Cards,
r#"{"id":"ecard2-3","expansion_sort_order":3}"#,
),
],
),
)],
};
assert_eq!(
scrydex_cached_prefix_len(&data, "ecard2").expect("contiguous prefix"),
3
);
assert_eq!(
scrydex_resume_position(10, 182).expect("ten-card page resumes"),
(2, 10)
);
assert_eq!(
scrydex_resume_position(7, 182).expect("partial page resumes without overlap"),
(2, 7)
);
data.cards[0].1.records.remove(1);
assert!(scrydex_cached_prefix_len(&data, "ecard2").is_err());
}
#[test]
fn card_data_requires_a_safe_set_filename() {
let mut data = valid_data();
data.cards[0].0 = "../aquapolis".to_owned();
assert!(validate_provider_data(&data).is_err());
}
#[test]
fn provider_data_cannot_mix_adapters_or_api_versions() {
let mut data = valid_data();
data.cards[0].1.provider = TCGDEX_PROVIDER.to_owned();
data.cards[0].1.api_version = TCGDEX_API_VERSION.to_owned();
assert!(validate_provider_data(&data).is_err());
}
#[test]
fn cli_contains_no_provider_http_transport() {
let sources = [
include_str!("lib.rs"),
include_str!("cache.rs"),
include_str!("main.rs"),
include_str!("../Cargo.toml"),
];
for forbidden in [
concat!("cu", "rl"),
concat!("req", "west"),
concat!("u", "req"),
concat!("hy", "per"),
concat!("api.just", "tcg.com"),
] {
assert!(
sources.iter().all(|source| !source.contains(forbidden)),
"the CLI must not contain provider HTTP transport: {forbidden}"
);
}
}
}