trane 0.28.0

//@<lp-example-2
//! Defines the system used to retrieve scores and rewards for units and come up with a final score.
//!
//! During performance testing, it was found that caching scores scores significantly improved the
//! performance of exercise scheduling.
//>@lp-example-2

use anyhow::{Result, anyhow};
use chrono::Utc;
use std::cell::RefCell;
use ustr::{Ustr, UstrMap, UstrSet};

use crate::{
    data::{ExerciseType, SchedulerOptions, UnitType},
    exercise_scorer::{ExerciseScorer, PowerLawScorer},
    reward_scorer::{RewardScorer, WeightedRewardScorer},
    scheduler::SchedulerData,
};

/// Stores information about a cached score.
#[derive(Clone)]
pub(super) struct CachedScore {
    /// The computed score.
    score: f32,

    /// The velocity of learning, a measure of how quickly the score is improving or worsening over
    /// trials.
    velocity: Option<f32>,

    /// The number of trials used to compute the score.
    num_trials: usize,

    /// The number of days since the last trial.
    last_seen: f32,
}

/// Contains the logic to score units based on their previous scores and rewards, as well as the
/// logic to cache those scores for efficiency.
pub(super) struct UnitScorer {
    /// A mapping of exercise ID to cached score.
    exercise_cache: RefCell<UstrMap<CachedScore>>,

    /// A mapping of lesson ID to cached score.
    lesson_cache: RefCell<UstrMap<Option<f32>>>,

    /// A mapping of course ID to cached score.
    course_cache: RefCell<UstrMap<Option<f32>>>,

    /// A mapping of lesson ID to cached average number of trials.
    lesson_trials_cache: RefCell<UstrMap<Option<f32>>>,

    /// A mapping of course ID to cached average number of trials.
    course_trials_cache: RefCell<UstrMap<Option<f32>>>,

    /// The data used by the scheduler.
    data: SchedulerData,

    /// The options used to schedule exercises.
    options: SchedulerOptions,

    /// The object used to compute the score of an exercise based on its previous trials.
    exercise_scorer: Box<dyn ExerciseScorer + Send + Sync>,

    /// The object used to compute the reward of a unit based on its previous rewards.
    reward_scorer: Box<dyn RewardScorer + Send + Sync>,
}

impl UnitScorer {
    /// Constructs a new score cache.
    pub(super) fn new(data: SchedulerData, options: SchedulerOptions) -> Self {
        Self {
            exercise_cache: RefCell::new(UstrMap::default()),
            lesson_cache: RefCell::new(UstrMap::default()),
            course_cache: RefCell::new(UstrMap::default()),
            lesson_trials_cache: RefCell::new(UstrMap::default()),
            course_trials_cache: RefCell::new(UstrMap::default()),
            data,
            options,
            exercise_scorer: Box::new(PowerLawScorer {}),
            reward_scorer: Box::new(WeightedRewardScorer {}),
        }
    }

    /// Removes the cached score for the given unit and all units affected by an update to its
    /// score.
    pub(super) fn invalidate_cached_score(&self, unit_id: Ustr) {
        // Remove the unit itself from all caches.
        self.exercise_cache.borrow_mut().remove(&unit_id);
        self.lesson_cache.borrow_mut().remove(&unit_id);
        self.course_cache.borrow_mut().remove(&unit_id);
        self.lesson_trials_cache.borrow_mut().remove(&unit_id);
        self.course_trials_cache.borrow_mut().remove(&unit_id);

        // Invalidate the caches depending on the type of the unit.
        let graph = self.data.unit_graph.read();
        match graph.get_unit_type(unit_id) {
            // If the unit is an exercise, invalidate the cached score of its lesson and course. If
            // the unit is a lesson, invalidate the cached score of its course.
            Some(UnitType::Exercise) => {
                if let Some(lesson_id) = graph.get_exercise_lesson(unit_id) {
                    self.lesson_cache.borrow_mut().remove(&lesson_id);
                    self.lesson_trials_cache.borrow_mut().remove(&lesson_id);
                    if let Some(course_id) = graph.get_lesson_course(lesson_id) {
                        self.course_cache.borrow_mut().remove(&course_id);
                        self.course_trials_cache.borrow_mut().remove(&course_id);
                    }
                }
            }
            // For lessons, invalidate the scores of all exercises in the lesson.
            Some(UnitType::Lesson) => {
                if let Some(course_id) = graph.get_lesson_course(unit_id) {
                    self.course_cache.borrow_mut().remove(&course_id);
                    self.course_trials_cache.borrow_mut().remove(&course_id);
                }
                if let Some(exercise_ids) = graph.get_lesson_exercises(unit_id) {
                    for exercise_id in exercise_ids.iter() {
                        self.exercise_cache.borrow_mut().remove(exercise_id);
                    }
                }
            }
            // For courses, invalidate the scores of all lessons and exercises in the course.
            Some(UnitType::Course) => {
                if let Some(lesson_ids) = graph.get_course_lessons(unit_id) {
                    for lesson_id in lesson_ids.iter() {
                        self.lesson_cache.borrow_mut().remove(lesson_id);
                        self.lesson_trials_cache.borrow_mut().remove(lesson_id);
                        if let Some(exercise_ids) = graph.get_lesson_exercises(*lesson_id) {
                            for exercise_id in exercise_ids.iter() {
                                self.exercise_cache.borrow_mut().remove(exercise_id);
                            }
                        }
                    }
                }
            }
            None => {}
        }
    }

    /// Removes the cached score for any unit with the given prefix.
    pub(super) fn invalidate_cached_scores_with_prefix(&self, prefix: &str) {
        // Remove the unit from the exercise, lesson, and course caches. This is safe to do even
        // though the unit is at most in one cache because the caches are disjoint.
        self.exercise_cache
            .borrow_mut()
            .retain(|unit_id, _| !unit_id.starts_with(prefix));
        self.lesson_cache
            .borrow_mut()
            .retain(|unit_id, _| !unit_id.starts_with(prefix));
        self.course_cache
            .borrow_mut()
            .retain(|unit_id, _| !unit_id.starts_with(prefix));
        self.lesson_trials_cache
            .borrow_mut()
            .retain(|unit_id, _| !unit_id.starts_with(prefix));
        self.course_trials_cache
            .borrow_mut()
            .retain(|unit_id, _| !unit_id.starts_with(prefix));
    }

    /// Returns the score for the given exercise.
    fn get_exercise_score(&self, exercise_id: Ustr) -> Result<f32> {
        // Return the cached score if it exists.
        let cached_score = self
            .exercise_cache
            .borrow()
            .get(&exercise_id)
            .map(|c| c.score);
        if let Some(score) = cached_score {
            return Ok(score);
        }

        // Retrieve the exercise's type and previous trials and compute its score.
        let exercise_type = self
            .data
            .course_library
            .read()
            .get_exercise_manifest(exercise_id)
            .map_or(ExerciseType::Procedural, |manifest| {
                manifest.exercise_type.clone()
            });
        let scores = self
            .data
            .practice_stats
            .read()
            .get_scores(exercise_id, self.options.num_trials)
            .unwrap_or_default();
        let score = self.exercise_scorer.score(exercise_type, &scores)?;

        // Retrieve the rewards for this exercise's lesson and course and compute the reward.
        let graph = self.data.unit_graph.read();
        let rewards = self.data.practice_rewards.read();
        let lesson_id = graph.get_exercise_lesson(exercise_id).unwrap_or_default();
        let lesson_rewards = rewards
            .get_rewards(lesson_id, self.options.num_rewards)
            .unwrap_or_default();
        let course_id = graph.get_lesson_course(lesson_id).unwrap_or_default();
        let course_rewards = rewards
            .get_rewards(course_id, self.options.num_rewards)
            .unwrap_or_default();
        let reward = self
            .reward_scorer
            .score_rewards(&course_rewards, &lesson_rewards)
            .unwrap_or_default();
        let now = Utc::now().timestamp();
        let last_seen = scores.first().map_or(0.0, |trial| {
            ((now - trial.timestamp) as f32 / 86_400.0).max(0.0)
        });

        // Apply the reward if it meets the criteria and cache the final score.
        let final_score = if self.reward_scorer.apply_reward(reward, &scores) {
            (score + reward).clamp(0.0, 5.0)
        } else {
            score
        };
        self.exercise_cache.borrow_mut().insert(
            exercise_id,
            CachedScore {
                score: final_score,
                velocity: self.exercise_scorer.velocity(&scores),
                num_trials: scores.len(),
                last_seen,
            },
        );
        Ok(final_score)
    }

    /// Returns the velocity of learning for the given exercise.
    pub(super) fn get_exercise_velocity(&self, exercise_id: Ustr) -> Result<Option<f32>> {
        // Return the cached value if it exists.
        let cached_velocity = self
            .exercise_cache
            .borrow()
            .get(&exercise_id)
            .and_then(|c| c.velocity);
        if let Some(velocity) = cached_velocity {
            return Ok(Some(velocity));
        }

        // Compute the exercise's score, which populates the cache. Then, retrieve the velocity of
        // learning from the cache.
        self.get_exercise_score(exercise_id)?;
        let cached_velocity = self
            .exercise_cache
            .borrow()
            .get(&exercise_id)
            .and_then(|s| s.velocity);
        Ok(cached_velocity)
    }

    /// Returns the number of trials that were considered when computing the score for the given
    /// exercise.
    pub(super) fn get_exercise_num_trials(&self, exercise_id: Ustr) -> Result<Option<usize>> {
        // Return the cached value if it exists.
        let cached_num_trials = self
            .exercise_cache
            .borrow()
            .get(&exercise_id)
            .map(|c| c.num_trials);
        if let Some(num_trials) = cached_num_trials {
            return Ok(Some(num_trials));
        }

        // Compute the exercise's score, which populates the cache. Then, retrieve the number of
        // trials from the cache.
        self.get_exercise_score(exercise_id)?;
        let cached_num_trials = self
            .exercise_cache
            .borrow()
            .get(&exercise_id)
            .map(|s| s.num_trials);
        Ok(cached_num_trials)
    }

    /// Returns the number of days since the last trial for the given exercise.
    pub(super) fn get_last_seen_days(&self, exercise_id: Ustr) -> Result<Option<f32>> {
        // Return the cached value if it exists.
        let cached_last_seen = self
            .exercise_cache
            .borrow()
            .get(&exercise_id)
            .map(|c| c.last_seen);
        if let Some(last_seen) = cached_last_seen {
            return Ok(Some(last_seen));
        }

        // Compute the exercise's score, which populates the cache. Then, retrieve the days since last
        // seen from the cache.
        self.get_exercise_score(exercise_id)?;
        let cached_last_seen = self
            .exercise_cache
            .borrow()
            .get(&exercise_id)
            .map(|s| s.last_seen);
        Ok(cached_last_seen)
    }

    /// Returns whether all the exercises in the unit have valid scores.
    pub(super) fn all_valid_exercises_have_scores(&self, unit_id: Ustr) -> bool {
        // Get all the valid exercises in the unit.
        let valid_exercises = self.data.all_valid_exercises(unit_id);
        if valid_exercises.is_empty() {
            return true;
        }

        // All valid exercises must have a score greater than 0.0.
        let scores: Vec<Result<f32>> = valid_exercises
            .into_iter()
            .map(|id| self.get_exercise_score(id))
            .collect();
        scores
            .into_iter()
            .all(|score| score.is_ok() && score.unwrap() > 0.0)
    }

    /// Returns whether the superseded unit can be considered as superseded by the superseding
    /// units.
    pub(super) fn is_superseded(&self, superseded_id: Ustr, superseding_ids: &UstrSet) -> bool {
        // Units with no superseding units are not superseded.
        if superseding_ids.is_empty() {
            return false;
        }

        // All the exercises from the superseded unit must have been seen at least once.
        if !self.all_valid_exercises_have_scores(superseded_id) {
            return false;
        }

        // All the superseding units must have a score equal or greater than the superseding score.
        let scores = superseding_ids
            .iter()
            .filter_map(|id| self.get_unit_score(*id).unwrap_or_default())
            .collect::<Vec<_>>();
        scores
            .iter()
            .all(|score| *score >= self.data.options.superseding_score)
    }

    /// Recursively check if each superseding unit has itself been superseded by another unit and
    /// replace them from the original set with those units.
    fn replace_superseding(&self, superseding_ids: &UstrSet) -> UstrSet {
        let mut result = UstrSet::default();
        for id in superseding_ids.iter().copied() {
            let superseding = self.data.get_superseding(id);
            if let Some(superseding) = superseding {
                // The unit has some superseding units of its own. If the unit has been superseded
                // by them, recursively call this function. Otherwise, add the unit to the result.
                if self.is_superseded(id, &superseding) {
                    result.extend(self.replace_superseding(&superseding));
                } else {
                    result.insert(id);
                }
            } else {
                // The unit has no superseding units, so add it to the result.
                result.insert(id);
            }
        }
        result
    }

    /// Get the initial superseding units and then recursively replace them if they have been
    /// superseded.
    pub(super) fn get_superseding_recursive(&self, unit_id: Ustr) -> Option<UstrSet> {
        let superseding_ids = self.data.get_superseding(unit_id);
        superseding_ids.map(|ids| self.replace_superseding(&ids))
    }

    /// Returns the average score of all the exercises in the given lesson.
    fn get_lesson_score(&self, lesson_id: Ustr) -> Result<Option<f32>> {
        // Return the cached score if it exists.
        let cached_score = self.lesson_cache.borrow().get(&lesson_id).copied();
        if let Some(score) = cached_score {
            return Ok(score);
        }

        // Check if the unit is blacklisted. A blacklisted unit has no score.
        let blacklist = self.data.blacklist.read();
        let blacklisted = blacklist.blacklisted(lesson_id);
        if blacklisted.unwrap_or(false) {
            self.lesson_cache.borrow_mut().insert(lesson_id, None);
            return Ok(None);
        }

        // Check if the lesson has been superseded. Superseded lessons have no score.
        let superseding_ids = self.get_superseding_recursive(lesson_id);
        if let Some(superseding_ids) = superseding_ids
            && self.is_superseded(lesson_id, &superseding_ids)
        {
            self.lesson_cache.borrow_mut().insert(lesson_id, None);
            return Ok(None);
        }

        // Compute the average score of all the exercises in the lesson.
        let exercises = self.data.unit_graph.read().get_lesson_exercises(lesson_id);
        let score = match exercises {
            None => {
                // A lesson with no exercises has no valid score.
                Ok(None)
            }
            Some(exercise_ids) => {
                // Compute the list of valid exercises. All blacklisted exercises are ignored.
                let valid_exercises = exercise_ids
                    .iter()
                    .copied()
                    .filter(|exercise_id| {
                        let blacklisted = blacklist.blacklisted(*exercise_id);
                        !blacklisted.unwrap_or(false)
                    })
                    .collect::<Vec<Ustr>>();

                if valid_exercises.is_empty() {
                    // If all exercises are blacklisted, the lesson has no valid score.
                    Ok(None)
                } else {
                    // Compute the average score of the valid exercises.
                    let avg_score: f32 = valid_exercises
                        .iter()
                        .map(|id| self.get_exercise_score(*id))
                        .sum::<Result<f32>>()?
                        / valid_exercises.len() as f32;
                    Ok(Some(avg_score))
                }
            }
        };

        // Update the cache with a valid score.
        if let Ok(score) = score {
            self.lesson_cache.borrow_mut().insert(lesson_id, score);
        }
        score
    }

    /// Returns the average score of all the lesson scores in the given course.
    fn get_course_score(&self, course_id: Ustr) -> Result<Option<f32>> {
        // Return the cached score if it exists.
        let cached_score = self.course_cache.borrow().get(&course_id).copied();
        if let Some(score) = cached_score {
            return Ok(score);
        }

        // Check if the unit is blacklisted. A blacklisted course has no valid score.
        let blacklisted = self.data.blacklist.read().blacklisted(course_id);
        if blacklisted.unwrap_or(false) {
            self.course_cache.borrow_mut().insert(course_id, None);
            return Ok(None);
        }

        // Check if the course has been superseded. Superseded courses have no score.
        let superseding_ids = self.get_superseding_recursive(course_id);
        if let Some(superseding_ids) = superseding_ids
            && self.is_superseded(course_id, &superseding_ids)
        {
            self.course_cache.borrow_mut().insert(course_id, None);
            return Ok(None);
        }

        // Compute the average score of all the lessons in the course.
        let lessons = self.data.unit_graph.read().get_course_lessons(course_id);
        let score = match lessons {
            None => {
                // A course with no lessons has no valid score.
                Ok(None)
            }
            Some(lesson_ids) => {
                // Collect all the valid scores from the course's lessons.
                let valid_lesson_scores = lesson_ids
                    .iter()
                    .copied()
                    .map(|lesson_id| self.get_lesson_score(lesson_id))
                    .filter(|score| {
                        // Filter out any lesson whose score is not valid.
                        if score.as_ref().unwrap_or(&None).is_none() {
                            return false;
                        }
                        true
                    })
                    .collect::<Result<Vec<_>>>()?;

                // Return an invalid score if all the lesson scores are invalid. This can happen if
                // all the lessons in the course are blacklisted.
                if valid_lesson_scores.is_empty() {
                    return Ok(None);
                }

                // Compute the average of the valid lesson scores.
                let avg_score: f32 = valid_lesson_scores
                    .iter()
                    .map(|s| s.unwrap_or_default())
                    .sum::<f32>()
                    / valid_lesson_scores.len() as f32;
                Ok(Some(avg_score))
            }
        };

        // Update the cache with a valid score.
        if let Ok(score) = score {
            self.course_cache.borrow_mut().insert(course_id, score);
        }
        score
    }

    /// Returns the score for the given unit. A return value of `Ok(None)` indicates that there is
    /// not a valid score for the unit, such as when the unit is blacklisted. Such a unit is
    /// considered a satisfied dependency.
    pub(super) fn get_unit_score(&self, unit_id: Ustr) -> Result<Option<f32>> {
        // Decide which method to call based on the unit type.
        let unit_type = self
            .data
            .unit_graph
            .read()
            .get_unit_type(unit_id)
            .ok_or(anyhow!("missing unit type for unit with ID {unit_id}"))?;
        match unit_type {
            UnitType::Course => self.get_course_score(unit_id),
            UnitType::Lesson => self.get_lesson_score(unit_id),
            UnitType::Exercise => self.get_exercise_score(unit_id).map(Some),
        }
    }

    /// Returns the average number of trials across all the exercises in the given lesson.
    pub(super) fn get_lesson_num_trials(&self, lesson_id: Ustr) -> Option<f32> {
        // Return the cached value if it exists.
        if let Some(cached) = self.lesson_trials_cache.borrow().get(&lesson_id) {
            return *cached;
        }

        // Get all the exercises in the lesson and filter those that are blacklisted.
        let blacklist = self.data.blacklist.read();
        let exercise_ids: Vec<Ustr> = self
            .data
            .unit_graph
            .read()
            .get_lesson_exercises(lesson_id)?
            .iter()
            .copied()
            .filter(|exercise_id| {
                let blacklisted = blacklist.blacklisted(*exercise_id);
                !blacklisted.unwrap_or(false)
            })
            .collect();

        // Compute the average number of trials across all the exercises in the lesson.
        let valid_exercise_trials: Vec<usize> = exercise_ids
            .iter()
            .filter_map(|exercise_id| self.get_exercise_num_trials(*exercise_id).unwrap_or(None))
            .collect();
        let result = if valid_exercise_trials.is_empty() {
            None
        } else {
            let total_num_trials: usize = valid_exercise_trials.iter().sum();
            Some(total_num_trials as f32 / valid_exercise_trials.len() as f32)
        };

        self.lesson_trials_cache
            .borrow_mut()
            .insert(lesson_id, result);
        result
    }

    /// Returns the average number of trials across all the lessons in the given course.
    pub(super) fn get_course_num_trials(&self, course_id: Ustr) -> Option<f32> {
        // Return the cached value if it exists.
        if let Some(cached) = self.course_trials_cache.borrow().get(&course_id) {
            return *cached;
        }

        // Get all the lessons in the course and filter those that are blacklisted or superseded.
        let blacklist = self.data.blacklist.read();
        let lesson_ids: Vec<Ustr> = self
            .data
            .unit_graph
            .read()
            .get_course_lessons(course_id)
            .unwrap_or_default()
            .iter()
            .copied()
            .filter(|lesson_id| {
                let superseding_ids = self.get_superseding_recursive(*lesson_id);
                let superseded = if let Some(superseding_ids) = superseding_ids {
                    self.is_superseded(*lesson_id, &superseding_ids)
                } else {
                    false
                };
                let blacklisted = blacklist.blacklisted(*lesson_id);
                !blacklisted.unwrap_or(false) && !superseded
            })
            .collect();

        // Compute the average number of trials across all the lessons in the course.
        let valid_lesson_trials: Vec<f32> = lesson_ids
            .iter()
            .filter_map(|lesson_id| self.get_lesson_num_trials(*lesson_id))
            .collect();
        let result = if valid_lesson_trials.is_empty() {
            None
        } else {
            let total_num_trials: f32 = valid_lesson_trials.iter().sum();
            Some(total_num_trials / valid_lesson_trials.len() as f32)
        };

        self.course_trials_cache
            .borrow_mut()
            .insert(course_id, result);
        result
    }

    /// Returns the number of trials for the unit. If the unit is a course or lesson, the average
    /// number of trials across the valid exercises in the unit is returned.
    pub(super) fn get_avg_trials(&self, unit_id: Ustr) -> Option<f32> {
        let unit_type = self.data.unit_graph.read().get_unit_type(unit_id);
        match unit_type {
            Some(UnitType::Course) => self.get_course_num_trials(unit_id),
            Some(UnitType::Lesson) => self.get_lesson_num_trials(unit_id),
            _ => None, // grcov-excl-line
        }
    }
}

#[cfg(test)]
#[cfg_attr(coverage, coverage(off))]
mod test {
    use anyhow::Result;
    use chrono::Utc;
    use std::{collections::BTreeMap, sync::LazyLock};
    use ustr::Ustr;

    use crate::{
        blacklist::Blacklist,
        data::{MasteryScore, SchedulerOptions},
        scheduler::{ExerciseScheduler, UnitScorer, unit_scorer::CachedScore},
        test_utils::*,
    };

    static NUM_EXERCISES: usize = 2;

    /// A simple set of courses to test the basic functionality of Trane.
    static TEST_LIBRARY: LazyLock<Vec<TestCourse>> = LazyLock::new(|| {
        vec![
            TestCourse {
                id: TestId(0, None, None),
                dependencies: vec![],
                superseded: vec![],
                encompassed: vec![],
                metadata: BTreeMap::default(),
                lessons: vec![
                    TestLesson {
                        id: TestId(0, Some(0), None),
                        dependencies: vec![],
                        superseded: vec![],
                        encompassed: vec![],
                        metadata: BTreeMap::default(),
                        num_exercises: NUM_EXERCISES,
                    },
                    TestLesson {
                        id: TestId(0, Some(1), None),
                        dependencies: vec![TestId(0, Some(0), None)],
                        encompassed: vec![],
                        superseded: vec![],
                        metadata: BTreeMap::default(),
                        num_exercises: NUM_EXERCISES,
                    },
                ],
            },
            TestCourse {
                id: TestId(1, None, None),
                dependencies: vec![TestId(0, None, None)],
                encompassed: vec![],
                superseded: vec![TestId(0, None, None)],
                metadata: BTreeMap::default(),
                lessons: vec![
                    TestLesson {
                        id: TestId(1, Some(0), None),
                        dependencies: vec![],
                        encompassed: vec![],
                        superseded: vec![],
                        metadata: BTreeMap::default(),
                        num_exercises: NUM_EXERCISES,
                    },
                    TestLesson {
                        id: TestId(1, Some(1), None),
                        dependencies: vec![TestId(1, Some(0), None)],
                        encompassed: vec![],
                        superseded: vec![TestId(1, Some(0), None)],
                        metadata: BTreeMap::default(),
                        num_exercises: NUM_EXERCISES,
                    },
                ],
            },
        ]
    });

    /// Verifies that a score of `None` is returned for a blacklisted course.
    #[test]
    fn blacklisted_course_score() -> Result<()> {
        let temp_dir = tempfile::tempdir()?;
        let mut library = init_test_simulation(temp_dir.path(), &TEST_LIBRARY)?;
        let scheduler_data = library.get_scheduler_data();
        let cache = UnitScorer::new(scheduler_data, SchedulerOptions::default());

        let course_id = Ustr::from("0");
        library.add_to_blacklist(course_id)?;
        assert_eq!(cache.get_course_score(course_id)?, None);
        Ok(())
    }

    /// Verifies that the score of a superseded course is None and is correctly cached.
    #[test]
    fn superseded_course_cached() -> Result<()> {
        let temp_dir = tempfile::tempdir()?;
        let library = init_test_simulation(temp_dir.path(), &TEST_LIBRARY)?;
        let scheduler_data = library.get_scheduler_data();
        let cache = UnitScorer::new(scheduler_data, SchedulerOptions::default());

        // Insert scores for every exercise to ensure course 0 has been superseded.
        let ts = Utc::now().timestamp();
        library.score_exercise(Ustr::from("0::0::0"), MasteryScore::Five, ts)?;
        library.score_exercise(Ustr::from("0::0::1"), MasteryScore::Five, ts)?;
        library.score_exercise(Ustr::from("0::1::0"), MasteryScore::Five, ts)?;
        library.score_exercise(Ustr::from("0::1::1"), MasteryScore::Five, ts)?;
        library.score_exercise(Ustr::from("1::0::0"), MasteryScore::Five, ts)?;
        library.score_exercise(Ustr::from("1::0::1"), MasteryScore::Five, ts)?;
        library.score_exercise(Ustr::from("1::1::0"), MasteryScore::Five, ts)?;
        library.score_exercise(Ustr::from("1::1::1"), MasteryScore::Five, ts)?;

        // Get the scores for course 0 twice. Once to populate the cache and once to retrieve the
        // cached value.
        assert_eq!(cache.get_course_score(Ustr::from("0"))?, None);
        assert_eq!(cache.get_course_score(Ustr::from("0"))?, None);
        Ok(())
    }

    /// Verifies that the score of a superseded lesson is None and is correctly cached.
    #[test]
    fn superseded_course_lesson_cached() -> Result<()> {
        let temp_dir = tempfile::tempdir()?;
        let library = init_test_simulation(temp_dir.path(), &TEST_LIBRARY)?;
        let scheduler_data = library.get_scheduler_data();
        let cache = UnitScorer::new(scheduler_data, SchedulerOptions::default());

        // Insert scores for every exercise to ensure lesson 1::0 has been superseded.
        let ts = Utc::now().timestamp();
        library.score_exercise(Ustr::from("0::0::0"), MasteryScore::Five, ts)?;
        library.score_exercise(Ustr::from("0::0::1"), MasteryScore::Five, ts)?;
        library.score_exercise(Ustr::from("0::1::0"), MasteryScore::Five, ts)?;
        library.score_exercise(Ustr::from("0::1::1"), MasteryScore::Five, ts)?;
        library.score_exercise(Ustr::from("1::0::0"), MasteryScore::Five, ts)?;
        library.score_exercise(Ustr::from("1::0::1"), MasteryScore::Five, ts)?;
        library.score_exercise(Ustr::from("1::1::0"), MasteryScore::Five, ts)?;
        library.score_exercise(Ustr::from("1::1::1"), MasteryScore::Five, ts)?;

        // Get the scores for lesson 1::0 twice. Once to populate the cache and once to retrieve the
        // cached value.
        assert_eq!(cache.get_lesson_score(Ustr::from("1::0"))?, None);
        assert_eq!(cache.get_lesson_score(Ustr::from("1::0"))?, None);
        Ok(())
    }

    /// Verifies that scores are correctly invalidated.
    #[test]
    fn invalidate_cached_scores() -> Result<()> {
        let temp_dir = tempfile::tempdir()?;
        let library = init_test_simulation(temp_dir.path(), &TEST_LIBRARY)?;
        let scheduler_data = library.get_scheduler_data();
        let cache = UnitScorer::new(scheduler_data, SchedulerOptions::default());

        // Insert some scores into the exercise and lesson caches.
        cache.exercise_cache.borrow_mut().insert(
            Ustr::from("a"),
            CachedScore {
                score: 5.0,
                velocity: None,
                num_trials: 1,
                last_seen: 0.0,
            },
        );
        cache.exercise_cache.borrow_mut().insert(
            Ustr::from("b::a"),
            CachedScore {
                score: 5.0,
                velocity: None,
                num_trials: 1,
                last_seen: 0.0,
            },
        );
        cache
            .lesson_cache
            .borrow_mut()
            .insert(Ustr::from("a::a"), Some(5.0));
        cache
            .lesson_cache
            .borrow_mut()
            .insert(Ustr::from("c::a"), Some(5.0));

        // Verify that the scores are present.
        assert_eq!(cache.get_exercise_score(Ustr::from("a"))?, 5.0);
        assert_eq!(cache.get_exercise_score(Ustr::from("b::a"))?, 5.0);
        assert_eq!(cache.get_lesson_score(Ustr::from("a::a"))?, Some(5.0));
        assert_eq!(cache.get_lesson_score(Ustr::from("c::a"))?, Some(5.0));

        // Invalidate prefix `a` and verify that the cached scores are removed.
        cache.invalidate_cached_scores_with_prefix("a");
        assert_eq!(cache.get_exercise_score(Ustr::from("a"))?, 0.0);
        assert_eq!(cache.get_exercise_score(Ustr::from("b::a"))?, 5.0);
        assert_eq!(cache.get_lesson_score(Ustr::from("a::a"))?, None);
        assert_eq!(cache.get_lesson_score(Ustr::from("c::a"))?, Some(5.0));

        // Invalidate units `b::a  and `c::a` and verify that the score is removed.
        cache.invalidate_cached_score(Ustr::from("b::a"));
        cache.invalidate_cached_score(Ustr::from("c::a"));
        assert_eq!(cache.get_exercise_score(Ustr::from("b::a"))?, 0.0);
        assert_eq!(cache.get_lesson_score(Ustr::from("c::a"))?, None);
        Ok(())
    }

    /// Verifies that the number of trials are cached along the exercise scores.
    #[test]
    fn get_num_trials() -> Result<()> {
        // Create a test library and send some scores.
        let temp_dir = tempfile::tempdir()?;
        let library = init_test_simulation(temp_dir.path(), &TEST_LIBRARY)?;
        let scheduler_data = library.get_scheduler_data();
        let cache = UnitScorer::new(scheduler_data, SchedulerOptions::default());
        let exercise_id = Ustr::from("0::0::0");
        library.score_exercise(exercise_id, MasteryScore::Four, 1)?;
        library.score_exercise(exercise_id, MasteryScore::Five, 2)?;

        // Retrieve the number of trials twice. The second time should hit the cache.
        assert_eq!(Some(2), cache.get_exercise_num_trials(exercise_id)?);
        assert_eq!(Some(2), cache.get_exercise_num_trials(exercise_id)?);

        // Add another score and invalidate the cache. The change in the number of trials should be
        // reflected.
        library.score_exercise(exercise_id, MasteryScore::Four, 3)?;
        cache.invalidate_cached_score(exercise_id);
        assert_eq!(Some(3), cache.get_exercise_num_trials(exercise_id)?);
        Ok(())
    }

    /// Verifies that the number of days since last seen is computed and cached along with the score.
    #[test]
    fn get_last_seen_days() -> Result<()> {
        let temp_dir = tempfile::tempdir()?;
        let library = init_test_simulation(temp_dir.path(), &TEST_LIBRARY)?;
        let scheduler_data = library.get_scheduler_data();
        let cache = UnitScorer::new(scheduler_data, SchedulerOptions::default());
        let exercise_id = Ustr::from("0::0::0");
        let two_days_ago = Utc::now().timestamp() - (2 * 86_400);
        library.score_exercise(exercise_id, MasteryScore::Three, two_days_ago)?;

        let last_seen = cache.get_last_seen_days(exercise_id)?;
        assert!((last_seen.unwrap_or_default() - 2.0).abs() < 0.5);

        library.score_exercise(exercise_id, MasteryScore::Four, Utc::now().timestamp())?;
        cache.invalidate_cached_score(exercise_id);
        let last_seen = cache.get_last_seen_days(exercise_id)?;
        assert!(last_seen.unwrap_or_default() < 1.0);
        Ok(())
    }
}