Struct TokenMultiset 

pub struct TokenMultiset(/* private fields */);

A multiset of token IDs stored as token -> occurrence count.

Implementations

impl TokenMultiset

pub fn from_token_ids(token_ids: &[TokenId]) -> Self

Create a TokenMultiset from a sequence of token IDs.

pub fn total_count(&self) -> usize

Total number of token occurrences in the multiset.

pub fn high_subset(&self, dictionary: &TokenDictionary) -> Self

Get a subset containing only high-value (legalese) tokens.

pub fn intersection(&self, other: &Self) -> Self

Materialize the multiset intersection with another TokenMultiset.

Methods from Deref<Target = HashMap<TokenId, usize>>

pub fn capacity(&self) -> usize

Returns the number of elements the map can hold without reallocating.

This number is a lower bound; the HashMap<K, V> might be able to hold more, but is guaranteed to be able to hold at least this many.

Examples

use std::collections::HashMap;
let map: HashMap<i32, i32> = HashMap::with_capacity(100);
assert!(map.capacity() >= 100);

pub fn keys(&self) -> Keys<'_, K, V>

An iterator visiting all keys in arbitrary order. The iterator element type is &'a K.

Examples

use std::collections::HashMap;

let map = HashMap::from([
    ("a", 1),
    ("b", 2),
    ("c", 3),
]);

for key in map.keys() {
    println!("{key}");
}

Performance

In the current implementation, iterating over keys takes O(capacity) time instead of O(len) because it internally visits empty buckets too.

pub fn values(&self) -> Values<'_, K, V>

An iterator visiting all values in arbitrary order. The iterator element type is &'a V.

Examples

use std::collections::HashMap;

let map = HashMap::from([
    ("a", 1),
    ("b", 2),
    ("c", 3),
]);

for val in map.values() {
    println!("{val}");
}

Performance

In the current implementation, iterating over values takes O(capacity) time instead of O(len) because it internally visits empty buckets too.

pub fn iter(&self) -> Iter<'_, K, V>

An iterator visiting all key-value pairs in arbitrary order. The iterator element type is (&'a K, &'a V).

Examples

use std::collections::HashMap;

let map = HashMap::from([
    ("a", 1),
    ("b", 2),
    ("c", 3),
]);

for (key, val) in map.iter() {
    println!("key: {key} val: {val}");
}

Performance

In the current implementation, iterating over map takes O(capacity) time instead of O(len) because it internally visits empty buckets too.

pub fn len(&self) -> usize

Returns the number of elements in the map.

Examples

use std::collections::HashMap;

let mut a = HashMap::new();
assert_eq!(a.len(), 0);
a.insert(1, "a");
assert_eq!(a.len(), 1);

pub fn is_empty(&self) -> bool

Returns true if the map contains no elements.

Examples

use std::collections::HashMap;

let mut a = HashMap::new();
assert!(a.is_empty());
a.insert(1, "a");
assert!(!a.is_empty());

pub fn hasher(&self) -> &S

Returns a reference to the map’s BuildHasher.

Examples

use std::collections::HashMap;
use std::hash::RandomState;

let hasher = RandomState::new();
let map: HashMap<i32, i32> = HashMap::with_hasher(hasher);
let hasher: &RandomState = map.hasher();

pub fn get<Q>(&self, k: &Q) -> Option<&V>

where K: Borrow<Q>, Q: Hash + Eq + ?Sized,

Returns a reference to the value corresponding to the key.

The key may be any borrowed form of the map’s key type, but Hash and Eq on the borrowed form must match those for the key type.

Examples

use std::collections::HashMap;

let mut map = HashMap::new();
map.insert(1, "a");
assert_eq!(map.get(&1), Some(&"a"));
assert_eq!(map.get(&2), None);

pub fn get_key_value<Q>(&self, k: &Q) -> Option<(&K, &V)>

where K: Borrow<Q>, Q: Hash + Eq + ?Sized,

Returns the key-value pair corresponding to the supplied key. This is potentially useful:

• for key types where non-identical keys can be considered equal;
• for getting the &K stored key value from a borrowed &Q lookup key; or
• for getting a reference to a key with the same lifetime as the collection.

The supplied key may be any borrowed form of the map’s key type, but Hash and Eq on the borrowed form must match those for the key type.

Examples

use std::collections::HashMap;
use std::hash::{Hash, Hasher};

#[derive(Clone, Copy, Debug)]
struct S {
    id: u32,
    name: &'static str, // ignored by equality and hashing operations
}

impl PartialEq for S {
    fn eq(&self, other: &S) -> bool {
        self.id == other.id
    }
}

impl Eq for S {}

impl Hash for S {
    fn hash<H: Hasher>(&self, state: &mut H) {
        self.id.hash(state);
    }
}

let j_a = S { id: 1, name: "Jessica" };
let j_b = S { id: 1, name: "Jess" };
let p = S { id: 2, name: "Paul" };
assert_eq!(j_a, j_b);

let mut map = HashMap::new();
map.insert(j_a, "Paris");
assert_eq!(map.get_key_value(&j_a), Some((&j_a, &"Paris")));
assert_eq!(map.get_key_value(&j_b), Some((&j_a, &"Paris"))); // the notable case
assert_eq!(map.get_key_value(&p), None);

pub fn contains_key<Q>(&self, k: &Q) -> bool

where K: Borrow<Q>, Q: Hash + Eq + ?Sized,

Returns true if the map contains a value for the specified key.

The key may be any borrowed form of the map’s key type, but Hash and Eq on the borrowed form must match those for the key type.

Examples

use std::collections::HashMap;

let mut map = HashMap::new();
map.insert(1, "a");
assert_eq!(map.contains_key(&1), true);
assert_eq!(map.contains_key(&2), false);

Trait Implementations

impl Archive for TokenMultiset

where HashMap<TokenId, usize>: Archive,

const COPY_OPTIMIZATION: CopyOptimization<Self>

An optimization flag that allows the bytes of this type to be copied directly to a writer instead of calling serialize.

type Archived = ArchivedTokenMultiset

The archived representation of this type.

type Resolver = TokenMultisetResolver

The resolver for this type. It must contain all the additional information from serializing needed to make the archived type from the normal type.

fn resolve(&self, resolver: Self::Resolver, out: Place<Self::Archived>)

Creates the archived version of this value at the given position and writes it to the given output.

impl Clone for TokenMultiset

fn clone(&self) -> TokenMultiset

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenMultiset

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for TokenMultiset

fn default() -> TokenMultiset

Returns the “default value” for a type.

impl Deref for TokenMultiset

type Target = HashMap<TokenId, usize>

The resulting type after dereferencing.

fn deref(&self) -> &Self::Target

Dereferences the value.

impl<__D: Fallible + ?Sized> Deserialize<TokenMultiset, __D> for Archived<TokenMultiset>

where HashMap<TokenId, usize>: Archive, <HashMap<TokenId, usize> as Archive>::Archived: Deserialize<HashMap<TokenId, usize>, __D>,

fn deserialize( &self, deserializer: &mut __D, ) -> Result<TokenMultiset, <__D as Fallible>::Error>

Deserializes using the given deserializer

impl FromIterator<(TokenId, usize)> for TokenMultiset

fn from_iter<T: IntoIterator<Item = (TokenId, usize)>>(iter: T) -> Self

Creates a value from an iterator.

impl PartialEq for TokenMultiset

fn eq(&self, other: &TokenMultiset) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl<__S: Fallible + ?Sized> Serialize<__S> for TokenMultiset

where HashMap<TokenId, usize>: Serialize<__S>,

fn serialize( &self, serializer: &mut __S, ) -> Result<<Self as Archive>::Resolver, <__S as Fallible>::Error>

Writes the dependencies for the object and returns a resolver that can create the archived type.

impl Eq for TokenMultiset

impl StructuralPartialEq for TokenMultiset