use std::io;
use std::hash::{BuildHasherDefault, Hash};
use std::collections::hash_map::DefaultHasher;
use super::Map;
use crate::coding::{Coding, Decoder, SerializableCoding, BuildCoding};
use super::conf::{MapConf, ValuesPreFiller};
use bitm::{BitAccess, BitVec};
use ph::stats::AccessStatsCollector;
use ph::{BuildDefaultSeededHasher, BuildSeededHasher};
use std::collections::HashMap;
use dyn_size_of::GetSize;
use minimum_redundancy::{BitsPerFragment, DecodingResult};
pub struct CMap<C , S = BuildDefaultSeededHasher> {
pub value_fragments: Map<S>,
pub value_coding: C,
}
impl<C: GetSize, S> GetSize for CMap<C, S> {
fn size_bytes_dyn(&self) -> usize {
self.value_fragments.size_bytes_dyn() + self.value_coding.size_bytes_dyn()
}
const USES_DYN_MEM: bool = Map::<S>::USES_DYN_MEM || C::USES_DYN_MEM;
}
impl<C: SerializableCoding, S> CMap<C, S> {
pub fn write_bytes(&self, bytes_per_value: usize) -> usize {
self.value_fragments.write_bytes() + self.value_coding.write_bytes(bytes_per_value)
}
pub fn write<F>(&self, output: &mut dyn io::Write, write_value: F) -> io::Result<()>
where F: FnMut(&mut dyn io::Write, &C::Value) -> io::Result<()>
{
self.value_fragments.write(output)?;
self.value_coding.write(output, write_value)
}
pub fn read_with_hasher<F>(input: &mut dyn io::Read, read_value: F, hasher: S) -> io::Result<Self>
where F: FnMut(&mut dyn io::Read) -> io::Result<C::Value>
{
Ok(Self {
value_fragments: Map::<S>::read_with_hasher(input, hasher)?,
value_coding: C::read(input, read_value)?
})
}
}
impl<C: SerializableCoding> CMap<C, BuildHasherDefault<DefaultHasher>> {
pub fn read<F>(input: &mut dyn io::Read, read_value: F) -> io::Result<Self>
where F: FnMut(&mut dyn io::Read) -> io::Result<C::Value>
{
Self::read_with_hasher(input, read_value, Default::default())
}
}
impl<C: Coding, S: BuildSeededHasher> CMap<C, S> {
pub fn try_from_mapf_with_coding_conf<'a, K, V, KvIntoIter, FKvIntoIter, BM>(
map: FKvIntoIter, value_coding: C, conf: MapConf<BM, S>, extra_bits_per_fragment: u8
) -> Option<Self>
where K: Hash + 'a,
V: std::borrow::Borrow<<C as Coding>::Value> + 'a,
KvIntoIter: IntoIterator<Item=(&'a K, &'a V)> + 'a,
FKvIntoIter: Fn() -> KvIntoIter,
BM: ValuesPreFiller {
let encoder = value_coding.encoder();
let keys_len = map().into_iter().map(|(_,v)| value_coding.len_of_encoded(&encoder, v) as usize).sum();
let mut values = Box::<[u64]>::with_zeroed_bits(value_coding.bits_per_fragment() as usize*keys_len);
let mut values_len = 0;
for (_, v) in map() {
for f in value_coding.fragments_of_encoded(&encoder, v) {
values.init_fragment(values_len, f as u64, value_coding.bits_per_fragment());
values_len += 1;
}
}
debug_assert_eq!(values_len, keys_len);
let r = Map::try_with_conf_fn(
|| map().into_iter()
.flat_map(|(k, v)|
value_coding.fragments_of_encoded(&encoder, v)
.enumerate().map(move |(i, f)| ((k, i as u8), f as u64)) ),
keys_len,
value_coding.bits_per_fragment()+extra_bits_per_fragment,
conf);
drop(encoder);
r.map(|value_fragments| Self { value_fragments, value_coding })
}
#[inline(always)]
pub fn try_from_map_with_coding_conf<K, MS, BM, V>(map: &HashMap<K, V, MS>, value_coding: C, conf: MapConf<BM, S>, bdz_extra_bits_per_fragment: u8) -> Option<Self>
where K: Hash, BM: ValuesPreFiller, V: std::borrow::Borrow<<C as Coding>::Value> {
Self::try_from_mapf_with_coding_conf(|| map, value_coding, conf, bdz_extra_bits_per_fragment)
}
#[inline(always)]
pub fn try_from_kv_with_coding_conf<K, BM, V>(keys: &[K], values: &[V], value_coding: C, conf: MapConf<BM, S>, bdz_extra_bits_per_fragment: u8) -> Option<Self>
where K: Hash, BM: ValuesPreFiller, V: std::borrow::Borrow<<C as Coding>::Value> {
Self::try_from_mapf_with_coding_conf(|| keys.iter().zip(values), value_coding, conf, bdz_extra_bits_per_fragment)
}
}
impl<C: Coding, > CMap<C> {
#[inline(always)]
pub fn try_from_map_with_coding<K, MS, V>(map: &HashMap<K, V, MS>, value_coding: C, bdz_extra_bits_per_fragment: u8) -> Option<Self>
where K: Hash, V: std::borrow::Borrow<<C as Coding>::Value> {
Self::try_from_map_with_coding_conf(map, value_coding, MapConf::<(), _>::default(), bdz_extra_bits_per_fragment)
}
#[inline(always)]
pub fn try_from_mapf_with_coding<'a, K, KvIntoIter, FKvIntoIter, V>(map: FKvIntoIter, value_coding: C, bdz_extra_bits_per_fragment: u8) -> Option<Self>
where K: Hash + 'a,
V: std::borrow::Borrow<<C as Coding>::Value> + 'a,
KvIntoIter: IntoIterator<Item=(&'a K, &'a V)> + 'a,
FKvIntoIter: Fn() -> KvIntoIter
{
Self::try_from_mapf_with_coding_conf(map, value_coding, MapConf::<(), _>::default(), bdz_extra_bits_per_fragment)
}
#[inline(always)]
pub fn try_from_kv_with_coding<K: Hash, V>(keys: &[K], values: &[V], value_coding: C, bdz_extra_bits_per_fragment: u8) -> Option<Self>
where K: Hash, V: std::borrow::Borrow<<C as Coding>::Value>
{
Self::try_from_kv_with_coding_conf(keys, values, value_coding, MapConf::<(), _>::default(), bdz_extra_bits_per_fragment)
}
}
impl<C: Coding, S: BuildSeededHasher> CMap<C, S> {
#[inline(always)]
pub fn try_from_map_with_builder_bpf_conf<K, MS, BM, BC>(map: &HashMap<K, C::Value, MS>, build_coding: &BC, bits_per_fragment: u8, conf: MapConf<BM, S>, bdz_extra_bits_per_fragment: u8) -> Option<Self>
where K: Hash,
BM: ValuesPreFiller,
BC: BuildCoding<C::Value, Coding=C>
{
Self::try_from_mapf_with_coding_conf(|| map,
build_coding.build_from_iter(map.values(), bits_per_fragment),
conf, bdz_extra_bits_per_fragment)
}
#[inline(always)]
pub fn try_from_map_with_builder_conf<K, MS, BM, BC>(map: &HashMap<K, C::Value, MS>, build_coding: &BC, conf: MapConf<BM, S>, bdz_extra_bits_per_fragment: u8) -> Option<Self>
where K: Hash,
BM: ValuesPreFiller,
BC: BuildCoding<C::Value, Coding=C>
{
Self::try_from_mapf_with_coding_conf(|| map,
build_coding.build_from_iter(map.values(), 0),
conf, bdz_extra_bits_per_fragment)
}
#[inline(always)]
pub fn try_from_kv_with_builder_bpf_conf<K, BM, BC>(keys: &[K], values: &[C::Value], build_coding: &BC, bits_per_fragment: u8, conf: MapConf<BM, S>, bdz_extra_bits_per_fragment: u8) -> Option<Self>
where K: Hash,
BM: ValuesPreFiller,
BC: BuildCoding<C::Value, Coding=C>
{
Self::try_from_kv_with_coding_conf(keys, values, build_coding.build_from_iter(values.iter(), bits_per_fragment), conf, bdz_extra_bits_per_fragment)
}
#[inline(always)]
pub fn try_from_kv_with_builder_conf<K, BM, BC>(keys: &[K], values: &[C::Value], build_coding: &BC, conf: MapConf<BM, S>, bdz_extra_bits_per_fragment: u8) -> Option<Self>
where K: Hash,
BM: ValuesPreFiller,
BC: BuildCoding<C::Value, Coding=C>
{
Self::try_from_kv_with_coding_conf(keys, values, build_coding.build_from_iter(values.iter(), 0), conf, bdz_extra_bits_per_fragment)
}
}
impl<V: Hash+Eq+Clone, S: BuildSeededHasher> CMap<minimum_redundancy::Coding<V>, S> {
#[inline(always)]
pub fn try_from_map_with_conf<K, MS, BM>(map: &HashMap<K, V, MS>, bits_per_fragment: u8, conf: MapConf<BM, S>, bdz_extra_bits_per_fragment: u8) -> Option<Self>
where K: Hash,
BM: ValuesPreFiller
{
Self::try_from_mapf_with_coding_conf(|| map,
minimum_redundancy::Coding::<V>::from_iter(BitsPerFragment(bits_per_fragment), map.values()),
conf, bdz_extra_bits_per_fragment)
}
#[inline(always)]
pub fn try_from_kv_with_conf<K, BM>(keys: &[K], values: &[V], bits_per_fragment: u8, conf: MapConf<BM, S>, bdz_extra_bits_per_fragment: u8) -> Option<Self>
where K: Hash, BM: ValuesPreFiller {
Self::try_from_kv_with_coding_conf(keys, values, minimum_redundancy::Coding::<V>::from_iter(BitsPerFragment(bits_per_fragment), values.iter()), conf, bdz_extra_bits_per_fragment)
}
}
impl<V: Hash+Eq+Clone> CMap<minimum_redundancy::Coding<V>> {
#[inline(always)]
pub fn try_from_map<K: Hash, MS>(map: &HashMap<K, V, MS>, bits_per_fragment: u8, bdz_extra_bits_per_fragment: u8) -> Option<Self> {
Self::try_from_map_with_conf(map, bits_per_fragment, MapConf::<(), _>::default(), bdz_extra_bits_per_fragment)
}
#[inline(always)]
pub fn try_from_mapf<'a, K, KvIntoIter, FKvIntoIter, BM>(map: FKvIntoIter, bits_per_fragment: u8, bdz_extra_bits_per_fragment: u8) -> Option<Self>
where K: Hash + 'a,
V: 'a,
KvIntoIter: IntoIterator<Item=(&'a K, &'a V)> + 'a,
FKvIntoIter: Fn() -> KvIntoIter
{
let value_coding = minimum_redundancy::Coding::<V>::from_iter(BitsPerFragment(bits_per_fragment), map().into_iter().map(|(_, v)| v));
Self::try_from_mapf_with_coding(map, value_coding, bdz_extra_bits_per_fragment)
}
#[inline(always)]
pub fn try_from_kv<K: Hash>(keys: &[K], values: &[V], bits_per_fragment: u8, bdz_extra_bits_per_fragment: u8) -> Option<Self> {
Self::try_from_kv_with_conf(keys, values, bits_per_fragment, MapConf::<(), _>::default(), bdz_extra_bits_per_fragment)
}
}
impl<C: Coding, S: BuildSeededHasher> CMap<C, S> {
pub fn get_stats<K: Hash + ?Sized, A: AccessStatsCollector>(&self, k: &K, access_stats: &mut A) -> Option<<<C as Coding>::Decoder<'_> as Decoder>::Decoded> {
let mut result_decoder = self.value_coding.decoder();
let mut fragment_nr = 0u8;
if self.value_fragments.bits_per_value == self.value_coding.bits_per_fragment() { loop {
match result_decoder.consume(self.value_fragments.get(&(k, fragment_nr)) as u8) {
DecodingResult::Value(v) => {
access_stats.found_on_level(fragment_nr as u32);
return Some(v)
},
DecodingResult::Invalid => {
access_stats.fail_on_level(fragment_nr as u32);
return None
},
DecodingResult::Incomplete => {}
}
fragment_nr += 1;
}
} else {
loop {
match result_decoder.consume_checked(self.value_fragments.get(&(k, fragment_nr)) as u8) {
DecodingResult::Value(v) => {
access_stats.found_on_level(fragment_nr as u32);
return Some(v)
},
DecodingResult::Invalid => {
access_stats.fail_on_level(fragment_nr as u32);
return None
},
DecodingResult::Incomplete => {}
}
fragment_nr += 1;
}
}
}
#[inline(always)] pub fn get<K: Hash + ?Sized>(&self, k: &K) -> Option<<<C as Coding>::Decoder<'_> as Decoder>::Decoded> {
self.get_stats(k, &mut ())
}
}
#[cfg(test)]
mod tests {
use super::*;
use maplit::hashmap;
fn bdzhmap_3pairs_conf<BM: ValuesPreFiller>(conf: MapConf<BM>, bits_per_fragment: u8, bdz_extra_bits_per_fragment: u8) {
let bdzhmap = CMap::try_from_map_with_conf(&hashmap!('a'=>0u8, 'b'=>3u8, 'c'=>8u8), bits_per_fragment, conf, bdz_extra_bits_per_fragment).unwrap();
assert_eq!(bdzhmap.get(&'a'), Some(&0));
assert_eq!(bdzhmap.get(&'b'), Some(&3));
assert_eq!(bdzhmap.get(&'c'), Some(&8));
assert_eq!(bdzhmap.value_fragments.bits_per_value, bits_per_fragment+bdz_extra_bits_per_fragment);
assert_eq!(bdzhmap.value_coding.bits_per_fragment(), bits_per_fragment);
}
#[test]
fn bdzhmap_3pairs_1bpf() {
bdzhmap_3pairs_conf(MapConf::new(), 1, 0);
bdzhmap_3pairs_conf(MapConf::new(), 1, 1);
bdzhmap_3pairs_conf(MapConf::new(), 1, 2);
}
#[test]
fn bdzhmap_3pairs_1bpf_bm123() {
bdzhmap_3pairs_conf(MapConf::pattern(123u64), 1, 0);
bdzhmap_3pairs_conf(MapConf::pattern(123u64), 1, 1);
bdzhmap_3pairs_conf(MapConf::pattern(123u64), 1, 2);
}
#[test]
fn bdzhmap_3pairs_2bpf() {
bdzhmap_3pairs_conf(MapConf::new(), 2, 0);
bdzhmap_3pairs_conf(MapConf::new(), 2, 1);
bdzhmap_3pairs_conf(MapConf::new(), 2, 2);
}
#[test]
fn bdzhmap_3pairs_2bpf_bm123() {
bdzhmap_3pairs_conf(MapConf::pattern(123u64), 2, 0);
bdzhmap_3pairs_conf(MapConf::pattern(123u64), 2, 1);
bdzhmap_3pairs_conf(MapConf::pattern(123u64), 2, 2);
}
fn bdzhmap_8pairs_conf<BM: ValuesPreFiller>(conf: MapConf<BM>, bits_per_fragment: u8, bdz_extra_bits_per_fragment: u8) {
let bdzhmap = CMap::try_from_map_with_conf(&hashmap!(
'a' => 1u8, 'b' => 2u8, 'c' => 1u8, 'd' => 3u8,
'e' => 4u8, 'f' => 1u8, 'g' => 5u8, 'h' => 6u8), bits_per_fragment, conf, bdz_extra_bits_per_fragment).unwrap();
assert_eq!(bdzhmap.get(&'a'), Some(&1));
assert_eq!(bdzhmap.get(&'b'), Some(&2));
assert_eq!(bdzhmap.get(&'c'), Some(&1));
assert_eq!(bdzhmap.get(&'d'), Some(&3));
assert_eq!(bdzhmap.get(&'e'), Some(&4));
assert_eq!(bdzhmap.get(&'f'), Some(&1));
assert_eq!(bdzhmap.get(&'g'), Some(&5));
assert_eq!(bdzhmap.get(&'h'), Some(&6));
assert_eq!(bdzhmap.value_fragments.bits_per_value, bits_per_fragment+bdz_extra_bits_per_fragment);
assert_eq!(bdzhmap.value_coding.bits_per_fragment(), bits_per_fragment);
}
#[test]
fn bdzhmap_8pairs_1bpf() {
bdzhmap_8pairs_conf(MapConf::new(), 1, 0);
bdzhmap_8pairs_conf(MapConf::new(), 1, 1);
bdzhmap_8pairs_conf(MapConf::new(), 1, 2);
}
#[test]
fn bdzhmap_8pairs_1bpf_bm123() {
bdzhmap_8pairs_conf(MapConf::pattern(123u64), 1, 0);
bdzhmap_8pairs_conf(MapConf::pattern(123u64), 1, 1);
bdzhmap_8pairs_conf(MapConf::pattern(123u64), 1, 2);
}
#[test]
fn bdzhmap_8pairs_2bpf() {
bdzhmap_8pairs_conf(MapConf::new(), 2, 0);
bdzhmap_8pairs_conf(MapConf::new(), 2, 1);
bdzhmap_8pairs_conf(MapConf::new(), 2, 2);
}
#[test]
fn bdzhmap_8pairs_2bpf_bm123() {
bdzhmap_8pairs_conf(MapConf::pattern(123u64), 2, 0);
bdzhmap_8pairs_conf(MapConf::pattern(123u64), 2, 1);
bdzhmap_8pairs_conf(MapConf::pattern(123u64), 2, 2);
}
#[test]
fn bdzhmap_8pairs_3bpf() {
bdzhmap_8pairs_conf(MapConf::new(), 3, 0);
bdzhmap_8pairs_conf(MapConf::new(), 3, 1);
bdzhmap_8pairs_conf(MapConf::new(), 3, 2);
}
#[test]
fn bdzhmap_8pairs_3bpf_bm123() {
bdzhmap_8pairs_conf(MapConf::pattern(123u64), 3, 0);
bdzhmap_8pairs_conf(MapConf::pattern(123u64), 3, 1);
bdzhmap_8pairs_conf(MapConf::pattern(123u64), 3, 2);
}
}