icbiadb 0.1.21

KV & data structures database
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136

use std::collections::HashMap;

use serde::Deserialize;

use crate::utils::*;
use crate::decl::types::*;
use crate::mem::{OwnedMemoryRecord, Memory};
use crate::parser;

use std::io::{SeekFrom};

#[derive(Debug, Deserialize)]
#[repr(C)]
pub struct Header {
	decl_len: u32,
	records_len: u64,
	decl_records_len: u64,
}

pub struct Reader<T: std::io::Read + std::io::Seek> {
	reader: T,
}

impl<T: std::io::Read + std::io::Seek> Reader<T> {
	pub fn new(reader: T) -> Self {
		Reader {
			reader: reader,
		}
	}

	pub fn read_to(&mut self, memory: &mut Memory) -> std::io::Result<()> {
		// TODO, move data initialization for memory upward in the function call stack
		let time = std::time::Instant::now();

		let header = match self.read_header() {
			Ok(h) => h,
			Err(_) => {
				// Initialize empty stuff if file is empty
				*memory.decl_get_mut() = DeclarationMap::new();
				*memory.kv_records_get_mut() = Vec::new();
				*memory.decl_records_get_mut() = HashMap::new();

				return Ok(())
			}
		};
		debug!("{:?}", header);

		// TODO
		// For some reason, reading the header reads 40 bytes, while writing 36 bytes(u32 + u128*2)
		// Changed u128 to u64, for some reason it reads 24 instead of 20
		self.reader.seek(SeekFrom::Start(20))?;

		// lu map<[u8], (records start, records length, records count)>
		let (lu_map, declarations) = if header.decl_len > 0 {
			self.read_declarations(header.decl_len).expect("[Reading declarations] Failed to declarations")
		} else {
			(HashMap::new(), DeclarationMap::new())
		};

		let kv_records = if header.records_len > 0 {
			self.read_kv_records(header.records_len).expect("[Reading KV records] Failed to read KV records")
		} else {
			Vec::new()
		};

		let decl_records = if header.decl_records_len > 0 {
			let mut gr = DeclarationRecords::new();

			for (name, _) in declarations.iter() {
				// rstart, rlen, rcount
				let (rstart, rlen, _) = lu_map[name];
				self.reader.seek(SeekFrom::Start(rstart))?;
				let records = self.read_decl_records(rlen).expect("[Reading decl records] Failed to read decl records");

				gr.insert(name.to_vec(), records);
			}

			gr
		} else {
			let mut hm = HashMap::new();

			for key in declarations.keys() {
				hm.insert(key.to_vec(), Vec::new());
			}

			hm
		};


		debug!("Loaded {} Declarations, {} KV records, {} Declared records in {:?}",
			declarations.len(), 
			kv_records.len(), 
			decl_records.values().map(|v| v.len()).sum::<usize>(),
			time.elapsed());
		
		*memory.decl_get_mut() = declarations;
		*memory.kv_records_get_mut() = kv_records;
		*memory.decl_records_get_mut() = decl_records;

		Ok(())
	}

	pub fn read_header(&mut self) -> std::io::Result<Header> {
		let header: Header = {
			let mut hbuf = [0u8; std::mem::size_of::<Header>()];
			self.reader.read_exact(&mut hbuf)?;
			deserialize(&hbuf)
		};

		Ok(header)
	}

	pub fn read_declarations(&mut self, len: u32) -> std::io::Result<(HashMap<Vec<u8>, (u64, u64, u64)>, DeclarationMap)> {
		let mut dbuf = vec![0u8; len as usize];
		self.reader.read_exact(&mut dbuf).expect("[Reading declarations]");
		debug!("[Reading declarations] Read {}/{}", dbuf.len(), len);
		Ok(parser::decl::extract_decls(&dbuf))
	}

	pub fn read_kv_records(&mut self, len: u64) -> std::io::Result<Vec<OwnedMemoryRecord>> {
		let mut dbuf = vec![0u8; len as usize];
		self.reader.read_exact(&mut dbuf)?;
		debug!("[Reading kv records] Read {}/{}", dbuf.len(), len);
		Ok(parser::kv::extract(&dbuf))
	}

	pub fn read_decl_records(&mut self, len: u64) -> std::io::Result<Vec<DeclarationRecord>> {
		let mut dbuf = vec![0u8; len as usize];
		self.reader.read_exact(&mut dbuf).expect("[Reading Decl records]");
		debug!("[Reading decl records] Read {}/{}", dbuf.len(), len);
		Ok(parser::decl::records::extract(&dbuf))
	}
}