1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
use std::fmt::Debug;
use std::fs::{File, OpenOptions};
use std::hash::Hash;
use std::io::{self, BufRead, Seek, SeekFrom, Write};
use std::path::PathBuf;
use std::sync::{Arc, Mutex};

use bincode::deserialize_from;
use hashbrown::HashMap;
use log::LevelFilter;
use rayon::prelude::*;
use serde::de::DeserializeOwned;
use serde::Serialize;
use simple_logger::SimpleLogger;
use time::macros::format_description;

use crate::types::BinaryKv;

#[derive(Debug, Clone)]
pub struct QuickConfiguration
{
    pub path: Option<PathBuf>,
    pub logs: bool,
    pub log_level: Option<LevelFilter>,
}

impl QuickConfiguration
{
    pub fn new(path: Option<PathBuf>, logs: bool, log_level: Option<LevelFilter>) -> Self
    {
        Self { path, logs, log_level }
    }
}

impl Default for QuickConfiguration
{
    fn default() -> Self
    {
        Self {
            path: Some(PathBuf::from("db.qkv")),
            logs: false,
            log_level: Some(LevelFilter::Info),
        }
    }
}

/// The default and recommended client to use. It is optimized for a specific schema and has multi-threading enabled by default.
///
/// It allows you to define a schema for your data, which will be used to serialize and deserialize
/// your data. The benefit is all operations are optimized for your data type, it also makes typings
/// easier to work with. Use this client when you want to work with data-modules that you have
/// defined. The mini client is good for storing generic data that could change frequently.
///
/// # Example
/// ```rust
/// use std::path::PathBuf;
///
/// use quick_kv::prelude::*;
/// use serde::{Deserialize, Serialize};
///
/// #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Hash, Clone)]
/// struct User
/// {
///     name: String,
///     age: u8,
/// }
///
/// let config = QuickConfiguration::new(Some(PathBuf::from("db.qkv")), true, None);
///
/// let mut client = QuickClient::<User>::new(Some(config)).unwrap();
///
/// let user = User {
///     name: "John".to_string(),
///     age: 20,
/// };
///
/// client.set("user", user.clone()).unwrap();
///
/// let user_from_db = client.get("user").unwrap().unwrap();
///
/// assert_eq!(user, user_from_db);
/// ```
#[cfg(feature = "full")]
#[derive(Debug, Clone)]
pub struct QuickClient<T>
where
    T: Serialize + DeserializeOwned + Clone + Debug + Eq + PartialEq + Hash + Send + Sync,
{
    pub file: Arc<Mutex<File>>,
    pub cache: Arc<Mutex<HashMap<String, BinaryKv<T>>>>,
    pub config: QuickConfiguration,
}

impl<T> QuickClient<T>
where
    T: Serialize + DeserializeOwned + Clone + Debug + Eq + PartialEq + Hash + Send + Sync,
{
    pub fn new(config: Option<QuickConfiguration>) -> std::io::Result<Self>
    {
        let config = match config {
            Some(config) => config,
            None => QuickConfiguration::default(),
        };

        if config.clone().logs {
            let log_level = match config.clone().log_level {
                Some(log_level) => log_level,
                None => QuickConfiguration::default().log_level.unwrap(),
            };
            SimpleLogger::new()
                .with_colors(true)
                .with_threads(true)
                .with_level(log_level)
                .with_timestamp_format(format_description!("[year]-[month]-[day] [hour]:[minute]:[second]"))
                .init()
                .unwrap();
        }

        let file = match OpenOptions::new()
            .read(true)
            .write(true)
            .create(true)
            .open(&config.clone().path.unwrap())
        {
            Ok(file) => file,
            Err(e) => {
                return Err(io::Error::new(io::ErrorKind::Other, format!("Error opening file: {:?}", e)));
            }
        };

        log::info!("QuickSchemaClient Initialized!");

        Ok(Self {
            file: Arc::new(Mutex::new(file)),
            cache: Arc::new(Mutex::new(HashMap::new())),
            config,
        })
    }

    pub fn get(&mut self, key: &str) -> std::io::Result<Option<T>>
    where
        T: Clone,
    {
        log::info!("[GET] Searching for key: {}", key);

        // Check if the key is in the cache first
        {
            let cache = self.cache.lock().unwrap();
            if let Some(entry) = cache.get(key) {
                log::debug!("[GET] Found cached key: {}", key);
                return Ok(Some(entry.value.clone()));
            }
        }

        // If not in the cache, lock the file for reading
        let mut file = match self.file.lock() {
            Ok(file) => file,
            Err(e) => {
                return Err(io::Error::new(io::ErrorKind::Other, format!("Error locking file: {:?}", e)));
            }
        };

        let mut reader = io::BufReader::new(&mut *file);

        // Set the position if the reader
        reader.seek(SeekFrom::Start(0))?;

        let key_clone = key.to_string();

        // Read and deserialize entries in parallel until the end of the file is reached
        let result = reader
            .lines()
            .par_bridge()
            .filter_map(|line| {
                if let Ok(line) = line {
                    let mut line_reader = io::Cursor::new(line);
                    match deserialize_from::<_, BinaryKv<T>>(&mut line_reader) {
                        Ok(BinaryKv { key: entry_key, value }) if key == entry_key => {
                            // Cache the deserialized entry
                            self.cache
                                .lock()
                                .unwrap()
                                .insert(key_clone.clone(), BinaryKv::new(key_clone.clone(), value.clone()));
                            log::debug!("[GET] Caching uncached key: {}", key_clone);

                            log::debug!("[GET] Found key: {}", key_clone);
                            Some(value)
                        }
                        Err(e) => {
                            if let bincode::ErrorKind::Io(io_err) = e.as_ref() {
                                if io_err.kind() == io::ErrorKind::UnexpectedEof {
                                    // Reached the end of the serialized data
                                    None
                                } else {
                                    None
                                }
                            } else {
                                None
                            }
                        }
                        _ => None,
                    }
                } else {
                    None
                }
            })
            .collect::<Vec<T>>();

        if result.is_empty() {
            log::debug!("[GET] Key not found: {}", key);
            return Ok(None);
        }

        log::info!("[GET] Key found: {}", key);

        Ok(Some(result[0].clone()))
    }

    pub fn set(&mut self, key: &str, value: T) -> std::io::Result<()>
    {
        log::info!("[SET] Setting key: {}", key);

        // First check if the data already exists; if so, update it instead
        {
            if self.cache.lock().unwrap().get(key).is_some() {
                log::debug!("[SET] Key already exists, updating {} instead", key);
                return self.update(key, value);
            }
        }

        let mut file = match self.file.lock() {
            Ok(file) => file,
            Err(e) => {
                return Err(io::Error::new(io::ErrorKind::Other, format!("Error locking file: {:?}", e)));
            }
        };

        let mut writer = io::BufWriter::new(&mut *file);

        let data = BinaryKv::new(key.to_string(), value.clone());
        // Serialize the data in parallel and wait for it to complete
        let serialized = match bincode::serialize(&data) {
            Ok(data) => data,
            Err(e) => {
                return Err(io::Error::new(
                    io::ErrorKind::Other,
                    format!("Error serializing data: {:?}", e),
                ));
            }
        };

        // Write the serialized data to the file
        writer.write_all(&serialized)?;
        writer.get_ref().sync_all()?;

        self.cache
            .lock()
            .unwrap()
            .insert(key.to_string(), BinaryKv::new(key.to_string(), value.clone()));

        log::info!("[SET] Key set: {}", key);

        Ok(())
    }

    pub fn delete(&mut self, key: &str) -> std::io::Result<()>
    {
        log::info!("[DELETE] Deleting key: {}", key);

        // If the key is not in the cache, dont do anything as it doesn't exist on the file.
        {
            if self.cache.lock().unwrap().remove(key).is_none() {
                return Ok(());
            }
        }

        let mut file = match self.file.lock() {
            Ok(file) => file,
            Err(e) => {
                return Err(io::Error::new(io::ErrorKind::Other, format!("Error locking file: {:?}", e)));
            }
        };

        let mut reader = io::BufReader::new(&mut *file);

        // Create a temporary buffer to store the updated data
        let mut updated_buffer = Vec::new();

        // Read and process entries
        loop {
            match deserialize_from::<_, BinaryKv<T>>(&mut reader) {
                Ok(BinaryKv { key: entry_key, .. }) if key != entry_key => {
                    // Keep entries that don't match the key
                    updated_buffer.extend_from_slice(reader.buffer());
                }
                Ok(_) => {
                    // Skip entries that match the key
                }
                Err(e) => {
                    if let bincode::ErrorKind::Io(io_err) = e.as_ref() {
                        if io_err.kind() == io::ErrorKind::UnexpectedEof {
                            // Reached the end of the serialized data
                            break;
                        }
                    }
                }
            }
        }

        // Close the file and open it in write mode for writing
        drop(reader); // Release the reader

        let mut writer = io::BufWriter::new(&mut *file);

        // Truncate the file and write the updated data back
        writer.seek(SeekFrom::Start(0))?;
        writer.write_all(&updated_buffer)?;
        writer.get_ref().sync_all()?;

        self.cache.lock().unwrap().remove(key);
        log::debug!("[DELETE] Cache deleted: {}", key);

        log::info!("[DELETE] Key deleted: {}", key);

        Ok(())
    }

    pub fn update(&mut self, key: &str, value: T) -> std::io::Result<()>
    {
        log::info!("[UPDATE] Updating key: {}", key);

        {
            if self.cache.lock().unwrap().get(key).is_none() {
                log::debug!("[UPDATE] Key not found, attempting to set {} instead", key);
                return self.set(key, value);
            };
        }

        let mut file = match self.file.lock() {
            Ok(file) => file,
            Err(e) => {
                return Err(io::Error::new(io::ErrorKind::Other, format!("Error locking file: {:?}", e)));
            }
        };

        let mut reader = io::BufReader::new(&mut *file);

        reader.seek(SeekFrom::Start(0))?;

        let mut updated_entries = Vec::new();
        let mut updated = false;

        // Read and process entries
        loop {
            match deserialize_from::<_, BinaryKv<T>>(&mut reader) {
                Ok(entry) => {
                    if key == entry.key {
                        // Update the value associated with the key
                        let mut updated_entry = entry.clone();
                        updated_entry.value = value.clone();
                        updated_entries.push(updated_entry);
                        updated = true;
                    } else {
                        updated_entries.push(entry);
                    }
                }
                Err(e) => {
                    if let bincode::ErrorKind::Io(io_err) = e.as_ref() {
                        if io_err.kind() == io::ErrorKind::UnexpectedEof {
                            // Reached the end of the serialized data
                            break;
                        }
                    }
                }
            }
        }

        if !updated {
            log::warn!(
                "[UPDATE] Key not found: {}. This should not trigger, if it did some cache may be invalid.",
                key
            );
            // Key not found
            return Err(io::Error::new(io::ErrorKind::Other, format!("Key not found: {}", key)));
        }

        // Close the file and open it in write mode
        drop(reader); // Release the reader

        // Reopen the file in write mode for writing
        let mut writer = io::BufWriter::new(&mut *file);

        // Truncate the file and write the updated data back
        writer.seek(SeekFrom::Start(0))?;
        for entry in updated_entries.iter() {
            let serialized = match bincode::serialize(entry) {
                Ok(data) => data,
                Err(e) => {
                    return Err(io::Error::new(
                        io::ErrorKind::Other,
                        format!("Error serializing data: {:?}", e),
                    ));
                }
            };
            writer.write_all(&serialized)?;
        }

        writer.get_ref().sync_all()?;

        // Update the cache
        self.cache
            .lock()
            .unwrap()
            .insert(key.to_string(), BinaryKv::new(key.to_string(), value.clone()));
        log::debug!("[UPDATE] Cache updated: {}", key);

        log::info!("[UPDATE] Key updated: {}", key);

        Ok(())
    }

    pub fn clear(&mut self) -> std::io::Result<()>
    {
        log::info!("[CLEAR] Clearing database");

        let mut file = match self.file.lock() {
            Ok(file) => file,
            Err(e) => {
                return Err(io::Error::new(io::ErrorKind::Other, format!("Error locking file: {:?}", e)));
            }
        };

        let mut writer = io::BufWriter::new(&mut *file);

        writer.get_mut().set_len(0)?;
        writer.seek(SeekFrom::Start(0))?;
        writer.get_ref().sync_all()?;

        self.cache.lock().unwrap().clear();
        log::debug!("[CLEAR] Cache cleared");

        log::info!("[CLEAR] Database cleared");

        Ok(())
    }

    pub fn get_all(&mut self) -> std::io::Result<Vec<BinaryKv<T>>>
    {
        log::info!("[GET_ALL] Fetching all data in db cache...");

        let cache = &self.cache.lock().unwrap();

        let all_results: Vec<BinaryKv<T>> = cache
            .par_iter() // Parallelize the iteration over key-value pairs
            .map(|(_, entry)| entry.clone()) // Clone each entry in parallel
            .collect();

        log::info!("[GET_ALL] Fetched all data in db");

        Ok(all_results)
    }

    pub fn get_many(&mut self, keys: Vec<String>) -> std::io::Result<Vec<BinaryKv<T>>>
    {
        log::info!("[GET_MANY] Fetching many keys from db cache...");

        let cache_guard = self.cache.lock().unwrap();

        let results: Vec<BinaryKv<T>> = keys
            .par_iter() // Parallelize the iteration over keys
            .filter_map(|key| cache_guard.get(key).cloned()) // Filter and clone entries in parallel
            .collect();

        log::info!("[GET_MANY] Fetched {} keys from db", results.len());

        Ok(results)
    }

    pub fn set_many(&mut self, values: Vec<BinaryKv<T>>) -> std::io::Result<()>
    {
        log::info!("[SET_MANY] Setting many keys in db...");

        // First check if the data already exist, if so, update it not set it again.
        // This will stop memory alloc errors.
        let mut to_update = Vec::new();

        {
            let cache_guard = self.cache.lock().unwrap();

            for entry in values.iter() {
                if cache_guard.get(&entry.key).is_some() {
                    to_update.push(entry.clone());
                }
            }
        }

        if !to_update.is_empty() {
            log::debug!(
                "[SET_MANY] Found {} keys that already exist, updating them instead of calling set",
                to_update.len()
            );
            self.update_many(to_update)?;
        }

        let mut file = match self.file.lock() {
            Ok(file) => file,
            Err(e) => {
                return Err(io::Error::new(io::ErrorKind::Other, format!("Error locking file: {:?}", e)));
            }
        };

        let mut writer = io::BufWriter::new(&mut *file);
        let mut serialized = Vec::new();

        for entry in values.iter() {
            serialized.push(BinaryKv::new(entry.key.clone(), entry.value.clone()))
        }

        log::debug!("[SET_MANY] Serialized {} keys", serialized.len());

        let serialized = match bincode::serialize(&serialized) {
            Ok(data) => data,
            Err(e) => {
                return Err(io::Error::new(
                    io::ErrorKind::Other,
                    format!("Error serializing data: {:?}", e),
                ));
            }
        };

        // Write the serialized data to the file
        writer.write_all(&serialized)?;
        writer.get_ref().sync_all()?;

        log::debug!("[SET_MANY] Wrote {} keys to file", serialized.len());

        {
            let mut cache_guard = self.cache.lock().unwrap();

            for entry in values.iter() {
                cache_guard.insert(entry.key.clone(), BinaryKv::new(entry.key.clone(), entry.value.clone()));
            }
        }

        log::info!("[SET_MANY] Set {} keys in db", values.len());

        Ok(())
    }

    pub fn delete_many(&mut self, keys: Vec<String>) -> std::io::Result<()>
    {
        log::info!("[DELETE_MANY] Deleting many keys from db...");

        {
            if self.cache.lock().unwrap().is_empty() {
                log::debug!("[DELETE_MANY] Cache is empty, nothing to delete");
                return Ok(());
            }
        }

        // First we check if any of the keys passed exist, before we search the file for them.
        let mut valid_keys = Vec::new();
        {
            let cache_guard = self.cache.lock().unwrap();

            for key in keys {
                if cache_guard.get(&key).is_some() {
                    valid_keys.push(key);
                }
            }
        }

        // Clone the valid_keys vector
        let vkc = valid_keys.clone();

        if valid_keys.is_empty() {
            log::debug!("[DELETE_MANY] No valid keys found, nothing to delete");
            return Ok(());
        }

        let mut file = match self.file.lock() {
            Ok(file) => file,
            Err(e) => {
                return Err(io::Error::new(io::ErrorKind::Other, format!("Error locking file: {:?}", e)));
            }
        };

        let mut reader = io::BufReader::new(&mut *file);

        // Create a temporary buffer to store the updated data
        let mut updated_buffer = Vec::new();

        // Read and process entries
        loop {
            match deserialize_from::<_, BinaryKv<T>>(&mut reader) {
                Ok(BinaryKv { key: entry_key, .. }) if valid_keys.contains(&entry_key) => {
                    // Keep entries that don't match the key
                    updated_buffer.extend_from_slice(reader.buffer());
                }
                Ok(_) => {
                    // Skip entries that match the key
                }
                Err(e) => {
                    if let bincode::ErrorKind::Io(io_err) = e.as_ref() {
                        if io_err.kind() == io::ErrorKind::UnexpectedEof {
                            // Reached the end of the serialized data
                            break;
                        }
                    }
                }
            }
        }

        // Close the file and open it in write mode for writing
        drop(reader); // Release the reader

        let mut writer = io::BufWriter::new(&mut *file);

        // Truncate the file and write the updated data back
        writer.seek(SeekFrom::Start(0))?;
        writer.write_all(&updated_buffer)?;
        writer.get_ref().sync_all()?;

        for key in valid_keys {
            self.cache.lock().unwrap().remove(&key);
        }

        log::info!("[DELETE_MANY] Deleted {} keys from db", vkc.len());

        Ok(())
    }

    pub fn update_many(&mut self, values: Vec<BinaryKv<T>>) -> std::io::Result<()>
    {
        log::info!("[UPDATE_MANY] Updating many keys in db...");

        let mut to_set = Vec::new();

        {
            let cache_guard = self.cache.lock().unwrap();

            for entry in values.iter() {
                if cache_guard.get(&entry.key).is_none() {
                    to_set.push(entry.clone());
                }
            }
        }

        if !to_set.is_empty() {
            log::debug!(
                "[UPDATE_MANY] Found {} keys that dont exist, setting them instead of calling update",
                to_set.len()
            );
            return self.set_many(to_set);
        }

        let mut file = match self.file.lock() {
            Ok(file) => file,
            Err(e) => {
                return Err(io::Error::new(io::ErrorKind::Other, format!("Error locking file: {:?}", e)));
            }
        };

        let mut reader = io::BufReader::new(&mut *file);

        reader.seek(SeekFrom::Start(0))?;

        let mut updated_entries = Vec::new();

        // Read and process entries
        loop {
            match deserialize_from::<_, BinaryKv<T>>(&mut reader) {
                Ok(entry) => {
                    if let Some(value) = values.iter().find(|v| v.key == entry.key) {
                        // Update the value associated with the key
                        let mut updated_entry = entry.clone();
                        updated_entry.value = value.value.clone();
                        updated_entries.push(updated_entry);
                    } else {
                        updated_entries.push(entry);
                    }
                }
                Err(e) => {
                    if let bincode::ErrorKind::Io(io_err) = e.as_ref() {
                        if io_err.kind() == io::ErrorKind::UnexpectedEof {
                            // Reached the end of the serialized data
                            break;
                        }
                    }
                }
            }
        }

        // Close the file and open it in write mode
        drop(reader); // Release the reader

        // Reopen the file in write mode for writing
        let mut writer = io::BufWriter::new(&mut *file);

        let mut serialized = Vec::new();

        for entry in updated_entries.iter() {
            serialized.push(BinaryKv::new(entry.key.clone(), entry.value.clone()))
        }

        let serialized = match bincode::serialize(&serialized) {
            Ok(data) => data,
            Err(e) => {
                return Err(io::Error::new(
                    io::ErrorKind::Other,
                    format!("Error serializing data: {:?}", e),
                ));
            }
        };

        log::debug!("[UPDATE_MANY] Serialized {} keys", serialized.len());

        // Truncate the file and write the updated data back
        writer.seek(SeekFrom::Start(0))?;
        writer.write_all(&serialized)?;
        writer.get_ref().sync_all()?;

        log::debug!("[UPDATE_MANY] Wrote {} keys to file", serialized.len());

        for entry in updated_entries.iter() {
            self.cache
                .lock()
                .unwrap()
                .insert(entry.key.clone(), BinaryKv::new(entry.key.clone(), entry.value.clone()));
        }

        log::info!("[UPDATE_MANY] Updated {} keys in db", values.len());

        Ok(())
    }
}