kmerutils/redisbase.rs
1//! key definitions and trait implementation for communicating with redis
2
3// TODO
4// SliceAnchorValueRedis and SliceAnchorKeyRedis MinhashKeyRedis should not have redis in their name
5// as the mechanism would be the same with any KV store RocksDB/TiKV
6// Only the trait implementation ToRedisArgs and FromRedisValue are specific to redis
7
8extern crate redis;
9use self::redis::{FromRedisValue, RedisWrite, ToRedisArgs};
10
11use std::str;
12
13use log::trace;
14
15// REDIS on port 6379 dedicated to (my) genomic processing
16// for key "properties"
17// "prop:user" value string for user name
18// "prop:fn" value string fastafile processed
19// "prop:fn:process" value string "anchor:minhash"
20// "prop:fn:process:nb_bases" value int
21// "prop:fn:process:slice_size" value int
22// "prop:fn:process:pos:readnum:slicepos" values : list of (kmer_value, kmer_count):
23
24// trait ToString implemented for u64 and u8 enable easy generation of string for this last key value as it
25// is implemented for T: fmt::Display + ?Sized and
26// For the inverse parse uses FromStr (Cf rust documentation)
27// String to u64 str.parse::<u64>()
28
29pub const FN_KEY: &str = "prop:fn";
30pub const PROCESS_KEY: &str = "prop:fn:process";
31pub const NB_BASES_KEY: &str = "prop:fn:process:bases";
32pub const SLICE_SIZE_KEY: &str = "prop:fn:process:ssize";
33pub const POS_KEY: &str = "prop:fn:process:readnum:slicepos";
34
35// for inverse indexing
36
37pub const MINHASH_1: &str = "prop:fn:process:minhash_1";
38pub const MINHASH_2: &str = "prop:fn:process:minhash_2";
39
40pub const SLICE_ANCHOR_KEY: &str = "prop:fn:process:ssize:bases:readnum:slicepos";
41
42// internal structure we decode from Redis via FromRedisValue to get value part of SliceAnchor
43// This enables getting the value for a given key.
44//
45
46#[derive(Debug)]
47pub struct SliceAnchorValueRedis {
48 pub hk_count: Vec<(u64, u8)>,
49}
50
51// beware each basic write_redis_args push a new vec<u8> coming fromm a call to into_bytes.
52impl redis::ToRedisArgs for SliceAnchorValueRedis {
53 fn write_redis_args<W: RedisWrite + ?Sized>(&self, out: &mut W) {
54 // minhash is a vector of ItemHash (u64) and count (u8)
55 let mut key: Vec<u8> = Vec::new();
56 let nb_kmer = self.hk_count.len();
57 for i in 0..nb_kmer {
58 if i > 0 {
59 key.push(b':');
60 }
61 key.extend_from_slice(self.hk_count[i].0.to_string().as_bytes());
62 key.push(b',');
63 key.extend_from_slice(self.hk_count[i].1.to_string().as_bytes());
64 if i < nb_kmer {
65 key.push(b':');
66 }
67 }
68 trace!(
69 "SliceAnchorValueRedis encoded sliceanchor for redis (kmer/count {:?}",
70 &key
71 );
72 out.write_arg(&key);
73 } // end of write_redis_args
74} // end impl redis::ToRedisArgs for SliceAnchorValueRedis
75
76// Given a redis Value, generate a vector of (invhashkmer, count)
77// corresponding to (InvHashedKmer<T>, u8)
78// We next have to go from u64 to InvHashedKmer<T> knowing the number of bases
79
80impl FromRedisValue for SliceAnchorValueRedis {
81 fn from_redis_value(v: &redis::Value) -> redis::RedisResult<Self> {
82 let mut retvec = Vec::<(u64, u8)>::new();
83 //
84 match v {
85 redis::Value::BulkString(bytes) => {
86 // We must be in this case as we encode all values in one flat string in ToRedisArgs for SliceAnchorValueRedis.
87 // We should not get a Value::Bulk
88 // now we must decode vecu8 byte by byte. Inverse of write_redis_args
89 let res = str::from_utf8(bytes);
90 if res.is_err() {
91 Err(redis::RedisError::from((
92 redis::ErrorKind::TypeError,
93 "Not a String",
94 )))
95 } else {
96 let str = res.unwrap();
97 let vcouple: Vec<&str> = str.split(':').collect(); // we get in v a vector of "kmer,count"
98 for couple in vcouple {
99 let vterms: Vec<&str> = couple.split(',').collect(); // we get in vterms[0] kmer, in vterm[1] count as strings
100 // convert from strings to u64 and u8
101 let resu64 = vterms[0].parse::<u64>();
102 let resu8 = vterms[1].parse::<u8>();
103 if resu64.is_err() || resu8.is_err() {
104 return Err(redis::RedisError::from((
105 redis::ErrorKind::TypeError,
106 "Cannot decode kmer or count",
107 )));
108 } else {
109 // everything is fine
110 let kmer_h = resu64.unwrap();
111 let count = resu8.unwrap();
112 retvec.push((kmer_h, count));
113 trace!(
114 "SliceAnchorValueRedis FromRedisValue pushing (kmer/count {:?} {:?}",
115 &kmer_h, &count
116 );
117 }
118 } // end of for in couple
119 Ok(SliceAnchorValueRedis { hk_count: retvec })
120 } // end else
121 } // case ref bytes
122 // else return error
123 _ => Err(redis::RedisError::from((
124 redis::ErrorKind::TypeError,
125 "Not a Vec<u8>",
126 ))),
127 } // end match
128 } // end fn decode_minhash_from_redis_value(v: &redis::Value)
129} // end impl FromRedisValue for SliceAnchorValueRedis
130
131// Structure for generating KEYS, implements ToRedisArgs
132
133#[derive(Clone, Debug)]
134pub struct SliceAnchorKeyRedis {
135 pub filename: String,
136 pub process: String,
137 pub slice_size: u32,
138 pub nb_bases: u16,
139 pub readnum: u32,
140 pub slicepos: u32,
141}
142
143impl ToRedisArgs for SliceAnchorKeyRedis {
144 // we concatenate all fields of SliceAnchorKeyRedis as string and return corresponding Vec<u8>
145 fn write_redis_args<W: RedisWrite + ?Sized>(&self, out: &mut W) {
146 let mut key: Vec<u8> = Vec::new();
147 // This suppose that filename is a valid utf-8 but filename are !? CAVEAT
148 key.extend_from_slice(self.filename.as_bytes());
149 key.push(b':');
150 key.extend_from_slice(self.process.as_bytes());
151 key.push(b':');
152 // transfer other numerical values as [u8]
153 key.extend_from_slice(self.slice_size.to_string().as_bytes());
154 key.push(b':');
155 key.extend_from_slice(self.nb_bases.to_string().as_bytes());
156 key.push(b':');
157 key.extend_from_slice(self.readnum.to_string().as_bytes());
158 key.push(b':');
159 key.extend_from_slice(self.slicepos.to_string().as_bytes());
160 //
161 out.write_arg(&key);
162 } // end of write_redis_args
163} // end of impl ToRedisArgs for SliceAnchorKeyRedis
164
165//=================================================================================
166// For reverse request. Which read have a slice with a given minhash.
167//=================================================================================
168
169#[derive(Clone, Debug)]
170pub struct MinhashKeyRedis {
171 pub filename: String,
172 pub process: String,
173 pub slice_size: u32,
174 pub nb_bases: u16,
175 pub minhash_val: u64,
176}
177
178impl ToRedisArgs for MinhashKeyRedis {
179 // we concatenate all fields of SliceAnchorKeyRedis as string and return corresponding Vec<u8>
180 fn write_redis_args<W: RedisWrite + ?Sized>(&self, out: &mut W) {
181 let mut key: Vec<u8> = Vec::new();
182 // This suppose that filename is a valid utf-8 but filename are !? CAVEAT
183 key.extend_from_slice(self.filename.as_bytes());
184 key.push(b':');
185 key.extend_from_slice(self.process.as_bytes());
186 key.push(b':');
187 // transfer other numerical values as [u8]
188 key.extend_from_slice(self.slice_size.to_string().as_bytes());
189 key.push(b':');
190 key.extend_from_slice(self.nb_bases.to_string().as_bytes());
191 key.push(b':');
192 key.extend_from_slice(self.minhash_val.to_string().as_bytes());
193 //
194 out.write_arg(&key);
195 } // end of write_redis_args
196} // end of impl ToRedisArgs for MinhashKeyRedis