veryl_parser/
fragment_codec.rs

1//! ID remapping for serialized analyzer fragments.
2//!
3//! Global IDs aren't reproducible across runs, so fragments store local
4//! ones: counter IDs (`TokenId`/`TextId`) as per-file window offsets
5//! (rebased on decode), interned IDs (`StrId`/`PathId`) as dictionary
6//! indices (re-interned on decode). The ID types' custom serde consults a
7//! thread-local session; inactive means passthrough, and an out-of-window
8//! ID errors out so the fragment is treated as non-cacheable.
9
10use crate::resource_table::{self, PathId, StrId, TokenId};
11use crate::text_table::TextId;
12use serde::de::Error as DeError;
13use serde::ser::Error as SerError;
14use serde::{Deserialize, Deserializer, Serialize, Serializer};
15use std::cell::RefCell;
16use std::collections::HashMap;
17use std::path::PathBuf;
18
19/// Half-open ID window `(start, end]` matching the pre/post counter values
20/// around a single file's parse + pass1.
21#[derive(Clone, Copy, Debug, Default)]
22pub struct IdWindow {
23    pub start: usize,
24    pub end: usize,
25}
26
27impl IdWindow {
28    pub fn count(&self) -> usize {
29        self.end - self.start
30    }
31
32    // Analyzer-side ids layer a sentinel-0 wire shift on top of this (see
33    // veryl_analyzer::fragment_codec); parser ids have no 0 sentinel.
34    pub fn encode(&self, id: usize, what: &str) -> Result<u64, String> {
35        if id > self.start && id <= self.end {
36            Ok((id - self.start - 1) as u64)
37        } else {
38            Err(format!(
39                "fragment codec: {what} {id} outside window ({}, {}]",
40                self.start, self.end
41            ))
42        }
43    }
44}
45
46/// Rebases decoded local IDs onto a freshly reserved counter range.
47#[derive(Clone, Copy, Debug, Default)]
48pub struct IdRebase {
49    pub base: usize,
50    pub count: usize,
51}
52
53impl IdRebase {
54    pub fn decode(&self, local: u64, what: &str) -> Result<usize, String> {
55        let local = local as usize;
56        if local < self.count {
57            Ok(self.base + local + 1)
58        } else {
59            Err(format!(
60                "fragment codec: {what} local id {local} out of range (count {})",
61                self.count
62            ))
63        }
64    }
65}
66
67#[derive(Default)]
68pub struct EncodeSession {
69    str_map: HashMap<StrId, u32>,
70    str_dict: Vec<String>,
71    path_map: HashMap<PathId, u32>,
72    path_dict: Vec<PathBuf>,
73    token_window: IdWindow,
74    text_window: IdWindow,
75}
76
77impl EncodeSession {
78    pub fn new(token_window: IdWindow, text_window: IdWindow) -> Self {
79        Self {
80            token_window,
81            text_window,
82            ..Default::default()
83        }
84    }
85
86    fn encode_str(&mut self, id: StrId) -> Result<u64, String> {
87        if let Some(x) = self.str_map.get(&id) {
88            return Ok(*x as u64);
89        }
90        let value = resource_table::get_str_value(id)
91            .ok_or_else(|| format!("fragment codec: unknown StrId {}", id.0))?;
92        let local = self.str_dict.len() as u32;
93        self.str_dict.push(value);
94        self.str_map.insert(id, local);
95        Ok(local as u64)
96    }
97
98    fn encode_path(&mut self, id: PathId) -> Result<u64, String> {
99        if let Some(x) = self.path_map.get(&id) {
100            return Ok(*x as u64);
101        }
102        let value = resource_table::get_path_value(id)
103            .ok_or_else(|| format!("fragment codec: unknown PathId {}", id.0))?;
104        let local = self.path_dict.len() as u32;
105        self.path_dict.push(value);
106        self.path_map.insert(id, local);
107        Ok(local as u64)
108    }
109}
110
111/// Dictionaries produced by an encode session. Stored alongside the encoded
112/// payload and used to seed the decode session.
113pub struct EncodeDicts {
114    pub strings: Vec<String>,
115    pub paths: Vec<PathBuf>,
116}
117
118pub struct DecodeSession {
119    strs: Vec<StrId>,
120    paths: Vec<PathId>,
121    token_rebase: IdRebase,
122    text_rebase: IdRebase,
123}
124
125impl DecodeSession {
126    /// Re-interns the dictionaries into the live tables and prepares rebases.
127    /// The caller must have reserved `token_rebase`/`text_rebase` ranges.
128    pub fn new(
129        strings: &[String],
130        paths: &[PathBuf],
131        token_rebase: IdRebase,
132        text_rebase: IdRebase,
133    ) -> Self {
134        let strs = strings
135            .iter()
136            .map(|x| resource_table::insert_str(x))
137            .collect();
138        let paths = paths
139            .iter()
140            .map(|x| resource_table::insert_path(x))
141            .collect();
142        Self {
143            strs,
144            paths,
145            token_rebase,
146            text_rebase,
147        }
148    }
149
150    fn decode_str(&self, local: u64) -> Result<StrId, String> {
151        self.strs
152            .get(local as usize)
153            .copied()
154            .ok_or_else(|| format!("fragment codec: StrId index {local} out of dictionary"))
155    }
156
157    fn decode_path(&self, local: u64) -> Result<PathId, String> {
158        self.paths
159            .get(local as usize)
160            .copied()
161            .ok_or_else(|| format!("fragment codec: PathId index {local} out of dictionary"))
162    }
163}
164
165thread_local!(static ENCODE: RefCell<Option<EncodeSession>> = const { RefCell::new(None) });
166thread_local!(static DECODE: RefCell<Option<DecodeSession>> = const { RefCell::new(None) });
167
168pub fn begin_encode(session: EncodeSession) {
169    ENCODE.with(|f| *f.borrow_mut() = Some(session));
170}
171
172/// Ends the encode session and returns the collected dictionaries.
173pub fn end_encode() -> Option<EncodeDicts> {
174    ENCODE.with(|f| {
175        f.borrow_mut().take().map(|x| EncodeDicts {
176            strings: x.str_dict,
177            paths: x.path_dict,
178        })
179    })
180}
181
182pub fn begin_decode(session: DecodeSession) {
183    DECODE.with(|f| *f.borrow_mut() = Some(session));
184}
185
186pub fn end_decode() {
187    DECODE.with(|f| *f.borrow_mut() = None);
188}
189
190fn with_encode<R>(f: impl FnOnce(Option<&mut EncodeSession>) -> R) -> R {
191    ENCODE.with(|x| f(x.borrow_mut().as_mut()))
192}
193
194fn with_decode<R>(f: impl FnOnce(Option<&DecodeSession>) -> R) -> R {
195    DECODE.with(|x| f(x.borrow().as_ref()))
196}
197
198impl Serialize for StrId {
199    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
200        let value = with_encode(|session| match session {
201            Some(session) => session.encode_str(*self),
202            None => Ok(self.0 as u64),
203        })
204        .map_err(S::Error::custom)?;
205        serializer.serialize_u64(value)
206    }
207}
208
209impl<'de> Deserialize<'de> for StrId {
210    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
211        let value = u64::deserialize(deserializer)?;
212        with_decode(|session| match session {
213            Some(session) => session.decode_str(value),
214            None => Ok(StrId(value as usize)),
215        })
216        .map_err(D::Error::custom)
217    }
218}
219
220impl Serialize for PathId {
221    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
222        let value = with_encode(|session| match session {
223            Some(session) => session.encode_path(*self),
224            None => Ok(self.0 as u64),
225        })
226        .map_err(S::Error::custom)?;
227        serializer.serialize_u64(value)
228    }
229}
230
231impl<'de> Deserialize<'de> for PathId {
232    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
233        let value = u64::deserialize(deserializer)?;
234        with_decode(|session| match session {
235            Some(session) => session.decode_path(value),
236            None => Ok(PathId(value as usize)),
237        })
238        .map_err(D::Error::custom)
239    }
240}
241
242impl Serialize for TokenId {
243    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
244        let value = with_encode(|session| match session {
245            Some(session) => session.token_window.encode(self.0, "TokenId"),
246            None => Ok(self.0 as u64),
247        })
248        .map_err(S::Error::custom)?;
249        serializer.serialize_u64(value)
250    }
251}
252
253impl<'de> Deserialize<'de> for TokenId {
254    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
255        let value = u64::deserialize(deserializer)?;
256        with_decode(|session| match session {
257            Some(session) => session.token_rebase.decode(value, "TokenId").map(TokenId),
258            None => Ok(TokenId(value as usize)),
259        })
260        .map_err(D::Error::custom)
261    }
262}
263
264impl Serialize for TextId {
265    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
266        let value = with_encode(|session| match session {
267            Some(session) => session.text_window.encode(self.0, "TextId"),
268            None => Ok(self.0 as u64),
269        })
270        .map_err(S::Error::custom)?;
271        serializer.serialize_u64(value)
272    }
273}
274
275impl<'de> Deserialize<'de> for TextId {
276    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
277        let value = u64::deserialize(deserializer)?;
278        with_decode(|session| match session {
279            Some(session) => session.text_rebase.decode(value, "TextId").map(TextId),
280            None => Ok(TextId(value as usize)),
281        })
282        .map_err(D::Error::custom)
283    }
284}
285
286#[cfg(test)]
287mod tests {
288    use super::*;
289    use crate::text_table::{self, TextInfo};
290    use crate::veryl_token::{Token, TokenSource};
291    use std::path::Path;
292
293    #[test]
294    fn token_roundtrip_rebases_ids() {
295        let token_start = resource_table::peek_token_id();
296        let text_start = text_table::peek_text_id();
297
298        let path = resource_table::insert_path(Path::new("roundtrip.veryl"));
299        let text = text_table::set_current_text(TextInfo {
300            text: "module A {}".to_string(),
301            path,
302        });
303        let source = TokenSource::File { path, text };
304        let a = Token::new("alpha", 1, 2, 5, 0, source);
305        let b = Token::new("beta", 2, 3, 4, 10, source);
306
307        let token_end = resource_table::peek_token_id();
308        let text_end = text_table::peek_text_id();
309        let token_count = token_end - token_start;
310        let text_count = text_end - text_start;
311
312        begin_encode(EncodeSession::new(
313            IdWindow {
314                start: token_start,
315                end: token_end,
316            },
317            IdWindow {
318                start: text_start,
319                end: text_end,
320            },
321        ));
322        let bytes = postcard::to_allocvec(&(a, b)).unwrap();
323        let dicts = end_encode().unwrap();
324        assert!(dicts.strings.contains(&"alpha".to_string()));
325        assert!(
326            dicts
327                .paths
328                .contains(&Path::new("roundtrip.veryl").to_path_buf())
329        );
330
331        let token_base = resource_table::reserve_token_ids(token_count);
332        let text_base = text_table::reserve_text_ids(text_count);
333        begin_decode(DecodeSession::new(
334            &dicts.strings,
335            &dicts.paths,
336            IdRebase {
337                base: token_base,
338                count: token_count,
339            },
340            IdRebase {
341                base: text_base,
342                count: text_count,
343            },
344        ));
345        let (a2, b2): (Token, Token) = postcard::from_bytes(&bytes).unwrap();
346        end_decode();
347
348        // IDs are rebased onto the reserved range, preserving relative order
349        assert_eq!(a2.id.0, token_base + (a.id.0 - token_start));
350        assert_eq!(b2.id.0, token_base + (b.id.0 - token_start));
351        assert!(a2.id < b2.id);
352        // interned values survive the roundtrip
353        assert_eq!(a2.text, resource_table::insert_str("alpha"));
354        assert_eq!(b2.text, resource_table::insert_str("beta"));
355        assert_eq!((a2.line, a2.column, a2.length, a2.pos), (1, 2, 5, 0));
356        match a2.source {
357            TokenSource::File { path, text } => {
358                assert_eq!(
359                    resource_table::get_path_value(path).unwrap(),
360                    Path::new("roundtrip.veryl").to_path_buf()
361                );
362                assert_eq!(text.0, text_base + 1);
363            }
364            _ => panic!("source kind changed"),
365        }
366    }
367
368    #[test]
369    fn out_of_window_token_fails_encode() {
370        let outside = Token::new("x", 0, 0, 1, 0, TokenSource::External);
371        let start = resource_table::peek_token_id();
372        let _inside = Token::new("y", 0, 0, 1, 0, TokenSource::External);
373        let end = resource_table::peek_token_id();
374
375        begin_encode(EncodeSession::new(
376            IdWindow { start, end },
377            IdWindow::default(),
378        ));
379        let result = postcard::to_allocvec(&outside);
380        end_encode();
381        assert!(result.is_err());
382    }
383
384    #[test]
385    fn passthrough_without_session() {
386        let bytes = postcard::to_allocvec(&TokenId(42)).unwrap();
387        let id: TokenId = postcard::from_bytes(&bytes).unwrap();
388        assert_eq!(id, TokenId(42));
389    }
390}
veryl_parser/fragment_codec.rs

veryl_parser/
fragment_codec.rs
