1use crate::context::{filter_by_min_level, ContextExtractor, Window};
45use crate::cooc::CooAccumulator;
46use crate::ppmi::{self, DEFAULT_FREQUENCY_FLOOR, DEFAULT_PPMI_THRESHOLD};
47use crate::store::{Store, TokenId};
48use crate::wheel::WheelState;
49use redb::{Database, ReadableDatabase, ReadableTable, ReadableTableMetadata, TableDefinition};
50use std::collections::{HashMap, HashSet};
51use std::fmt;
52use std::path::Path;
53
54pub const ROOT_WINDOW_REVOLUTIONS: u32 = 3;
56pub const WEIGHT_CUT_QUANTILE: f64 = 0.25;
58pub const ONE_CONTEXT_DIVERSITY: usize = 2;
62
63#[derive(Debug, Clone, Copy, PartialEq, Eq)]
64pub enum FocusEvent {
65 Focus { token: TokenId, revolution: u32 },
66 Unfocus { token: TokenId, revolution: u32 },
67}
68
69#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
73pub struct MarkOutcome {
74 pub newly_focused: usize,
75 pub newly_unfocused: usize,
76 pub unchanged: usize,
77}
78
79#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
81pub struct WeightCutOutcome {
82 pub borderline_count: usize,
83 pub cut_count: usize,
84}
85
86#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
88pub struct FocusConcept {
89 pub token_id: TokenId,
90 pub text: String,
91 pub level: u8,
92 pub ppmi_weight: f64,
93}
94
95#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
97pub struct FocusSet {
98 pub revolution: u32,
99 pub tokens: HashSet<TokenId>,
100 pub top_concepts: Vec<FocusConcept>,
101}
102
103impl FocusSet {
104 pub fn new(revolution: u32, tokens: HashSet<TokenId>, top_concepts: Vec<FocusConcept>) -> Self {
105 Self {
106 revolution,
107 tokens,
108 top_concepts,
109 }
110 }
111
112 pub fn empty() -> Self {
113 Self {
114 revolution: 0,
115 tokens: HashSet::new(),
116 top_concepts: Vec::new(),
117 }
118 }
119
120 #[inline]
121 pub fn contains(&self, id: TokenId) -> bool {
122 self.tokens.contains(&id)
123 }
124
125 #[inline]
126 pub fn len(&self) -> usize {
127 self.tokens.len()
128 }
129
130 #[inline]
131 pub fn is_empty(&self) -> bool {
132 self.tokens.is_empty()
133 }
134
135 pub fn tokens(&self) -> &HashSet<TokenId> {
136 &self.tokens
137 }
138
139 pub fn top_concepts(&self) -> &[FocusConcept] {
140 &self.top_concepts
141 }
142
143 pub fn revolution(&self) -> u32 {
144 self.revolution
145 }
146}
147
148pub fn root_set(wheel: &WheelState) -> HashSet<TokenId> {
152 let mut roots = HashSet::with_capacity(256 + wheel.last_touched_revolution.len());
153 let factory_count = 256.min(wheel.store.len() as TokenId);
154 for id in 0..factory_count {
155 if wheel.store.level_of(id) == 0 {
156 roots.insert(id);
157 }
158 }
159 for (&id, &last) in &wheel.last_touched_revolution {
160 if wheel.revolution.saturating_sub(last) < ROOT_WINDOW_REVOLUTIONS {
161 roots.insert(id);
162 }
163 }
164 roots
165}
166
167pub fn reachable_set(store: &Store, roots: &HashSet<TokenId>) -> HashSet<TokenId> {
169 let mut reachable: HashSet<TokenId> = HashSet::new();
170 let mut stack: Vec<TokenId> = roots.iter().copied().collect();
171 while let Some(id) = stack.pop() {
172 if !reachable.insert(id) {
173 continue;
174 }
175 let (level, children) = store.get(id);
176 if level > 0 {
177 for &c in children {
178 if !reachable.contains(&c) {
179 stack.push(c);
180 }
181 }
182 }
183 }
184 reachable
185}
186
187#[derive(Default)]
192pub struct DeathLog {
193 events: Vec<FocusEvent>,
194 focused: HashSet<TokenId>,
195}
196
197impl DeathLog {
198 pub fn new() -> Self {
199 Self::default()
200 }
201
202 pub fn from_parts(events: Vec<FocusEvent>, focused: HashSet<TokenId>) -> Self {
203 Self { events, focused }
204 }
205
206 pub fn mark(&mut self, wheel: &WheelState) -> MarkOutcome {
211 let roots = root_set(wheel);
212 let reachable = reachable_set(&wheel.store, &roots);
213 let mut outcome = MarkOutcome::default();
214
215 let mut newly_focused: Vec<TokenId> = reachable
216 .iter()
217 .copied()
218 .filter(|id| !self.focused.contains(id))
219 .collect();
220 let mut newly_unfocused: Vec<TokenId> = self
221 .focused
222 .iter()
223 .copied()
224 .filter(|id| !reachable.contains(id))
225 .collect();
226
227 outcome.newly_focused = newly_focused.len();
228 outcome.newly_unfocused = newly_unfocused.len();
229 outcome.unchanged = wheel.store.len().saturating_sub(outcome.newly_focused + outcome.newly_unfocused);
230
231 newly_focused.sort_unstable();
232 newly_unfocused.sort_unstable();
233
234 let mut i = 0;
235 let mut j = 0;
236 while i < newly_focused.len() && j < newly_unfocused.len() {
237 if newly_focused[i] < newly_unfocused[j] {
238 let id = newly_focused[i];
239 self.events.push(FocusEvent::Focus { token: id, revolution: wheel.revolution });
240 self.focused.insert(id);
241 i += 1;
242 } else {
243 let id = newly_unfocused[j];
244 self.events.push(FocusEvent::Unfocus { token: id, revolution: wheel.revolution });
245 self.focused.remove(&id);
246 j += 1;
247 }
248 }
249 while i < newly_focused.len() {
250 let id = newly_focused[i];
251 self.events.push(FocusEvent::Focus { token: id, revolution: wheel.revolution });
252 self.focused.insert(id);
253 i += 1;
254 }
255 while j < newly_unfocused.len() {
256 let id = newly_unfocused[j];
257 self.events.push(FocusEvent::Unfocus { token: id, revolution: wheel.revolution });
258 self.focused.remove(&id);
259 j += 1;
260 }
261
262 outcome
263 }
264
265 pub fn snapshot(&self, wheel: &WheelState, max_concepts: usize) -> FocusSet {
268 let mut top_concepts = Vec::new();
269 if max_concepts > 0 && !wheel.sequence.ids.is_empty() {
270 let window = Window::range(2, 5);
271 let events = window.extract(&wheel.store, &wheel.sequence.ids);
272 let filtered = filter_by_min_level(&wheel.store, events, 1);
273 let mut coo = CooAccumulator::new();
274 coo.accumulate(&filtered);
275 let csr = coo.bake_to_csr();
276 let valid = ppmi::valid_links(&csr, 1, 0.0);
277
278 let mut weights: HashMap<TokenId, f64> = HashMap::new();
279 for link in &valid {
280 if self.focused.contains(&link.token) && self.focused.contains(&link.context) {
281 *weights.entry(link.token).or_insert(0.0) += link.ppmi;
282 }
283 }
284
285 let mut candidate_ids: Vec<TokenId> = self
286 .focused
287 .iter()
288 .copied()
289 .filter(|&id| wheel.store.level_of(id) >= 1)
290 .collect();
291
292 candidate_ids.sort_by(|&a, &b| {
294 let wa = weights.get(&a).copied().unwrap_or(0.0);
295 let wb = weights.get(&b).copied().unwrap_or(0.0);
296 wb.partial_cmp(&wa)
297 .unwrap_or(std::cmp::Ordering::Equal)
298 .then_with(|| wheel.store.level_of(b).cmp(&wheel.store.level_of(a)))
299 .then_with(|| a.cmp(&b))
300 });
301
302 for &id in candidate_ids.iter().take(max_concepts) {
303 let raw_bytes = wheel.store.bytes_of(id);
304 let text = String::from_utf8_lossy(&raw_bytes).to_string();
305 let ppmi_weight = weights.get(&id).copied().unwrap_or(0.0);
306 let level = wheel.store.level_of(id) as u8;
307 top_concepts.push(FocusConcept {
308 token_id: id,
309 text,
310 level,
311 ppmi_weight,
312 });
313 }
314 }
315
316 FocusSet {
317 revolution: wheel.revolution,
318 tokens: self.focused.clone(),
319 top_concepts,
320 }
321 }
322
323 pub fn weight_cut(&mut self, wheel: &WheelState) -> WeightCutOutcome {
331 let roots = root_set(wheel);
332 let reachable = reachable_set(&wheel.store, &roots);
333 let borderline: Vec<TokenId> = reachable.iter().copied().filter(|id| !roots.contains(id)).collect();
334
335 let window = Window::range(2, 5);
338 let events = window.extract(&wheel.store, &wheel.sequence.ids);
339 let filtered = filter_by_min_level(&wheel.store, events, 2);
340 let mut coo = CooAccumulator::new();
341 coo.accumulate(&filtered);
342 let csr = coo.bake_to_csr();
343 let valid = ppmi::valid_links(&csr, DEFAULT_FREQUENCY_FLOOR, DEFAULT_PPMI_THRESHOLD);
344
345 let mut weight: HashMap<TokenId, f64> = HashMap::new();
346 let mut diversity: HashMap<TokenId, usize> = HashMap::new();
347 for entry in &valid {
348 if reachable.contains(&entry.token) && reachable.contains(&entry.context) {
350 *weight.entry(entry.token).or_insert(0.0) += entry.ppmi;
351 *diversity.entry(entry.token).or_insert(0) += 1;
352 }
353 }
354
355 let mut scored: Vec<(TokenId, f64, usize)> = borderline
356 .iter()
357 .map(|&id| {
358 (
359 id,
360 weight.get(&id).copied().unwrap_or(0.0),
361 diversity.get(&id).copied().unwrap_or(0),
362 )
363 })
364 .collect();
365 scored.sort_by(|a, b| {
369 a.1.partial_cmp(&b.1)
370 .unwrap()
371 .then_with(|| a.2.cmp(&b.2))
372 .then_with(|| a.0.cmp(&b.0))
373 });
374
375 let cut_count = ((scored.len() as f64) * WEIGHT_CUT_QUANTILE).round() as usize;
376 let mut newly_unfocused = 0usize;
377 for &(id, _, _) in scored.iter().take(cut_count) {
378 if self.focused.contains(&id) {
379 self.events.push(FocusEvent::Unfocus { token: id, revolution: wheel.revolution });
380 self.focused.remove(&id);
381 newly_unfocused += 1;
382 }
383 }
384
385 WeightCutOutcome {
386 borderline_count: borderline.len(),
387 cut_count: newly_unfocused,
388 }
389 }
390
391 pub fn events(&self) -> &[FocusEvent] {
392 &self.events
393 }
394
395 pub fn is_focused(&self, token: TokenId) -> bool {
396 self.focused.contains(&token)
397 }
398
399 pub fn focus(&mut self, token: TokenId, revolution: u64) {
401 if !self.focused.contains(&token) {
402 self.events.push(FocusEvent::Focus { token, revolution: revolution as u32 });
403 self.focused.insert(token);
404 }
405 }
406
407
408 pub fn focused_set(&self) -> &HashSet<TokenId> {
409 &self.focused
410 }
411
412
413 pub fn replay(events: &[FocusEvent]) -> HashSet<TokenId> {
417 let mut focused = HashSet::new();
418 for &event in events {
419 match event {
420 FocusEvent::Focus { token, .. } => {
421 focused.insert(token);
422 }
423 FocusEvent::Unfocus { token, .. } => {
424 focused.remove(&token);
425 }
426 }
427 }
428 focused
429 }
430}
431
432pub(crate) const DEATH_LOG_TABLE: TableDefinition<u32, (u8, u32, u32)> = TableDefinition::new("death_log");
433
434#[derive(Debug)]
435pub enum DeathLoadError {
436 Redb(redb::Error),
437}
438
439impl fmt::Display for DeathLoadError {
440 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
441 match self {
442 DeathLoadError::Redb(e) => write!(f, "redb: {e}"),
443 }
444 }
445}
446
447impl std::error::Error for DeathLoadError {}
448
449impl From<redb::Error> for DeathLoadError {
450 fn from(e: redb::Error) -> Self {
451 DeathLoadError::Redb(e)
452 }
453}
454
455pub(crate) fn encode_event(event: FocusEvent) -> (u8, u32, u32) {
456 match event {
457 FocusEvent::Focus { token, revolution } => (0, token, revolution),
458 FocusEvent::Unfocus { token, revolution } => (1, token, revolution),
459 }
460}
461
462pub(crate) fn decode_event(raw: (u8, u32, u32)) -> FocusEvent {
463 let (kind, token, revolution) = raw;
464 match kind {
465 0 => FocusEvent::Focus { token, revolution },
466 1 => FocusEvent::Unfocus { token, revolution },
467 other => panic!("неизвестный тип записи лога фокуса: {other} (повреждённый файл?)"),
468 }
469}
470
471pub fn save(log: &DeathLog, path: &Path) -> Result<(), redb::Error> {
476 let db = Database::create(path)?;
477 let write_txn = db.begin_write()?;
478 {
479 let mut table = write_txn.open_table(DEATH_LOG_TABLE)?;
480 let already_persisted = table.len()? as u32;
481 for (i, &event) in log.events.iter().enumerate().skip(already_persisted as usize) {
482 table.insert(i as u32, encode_event(event))?;
483 }
484 }
485 write_txn.commit()?;
486 Ok(())
487}
488
489pub fn load(path: &Path) -> Result<DeathLog, DeathLoadError> {
493 let db = Database::open(path).map_err(redb::Error::from)?;
494 let read_txn = db.begin_read().map_err(redb::Error::from)?;
495 let table = read_txn.open_table(DEATH_LOG_TABLE).map_err(redb::Error::from)?;
496
497 let mut events = Vec::new();
498 for entry in table.iter().map_err(redb::Error::from)? {
499 let (_key, value) = entry.map_err(redb::Error::from)?;
500 events.push(decode_event(value.value()));
501 }
502
503 let focused = DeathLog::replay(&events);
504 Ok(DeathLog { events, focused })
505}
506
507#[cfg(test)]
508mod tests {
509 use super::*;
510 use crate::merge::TieBreak;
511
512 #[test]
513 fn factory_tokens_are_always_roots_and_reachable() {
514 let wheel = WheelState::new();
515 let roots = root_set(&wheel);
516 for b in 0..256u32 {
517 assert!(roots.contains(&b), "байт {b} обязан быть корнем — фабричное дно бессмертно");
518 }
519 }
520
521 #[test]
522 fn token_outside_root_window_is_unfocused_but_reachable_child_stays_focused() {
523 let mut wheel = WheelState::new();
524 wheel.turn(b"ab", 2, TieBreak::BirthOrder);
526 let a = wheel.store.intern(0, &[b'a' as u32]);
527 let b = wheel.store.intern(0, &[b'b' as u32]);
528 let ab = wheel.store.intern(1, &[a, b]);
529 for _ in 0..ROOT_WINDOW_REVOLUTIONS {
531 wheel.turn(b"xyz", 2, TieBreak::BirthOrder);
532 }
533
534 let mut log = DeathLog::new();
535 log.mark(&wheel);
536
537 assert!(!log.is_focused(ab), "'ab' обязан выйти из фокуса — не был в своде последних {ROOT_WINDOW_REVOLUTIONS} оборотов и не достижим");
540 assert!(log.is_focused(a));
543 assert!(log.is_focused(b));
544 }
545
546 #[test]
547 fn resurrection_is_a_later_focus_event_not_a_special_mechanism() {
548 let mut wheel = WheelState::new();
549 wheel.turn(b"ab ab", 2, TieBreak::BirthOrder);
550 let mut log = DeathLog::new();
551 log.mark(&wheel);
552
553 let ab = wheel.sequence.ids.iter().copied().find(|&id| wheel.store.bytes_of(id) == b"ab").expect("'ab' должен был слиться в композит");
554 assert!(log.is_focused(ab));
555
556 for _ in 0..ROOT_WINDOW_REVOLUTIONS {
557 wheel.turn(b"zzz", 2, TieBreak::BirthOrder);
558 }
559 log.mark(&wheel);
560 assert!(!log.is_focused(ab), "должен выйти из фокуса без новых встреч");
561
562 wheel.turn(b"ab ab", 2, TieBreak::BirthOrder);
564 log.mark(&wheel);
565 assert!(log.is_focused(ab), "новая встреча обязана вернуть в фокус — Focus, не отдельный механизм");
566
567 let transitions: Vec<FocusEvent> = log.events().iter().copied().filter(|e| matches!(e, FocusEvent::Focus { token, .. } | FocusEvent::Unfocus { token, .. } if *token == ab)).collect();
569 assert!(transitions.len() >= 3, "минимум Focus->Unfocus->Focus на этот токен, получили {:?}", transitions);
570 }
571
572 #[test]
573 fn mark_outcome_reports_three_natural_results() {
574 let mut wheel = WheelState::new();
575 wheel.turn(b"hello world", 2, TieBreak::BirthOrder);
576 let mut log = DeathLog::new();
577 let first = log.mark(&wheel);
578 assert_eq!(first.unchanged, 0, "первый mark -> всё либо новое, либо нет");
579 assert!(first.newly_focused > 0);
580
581 let second = log.mark(&wheel); assert_eq!(second.newly_focused, 0);
583 assert_eq!(second.newly_unfocused, 0);
584 assert!(second.unchanged > 0, "повторный mark без изменений Колеса обязан дать unchanged, не Focus заново");
585 }
586
587 #[test]
588 fn weight_cut_removes_roughly_a_quarter_of_borderline_tokens() {
589 let mut wheel = WheelState::new();
590 let text = b"the cat sat on the mat, the cat sat on the mat, the cat ran to the mat, \
591 the dog sat on the log, the dog sat on the log, the dog ran to the log, \
592 a bird flew over the tree, a bird flew over the tree, a bird sang in the tree";
593 wheel.turn(text, 2, TieBreak::BirthOrder);
594 let mut log = DeathLog::new();
595 log.mark(&wheel);
596
597 let outcome = log.weight_cut(&wheel);
598 assert!(outcome.borderline_count > 0, "должны быть пограничные (достижимые не-корни) на таком тексте");
599 let expected = ((outcome.borderline_count as f64) * WEIGHT_CUT_QUANTILE).round() as usize;
600 assert_eq!(outcome.cut_count, expected, "срез обязан снять ровно квантиль 25% (с округлением)");
601 }
602
603 #[test]
604 fn weight_cut_only_acts_on_borderline_not_on_roots() {
605 let mut wheel = WheelState::new();
606 wheel.turn(b"ab ab cd cd ab cd ef ef gh gh ef gh", 2, TieBreak::BirthOrder);
607 let mut log = DeathLog::new();
608 log.mark(&wheel);
609 let roots_before = root_set(&wheel);
610
611 log.weight_cut(&wheel);
612
613 for &root in &roots_before {
616 assert!(log.is_focused(root), "срез по весу не имеет права снять корень");
617 }
618 }
619
620 #[test]
621 fn replay_reconstructs_focused_set_bit_for_bit() {
622 let mut wheel = WheelState::new();
623 wheel.turn(b"the cat sat on the mat", 2, TieBreak::BirthOrder);
624 let mut log = DeathLog::new();
625 log.mark(&wheel);
626 for _ in 0..ROOT_WINDOW_REVOLUTIONS + 1 {
627 wheel.turn(b"zzz zzz zzz", 2, TieBreak::BirthOrder);
628 }
629 log.mark(&wheel);
630
631 let replayed = DeathLog::replay(log.events());
632 assert_eq!(replayed, *log.focused_set(), "replay обязан дать то же множество, что накопленное состояние");
633 }
634
635 fn tmp_db_path(name: &str) -> std::path::PathBuf {
636 let mut p = std::env::temp_dir();
637 p.push(format!("shelf_death_test_{}_{}.redb", name, std::process::id()));
638 let _ = std::fs::remove_file(&p);
639 p
640 }
641
642 #[test]
643 fn persist_round_trip_matches_in_memory_state() {
644 let mut wheel = WheelState::new();
645 wheel.turn(b"ab ab cd cd ab cd", 2, TieBreak::BirthOrder);
646 let mut log = DeathLog::new();
647 log.mark(&wheel);
648 for _ in 0..ROOT_WINDOW_REVOLUTIONS + 1 {
649 wheel.turn(b"zzz", 2, TieBreak::BirthOrder);
650 }
651 log.mark(&wheel);
652
653 let path = tmp_db_path("round_trip");
654 save(&log, &path).expect("save");
655 let loaded = load(&path).expect("load");
656
657 assert_eq!(loaded.events(), log.events());
658 assert_eq!(*loaded.focused_set(), *log.focused_set());
659
660 let _ = std::fs::remove_file(&path);
661 }
662
663 #[test]
664 fn incremental_save_matches_one_shot() {
665 let mut wheel = WheelState::new();
666 wheel.turn(b"ab ab", 2, TieBreak::BirthOrder);
667 let mut log = DeathLog::new();
668 log.mark(&wheel);
669
670 let incremental_path = tmp_db_path("incremental");
671 save(&log, &incremental_path).expect("первая дозапись");
672
673 for _ in 0..ROOT_WINDOW_REVOLUTIONS + 1 {
674 wheel.turn(b"cd cd", 2, TieBreak::BirthOrder);
675 }
676 log.mark(&wheel);
677 save(&log, &incremental_path).expect("вторая дозапись");
678
679 let one_shot_path = tmp_db_path("one_shot");
680 save(&log, &one_shot_path).expect("контрольный одноразовый save");
681
682 let loaded_incremental = load(&incremental_path).expect("load инкрементального");
683 let loaded_one_shot = load(&one_shot_path).expect("load одноразового");
684 assert_eq!(loaded_incremental.events(), loaded_one_shot.events());
685
686 let _ = std::fs::remove_file(&incremental_path);
687 let _ = std::fs::remove_file(&one_shot_path);
688 }
689
690 #[test]
691 fn test_focus_set_snapshot() {
692 let mut wheel = WheelState::new();
693 wheel.turn(b"the memory engine of axmg is fast and reliable", 2, TieBreak::BirthOrder);
694 let mut log = DeathLog::new();
695 log.mark(&wheel);
696
697 let snapshot = log.snapshot(&wheel, 10);
698 assert_eq!(snapshot.revolution(), wheel.revolution);
699 assert_eq!(snapshot.len(), log.focused_set().len());
700 assert!(!snapshot.is_empty());
701 for &id in log.focused_set() {
702 assert!(snapshot.contains(id));
703 }
704 }
705}