eggress_testkit/
strict_observations.rs

1use serde::{Deserialize, Serialize};
2
3/// Schema version for strict observations
4pub const STRICT_OBSERVATION_SCHEMA_VERSION: u32 = 1;
5
6/// Environment metadata captured with each observation
7#[derive(Debug, Clone, Serialize, Deserialize)]
8pub struct EnvironmentMeta {
9    pub pproxy_version: Option<String>,
10    pub eggress_version: String,
11    pub python_version: String,
12    pub os: String,
13    pub arch: String,
14    pub interpreter: String,
15}
16
17/// A strict observation emitted by either oracle or candidate runner
18#[derive(Debug, Clone, Serialize, Deserialize)]
19pub struct StrictObservation {
20    pub schema_version: u32,
21    pub scenario_id: String,
22    pub runner: RunnerKind,
23    pub environment: EnvironmentMeta,
24    pub import_result: ImportResult,
25    pub stdout: Vec<String>,
26    pub stderr: Vec<String>,
27    pub exit_code: Option<i32>,
28    pub duration_ms: u64,
29    pub signature: Option<CallableSignature>,
30    pub is_coroutine: Option<bool>,
31    pub return_shape: Option<String>,
32    pub attributes: Vec<String>,
33    pub exception: Option<ExceptionInfo>,
34    pub protocol_observation: Option<ProtocolObservation>,
35    pub cleanup: CleanupInfo,
36    pub warnings: Vec<String>,
37}
38
39#[derive(Debug, Clone, Serialize, Deserialize)]
40#[serde(rename_all = "snake_case")]
41pub enum RunnerKind {
42    Oracle,
43    Candidate,
44}
45
46#[derive(Debug, Clone, Serialize, Deserialize)]
47#[serde(rename_all = "snake_case")]
48pub enum ImportResult {
49    Success,
50    ModuleNotFound,
51    ImportError,
52    SyntaxError,
53    Other,
54}
55
56#[derive(Debug, Clone, Serialize, Deserialize)]
57pub struct CallableSignature {
58    pub name: String,
59    pub positional_args: Vec<String>,
60    pub keyword_args: Vec<String>,
61    pub defaults: Vec<Option<String>>,
62    pub return_annotation: Option<String>,
63}
64
65#[derive(Debug, Clone, Serialize, Deserialize)]
66pub struct ExceptionInfo {
67    pub class_name: String,
68    pub message_category: String,
69    pub stage: String,
70    pub raw_message: String,
71}
72
73#[derive(Debug, Clone, Serialize, Deserialize)]
74pub struct ProtocolObservation {
75    pub protocol: String,
76    pub connection_result: String,
77    pub bytes_sent: u64,
78    pub bytes_received: u64,
79    pub status_code: Option<u16>,
80}
81
82#[derive(Debug, Clone, Serialize, Deserialize)]
83pub struct CleanupInfo {
84    pub processes_cleaned: bool,
85    pub sockets_cleaned: bool,
86    pub files_cleaned: bool,
87    pub leftover: Vec<String>,
88}
89
90#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
91pub struct ComparisonResult {
92    pub field: String,
93    pub oracle_value: String,
94    pub candidate_value: String,
95    pub matched: bool,
96    pub mismatch_kind: Option<MismatchKind>,
97    pub classification: MismatchClassification,
98}
99
100#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
101#[serde(rename_all = "snake_case")]
102pub enum MismatchKind {
103    ExactMismatch,
104    StructuralMismatch,
105    MissingInCandidate,
106    MissingInOracle,
107    TypeMismatch,
108    SignatureMismatch,
109    NotExecuted,
110}
111
112#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
113#[serde(rename_all = "snake_case")]
114pub enum MismatchClassification {
115    CandidateDefect,
116    OracleExecutionDefect,
117    HarnessDefect,
118    KnownUpstreamDefect,
119    PlatformConstraint,
120    ManifestDefect,
121    ApprovedNormalization,
122    Unclassified,
123}
124
125impl ComparisonResult {
126    pub fn matched(field: &str, oracle_value: &str, candidate_value: &str) -> Self {
127        Self {
128            field: field.to_string(),
129            oracle_value: oracle_value.to_string(),
130            candidate_value: candidate_value.to_string(),
131            matched: true,
132            mismatch_kind: None,
133            classification: MismatchClassification::Unclassified,
134        }
135    }
136
137    pub fn mismatched(
138        field: &str,
139        oracle_value: &str,
140        candidate_value: &str,
141        kind: MismatchKind,
142        classification: MismatchClassification,
143    ) -> Self {
144        Self {
145            field: field.to_string(),
146            oracle_value: oracle_value.to_string(),
147            candidate_value: candidate_value.to_string(),
148            matched: false,
149            mismatch_kind: Some(kind),
150            classification,
151        }
152    }
153}
154
155impl StrictObservation {
156    pub fn oracle(
157        scenario_id: &str,
158        environment: EnvironmentMeta,
159        import_result: ImportResult,
160    ) -> Self {
161        Self {
162            schema_version: STRICT_OBSERVATION_SCHEMA_VERSION,
163            scenario_id: scenario_id.to_string(),
164            runner: RunnerKind::Oracle,
165            environment,
166            import_result,
167            stdout: Vec::new(),
168            stderr: Vec::new(),
169            exit_code: None,
170            duration_ms: 0,
171            signature: None,
172            is_coroutine: None,
173            return_shape: None,
174            attributes: Vec::new(),
175            exception: None,
176            protocol_observation: None,
177            cleanup: CleanupInfo {
178                processes_cleaned: false,
179                sockets_cleaned: false,
180                files_cleaned: false,
181                leftover: Vec::new(),
182            },
183            warnings: Vec::new(),
184        }
185    }
186
187    pub fn candidate(
188        scenario_id: &str,
189        environment: EnvironmentMeta,
190        import_result: ImportResult,
191    ) -> Self {
192        Self {
193            schema_version: STRICT_OBSERVATION_SCHEMA_VERSION,
194            scenario_id: scenario_id.to_string(),
195            runner: RunnerKind::Candidate,
196            environment,
197            import_result,
198            stdout: Vec::new(),
199            stderr: Vec::new(),
200            exit_code: None,
201            duration_ms: 0,
202            signature: None,
203            is_coroutine: None,
204            return_shape: None,
205            attributes: Vec::new(),
206            exception: None,
207            protocol_observation: None,
208            cleanup: CleanupInfo {
209                processes_cleaned: false,
210                sockets_cleaned: false,
211                files_cleaned: false,
212                leftover: Vec::new(),
213            },
214            warnings: Vec::new(),
215        }
216    }
217}
218
219#[cfg(test)]
220mod tests {
221    use super::*;
222
223    fn test_env() -> EnvironmentMeta {
224        EnvironmentMeta {
225            pproxy_version: Some("2.7.9".to_string()),
226            eggress_version: "1.0.1".to_string(),
227            python_version: "3.11.0".to_string(),
228            os: "macos".to_string(),
229            arch: "aarch64".to_string(),
230            interpreter: "cpython".to_string(),
231        }
232    }
233
234    #[test]
235    fn strict_observation_json_roundtrip() {
236        let obs = StrictObservation::oracle("test.1", test_env(), ImportResult::Success);
237        let json = serde_json::to_string(&obs).unwrap();
238        let parsed: StrictObservation = serde_json::from_str(&json).unwrap();
239        assert_eq!(parsed.schema_version, 1);
240        assert_eq!(parsed.scenario_id, "test.1");
241        assert!(matches!(parsed.runner, RunnerKind::Oracle));
242        assert!(matches!(parsed.import_result, ImportResult::Success));
243    }
244
245    #[test]
246    fn candidate_runner_serde() {
247        let obs = StrictObservation::candidate("test.2", test_env(), ImportResult::ModuleNotFound);
248        let json = serde_json::to_string(&obs).unwrap();
249        assert!(json.contains("\"candidate\""));
250        let parsed: StrictObservation = serde_json::from_str(&json).unwrap();
251        assert!(matches!(parsed.runner, RunnerKind::Candidate));
252        assert!(matches!(parsed.import_result, ImportResult::ModuleNotFound));
253    }
254
255    #[test]
256    fn comparison_result_matched() {
257        let r = ComparisonResult::matched("field", "a", "a");
258        assert!(r.matched);
259        assert!(r.mismatch_kind.is_none());
260    }
261
262    #[test]
263    fn comparison_result_mismatched() {
264        let r = ComparisonResult::mismatched(
265            "field",
266            "oracle_val",
267            "cand_val",
268            MismatchKind::ExactMismatch,
269            MismatchClassification::CandidateDefect,
270        );
271        assert!(!r.matched);
272        assert_eq!(r.mismatch_kind, Some(MismatchKind::ExactMismatch));
273        assert_eq!(r.classification, MismatchClassification::CandidateDefect);
274    }
275
276    #[test]
277    fn environment_meta_json_roundtrip() {
278        let env = test_env();
279        let json = serde_json::to_string(&env).unwrap();
280        let parsed: EnvironmentMeta = serde_json::from_str(&json).unwrap();
281        assert_eq!(parsed.pproxy_version, Some("2.7.9".to_string()));
282        assert_eq!(parsed.arch, "aarch64");
283    }
284
285    #[test]
286    fn cleanup_info_defaults() {
287        let cleanup = CleanupInfo {
288            processes_cleaned: false,
289            sockets_cleaned: false,
290            files_cleaned: false,
291            leftover: vec!["/tmp/stale".to_string()],
292        };
293        let json = serde_json::to_string(&cleanup).unwrap();
294        let parsed: CleanupInfo = serde_json::from_str(&json).unwrap();
295        assert!(!parsed.processes_cleaned);
296        assert_eq!(parsed.leftover.len(), 1);
297    }
298
299    #[test]
300    fn exception_info_roundtrip() {
301        let exc = ExceptionInfo {
302            class_name: "ConnectionRefusedError".to_string(),
303            message_category: "connection_refused".to_string(),
304            stage: "connect".to_string(),
305            raw_message: "[Errno 61] Connection refused".to_string(),
306        };
307        let json = serde_json::to_string(&exc).unwrap();
308        let parsed: ExceptionInfo = serde_json::from_str(&json).unwrap();
309        assert_eq!(parsed.class_name, "ConnectionRefusedError");
310        assert_eq!(parsed.message_category, "connection_refused");
311    }
312
313    #[test]
314    fn callable_signature_roundtrip() {
315        let sig = CallableSignature {
316            name: "connect".to_string(),
317            positional_args: vec!["host".to_string(), "port".to_string()],
318            keyword_args: vec!["timeout".to_string()],
319            defaults: vec![None, None, Some("30".to_string())],
320            return_annotation: Some("Connection".to_string()),
321        };
322        let json = serde_json::to_string(&sig).unwrap();
323        let parsed: CallableSignature = serde_json::from_str(&json).unwrap();
324        assert_eq!(parsed.positional_args.len(), 2);
325        assert_eq!(parsed.defaults[2], Some("30".to_string()));
326    }
327
328    #[test]
329    fn protocol_observation_roundtrip() {
330        let proto = ProtocolObservation {
331            protocol: "socks5".to_string(),
332            connection_result: "success".to_string(),
333            bytes_sent: 1024,
334            bytes_received: 2048,
335            status_code: Some(0),
336        };
337        let json = serde_json::to_string(&proto).unwrap();
338        let parsed: ProtocolObservation = serde_json::from_str(&json).unwrap();
339        assert_eq!(parsed.bytes_sent, 1024);
340        assert_eq!(parsed.status_code, Some(0));
341    }
342
343    #[test]
344    fn mismatch_kind_serde() {
345        let kinds = vec![
346            MismatchKind::ExactMismatch,
347            MismatchKind::StructuralMismatch,
348            MismatchKind::MissingInCandidate,
349            MismatchKind::MissingInOracle,
350            MismatchKind::TypeMismatch,
351            MismatchKind::SignatureMismatch,
352        ];
353        for kind in kinds {
354            let json = serde_json::to_string(&kind).unwrap();
355            let parsed: MismatchKind = serde_json::from_str(&json).unwrap();
356            assert_eq!(serde_json::to_string(&parsed).unwrap(), json);
357        }
358    }
359
360    #[test]
361    fn mismatch_classification_serde() {
362        let classes = vec![
363            MismatchClassification::CandidateDefect,
364            MismatchClassification::OracleExecutionDefect,
365            MismatchClassification::HarnessDefect,
366            MismatchClassification::KnownUpstreamDefect,
367            MismatchClassification::PlatformConstraint,
368            MismatchClassification::ManifestDefect,
369            MismatchClassification::ApprovedNormalization,
370            MismatchClassification::Unclassified,
371        ];
372        for cls in classes {
373            let json = serde_json::to_string(&cls).unwrap();
374            let parsed: MismatchClassification = serde_json::from_str(&json).unwrap();
375            assert_eq!(serde_json::to_string(&parsed).unwrap(), json);
376        }
377    }
378
379    #[test]
380    fn full_observation_with_signature_and_exception() {
381        let mut obs = StrictObservation::oracle("full.test", test_env(), ImportResult::Success);
382        obs.signature = Some(CallableSignature {
383            name: "proxy".to_string(),
384            positional_args: vec![],
385            keyword_args: vec!["port".to_string()],
386            defaults: vec![Some("8080".to_string())],
387            return_annotation: None,
388        });
389        obs.is_coroutine = Some(false);
390        obs.return_shape = Some("Connection".to_string());
391        obs.attributes = vec!["public".to_string(), "async".to_string()];
392        obs.exception = Some(ExceptionInfo {
393            class_name: "TimeoutError".to_string(),
394            message_category: "timeout".to_string(),
395            stage: "handshake".to_string(),
396            raw_message: "timed out".to_string(),
397        });
398        obs.protocol_observation = Some(ProtocolObservation {
399            protocol: "http".to_string(),
400            connection_result: "timeout".to_string(),
401            bytes_sent: 0,
402            bytes_received: 0,
403            status_code: None,
404        });
405        obs.cleanup = CleanupInfo {
406            processes_cleaned: true,
407            sockets_cleaned: true,
408            files_cleaned: false,
409            leftover: vec!["/tmp/x".to_string()],
410        };
411        obs.warnings = vec!["deprecated_api".to_string()];
412        obs.stdout = vec!["info: started".to_string()];
413        obs.stderr = vec!["warn: slow".to_string()];
414        obs.exit_code = Some(0);
415        obs.duration_ms = 150;
416
417        let json = serde_json::to_string(&obs).unwrap();
418        let parsed: StrictObservation = serde_json::from_str(&json).unwrap();
419        assert!(parsed.signature.is_some());
420        assert_eq!(parsed.signature.unwrap().name, "proxy");
421        assert!(parsed.exception.is_some());
422        assert_eq!(parsed.exception.unwrap().message_category, "timeout");
423        assert!(parsed.protocol_observation.is_some());
424        assert_eq!(parsed.warnings.len(), 1);
425        assert_eq!(parsed.duration_ms, 150);
426    }
427}
eggress_testkit/strict_observations.rs

eggress_testkit/
strict_observations.rs
