eggress_testkit/
strict_observations.rs

1use serde::{Deserialize, Serialize};
2
3/// Schema version for strict observations
4pub const STRICT_OBSERVATION_SCHEMA_VERSION: u32 = 1;
5
6/// Environment metadata captured with each observation
7#[derive(Debug, Clone, Serialize, Deserialize)]
8pub struct EnvironmentMeta {
9    pub pproxy_version: Option<String>,
10    pub eggress_version: String,
11    pub python_version: String,
12    pub os: String,
13    pub arch: String,
14    pub interpreter: String,
15}
16
17/// A strict observation emitted by either oracle or candidate runner
18#[derive(Debug, Clone, Serialize, Deserialize)]
19pub struct StrictObservation {
20    pub schema_version: u32,
21    pub scenario_id: String,
22    pub runner: RunnerKind,
23    pub environment: EnvironmentMeta,
24    pub import_result: ImportResult,
25    pub stdout: Vec<String>,
26    pub stderr: Vec<String>,
27    pub exit_code: Option<i32>,
28    pub duration_ms: u64,
29    pub signature: Option<CallableSignature>,
30    pub is_coroutine: Option<bool>,
31    pub return_shape: Option<String>,
32    pub attributes: Vec<String>,
33    pub exception: Option<ExceptionInfo>,
34    pub protocol_observation: Option<ProtocolObservation>,
35    pub cleanup: CleanupInfo,
36    pub warnings: Vec<String>,
37}
38
39#[derive(Debug, Clone, Serialize, Deserialize)]
40#[serde(rename_all = "snake_case")]
41pub enum RunnerKind {
42    Oracle,
43    Candidate,
44}
45
46#[derive(Debug, Clone, Serialize, Deserialize)]
47#[serde(rename_all = "snake_case")]
48pub enum ImportResult {
49    Success,
50    ModuleNotFound,
51    ImportError,
52    SyntaxError,
53    Other,
54}
55
56#[derive(Debug, Clone, Serialize, Deserialize)]
57pub struct CallableSignature {
58    pub name: String,
59    pub positional_args: Vec<String>,
60    pub keyword_args: Vec<String>,
61    pub defaults: Vec<Option<String>>,
62    pub return_annotation: Option<String>,
63}
64
65#[derive(Debug, Clone, Serialize, Deserialize)]
66pub struct ExceptionInfo {
67    pub class_name: String,
68    pub message_category: String,
69    pub stage: String,
70    pub raw_message: String,
71}
72
73#[derive(Debug, Clone, Serialize, Deserialize)]
74pub struct ProtocolObservation {
75    pub protocol: String,
76    pub connection_result: String,
77    pub bytes_sent: u64,
78    pub bytes_received: u64,
79    pub status_code: Option<u16>,
80}
81
82#[derive(Debug, Clone, Serialize, Deserialize)]
83pub struct CleanupInfo {
84    pub processes_cleaned: bool,
85    pub sockets_cleaned: bool,
86    pub files_cleaned: bool,
87    pub leftover: Vec<String>,
88}
89
90#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
91pub struct ComparisonResult {
92    pub field: String,
93    pub oracle_value: String,
94    pub candidate_value: String,
95    pub matched: bool,
96    pub mismatch_kind: Option<MismatchKind>,
97    pub classification: MismatchClassification,
98}
99
100#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
101#[serde(rename_all = "snake_case")]
102pub enum MismatchKind {
103    ExactMismatch,
104    StructuralMismatch,
105    MissingInCandidate,
106    MissingInOracle,
107    TypeMismatch,
108    SignatureMismatch,
109}
110
111#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
112#[serde(rename_all = "snake_case")]
113pub enum MismatchClassification {
114    CandidateDefect,
115    OracleExecutionDefect,
116    HarnessDefect,
117    KnownUpstreamDefect,
118    PlatformConstraint,
119    ManifestDefect,
120    ApprovedNormalization,
121    Unclassified,
122}
123
124impl ComparisonResult {
125    pub fn matched(field: &str, oracle_value: &str, candidate_value: &str) -> Self {
126        Self {
127            field: field.to_string(),
128            oracle_value: oracle_value.to_string(),
129            candidate_value: candidate_value.to_string(),
130            matched: true,
131            mismatch_kind: None,
132            classification: MismatchClassification::Unclassified,
133        }
134    }
135
136    pub fn mismatched(
137        field: &str,
138        oracle_value: &str,
139        candidate_value: &str,
140        kind: MismatchKind,
141        classification: MismatchClassification,
142    ) -> Self {
143        Self {
144            field: field.to_string(),
145            oracle_value: oracle_value.to_string(),
146            candidate_value: candidate_value.to_string(),
147            matched: false,
148            mismatch_kind: Some(kind),
149            classification,
150        }
151    }
152}
153
154impl StrictObservation {
155    pub fn oracle(
156        scenario_id: &str,
157        environment: EnvironmentMeta,
158        import_result: ImportResult,
159    ) -> Self {
160        Self {
161            schema_version: STRICT_OBSERVATION_SCHEMA_VERSION,
162            scenario_id: scenario_id.to_string(),
163            runner: RunnerKind::Oracle,
164            environment,
165            import_result,
166            stdout: Vec::new(),
167            stderr: Vec::new(),
168            exit_code: None,
169            duration_ms: 0,
170            signature: None,
171            is_coroutine: None,
172            return_shape: None,
173            attributes: Vec::new(),
174            exception: None,
175            protocol_observation: None,
176            cleanup: CleanupInfo {
177                processes_cleaned: false,
178                sockets_cleaned: false,
179                files_cleaned: false,
180                leftover: Vec::new(),
181            },
182            warnings: Vec::new(),
183        }
184    }
185
186    pub fn candidate(
187        scenario_id: &str,
188        environment: EnvironmentMeta,
189        import_result: ImportResult,
190    ) -> Self {
191        Self {
192            schema_version: STRICT_OBSERVATION_SCHEMA_VERSION,
193            scenario_id: scenario_id.to_string(),
194            runner: RunnerKind::Candidate,
195            environment,
196            import_result,
197            stdout: Vec::new(),
198            stderr: Vec::new(),
199            exit_code: None,
200            duration_ms: 0,
201            signature: None,
202            is_coroutine: None,
203            return_shape: None,
204            attributes: Vec::new(),
205            exception: None,
206            protocol_observation: None,
207            cleanup: CleanupInfo {
208                processes_cleaned: false,
209                sockets_cleaned: false,
210                files_cleaned: false,
211                leftover: Vec::new(),
212            },
213            warnings: Vec::new(),
214        }
215    }
216}
217
218#[cfg(test)]
219mod tests {
220    use super::*;
221
222    fn test_env() -> EnvironmentMeta {
223        EnvironmentMeta {
224            pproxy_version: Some("2.7.9".to_string()),
225            eggress_version: "1.0.1".to_string(),
226            python_version: "3.11.0".to_string(),
227            os: "macos".to_string(),
228            arch: "aarch64".to_string(),
229            interpreter: "cpython".to_string(),
230        }
231    }
232
233    #[test]
234    fn strict_observation_json_roundtrip() {
235        let obs = StrictObservation::oracle("test.1", test_env(), ImportResult::Success);
236        let json = serde_json::to_string(&obs).unwrap();
237        let parsed: StrictObservation = serde_json::from_str(&json).unwrap();
238        assert_eq!(parsed.schema_version, 1);
239        assert_eq!(parsed.scenario_id, "test.1");
240        assert!(matches!(parsed.runner, RunnerKind::Oracle));
241        assert!(matches!(parsed.import_result, ImportResult::Success));
242    }
243
244    #[test]
245    fn candidate_runner_serde() {
246        let obs = StrictObservation::candidate("test.2", test_env(), ImportResult::ModuleNotFound);
247        let json = serde_json::to_string(&obs).unwrap();
248        assert!(json.contains("\"candidate\""));
249        let parsed: StrictObservation = serde_json::from_str(&json).unwrap();
250        assert!(matches!(parsed.runner, RunnerKind::Candidate));
251        assert!(matches!(parsed.import_result, ImportResult::ModuleNotFound));
252    }
253
254    #[test]
255    fn comparison_result_matched() {
256        let r = ComparisonResult::matched("field", "a", "a");
257        assert!(r.matched);
258        assert!(r.mismatch_kind.is_none());
259    }
260
261    #[test]
262    fn comparison_result_mismatched() {
263        let r = ComparisonResult::mismatched(
264            "field",
265            "oracle_val",
266            "cand_val",
267            MismatchKind::ExactMismatch,
268            MismatchClassification::CandidateDefect,
269        );
270        assert!(!r.matched);
271        assert_eq!(r.mismatch_kind, Some(MismatchKind::ExactMismatch));
272        assert_eq!(r.classification, MismatchClassification::CandidateDefect);
273    }
274
275    #[test]
276    fn environment_meta_json_roundtrip() {
277        let env = test_env();
278        let json = serde_json::to_string(&env).unwrap();
279        let parsed: EnvironmentMeta = serde_json::from_str(&json).unwrap();
280        assert_eq!(parsed.pproxy_version, Some("2.7.9".to_string()));
281        assert_eq!(parsed.arch, "aarch64");
282    }
283
284    #[test]
285    fn cleanup_info_defaults() {
286        let cleanup = CleanupInfo {
287            processes_cleaned: false,
288            sockets_cleaned: false,
289            files_cleaned: false,
290            leftover: vec!["/tmp/stale".to_string()],
291        };
292        let json = serde_json::to_string(&cleanup).unwrap();
293        let parsed: CleanupInfo = serde_json::from_str(&json).unwrap();
294        assert!(!parsed.processes_cleaned);
295        assert_eq!(parsed.leftover.len(), 1);
296    }
297
298    #[test]
299    fn exception_info_roundtrip() {
300        let exc = ExceptionInfo {
301            class_name: "ConnectionRefusedError".to_string(),
302            message_category: "connection_refused".to_string(),
303            stage: "connect".to_string(),
304            raw_message: "[Errno 61] Connection refused".to_string(),
305        };
306        let json = serde_json::to_string(&exc).unwrap();
307        let parsed: ExceptionInfo = serde_json::from_str(&json).unwrap();
308        assert_eq!(parsed.class_name, "ConnectionRefusedError");
309        assert_eq!(parsed.message_category, "connection_refused");
310    }
311
312    #[test]
313    fn callable_signature_roundtrip() {
314        let sig = CallableSignature {
315            name: "connect".to_string(),
316            positional_args: vec!["host".to_string(), "port".to_string()],
317            keyword_args: vec!["timeout".to_string()],
318            defaults: vec![None, None, Some("30".to_string())],
319            return_annotation: Some("Connection".to_string()),
320        };
321        let json = serde_json::to_string(&sig).unwrap();
322        let parsed: CallableSignature = serde_json::from_str(&json).unwrap();
323        assert_eq!(parsed.positional_args.len(), 2);
324        assert_eq!(parsed.defaults[2], Some("30".to_string()));
325    }
326
327    #[test]
328    fn protocol_observation_roundtrip() {
329        let proto = ProtocolObservation {
330            protocol: "socks5".to_string(),
331            connection_result: "success".to_string(),
332            bytes_sent: 1024,
333            bytes_received: 2048,
334            status_code: Some(0),
335        };
336        let json = serde_json::to_string(&proto).unwrap();
337        let parsed: ProtocolObservation = serde_json::from_str(&json).unwrap();
338        assert_eq!(parsed.bytes_sent, 1024);
339        assert_eq!(parsed.status_code, Some(0));
340    }
341
342    #[test]
343    fn mismatch_kind_serde() {
344        let kinds = vec![
345            MismatchKind::ExactMismatch,
346            MismatchKind::StructuralMismatch,
347            MismatchKind::MissingInCandidate,
348            MismatchKind::MissingInOracle,
349            MismatchKind::TypeMismatch,
350            MismatchKind::SignatureMismatch,
351        ];
352        for kind in kinds {
353            let json = serde_json::to_string(&kind).unwrap();
354            let parsed: MismatchKind = serde_json::from_str(&json).unwrap();
355            assert_eq!(serde_json::to_string(&parsed).unwrap(), json);
356        }
357    }
358
359    #[test]
360    fn mismatch_classification_serde() {
361        let classes = vec![
362            MismatchClassification::CandidateDefect,
363            MismatchClassification::OracleExecutionDefect,
364            MismatchClassification::HarnessDefect,
365            MismatchClassification::KnownUpstreamDefect,
366            MismatchClassification::PlatformConstraint,
367            MismatchClassification::ManifestDefect,
368            MismatchClassification::ApprovedNormalization,
369            MismatchClassification::Unclassified,
370        ];
371        for cls in classes {
372            let json = serde_json::to_string(&cls).unwrap();
373            let parsed: MismatchClassification = serde_json::from_str(&json).unwrap();
374            assert_eq!(serde_json::to_string(&parsed).unwrap(), json);
375        }
376    }
377
378    #[test]
379    fn full_observation_with_signature_and_exception() {
380        let mut obs = StrictObservation::oracle("full.test", test_env(), ImportResult::Success);
381        obs.signature = Some(CallableSignature {
382            name: "proxy".to_string(),
383            positional_args: vec![],
384            keyword_args: vec!["port".to_string()],
385            defaults: vec![Some("8080".to_string())],
386            return_annotation: None,
387        });
388        obs.is_coroutine = Some(false);
389        obs.return_shape = Some("Connection".to_string());
390        obs.attributes = vec!["public".to_string(), "async".to_string()];
391        obs.exception = Some(ExceptionInfo {
392            class_name: "TimeoutError".to_string(),
393            message_category: "timeout".to_string(),
394            stage: "handshake".to_string(),
395            raw_message: "timed out".to_string(),
396        });
397        obs.protocol_observation = Some(ProtocolObservation {
398            protocol: "http".to_string(),
399            connection_result: "timeout".to_string(),
400            bytes_sent: 0,
401            bytes_received: 0,
402            status_code: None,
403        });
404        obs.cleanup = CleanupInfo {
405            processes_cleaned: true,
406            sockets_cleaned: true,
407            files_cleaned: false,
408            leftover: vec!["/tmp/x".to_string()],
409        };
410        obs.warnings = vec!["deprecated_api".to_string()];
411        obs.stdout = vec!["info: started".to_string()];
412        obs.stderr = vec!["warn: slow".to_string()];
413        obs.exit_code = Some(0);
414        obs.duration_ms = 150;
415
416        let json = serde_json::to_string(&obs).unwrap();
417        let parsed: StrictObservation = serde_json::from_str(&json).unwrap();
418        assert!(parsed.signature.is_some());
419        assert_eq!(parsed.signature.unwrap().name, "proxy");
420        assert!(parsed.exception.is_some());
421        assert_eq!(parsed.exception.unwrap().message_category, "timeout");
422        assert!(parsed.protocol_observation.is_some());
423        assert_eq!(parsed.warnings.len(), 1);
424        assert_eq!(parsed.duration_ms, 150);
425    }
426}
eggress_testkit/strict_observations.rs

eggress_testkit/
strict_observations.rs
