eggress_testkit/
strict_comparators.rs

1use std::collections::HashSet;
2
3use crate::strict_observations::{
4    ComparisonResult, MismatchClassification, MismatchKind, StrictObservation,
5};
6
7// ---------------------------------------------------------------------------
8// compare_exact_json
9// ---------------------------------------------------------------------------
10
11pub fn compare_exact_json(
12    oracle: &StrictObservation,
13    candidate: &StrictObservation,
14) -> Vec<ComparisonResult> {
15    let oracle_json = serde_json::to_string(oracle).unwrap_or_default();
16    let candidate_json = serde_json::to_string(candidate).unwrap_or_default();
17    vec![if oracle_json == candidate_json {
18        ComparisonResult::matched("exact_json", &oracle_json, &candidate_json)
19    } else {
20        ComparisonResult::mismatched(
21            "exact_json",
22            &oracle_json,
23            &candidate_json,
24            MismatchKind::ExactMismatch,
25            MismatchClassification::Unclassified,
26        )
27    }]
28}
29
30// ---------------------------------------------------------------------------
31// compare_namespace_set
32// ---------------------------------------------------------------------------
33
34pub fn compare_namespace_set(
35    oracle_imports: &[String],
36    candidate_imports: &[String],
37) -> Vec<ComparisonResult> {
38    let oracle_set: HashSet<&str> = oracle_imports.iter().map(|s| s.as_str()).collect();
39    let candidate_set: HashSet<&str> = candidate_imports.iter().map(|s| s.as_str()).collect();
40    let mut results = Vec::new();
41
42    let oracle_str = format!("{:?}", oracle_set);
43    let candidate_str = format!("{:?}", candidate_set);
44
45    if oracle_set == candidate_set {
46        results.push(ComparisonResult::matched(
47            "namespace_set",
48            &oracle_str,
49            &candidate_str,
50        ));
51    } else {
52        let missing_in_candidate: Vec<_> = oracle_set.difference(&candidate_set).collect();
53        let extra_in_candidate: Vec<_> = candidate_set.difference(&oracle_set).collect();
54
55        if !missing_in_candidate.is_empty() {
56            results.push(ComparisonResult::mismatched(
57                "namespace_set.missing_in_candidate",
58                &format!("{:?}", missing_in_candidate),
59                &candidate_str,
60                MismatchKind::MissingInCandidate,
61                MismatchClassification::CandidateDefect,
62            ));
63        }
64        if !extra_in_candidate.is_empty() {
65            results.push(ComparisonResult::mismatched(
66                "namespace_set.extra_in_candidate",
67                &oracle_str,
68                &format!("{:?}", extra_in_candidate),
69                MismatchKind::MissingInOracle,
70                MismatchClassification::ApprovedNormalization,
71            ));
72        }
73    }
74
75    results
76}
77
78// ---------------------------------------------------------------------------
79// compare_signature
80// ---------------------------------------------------------------------------
81
82pub fn compare_signature(
83    oracle: &StrictObservation,
84    candidate: &StrictObservation,
85) -> Vec<ComparisonResult> {
86    let mut results = Vec::new();
87
88    match (&oracle.signature, &candidate.signature) {
89        (Some(oracle_sig), Some(candidate_sig)) => {
90            let oracle_str = serde_json::to_string(oracle_sig).unwrap_or_default();
91            let candidate_str = serde_json::to_string(candidate_sig).unwrap_or_default();
92            if oracle_str == candidate_str {
93                results.push(ComparisonResult::matched(
94                    "signature",
95                    &oracle_str,
96                    &candidate_str,
97                ));
98            } else {
99                if oracle_sig.positional_args != candidate_sig.positional_args {
100                    results.push(ComparisonResult::mismatched(
101                        "signature.positional_args",
102                        &format!("{:?}", oracle_sig.positional_args),
103                        &format!("{:?}", candidate_sig.positional_args),
104                        MismatchKind::SignatureMismatch,
105                        MismatchClassification::CandidateDefect,
106                    ));
107                }
108                if oracle_sig.keyword_args != candidate_sig.keyword_args {
109                    results.push(ComparisonResult::mismatched(
110                        "signature.keyword_args",
111                        &format!("{:?}", oracle_sig.keyword_args),
112                        &format!("{:?}", candidate_sig.keyword_args),
113                        MismatchKind::SignatureMismatch,
114                        MismatchClassification::CandidateDefect,
115                    ));
116                }
117                if oracle_sig.defaults != candidate_sig.defaults {
118                    results.push(ComparisonResult::mismatched(
119                        "signature.defaults",
120                        &format!("{:?}", oracle_sig.defaults),
121                        &format!("{:?}", candidate_sig.defaults),
122                        MismatchKind::SignatureMismatch,
123                        MismatchClassification::CandidateDefect,
124                    ));
125                }
126                if oracle_sig.return_annotation != candidate_sig.return_annotation {
127                    results.push(ComparisonResult::mismatched(
128                        "signature.return_annotation",
129                        oracle_sig.return_annotation.as_deref().unwrap_or("None"),
130                        candidate_sig.return_annotation.as_deref().unwrap_or("None"),
131                        MismatchKind::SignatureMismatch,
132                        MismatchClassification::CandidateDefect,
133                    ));
134                }
135            }
136        }
137        (Some(oracle_sig), None) => {
138            results.push(ComparisonResult::mismatched(
139                "signature",
140                &serde_json::to_string(oracle_sig).unwrap_or_default(),
141                "missing",
142                MismatchKind::MissingInCandidate,
143                MismatchClassification::CandidateDefect,
144            ));
145        }
146        (None, Some(candidate_sig)) => {
147            results.push(ComparisonResult::mismatched(
148                "signature",
149                "missing",
150                &serde_json::to_string(candidate_sig).unwrap_or_default(),
151                MismatchKind::MissingInOracle,
152                MismatchClassification::OracleExecutionDefect,
153            ));
154        }
155        (None, None) => {
156            results.push(ComparisonResult::matched("signature", "None", "None"));
157        }
158    }
159
160    results
161}
162
163// ---------------------------------------------------------------------------
164// compare_callable_kind
165// ---------------------------------------------------------------------------
166
167pub fn compare_callable_kind(
168    oracle: &StrictObservation,
169    candidate: &StrictObservation,
170) -> Vec<ComparisonResult> {
171    let oracle_is_coroutine = oracle.is_coroutine;
172    let candidate_is_coroutine = candidate.is_coroutine;
173    vec![match (oracle_is_coroutine, candidate_is_coroutine) {
174        (Some(o), Some(c)) if o == c => {
175            ComparisonResult::matched("callable_kind", &format!("{}", o), &format!("{}", c))
176        }
177        (Some(o), Some(c)) => ComparisonResult::mismatched(
178            "callable_kind",
179            &format!("coroutine={}", o),
180            &format!("coroutine={}", c),
181            MismatchKind::TypeMismatch,
182            MismatchClassification::CandidateDefect,
183        ),
184        (None, None) => ComparisonResult::matched("callable_kind", "unknown", "unknown"),
185        (Some(o), None) => ComparisonResult::mismatched(
186            "callable_kind",
187            &format!("coroutine={}", o),
188            "unknown",
189            MismatchKind::MissingInCandidate,
190            MismatchClassification::CandidateDefect,
191        ),
192        (None, Some(c)) => ComparisonResult::mismatched(
193            "callable_kind",
194            "unknown",
195            &format!("coroutine={}", c),
196            MismatchKind::MissingInOracle,
197            MismatchClassification::OracleExecutionDefect,
198        ),
199    }]
200}
201
202// ---------------------------------------------------------------------------
203// compare_exception
204// ---------------------------------------------------------------------------
205
206pub fn compare_exception(
207    oracle: &StrictObservation,
208    candidate: &StrictObservation,
209) -> Vec<ComparisonResult> {
210    let mut results = Vec::new();
211
212    match (&oracle.exception, &candidate.exception) {
213        (Some(oracle_exc), Some(candidate_exc)) => {
214            if oracle_exc.class_name == candidate_exc.class_name {
215                results.push(ComparisonResult::matched(
216                    "exception.class_name",
217                    &oracle_exc.class_name,
218                    &candidate_exc.class_name,
219                ));
220            } else {
221                results.push(ComparisonResult::mismatched(
222                    "exception.class_name",
223                    &oracle_exc.class_name,
224                    &candidate_exc.class_name,
225                    MismatchKind::StructuralMismatch,
226                    MismatchClassification::CandidateDefect,
227                ));
228            }
229
230            if oracle_exc.message_category == candidate_exc.message_category {
231                results.push(ComparisonResult::matched(
232                    "exception.message_category",
233                    &oracle_exc.message_category,
234                    &candidate_exc.message_category,
235                ));
236            } else {
237                results.push(ComparisonResult::mismatched(
238                    "exception.message_category",
239                    &oracle_exc.message_category,
240                    &candidate_exc.message_category,
241                    MismatchKind::StructuralMismatch,
242                    MismatchClassification::CandidateDefect,
243                ));
244            }
245        }
246        (Some(oracle_exc), None) => {
247            results.push(ComparisonResult::mismatched(
248                "exception",
249                &oracle_exc.class_name,
250                "no_exception",
251                MismatchKind::MissingInCandidate,
252                MismatchClassification::CandidateDefect,
253            ));
254        }
255        (None, Some(candidate_exc)) => {
256            results.push(ComparisonResult::mismatched(
257                "exception",
258                "no_exception",
259                &candidate_exc.class_name,
260                MismatchKind::MissingInOracle,
261                MismatchClassification::OracleExecutionDefect,
262            ));
263        }
264        (None, None) => {
265            results.push(ComparisonResult::matched("exception", "none", "none"));
266        }
267    }
268
269    results
270}
271
272// ---------------------------------------------------------------------------
273// compare_protocol_wire
274// ---------------------------------------------------------------------------
275
276pub fn compare_protocol_wire(
277    oracle: &StrictObservation,
278    candidate: &StrictObservation,
279) -> Vec<ComparisonResult> {
280    let mut results = Vec::new();
281
282    match (
283        &oracle.protocol_observation,
284        &candidate.protocol_observation,
285    ) {
286        (Some(oracle_proto), Some(candidate_proto)) => {
287            if oracle_proto.protocol == candidate_proto.protocol {
288                results.push(ComparisonResult::matched(
289                    "protocol_wire.protocol",
290                    &oracle_proto.protocol,
291                    &candidate_proto.protocol,
292                ));
293            } else {
294                results.push(ComparisonResult::mismatched(
295                    "protocol_wire.protocol",
296                    &oracle_proto.protocol,
297                    &candidate_proto.protocol,
298                    MismatchKind::StructuralMismatch,
299                    MismatchClassification::CandidateDefect,
300                ));
301            }
302
303            if oracle_proto.connection_result == candidate_proto.connection_result {
304                results.push(ComparisonResult::matched(
305                    "protocol_wire.connection_result",
306                    &oracle_proto.connection_result,
307                    &candidate_proto.connection_result,
308                ));
309            } else {
310                results.push(ComparisonResult::mismatched(
311                    "protocol_wire.connection_result",
312                    &oracle_proto.connection_result,
313                    &candidate_proto.connection_result,
314                    MismatchKind::StructuralMismatch,
315                    MismatchClassification::CandidateDefect,
316                ));
317            }
318
319            let oracle_sent = oracle_proto.bytes_sent.to_string();
320            let candidate_sent = candidate_proto.bytes_sent.to_string();
321            if oracle_proto.bytes_sent == candidate_proto.bytes_sent {
322                results.push(ComparisonResult::matched(
323                    "protocol_wire.bytes_sent",
324                    &oracle_sent,
325                    &candidate_sent,
326                ));
327            } else {
328                results.push(ComparisonResult::mismatched(
329                    "protocol_wire.bytes_sent",
330                    &oracle_sent,
331                    &candidate_sent,
332                    MismatchKind::ExactMismatch,
333                    MismatchClassification::CandidateDefect,
334                ));
335            }
336
337            let oracle_recv = oracle_proto.bytes_received.to_string();
338            let candidate_recv = candidate_proto.bytes_received.to_string();
339            if oracle_proto.bytes_received == candidate_proto.bytes_received {
340                results.push(ComparisonResult::matched(
341                    "protocol_wire.bytes_received",
342                    &oracle_recv,
343                    &candidate_recv,
344                ));
345            } else {
346                results.push(ComparisonResult::mismatched(
347                    "protocol_wire.bytes_received",
348                    &oracle_recv,
349                    &candidate_recv,
350                    MismatchKind::ExactMismatch,
351                    MismatchClassification::CandidateDefect,
352                ));
353            }
354
355            let oracle_status = oracle_proto.status_code.map(|c| c.to_string());
356            let candidate_status = candidate_proto.status_code.map(|c| c.to_string());
357            if oracle_status == candidate_status {
358                results.push(ComparisonResult::matched(
359                    "protocol_wire.status_code",
360                    oracle_status.as_deref().unwrap_or("None"),
361                    candidate_status.as_deref().unwrap_or("None"),
362                ));
363            } else {
364                results.push(ComparisonResult::mismatched(
365                    "protocol_wire.status_code",
366                    oracle_status.as_deref().unwrap_or("None"),
367                    candidate_status.as_deref().unwrap_or("None"),
368                    MismatchKind::ExactMismatch,
369                    MismatchClassification::CandidateDefect,
370                ));
371            }
372        }
373        (Some(oracle_proto), None) => {
374            results.push(ComparisonResult::mismatched(
375                "protocol_wire",
376                &oracle_proto.protocol,
377                "no_observation",
378                MismatchKind::MissingInCandidate,
379                MismatchClassification::CandidateDefect,
380            ));
381        }
382        (None, Some(candidate_proto)) => {
383            results.push(ComparisonResult::mismatched(
384                "protocol_wire",
385                "no_observation",
386                &candidate_proto.protocol,
387                MismatchKind::MissingInOracle,
388                MismatchClassification::OracleExecutionDefect,
389            ));
390        }
391        (None, None) => {
392            results.push(ComparisonResult::matched("protocol_wire", "none", "none"));
393        }
394    }
395
396    results
397}
398
399// ---------------------------------------------------------------------------
400// compare_cli_flag
401// ---------------------------------------------------------------------------
402
403pub fn compare_cli_flag(
404    flag_name: &str,
405    oracle_parse_result: &Result<String, String>,
406    candidate_parse_result: &Result<String, String>,
407) -> Vec<ComparisonResult> {
408    let oracle_str = match oracle_parse_result {
409        Ok(v) => format!("Ok({})", v),
410        Err(e) => format!("Err({})", e),
411    };
412    let candidate_str = match candidate_parse_result {
413        Ok(v) => format!("Ok({})", v),
414        Err(e) => format!("Err({})", e),
415    };
416
417    vec![if oracle_parse_result == candidate_parse_result {
418        ComparisonResult::matched(
419            &format!("cli_flag.{}", flag_name),
420            &oracle_str,
421            &candidate_str,
422        )
423    } else {
424        let kind = match (oracle_parse_result, candidate_parse_result) {
425            (Ok(_), Err(_)) | (Err(_), Ok(_)) => MismatchKind::StructuralMismatch,
426            (Err(_), Err(_)) => MismatchKind::ExactMismatch,
427            (Ok(_), Ok(_)) => MismatchKind::ExactMismatch,
428        };
429        ComparisonResult::mismatched(
430            &format!("cli_flag.{}", flag_name),
431            &oracle_str,
432            &candidate_str,
433            kind,
434            MismatchClassification::CandidateDefect,
435        )
436    }]
437}
438
439// ---------------------------------------------------------------------------
440// compare_cipher_roundtrip
441// ---------------------------------------------------------------------------
442
443fn to_hex(bytes: &[u8]) -> String {
444    bytes.iter().fold(String::new(), |mut output, byte| {
445        use std::fmt::Write;
446        write!(&mut output, "{byte:02x}").expect("writing to String is infallible");
447        output
448    })
449}
450
451pub fn compare_cipher_roundtrip(
452    cipher_name: &str,
453    oracle_input: &[u8],
454    oracle_output: &[u8],
455    candidate_output: &[u8],
456) -> Vec<ComparisonResult> {
457    let mut results = Vec::new();
458
459    if oracle_output == candidate_output {
460        results.push(ComparisonResult::matched(
461            &format!("cipher_roundtrip.{}.encrypt", cipher_name),
462            &to_hex(oracle_output),
463            &to_hex(candidate_output),
464        ));
465    } else {
466        results.push(ComparisonResult::mismatched(
467            &format!("cipher_roundtrip.{}.encrypt", cipher_name),
468            &to_hex(oracle_output),
469            &to_hex(candidate_output),
470            MismatchKind::ExactMismatch,
471            MismatchClassification::CandidateDefect,
472        ));
473    }
474
475    results.push(ComparisonResult::matched(
476        &format!("cipher_roundtrip.{}.input", cipher_name),
477        &to_hex(oracle_input),
478        &to_hex(oracle_input),
479    ));
480
481    results
482}
483
484// ---------------------------------------------------------------------------
485// compare_process_lifecycle
486// ---------------------------------------------------------------------------
487
488pub fn compare_process_lifecycle(
489    oracle: &StrictObservation,
490    candidate: &StrictObservation,
491) -> Vec<ComparisonResult> {
492    let mut results = Vec::new();
493
494    let oracle_exit = oracle.exit_code.map(|c| c.to_string());
495    let candidate_exit = candidate.exit_code.map(|c| c.to_string());
496    if oracle_exit == candidate_exit {
497        results.push(ComparisonResult::matched(
498            "process_lifecycle.exit_code",
499            oracle_exit.as_deref().unwrap_or("None"),
500            candidate_exit.as_deref().unwrap_or("None"),
501        ));
502    } else {
503        results.push(ComparisonResult::mismatched(
504            "process_lifecycle.exit_code",
505            oracle_exit.as_deref().unwrap_or("None"),
506            candidate_exit.as_deref().unwrap_or("None"),
507            MismatchKind::ExactMismatch,
508            MismatchClassification::CandidateDefect,
509        ));
510    }
511
512    let oracle_clean = !oracle.cleanup.leftover.is_empty();
513    let candidate_clean = !candidate.cleanup.leftover.is_empty();
514    if oracle_clean == candidate_clean {
515        results.push(ComparisonResult::matched(
516            "process_lifecycle.cleanup",
517            &format!("{}", oracle_clean),
518            &format!("{}", candidate_clean),
519        ));
520    } else {
521        results.push(ComparisonResult::mismatched(
522            "process_lifecycle.cleanup",
523            &format!("leftover={}", oracle_clean),
524            &format!("leftover={}", candidate_clean),
525            MismatchKind::StructuralMismatch,
526            MismatchClassification::CandidateDefect,
527        ));
528    }
529
530    results
531}
532
533// ---------------------------------------------------------------------------
534// compare_failure_class
535// ---------------------------------------------------------------------------
536
537pub fn compare_failure_class(
538    oracle: &StrictObservation,
539    candidate: &StrictObservation,
540) -> Vec<ComparisonResult> {
541    let oracle_category = oracle
542        .exception
543        .as_ref()
544        .map(|e| e.message_category.as_str())
545        .unwrap_or("none");
546    let candidate_category = candidate
547        .exception
548        .as_ref()
549        .map(|e| e.message_category.as_str())
550        .unwrap_or("none");
551
552    vec![if oracle_category == candidate_category {
553        ComparisonResult::matched("failure_class", oracle_category, candidate_category)
554    } else {
555        ComparisonResult::mismatched(
556            "failure_class",
557            oracle_category,
558            candidate_category,
559            MismatchKind::StructuralMismatch,
560            MismatchClassification::Unclassified,
561        )
562    }]
563}
564
565// ---------------------------------------------------------------------------
566// compare_composition_validity
567// ---------------------------------------------------------------------------
568
569pub fn compare_composition_validity(
570    composition_key: &str,
571    oracle_accepted: bool,
572    candidate_accepted: bool,
573) -> Vec<ComparisonResult> {
574    vec![if oracle_accepted == candidate_accepted {
575        ComparisonResult::matched(
576            &format!("composition_validity.{}", composition_key),
577            &format!("{}", oracle_accepted),
578            &format!("{}", candidate_accepted),
579        )
580    } else {
581        ComparisonResult::mismatched(
582            &format!("composition_validity.{}", composition_key),
583            &format!("accepted={}", oracle_accepted),
584            &format!("accepted={}", candidate_accepted),
585            MismatchKind::StructuralMismatch,
586            MismatchClassification::CandidateDefect,
587        )
588    }]
589}
590
591// ---------------------------------------------------------------------------
592// run_comparator (dispatcher)
593// ---------------------------------------------------------------------------
594
595pub fn run_comparator(
596    comparator_name: &str,
597    oracle: &StrictObservation,
598    candidate: &StrictObservation,
599) -> Vec<ComparisonResult> {
600    match comparator_name {
601        "compare_exact_json" => compare_exact_json(oracle, candidate),
602        "compare_signature" => compare_signature(oracle, candidate),
603        "compare_callable_kind" => compare_callable_kind(oracle, candidate),
604        "compare_exception" => compare_exception(oracle, candidate),
605        "compare_protocol_wire" => compare_protocol_wire(oracle, candidate),
606        "compare_process_lifecycle" => compare_process_lifecycle(oracle, candidate),
607        "compare_failure_class" => compare_failure_class(oracle, candidate),
608        "compare_namespace_set"
609        | "compare_cli_flag"
610        | "compare_cipher_roundtrip"
611        | "compare_composition_validity" => {
612            vec![ComparisonResult::mismatched(
613                comparator_name,
614                "not_executed",
615                "requires_extra_args",
616                MismatchKind::NotExecuted,
617                MismatchClassification::HarnessDefect,
618            )]
619        }
620        _ => vec![ComparisonResult::mismatched(
621            comparator_name,
622            "unknown_comparator",
623            comparator_name,
624            MismatchKind::StructuralMismatch,
625            MismatchClassification::HarnessDefect,
626        )],
627    }
628}
629
630#[cfg(test)]
631mod tests {
632    use super::*;
633    use crate::strict_observations::{
634        CallableSignature, CleanupInfo, EnvironmentMeta, ExceptionInfo, ImportResult,
635        MismatchClassification, ProtocolObservation, StrictObservation,
636    };
637
638    fn test_env() -> EnvironmentMeta {
639        EnvironmentMeta {
640            pproxy_version: Some("2.7.9".to_string()),
641            eggress_version: "1.0.1".to_string(),
642            python_version: "3.11.0".to_string(),
643            os: "macos".to_string(),
644            arch: "aarch64".to_string(),
645            interpreter: "cpython".to_string(),
646        }
647    }
648
649    fn oracle_obs() -> StrictObservation {
650        StrictObservation::oracle("test.1", test_env(), ImportResult::Success)
651    }
652
653    fn candidate_obs() -> StrictObservation {
654        StrictObservation::candidate("test.1", test_env(), ImportResult::Success)
655    }
656
657    #[test]
658    fn compare_exact_json_match() {
659        let o = oracle_obs();
660        let c = oracle_obs();
661        let results = compare_exact_json(&o, &c);
662        assert_eq!(results.len(), 1);
663        assert!(results[0].matched);
664    }
665
666    #[test]
667    fn compare_exact_json_mismatch() {
668        let o = oracle_obs();
669        let mut c = candidate_obs();
670        c.warnings.push("extra".to_string());
671        let results = compare_exact_json(&o, &c);
672        assert_eq!(results.len(), 1);
673        assert!(!results[0].matched);
674        assert_eq!(results[0].mismatch_kind, Some(MismatchKind::ExactMismatch));
675    }
676
677    #[test]
678    fn compare_namespace_set_identical() {
679        let imports = vec!["pproxy".to_string(), "pproxy.Connection".to_string()];
680        let results = compare_namespace_set(&imports, &imports);
681        assert_eq!(results.len(), 1);
682        assert!(results[0].matched);
683    }
684
685    #[test]
686    fn compare_namespace_set_missing_in_candidate() {
687        let oracle = vec!["pproxy".to_string(), "pproxy.Server".to_string()];
688        let candidate = vec!["pproxy".to_string()];
689        let results = compare_namespace_set(&oracle, &candidate);
690        assert!(!results.is_empty());
691        assert!(results.iter().any(|r| !r.matched));
692    }
693
694    #[test]
695    fn compare_namespace_set_extra_in_candidate() {
696        let oracle = vec!["pproxy".to_string()];
697        let candidate = vec!["pproxy".to_string(), "pproxy.extra".to_string()];
698        let results = compare_namespace_set(&oracle, &candidate);
699        assert!(!results.is_empty());
700        assert!(results.iter().any(|r| !r.matched));
701    }
702
703    #[test]
704    fn dispatcher_does_not_fabricate_unparameterized_comparisons() {
705        let results = run_comparator("compare_cli_flag", &oracle_obs(), &candidate_obs());
706        assert_eq!(results.len(), 1);
707        assert!(!results[0].matched);
708        assert_eq!(results[0].mismatch_kind, Some(MismatchKind::NotExecuted));
709        assert_eq!(
710            results[0].classification,
711            MismatchClassification::HarnessDefect
712        );
713    }
714
715    #[test]
716    fn compare_signature_match() {
717        let mut o = oracle_obs();
718        o.signature = Some(CallableSignature {
719            name: "connect".to_string(),
720            positional_args: vec!["host".to_string()],
721            keyword_args: vec![],
722            defaults: vec![],
723            return_annotation: None,
724        });
725        let mut c = candidate_obs();
726        c.signature = Some(CallableSignature {
727            name: "connect".to_string(),
728            positional_args: vec!["host".to_string()],
729            keyword_args: vec![],
730            defaults: vec![],
731            return_annotation: None,
732        });
733        let results = compare_signature(&o, &c);
734        assert!(results.iter().all(|r| r.matched));
735    }
736
737    #[test]
738    fn compare_signature_mismatch_positional_args() {
739        let mut o = oracle_obs();
740        o.signature = Some(CallableSignature {
741            name: "connect".to_string(),
742            positional_args: vec!["host".to_string(), "port".to_string()],
743            keyword_args: vec![],
744            defaults: vec![],
745            return_annotation: None,
746        });
747        let mut c = candidate_obs();
748        c.signature = Some(CallableSignature {
749            name: "connect".to_string(),
750            positional_args: vec!["host".to_string()],
751            keyword_args: vec![],
752            defaults: vec![],
753            return_annotation: None,
754        });
755        let results = compare_signature(&o, &c);
756        assert!(results.iter().any(|r| !r.matched));
757    }
758
759    #[test]
760    fn compare_signature_missing_in_candidate() {
761        let mut o = oracle_obs();
762        o.signature = Some(CallableSignature {
763            name: "f".to_string(),
764            positional_args: vec![],
765            keyword_args: vec![],
766            defaults: vec![],
767            return_annotation: None,
768        });
769        let results = compare_signature(&o, &candidate_obs());
770        assert_eq!(results.len(), 1);
771        assert!(!results[0].matched);
772        assert_eq!(
773            results[0].mismatch_kind,
774            Some(MismatchKind::MissingInCandidate)
775        );
776    }
777
778    #[test]
779    fn compare_signature_both_missing() {
780        let results = compare_signature(&oracle_obs(), &candidate_obs());
781        assert_eq!(results.len(), 1);
782        assert!(results[0].matched);
783    }
784
785    #[test]
786    fn compare_callable_kind_match() {
787        let mut o = oracle_obs();
788        o.is_coroutine = Some(true);
789        let mut c = candidate_obs();
790        c.is_coroutine = Some(true);
791        let results = compare_callable_kind(&o, &c);
792        assert_eq!(results.len(), 1);
793        assert!(results[0].matched);
794    }
795
796    #[test]
797    fn compare_callable_kind_mismatch() {
798        let mut o = oracle_obs();
799        o.is_coroutine = Some(true);
800        let mut c = candidate_obs();
801        c.is_coroutine = Some(false);
802        let results = compare_callable_kind(&o, &c);
803        assert_eq!(results.len(), 1);
804        assert!(!results[0].matched);
805        assert_eq!(results[0].mismatch_kind, Some(MismatchKind::TypeMismatch));
806    }
807
808    #[test]
809    fn compare_callable_kind_both_unknown() {
810        let results = compare_callable_kind(&oracle_obs(), &candidate_obs());
811        assert_eq!(results.len(), 1);
812        assert!(results[0].matched);
813    }
814
815    #[test]
816    fn compare_exception_match() {
817        let mut o = oracle_obs();
818        o.exception = Some(ExceptionInfo {
819            class_name: "TimeoutError".to_string(),
820            message_category: "timeout".to_string(),
821            stage: "connect".to_string(),
822            raw_message: "timed out".to_string(),
823        });
824        let mut c = candidate_obs();
825        c.exception = Some(ExceptionInfo {
826            class_name: "TimeoutError".to_string(),
827            message_category: "timeout".to_string(),
828            stage: "connect".to_string(),
829            raw_message: "timed out".to_string(),
830        });
831        let results = compare_exception(&o, &c);
832        assert!(results.iter().all(|r| r.matched));
833    }
834
835    #[test]
836    fn compare_exception_class_mismatch() {
837        let mut o = oracle_obs();
838        o.exception = Some(ExceptionInfo {
839            class_name: "TimeoutError".to_string(),
840            message_category: "timeout".to_string(),
841            stage: "connect".to_string(),
842            raw_message: "timed out".to_string(),
843        });
844        let mut c = candidate_obs();
845        c.exception = Some(ExceptionInfo {
846            class_name: "ConnectionRefusedError".to_string(),
847            message_category: "connection_refused".to_string(),
848            stage: "connect".to_string(),
849            raw_message: "refused".to_string(),
850        });
851        let results = compare_exception(&o, &c);
852        assert!(results.iter().any(|r| !r.matched));
853    }
854
855    #[test]
856    fn compare_exception_oracle_only() {
857        let mut o = oracle_obs();
858        o.exception = Some(ExceptionInfo {
859            class_name: "Error".to_string(),
860            message_category: "timeout".to_string(),
861            stage: "connect".to_string(),
862            raw_message: "err".to_string(),
863        });
864        let results = compare_exception(&o, &candidate_obs());
865        assert!(!results.is_empty());
866        assert!(results.iter().any(|r| !r.matched));
867    }
868
869    #[test]
870    fn compare_exception_candidate_only() {
871        let mut c = candidate_obs();
872        c.exception = Some(ExceptionInfo {
873            class_name: "Error".to_string(),
874            message_category: "timeout".to_string(),
875            stage: "connect".to_string(),
876            raw_message: "err".to_string(),
877        });
878        let results = compare_exception(&oracle_obs(), &c);
879        assert!(!results.is_empty());
880        assert!(results.iter().any(|r| !r.matched));
881    }
882
883    #[test]
884    fn compare_exception_both_none() {
885        let results = compare_exception(&oracle_obs(), &candidate_obs());
886        assert_eq!(results.len(), 1);
887        assert!(results[0].matched);
888    }
889
890    #[test]
891    fn compare_protocol_wire_match() {
892        let mut o = oracle_obs();
893        o.protocol_observation = Some(ProtocolObservation {
894            protocol: "socks5".to_string(),
895            connection_result: "success".to_string(),
896            bytes_sent: 100,
897            bytes_received: 200,
898            status_code: Some(0),
899        });
900        let mut c = candidate_obs();
901        c.protocol_observation = Some(ProtocolObservation {
902            protocol: "socks5".to_string(),
903            connection_result: "success".to_string(),
904            bytes_sent: 100,
905            bytes_received: 200,
906            status_code: Some(0),
907        });
908        let results = compare_protocol_wire(&o, &c);
909        assert!(results.iter().all(|r| r.matched));
910    }
911
912    #[test]
913    fn compare_protocol_wire_mismatch_bytes() {
914        let mut o = oracle_obs();
915        o.protocol_observation = Some(ProtocolObservation {
916            protocol: "socks5".to_string(),
917            connection_result: "success".to_string(),
918            bytes_sent: 100,
919            bytes_received: 200,
920            status_code: None,
921        });
922        let mut c = candidate_obs();
923        c.protocol_observation = Some(ProtocolObservation {
924            protocol: "socks5".to_string(),
925            connection_result: "success".to_string(),
926            bytes_sent: 100,
927            bytes_received: 300,
928            status_code: None,
929        });
930        let results = compare_protocol_wire(&o, &c);
931        assert!(results.iter().any(|r| !r.matched));
932    }
933
934    #[test]
935    fn compare_protocol_wire_missing_in_candidate() {
936        let mut o = oracle_obs();
937        o.protocol_observation = Some(ProtocolObservation {
938            protocol: "http".to_string(),
939            connection_result: "success".to_string(),
940            bytes_sent: 0,
941            bytes_received: 0,
942            status_code: Some(200),
943        });
944        let results = compare_protocol_wire(&o, &candidate_obs());
945        assert!(!results.is_empty());
946        assert!(results.iter().any(|r| !r.matched));
947    }
948
949    #[test]
950    fn compare_protocol_wire_both_none() {
951        let results = compare_protocol_wire(&oracle_obs(), &candidate_obs());
952        assert_eq!(results.len(), 1);
953        assert!(results[0].matched);
954    }
955
956    #[test]
957    fn compare_cli_flag_ok_match() {
958        let results = compare_cli_flag("--port", &Ok("8080".to_string()), &Ok("8080".to_string()));
959        assert_eq!(results.len(), 1);
960        assert!(results[0].matched);
961    }
962
963    #[test]
964    fn compare_cli_flag_ok_mismatch() {
965        let results = compare_cli_flag("--port", &Ok("8080".to_string()), &Ok("9090".to_string()));
966        assert_eq!(results.len(), 1);
967        assert!(!results[0].matched);
968    }
969
970    #[test]
971    fn compare_cli_flag_err_match() {
972        let results = compare_cli_flag(
973            "--daemon",
974            &Err("unsupported".to_string()),
975            &Err("unsupported".to_string()),
976        );
977        assert_eq!(results.len(), 1);
978        assert!(results[0].matched);
979    }
980
981    #[test]
982    fn compare_cli_flag_structural_mismatch() {
983        let results = compare_cli_flag(
984            "--flag",
985            &Ok("value".to_string()),
986            &Err("rejected".to_string()),
987        );
988        assert_eq!(results.len(), 1);
989        assert!(!results[0].matched);
990        assert_eq!(
991            results[0].mismatch_kind,
992            Some(MismatchKind::StructuralMismatch)
993        );
994    }
995
996    #[test]
997    fn compare_cipher_roundtrip_match() {
998        let input = b"hello world";
999        let encrypted = b"encrypted_bytes";
1000        let results = compare_cipher_roundtrip("aes_256_gcm", input, encrypted, encrypted);
1001        assert!(results.iter().all(|r| r.matched));
1002    }
1003
1004    #[test]
1005    fn compare_cipher_roundtrip_mismatch() {
1006        let input = b"hello world";
1007        let oracle_enc = b"oracle_encrypted";
1008        let candidate_enc = b"cand_encrypted";
1009        let results = compare_cipher_roundtrip("aes_256_gcm", input, oracle_enc, candidate_enc);
1010        assert!(results.iter().any(|r| !r.matched));
1011    }
1012
1013    #[test]
1014    fn compare_process_lifecycle_match() {
1015        let o = oracle_obs();
1016        let c = candidate_obs();
1017        let results = compare_process_lifecycle(&o, &c);
1018        assert!(results.iter().all(|r| r.matched));
1019    }
1020
1021    #[test]
1022    fn compare_process_lifecycle_exit_code_mismatch() {
1023        let mut o = oracle_obs();
1024        o.exit_code = Some(0);
1025        let mut c = candidate_obs();
1026        c.exit_code = Some(1);
1027        let results = compare_process_lifecycle(&o, &c);
1028        assert!(results.iter().any(|r| !r.matched));
1029    }
1030
1031    #[test]
1032    fn compare_process_lifecycle_cleanup_mismatch() {
1033        let mut o = oracle_obs();
1034        o.cleanup = CleanupInfo {
1035            processes_cleaned: true,
1036            sockets_cleaned: true,
1037            files_cleaned: true,
1038            leftover: vec![],
1039        };
1040        let mut c = candidate_obs();
1041        c.cleanup = CleanupInfo {
1042            processes_cleaned: false,
1043            sockets_cleaned: false,
1044            files_cleaned: false,
1045            leftover: vec!["/tmp/stale".to_string()],
1046        };
1047        let results = compare_process_lifecycle(&o, &c);
1048        assert!(results.iter().any(|r| !r.matched));
1049    }
1050
1051    #[test]
1052    fn compare_failure_class_match() {
1053        let mut o = oracle_obs();
1054        o.exception = Some(ExceptionInfo {
1055            class_name: "TimeoutError".to_string(),
1056            message_category: "timeout".to_string(),
1057            stage: "connect".to_string(),
1058            raw_message: "timed out".to_string(),
1059        });
1060        let mut c = candidate_obs();
1061        c.exception = Some(ExceptionInfo {
1062            class_name: "TimeoutError".to_string(),
1063            message_category: "timeout".to_string(),
1064            stage: "connect".to_string(),
1065            raw_message: "timed out".to_string(),
1066        });
1067        let results = compare_failure_class(&o, &c);
1068        assert_eq!(results.len(), 1);
1069        assert!(results[0].matched);
1070    }
1071
1072    #[test]
1073    fn compare_failure_class_mismatch() {
1074        let mut o = oracle_obs();
1075        o.exception = Some(ExceptionInfo {
1076            class_name: "TimeoutError".to_string(),
1077            message_category: "timeout".to_string(),
1078            stage: "connect".to_string(),
1079            raw_message: "timed out".to_string(),
1080        });
1081        let mut c = candidate_obs();
1082        c.exception = Some(ExceptionInfo {
1083            class_name: "ConnectionRefusedError".to_string(),
1084            message_category: "connection_refused".to_string(),
1085            stage: "connect".to_string(),
1086            raw_message: "refused".to_string(),
1087        });
1088        let results = compare_failure_class(&o, &c);
1089        assert_eq!(results.len(), 1);
1090        assert!(!results[0].matched);
1091    }
1092
1093    #[test]
1094    fn compare_failure_class_both_none() {
1095        let results = compare_failure_class(&oracle_obs(), &candidate_obs());
1096        assert_eq!(results.len(), 1);
1097        assert!(results[0].matched);
1098    }
1099
1100    #[test]
1101    fn compare_composition_validity_match() {
1102        let results = compare_composition_validity("socks5->http", true, true);
1103        assert_eq!(results.len(), 1);
1104        assert!(results[0].matched);
1105    }
1106
1107    #[test]
1108    fn compare_composition_validity_mismatch() {
1109        let results = compare_composition_validity("socks5->http", true, false);
1110        assert_eq!(results.len(), 1);
1111        assert!(!results[0].matched);
1112    }
1113
1114    #[test]
1115    fn run_comparator_exact_json() {
1116        let o = oracle_obs();
1117        let c = candidate_obs();
1118        let results = run_comparator("compare_exact_json", &o, &c);
1119        assert_eq!(results.len(), 1);
1120    }
1121
1122    #[test]
1123    fn run_comparator_signature() {
1124        let results = run_comparator("compare_signature", &oracle_obs(), &candidate_obs());
1125        assert_eq!(results.len(), 1);
1126    }
1127
1128    #[test]
1129    fn run_comparator_callable_kind() {
1130        let results = run_comparator("compare_callable_kind", &oracle_obs(), &candidate_obs());
1131        assert_eq!(results.len(), 1);
1132    }
1133
1134    #[test]
1135    fn run_comparator_exception() {
1136        let results = run_comparator("compare_exception", &oracle_obs(), &candidate_obs());
1137        assert_eq!(results.len(), 1);
1138    }
1139
1140    #[test]
1141    fn run_comparator_protocol_wire() {
1142        let results = run_comparator("compare_protocol_wire", &oracle_obs(), &candidate_obs());
1143        assert_eq!(results.len(), 1);
1144    }
1145
1146    #[test]
1147    fn run_comparator_process_lifecycle() {
1148        let results = run_comparator("compare_process_lifecycle", &oracle_obs(), &candidate_obs());
1149        assert_eq!(results.len(), 2);
1150    }
1151
1152    #[test]
1153    fn run_comparator_failure_class() {
1154        let results = run_comparator("compare_failure_class", &oracle_obs(), &candidate_obs());
1155        assert_eq!(results.len(), 1);
1156    }
1157
1158    #[test]
1159    fn run_comparator_unknown() {
1160        let results = run_comparator("bogus_comparator", &oracle_obs(), &candidate_obs());
1161        assert_eq!(results.len(), 1);
1162        assert!(!results[0].matched);
1163        assert_eq!(
1164            results[0].classification,
1165            MismatchClassification::HarnessDefect
1166        );
1167    }
1168
1169    #[test]
1170    fn run_comparator_extra_args_passthrough() {
1171        let results = run_comparator("compare_namespace_set", &oracle_obs(), &candidate_obs());
1172        assert_eq!(results.len(), 1);
1173        assert!(!results[0].matched);
1174        assert_eq!(results[0].mismatch_kind, Some(MismatchKind::NotExecuted));
1175    }
1176}
eggress_testkit/strict_comparators.rs

eggress_testkit/
strict_comparators.rs
