rustev-eval 0.1.0

Rustev's offline evaluation: bounded replay bundles, historical reproduction, candidate comparison, datasets, metrics and calibration fitting.
Documentation
1
{"core":{"recorded":{"decision_id":"lodging-emb","definition_id":"sha256:87f9e5dafd3d025e4c2133513522cffd7b728d023410a23011cfb19fdb13fec2","evaluation_time_ms":1790164800000,"inputs":[{"field":"fx.rates","status":{"resolved":{"as_of_ms":1790164740000,"provenance":"system-of-record","source":"synthetic"}}},{"field":"inventory.candidates","status":{"resolved":{"as_of_ms":1790164740000,"provenance":"third-party","source":"synthetic"}}},{"field":"traveler.claims","status":{"resolved":{"as_of_ms":1790078400000,"provenance":"attributed-claim","source":"synthetic"}}},{"field":"trip.budget","status":{"resolved":{"as_of_ms":1790164740000,"provenance":"user-supplied","source":"synthetic"}}},{"field":"trip.dates","status":{"resolved":{"as_of_ms":1790164740000,"provenance":"user-supplied","source":"synthetic"}}},{"field":"trip.party_size","status":{"resolved":{"as_of_ms":1790164740000,"provenance":"user-supplied","source":"synthetic"}}},{"field":"trip.requires_step_free","status":{"resolved":{"as_of_ms":1790164740000,"provenance":"user-supplied","source":"synthetic"}}},{"field":"trip.text","status":{"resolved":{"as_of_ms":1790164740000,"provenance":"user-supplied","source":"synthetic"}}}],"judgment":{"derivation":"mixed-derived","evaluation_time_ms":1790164800000,"lineage":{"inputs":["fx.rates","inventory.candidates","traveler.claims","trip.budget","trip.dates","trip.party_size","trip.requires_step_free","trip.text"],"steps":["active_claims","eligible","missing_trip_fields","ranking","shortlist","suitability","supports_preference"]},"notices":[],"outcome":{"propose":{"action":"present_ranked_lodging","params":{"eligibility":{"filtered":{"counts":{},"eligible":["c-1","c-2","c-3"],"excluded":[]}},"ranking":{"ranking":{"entries":[{"candidate":"c-2","components":{"preference":0.2689414213699951,"price":1.0,"suitability":0.4205318233453525},"position":1,"score":0.4606302576886033},{"candidate":"c-1","components":{"preference":0.2689414213699951,"price":0.5,"suitability":0.347430536714805},"position":2,"score":0.33869987169943905},{"candidate":"c-3","components":{"preference":0.2689414213699951,"price":0.0,"suitability":0.160727383224561},"position":3,"score":0.18268892565236586}],"excluded":[]}},"shortlist":{"shortlist":{"excluded":[],"ids":["c-2","c-1","c-3"],"truncated":0}}}}},"plan_id":"sha256:08467527cf3d0222f0e7071819d5a7d04467ebf7d9da2619365f647aa2303642","schema":"rustev.judgment/1","snapshot_id":"sha256:3cbe25eaaed2094df97b7809eee0166592a3038211da73550de688a94dff2c86","trace":["rule:ranked"]},"plan_id":"sha256:08467527cf3d0222f0e7071819d5a7d04467ebf7d9da2619365f647aa2303642","schema":"rustev.evidence/1","snapshot_id":"sha256:3cbe25eaaed2094df97b7809eee0166592a3038211da73550de688a94dff2c86","steps":[{"instance":[],"status":{"resolved":{"derivation":"exact-derived","kind":"list<text>"}},"step":"missing_trip_fields"},{"instance":[],"status":{"resolved":{"derivation":"exact-derived","kind":"filtered"}},"step":"eligible"},{"instance":[],"status":{"resolved":{"derivation":"exact-derived","kind":"shortlist"}},"step":"shortlist"},{"instance":[],"status":{"resolved":{"derivation":"exact-derived","kind":"filtered"}},"step":"active_claims"},{"instance":[],"status":{"resolved":{"derivation":"model-derived","kind":"distribution"}},"step":"trip_intent"},{"instance":["c-1","k-1"],"status":{"resolved":{"derivation":"model-derived","kind":"distribution"}},"step":"supports_preference"},{"instance":["c-1","k-2"],"status":{"resolved":{"derivation":"model-derived","kind":"distribution"}},"step":"supports_preference"},{"instance":["c-2","k-1"],"status":{"resolved":{"derivation":"model-derived","kind":"distribution"}},"step":"supports_preference"},{"instance":["c-2","k-2"],"status":{"resolved":{"derivation":"model-derived","kind":"distribution"}},"step":"supports_preference"},{"instance":["c-3","k-1"],"status":{"resolved":{"derivation":"model-derived","kind":"distribution"}},"step":"supports_preference"},{"instance":["c-3","k-2"],"status":{"resolved":{"derivation":"model-derived","kind":"distribution"}},"step":"supports_preference"},{"instance":["c-1"],"status":{"resolved":{"derivation":"model-derived","kind":"ordinal_level(distribution)"}},"step":"suitability"},{"instance":["c-2"],"status":{"resolved":{"derivation":"model-derived","kind":"ordinal_level(distribution)"}},"step":"suitability"},{"instance":["c-3"],"status":{"resolved":{"derivation":"model-derived","kind":"ordinal_level(distribution)"}},"step":"suitability"},{"instance":[],"status":{"resolved":{"derivation":"mixed-derived","kind":"ranking"}},"step":"ranking"}]}},"cost":{"bound_violations":0,"estimated":0,"final_cost":"known","liability":0,"limit":0,"mode":"unlimited","observed":20,"within_guaranteed_cap":false},"decision_id":"lodging-emb","execution":"none","plan_id":"sha256:08467527cf3d0222f0e7071819d5a7d04467ebf7d9da2619365f647aa2303642","requests":[{"attempts":[{"artifact":"sha256:3374000791e29cca4b8625e4c6f453e338c073becc0b8d55df31aea6796ca0ec","attempt_id":"lodging-emb/trip_intent//1","backend_id":"synthetic-rules-lodging","cancellation":"not_requested","cost":{"bound":{"bounded":{"max_units":2}},"charge":{"observed":{"units":2}},"reserved":2},"dispatched_ms":1,"end":"output","ended_ms":1,"remote":"finished","target":0}],"instance":[],"result":{"output":{"target":0}},"step":"trip_intent","transitions":[]},{"attempts":[{"artifact":"sha256:3374000791e29cca4b8625e4c6f453e338c073becc0b8d55df31aea6796ca0ec","attempt_id":"lodging-emb/supports_preference/c-1,k-1/1","backend_id":"synthetic-rules-lodging","cancellation":"not_requested","cost":{"bound":{"bounded":{"max_units":2}},"charge":{"observed":{"units":2}},"reserved":2},"dispatched_ms":5,"end":"output","ended_ms":5,"remote":"finished","target":0}],"instance":["c-1","k-1"],"result":{"output":{"target":0}},"step":"supports_preference","transitions":[]},{"attempts":[{"artifact":"sha256:3374000791e29cca4b8625e4c6f453e338c073becc0b8d55df31aea6796ca0ec","attempt_id":"lodging-emb/supports_preference/c-1,k-2/1","backend_id":"synthetic-rules-lodging","cancellation":"not_requested","cost":{"bound":{"bounded":{"max_units":2}},"charge":{"observed":{"units":2}},"reserved":2},"dispatched_ms":5,"end":"output","ended_ms":5,"remote":"finished","target":0}],"instance":["c-1","k-2"],"result":{"output":{"target":0}},"step":"supports_preference","transitions":[]},{"attempts":[{"artifact":"sha256:3374000791e29cca4b8625e4c6f453e338c073becc0b8d55df31aea6796ca0ec","attempt_id":"lodging-emb/supports_preference/c-2,k-1/1","backend_id":"synthetic-rules-lodging","cancellation":"not_requested","cost":{"bound":{"bounded":{"max_units":2}},"charge":{"observed":{"units":2}},"reserved":2},"dispatched_ms":1,"end":"output","ended_ms":1,"remote":"finished","target":0}],"instance":["c-2","k-1"],"result":{"output":{"target":0}},"step":"supports_preference","transitions":[]},{"attempts":[{"artifact":"sha256:3374000791e29cca4b8625e4c6f453e338c073becc0b8d55df31aea6796ca0ec","attempt_id":"lodging-emb/supports_preference/c-2,k-2/1","backend_id":"synthetic-rules-lodging","cancellation":"not_requested","cost":{"bound":{"bounded":{"max_units":2}},"charge":{"observed":{"units":2}},"reserved":2},"dispatched_ms":2,"end":"output","ended_ms":2,"remote":"finished","target":0}],"instance":["c-2","k-2"],"result":{"output":{"target":0}},"step":"supports_preference","transitions":[]},{"attempts":[{"artifact":"sha256:3374000791e29cca4b8625e4c6f453e338c073becc0b8d55df31aea6796ca0ec","attempt_id":"lodging-emb/supports_preference/c-3,k-1/1","backend_id":"synthetic-rules-lodging","cancellation":"not_requested","cost":{"bound":{"bounded":{"max_units":2}},"charge":{"observed":{"units":2}},"reserved":2},"dispatched_ms":3,"end":"output","ended_ms":3,"remote":"finished","target":0}],"instance":["c-3","k-1"],"result":{"output":{"target":0}},"step":"supports_preference","transitions":[]},{"attempts":[{"artifact":"sha256:3374000791e29cca4b8625e4c6f453e338c073becc0b8d55df31aea6796ca0ec","attempt_id":"lodging-emb/supports_preference/c-3,k-2/1","backend_id":"synthetic-rules-lodging","cancellation":"not_requested","cost":{"bound":{"bounded":{"max_units":2}},"charge":{"observed":{"units":2}},"reserved":2},"dispatched_ms":3,"end":"output","ended_ms":3,"remote":"finished","target":0}],"instance":["c-3","k-2"],"result":{"output":{"target":0}},"step":"supports_preference","transitions":[]},{"attempts":[{"artifact":"sha256:3374000791e29cca4b8625e4c6f453e338c073becc0b8d55df31aea6796ca0ec","attempt_id":"lodging-emb/suitability/c-1/1","backend_id":"synthetic-rules-lodging","cancellation":"not_requested","cost":{"bound":{"bounded":{"max_units":2}},"charge":{"observed":{"units":2}},"reserved":2},"dispatched_ms":3,"end":"output","ended_ms":4,"remote":"finished","target":0}],"instance":["c-1"],"result":{"output":{"target":0}},"step":"suitability","transitions":[]},{"attempts":[{"artifact":"sha256:3374000791e29cca4b8625e4c6f453e338c073becc0b8d55df31aea6796ca0ec","attempt_id":"lodging-emb/suitability/c-2/1","backend_id":"synthetic-rules-lodging","cancellation":"not_requested","cost":{"bound":{"bounded":{"max_units":2}},"charge":{"observed":{"units":2}},"reserved":2},"dispatched_ms":4,"end":"output","ended_ms":4,"remote":"finished","target":0}],"instance":["c-2"],"result":{"output":{"target":0}},"step":"suitability","transitions":[]},{"attempts":[{"artifact":"sha256:3374000791e29cca4b8625e4c6f453e338c073becc0b8d55df31aea6796ca0ec","attempt_id":"lodging-emb/suitability/c-3/1","backend_id":"synthetic-rules-lodging","cancellation":"not_requested","cost":{"bound":{"bounded":{"max_units":2}},"charge":{"observed":{"units":2}},"reserved":2},"dispatched_ms":4,"end":"output","ended_ms":5,"remote":"finished","target":0}],"instance":["c-3"],"result":{"output":{"target":0}},"step":"suitability","transitions":[]}],"schema":"rustev.run/1","termination":"judged","timing":{"deadline_expired":false,"deadline_ms":2000,"elapsed_ms":6,"queued_ms":1}}