batch-mode-batch-triple 0.2.4

Manages batch file triples (input, output, error, metadata) in batch processing systems, including file validation, error handling, and file movement.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
// ---------------- [ File: batch-mode-batch-triple/src/batch_file_triple.rs ]
crate::ix!();

/// Represents the batch files associated with a specific index.
#[derive(Builder,Getters,Clone)]
#[getset(get="pub")]
#[builder(setter(into,strip_option))]
pub struct BatchFileTriple {
    index:               BatchIndex,

    #[builder(default)]
    input:               Option<PathBuf>,

    #[builder(default)]
    output:              Option<PathBuf>,

    #[builder(default)]
    error:               Option<PathBuf>,

    #[builder(default)]
    associated_metadata: Option<PathBuf>,

    #[builder(default)]
    seed_manifest:       Option<PathBuf>,

    workspace:           Arc<dyn BatchWorkspaceInterface>,
}

unsafe impl Send for BatchFileTriple {}
unsafe impl Sync for BatchFileTriple {}

impl Debug for BatchFileTriple {

    fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
        f.debug_struct("BatchFileTriple")
            .field("index",  &self.index)
            .field("input",  &self.input)
            .field("output", &self.output)
            .field("error",  &self.error)
            .field("associated_metadata", &self.associated_metadata)
            .field("seed_manifest", &self.seed_manifest)
            .finish()
    }
}

impl PartialOrd for BatchFileTriple {

    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
        self.index.partial_cmp(&other.index)
    }
}

impl PartialEq for BatchFileTriple {

    fn eq(&self, other: &BatchFileTriple) -> bool { 
        self.index.eq(&other.index) 
            &&
        self.input.eq(&other.input) 
            &&
        self.output.eq(&other.output) 
            &&
        self.error.eq(&other.error) 
            &&
        self.associated_metadata.eq(&other.associated_metadata) 
            &&
        self.seed_manifest.eq(&other.seed_manifest) 
    }
}

impl Eq for BatchFileTriple {}

impl Ord for BatchFileTriple {

    fn cmp(&self, other: &Self) -> Ordering {
        self.index.cmp(&other.index)
    }
}

impl BatchFileTriple {
    /// A convenience constructor for tests that supply a custom workspace. 
    /// Everything else is None, and we assign a dummy index.
    pub fn new_for_test_with_workspace(workspace: Arc<dyn BatchWorkspaceInterface>) -> Self {
        trace!("Constructing a test triple with a custom workspace only");
        let index = BatchIndex::Usize(9999);
        Self::new_direct(&index, None, None, None, None, None, workspace)
    }

    /// Some tests referred to “new_for_test_empty()”. We define it here 
    /// as a convenience constructor that sets everything to None, 
    /// with a dummy index and a MockBatchWorkspace.
    pub fn new_for_test_empty() -> Self {
        let index = BatchIndex::Usize(9999);
        let workspace = Arc::new(MockBatchWorkspace::default());
        Self::new_direct(&index, None, None, None, None, None, workspace)
    }

    /// Some tests set the index after constructing. We add a trivial setter:
    pub fn set_index(&mut self, new_index: BatchIndex) {
        self.index = new_index;
    }

    pub fn effective_input_filename(&self) -> PathBuf {
        if let Some(path) = self.input() {
            // If the user/test code explicitly set the input path, use it
            path.clone()
        } else {
            // Otherwise, fall back to workspace
            self.workspace.input_filename(&self.index)
        }
    }

    pub fn effective_output_filename(&self) -> PathBuf {
        if let Some(path) = self.output() {
            path.clone()
        } else {
            self.workspace.output_filename(&self.index)
        }
    }

    pub fn effective_error_filename(&self) -> PathBuf {
        if let Some(path) = self.error() {
            path.clone()
        } else {
            self.workspace.error_filename(&self.index)
        }
    }

    pub fn effective_metadata_filename(&self) -> PathBuf {
        if let Some(path) = self.associated_metadata() {
            path.clone()
        } else {
            self.workspace.metadata_filename(&self.index)
        }
    }

    pub fn effective_seed_manifest_filename(&self) -> PathBuf {
        if let Some(path) = self.seed_manifest() {
            path.clone()
        } else {
            self.workspace.seed_manifest_filename(&self.index)
        }
    }
}

impl BatchFileTriple {

    pub fn new_for_test_unique(workspace: Arc<dyn BatchWorkspaceInterface>) -> Self {
        
        // Now build the triple, but override the “index” or “output filename” with something unique:
        let triple = BatchFileTriple::new_direct(
            // Or pick some new function signature. For now, we pass a mocked index:
            &BatchIndex::new(/*this is random uuid4 */),
            None, None, None, None, None,
            workspace,
        );
        
        // If you prefer, also set triple metadata path, etc. 
        triple
    }

    pub fn new_for_test_with_metadata_path_unique(metadata_path: PathBuf) -> Self {
        // Any random generator or unique ID logic. We'll just do a
        // thread‐local counter or random number for demonstration:
        use std::sync::atomic::{AtomicU64, Ordering};
        static COUNTER: AtomicU64 = AtomicU64::new(1);
        let unique_num = COUNTER.fetch_add(1, Ordering::SeqCst);

        // Then we create an index with that unique number, so the default
        // filenames become "mock_error_{unique_num}.json" etc.
        let index = BatchIndex::Usize(unique_num as usize);

        let triple = Self::new_direct(
            &index,
            None, // no forced input path
            None, // no forced output path
            None, // no forced error path
            Some(metadata_path.clone()),
            None,
            std::sync::Arc::new(MockBatchWorkspace::default()), // or however you handle workspace
        );
        triple
    }

    delegate!{
        to self.workspace {
            pub fn get_done_directory(&self) -> &PathBuf;
        }
    }

    pub fn set_output_path(&mut self, path: Option<PathBuf>) {
        trace!("Setting 'output' path to {:?}", path);
        self.output = path;
    }

    pub fn set_input_path(&mut self, path: Option<PathBuf>) {
        trace!("Setting 'input' path to {:?}", path);
        self.input = path;
    }

    pub fn set_metadata_path(&mut self, path: Option<PathBuf>) {
        trace!("Setting 'associated_metadata' path to {:?}", path);
        self.associated_metadata = path;
    }

    pub fn set_error_path(&mut self, path: Option<PathBuf>) {
        trace!("Setting 'error' path to {:?}", path);
        self.error = path;
    }

    pub fn set_seed_manifest_path(&mut self, path: Option<PathBuf>) {
        trace!("Setting 'seed_manifest' path to {:?}", path);
        self.seed_manifest = path;
    }

    pub fn all_are_none(&self) -> bool {
        trace!("Checking if input, output, and error are all None for batch index={:?}", self.index);
        self.input.is_none() && self.output.is_none() && self.error.is_none()
    }

    //--------------------------------------------
    pub fn new_with_requests(
        requests:  &[LanguageModelBatchAPIRequest], 
        workspace: Arc<dyn BatchWorkspaceInterface>
    ) -> Result<Self,BatchInputCreationError> {

        trace!("Creating new batch triple with provided requests (count={}) in workspace={:?}", requests.len(), workspace);
        let index = BatchIndex::new();

        let batch_input_filename    = workspace.input_filename(&index);
        let batch_output_filename   = workspace.output_filename(&index);
        let batch_error_filename    = workspace.error_filename(&index);
        let batch_metadata_filename = workspace.metadata_filename(&index);
        let batch_seed_manifest_filename = workspace.seed_manifest_filename(&index);

        info!("Creating new batch input file at {:?} with {} requests", batch_input_filename, requests.len());

        batch_mode_batch_scribe::create_batch_input_file(&requests,&batch_input_filename)?;

        trace!("Writing seed manifest at {:?}", batch_seed_manifest_filename);

        batch_mode_batch_scribe::write_seed_manifest(&batch_seed_manifest_filename, requests)?;

        // dev-only checks
        assert!(batch_input_filename.exists());
        assert!(!batch_output_filename.exists());
        assert!(!batch_error_filename.exists());
        assert!(!batch_metadata_filename.exists());
        assert!(batch_seed_manifest_filename.exists());

        Ok(Self {
            index,
            input:               Some(batch_input_filename),
            output:              None,
            error:               None,
            associated_metadata: None,
            seed_manifest:       Some(batch_seed_manifest_filename),
            workspace,
        })
    }

    pub fn new_direct(
        index:               &BatchIndex, 
        input:               Option<PathBuf>, 
        output:              Option<PathBuf>, 
        error:               Option<PathBuf>, 
        associated_metadata: Option<PathBuf>, 
        seed_manifest:       Option<PathBuf>, 
        workspace:           Arc<dyn BatchWorkspaceInterface>
    ) -> Self {
        trace!(
            "Constructing BatchFileTriple::new_direct with index={:?}, input={:?}, output={:?}, error={:?}, metadata={:?}",
            index, input, output, error, associated_metadata
        );
        Self { 
            index: index.clone(), 
            input, 
            output, 
            error, 
            associated_metadata, 
            seed_manifest, 
            workspace 
        }
    }

    /// A convenience constructor used by certain unit tests that only need
    /// to set `associated_metadata` while leaving other paths as None.
    /// We assign a dummy `BatchIndex` and a default MockBatchWorkspace (or any real workspace).
    pub fn new_for_test_with_metadata_path(metadata_path: PathBuf) -> Self {
        trace!(
            "Constructing a test triple with just an associated metadata path: {:?}",
            metadata_path
        );

        let index = BatchIndex::Usize(9999);
        let workspace = Arc::new(MockBatchWorkspace::default());

        Self::new_direct(
            &index,
            None,                 // no input file
            None,                 // no output file
            None,                 // no error file
            Some(metadata_path),  // test sets an associated metadata path
            None,
            workspace
        )
    }

    /// A convenience constructor used by certain unit tests that need to set
    /// specific input, output, and error paths directly (often to temp files).
    /// We assign a dummy `BatchIndex` and a default MockBatchWorkspace.
    pub fn new_for_test_with_in_out_err_paths(
        workspace: Arc<dyn BatchWorkspaceInterface>,
        input:     PathBuf,
        output:    Option<PathBuf>,
        error:     Option<PathBuf>,
    ) -> Self {
        trace!(
            "Constructing a test triple with input={:?}, output={:?}, error={:?}",
            input,
            output,
            error
        );

        let index = BatchIndex::Usize(9999);

        info!(
            "Created new_for_test_with_in_out_err_paths triple with index={:?} in a mock workspace",
            index
        );

        Self::new_direct(
            &index,
            Some(input),
            output,
            error,
            None,
            None,
            workspace,
        )
    }
}

#[cfg(test)]
mod batch_file_triple_filename_accessors_exhaustive_tests {
    use super::*;

    #[traced_test]
    fn input_filename_returns_correct_path() {
        trace!("===== BEGIN TEST: input_filename_returns_correct_path =====");
        let workspace = Arc::new(MockBatchWorkspace::default());
        let triple = BatchFileTriple::new_direct(
            &BatchIndex::new(),
            None,None,None,None,None,
            workspace.clone()
        );
        let path = triple.effective_input_filename();
        debug!("Returned path: {:?}", path);
        pretty_assert_eq!(path, workspace.input_filename(&triple.index()), "Should match workspace input filename");
        trace!("===== END TEST: effective_input_filename_returns_correct_path =====");
    }

    #[traced_test]
    fn output_filename_returns_correct_path() {
        trace!("===== BEGIN TEST: output_filename_returns_correct_path =====");
        let workspace = Arc::new(MockBatchWorkspace::default());
        let triple = BatchFileTriple::new_direct(
            &BatchIndex::new(),
            None,None,None,None,None,
            workspace.clone()
        );
        let path = triple.effective_output_filename();
        debug!("Returned path: {:?}", path);
        pretty_assert_eq!(path, workspace.output_filename(&triple.index()), "Should match workspace output filename");
        trace!("===== END TEST: output_filename_returns_correct_path =====");
    }

    #[traced_test]
    fn error_filename_returns_correct_path() {
        trace!("===== BEGIN TEST: error_filename_returns_correct_path =====");
        let workspace = Arc::new(MockBatchWorkspace::default());
        let triple = BatchFileTriple::new_direct(
            &BatchIndex::new(),
            None,None,None,None,None,
            workspace.clone()
        );
        let path = triple.effective_error_filename();
        debug!("Returned path: {:?}", path);
        pretty_assert_eq!(path, workspace.error_filename(&triple.index()), "Should match workspace error filename");
        trace!("===== END TEST: error_filename_returns_correct_path =====");
    }

    #[traced_test]
    fn metadata_filename_returns_correct_path() {
        trace!("===== BEGIN TEST: metadata_filename_returns_correct_path =====");
        let workspace = Arc::new(MockBatchWorkspace::default());
        let triple = BatchFileTriple::new_direct(
            &BatchIndex::new(),
            None,None,None,None,None,
            workspace.clone()
        );
        let path = triple.effective_metadata_filename();
        debug!("Returned path: {:?}", path);
        pretty_assert_eq!(path, workspace.metadata_filename(&triple.index()), "Should match workspace metadata filename");
        trace!("===== END TEST: metadata_filename_returns_correct_path =====");
    }

    #[traced_test]
    fn set_output_path_updates_field() {
        trace!("===== BEGIN TEST: set_output_path_updates_field =====");
        let workspace = Arc::new(MockBatchWorkspace::default());
        let mut triple = BatchFileTriple::new_direct(
            &BatchIndex::new(),
            None,None,None,None,None,
            workspace
        );
        let new_path = Some(PathBuf::from("test_output.json"));
        triple.set_output_path(new_path.clone());
        debug!("Updated triple: {:?}", triple);
        pretty_assert_eq!(*triple.output(), new_path, "Output path should be updated");
        trace!("===== END TEST: set_output_path_updates_field =====");
    }

    #[traced_test]
    fn set_error_path_updates_field() {
        trace!("===== BEGIN TEST: set_error_path_updates_field =====");
        let workspace = Arc::new(MockBatchWorkspace::default());
        let mut triple = BatchFileTriple::new_direct(
            &BatchIndex::new(),
            None,None,None,None,None,
            workspace
        );
        let new_path = Some(PathBuf::from("test_error.json"));
        triple.set_error_path(new_path.clone());
        debug!("Updated triple: {:?}", triple);
        pretty_assert_eq!(*triple.error(), new_path, "Error path should be updated");
        trace!("===== END TEST: set_error_path_updates_field =====");
    }

    #[traced_test]
    fn all_are_none_returns_true_when_no_paths_present() {
        trace!("===== BEGIN TEST: all_are_none_returns_true_when_no_paths_present =====");
        let workspace = Arc::new(MockBatchWorkspace::default());
        let triple = BatchFileTriple::new_direct(
            &BatchIndex::new(),
            None,None,None,None,None,
            workspace
        );
        debug!("Triple with all None: {:?}", triple);
        assert!(triple.all_are_none(), "Should return true when all fields are None");
        trace!("===== END TEST: all_are_none_returns_true_when_no_paths_present =====");
    }

    #[traced_test]
    fn all_are_none_returns_false_when_any_path_present() {
        trace!("===== BEGIN TEST: all_are_none_returns_false_when_any_path_present =====");
        let workspace = Arc::new(MockBatchWorkspace::default());
        let triple = BatchFileTriple::new_direct(
            &BatchIndex::new(),
            Some(PathBuf::from("some_input.json")),
            None,None,None,None,
            workspace
        );
        debug!("Triple with input path: {:?}", triple);
        assert!(!triple.all_are_none(), "Should return false when any field is present");
        trace!("===== END TEST: all_are_none_returns_false_when_any_path_present =====");
    }

    #[traced_test]
    fn new_with_requests_creates_input_file_and_none_for_others() {
        trace!("===== BEGIN TEST: new_with_requests_creates_input_file_and_none_for_others =====");
        let workspace = Arc::new(MockBatchWorkspace::default());
        let requests = vec![LanguageModelBatchAPIRequest::mock("req-1")];

        let triple_res = BatchFileTriple::new_with_requests(&requests, workspace.clone());
        debug!("Resulting triple: {:?}", triple_res);
        assert!(triple_res.is_ok(), "Should succeed in creating new batch file triple");
        let triple = triple_res.unwrap();

        // Confirm the input file is set and presumably exists in the mock
        assert!(triple.input().is_some(), "Input should not be None");
        assert!(triple.output().is_none(), "Output should be None initially");
        assert!(triple.error().is_none(), "Error should be None initially");

        trace!("===== END TEST: new_with_requests_creates_input_file_and_none_for_others =====");
    }

    #[traced_test]
    fn new_with_requests_fails_if_input_cannot_be_created() {
        trace!("===== BEGIN TEST: new_with_requests_fails_if_input_cannot_be_created =====");
        // This scenario might require a custom workspace that fails file creation.
        let workspace = Arc::new(FailingWorkspace {});
        let requests = vec![LanguageModelBatchAPIRequest::mock("req-2")];

        let triple_res = BatchFileTriple::new_with_requests(&requests, workspace);
        debug!("Resulting triple: {:?}", triple_res);
        assert!(triple_res.is_err(), "Should fail when input file can't be created");

        trace!("===== END TEST: new_with_requests_fails_if_input_cannot_be_created =====");
    }

    #[traced_test]
    fn new_direct_sets_all_fields_as_provided() {
        trace!("===== BEGIN TEST: new_direct_sets_all_fields_as_provided =====");
        let index    = BatchIndex::new();
        let input    = Some(PathBuf::from("input.json"));
        let output   = Some(PathBuf::from("output.json"));
        let error    = Some(PathBuf::from("error.json"));
        let metadata = Some(PathBuf::from("metadata.json"));
        let seed_manifest = Some(PathBuf::from("seed_manifest.json"));

        let workspace = Arc::new(MockBatchWorkspace::default());
        let triple = BatchFileTriple::new_direct(
            &index,
            input.clone(), output.clone(), error.clone(), metadata.clone(), seed_manifest.clone(),
            workspace
        );
        debug!("Constructed triple: {:?}", triple);

        pretty_assert_eq!(triple.index(), &index, "Index should match");
        pretty_assert_eq!(*triple.input(), input, "Input path mismatch");
        pretty_assert_eq!(*triple.output(), output, "Output path mismatch");
        pretty_assert_eq!(*triple.error(), error, "Error path mismatch");
        pretty_assert_eq!(*triple.associated_metadata(), metadata, "Metadata path mismatch");
        trace!("===== END TEST: new_direct_sets_all_fields_as_provided =====");
    }

    #[traced_test]
    fn batch_file_triple_partial_eq_and_ord_work_as_expected() {
        trace!("===== BEGIN TEST: batch_file_triple_partial_eq_and_ord_work_as_expected =====");
        let idx1 = BatchIndex::new();
        let idx2 = BatchIndex::new();

        let triple1 = BatchFileTriple::new_direct(
            &idx1,
            None, None, None, None, None,
            Arc::new(MockBatchWorkspace::default())
        );
        let triple2 = BatchFileTriple::new_direct(
            &idx2,
            None, None, None, None, None,
            Arc::new(MockBatchWorkspace::default())
        );

        // Equality vs difference
        assert_ne!(triple1, triple2, "Distinct indexes should not be equal");
        
        // Ordering checks (the actual ordering depends on the underlying BatchIndex logic)
        let ordering = triple1.cmp(&triple2);
        debug!("Ordering result: {:?}", ordering);
        assert!(
            ordering == std::cmp::Ordering::Less 
            || ordering == std::cmp::Ordering::Greater,
            "They should have a total order"
        );

        trace!("===== END TEST: batch_file_triple_partial_eq_and_ord_work_as_expected =====");
    }
}