ironwork_exec/
files.rs

1//! Files. ASSIGN names a DD, and only the operator maps a DD to a host file, as JCL does, so a
2//! program reaches no file it was not given. A binary DD holds z/OS records byte for byte:
3//! fixed-length records back to back, or variable-length records each behind a 4-byte RDW. A text
4//! DD holds UTF-8 lines, converted through the program's code page. Sequential files stream;
5//! indexed and relative files are held in memory (see [`Keyed`]).
6
7use std::cmp::Ordering;
8use std::collections::{BTreeMap, BTreeSet, HashMap};
9use std::fs::{File, OpenOptions};
10use std::io::{self, BufRead, BufReader, BufWriter, Read, Write};
11use std::ops::Bound;
12use std::path::PathBuf;
13use syntax::ast::OpenMode;
14use zarch::ebcdic::{self, CodePage};
15
16#[derive(Clone, Copy, Debug, PartialEq, Eq)]
17pub enum Format {
18    Fixed,
19    Variable,
20    Text,
21}
22
23#[derive(Clone, Debug, PartialEq, Eq)]
24pub struct Dd {
25    pub path: PathBuf,
26    pub format: Option<Format>,
27}
28
29/// The DDs a run may use: given explicitly, or read from `DD_<NAME>` in the environment.
30#[derive(Clone, Debug, Default)]
31pub struct Dds {
32    given: HashMap<String, Dd>,
33    environment: bool,
34}
35
36fn dd_value(value: &str) -> Result<Dd, String> {
37    let (path, format) = match value.rsplit_once(':') {
38        Some((p, "text")) => (p, Some(Format::Text)),
39        Some((p, "fixed" | "f" | "F")) => (p, Some(Format::Fixed)),
40        Some((p, "variable" | "v" | "V")) => (p, Some(Format::Variable)),
41        _ => (value, None),
42    };
43    if path.is_empty() {
44        return Err(format!("DD {value}: no path"));
45    }
46    Ok(Dd { path: PathBuf::from(path), format })
47}
48
49impl Dds {
50    /// DDs from `NAME=path[:text|:fixed|:variable]` specs, and from the environment when `environment`.
51    pub fn new(specs: &[String], environment: bool) -> Result<Self, String> {
52        let mut given = HashMap::new();
53        for spec in specs {
54            let (name, value) = spec.split_once('=').ok_or_else(|| format!("--dd {spec}: expected NAME=path"))?;
55            given.insert(name.to_ascii_uppercase(), dd_value(value)?);
56        }
57        Ok(Self { given, environment })
58    }
59
60    pub fn get(&self, name: &str) -> Option<Dd> {
61        if let Some(dd) = self.given.get(name) {
62            return Some(dd.clone());
63        }
64        if !self.environment {
65            return None;
66        }
67        std::env::var(format!("DD_{name}")).ok().and_then(|v| dd_value(&v).ok())
68    }
69}
70
71enum Handle {
72    Reader(BufReader<File>),
73    Writer(BufWriter<File>),
74    Keyed(Box<Keyed>),
75    Empty,
76}
77
78/// Where a key lies in a record.
79#[derive(Clone, Copy, Debug, PartialEq, Eq)]
80pub struct KeySpan {
81    pub offset: usize,
82    pub len: usize,
83}
84
85impl KeySpan {
86    pub fn of(&self, record: &[u8]) -> Vec<u8> {
87        let mut key: Vec<u8> = record.iter().skip(self.offset).take(self.len).copied().collect();
88        key.resize(self.len, ebcdic::SPACE);
89        key
90    }
91}
92
93/// How the records of a file held in memory are keyed. Record numbers and positions are held as
94/// eight big-endian bytes, so that they sort.
95#[derive(Clone, Debug, PartialEq, Eq)]
96pub enum Keying {
97    /// A sequential file opened I-O, by position.
98    Position,
99    /// A relative file, by record number. In the DD an empty slot is a record of no bytes, or of
100    /// zero bytes only.
101    Relative,
102    /// An indexed file: the prime key, then each alternate key and whether it allows duplicates.
103    Indexed { prime: KeySpan, alternates: Vec<(KeySpan, bool)> },
104}
105
106/// The highest relative record number: past it WRITE is a boundary violation (status 24). It
107/// stands in for the space a cluster is defined with, so that a stray record number cannot make
108/// CLOSE write millions of empty slots.
109pub const MAX_RELATIVE: u64 = 16_777_215;
110
111pub fn record_number(n: u64) -> Vec<u8> {
112    n.to_be_bytes().to_vec()
113}
114
115pub fn number_of(key: &[u8]) -> u64 {
116    u64::from_be_bytes(key.try_into().unwrap_or([0; 8]))
117}
118
119/// An alternate index entry: alternate key, arrival number, prime key. Duplicates come back in
120/// the order they arrived, as VSAM keeps them; records loaded from a DD arrive in prime-key order,
121/// as BLDINDEX builds an alternate index.
122type Entry = (Vec<u8>, u64, Vec<u8>);
123
124struct Alternate {
125    span: KeySpan,
126    duplicates: bool,
127    index: BTreeSet<Entry>,
128    arrival: HashMap<Vec<u8>, u64>,
129}
130
131impl Alternate {
132    fn holders<'a>(&'a self, value: &'a [u8]) -> impl Iterator<Item = &'a Entry> {
133        self.index.range((value.to_vec(), 0, Vec::new())..).take_while(move |(v, ..)| v == value)
134    }
135}
136
137/// The file position indicator.
138#[derive(Clone, Debug)]
139enum Cursor {
140    /// After OPEN: READ NEXT reads the first record by the prime key.
141    First,
142    /// A key of reference (0 the prime key, then each alternate), a place in its index, and
143    /// whether the record there is itself next (after START) or was the last read.
144    At { which: usize, at: Entry, inclusive: bool },
145    /// After an AT END condition or a START that found nothing: READ NEXT fails with status 46.
146    Undefined,
147}
148
149/// A record READ found.
150pub struct Found {
151    pub key: Vec<u8>,
152    pub record: Vec<u8>,
153    /// The next record by the alternate key of reference has the same key (status 02).
154    pub duplicate: bool,
155}
156
157/// An indexed or relative file, or a sequential file opened I-O, held in memory from OPEN to
158/// CLOSE. The DD holds the records in key (or record-number) order, as an IDCAMS REPRO unload of a
159/// KSDS or RRDS does, and CLOSE writes them back that way when they changed.
160pub struct Keyed {
161    pub keying: Keying,
162    records: BTreeMap<Vec<u8>, Vec<u8>>,
163    alternates: Vec<Alternate>,
164    arrivals: u64,
165    cursor: Cursor,
166    /// The key of the record the last statement on the file, a successful READ, returned: what a
167    /// sequential REWRITE or DELETE acts on.
168    pub last_read: Option<Vec<u8>>,
169    dirty: bool,
170    path: Option<PathBuf>,
171    record_len: usize,
172    page: &'static CodePage,
173}
174
175impl Keyed {
176    fn new(keying: Keying, path: Option<PathBuf>, record_len: usize, page: &'static CodePage) -> Self {
177        let alternates = match &keying {
178            Keying::Indexed { alternates, .. } => {
179                alternates.iter().map(|&(span, duplicates)| Alternate { span, duplicates, index: BTreeSet::new(), arrival: HashMap::new() }).collect()
180            }
181            _ => Vec::new(),
182        };
183        Self { keying, records: BTreeMap::new(), alternates, arrivals: 0, cursor: Cursor::First, last_read: None, dirty: true, path, record_len, page }
184    }
185
186    pub fn prime_key(&self, record: &[u8]) -> Option<Vec<u8>> {
187        match &self.keying {
188            Keying::Indexed { prime, .. } => Some(prime.of(record)),
189            _ => None,
190        }
191    }
192
193    pub fn highest_key(&self) -> Option<&Vec<u8>> {
194        self.records.keys().next_back()
195    }
196
197    pub fn record(&self, key: &[u8]) -> Option<&Vec<u8>> {
198        self.records.get(key)
199    }
200
201    fn key_len(&self, which: usize) -> usize {
202        match (&self.keying, which) {
203            (Keying::Indexed { prime, .. }, 0) => prime.len,
204            (Keying::Indexed { .. }, n) => self.alternates[n - 1].span.len,
205            _ => 8,
206        }
207    }
208
209    /// The record whose key `which` is `value`: for an alternate key, the first of any duplicates.
210    pub fn get(&self, which: usize, value: &[u8]) -> Option<Found> {
211        if which == 0 {
212            return self.records.get(value).map(|r| Found { key: value.to_vec(), record: r.clone(), duplicate: false });
213        }
214        let mut hits = self.alternates[which - 1].holders(value);
215        let key = hits.next()?.2.clone();
216        Some(Found { record: self.records[&key].clone(), key, duplicate: hits.next().is_some() })
217    }
218
219    /// Adds a record, or replaces the one with its key: Err with status 22 when it would take a
220    /// unique alternate key another record holds. Ok(true) when it shares an alternate key that
221    /// allows duplicates (status 02).
222    fn put(&mut self, key: Vec<u8>, record: Vec<u8>) -> Result<bool, &'static str> {
223        let mut shared = false;
224        for alt in &self.alternates {
225            let value = alt.span.of(&record);
226            let others = alt.holders(&value).any(|(_, _, k)| *k != key);
227            if others && !alt.duplicates {
228                return Err("22");
229            }
230            shared |= others;
231        }
232        let old = self.records.get(&key).cloned();
233        let mut arrivals = self.arrivals;
234        for alt in &mut self.alternates {
235            let now = alt.span.of(&record);
236            if let Some(old) = &old {
237                let was = alt.span.of(old);
238                if was == now {
239                    continue;
240                }
241                let seq = alt.arrival[&key];
242                alt.index.remove(&(was, seq, key.clone()));
243            }
244            alt.index.insert((now, arrivals, key.clone()));
245            alt.arrival.insert(key.clone(), arrivals);
246            arrivals += 1;
247        }
248        self.arrivals = arrivals;
249        self.records.insert(key, record);
250        self.dirty = true;
251        Ok(shared)
252    }
253
254    /// Adds a record: Err with status 22 when its prime key, or a unique alternate key, is already
255    /// there; Ok(true) when it shares an alternate key that allows duplicates (status 02).
256    pub fn insert(&mut self, key: Vec<u8>, record: Vec<u8>) -> Result<bool, &'static str> {
257        if self.records.contains_key(&key) {
258            return Err("22");
259        }
260        self.put(key, record)
261    }
262
263    /// Replaces a record: Err with status 23 when none has its key, 22 as for [`Keyed::insert`].
264    pub fn replace(&mut self, key: Vec<u8>, record: Vec<u8>) -> Result<bool, &'static str> {
265        if !self.records.contains_key(&key) {
266            return Err("23");
267        }
268        self.put(key, record)
269    }
270
271    pub fn remove(&mut self, key: &[u8]) -> Option<Vec<u8>> {
272        let record = self.records.remove(key)?;
273        for alt in &mut self.alternates {
274            if let Some(seq) = alt.arrival.remove(key) {
275                alt.index.remove(&(alt.span.of(&record), seq, key.to_vec()));
276            }
277        }
278        self.dirty = true;
279        Some(record)
280    }
281
282    /// START: positions at the first record whose key `which`, compared over the length of `value`
283    /// (a longer `value` is cut to the key's length), is `wanted` to `value`, or equal when
284    /// `or_equal`. False, and no position, when there is none.
285    pub fn start(&mut self, which: usize, wanted: Ordering, or_equal: bool, value: &[u8]) -> bool {
286        let value = &value[..value.len().min(self.key_len(which))];
287        let fits = |k: &[u8]| {
288            let o = k[..value.len().min(k.len())].cmp(value);
289            o == wanted || (or_equal && o == Ordering::Equal)
290        };
291        let from = (value.to_vec(), 0, Vec::new());
292        let found = match which {
293            0 => self.records.range(from.0..).map(|(k, _)| k).find(|k| fits(k)).map(|k| (k.clone(), 0, Vec::new())),
294            n => self.alternates[n - 1].index.range(from..).find(|(k, ..)| fits(k)).cloned(),
295        };
296        self.last_read = None;
297        match found {
298            Some(at) => {
299                self.cursor = Cursor::At { which, at, inclusive: true };
300                true
301            }
302            None => {
303                self.lose_position();
304                false
305            }
306        }
307    }
308
309    /// After a READ that found nothing: READ NEXT has no valid next record.
310    pub fn lose_position(&mut self) {
311        self.cursor = Cursor::Undefined;
312        self.last_read = None;
313    }
314
315    /// Makes the record just read by key `which` the one READ NEXT goes on from.
316    pub fn read_at(&mut self, which: usize, key: &[u8]) {
317        let at = match which {
318            0 => (key.to_vec(), 0, Vec::new()),
319            n => {
320                let alt = &self.alternates[n - 1];
321                (alt.span.of(&self.records[key]), alt.arrival[key], key.to_vec())
322            }
323        };
324        self.cursor = Cursor::At { which, at, inclusive: false };
325        self.last_read = Some(key.to_vec());
326    }
327
328    /// The first record by prime key after `from` (at it too when `inclusive`), or before it when
329    /// `backward`: what a CICS browse reads, holding its own position rather than the file's.
330    pub fn seek(&self, from: &[u8], inclusive: bool, backward: bool) -> Option<(Vec<u8>, Vec<u8>)> {
331        let edge = if inclusive { Bound::Included(from.to_vec()) } else { Bound::Excluded(from.to_vec()) };
332        let mut hits = if backward { self.records.range((Bound::Unbounded, edge)) } else { self.records.range((edge, Bound::Unbounded)) };
333        let hit = if backward { hits.next_back() } else { hits.next() };
334        hit.map(|(k, r)| (k.clone(), r.clone()))
335    }
336
337    /// READ NEXT, or READ PREVIOUS when `backward`: None at the end, Err(46) with no position.
338    pub fn step(&mut self, backward: bool) -> Result<Option<Found>, &'static str> {
339        let (which, bound) = match &self.cursor {
340            Cursor::Undefined => return Err("46"),
341            Cursor::First => (0, Bound::Unbounded),
342            Cursor::At { which, at, inclusive: true } => (*which, Bound::Included(at.clone())),
343            Cursor::At { which, at, inclusive: false } => (*which, Bound::Excluded(at.clone())),
344        };
345        let range = if backward { (Bound::Unbounded, bound) } else { (bound, Bound::Unbounded) };
346        let hit = if which == 0 {
347            let mut keys = self.records.range((range.0.map(|(k, ..)| k), range.1.map(|(k, ..)| k))).map(|(k, _)| k);
348            let key = if backward { keys.next_back() } else { keys.next() };
349            key.map(|k| (k.clone(), false))
350        } else {
351            let mut entries = self.alternates[which - 1].index.range(range);
352            let entry = if backward { entries.next_back() } else { entries.next() };
353            entry.map(|(value, _, key)| {
354                let after = if backward { entries.next_back() } else { entries.next() };
355                (key.clone(), after.is_some_and(|(v, ..)| v == value))
356            })
357        };
358        let Some((key, duplicate)) = hit else {
359            self.lose_position();
360            return Ok(None);
361        };
362        self.read_at(which, &key);
363        Ok(Some(Found { record: self.records[&key].clone(), key, duplicate }))
364    }
365
366    fn inbound(&self, raw: Vec<u8>, format: Format) -> Vec<u8> {
367        let mut record = match format {
368            Format::Text => {
369                let unknown = self.page.encode_char('?').unwrap_or(0x6F);
370                String::from_utf8_lossy(&raw).chars().map(|c| self.page.encode_char(c).unwrap_or(unknown)).collect()
371            }
372            _ => raw,
373        };
374        if format != Format::Variable {
375            record.resize(self.record_len.max(record.len()), ebcdic::SPACE);
376        }
377        record
378    }
379
380    fn outbound(&self, record: &[u8], format: Format) -> Vec<u8> {
381        match format {
382            Format::Text => format!("{}\n", self.page.decode(record).trim_end()).into_bytes(),
383            Format::Fixed => {
384                let mut r = record.to_vec();
385                r.resize(self.record_len, ebcdic::SPACE);
386                r
387            }
388            Format::Variable => record.to_vec(),
389        }
390    }
391
392    fn save(&self, format: Format) -> io::Result<()> {
393        let Some(path) = self.path.as_ref().filter(|_| self.dirty) else { return Ok(()) };
394        let mut out = Open { mode: OpenMode::Output, format, handle: Handle::Writer(BufWriter::new(File::create(path)?)) };
395        let empty = match format {
396            Format::Fixed => vec![0; self.record_len],
397            Format::Variable => Vec::new(),
398            Format::Text => b"\n".to_vec(),
399        };
400        let mut next = 1;
401        for (key, record) in &self.records {
402            if self.keying == Keying::Relative {
403                for _ in next..number_of(key) {
404                    out.write(&empty)?;
405                }
406                next = number_of(key) + 1;
407            }
408            out.write(&self.outbound(record, format))?;
409        }
410        out.close()
411    }
412}
413
414pub struct Open {
415    pub mode: OpenMode,
416    pub format: Format,
417    handle: Handle,
418}
419
420/// What a READ found.
421pub enum Record {
422    Data(Vec<u8>),
423    /// A record whose length did not match: the bytes read, for FILE STATUS 04.
424    WrongLength(Vec<u8>),
425    End,
426}
427
428/// Opens a file held in memory. With no DD (an OPTIONAL file) or no file at the DD's path, it
429/// starts empty; OUTPUT always starts empty. `record_len` is the longest record.
430pub fn open_keyed(dd: Option<&Dd>, mode: OpenMode, format: Format, keying: Keying, record_len: usize, page: &'static CodePage) -> io::Result<Open> {
431    let path = dd.filter(|_| mode != OpenMode::Input).map(|d| d.path.clone());
432    let mut keyed = Keyed::new(keying, path, record_len, page);
433    if let Some(dd) = dd.filter(|d| mode != OpenMode::Output && d.path.exists()) {
434        let mut reader = open(dd, OpenMode::Input, format)?;
435        let mut number = 0u64;
436        loop {
437            let raw = match reader.read(record_len)? {
438                Record::End => break,
439                Record::Data(r) | Record::WrongLength(r) => r,
440            };
441            number += 1;
442            if keyed.keying == Keying::Relative && raw.iter().all(|&b| b == 0) {
443                continue;
444            }
445            let record = keyed.inbound(raw, format);
446            let key = keyed.prime_key(&record).unwrap_or_else(|| record_number(number));
447            keyed.insert(key, record).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, format!("{}: two records with one key", dd.path.display())))?;
448        }
449        keyed.dirty = false;
450    }
451    Ok(Open { mode, format, handle: Handle::Keyed(Box::new(keyed)) })
452}
453
454pub fn open(dd: &Dd, mode: OpenMode, format: Format) -> io::Result<Open> {
455    let handle = match mode {
456        OpenMode::Input => Handle::Reader(BufReader::new(File::open(&dd.path)?)),
457        OpenMode::Output => Handle::Writer(BufWriter::new(File::create(&dd.path)?)),
458        OpenMode::Extend => Handle::Writer(BufWriter::new(OpenOptions::new().append(true).create(true).open(&dd.path)?)),
459        OpenMode::InputOutput => return Err(io::Error::new(io::ErrorKind::Unsupported, "OPEN I-O of a line-sequential file")),
460    };
461    Ok(Open { mode, format, handle })
462}
463
464/// An OPTIONAL input file with no DD: every READ is at end.
465pub fn absent() -> Open {
466    Open { mode: OpenMode::Input, format: Format::Fixed, handle: Handle::Empty }
467}
468
469impl Open {
470    pub fn keyed(&mut self) -> Option<&mut Keyed> {
471        match &mut self.handle {
472            Handle::Keyed(k) => Some(k),
473            _ => None,
474        }
475    }
476
477    pub fn is_keyed(&self) -> bool {
478        matches!(self.handle, Handle::Keyed(_))
479    }
480
481    pub fn read(&mut self, fixed_len: usize) -> io::Result<Record> {
482        let reader = match &mut self.handle {
483            Handle::Reader(r) => r,
484            Handle::Empty => return Ok(Record::End),
485            Handle::Writer(_) | Handle::Keyed(_) => return Err(io::Error::other("a sequential READ of a file not open for it")),
486        };
487        match self.format {
488            Format::Fixed => {
489                let mut buf = Vec::with_capacity(fixed_len);
490                reader.by_ref().take(fixed_len as u64).read_to_end(&mut buf)?;
491                Ok(match buf.len() {
492                    0 => Record::End,
493                    n if n == fixed_len => Record::Data(buf),
494                    _ => Record::WrongLength(buf),
495                })
496            }
497            Format::Variable => {
498                let mut rdw = [0u8; 4];
499                match reader.read_exact(&mut rdw) {
500                    Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(Record::End),
501                    other => other?,
502                }
503                let len = u16::from_be_bytes([rdw[0], rdw[1]]) as usize;
504                if len < 4 {
505                    return Err(io::Error::new(io::ErrorKind::InvalidData, format!("an RDW of length {len}")));
506                }
507                let mut buf = vec![0u8; len - 4];
508                reader.read_exact(&mut buf)?;
509                Ok(Record::Data(buf))
510            }
511            Format::Text => {
512                let mut line = Vec::new();
513                if reader.read_until(b'\n', &mut line)? == 0 {
514                    return Ok(Record::End);
515                }
516                while matches!(line.last(), Some(b'\n' | b'\r')) {
517                    line.pop();
518                }
519                Ok(Record::Data(line))
520            }
521        }
522    }
523
524    pub fn write(&mut self, bytes: &[u8]) -> io::Result<()> {
525        let Handle::Writer(w) = &mut self.handle else {
526            return Err(io::Error::other("WRITE to a file not opened for output"));
527        };
528        match self.format {
529            Format::Fixed | Format::Text => w.write_all(bytes),
530            Format::Variable => {
531                let len = u16::try_from(bytes.len() + 4).map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "a record over 32,760 bytes"))?;
532                w.write_all(&len.to_be_bytes())?;
533                w.write_all(&[0, 0])?;
534                w.write_all(bytes)
535            }
536        }
537    }
538
539    pub fn close(self) -> io::Result<()> {
540        match self.handle {
541            Handle::Writer(mut w) => w.flush(),
542            Handle::Keyed(k) => k.save(self.format),
543            _ => Ok(()),
544        }
545    }
546}
547
548#[cfg(test)]
549mod tests {
550    use super::*;
551
552    #[test]
553    fn dd_specs() {
554        let dds = Dds::new(&["IN=/tmp/a.txt:text".into(), "out=/tmp/b.dat".into()], false).unwrap();
555        assert_eq!(dds.get("IN"), Some(Dd { path: "/tmp/a.txt".into(), format: Some(Format::Text) }));
556        assert_eq!(dds.get("OUT").unwrap().format, None);
557        assert!(dds.get("NONE").is_none());
558        assert!(Dds::new(&["NOEQUALS".into()], false).is_err());
559    }
560
561    #[test]
562    fn variable_records_round_trip_through_rdws() {
563        let path = std::env::temp_dir().join(format!("ironwork-vb-{}", std::process::id()));
564        let dd = Dd { path: path.clone(), format: None };
565        let mut out = open(&dd, OpenMode::Output, Format::Variable).unwrap();
566        out.write(b"AB").unwrap();
567        out.write(b"CDEF").unwrap();
568        out.close().unwrap();
569        assert_eq!(std::fs::read(&path).unwrap(), [0, 6, 0, 0, b'A', b'B', 0, 8, 0, 0, b'C', b'D', b'E', b'F']);
570        let mut input = open(&dd, OpenMode::Input, Format::Variable).unwrap();
571        assert!(matches!(input.read(0).unwrap(), Record::Data(d) if d == b"AB"));
572        assert!(matches!(input.read(0).unwrap(), Record::Data(d) if d == b"CDEF"));
573        assert!(matches!(input.read(0).unwrap(), Record::End));
574    }
575}
ironwork_exec/files.rs

ironwork_exec/
files.rs
