ironwork_syntax/
extended.rs

1//! What `--compliance extended` does to the tokens before the parser reads them: each constant
2//! entry comes out and every later use of its name stands for its value, literals joined by `&`
3//! become one literal, and BINARY-SHORT, BINARY-LONG and BINARY-DOUBLE become COMP-5 PICTUREs
4//! (docs/compliance.md).
5
6use crate::lexer::{Tok, Token};
7use crate::{Error, Pos};
8use std::collections::HashMap;
9
10pub const CONSTANT: &str = "IWX0002-W constant entry (Micro Focus and GnuCOBOL; Enterprise COBOL has no level 78 and no CONSTANT clause)";
11pub const CONCATENATION: &str = "IWX0004-W literal concatenation with & (Micro Focus and GnuCOBOL; Enterprise COBOL has none)";
12pub const BINARY_USAGE: &str = "IWX0005-W the COBOL 2002 binary usage (Micro Focus and GnuCOBOL; not Enterprise COBOL's)";
13pub const NO_IDENTIFICATION_HEADER: &str = "IWX0006-W PROGRAM-ID with no IDENTIFICATION DIVISION header before it (COBOL 2002, Micro Focus and GnuCOBOL; Enterprise COBOL requires the header)";
14pub const ASSIGN_ITEM: &str = "IWX0007-W ASSIGN to a data item (Micro Focus and GnuCOBOL; Enterprise COBOL's assignment-name is never a data item)";
15
16/// BINARY-SHORT, BINARY-LONG and BINARY-DOUBLE, and the COMP-5 PICTURE each is: two, four and eight
17/// bytes of native binary.
18const BINARY_USAGES: &[(&str, &str)] = &[("BINARY-SHORT", "9(4)"), ("BINARY-LONG", "9(9)"), ("BINARY-DOUBLE", "9(18)")];
19
20const FIGURATIVES: &[&str] = &["ZERO", "ZEROS", "ZEROES", "SPACE", "SPACES", "HIGH-VALUE", "HIGH-VALUES", "LOW-VALUE", "LOW-VALUES", "QUOTE", "QUOTES", "NULL", "NULLS"];
21
22/// The tokens with constant entries taken out, their names replaced by their values, `&`
23/// concatenations joined and the binary usages rewritten. `cards` are the CBL and PROCESS options,
24/// whose code page reads a hexadecimal literal joined to an alphanumeric one.
25pub fn rewrite(tokens: Vec<Token>, cards: &[String]) -> Result<Vec<Token>, Error> {
26    let mut options = numeric::Options::default();
27    for card in cards {
28        options.apply(card).ok();
29    }
30    let mut r = Rewrite { tokens, at: 0, out: Vec::new(), constants: HashMap::new(), pending: Vec::new(), options };
31    let mut data = false;
32    while r.at < r.tokens.len() {
33        let division = r.tokens.get(r.at + 1).is_some_and(|t| matches!(&t.tok, Tok::Word(w) if w == "DIVISION"));
34        match &r.tokens[r.at].tok {
35            Tok::Word(w) if division => data = w == "DATA",
36            Tok::Number(n) if data && r.out.last().is_none_or(|t| t.tok == Tok::Period) && matches!(r.tokens.get(r.at + 1).map(|t| &t.tok), Some(Tok::Word(_))) && (n == "78" || r.word_at(2) == Some("CONSTANT") && matches!(n.as_str(), "01" | "1")) => {
37                r.constant()?;
38                continue;
39            }
40            Tok::Word(w) if data && (w == "BINARY-CHAR" || BINARY_USAGES.iter().any(|(u, _)| u == w)) => {
41                r.binary_usage()?;
42                continue;
43            }
44            Tok::Word(w) if w == "PROGRAM-ID" && !ends_with_header(&r.out) => r.identification_header(),
45            _ => {}
46        }
47        let value = r.value()?;
48        r.push(value);
49    }
50    if let Some(last) = r.out.last_mut() {
51        last.messages.append(&mut r.pending);
52    }
53    Ok(r.out)
54}
55
56struct Rewrite {
57    tokens: Vec<Token>,
58    at: usize,
59    out: Vec<Token>,
60    constants: HashMap<String, Tok>,
61    /// Messages of tokens taken out, for the next token kept.
62    pending: Vec<Error>,
63    /// The cards' options, whose code page reads a hexadecimal literal joined to an alphanumeric one.
64    options: numeric::Options,
65}
66
67impl Rewrite {
68    fn push(&mut self, mut token: Token) {
69        token.messages.splice(0..0, self.pending.drain(..));
70        self.out.push(token);
71    }
72
73    /// Token `k`, a constant's name replaced by its value and a PICTURE's `(name)` by the number.
74    fn substituted(&self, k: usize) -> Token {
75        let mut token = self.tokens[k].clone();
76        match &token.tok {
77            Tok::Word(w) => {
78                if let Some(value) = self.constants.get(w) {
79                    token.tok = value.clone();
80                    token.spelled = None;
81                }
82            }
83            Tok::Pic(p) => {
84                if let Some(p) = picture(p, &self.constants) {
85                    token.tok = Tok::Pic(p);
86                }
87            }
88            _ => {}
89        }
90        token
91    }
92
93    /// The token at `at`, substituted, and the literals any `&` joins to it.
94    fn value(&mut self) -> Result<Token, Error> {
95        let mut left = self.substituted(self.at);
96        self.at += 1;
97        while self.tokens.get(self.at).is_some_and(|t| t.tok == Tok::Ampersand) {
98            let amp = self.tokens[self.at].clone();
99            if self.at + 1 >= self.tokens.len() {
100                return Err(Error::at(amp.pos, "& with no literal after it"));
101            }
102            let right = self.substituted(self.at + 1);
103            self.at += 2;
104            left = join(left, amp, right, |bytes| self.options.code_page().decode(bytes))?;
105        }
106        Ok(left)
107    }
108
109    fn word_at(&self, ahead: usize) -> Option<&str> {
110        match self.tokens.get(self.at + ahead).map(|t| &t.tok) {
111            Some(Tok::Word(w)) => Some(w),
112            _ => None,
113        }
114    }
115
116    /// A constant entry from its level number, `78 name [IS] [GLOBAL] VALUE [IS] value.` or
117    /// `01 name CONSTANT [IS] [GLOBAL] [AS] value.`, where the value is a literal, a figurative
118    /// constant, a constant already defined, or literals joined by `&`. The entry is taken out of
119    /// the tokens.
120    fn constant(&mut self) -> Result<(), Error> {
121        let level = self.tokens[self.at].clone();
122        let named = self.tokens[self.at + 1].clone();
123        let Tok::Word(name) = named.tok else { unreachable!("the caller saw a word") };
124        let seventy_eight = level.tok == Tok::Number("78".into());
125        self.at += if seventy_eight { 2 } else { 3 };
126        self.pending.extend(level.messages);
127        self.pending.extend(named.messages);
128        let refused = |at: Pos, why: &str| Error::at(at, format!("constant {name}: {why}"));
129        match (self.word_at(0), self.word_at(1)) {
130            (Some("IS"), Some("GLOBAL")) => self.at += 2,
131            (Some("GLOBAL"), _) => self.at += 1,
132            _ => {}
133        }
134        let here = self.tokens.get(self.at).map_or(named.pos, |t| t.pos);
135        if seventy_eight {
136            if self.word_at(0) != Some("VALUE") {
137                return Err(refused(here, "a level-78 entry is VALUE and its value, then a period"));
138            }
139            self.at += 1;
140            if self.word_at(0) == Some("IS") {
141                self.at += 1;
142            }
143        } else if self.word_at(0) == Some("AS") {
144            self.at += 1;
145        }
146        if self.at >= self.tokens.len() {
147            return Err(refused(here, "VALUE with no value"));
148        }
149        let value = self.value()?;
150        let literal = match &value.tok {
151            Tok::Alnum(_) | Tok::Hex(_) | Tok::National(_) | Tok::Number(_) => true,
152            Tok::Word(w) => FIGURATIVES.contains(&w.as_str()),
153            _ => false,
154        };
155        let ended = self.tokens.get(self.at).is_some_and(|t| t.tok == Tok::Period);
156        if !literal || !ended {
157            let at = if literal { self.tokens.get(self.at).map_or(value.pos, |t| t.pos) } else { value.pos };
158            return Err(refused(at, "the value is a literal, a figurative constant, a constant defined before, or literals joined by &; ironwork computes no expression there"));
159        }
160        self.pending.extend(self.tokens[self.at].messages.iter().cloned());
161        self.at += 1;
162        self.pending.push(Error::warning(level.pos, format!("{CONSTANT}: {name} stands for its value wherever it is used after this entry")));
163        self.pending.extend(value.messages);
164        self.constants.insert(name, value.tok);
165        Ok(())
166    }
167
168    /// `IDENTIFICATION DIVISION.` before a PROGRAM-ID that has none, which it means.
169    fn identification_header(&mut self) {
170        let at = self.tokens[self.at].clone();
171        let made = |tok: Tok, messages: Vec<Error>| Token { tok, pos: at.pos, area_a: at.area_a, spelled: None, after_comma: false, messages };
172        let warning = Error::warning(at.pos, format!("{NO_IDENTIFICATION_HEADER}: the program reads as though IDENTIFICATION DIVISION. came before it"));
173        self.push(made(Tok::Word("IDENTIFICATION".into()), vec![warning]));
174        self.push(made(Tok::Word("DIVISION".into()), Vec::new()));
175        self.push(made(Tok::Period, Vec::new()));
176    }
177
178    /// `[USAGE [IS]] BINARY-SHORT|BINARY-LONG|BINARY-DOUBLE [SIGNED|UNSIGNED]` in a data entry, as
179    /// `PIC S9(n) COMP-5`, or `PIC 9(n) COMP-5` when UNSIGNED: SIGNED is the default.
180    fn binary_usage(&mut self) -> Result<(), Error> {
181        let token = self.tokens[self.at].clone();
182        let Tok::Word(usage) = &token.tok else { unreachable!("the caller saw a word") };
183        let Some(&(_, digits)) = BINARY_USAGES.iter().find(|(u, _)| u == usage) else {
184            return Err(Error::at(token.pos, "BINARY-CHAR is a one-byte binary item, and ironwork's binary items are two, four or eight bytes, as Enterprise COBOL's are"));
185        };
186        let mut messages = Vec::new();
187        if self.out.last().is_some_and(|t| t.tok == Tok::Word("IS".into())) && self.out.len() >= 2 && self.out[self.out.len() - 2].tok == Tok::Word("USAGE".into()) {
188            messages.extend(self.out.pop().map(|t| t.messages).unwrap_or_default());
189        }
190        if self.out.last().is_some_and(|t| t.tok == Tok::Word("USAGE".into())) {
191            messages.extend(self.out.pop().map(|t| t.messages).unwrap_or_default());
192        }
193        self.at += 1;
194        let signed = match self.tokens.get(self.at).map(|t| &t.tok) {
195            Some(Tok::Word(w)) if w == "SIGNED" || w == "UNSIGNED" => {
196                self.at += 1;
197                w == "SIGNED"
198            }
199            _ => true,
200        };
201        let picture = format!("{}{digits}", if signed { "S" } else { "" });
202        let shown = format!("{BINARY_USAGE}: {usage}{} is read as PIC {picture} COMP-5", if signed { "" } else { " UNSIGNED" });
203        messages.push(Error::warning(token.pos, shown));
204        messages.extend(token.messages.iter().cloned());
205        let made = |tok: Tok, messages: Vec<Error>| Token { tok, pos: token.pos, area_a: false, spelled: None, after_comma: false, messages };
206        self.push(made(Tok::Word("PIC".into()), messages));
207        self.push(made(Tok::Pic(picture), Vec::new()));
208        self.push(made(Tok::Word("COMP-5".into()), Vec::new()));
209        Ok(())
210    }
211}
212
213/// Whether the tokens end with `IDENTIFICATION DIVISION.` or `ID DIVISION.`.
214fn ends_with_header(tokens: &[Token]) -> bool {
215    matches!(tokens, [.., a, b, c] if matches!(&a.tok, Tok::Word(w) if w == "IDENTIFICATION" || w == "ID") && b.tok == Tok::Word("DIVISION".into()) && c.tok == Tok::Period)
216}
217
218/// `(name)` in a PICTURE, where `name` is a constant whose value is an unsigned integer, with the
219/// integer in its place.
220fn picture(text: &str, constants: &HashMap<String, Tok>) -> Option<String> {
221    let (mut out, mut rest, mut changed) = (String::new(), text, false);
222    while let Some(open) = rest.find('(') {
223        let Some(close) = rest[open..].find(')').map(|c| open + c) else { break };
224        match constants.get(&rest[open + 1..close]) {
225            Some(Tok::Number(n)) if n.bytes().all(|b| b.is_ascii_digit()) => {
226                out.push_str(&rest[..=open]);
227                out.push_str(n);
228                out.push(')');
229                changed = true;
230            }
231            _ => out.push_str(&rest[..=close]),
232        }
233        rest = &rest[close + 1..];
234    }
235    out.push_str(rest);
236    changed.then_some(out)
237}
238
239/// `left & right` as one literal, at `left`'s place. Alphanumeric and hexadecimal literals join
240/// into an alphanumeric one, a hexadecimal one's bytes read in the program's code page, two
241/// hexadecimal literals into a hexadecimal one, and national literals into a national one.
242fn join(left: Token, amp: Token, right: Token, decode: impl Fn(&[u8]) -> String) -> Result<Token, Error> {
243    let tok = match (&left.tok, &right.tok) {
244        (Tok::Alnum(a), Tok::Alnum(b)) => Tok::Alnum(format!("{a}{b}")),
245        (Tok::Hex(a), Tok::Hex(b)) => Tok::Hex([a.as_slice(), b].concat()),
246        (Tok::Alnum(a), Tok::Hex(b)) => Tok::Alnum(format!("{a}{}", decode(b))),
247        (Tok::Hex(a), Tok::Alnum(b)) => Tok::Alnum(format!("{}{b}", decode(a))),
248        (Tok::National(a), Tok::National(b)) => Tok::National(format!("{a}{b}")),
249        _ => return Err(Error::at(amp.pos, "& joins two alphanumeric or hexadecimal literals, or two national literals, either of which may be a level-78 constant standing for one")),
250    };
251    let mut messages = left.messages;
252    messages.push(Error::warning(amp.pos, format!("{CONCATENATION}: the literals on either side are one literal")));
253    messages.extend(amp.messages);
254    messages.extend(right.messages);
255    Ok(Token { tok, pos: left.pos, area_a: left.area_a, spelled: None, after_comma: left.after_comma, messages })
256}
257
258#[cfg(test)]
259mod tests {
260    use crate::ast::{Literal, Usage};
261    use crate::copy::Libraries;
262    use numeric::Compliance;
263
264    fn source(data: &str, procedure: &str) -> String {
265        format!("       IDENTIFICATION DIVISION.\n       PROGRAM-ID. T.\n       DATA DIVISION.\n       WORKING-STORAGE SECTION.\n{data}       PROCEDURE DIVISION.\n{procedure}           GOBACK.\n")
266    }
267
268    fn extended(text: &str) -> Result<crate::ast::Program, crate::Error> {
269        crate::parse_with(text, &Libraries::default().with_compliance(Compliance::Extended))
270    }
271
272    #[test]
273    fn a_constant_stands_for_its_value_in_pictures_values_occurs_and_statements() {
274        let data = concat!(
275            "       78  MAX-LEN VALUE 3.\n",
276            "       78  GREETING IS GLOBAL VALUE IS 'AB' & X'C1'.\n",
277            "       01  QUOTED CONSTANT AS 'Q'.\n",
278            "       01  NOTHING CONSTANT GLOBAL SPACE.\n",
279            "       01  X PIC X(MAX-LEN) VALUE GREETING.\n",
280            "       01  G.\n",
281            "           05 T OCCURS MAX-LEN TIMES PIC X VALUE NOTHING.\n",
282        );
283        let p = extended(&source(data, "           MOVE QUOTED TO X.\n")).unwrap();
284        let names: Vec<&str> = p.working_storage.iter().filter_map(|e| e.name.as_deref()).collect();
285        assert_eq!(names, ["X", "G", "T"]);
286        let x = &p.working_storage[0];
287        assert_eq!((x.picture.as_deref(), x.value.clone()), (Some("X(3)"), Some(Literal::Alnum("ABA".into()))));
288        assert_eq!(p.working_storage[2].occurs, Some(3));
289        let warnings: Vec<(u32, String)> = p.messages.iter().map(|m| (m.pos.line, m.message.clone())).collect();
290        assert_eq!(warnings.len(), 5, "{warnings:?}");
291        assert!(warnings[..2].iter().all(|(_, m)| m.starts_with(super::CONSTANT)) && warnings[0].0 == 5 && warnings[0].1.ends_with("MAX-LEN stands for its value wherever it is used after this entry"));
292        assert!(warnings[2].1.starts_with(super::CONCATENATION) && warnings[2].0 == 6);
293        assert!(warnings[3..].iter().all(|(_, m)| m.starts_with(super::CONSTANT)));
294        assert!(format!("{:?}", p.paragraphs[0].statements[0]).contains("Alnum(\"Q\")"));
295    }
296
297    #[test]
298    fn a_constant_whose_value_is_an_expression_or_missing_is_refused_by_name() {
299        let refused = |data: &str| extended(&source(data, "")).unwrap_err().message;
300        assert_eq!(refused("       78  N VALUE 1 + 2.\n"), "constant N: the value is a literal, a figurative constant, a constant defined before, or literals joined by &; ironwork computes no expression there");
301        assert_eq!(refused("       78  N PIC 9 VALUE 1.\n"), "constant N: a level-78 entry is VALUE and its value, then a period");
302        assert!(refused("       01  N PIC X VALUE 'A' & B.\n").starts_with("& joins two alphanumeric or hexadecimal literals"));
303        assert!(refused("       01  N PIC X VALUE 'A' & N'B'.\n").starts_with("& joins two alphanumeric or hexadecimal literals"));
304    }
305
306    #[test]
307    fn the_binary_usages_are_comp_5_pictures_and_binary_char_is_refused() {
308        let data = "       01  A USAGE IS BINARY-LONG.\n       01  B BINARY-SHORT UNSIGNED VALUE 7.\n       01  C BINARY-DOUBLE SIGNED.\n";
309        let p = extended(&source(data, "")).unwrap();
310        let read: Vec<(Option<&str>, Option<Usage>)> = p.working_storage.iter().map(|e| (e.picture.as_deref(), e.usage)).collect();
311        assert_eq!(read, [(Some("S9(9)"), Some(Usage::NativeBinary)), (Some("9(4)"), Some(Usage::NativeBinary)), (Some("S9(18)"), Some(Usage::NativeBinary))]);
312        assert_eq!(p.working_storage[1].value, Some(Literal::Number("7".into())));
313        let shown: Vec<String> = p.messages.iter().map(|m| m.message.clone()).collect();
314        assert_eq!(shown[1], format!("{}: BINARY-SHORT UNSIGNED is read as PIC 9(4) COMP-5", super::BINARY_USAGE));
315        let refused = extended(&source("       01  D BINARY-CHAR.\n", "")).unwrap_err();
316        assert!(refused.message.starts_with("BINARY-CHAR is a one-byte binary item") && refused.pos.line == 5);
317    }
318
319    #[test]
320    fn strict_reads_none_of_it() {
321        let strict = |data: &str| crate::parse(&source(data, ""));
322        assert_eq!(strict("       01  N PIC X(2) VALUE 'A' & 'B'.\n").unwrap_err().message, "literal concatenation with & is not Enterprise COBOL's");
323        assert_eq!(strict("       78  N VALUE 1.\n").unwrap().working_storage[0].level, 78);
324        assert!(strict("       01  A BINARY-LONG.\n").unwrap_err().message.contains("BINARY-LONG is not a data description clause"));
325    }
326}
ironwork_syntax/extended.rs

ironwork_syntax/
extended.rs
