ironwork_syntax/
extended.rs

1//! What `--compliance extended` does to the tokens before the parser reads them: each constant
2//! entry comes out and every later use of its name stands for its value, literals joined by `&`
3//! become one literal, and BINARY-SHORT, BINARY-LONG and BINARY-DOUBLE become COMP-5 PICTUREs
4//! (docs/compliance.md).
5
6use crate::lexer::{Tok, Token};
7use crate::{Error, Pos};
8use std::collections::HashMap;
9
10pub const CONSTANT: &str = "IWX0002-W constant entry (Micro Focus and GnuCOBOL; Enterprise COBOL has no level 78 and no CONSTANT clause)";
11pub const CONCATENATION: &str = "IWX0004-W literal concatenation with & (Micro Focus and GnuCOBOL; Enterprise COBOL has none)";
12pub const BINARY_USAGE: &str = "IWX0005-W the COBOL 2002 binary usage (Micro Focus and GnuCOBOL; not Enterprise COBOL's)";
13pub const NO_IDENTIFICATION_HEADER: &str = "IWX0006-W PROGRAM-ID with no IDENTIFICATION DIVISION header before it (COBOL 2002, Micro Focus and GnuCOBOL; Enterprise COBOL requires the header)";
14
15/// BINARY-SHORT, BINARY-LONG and BINARY-DOUBLE, and the COMP-5 PICTURE each is: two, four and eight
16/// bytes of native binary.
17const BINARY_USAGES: &[(&str, &str)] = &[("BINARY-SHORT", "9(4)"), ("BINARY-LONG", "9(9)"), ("BINARY-DOUBLE", "9(18)")];
18
19const FIGURATIVES: &[&str] = &["ZERO", "ZEROS", "ZEROES", "SPACE", "SPACES", "HIGH-VALUE", "HIGH-VALUES", "LOW-VALUE", "LOW-VALUES", "QUOTE", "QUOTES", "NULL", "NULLS"];
20
21/// The tokens with constant entries taken out, their names replaced by their values, `&`
22/// concatenations joined and the binary usages rewritten. `cards` are the CBL and PROCESS options,
23/// whose code page reads a hexadecimal literal joined to an alphanumeric one.
24pub fn rewrite(tokens: Vec<Token>, cards: &[String]) -> Result<Vec<Token>, Error> {
25    let mut options = numeric::Options::default();
26    for card in cards {
27        options.apply(card).ok();
28    }
29    let mut r = Rewrite { tokens, at: 0, out: Vec::new(), constants: HashMap::new(), pending: Vec::new(), options };
30    let mut data = false;
31    while r.at < r.tokens.len() {
32        let division = r.tokens.get(r.at + 1).is_some_and(|t| matches!(&t.tok, Tok::Word(w) if w == "DIVISION"));
33        match &r.tokens[r.at].tok {
34            Tok::Word(w) if division => data = w == "DATA",
35            Tok::Number(n) if data && r.out.last().is_none_or(|t| t.tok == Tok::Period) && matches!(r.tokens.get(r.at + 1).map(|t| &t.tok), Some(Tok::Word(_))) && (n == "78" || r.word_at(2) == Some("CONSTANT") && matches!(n.as_str(), "01" | "1")) => {
36                r.constant()?;
37                continue;
38            }
39            Tok::Word(w) if data && (w == "BINARY-CHAR" || BINARY_USAGES.iter().any(|(u, _)| u == w)) => {
40                r.binary_usage()?;
41                continue;
42            }
43            Tok::Word(w) if w == "PROGRAM-ID" && !ends_with_header(&r.out) => r.identification_header(),
44            _ => {}
45        }
46        let value = r.value()?;
47        r.push(value);
48    }
49    if let Some(last) = r.out.last_mut() {
50        last.messages.append(&mut r.pending);
51    }
52    Ok(r.out)
53}
54
55struct Rewrite {
56    tokens: Vec<Token>,
57    at: usize,
58    out: Vec<Token>,
59    constants: HashMap<String, Tok>,
60    /// Messages of tokens taken out, for the next token kept.
61    pending: Vec<Error>,
62    /// The cards' options, whose code page reads a hexadecimal literal joined to an alphanumeric one.
63    options: numeric::Options,
64}
65
66impl Rewrite {
67    fn push(&mut self, mut token: Token) {
68        token.messages.splice(0..0, self.pending.drain(..));
69        self.out.push(token);
70    }
71
72    /// Token `k`, a constant's name replaced by its value and a PICTURE's `(name)` by the number.
73    fn substituted(&self, k: usize) -> Token {
74        let mut token = self.tokens[k].clone();
75        match &token.tok {
76            Tok::Word(w) => {
77                if let Some(value) = self.constants.get(w) {
78                    token.tok = value.clone();
79                    token.spelled = None;
80                }
81            }
82            Tok::Pic(p) => {
83                if let Some(p) = picture(p, &self.constants) {
84                    token.tok = Tok::Pic(p);
85                }
86            }
87            _ => {}
88        }
89        token
90    }
91
92    /// The token at `at`, substituted, and the literals any `&` joins to it.
93    fn value(&mut self) -> Result<Token, Error> {
94        let mut left = self.substituted(self.at);
95        self.at += 1;
96        while self.tokens.get(self.at).is_some_and(|t| t.tok == Tok::Ampersand) {
97            let amp = self.tokens[self.at].clone();
98            if self.at + 1 >= self.tokens.len() {
99                return Err(Error::at(amp.pos, "& with no literal after it"));
100            }
101            let right = self.substituted(self.at + 1);
102            self.at += 2;
103            left = join(left, amp, right, |bytes| self.options.code_page().decode(bytes))?;
104        }
105        Ok(left)
106    }
107
108    fn word_at(&self, ahead: usize) -> Option<&str> {
109        match self.tokens.get(self.at + ahead).map(|t| &t.tok) {
110            Some(Tok::Word(w)) => Some(w),
111            _ => None,
112        }
113    }
114
115    /// A constant entry from its level number, `78 name [IS] [GLOBAL] VALUE [IS] value.` or
116    /// `01 name CONSTANT [IS] [GLOBAL] [AS] value.`, where the value is a literal, a figurative
117    /// constant, a constant already defined, or literals joined by `&`. The entry is taken out of
118    /// the tokens.
119    fn constant(&mut self) -> Result<(), Error> {
120        let level = self.tokens[self.at].clone();
121        let named = self.tokens[self.at + 1].clone();
122        let Tok::Word(name) = named.tok else { unreachable!("the caller saw a word") };
123        let seventy_eight = level.tok == Tok::Number("78".into());
124        self.at += if seventy_eight { 2 } else { 3 };
125        self.pending.extend(level.messages);
126        self.pending.extend(named.messages);
127        let refused = |at: Pos, why: &str| Error::at(at, format!("constant {name}: {why}"));
128        match (self.word_at(0), self.word_at(1)) {
129            (Some("IS"), Some("GLOBAL")) => self.at += 2,
130            (Some("GLOBAL"), _) => self.at += 1,
131            _ => {}
132        }
133        let here = self.tokens.get(self.at).map_or(named.pos, |t| t.pos);
134        if seventy_eight {
135            if self.word_at(0) != Some("VALUE") {
136                return Err(refused(here, "a level-78 entry is VALUE and its value, then a period"));
137            }
138            self.at += 1;
139            if self.word_at(0) == Some("IS") {
140                self.at += 1;
141            }
142        } else if self.word_at(0) == Some("AS") {
143            self.at += 1;
144        }
145        if self.at >= self.tokens.len() {
146            return Err(refused(here, "VALUE with no value"));
147        }
148        let value = self.value()?;
149        let literal = match &value.tok {
150            Tok::Alnum(_) | Tok::Hex(_) | Tok::National(_) | Tok::Number(_) => true,
151            Tok::Word(w) => FIGURATIVES.contains(&w.as_str()),
152            _ => false,
153        };
154        let ended = self.tokens.get(self.at).is_some_and(|t| t.tok == Tok::Period);
155        if !literal || !ended {
156            let at = if literal { self.tokens.get(self.at).map_or(value.pos, |t| t.pos) } else { value.pos };
157            return Err(refused(at, "the value is a literal, a figurative constant, a constant defined before, or literals joined by &; ironwork computes no expression there"));
158        }
159        self.pending.extend(self.tokens[self.at].messages.iter().cloned());
160        self.at += 1;
161        self.pending.push(Error::warning(level.pos, format!("{CONSTANT}: {name} stands for its value wherever it is used after this entry")));
162        self.pending.extend(value.messages);
163        self.constants.insert(name, value.tok);
164        Ok(())
165    }
166
167    /// `IDENTIFICATION DIVISION.` before a PROGRAM-ID that has none, which it means.
168    fn identification_header(&mut self) {
169        let at = self.tokens[self.at].clone();
170        let made = |tok: Tok, messages: Vec<Error>| Token { tok, pos: at.pos, area_a: at.area_a, spelled: None, after_comma: false, messages };
171        let warning = Error::warning(at.pos, format!("{NO_IDENTIFICATION_HEADER}: the program reads as though IDENTIFICATION DIVISION. came before it"));
172        self.push(made(Tok::Word("IDENTIFICATION".into()), vec![warning]));
173        self.push(made(Tok::Word("DIVISION".into()), Vec::new()));
174        self.push(made(Tok::Period, Vec::new()));
175    }
176
177    /// `[USAGE [IS]] BINARY-SHORT|BINARY-LONG|BINARY-DOUBLE [SIGNED|UNSIGNED]` in a data entry, as
178    /// `PIC S9(n) COMP-5`, or `PIC 9(n) COMP-5` when UNSIGNED: SIGNED is the default.
179    fn binary_usage(&mut self) -> Result<(), Error> {
180        let token = self.tokens[self.at].clone();
181        let Tok::Word(usage) = &token.tok else { unreachable!("the caller saw a word") };
182        let Some(&(_, digits)) = BINARY_USAGES.iter().find(|(u, _)| u == usage) else {
183            return Err(Error::at(token.pos, "BINARY-CHAR is a one-byte binary item, and ironwork's binary items are two, four or eight bytes, as Enterprise COBOL's are"));
184        };
185        let mut messages = Vec::new();
186        if self.out.last().is_some_and(|t| t.tok == Tok::Word("IS".into())) && self.out.len() >= 2 && self.out[self.out.len() - 2].tok == Tok::Word("USAGE".into()) {
187            messages.extend(self.out.pop().map(|t| t.messages).unwrap_or_default());
188        }
189        if self.out.last().is_some_and(|t| t.tok == Tok::Word("USAGE".into())) {
190            messages.extend(self.out.pop().map(|t| t.messages).unwrap_or_default());
191        }
192        self.at += 1;
193        let signed = match self.tokens.get(self.at).map(|t| &t.tok) {
194            Some(Tok::Word(w)) if w == "SIGNED" || w == "UNSIGNED" => {
195                self.at += 1;
196                w == "SIGNED"
197            }
198            _ => true,
199        };
200        let picture = format!("{}{digits}", if signed { "S" } else { "" });
201        let shown = format!("{BINARY_USAGE}: {usage}{} is read as PIC {picture} COMP-5", if signed { "" } else { " UNSIGNED" });
202        messages.push(Error::warning(token.pos, shown));
203        messages.extend(token.messages.iter().cloned());
204        let made = |tok: Tok, messages: Vec<Error>| Token { tok, pos: token.pos, area_a: false, spelled: None, after_comma: false, messages };
205        self.push(made(Tok::Word("PIC".into()), messages));
206        self.push(made(Tok::Pic(picture), Vec::new()));
207        self.push(made(Tok::Word("COMP-5".into()), Vec::new()));
208        Ok(())
209    }
210}
211
212/// Whether the tokens end with `IDENTIFICATION DIVISION.` or `ID DIVISION.`.
213fn ends_with_header(tokens: &[Token]) -> bool {
214    matches!(tokens, [.., a, b, c] if matches!(&a.tok, Tok::Word(w) if w == "IDENTIFICATION" || w == "ID") && b.tok == Tok::Word("DIVISION".into()) && c.tok == Tok::Period)
215}
216
217/// `(name)` in a PICTURE, where `name` is a constant whose value is an unsigned integer, with the
218/// integer in its place.
219fn picture(text: &str, constants: &HashMap<String, Tok>) -> Option<String> {
220    let (mut out, mut rest, mut changed) = (String::new(), text, false);
221    while let Some(open) = rest.find('(') {
222        let Some(close) = rest[open..].find(')').map(|c| open + c) else { break };
223        match constants.get(&rest[open + 1..close]) {
224            Some(Tok::Number(n)) if n.bytes().all(|b| b.is_ascii_digit()) => {
225                out.push_str(&rest[..=open]);
226                out.push_str(n);
227                out.push(')');
228                changed = true;
229            }
230            _ => out.push_str(&rest[..=close]),
231        }
232        rest = &rest[close + 1..];
233    }
234    out.push_str(rest);
235    changed.then_some(out)
236}
237
238/// `left & right` as one literal, at `left`'s place. Alphanumeric and hexadecimal literals join
239/// into an alphanumeric one, a hexadecimal one's bytes read in the program's code page, two
240/// hexadecimal literals into a hexadecimal one, and national literals into a national one.
241fn join(left: Token, amp: Token, right: Token, decode: impl Fn(&[u8]) -> String) -> Result<Token, Error> {
242    let tok = match (&left.tok, &right.tok) {
243        (Tok::Alnum(a), Tok::Alnum(b)) => Tok::Alnum(format!("{a}{b}")),
244        (Tok::Hex(a), Tok::Hex(b)) => Tok::Hex([a.as_slice(), b].concat()),
245        (Tok::Alnum(a), Tok::Hex(b)) => Tok::Alnum(format!("{a}{}", decode(b))),
246        (Tok::Hex(a), Tok::Alnum(b)) => Tok::Alnum(format!("{}{b}", decode(a))),
247        (Tok::National(a), Tok::National(b)) => Tok::National(format!("{a}{b}")),
248        _ => return Err(Error::at(amp.pos, "& joins two alphanumeric or hexadecimal literals, or two national literals, either of which may be a level-78 constant standing for one")),
249    };
250    let mut messages = left.messages;
251    messages.push(Error::warning(amp.pos, format!("{CONCATENATION}: the literals on either side are one literal")));
252    messages.extend(amp.messages);
253    messages.extend(right.messages);
254    Ok(Token { tok, pos: left.pos, area_a: left.area_a, spelled: None, after_comma: left.after_comma, messages })
255}
256
257#[cfg(test)]
258mod tests {
259    use crate::ast::{Literal, Usage};
260    use crate::copy::Libraries;
261    use numeric::Compliance;
262
263    fn source(data: &str, procedure: &str) -> String {
264        format!("       IDENTIFICATION DIVISION.\n       PROGRAM-ID. T.\n       DATA DIVISION.\n       WORKING-STORAGE SECTION.\n{data}       PROCEDURE DIVISION.\n{procedure}           GOBACK.\n")
265    }
266
267    fn extended(text: &str) -> Result<crate::ast::Program, crate::Error> {
268        crate::parse_with(text, &Libraries::default().with_compliance(Compliance::Extended))
269    }
270
271    #[test]
272    fn a_constant_stands_for_its_value_in_pictures_values_occurs_and_statements() {
273        let data = concat!(
274            "       78  MAX-LEN VALUE 3.\n",
275            "       78  GREETING IS GLOBAL VALUE IS 'AB' & X'C1'.\n",
276            "       01  QUOTED CONSTANT AS 'Q'.\n",
277            "       01  NOTHING CONSTANT GLOBAL SPACE.\n",
278            "       01  X PIC X(MAX-LEN) VALUE GREETING.\n",
279            "       01  G.\n",
280            "           05 T OCCURS MAX-LEN TIMES PIC X VALUE NOTHING.\n",
281        );
282        let p = extended(&source(data, "           MOVE QUOTED TO X.\n")).unwrap();
283        let names: Vec<&str> = p.working_storage.iter().filter_map(|e| e.name.as_deref()).collect();
284        assert_eq!(names, ["X", "G", "T"]);
285        let x = &p.working_storage[0];
286        assert_eq!((x.picture.as_deref(), x.value.clone()), (Some("X(3)"), Some(Literal::Alnum("ABA".into()))));
287        assert_eq!(p.working_storage[2].occurs, Some(3));
288        let warnings: Vec<(u32, String)> = p.messages.iter().map(|m| (m.pos.line, m.message.clone())).collect();
289        assert_eq!(warnings.len(), 5, "{warnings:?}");
290        assert!(warnings[..2].iter().all(|(_, m)| m.starts_with(super::CONSTANT)) && warnings[0].0 == 5 && warnings[0].1.ends_with("MAX-LEN stands for its value wherever it is used after this entry"));
291        assert!(warnings[2].1.starts_with(super::CONCATENATION) && warnings[2].0 == 6);
292        assert!(warnings[3..].iter().all(|(_, m)| m.starts_with(super::CONSTANT)));
293        assert!(format!("{:?}", p.paragraphs[0].statements[0]).contains("Alnum(\"Q\")"));
294    }
295
296    #[test]
297    fn a_constant_whose_value_is_an_expression_or_missing_is_refused_by_name() {
298        let refused = |data: &str| extended(&source(data, "")).unwrap_err().message;
299        assert_eq!(refused("       78  N VALUE 1 + 2.\n"), "constant N: the value is a literal, a figurative constant, a constant defined before, or literals joined by &; ironwork computes no expression there");
300        assert_eq!(refused("       78  N PIC 9 VALUE 1.\n"), "constant N: a level-78 entry is VALUE and its value, then a period");
301        assert!(refused("       01  N PIC X VALUE 'A' & B.\n").starts_with("& joins two alphanumeric or hexadecimal literals"));
302        assert!(refused("       01  N PIC X VALUE 'A' & N'B'.\n").starts_with("& joins two alphanumeric or hexadecimal literals"));
303    }
304
305    #[test]
306    fn the_binary_usages_are_comp_5_pictures_and_binary_char_is_refused() {
307        let data = "       01  A USAGE IS BINARY-LONG.\n       01  B BINARY-SHORT UNSIGNED VALUE 7.\n       01  C BINARY-DOUBLE SIGNED.\n";
308        let p = extended(&source(data, "")).unwrap();
309        let read: Vec<(Option<&str>, Option<Usage>)> = p.working_storage.iter().map(|e| (e.picture.as_deref(), e.usage)).collect();
310        assert_eq!(read, [(Some("S9(9)"), Some(Usage::NativeBinary)), (Some("9(4)"), Some(Usage::NativeBinary)), (Some("S9(18)"), Some(Usage::NativeBinary))]);
311        assert_eq!(p.working_storage[1].value, Some(Literal::Number("7".into())));
312        let shown: Vec<String> = p.messages.iter().map(|m| m.message.clone()).collect();
313        assert_eq!(shown[1], format!("{}: BINARY-SHORT UNSIGNED is read as PIC 9(4) COMP-5", super::BINARY_USAGE));
314        let refused = extended(&source("       01  D BINARY-CHAR.\n", "")).unwrap_err();
315        assert!(refused.message.starts_with("BINARY-CHAR is a one-byte binary item") && refused.pos.line == 5);
316    }
317
318    #[test]
319    fn strict_reads_none_of_it() {
320        let strict = |data: &str| crate::parse(&source(data, ""));
321        assert_eq!(strict("       01  N PIC X(2) VALUE 'A' & 'B'.\n").unwrap_err().message, "literal concatenation with & is not Enterprise COBOL's");
322        assert_eq!(strict("       78  N VALUE 1.\n").unwrap().working_storage[0].level, 78);
323        assert!(strict("       01  A BINARY-LONG.\n").unwrap_err().message.contains("BINARY-LONG is not a data description clause"));
324    }
325}
ironwork_syntax/extended.rs

ironwork_syntax/
extended.rs
