1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
//! This module contains [Generator](#Generator) - config-driven code generation,
//! and [CodeGen](#CodeGen), the trait for language-specific code-driven code generation.
//!
//!
use crate::Bytes;
use crate::{
    codegen_go::GoCodeGen,
    codegen_rust::RustCodeGen,
    config::{CodegenConfig, LanguageConfig, OutputFile, OutputLanguage},
    docgen::DocGen,
    error::{Error, Result},
    model::CommentKind,
    render::Renderer,
    writer::Writer,
    JsonValue, ParamMap, TomlValue,
};
use atelier_core::model::{Identifier, Model};
use std::borrow::Borrow;
use std::collections::BTreeMap;
use std::io::Write;
use std::path::{Path, PathBuf};

/// Common templates - not language specific
pub const COMMON_TEMPLATES: &[(&str, &str)] = &[
    (
        "codegen.toml",
        include_str!("../templates/codegen.toml.hbs"),
    ),
    (
        "hello.smithy",
        include_str!("../templates/hello.smithy.hbs"),
    ),
    (
        "Makefile.interface",
        include_str!("../templates/Makefile.interface.hbs"),
    ),
];

/// A Generator is a data-driven wrapper around code generator implementations,
/// There are two main modes of generation:
/// handlebars template-driven, and code-driven. For the latter, you implement
/// a trait CodeGen, which gets callbacks for various parts of the code generation.
/// One parameter to CodeGen is the output file name, which can be used if code is
/// to be generated across several files. (for example, if you want
/// one file to define interfaces and a separate file to define implementation classe).
/// Handlebars-driven code generation may be more well-suited for files such as Makefiles
/// and project config files that don't need a huge amount of customization,
/// but they can also be used to generate 100% of an interface. You decide!
#[derive(Debug, Default)]
pub struct Generator {}

impl<'model> Generator {
    /// Perform code generation on model, iterating through each configured OutputFile.
    /// Each file to be generated is either based on a handlebars
    /// template, which is generated with the Renderer class,
    /// or is generated by an implemented of the CodeGen implementation.
    /// Function parameters:
    /// - model - the smithy model
    /// - config - CodeConfig (usually loaded from a codegen.toml file)
    ///     - N.B. all relative paths (template dirs and the output_dir parameter) are
    ///            adjusted to be relative to config.base_dir.
    /// - templates - list of additional templates to register with the handlebars engine
    ///          (The templates parameter of config is ignored. to use a list of file
    ///          paths, call templates_from_dir() to load them and generate this parameter)
    /// - output_dir - top-level folder containing all output files
    /// - create - whether the user intends to create a new project (true) or just update (false).
    ///   The generator does not check for existence of an output file before overwriting. Usually the
    ///   `create` flag is set to true for the first pass when project files are created, after which
    ///   time the project file can be manually edited, and only the main codegen output will be updated.
    ///   The default (create is false/the flag is unspecified)) changes the fewest files
    pub fn gen(
        &self,
        model: Option<&'model Model>,
        config: CodegenConfig,
        templates: Vec<(String, String)>,
        output_dir: &Path,
        defines: Vec<(String, TomlValue)>,
    ) -> Result<()> {
        let mut json_model = match model {
            Some(model) => atelier_json::model_to_json(model),
            None => JsonValue::default(),
        };
        let output_dir = if output_dir.is_absolute() {
            output_dir.to_path_buf()
        } else {
            config.base_dir.join(output_dir)
        };
        // create one renderer so we only need to parse templates once
        let mut renderer = Renderer::default();

        // if defines.iter().any(|(k, _)| k == "create_interface") {

        for (name, template) in COMMON_TEMPLATES.iter() {
            renderer.add_template((name, template))?;
        }
        std::fs::create_dir_all(&output_dir).map_err(|e| {
            Error::Io(format!(
                "creating directory {}: {}",
                &output_dir.display(),
                e
            ))
        })?;

        for (language, mut lc) in config.languages.into_iter() {
            if !config.output_languages.is_empty() && !config.output_languages.contains(&language) {
                // if user specified list of languages, only generate code for those languages
                continue;
            }
            // add templates from <lang>.templates
            if let Some(template_dir) = &lc.templates {
                let template_dir = if template_dir.is_absolute() {
                    template_dir.clone()
                } else {
                    config.base_dir.join(template_dir)
                };
                for (name, tmpl) in templates_from_dir(&template_dir)? {
                    renderer.add_template((&name, &tmpl))?;
                }
            }
            // add templates from cli
            for (name, template) in templates.iter() {
                renderer.add_template((name, template))?;
            }
            // append language output_dir to project output_dir
            let output_dir = output_dir.join(&lc.output_dir);

            // add command-line overrides
            for (k, v) in defines.iter() {
                lc.parameters.insert(k.to_string(), v.clone());
            }
            let base_params: BTreeMap<String, JsonValue> = to_json(&lc.parameters)?;

            let mut cgen = gen_for_language(&language, model);

            // initialize generator
            cgen.init(model, &lc, &output_dir, &mut renderer)?;

            // A common param dictionary is shared (read-only) by the renderer and the code generator,
            // Parameters include the following:
            //   - "model" - the entire model, generated by parsing one or more smithy files,
            //               and converted to json-ast format
            //   - "_file" - the output file (path is relative to the output directory
            //   - LanguageConfig.parameters , applicable to all files for this output language
            //   - OutputFile.parameters - file-specific parameters
            //   The latter two are added in that order, so that a per-file parameter can
            //   override per-language settings.

            // The parameter dict is cleared after each iteration to avoid one file's override
            // from leaking into the next file in the iteration. There are two "optimiations"
            // in the loop below.
            // - The handlebars renderer only parses templates once, so it is shared across output files,
            // - The smithy model is parsed and validated once. After using it for one file, we pull it
            //   out of the params map, save it aside, and then re-insert it the next time.

            // list of files that were created or updated on this run
            let mut updated_files = Vec::new();

            for file_config in lc.files.iter() {
                // for conditional files (with the `if_defined` property), check that we have the right conditions
                if let Some(TomlValue::String(key)) = file_config.params.get("if_defined") {
                    match lc.parameters.get(key) {
                        None | Some(TomlValue::Boolean(false)) => {
                            // not defined, do not generate
                            continue;
                        }
                        Some(_) => {}
                    }
                }
                let mut params = base_params.clone();
                params.insert("model".to_string(), json_model);

                let file_params: BTreeMap<String, JsonValue> = to_json(&file_config.params)?;
                params.extend(file_params.into_iter());
                params.insert(
                    "_file".to_string(),
                    JsonValue::String(file_config.path.to_string_lossy().to_string()),
                );

                let out_path = output_dir.join(&file_config.path);
                let parent = out_path.parent().unwrap();
                std::fs::create_dir_all(parent).map_err(|e| {
                    Error::Io(format!("creating directory {}: {}", parent.display(), e))
                })?;

                // generate output using either hbs or CodeGen
                if let Some(hbs) = &file_config.hbs {
                    let mut out = std::fs::File::create(&out_path).map_err(|e| {
                        Error::Io(format!("creating file {}: {}", &out_path.display(), e))
                    })?;
                    renderer.render(hbs, &params, &mut out)?;
                    out.flush()?;
                } else if let Some(model) = model {
                    let bytes = cgen.generate_file(model, file_config, &params)?;
                    std::fs::write(&out_path, &bytes).map_err(|e| {
                        Error::Io(format!("writing output file {}: {}", out_path.display(), e))
                    })?;
                };
                updated_files.push(out_path);
                // retrieve json_model for the next iteration
                json_model = params.remove("model").unwrap();
            }
            cgen.format(updated_files, &lc.parameters)?;
        }

        Ok(())
    }
}

fn gen_for_language<'model>(
    language: &OutputLanguage,
    model: Option<&'model Model>,
) -> Box<dyn CodeGen + 'model> {
    match language {
        OutputLanguage::Rust => Box::new(RustCodeGen::new(model)),
        OutputLanguage::Html => Box::new(DocGen::default()),
        OutputLanguage::Poly => Box::new(PolyGen::default()),
        OutputLanguage::Go => Box::new(GoCodeGen::new(model)),
        _ => {
            crate::error::print_warning(&format!("Target language {} not implemented", language));
            Box::new(NoCodeGen::default())
        }
    }
}

/// A Codegen is used to generate source for a Smithy Model
/// The generator will invoke these functions (in order)
/// - init()
/// - write_source_file_header()
/// - declare_types()
/// - write_services()
/// - finalize()
///
pub(crate) trait CodeGen {
    /// Initialize code generator and renderer for language output.j
    /// This hook is called before any code is generated and can be used to initialize code generator
    /// and/or perform additional processing before output files are created.
    #[allow(unused_variables)]
    fn init(
        &mut self,
        model: Option<&Model>,
        lc: &LanguageConfig,
        output_dir: &Path,
        renderer: &mut Renderer,
    ) -> std::result::Result<(), Error> {
        Ok(())
    }

    /// This entrypoint drives output-file-specific code generation.
    /// This default implementation invokes `init_file`, `write_source_file_header`, `declare_types`, `write_services`, and `finalize`.
    /// The return value is Bytes containing the data that should be written to the output file.
    fn generate_file(
        &mut self,
        model: &Model,
        file_config: &OutputFile,
        params: &ParamMap,
    ) -> Result<Bytes> {
        let mut w: Writer = Writer::default();

        self.init_file(&mut w, model, file_config, params)?;
        self.write_source_file_header(&mut w, model, params)?;
        self.declare_types(&mut w, model, params)?;
        self.write_services(&mut w, model, params)?;
        self.finalize(&mut w)
    }

    /// Perform any initialization required prior to code generation for a file
    /// `model` may be used to check model metadata
    /// `id` is a tag from codegen.toml that indicates which source file is to be written
    /// `namespace` is the namespace in the model to generate
    #[allow(unused_variables)]
    fn init_file(
        &mut self,
        w: &mut Writer,
        model: &Model,
        file_config: &OutputFile,
        params: &ParamMap,
    ) -> Result<()> {
        Ok(())
    }

    /// generate the source file header
    #[allow(unused_variables)]
    fn write_source_file_header(
        &mut self,
        w: &mut Writer,
        model: &Model,
        params: &ParamMap,
    ) -> Result<()> {
        Ok(())
    }

    /// Write declarations for simple types, maps, and structures
    #[allow(unused_variables)]
    fn declare_types(&mut self, w: &mut Writer, model: &Model, params: &ParamMap) -> Result<()> {
        Ok(())
    }

    /// Write service declarations and implementation stubs
    #[allow(unused_variables)]
    fn write_services(&mut self, w: &mut Writer, model: &Model, params: &ParamMap) -> Result<()> {
        Ok(())
    }

    /// Complete generation and return the output bytes
    fn finalize(&mut self, w: &mut Writer) -> Result<Bytes> {
        Ok(w.take().freeze())
    }

    /// Write documentation for item
    #[allow(unused_variables)]
    fn write_documentation(&mut self, mut w: &mut Writer, _id: &Identifier, text: &str) {
        for line in text.split('\n') {
            // remove whitespace from end of line
            let line = line.trim_end_matches(|c| c == '\r' || c == ' ' || c == '\t');
            self.write_comment(&mut w, CommentKind::Documentation, line);
        }
    }

    /// Writes single-line comment beginning with '// '
    /// Can be overridden if more specific kinds are needed
    #[allow(unused_variables)]
    fn write_comment(&mut self, w: &mut Writer, kind: CommentKind, line: &str) {
        w.write(b"// ");
        w.write(line);
        w.write(b"\n");
    }

    fn write_ident(&self, w: &mut Writer, id: &Identifier) {
        w.write(&self.to_type_name(&id.to_string()));
    }

    // Writes info the the current output writer
    //fn write(&mut self, bytes: impl ToBytes);

    // Returns the current buffer, zeroing out self
    //fn take(&mut self) -> BytesMut;

    /// returns file extension of source files for this language
    fn get_file_extension(&self) -> &'static str {
        ""
    }

    /// Convert type name to its target-language-idiomatic case style
    /// The default implementation uses UpperPascalCase
    fn to_type_name(&self, s: &str) -> String {
        crate::strings::to_pascal_case(s)
    }

    /// Convert method name to its target-language-idiomatic case style
    /// Default implementation uses snake_case
    fn to_method_name(&self, method_id: &Identifier) -> String {
        crate::strings::to_snake_case(&method_id.to_string())
    }

    /// Convert field name to its target-language-idiomatic case style
    /// Default implementation uses snake_case
    fn to_field_name(&self, member_id: &Identifier) -> std::result::Result<String, Error> {
        Ok(crate::strings::to_snake_case(&member_id.to_string()))
    }

    /// The operation name used in dispatch, from method
    /// The default implementation is provided and should not be overridden
    fn op_dispatch_name(&self, id: &Identifier) -> String {
        crate::strings::to_pascal_case(&id.to_string())
    }

    /// The full operation name with service prefix, with surrounding quotes
    /// The default implementation is provided and should not be overridden
    fn full_dispatch_name(&self, service_id: &Identifier, method_id: &Identifier) -> String {
        format!(
            "\"{}.{}\"",
            &self.to_type_name(&service_id.to_string()),
            &self.op_dispatch_name(method_id)
        )
    }

    fn source_formatter(&self) -> Result<Box<dyn SourceFormatter>> {
        Ok(Box::new(crate::format::NullFormatter::default()))
    }

    /// After code generation has completed for all files, this method is called once per output language
    /// to allow code formatters to run. The `files` parameter contains a list of all files written or updated.
    fn format(
        &mut self,
        files: Vec<PathBuf>,
        lc_params: &BTreeMap<String, TomlValue>,
    ) -> Result<()> {
        // if we just created an interface project, don't run rustfmt yet
        // because we haven't generated the other rust file yet, so rustfmt will fail.
        if !lc_params.contains_key("create_interface") {
            // make a list of all output files with ".rs" extension so we can fix formatting with rustfmt
            // minor nit: we don't check the _config-only flag so there could be some false positives here, but rustfmt is safe to use anyway
            let formatter = self.source_formatter()?;

            let sources = files
                .into_iter()
                .filter(|f| formatter.include(f))
                .collect::<Vec<PathBuf>>();

            if !sources.is_empty() {
                ensure_files_exist(&sources)?;

                let file_names: Vec<std::borrow::Cow<'_, str>> =
                    sources.iter().map(|p| p.to_string_lossy()).collect();
                let borrowed = file_names.iter().map(|s| s.borrow()).collect::<Vec<&str>>();
                formatter.run(&borrowed)?;
            }
        }
        Ok(())
    }
}

/// Formats source code
#[allow(unused_variables)]
pub trait SourceFormatter {
    /// run formatter on all files
    /// Default implementation does nothing
    fn run(&self, source_files: &[&str]) -> Result<()> {
        Ok(())
    }

    /// returns true if the file should be included in the set to be formatted
    /// default implementation returns false for all files
    fn include(&self, path: &std::path::Path) -> bool {
        true
    }
}

/// confirm all files are present, otherwise return error
fn ensure_files_exist(source_files: &[std::path::PathBuf]) -> Result<()> {
    let missing = source_files
        .iter()
        .filter(|p| !p.is_file())
        .map(|p| p.to_string_lossy().into_owned())
        .collect::<Vec<String>>();
    if !missing.is_empty() {
        return Err(Error::Formatter(format!(
            "missing source file(s) '{}'",
            missing.join(",")
        )));
    }
    Ok(())
}

#[derive(Debug, Default)]
struct PolyGen {}
impl CodeGen for PolyGen {}

// convert from TOML map to JSON map so it's usable by handlebars
//pub fn toml_to_json(map: &BTreeMap<String, TomlValue>) -> Result<ParamMap> {
//    let s = serde_json::to_string(map)?;
//    let value: ParamMap = serde_json::from_str(&s)?;
//    Ok(value)
//}

/// Converts a type to json
pub fn to_json<S: serde::Serialize, T: serde::de::DeserializeOwned>(val: S) -> Result<T> {
    let s = serde_json::to_string(&val)?;
    Ok(serde_json::from_str(&s)?)
}

/// Search a folder recursively for files ending with the provided extension
/// Filenames must be utf-8 characters
pub fn find_files(dir: &Path, extension: &str) -> Result<Vec<PathBuf>> {
    if dir.is_dir() {
        let mut results = Vec::new();
        for entry in std::fs::read_dir(dir)
            .map_err(|e| Error::Io(format!("reading directory {}: {}", dir.display(), e)))?
        {
            let entry = entry?;
            let path = entry.path();
            if path.is_dir() {
                results.append(&mut find_files(&path, extension)?);
            } else {
                let ext = path
                    .extension()
                    .map(|s| s.to_string_lossy().to_string())
                    .unwrap_or_default();
                if ext == extension {
                    results.push(path)
                }
            }
        }
        Ok(results)
    } else if dir.is_file()
        && &dir
            .extension()
            .map(|s| s.to_string_lossy().to_string())
            .unwrap_or_default()
            == "smithy"
    {
        Ok(vec![dir.to_owned()])
    } else {
        Err(Error::Other(format!(
            "'{}' is not a valid folder or '.{}' file",
            dir.display(),
            extension
        )))
    }
}

/// Add all templates from the specified folder, using the base file name
/// as the template name. For example, "header.hbs" will be registered as "header"
#[cfg(not(target_arch = "wasm32"))]
pub fn templates_from_dir(start: &std::path::Path) -> Result<Vec<(String, String)>> {
    let mut templates = Vec::new();

    for path in crate::gen::find_files(start, "hbs")?.iter() {
        let stem = path
            .file_stem()
            .map(|s| s.to_string_lossy().to_string())
            .unwrap_or_default();
        if !stem.is_empty() {
            let template = std::fs::read_to_string(path)
                .map_err(|e| Error::Io(format!("reading template {}: {}", path.display(), e)))?;
            templates.push((stem, template));
        }
    }
    Ok(templates)
}

#[derive(Default)]
struct NoCodeGen {}
impl CodeGen for NoCodeGen {
    fn get_file_extension(&self) -> &'static str {
        ""
    }
}