bfom_lib/

lib.rs

mod line_type;
pub mod utilities;

use line_type::{block_line_type, HtmlType, LineType};

use std::{
  collections::HashMap,
  fs::File,
  io::{BufRead, BufReader, Error},
  path::Path,
};

// for testing

// for converting
#[derive(Debug, Clone)]
struct NodeData {
  // h1/p/pre/etc
  // plain text is none
  tag: String,

  // any misc data, depends on teh tag
  misc: Option<String>,
  // the raw data inside this node
  text: Option<String>,

  contents: Vec<NodeData>,
}

#[derive(Debug, Clone)]
struct ReferenceLink {
  url: String,
  title: Option<String>,
}

#[derive(Debug)]
pub struct Converter {
  html_void: Vec<String>,
  references: HashMap<String, ReferenceLink>,
  indentation: usize,
}

#[derive(Debug)]
pub struct FrontMatter {
  pub date: Option<String>,
  pub description: Option<String>,
  pub title: Option<String>,
  pub slug: Option<String>,
  pub author: Option<String>,
  pub draft: bool,
  pub slides: bool,
  pub skip_index: bool,
}

// takes the link provided, converts it
impl Converter {
  pub fn new(html_void: Vec<String>, indentation: usize) -> Self {
    Converter {
      html_void,
      references: Default::default(),
      indentation,
    }
  }

  // this manages the conversion
  pub fn convert_file(&mut self, input: &Path, depth: usize) -> Result<(String, FrontMatter), Error> {
    // read file into memory
    let lines = self.file_read(input)?;

    // grab the reference lines out of it
    let lines_processed_references = self.reference_link_get(lines);

    // pull out teh fm data
    let (lines_processed_fm, fm) = self.fm_get(lines_processed_references);

    // get teh block elements
    // frontmatter is only valid at a top level one
    let blocks = self.block_process(lines_processed_fm)?;

    // merge it all together
    let processed = self.block_merge(blocks, depth, fm.slides);

    // clean up references
    self.references = Default::default();

    Ok((processed, fm))
  }

  // load the file into working memory
  fn file_read(&self, path: &Path) -> Result<Vec<String>, Error> {
    // read file into memory
    //println!("In file {}", path);
    // read the file into memory
    let input = File::open(path)?;
    let buffered = BufReader::new(input);

    let lines_all = buffered
      .lines()
      .map(|l| {
        let unwrapped = l.expect("Could not parse line");
        unwrapped.replace('\u{0000}', "\u{FFFD}")
      })
      .collect::<Vec<String>>();

    Ok(lines_all)
  }

  fn reference_link_get(&mut self, lines: Vec<String>) -> Vec<String> {
    let mut result: Vec<String> = vec![];

    // iterate through the lines
    // if one matches the format it is not added to the result and added to teh link hash, if its not already there

    let mut references: HashMap<String, ReferenceLink> = HashMap::new();

    'outer: for line in lines {
      // only lines that start with [ are considered reference, number of spaces is irrelevant
      if !line.trim_start().starts_with('[') {
        result.push(line);
        continue;
      }

      // now check of the rest of the line matches

      // form manging position
      let characters: Vec<char> = line.trim_start().chars().collect();
      // starts at 1 to skip the first [
      let mut index_char = 1;

      // for getting the identifier
      let mut reference_vec: Vec<char> = vec![];
      loop {
        if index_char >= characters.len() {
          break;
        }

        let character = characters[index_char];

        match character {
          ']' => {
            // check if last character is a backslash
            if characters[index_char - 1] == '\\' {
              // remove teh slash
              // add the bracket
              reference_vec.pop();
              reference_vec.push(character);
            } else {
              // increment to account for this
              index_char += 1;
              break;
            }
          }
          _ => reference_vec.push(character),
        }
        index_char += 1;
      }

      // need to have an identifier
      if reference_vec.is_empty() {
        result.push(line);
        continue 'outer;
      }

      // check if next character is ':' (required)
      if index_char >= characters.len() {
        result.push(line);
        continue 'outer;
      }
      if characters[index_char] != ':' {
        // add to normal output
        result.push(line);
        continue 'outer;
      } else {
        // if ir is :
        index_char += 1;
      }

      // check if next character is whitespace, skip past these
      loop {
        if index_char >= characters.len() {
          break;
        }
        let character = characters[index_char];

        if character == ' ' || character == '\t' {
          index_char += 1;
        } else {
          break;
        }
      }

      // now get teh url
      let mut url_vec: Vec<char> = vec![];
      // (start, end)
      let mut angle_brackets = (false, false);
      loop {
        if index_char >= characters.len() {
          break;
        }
        let character = characters[index_char];

        match character {
          '<' => {
            // opening is only allowed on teh first chr
            if !angle_brackets.0 {
              angle_brackets = (true, false);
            } else {
              url_vec.push(character)
            }
          }
          '>' => {
            // check if it was opened
            if angle_brackets.0 {
              // then this closes it out

              // check if its preceded by backslash, its action is ignored
              if characters[index_char - 1] == '\\' {
                // remove teh slash
                url_vec.pop();
                url_vec.push(character);
              } else {
                // actually closes it out
                angle_brackets = (true, true);

                index_char += 1;
                break;
              }
            } else {
              url_vec.push(character);
            }
          }
          ' ' | '\t' => {
            // space breaks the url, unless its in an angled bracket

            if angle_brackets.0 {
              url_vec.push(character)
            } else {
              break;
            }
          }

          _ => url_vec.push(character),
        }

        index_char += 1;
      }

      if url_vec.is_empty() {
        // check if its angle brackets
        if !angle_brackets.1 {
          result.push(line);
          continue 'outer;
        }
      }

      // if angle brackets were opened but never closed
      if angle_brackets.0 && !angle_brackets.1 {
        result.push(line);
        continue 'outer;
      }

      // check for whitespace again, but this time it must be at least 1 if its followed by an open quotation
      let mut whitespace = 0;
      loop {
        if index_char >= characters.len() {
          break;
        }
        let character = characters[index_char];

        match character {
          ' ' | '\t' => {
            index_char += 1;
            whitespace += 1;
          }
          '\'' | '"' => {
            if whitespace == 0 {
              result.push(line);
              continue 'outer;
            } else {
              break;
            }
          }
          _ => break,
        }
      }

      // find the optional title next

      // first char must be either ' or "

      let mut title_quote: Option<char> = None;
      if index_char < characters.len() {
        let character = characters[index_char];
        match character {
          '\'' | '"' => title_quote = Some(character),
          _ => {}
        }
        index_char += 1;
      }

      let mut title_vec: Vec<char> = vec![];

      if let Some(delimiter) = title_quote {
        // starts off being opened
        let mut title_closed = false;

        loop {
          if index_char >= characters.len() {
            break;
          }
          let character = characters[index_char];

          if character == delimiter {
            // check if preceding was backslash

            if characters[index_char - 1] == '\\' {
              // remove teh slash
              // add the bracket
              title_vec.pop();
              title_vec.push(character);
            } else {
              title_closed = true;

              // increment to account for this
              //index_char += 1;
              break;
            }
          } else {
            title_vec.push(character)
          }

          index_char += 1;
        }

        // check if it was closed properly
        if !title_closed {
          // title is optional so just clear it instead of revoking teh line
          title_vec = vec![];
        }
      }

      // tidy up
      let reference: String = reference_vec.iter().collect();
      let url: String = url_vec.iter().collect();

      let title: Option<String> = if title_vec.is_empty() { None } else { Some(title_vec.iter().collect()) };

      let reference_data = ReferenceLink {
        url,
        title,
      };

      references.insert(reference, reference_data);
    }

    self.references = references;

    result
  }

  fn fm_get(&mut self, lines: Vec<String>) -> (Vec<String>, FrontMatter) {
    let mut fm = FrontMatter {
      date: None,
      description: None,
      title: None,
      slug: None,
      author: None,
      draft: false,
      slides: false,
      skip_index: false,
    };

    let mut result: Vec<String> = vec![];

    let mut fm_processed = false;
    let mut fm_in_progress = false;

    // get the language
    let mut closer: Vec<char> = vec![];
    let mut pre_closer = String::default();
    for line in lines {
      if block_line_type(&line, &[]) == LineType::FrontMatter && !fm_in_progress && !fm_processed {
        fm_in_progress = true;
      }
      if !fm_in_progress {
        result.push(line);
        continue;
      }

      /*
      All front matter blocks are fenced.
      They start and end with at least 3 plusses (+++).

      Everything inside can have further md applied, though its returned as a String.
      */

      if closer.is_empty() {
        for character in line.trim().chars() {
          match character {
            '+' => closer.push(character),
            _ => {
              break;
            }
          }
        }
        pre_closer = closer.iter().collect::<String>().trim().to_string();

        // nothing more left to do on this line
        continue;
      }

      let trimmed = line.trim();

      if trimmed == pre_closer.as_str() {
        fm_in_progress = false;
        fm_processed = true;
        continue;
      }

      // start off with default value
      let mut stripped = String::default();

      for substring in ["date", "slug", "title", "description", "author", "draft", "slides", "skip_index"] {
        if trimmed.starts_with(substring) {
          stripped = trimmed.replace(substring, "")
        }
      }

      // extract data from teh line, each line can has the format of
      // attribute = value
      // spaces
      let mut characters = vec![];
      let mut equals = false;
      for c in stripped.chars() {
        match c {
          '=' => {
            if !equals {
              equals = true;
            } else {
              characters.push(c);
            }
          }
          _ => {
            if equals {
              characters.push(c);
            }
          }
        }
      }

      let tmp = characters.iter().collect::<String>();
      let mut value_tmp = tmp.trim();
      if (value_tmp.starts_with('\'') && value_tmp.ends_with('\'')) || (value_tmp.starts_with('"') && value_tmp.ends_with('"')) {
        value_tmp = &value_tmp[1..value_tmp.len() - 1];
      }
      let value = value_tmp.trim().to_owned();
      let formatted = self.inline_process(value.clone(), 0);

      // description
      if trimmed.starts_with("date") {
        fm.date = Some(formatted);
      } else if trimmed.starts_with("slug") {
        fm.slug = Some(value);
      } else if trimmed.starts_with("title") {
        fm.title = Some(formatted);
      } else if trimmed.starts_with("description") {
        fm.description = Some(formatted);
      } else if trimmed.starts_with("author") {
        fm.author = Some(formatted);
      } else if trimmed.starts_with("draft") && value.to_ascii_lowercase().as_str() == "true" {
        fm.draft = true;
      } else if trimmed.starts_with("slides") && value.to_ascii_lowercase().as_str() == "true" {
        fm.slides = true;
      } else if trimmed.starts_with("skip_index") && value.to_ascii_lowercase().as_str() == "true" {
        fm.skip_index = true;
      };
    }

    (result, fm)
  }

  fn block_process(&self, lines: Vec<String>) -> Result<Vec<NodeData>, Error> {
    // this gets most of the structure done

    // what gets returned
    let mut node_data: Vec<NodeData> = vec![];

    // these handle dealing with teh current lines
    let mut row: isize = 0;

    // These keep track of info about the current block
    let mut block_excluded: Vec<String> = vec![];

    // (row, col)
    let mut html_end_col = 0;

    loop {
      // ensure that row is in bounds
      if row >= lines.len() as isize {
        break;
      }

      let line = if html_end_col != 0 { &lines[row as usize][html_end_col..] } else { lines[row as usize].as_str() };

      //println!("Start:  Top: {}. Row {} of {}. Block: {:?}, Current Line: {}. Line: {}. End?: {}", _top_layer, row, lines.len(), block_type, line_current_type.0, line, line_current_is_end);

      // an array of this line and all the remaining lines to be sent off to the sub functions
      let lines_remaining = &lines[(row as usize)..];

      // partial is for paragraphs and html
      let mut lines_remaining_partial = vec![line.to_string()];

      if ((row as usize) + 1) < lines.len() {
        let remainder = &lines[((row as usize) + 1)..];
        lines_remaining_partial.extend(remainder.to_vec());
      }

      let mut clean_up = false;
      // now process based on what type of line it is
      match block_line_type(line, &block_excluded) {
        LineType::Header => {
          node_data.push(self.block_process_headers(line));

          // go straight to the next row
          row += 1;

          clean_up = true;
        }
        LineType::HorizontalRule => {
          /*
              hr are simple, no content or anything.
          */

          node_data.push(NodeData {
            tag: "hr".to_string(),
            text: None,
            misc: None,
            contents: vec![],
          });

          // go straight to the next row
          row += 1;

          clean_up = true;
        }

        // blocks from here on out
        LineType::Paragraph => {
          if line.is_empty() {
            row += 1;
            continue;
          }

          if let Some((node, offset)) = self.block_process_paragraph(&lines_remaining_partial, 0, &block_excluded) {
            node_data.push(node);

            row += offset;

            clean_up = true;
          } else {
            block_excluded.push("p".to_string());
          }
        }
        LineType::Preformatted => {
          if let Some((node, offset)) = self.block_process_pre(lines_remaining) {
            node_data.push(node);

            // +1 as to not include the closing fence
            row += offset + 1;

            clean_up = true;
          } else {
            block_excluded.push("pre".to_string());
          }
        }

        LineType::FrontMatter => {
          // no longer handled here
        }

        LineType::BlockQuote => {
          if let Some((node, offset)) = self.block_process_blockquote(lines_remaining) {
            node_data.push(node);

            row += offset;

            clean_up = true;
          } else {
            block_excluded.push("blockquote".to_string());
          }
        }

        LineType::UL(_, _) | LineType::OL(_, _) => {
          if let Some((node, offset)) = self.block_process_lists(lines_remaining) {
            node_data.push(node);

            row += offset;

            clean_up = true;
          } else {
            block_excluded.push("ul".to_string());
          }
        }

        LineType::Html(html_type) => {
          if let Some((node, offset, col)) = self.block_process_html(&lines_remaining_partial, html_type) {
            node_data.push(node);

            if col > 0 {
              // repeat teh current line
              row += offset - 1;
              html_end_col = col;
            } else {
              row += offset;
              html_end_col = 0;
            }

            // clear the excluded blocks because this completed successfully
            block_excluded = vec![];
          } else {
            match html_type {
              HtmlType::Normal => {
                block_excluded.push("html".to_string());
              }
              HtmlType::Comment => {
                block_excluded.push("html_comment".to_string());
              }
              HtmlType::CData => {
                block_excluded.push("html_cdata".to_string());
              }
            }
          }
          // either way continue to the next/same row
          continue;
        }

        LineType::Table => {
          if let Some((node, offset)) = self.block_process_table(lines_remaining) {
            node_data.push(node);

            row += offset;

            clean_up = true;
          } else {
            block_excluded.push("table".to_string());
          }
        }
      }

      // this is the clean up for everything but html
      if clean_up {
        // clean up
        block_excluded = vec![];
        html_end_col = 0;

        continue;
      }

      // this is to catch anything that somehow slipped through
      row += 1;
    }
    Ok(node_data)
  }

  fn block_process_headers(&self, line: &str) -> NodeData {
    /*
        Headers are pretty easy to process.
        Strip leading whitespace.
        Count how many #'s there are at the start
        This either caps at 6 or when there is a non # character
        Anything after the last #] is counted as content
    */

    let mut count = 0;
    let mut broken = false;
    let mut content_array: Vec<char> = vec![];
    for character in line.trim().chars() {
      match character {
        '#' => {
          if !broken && count < 6 {
            count += 1
          } else {
            content_array.push(character);
          }
        }
        _ => {
          broken = true;
          content_array.push(character);
        }
      }
    }

    // for this the tag is depending on the number of #'s at the beginning
    let tag = format!("h{}", &count);

    // content is everything after the break
    let content_string = content_array.to_vec().iter().collect::<String>();
    let text = Some(content_string.as_str().trim().to_string());

    NodeData {
      tag,
      text,
      misc: None,
      contents: vec![],
    }
  }

  fn block_process_pre(&self, lines: &[String]) -> Option<(NodeData, isize)> {
    /*
        All pre blocks are fenced.
        They start with at least 3 backticks (`)and an optional language identifier.
        They close out with an equal number of backticks as the opener

        Opening square brackets '<' are replaced with '&lt;'.

        Everything inside is preformatted text and no further processing is done.
    */

    let line_first = &lines[0];

    // get the language
    let mut pre_language_tmp: Vec<char> = vec![];
    let mut closer: Vec<char> = vec![];
    let mut space = false;
    for character in line_first.trim().chars() {
      match character {
        // skip this one
        '`' => {
          if !space {
            closer.push(character)
          }

          continue;
        }
        ' ' | '\t' => {
          // first one mark it as a space
          if !space {
            space = true;
          } else {
            pre_language_tmp.push(character);
          }
        }
        _ => {
          pre_language_tmp.push(character);
        }
      }
    }

    let pre_language = if pre_language_tmp.is_empty() {
      None
    } else {
      let language = pre_language_tmp.iter().collect::<String>().trim().to_string();
      if language.is_empty() {
        None
      } else {
        Some(language)
      }
    };

    // set the closer it is looking for
    let pre_closer = closer.iter().collect::<String>().trim().to_string();

    // start on the second row
    let mut row = 1;

    let mut block_lines = vec![];
    let mut finished = false;
    loop {
      if row >= lines.len() as isize {
        break;
      }
      let line = &lines[row as usize];

      if line.trim() == pre_closer.as_str() {
        // end of block, now tidy up
        finished = true;
        break;
      } else {
        // add the lines to teh block
        block_lines.push(line.clone());

        // if its the last line then its caught alter down
      }
      row += 1;
    }

    // if successful it delivers teh NodeData and the offset, failure is None

    if finished {
      let text = block_lines.join("\n").replace('<', "&lt;").replace('>', "&gt;");
      let node = NodeData {
        tag: "pre".to_string(),
        text: Some(text),
        misc: pre_language,
        contents: vec![],
      };
      Some((node, row))
    } else {
      None
    }
  }

  fn block_process_blockquote(&self, lines: &[String]) -> Option<(NodeData, isize)> {
    /*
        Like Paragraphs, Blockquotes dont have a failure condition.
        Also it is one of the few blocks that can have other blocks inside itself
        This is because lazy continuation is not allowed each like starts with >

        The first > is stripped off of each line and the result is added to an array.
        On the final row with a > the contents are recursively added back into the block_process function.
    */

    let mut row = 0;

    let mut block_lines = vec![];
    loop {
      if row >= lines.len() as isize {
        break;
      }
      let line = &lines[row as usize];

      // > indicates that it is a blockquote, >! indicates a spoiler
      if line.starts_with('>') && !line.starts_with(">!") {
        let cleaned = line.replacen('>', "", 1).replacen(' ', "", 1);
        block_lines.push(cleaned);
      } else {
        break;
      }

      row += 1;
    }

    if !block_lines.is_empty() {
      let contents = self
        .block_process(block_lines)
        // if it errors then fail fairly gracefully
        .unwrap_or_default();

      let node = NodeData {
        tag: "blockquote".to_string(),
        text: None,
        misc: None,
        contents,
      };

      Some((node, row))
    } else {
      None
    }
  }

  fn block_process_table(&self, lines: &[String]) -> Option<(NodeData, isize)> {
    /*
        Mostly taken from https://github.github.com/gfm/#tables-extension- with a few changes:
        First character on a line is always |, while a trailing | isn't explicitly required it can look better.

        The delimiter line can occur anywhere, even before the header.
        This is because it is used to set the alignment.
    */

    let mut table_rows = vec![];
    let mut table_alignment: HashMap<i32, String> = HashMap::new();
    let mut table_header_length = 0;

    let mut row = 0;
    loop {
      if row >= lines.len() as isize {
        break;
      }
      let line = &lines[row as usize];

      // break early if its not a table
      if !line.trim().starts_with('|') {
        break;
      }

      // deal with alignment rows first
      if line.replace(['|', '-', ':'], "").trim() == "" {
        // will use https://developer.mozilla.org/en-US/docs/Web/CSS/text-align as the way github does it is no longer valid

        // alignment will impact subsequent lines, unless overwritten
        table_alignment = self.block_process_table_alignment(line);

        row += 1;
        continue;
      }

      let mut row_contents: Vec<NodeData> = vec![];
      let mut content: Vec<char> = vec![];
      let mut last_character: Option<char> = None;
      let mut current_col = 0;

      // first row has cols with th, every other row is td
      let tag_col = if table_rows.is_empty() { "th".to_string() } else { "td".to_string() };

      let tag_row = "tr".to_string();

      for character in line.trim_start().trim_end_matches('|').chars() {
        match character {
          '|' => {
            if let Some(x) = last_character {
              if x == '\\' {
                // remove the backslash
                content.pop();
                content.push(character);
                last_character = Some(character);
                continue;
              }
            }

            if current_col > 0 {
              let text: String = content.iter().collect();
              let alignment = table_alignment.get(&{ current_col }).cloned();

              // add current
              row_contents.push(NodeData {
                tag: tag_col.clone(),
                misc: alignment,
                text: Some(text),
                contents: vec![],
              });
            }

            // clear teh existing data (since its previously used
            content = vec![];

            // mark it as opening another col
            current_col += 1;

            if table_rows.is_empty() {
              table_header_length = current_col;
            } else if current_col > table_header_length {
              // if it isn't the header check if the new number exceeds what teh header gave
              break;
            }
          }
          _ => content.push(character),
        }
        last_character = Some(character);
      }

      // tidy up any trailing content
      if !content.is_empty() {
        let text: String = content.iter().collect();
        let alignment = table_alignment.get(&{ current_col }).cloned();

        // add current
        row_contents.push(NodeData {
          tag: tag_col.clone(),
          misc: alignment,
          text: Some(text),
          contents: vec![],
        });
      }

      table_rows.push(NodeData {
        tag: tag_row,
        misc: None,
        text: None,
        contents: row_contents,
      });

      row += 1;
    }

    if !table_rows.is_empty() {
      let table_header = vec![table_rows[0].clone()];

      let table_body = if table_rows.len() > 1 { table_rows[1..].to_vec() } else { vec![] };

      let table = NodeData {
        tag: "table".to_string(),
        misc: None,
        text: None,
        contents: vec![
          NodeData {
            tag: "thead".to_string(),
            misc: None,
            text: None,
            contents: table_header,
          },
          NodeData {
            tag: "tbody".to_string(),
            misc: None,
            text: None,
            contents: table_body,
          },
        ],
      };

      Some((table, row))
    } else {
      None
    }
  }

  fn block_process_lists(&self, lines: &[String]) -> Option<(NodeData, isize)> {
    /*
        All types of lists are comprised of li elements.
        These li can contain other blocks.
        Because of this it makes it a tad trickier to process compared to other blocks.

        Thankfully since there is no lazy continuation there are two ways to see if a line is part of a li.
            1. It is the first line of a li.
            2. It has an indent specified by the first line of the li.

        Like blockquotes the li contents are recursively proceed in the block_process function.

    */

    // manage teh current li
    let mut li_active = false;
    let mut li_number: Option<String> = None;
    let mut li_type = LineType::Paragraph;
    let mut li_indent = 0;
    // current lines in teh list item, list items can have other blocks inside them
    let mut li_lines = vec![];
    let mut li_finished = false;

    let mut li_array = vec![];

    let mut row = 0;
    loop {
      let mut finished = false;

      if row >= lines.len() as isize {
        break;
      }
      let line = &lines[row as usize];

      let list_type = block_line_type(line, &[]);

      if !li_active {
        // deal with first line of a li

        match list_type {
          LineType::OL(indent, number) => {
            li_indent = indent;
            li_number = Some(number.to_string());
          }
          LineType::UL(indent, _) => {
            li_indent = indent;
          }
          _ => {}
        };

        let vec_chars: Vec<char> = line.chars().collect();
        // remove lists_indent's worth ofd characters
        let trimmed = &vec_chars[li_indent..];
        let cleaned = trimmed.iter().collect::<String>();

        li_lines.push(cleaned);
        li_type = list_type;

        // next round will not be on this branch of the if
        li_active = true;
      } else {
        // second line of the lists

        let leading_spaces = String::from_utf8(vec![b' '; li_indent]).unwrap_or_default();

        if line.starts_with(&leading_spaces) {
          // if it starts with lists_indent spaces its staying in teh same li
          let trimmed = line.replacen(' ', "", li_indent);
          li_lines.push(trimmed);
        } else if line.is_empty() {
          // close out teh existing list item
          li_finished = true;
          finished = true;
        } else if compare_lists(list_type, li_type) {
          // get lists done properly, dont close out too fast
          li_finished = true;

          // repeat the current line next time
          row -= 1;
        } else {
          // line is not part of the ul or li
          // close out teh li and ul
          li_finished = true;
          finished = true;
        }
      }

      if (row + 1) >= lines.len() as isize {
        li_finished = true;
      }

      if li_finished {
        if li_lines.len() == 1 {
          li_array.push(NodeData {
            tag: "li".to_string(),
            text: Some(li_lines[0].clone()),
            misc: li_number.clone(),
            contents: vec![],
          });
        } else {
          let contents = self
            .block_process(li_lines.clone())
            // if it errors then fail fairly gracefully
            .unwrap_or_default();

          li_array.push(NodeData {
            tag: "li".to_string(),
            text: None,
            misc: li_number.clone(),
            contents,
          });
        }

        // reset for next round
        li_finished = false;
        li_active = false;
        li_lines = vec![];
      }

      if finished {
        break;
      }

      row += 1;
    }

    if !li_array.is_empty() {
      let list_type = match li_number {
        None => "ul".to_string(),
        Some(_) => "ol".to_string(),
      };
      let node = NodeData {
        tag: list_type,
        text: None,
        misc: None,
        contents: li_array,
      };

      Some((node, row))
    } else {
      None
    }
  }

  fn block_process_paragraph(&self, lines: &[String], row_start: isize, block_excluded: &[String]) -> Option<(NodeData, isize)> {
    /*
        Paragraphs are the catch all.
        If something does not fit in anything else its a paragraph.

        A paragraph ends on a blank like or the start of a new block.

        The extra parameters is due to the fact that some html can be considered part of a paragraph and to facilitate passthroughs.
        For example:
            <a href="">link</a> in paragraph

        Paragraphs have the advantage of having no failure, they don't need to search for a closer
    */

    let mut block_lines = lines[..(row_start as usize)].to_vec();

    let mut row = row_start;
    let mut first_run = true;
    loop {
      if row >= lines.len() as isize {
        break;
      }

      let line = &lines[row as usize];

      // // a line gap is instant break
      if line.is_empty() {
        break;
      }

      if block_lines.is_empty() {
        let line_to_push = if block_excluded.contains(&"html".to_string()) || block_excluded.contains(&"html_comment".to_string()) || block_excluded.contains(&"html_cdata".to_string()) {
          // strip first < from it, &gt;
          line.replacen('<', "&gt;", 1).to_string()
        } else {
          line.to_string()
        };

        // add it to the tmp array
        block_lines.push(line_to_push);
      } else {
        match block_line_type(line, &[]) {
          LineType::Paragraph => {}
          _ => {
            // if its not a paragraph and if its not the first run

            if !first_run {
              break;
            }
          }
        }

        block_lines.push(line.to_string());
      }

      first_run = false;
      row += 1;
    }

    if !block_lines.is_empty() {
      let node = NodeData {
        tag: 'p'.to_string(),
        text: Some(block_lines.join("\n")),
        misc: None,
        contents: vec![],
      };

      Some((node, row))
    } else {
      None
    }
  }

  fn block_process_html(&self, lines: &[String], html_type: HtmlType) -> Option<(NodeData, isize, usize)> {
    /*
        HTML blocks are identified by a line starting with:
            * <!--
            * <![CDATA[
            * < followed by any alphanumeric character
        Based on the identifier an opener and a closer are set.

        Both are set in order to handle nestled html:
            <div><div>inner content</div></div>


        Blocks can also end mid way through a line so keeping track of the last col is important for them:
            <div>first</div><div>second</div>


    */

    let opener;
    let closer;
    let tag;

    if !lines.is_empty() {
      if let Some((inner_tag, inner_opener, inner_closer)) = self.block_process_html_tags(&lines[0], html_type) {
        tag = inner_tag;
        opener = inner_opener;
        closer = inner_closer;
      } else {
        return None;
      }
    } else {
      return None;
    };

    let mut block_lines = vec![];

    let mut html_depth = 0;

    let mut row = 0;
    let mut index = 0;
    let mut html_end_col = 0;
    let mut finished = false;
    loop {
      if row >= lines.len() as isize {
        break;
      }
      let line = &lines[row as usize];

      // to get html blocks each row has to be scanned for both openers and closers

      // set the flags for this line
      let mut valid_chars_closing = 0;
      let mut valid_chars_opening = 0;
      // reset the index
      index = 0;

      let line_chars: Vec<char> = line.chars().collect();

      loop {
        if closer.is_empty() {
          break;
        }
        if opener.is_empty() {
          break;
        }
        if index >= line_chars.len() {
          break;
        }

        let character = line_chars[index];

        // set it to be used next round
        index += 1;

        if opener[valid_chars_opening] == character {
          // character is valid,
          valid_chars_opening += 1;

          if valid_chars_opening == opener.len() {
            html_depth += 1;
            valid_chars_opening = 0;
          }
        } else {
          // reset
          valid_chars_opening = 0;
        }

        if closer[valid_chars_closing] == character {
          // character is valid,
          valid_chars_closing += 1;

          if valid_chars_closing == closer.len() {
            html_depth -= 1;
            if html_depth == 0 {
              finished = true;
              break;
            }
            // reset it
            valid_chars_closing = 0;
          }
        } else {
          // reset
          valid_chars_closing = 0;
        }
      }

      if finished {
        break;
      } else {
        block_lines.push(line.to_string())
      }

      row += 1;
    }

    // its marked finished if its closed out properly
    if finished {
      let mut partial_line = false;
      let mut permitted_in_p = false;
      let forbidden_tag = false; // how handling forbidden tags would be

      let line = &lines[row as usize];

      // check if there is anything left on teh line
      if line[index..].trim() != "" {
        partial_line = true;

        // check if the tag is permitted in paragraphs
        // https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/Content_categories#phrasing_content
        // https://www.w3.org/TR/html52/dom.html#phrasing-content
        let allowed_tags_p = vec![
          // pure html stuff
          "abbr",
          "audio",
          "b",
          "button",
          "canvas",
          "cite",
          "code",
          "data",
          "datalist",
          "dfn",
          "em",
          "embed",
          "i",
          "iframe",
          "img",
          "input",
          "label",
          "mark",
          "math",
          "meter",
          "noscript",
          "object",
          "output",
          "picture",
          "progress",
          "q",
          "ruby",
          "samp",
          "script",
          "select",
          "small",
          "span",
          "string",
          "sub",
          "sup",
          "svg",
          "textarea",
          "time",
          "u",
          "var",
          "video",
          "wbr",
          // according to dn these are not always applicable
          "a",
          "del",
          "ins",
          "map",
          // these require an itemprop attribute, however not looking for these
          // "link", "mata",

          // these are the "custom" tags for misc html stuff
          "custom_comment",
          "custom_cdata",
        ];

        if allowed_tags_p.contains(&tag.as_str()) {
          permitted_in_p = true;
        }
      }

      // add list of forbidden tags here
      if forbidden_tag {
        // forbidden
      }

      // tidy up the closing of html here
      if partial_line {
        if permitted_in_p {
          // change the type to paragraph

          // switch over to paragraph, return that result
          return if let Some((node, offset)) = self.block_process_paragraph(lines, row, &[]) {
            Some((node, offset, 0))
          } else {
            // was not a code block, exclude it and try the same line again
            None
          };
        } else {
          // the first part is always added to teh tmp array
          let tmp_line = &line[..index];
          block_lines.push(tmp_line.to_string());

          // decide what to do with teh rest

          // get the type of the remaining line
          // if its html then its processed
          // else it is completely skipped
          let remaining = &line[index..];

          if let LineType::Html(_) = block_line_type(remaining, &[]) {
            html_end_col = index;
          }
        }
      } else {
        // add whole line to teh array
        block_lines.push(line.to_string());
      }

      // close up the html tag

      let node = NodeData {
        tag: "html".to_string(),
        // gonna do no more processing on this, just going to pass it straight through
        text: Some(block_lines.join("\n")),
        misc: None,
        contents: vec![],
      };

      // +1 to include the last row
      Some((node, row + 1, html_end_col))
    } else {
      None
    }
  }

  fn block_process_html_tags(&self, line: &str, html_type: HtmlType) -> Option<(String, Vec<char>, Vec<char>)> {
    let mut opener;
    let mut closer;
    let tag;

    match html_type {
      HtmlType::Normal => {
        // type of tag isn't known here

        // this first line is to find the tag name
        let mut tag_vec: Vec<char> = vec![];

        // 1 to skip the opening < in teh first run
        let mut index = 1;
        let line_chars: Vec<char> = line.trim_start().chars().collect();
        loop {
          if index >= line_chars.len() {
            break;
          }

          let character = line_chars[index];

          // set it to be used next round
          index += 1;
          match character {
            'a'..='z' | 'A'..='Z' | '0'..='9' => {
              tag_vec.push(character);
            }
            _ => {
              // not a valid tag character
              break;
            }
          }
        }

        // not a valid tag
        if tag_vec.is_empty() {
          // repeat teh current line, but not as a html tag
          return None;
        }

        // use html_void to see what type of closing it has
        // if its in html_void then its />
        // else its </tag>

        tag = tag_vec.iter().collect();

        // check for nestled html
        // <div>1<div>2</div></div>
        //html_depth += 1;

        opener = vec!['<'];
        opener.extend(&tag_vec);

        // these are the self-closing tags
        if self.html_void.contains(&tag) {
          closer = vec!['/', '>'];
        } else {
          closer = vec!['<', '/'];
          closer.extend(&tag_vec);
          closer.push('>');
        }
      }
      HtmlType::Comment => {
        // -->
        opener = vec!['<', '!', '-', '-'];
        closer = vec!['-', '-', '>'];
        tag = "custom_comment".to_string();
      }
      HtmlType::CData => {
        // ]]>
        opener = vec!['<', '!', '[', 'C', 'D', 'A', 'T', 'A', '['];
        closer = vec![']', ']', '>'];
        tag = "custom_cdata".to_string();
      }
    }

    Some((tag, opener, closer))
  }

  fn block_process_table_alignment(&self, line: &str) -> HashMap<i32, String> {
    let mut alignment: HashMap<i32, String> = HashMap::new();

    let mut last_character: Option<char> = None;
    let mut start = false;
    let mut end = false;
    let mut counter = 0;
    for character in line.trim_end_matches('|').chars() {
      match character {
        '|' => {
          // skip first character
          if last_character.is_none() {
            // dont forget to set it here
            last_character = Some(character);
            continue;
          }

          counter += 1;

          if start && end {
            alignment.insert(counter, "center".to_string());
          } else if start {
            alignment.insert(counter, "left".to_string());
          } else if end {
            alignment.insert(counter, "right".to_string());
          } else {
            // dont insert
          }

          // reset for next one
          start = false;
          end = false;
        }
        ':' => {
          // if preceded by a space or bar then its a start

          // if preceded by - then its a end

          if let Some(last) = last_character {
            if last == ' ' || last == '|' {
              start = true;
            }
            if last == '-' {
              end = true;
            }
          }
        }
        _ => {
          // nothing should happen here
        }
      }

      last_character = Some(character);
    }

    counter += 1;
    // tidy up the trailing
    if start && end {
      alignment.insert(counter, "center".to_string());
    } else if start {
      alignment.insert(counter, "left".to_string());
    } else if end {
      alignment.insert(counter, "right".to_string());
    } else {
      // dont insert
    }

    alignment
  }

  fn block_merge(&self, nodes: Vec<NodeData>, mut depth: usize, slides: bool) -> String {
    let mut result: Vec<String> = vec![];

    let mut slide_number = 0;

    if slides {
      // let indented = format!("{:indent$}{}", "", x_opening, indent = (depth * self.indentation));
      result.push(format!("{:indent$}<section class='slide' id='{}' style='display:none'>", "", slide_number, indent = (depth * self.indentation)));

      depth += 1;
    }

    for node in nodes {
      let tag = node.tag.as_str();
      let mut slide = false;
      let (opening, content, closing) = match tag {
        // html just gets passed straight through
        "html" => {
          if let Some(text) = node.text {
            result.push(text);
          }
          (None, None, None)
        }
        // simple stuff first, no recursion

        "h1" | "h2" | "h3" | "h4" | "h5" | "h6" => {
          if let Some(text) = node.text {
            let processed = self.inline_process(text, 0);

            // no attributes or anything
            let formatted = format!("<{}>{}</{}>", tag, processed, tag);

            (Some(formatted), None, None)
          } else {
            (None, None, None)
          }
        }

        "p" => {
          if let Some(text) = node.text {
            let processed = self.inline_process(text, depth + 1);

            // no attributes or anything
            let open = format!("<{}>", tag);
            let close = format!("</{}>", tag);

            (Some(open), Some(processed), Some(close))
          } else {
            (None, None, None)
          }
        }

        "hr" => {
          if !slides {
            // simplest of them all
            (Some("<hr />".to_string()), None, None)
          } else {
            // starting teh next slide
            slide_number += 1;
            slide = true;
            (Some("</section>".to_string()), None, Some(format!("<section class='slide' id='{}' style='display:none'>", slide_number)))
          }
        }

        "pre" => {
          if let Some(contents) = node.text {

            // misc for fenced code blocks is used for the language
            let formatted = if let Some(language) = node.misc {
              // using class="language-{language}" is a pseudo standard
              format!("<pre><code class=\"language-{}\">\n{}\n</code></pre>", language, contents)
            } else {
              format!("<pre><code>\n{}\n</code></pre>", contents)
            };

            result.push(formatted);
          }
          (None, None, None)
        }

        ///////////////////////////
        // recursive beyond this //
        ///////////////////////////

        // table
        "table" | "thead" | "tbody" | "tr" |
        // blockquote always contains other stuff
        "blockquote" |
        // lists
        "ul" | "ol"
        => {
          let processed = self.block_merge(node.contents, depth + 1, false);
          let open = format!("<{}>", tag);
          let close = format!("</{}>", tag);
          (Some(open), Some(processed), Some(close))
        }

        "th" | "td" => {
          let processed = if let Some(text) = node.text {
            self.inline_process(text, depth + 1)
          } else {
            "".to_string()
          };

          let open = if let Some(alignment) = node.misc {
            // text-align: alignment;
            // https://developer.mozilla.org/en-US/docs/Web/CSS/text-align
            format!("<{} style=\"text-align: {};\">", tag, alignment)
          } else {
            format!("<{}>", tag)
          };

          let close = format!("</{}>", tag);

          (Some(open), Some(processed), Some(close))
        }

        "li" => {
          // this handles the contents for both ordered and unordered lists

          let processed = if !node.contents.is_empty() {
            self.block_merge(node.contents, depth + 1, false)
          } else if let Some(text) = node.text {
            self.inline_process(text, depth + 1)
          } else {
            "".to_string()
          };


          let open = if let Some(value) = node.misc {
            format!("<{} value=\"{}\">", tag, value)
          } else {
            format!("<{}>", tag)
          };

          let close = format!("</{}>", tag);

          (Some(open), Some(processed), Some(close))
        }

        _ => {
          (None, None, None)
        }
      };

      if slide {
        depth -= 1;
      }
      if let Some(x_opening) = opening {
        let indented = format!("{:indent$}{}", "", x_opening, indent = (depth * self.indentation));
        result.push(indented)
      }
      if let Some(x_content) = content {
        let indented = format!("{:indent$}{}", "", x_content, indent = 0);
        // already indented in its section, specifically text content
        result.push(indented)
      }
      if let Some(x_closing) = closing {
        let indented = format!("{:indent$}{}", "", x_closing, indent = (depth * self.indentation));
        result.push(indented)
      }
      if slide {
        depth += 1;
      }
    }

    if slides {
      // let indented = format!("{:indent$}{}", "", x_opening, indent = (depth * self.indentation));
      result.push(format!("{:indent$}</section>", "", indent = ((depth - 1) * self.indentation)));
    }

    result.join("\n")
  }

  fn inline_process(&self, input: String, depth: usize) -> String {
    // convert it into nodes
    let nodes = self.inline_spans(input);

    // merge them together according to the type of node
    let merged = Converter::inline_merge(nodes);

    // add line breaks if they are required
    // replace "  \n" with "\n<br />\n"
    let line_breaks_added = merged.replace("  \n", "\n<br />\n");

    // add the appropriate indentation to make it readable as raw html
    self.inline_indention(line_breaks_added, depth)
  }

  // this breaks down teh spans
  fn inline_spans(&self, input: String) -> Vec<NodeData> {
    /*

        //em//
        **strong**
        __underline__
        ~~strikethrough~~
        >!spoiler text!< class=md-spoiler-text , hiding it is handled in css, using teh format from Reddit
        ``code``
        <<autolink>>

        yes these are breaking changes with commonmark right back to daringfireball's one.
        main reason is conformity.
        All of these have a specific opening and closing tag (em/strong from previous implementations is tricky)
        identifiers dont do multiple jobs, _ was used in place of * in previous versions


        contents of html are checked for markdown


        [link][]
        ![image]

        Valid:
            Normal:
                []()
                [](<>)
                [text](url "title")
                [text](<url with spaces>)

            Reference:
                [identifier]: /url "title"
                [text][identifier]

    text is subject to markdown
        ye some breaking changes here



        >! spoilered  **strong spoilered //emp strong spoilered// __not underlined** !< __
        >!![]!<

     */

    let mut node_data: Vec<NodeData> = vec![];

    let mut characters: Vec<char> = input.chars().collect();

    let mut index = 0;

    let mut span_closer: Vec<char> = vec![];
    let mut span_start = 0;

    /*
    normal = stuff created by the delimiters
    html is any html blocks
    url is anything that matches a url [
    image is like the url but starts with ![
    */

    // marks the start of a span
    let mut span_active = false;
    let mut span_finished = false;
    let mut span_reset = false;

    // for allocating where items go
    let mut span_text = true; // defaults to this,
    let mut span_normal = false;
    let mut span_html = false;

    let mut span_normal_type: Option<String> = None;

    let mut html_tag_vec = vec![];
    let mut html_tag_complete = false;

    loop {
      if index >= characters.len() {
        break;
      }

      let character = characters[index];
      let character_next = if (index + 1) < characters.len() { Some(characters[index + 1]) } else { None };
      let character_last_escape = if (index as isize - 1) > 0 { characters[index - 1] == '\\' } else { false };

      // this gets teh first section of a span
      if !span_active {
        match character {
          // em
          '/' => {
            // /
            if let Some(next) = character_next {
              if next == '/' {
                if character_last_escape {
                  // remove the backslash
                  characters.remove(index - 1);

                  // since everything moved to the left so staying still is teh same as the normal +1 for the index
                  // the -1 here will cancel out the +1 later
                  index -= 1;
                } else {
                  span_normal_type = Some("em".to_string());
                  span_closer = vec!['/', '/'];
                  span_active = true;
                  span_normal = true;
                  span_text = false;
                }
              }
            }
          }
          // strong
          '*' => {
            // *
            if let Some(next) = character_next {
              if next == '*' {
                if character_last_escape {
                  // remove the backslash
                  characters.remove(index - 1);

                  // since everything moved to the left so staying still is teh same as the normal +1 for the index
                  // the -1 here will cancel out the +1 later
                  index -= 1;
                } else {
                  span_normal_type = Some("strong".to_string());
                  span_closer = vec!['*', '*'];
                  span_active = true;
                  span_normal = true;
                  span_text = false;
                }
              }
            }
          }
          // underline
          '_' => {
            // _
            if let Some(next) = character_next {
              if next == '_' {
                if character_last_escape {
                  // remove the backslash
                  characters.remove(index - 1);

                  // since everything moved to the left so staying still is teh same as the normal +1 for the index
                  // the -1 here will cancel out the +1 later
                  index -= 1;
                } else {
                  span_normal_type = Some("u".to_string());
                  span_closer = vec!['_', '_'];
                  span_active = true;
                  span_normal = true;
                  span_text = false;
                }
              }
            }
          }
          // strikethrough
          '~' => {
            // ~
            if let Some(next) = character_next {
              if next == '~' {
                if character_last_escape {
                  // remove the backslash
                  characters.remove(index - 1);

                  // since everything moved to the left so staying still is teh same as the normal +1 for the index
                  // the -1 here will cancel out the +1 later
                  index -= 1;
                } else {
                  span_normal_type = Some("s".to_string());
                  span_closer = vec!['~', '~'];
                  span_active = true;
                  span_normal = true;
                  span_text = false;
                }
              }
            }
          }
          // code
          '`' => {
            // `
            if let Some(next) = character_next {
              if next == '`' {
                if character_last_escape {
                  // remove the backslash
                  characters.remove(index - 1);

                  // since everything moved to the left so staying still is teh same as the normal +1 for the index
                  // the -1 here will cancel out the +1 later
                  index -= 1;
                } else {
                  span_normal_type = Some("code".to_string());
                  span_closer = vec!['`', '`'];
                  span_active = true;
                  span_normal = true;
                  span_text = false;
                }
              }
            }
          }
          '>' => {
            // >
            // only one that breaks teh double mould
            if let Some(next) = character_next {
              if next == '!' {
                if character_last_escape {
                  // remove the backslash
                  characters.remove(index - 1);

                  // since everything moved to the left so staying still is teh same as the normal +1 for the index
                  // the -1 here will cancel out the +1 later
                  index -= 1;
                } else {
                  span_normal_type = Some("spoiler".to_string());
                  span_closer = vec!['!', '<'];
                  span_active = true;
                  span_normal = true;
                  span_text = false;
                }
              }
            }
          }
          '<' => {
            // set html first
            span_active = true;
            span_html = true;
            span_text = false;

            if let Some(next) = character_next {
              // check if autolink
              match next {
                '<' => {
                  // undo the html flag set above
                  span_html = false;

                  // set info for the closing
                  span_normal_type = Some("autolink".to_string());
                  span_closer = vec!['>', '>'];
                  span_normal = true;
                }
                ' ' | '\t' | '/' | '>' => {
                  // not a html tag
                  span_active = false;
                  span_html = false;
                  span_text = true;
                }
                _ => {}
              }
            }

            if span_active && character_last_escape {
              // reset teh flags
              span_active = false;
              span_html = false;
              span_text = true;
              span_normal_type = None;
              span_closer = vec![];
              span_normal = false;

              characters.remove(index - 1);

              // since everything moved to the left so staying still is teh same as the normal +1 for the index
              // the -1 here will cancel out the +1 later
              index -= 1;
            }
          }
          '[' => {
            if character_last_escape {
              // remove the backslash
              characters.remove(index - 1);

              // since everything moved to the left so staying still is teh same as the normal +1 for the index
              // the -1 here will cancel out the +1 later
              index -= 1;
            } else if let Some((node, offset)) = self.inline_spans_links(&characters[index..], "a".to_string()) {
              // set teh stuff before to be a text
              let text: String = characters[span_start..index].iter().collect();
              node_data.push(NodeData {
                tag: "text".to_string(),
                misc: None,
                text: Some(text),
                contents: vec![],
              });

              // add the data to teh array
              node_data.push(node);
              index += offset;

              // set the enw span start
              span_start = index + 1;
            }
          }
          '!' => {
            if let Some(next) = character_next {
              if next == '[' {
                if character_last_escape {
                  // remove the backslash
                  characters.remove(index - 1);

                  // since everything moved to the left so staying still is teh same as the normal +1 for the index
                  // the -1 here will cancel out the +1 later
                  index -= 1;
                } else {
                  // images require a +1 to the offsets to take into account teh !
                  if let Some((node, offset)) = self.inline_spans_links(&characters[(index + 1)..], "img".to_string()) {
                    // set teh stuff before to be a text
                    let text: String = characters[span_start..index].iter().collect();
                    node_data.push(NodeData {
                      tag: "text".to_string(),
                      misc: None,
                      text: Some(text),
                      contents: vec![],
                    });

                    // add the data to teh array
                    node_data.push(node);
                    index += offset + 1;

                    // set the new span start
                    span_start = index + 2;
                  }
                }
              }
            }
          }

          _ => {
            // do nothing
          }
        }

        if span_active {
          // tidy up teh last text span here
          // bundle up everything between span_start and index into a string
          let text: String = characters[span_start..index].iter().collect();
          node_data.push(NodeData {
            tag: "text".to_string(),
            misc: None,
            text: Some(text),
            contents: vec![],
          });

          span_start = index;

          // skip to the next character if html, two if its
          if span_normal {
            index += 2;
          } else {
            index += 1;
          }

          continue;
        }
      }

      // this captures the content of the spans
      if span_active {
        if span_normal {
          // this aught to be pretty easy, just find the tail end of the span, bundle it up into NodeData

          if span_closer[0] == character {
            if character_last_escape {
              characters.remove(index - 1);

              // since everything moved to the left so staying still is teh same as the normal +1 for the index
              // the -1 here will cancel out the +1 later
              index -= 1;
            } else if let Some(next) = character_next {
              if span_closer[1] == next {
                // mark it as finished
                span_finished = true;
              }
            }
          }
        }

        if span_html {
          // of the tag isn't complete then we need to find it
          if !html_tag_complete {
            match character {
              ' ' | '\t' => {
                html_tag_complete = true;
              }
              '/' | '>' => {
                // check last character, if it was a backslash then remove the backslash and add the current character
                if characters[index - 1] == '\\' {
                  // remove teh slash
                  // add the bracket
                  html_tag_vec.pop();
                  html_tag_vec.push(character);
                } else {
                  html_tag_complete = true;
                }
              }
              _ => {
                // as long as there are no spaces
                html_tag_vec.push(character);
              }
            }
          }
          if html_tag_complete {
            if span_closer.is_empty() {
              // sets teh
              let tag: String = html_tag_vec.iter().collect();

              if self.html_void.contains(&tag) {
                span_closer = vec!['/', '>'];
              } else {
                span_closer = vec!['<', '/'];
                span_closer.extend(&html_tag_vec);
                span_closer.push('>');
              }
              // reverse it so it can be used easier down below
              span_closer.reverse();
            }
            // using tag_closing find the end of the html span
            // check if this is the

            // starts off true
            let mut matches = true;
            for (position, closing_char) in span_closer.iter().enumerate() {
              let index_new: isize = (index as isize) - (position as isize);
              if index_new < 0 {
                span_reset = true;
                matches = false;
                break;
              }

              if &characters[index_new as usize] != closing_char {
                matches = false;
                break;
              }
            }
            if matches {
              span_finished = true;
            }
          }
        }
      }

      if character_next.is_none() && !span_finished {
        if span_text {
          span_finished = true;
        } else {
          span_reset = true;
        }
      }

      if span_reset {
        // if the first character is < then replace it with &lt;
        if characters[span_start] == '<' {
          let replacement_char = ['&', 'l', 't', ';'];
          characters.splice(span_start..=span_start, replacement_char.iter().cloned());
        }

        span_closer = vec![];

        span_active = false;
        span_finished = false;
        span_reset = false; // reset this flag

        span_normal = false;
        span_html = false;
        span_text = true;

        span_normal_type = None;

        html_tag_vec = vec![];
        html_tag_complete = false;

        // reset back to the start of the span
        index = span_start;
        // skip to next character so it wont re-analyse the same set of characters
        index += 1;

        continue;
      }

      if span_finished {
        if span_text {
          let text: String = characters[span_start..=index].iter().collect();
          node_data.push(NodeData {
            tag: "text".to_string(),
            misc: None,
            text: Some(text),
            contents: vec![],
          });

          span_start = index;

          // skip to the next character if html, two if its
          if span_normal {
            index += 1;
          } else {
            index += 0;
          }
        }

        if span_normal {
          // the offset of 2 is to exclude teh delimiter
          let text_raw: String = characters[(span_start + 2)..=(index - 1)].iter().collect();

          let span_type = if let Some(span) = span_normal_type { span } else { "text".to_string() };

          let (text, contents) = match span_type.as_str() {
            "text" => (Some(text_raw), vec![]),
            "code" => (Some(text_raw), vec![]),
            "autolink" => (Some(text_raw), vec![]),
            _ => (None, self.inline_spans(text_raw)),
          };

          node_data.push(NodeData {
            tag: span_type,
            misc: None,
            text,
            contents,
          });

          // skip to after the current delimiter
          span_start = index + 2;

          // skip to the next character if html, two if its

          index += 1;
        }

        if span_html {
          let text: String = characters[span_start..=index].iter().collect();

          node_data.push(NodeData {
            tag: "html".to_string(),
            misc: None,
            text: Some(text),
            contents: vec![],
          });

          span_start = index + 1;
        }

        span_closer = vec![];

        span_active = false;
        span_finished = false;

        span_normal = false;
        span_html = false;
        span_text = true;

        span_normal_type = None;

        html_tag_vec = vec![];
        html_tag_complete = false;
      }

      index += 1;
    }

    // catch anything at the end
    if characters.is_empty() || span_start < (characters.len() - 1) {
      let text: String = characters[span_start..].iter().collect();
      node_data.push(NodeData {
        tag: "text".to_string(),
        misc: None,
        text: Some(text),
        contents: vec![],
      });
    }

    node_data
  }

  fn inline_spans_links(&self, characters: &[char], tag: String) -> Option<(NodeData, usize)> {
    let references = self.references.clone();

    // index starts at 1 as first character is always the opener [
    let mut index = 1;

    let mut block_first = vec![];

    let mut link_type = None;
    loop {
      if index >= characters.len() {
        break;
      }

      let character = characters[index];
      let character_next = if (index + 1) < characters.len() { Some(characters[index + 1]) } else { None };
      let character_last_escape = if (index as isize - 1) > 0 { characters[index - 1] == '\\' } else { false };

      match character {
        ']' => {
          if character_last_escape {
            block_first.pop();
            block_first.push(character)
          } else {
            // find the type of link
            if let Some(next) = character_next {
              match next {
                '(' => {
                  link_type = Some("classic".to_string());
                  // going to need to take a look at the contents
                  index += 2;
                }
                '[' => {
                  link_type = Some("reference_named".to_string());
                  index += 2;
                }
                _ => {
                  // check if the link_text is the same as the reference links in self.references
                  link_type = Some("reference_anon".to_string());
                }
              }
            } else {
              // set up to test if its teh anon type
              link_type = Some("reference_anon".to_string());
            }

            // regardless break it here
            break;
          }
        }

        _ => block_first.push(character),
      }

      index += 1;
    }

    // deal with teh anon references
    if let Some(type_) = &link_type {
      // only concerned with teh anon_test one this time
      if let "reference_anon" = type_.as_str() {
        let reference: String = block_first.iter().collect();

        return match references.get(&reference) {
          Some(link_data) => {
            let url = link_data.url.clone();

            let contents = if let Some(title) = &link_data.title {
              // if it has a title then it will override the reference for teh contents
              self.inline_spans(title.clone())
            } else {
              // reference us used as the contents if its an anon one
              self.inline_spans(reference)
            };

            let node = NodeData {
              tag,
              misc: link_data.title.clone(),
              // text here is used as the url
              text: Some(url),
              contents,
            };

            Some((node, index))
          }
          None => {
            // not a link
            None
          }
        };
      }
    }

    let mut block_second = vec![];
    let mut angle_brackets = false;
    loop {
      if index >= characters.len() {
        break;
      }

      let character = characters[index];
      let character_last_escape = if (index as isize - 1) > 0 { characters[index - 1] == '\\' } else { false };

      // do stuff here

      if let Some(type_) = &link_type {
        match type_.as_str() {
          "classic" => {
            match character {
              '<' => {
                if block_second.is_empty() {
                  angle_brackets = true;
                }

                // add it regardless, can be easily removed later
                block_second.push(character)
              }
              ' ' | '\t' => {
                if angle_brackets {
                  block_second.push(character)
                } else if block_second.is_empty() {
                  // do nothing,
                } else {
                  // next loop will start on the next character
                  index += 1;
                  // its teh end of the url
                  break;
                }
              }
              ')' => {
                if angle_brackets {
                  block_second.push(character)
                } else if character_last_escape {
                  block_second.pop();
                  block_second.push(character);
                } else {
                  // this marks the classic link as finished
                  // so make it and finish early

                  let text_raw: String = block_first.iter().collect();
                  let contents = self.inline_spans(text_raw);
                  let url: String = block_second.iter().collect();
                  let node = NodeData {
                    tag,
                    misc: None,
                    text: Some(url),
                    contents,
                  };

                  return Some((node, index));
                }
              }
              '\n' => {
                // all inline links must be on the one line, not split over multiple
                // saves a lot of complexity
                return None;
              }
              '>' => {
                if angle_brackets {
                  // marks teh end of the span
                  // however if last char is a backslash its ignored
                  if character_last_escape {
                    block_second.pop();
                    block_second.push(character);
                  } else {
                    // remove the first char which is a <
                    block_second.remove(0);
                    break;
                  }
                } else {
                  block_second.push(character)
                }
              }
              _ => block_second.push(character),
            }
          }
          "reference_named" => match character {
            ']' => {
              if character_last_escape {
                block_second.pop();
                block_second.push(character);
              } else {
                break;
              }
            }
            _ => block_second.push(character),
          },
          _ => {
            // only classic and reference_named should show up
          }
        }
      }

      index += 1;
    }

    // tidy up the references
    if let Some(type_) = &link_type {
      // not for classic
      if let "reference_named" = type_.as_str() {
        let reference: String = block_second.iter().collect();

        return match references.get(&reference) {
          Some(link_data) => {
            let text_raw: String = block_first.iter().collect();
            let contents = self.inline_spans(text_raw);

            let node = NodeData {
              tag,
              misc: link_data.title.clone(),
              // text here is used as the url
              text: Some(link_data.url.clone()),
              contents,
            };

            Some((node, index))
          }
          None => {
            // not a link
            None
          }
        };
      }
    }

    //
    /*
    now just left with classic

    // these are the three general patterns to find
    [](<>)
    [](/url )
    [](/url "title")
     */

    // this handles the title
    let mut block_third = vec![];
    let mut delimiter = (' ', false, false);
    loop {
      if index >= characters.len() {
        break;
      }

      let character = characters[index];
      let character_last_escape = if (index as isize - 1) > 0 { characters[index - 1] == '\\' } else { false };

      match character {
        // manage delimiter
        '\'' | '"' => {
          if !delimiter.1 {
            // set the delimiter
            delimiter = (character, true, false);
          } else if delimiter.0 == character {
            // check if last character was an escape
            if character_last_escape {
              block_third.pop();
              block_third.push(character);
            } else {
              // mark it closed
              delimiter = (character, true, true);
            }
          } else {
            block_third.push(character);
          }
        }

        //
        ')' => {
          if delimiter.1 && !delimiter.2 {
            // if delimiter is open then add it to teh array
            block_third.push(character);
          } else {
            // else check if its escaped
            if character_last_escape {
              block_third.pop();
              block_third.push(character);
            } else {
              // this marks the classic link as finished
              // so make it and finish early

              let text_raw: String = block_first.iter().collect();
              let contents = self.inline_spans(text_raw);

              let url: String = block_second.iter().collect();

              let title = if block_third.is_empty() {
                None
              } else {
                let title_tmp: String = block_third.iter().collect();
                Some(title_tmp)
              };

              let node = NodeData {
                tag,
                misc: title,
                text: Some(url),
                contents,
              };

              return Some((node, index));
            }
          }
        }

        ' ' | '\t' => {
          if delimiter.1 && !delimiter.2 {
            // if delimiter is open then add it to teh array
            block_third.push(character);
          } else {
            // do nothing
          }
        }
        _ => {
          block_third.push(character);
        }
      }

      index += 1;
    }

    // if its gone to teh end without being closed out then its invalid

    None
  }

  fn inline_merge(nodes: Vec<NodeData>) -> String {
    let mut result: Vec<String> = vec![];

    for node in nodes {
      let tag = node.tag.as_str();
      let (opening, content, closing) = match tag {
        // html  and text just gets passed straight through
        "html" | "text" => {
          if let Some(text) = node.text {
            result.push(text);
          }
          (None, None, None)
        }
        // treat this basically the same as html above
        "autolink" => {
          if let Some(text) = node.text {
            let (cleaned, mail, tel) = if text.starts_with("mailto:") {
              (text.replacen("mailto:", "", 1), true, false)
            } else if text.starts_with("MAILTO:") {
              (text.replacen("MAILTO:", "", 1), true, false)
            } else if text.starts_with("tel:") {
              (text.replacen("tel:", "", 1), false, true)
            } else if text.starts_with("TEL:") {
              (text.replacen("TEL:", "", 1), false, true)
            } else {
              (text, false, false)
            };

            // check if it contains @
            let formatted = if cleaned.contains('@') || mail {
              format!("<a target='_blank' rel='noopener noreferrer' href='mailto:{}'>{}</a>", &cleaned, &cleaned)
            } else if tel {
              format!("<a target='_blank' rel='noopener noreferrer' href='tel:{}'>{}</a>", &cleaned, &cleaned)
            } else {
              format!("<a target='_blank' rel='noopener noreferrer' href='{}'>{}</a>", &cleaned, &cleaned)
            };

            result.push(formatted);
          }
          (None, None, None)
        }

        // set tags and no recursion
        "code" => {
          if let Some(text) = node.text {
            // no attributes or anything recursive
            let open = format!("<{}>", tag);
            let close = format!("</{}>", tag);

            let cleaned = text.replace('<', "&lt;").replace('>', "&gt;");

            (Some(open), Some(cleaned), Some(close))
          } else {
            (None, None, None)
          }
        }

        // recursive
        "em" | "strong" | "u" | "s" => {
          let processed = Converter::inline_merge(node.contents);
          let open = format!("<{}>", tag);
          let close = format!("</{}>", tag);
          (Some(open), Some(processed), Some(close))
        }

        // spoiler is a span with a class of class="md-spoiler" on it as there is no html element for spoilers and must be done using css
        "spoiler" => {
          let processed = Converter::inline_merge(node.contents);
          let open = "<span class='md-spoiler'>".to_string();
          let close = "</span>".to_string();
          (Some(open), Some(processed), Some(close))
        }

        "a" => {
          let url = node.text.unwrap_or_default();
          let open = if let Some(title) = node.misc {
            format!("<a target='_blank' rel='noopener noreferrer' href='{}' title='{}'>", url, title)
          } else {
            format!("<a target='_blank' rel='noopener noreferrer' href='{}'>", url)
          };

          let processed = Converter::inline_merge(node.contents);
          let close = "</a>".to_string();

          (Some(open), Some(processed), Some(close))
        }

        "img" => {
          let url = node.text.unwrap_or_default();
          let alt = Converter::inline_merge(node.contents);
          let open = if let Some(title) = node.misc {
            format!("<img src='{}' alt='{}' title='{}' />", url, alt, title)
          } else {
            format!("<img src='{}' alt='{}' />", url, alt)
          };

          (Some(open), None, None)
        }

        _ => (None, None, None),
      };

      if let Some(data) = opening {
        result.push(data)
      }
      if let Some(data) = content {
        result.push(data)
      }
      if let Some(data) = closing {
        result.push(data)
      }
    }

    result.join("")
  }

  fn inline_indention(&self, input: String, depth: usize) -> String {
    let mut result: Vec<String> = vec![];

    let lines = input.split('\n').collect::<Vec<_>>();
    for line in lines {
      let indented = format!("{:indent$}{}", "", line, indent = (depth * self.indentation));
      result.push(indented);
    }

    result.join("\n")
  }
}
//

fn compare_lists(list_type: LineType, li_type: LineType) -> bool {
  if let LineType::OL(_, _) = list_type {
    if let LineType::OL(_, _) = li_type {
      return true;
    }
  }

  if let LineType::UL(_, _) = list_type {
    if let LineType::UL(_, _) = li_type {
      return true;
    }
  }

  false
}