willbooster-tree-sitter 7.0.20

#include "./length.h"
#include "./lexer.h"
#include "./unicode.h"

#include "tree_sitter/api.h"

#include <stdarg.h>
#include <stdio.h>
#include <string.h>

#define LOG(message, character)              \
  if (self->logger.log) {                    \
    snprintf(                                \
      self->debug_buffer,                    \
      TREE_SITTER_SERIALIZATION_BUFFER_SIZE, \
      32 <= character && character < 127 ?   \
        message " character:'%c'" :          \
        message " character:%d",             \
      character                              \
    );                                       \
    self->logger.log(                        \
      self->logger.payload,                  \
      TSLogTypeLex,                          \
      self->debug_buffer                     \
    );                                       \
  }

static const int32_t BYTE_ORDER_MARK = 0xFEFF;

static const TSRange DEFAULT_RANGE = {
  .start_point = {
    .row = 0,
    .column = 0,
  },
  .end_point = {
    .row = UINT32_MAX,
    .column = UINT32_MAX,
  },
  .start_byte = 0,
  .end_byte = UINT32_MAX
};

static void ts_lexer__set_column_data(Lexer *self, uint32_t val) {
  self->column_data.valid = true;
  self->column_data.value = val;
}

static void ts_lexer__increment_column_data(Lexer *self) {
  if (self->column_data.valid) {
    self->column_data.value++;
  }
}

static void ts_lexer__invalidate_column_data(Lexer *self) {
  self->display_column_data.valid = false;
  self->column_data.valid = false;
  self->column_data.value = 0;
}

// Check if the lexer has reached EOF. This state is stored
// by setting the lexer's `current_included_range_index` such that
// it has consumed all of its available ranges.
static bool ts_lexer__eof(const TSLexer *_self) {
  Lexer *self = (Lexer *)_self;
  return self->current_included_range_index == self->included_range_count;
}

// Clear the currently stored chunk of source code, because the lexer's
// position has changed.
static void ts_lexer__clear_chunk(Lexer *self) {
  self->chunk = NULL;
  self->chunk_size = 0;
  self->chunk_start = 0;
}

// Call the lexer's input callback to obtain a new chunk of source code
// for the current position.
static void ts_lexer__get_chunk(Lexer *self) {
  self->chunk_start = self->current_position.bytes;
  self->chunk = self->input.read(
    self->input.payload,
    self->current_position.bytes,
    self->current_position.extent,
    &self->chunk_size
  );
  if (!self->chunk_size) {
    self->current_included_range_index = self->included_range_count;
    self->chunk = NULL;
  }
}

// Decode a UTF-8 character whose bytes are split across input chunks, which
// may each be shorter than the character, by copying them into a buffer that
// becomes the current chunk.
static void ts_lexer__join_split_utf8_character(
  Lexer *self,
  const uint8_t *chunk,
  uint32_t size
) {
  uint32_t character_size = 1 + U8_COUNT_TRAIL_BYTES(chunk[0]);
  memmove(self->lookahead_buffer, chunk, size);
  // A decode error that consumed every byte read so far may only mean that the
  // character continues in the next chunk.
  while (
    self->data.lookahead == TS_DECODE_ERROR &&
    self->lookahead_size == size &&
    size < character_size
  ) {
    TSPoint point = self->current_position.extent;
    point.column += size;
    uint32_t bytes_read = 0;
    const char *next = self->input.read(
      self->input.payload, self->current_position.bytes + size, point, &bytes_read
    );
    if (!bytes_read) break;
    if (bytes_read > character_size - size) bytes_read = character_size - size;
    memcpy(self->lookahead_buffer + size, next, bytes_read);
    size += bytes_read;
    self->lookahead_size = ts_decode_utf8(
      (const uint8_t *)self->lookahead_buffer, size, &self->data.lookahead
    );
  }
  // Each read may invalidate the buffer of the previous one, so the copied
  // bytes replace the chunk.
  self->chunk = self->lookahead_buffer;
  self->chunk_start = self->current_position.bytes;
  self->chunk_size = size;
}

// Decode the next unicode character in the current chunk of source code.
// This assumes that the lexer has already retrieved a chunk of source
// code that spans the current position.
static void ts_lexer__get_lookahead(Lexer *self) {
  uint32_t position_in_chunk = self->current_position.bytes - self->chunk_start;
  uint32_t size = self->chunk_size - position_in_chunk;

  if (size == 0) {
    self->lookahead_size = 1;
    self->data.lookahead = '\0';
    return;
  }

  const uint8_t *chunk = (const uint8_t *)self->chunk + position_in_chunk;

  if (self->input.encoding == TSInputEncodingUTF8 && chunk[0] < 0x80) {
    self->data.lookahead = chunk[0];
    self->lookahead_size = 1;
    return;
  }

  TSDecodeFunction decode =
    self->input.encoding == TSInputEncodingUTF8    ? ts_decode_utf8     :
    self->input.encoding == TSInputEncodingUTF16LE ? ts_decode_utf16_le :
    self->input.encoding == TSInputEncodingUTF16BE ? ts_decode_utf16_be : self->input.decode;

  self->lookahead_size = decode(chunk, size, &self->data.lookahead);

  // The chunk may end in the middle of a multi-byte character.
  if (self->data.lookahead == TS_DECODE_ERROR && size < 4) {
    if (self->input.encoding == TSInputEncodingUTF8 && U8_IS_LEAD(chunk[0])) {
      ts_lexer__join_split_utf8_character(self, chunk, size);
    } else {
      ts_lexer__get_chunk(self);
      self->lookahead_size = decode(
        (const uint8_t *)self->chunk, self->chunk_size, &self->data.lookahead
      );
    }
  }

  if (self->data.lookahead == TS_DECODE_ERROR) {
    self->lookahead_size = 1;
  }
}

static void ts_lexer_goto(Lexer *self, Length position) {
  if (position.bytes != self->current_position.bytes) {
    ts_lexer__invalidate_column_data(self);
  }

  self->current_position = position;

  unsigned first_range = 0;
  if (self->included_range_count > 1) {
    first_range = self->current_included_range_index;
    if (
      first_range > 0 && first_range <= self->included_range_count &&
      self->included_ranges[first_range - 1].end_byte > position.bytes
    ) {
      first_range--;
    }
    if (
      first_range >= self->included_range_count ||
      self->included_ranges[first_range].end_byte <= position.bytes ||
      (first_range > 0 && self->included_ranges[first_range - 1].end_byte > position.bytes)
    ) {
      first_range = 0;
      unsigned end = self->included_range_count;
      while (first_range < end) {
        unsigned mid = first_range + (end - first_range) / 2;
        if (self->included_ranges[mid].end_byte <= position.bytes) {
          first_range = mid + 1;
        } else {
          end = mid;
        }
      }
    }
  }

  bool found_included_range = false;
  for (unsigned i = first_range; i < self->included_range_count; i++) {
    TSRange *included_range = &self->included_ranges[i];
    if (
      included_range->end_byte > self->current_position.bytes &&
      included_range->end_byte > included_range->start_byte
    ) {
      if (included_range->start_byte >= self->current_position.bytes) {
        self->current_position = (Length) {
          .bytes = included_range->start_byte,
          .extent = included_range->start_point,
        };
      }

      self->current_included_range_index = i;
      found_included_range = true;
      break;
    }
  }

  if (found_included_range) {
    // If the current position is outside of the current chunk of text,
    // then clear out the current chunk of text.
    if (self->chunk && (
      self->current_position.bytes < self->chunk_start ||
      self->current_position.bytes >= self->chunk_start + self->chunk_size
    )) {
      ts_lexer__clear_chunk(self);
    }

    self->lookahead_size = 0;
    self->data.lookahead = '\0';
  }

  // If the given position is beyond any of included ranges, move to the EOF
  // state - past the end of the included ranges.
  else {
    self->current_included_range_index = self->included_range_count;
    TSRange *last_included_range = &self->included_ranges[self->included_range_count - 1];
    self->current_position = (Length) {
      .bytes = last_included_range->end_byte,
      .extent = last_included_range->end_point,
    };
    ts_lexer__clear_chunk(self);
    self->lookahead_size = 1;
    self->data.lookahead = '\0';
  }
}

static void ts_lexer__advance_display_column(Lexer *self) {
  if (!self->display_column_data.valid) return;
  uint32_t column = self->display_column_data.value;
  switch (self->data.lookahead) {
    case '\n':
    case '\r':
    case '\f':
      column = 0;
      break;
    case '\t': {
      uint32_t increment = self->display_tab_width - column % self->display_tab_width;
      column = increment > UINT32_MAX - column ? UINT32_MAX : column + increment;
      break;
    }
    default:
      if (column < UINT32_MAX && !(self->current_position.bytes == 0 && self->data.lookahead == BYTE_ORDER_MARK)) column++;
      break;
  }
  self->display_column_data.value = column;
}

static void ts_lexer__do_advance(Lexer *self, bool skip) {
  if (self->lookahead_size) {
    ts_lexer__advance_display_column(self);
    if (self->data.lookahead == '\n') {
      self->current_position.extent.row++;
      self->current_position.extent.column = 0;
      ts_lexer__set_column_data(self, 0);
    } else {
      bool is_bom = self->current_position.bytes == 0 && 
        self->data.lookahead == BYTE_ORDER_MARK;
      if (!is_bom) ts_lexer__increment_column_data(self);
      self->current_position.extent.column += self->lookahead_size;
    }
    self->current_position.bytes += self->lookahead_size;
  }

  const TSRange *current_range = &self->included_ranges[self->current_included_range_index];
  while (
    self->current_position.bytes >= current_range->end_byte ||
    current_range->end_byte == current_range->start_byte
  ) {
    if (self->current_included_range_index < self->included_range_count) {
      self->current_included_range_index++;
    }
    if (self->current_included_range_index < self->included_range_count) {
      current_range++;
      if (current_range->start_point.row != self->current_position.extent.row) {
        self->display_column_data.value = 0;
      }
      self->current_position = (Length) {
        current_range->start_byte,
        current_range->start_point,
      };
    } else {
      current_range = NULL;
      break;
    }
  }

  if (skip) self->token_start_position = self->current_position;

  if (current_range) {
    if (
      self->current_position.bytes < self->chunk_start ||
      self->current_position.bytes >= self->chunk_start + self->chunk_size
    ) {
      ts_lexer__get_chunk(self);
    }
    ts_lexer__get_lookahead(self);
  } else {
    ts_lexer__clear_chunk(self);
    self->data.lookahead = '\0';
    self->lookahead_size = 1;
  }
}

// Advance to the next character in the source code, retrieving a new
// chunk of source code if needed.
static void ts_lexer__advance(TSLexer *_self, bool skip) {
  Lexer *self = (Lexer *)_self;
  if (!self->chunk) return;

  if (skip) {
    LOG("skip", self->data.lookahead)
  } else {
    LOG("consume", self->data.lookahead)
  }

  const uint32_t next_position = self->current_position.bytes + 1;
  const uint32_t current_range_end =
    self->included_ranges[self->current_included_range_index].end_byte;
  if (
    self->input.encoding == TSInputEncodingUTF8 &&
    self->lookahead_size == 1 &&
    self->data.lookahead != '\n' &&
    next_position < current_range_end &&
    next_position < self->chunk_start + self->chunk_size
  ) {
    uint8_t next_byte = (uint8_t)self->chunk[next_position - self->chunk_start];
    if (next_byte < 0x80) {
      ts_lexer__advance_display_column(self);
      ts_lexer__increment_column_data(self);
      self->current_position.bytes++;
      self->current_position.extent.column++;
      if (skip) self->token_start_position = self->current_position;
      self->data.lookahead = next_byte;
      return;
    }
  }

  if (
    (self->input.encoding == TSInputEncodingUTF16LE ||
     self->input.encoding == TSInputEncodingUTF16BE) &&
    self->lookahead_size == 2 &&
    (uint32_t)self->data.lookahead < 0x80 &&
    self->data.lookahead != '\n' &&
    self->current_position.bytes <= UINT32_MAX - 4
  ) {
    const uint32_t next_utf16_position = self->current_position.bytes + 2;
    if (
      next_utf16_position + 1 < current_range_end &&
      next_utf16_position + 1 < self->chunk_start + self->chunk_size
    ) {
      const uint8_t *next = (const uint8_t *)self->chunk + next_utf16_position - self->chunk_start;
      uint16_t next_character = self->input.encoding == TSInputEncodingUTF16LE
        ? (uint16_t)next[0] | (uint16_t)next[1] << 8
        : (uint16_t)next[0] << 8 | (uint16_t)next[1];
      if (next_character < 0x80) {
        ts_lexer__advance_display_column(self);
        ts_lexer__increment_column_data(self);
        self->current_position.bytes = next_utf16_position;
        self->current_position.extent.column += 2;
        if (skip) self->token_start_position = self->current_position;
        self->data.lookahead = next_character;
        return;
      }
    }
  }

  ts_lexer__do_advance(self, skip);
}

static void ts_lexer__get_position(const TSLexer *_self, TSLexerPosition *position) {
  const Lexer *self = (const Lexer *)_self;
  *position = (TSLexerPosition) {
    self->current_position.bytes,
    self->current_position.extent.row,
    self->current_position.extent.column,
  };
}

static void ts_lexer__mark_start(TSLexer *_self, const TSLexerPosition *position) {
  Lexer *self = (Lexer *)_self;
  self->marked_start_position = (Length) {position->bytes, {position->row, position->column}};
  self->token_start_position = self->marked_start_position;
}

static void ts_lexer__mark_end_at(TSLexer *_self, const TSLexerPosition *position) {
  Lexer *self = (Lexer *)_self;
  self->token_end_position = (Length) {position->bytes, {position->row, position->column}};
}

static void ts_lexer__get_mark_end(const TSLexer *_self, TSLexerPosition *position) {
  const Lexer *self = (const Lexer *)_self;
  *position = (TSLexerPosition) {
    self->token_end_position.bytes,
    self->token_end_position.extent.row,
    self->token_end_position.extent.column,
  };
}

static void ts_lexer__mark_end(TSLexer *_self) {
  Lexer *self = (Lexer *)_self;
  if (!ts_lexer__eof(&self->data)) {
    // If the lexer is right at the beginning of included range,
    // then the token should be considered to end at the *end* of the
    // previous included range, rather than here.
    TSRange *current_included_range = &self->included_ranges[
      self->current_included_range_index
    ];
    if (
      self->current_included_range_index > 0 &&
      self->current_position.bytes == current_included_range->start_byte
    ) {
      TSRange *previous_included_range = current_included_range - 1;
      self->token_end_position = (Length) {
        previous_included_range->end_byte,
        previous_included_range->end_point,
      };
      return;
    }
  }
  self->token_end_position = self->current_position;
}

static void ts_lexer__replay_columns(Lexer *self) {
  uint32_t goal_byte = self->current_position.bytes;
  Length start = {
    goal_byte - self->current_position.extent.column,
    {self->current_position.extent.row, 0},
  };
  ts_lexer_goto(self, start);
  ts_lexer__set_column_data(self, 0);
  self->display_column_data = (ColumnData) {
    .value = 0,
    .valid = self->display_tab_width != 0,
  };
  ts_lexer__get_chunk(self);
  if (!ts_lexer__eof(&self->data)) {
    ts_lexer__get_lookahead(self);
    while (self->current_position.bytes < goal_byte && !ts_lexer__eof(&self->data) && self->chunk) {
      ts_lexer__do_advance(self, false);
    }
  }
}

static uint32_t ts_lexer__get_column(TSLexer *_self) {
  Lexer *self = (Lexer *)_self;
  self->did_get_column = true;
  if (!self->column_data.valid) ts_lexer__replay_columns(self);
  return self->column_data.value;
}

static uint32_t ts_lexer__get_display_column(TSLexer *_self, uint32_t tab_width) {
  Lexer *self = (Lexer *)_self;
  self->did_get_column = true;
  if (!tab_width) tab_width = 1;
  if (!self->display_column_data.valid || self->display_tab_width != tab_width) {
    ColumnData column_data = self->column_data;
    self->display_tab_width = tab_width;
    ts_lexer__replay_columns(self);
    self->column_data = column_data;
  }
  return self->display_column_data.value;
}

// Is the lexer at a boundary between two disjoint included ranges of
// source code? This is exposed as an API because some languages' external
// scanners need to perform custom actions at these boundaries.
static bool ts_lexer__is_at_included_range_start(const TSLexer *_self) {
  const Lexer *self = (const Lexer *)_self;
  if (self->current_included_range_index >= self->included_range_count) return false;
  const TSRange *current_range = &self->included_ranges[self->current_included_range_index];
  if (self->current_position.bytes != current_range->start_byte) return false;

  uint32_t previous = self->current_included_range_index;
  while (previous > 0 &&
         self->included_ranges[previous - 1].start_byte == self->included_ranges[previous - 1].end_byte) {
    previous--;
  }
  return previous == 0 || self->included_ranges[previous - 1].end_byte < current_range->start_byte;
}

static void ts_lexer__log(const TSLexer *_self, const char *fmt, ...) {
  Lexer *self = (Lexer *)_self;
  va_list args;
  va_start(args, fmt);
  if (self->logger.log) {
    vsnprintf(self->debug_buffer, TREE_SITTER_SERIALIZATION_BUFFER_SIZE, fmt, args);
    self->logger.log(self->logger.payload, TSLogTypeLex, self->debug_buffer);
  }
  va_end(args);
}

void ts_lexer_init(Lexer *self) {
  *self = (Lexer) {
    .data = {
      // The lexer's methods are stored as struct fields so that generated
      // parsers can call them without needing to be linked against this
      // library.
      .advance = ts_lexer__advance,
      .mark_end = ts_lexer__mark_end,
      .get_position = ts_lexer__get_position,
      .mark_start = ts_lexer__mark_start,
      .mark_end_at = ts_lexer__mark_end_at,
      .get_mark_end = ts_lexer__get_mark_end,
      .get_column = ts_lexer__get_column,
      .get_display_column = ts_lexer__get_display_column,
      .is_at_included_range_start = ts_lexer__is_at_included_range_start,
      .eof = ts_lexer__eof,
      .log = ts_lexer__log,
      .lookahead = 0,
      .result_symbol = 0,
    },
    .chunk = NULL,
    .chunk_size = 0,
    .chunk_start = 0,
    .current_position = {0, {0, 0}},
    .logger = {
      .payload = NULL,
      .log = NULL
    },
    .included_ranges = NULL,
    .included_range_count = 0,
    .current_included_range_index = 0,
    .did_get_column = false,
    .column_data = {
      .valid = false,
      .value = 0
    }
  };
  ts_lexer_set_included_ranges(self, NULL, 0);
}

void ts_lexer_delete(Lexer *self) {
  ts_free(self->included_ranges);
}

void ts_lexer_set_input(Lexer *self, TSInput input) {
  self->input = input;
  self->display_column_data.valid = false;
  ts_lexer__clear_chunk(self);
  ts_lexer_goto(self, self->current_position);
}

// Move the lexer to the given position. This doesn't do any work
// if the parser is already at the given position.
void ts_lexer_reset(Lexer *self, Length position) {
  if (position.bytes != self->current_position.bytes) {
    ts_lexer_goto(self, position);
  }
}

void ts_lexer_start(Lexer *self) {
  self->token_start_position = self->current_position;
  self->token_end_position = LENGTH_UNDEFINED;
  self->marked_start_position = LENGTH_UNDEFINED;
  self->data.result_symbol = 0;
  self->did_get_column = false;
  if (!ts_lexer__eof(&self->data)) {
    if (!self->chunk_size) ts_lexer__get_chunk(self);
    if (!self->lookahead_size) ts_lexer__get_lookahead(self);
    if (self->current_position.bytes == 0) {
      if (self->data.lookahead == BYTE_ORDER_MARK) {
        ts_lexer__advance(&self->data, true);
      }
      ts_lexer__set_column_data(self, 0);
    }
  }
}

void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte) {
  if (!length_is_undefined(self->marked_start_position)) {
    self->token_start_position = self->marked_start_position;
  }
  if (length_is_undefined(self->token_end_position)) {
    ts_lexer__mark_end(&self->data);
  }

  // If the token ended at an included range boundary, then its end position
  // will have been reset to the end of the preceding range. Reset the start
  // position to match.
  if (self->token_end_position.bytes < self->token_start_position.bytes) {
    self->token_start_position = self->token_end_position;
  }

  // The lexer has read every byte of the lookahead character, which can be
  // longer than one byte.
  uint32_t current_lookahead_end_byte =
    self->current_position.bytes + (self->lookahead_size > 1 ? self->lookahead_size : 1);

  // In order to determine that a byte sequence is invalid UTF8 or UTF16,
  // the character decoding algorithm may have looked at the following byte.
  // Therefore, the next byte *after* the current (invalid) character
  // affects the interpretation of the current character.
  if (self->data.lookahead == TS_DECODE_ERROR) {
    current_lookahead_end_byte += 4; // the maximum number of bytes read to identify an invalid code point
  }

  if (current_lookahead_end_byte > *lookahead_end_byte) {
    *lookahead_end_byte = current_lookahead_end_byte;
  }
}

void ts_lexer_mark_end(Lexer *self) {
  ts_lexer__mark_end(&self->data);
}

bool ts_lexer_set_included_ranges(
  Lexer *self,
  const TSRange *ranges,
  uint32_t count
) {
  if (count == 0 || !ranges) {
    ranges = &DEFAULT_RANGE;
    count = 1;
  } else {
    uint32_t previous_byte = 0;
    for (unsigned i = 0; i < count; i++) {
      const TSRange *range = &ranges[i];
      if (
        range->start_byte < previous_byte ||
        range->end_byte < range->start_byte
      ) return false;
      previous_byte = range->end_byte;
    }
  }

  size_t size = count * sizeof(TSRange);
  self->included_ranges = ts_realloc(self->included_ranges, size);
  memcpy(self->included_ranges, ranges, size);
  self->included_range_count = count;
  self->display_column_data.valid = false;
  ts_lexer_goto(self, self->current_position);
  return true;
}

TSRange *ts_lexer_included_ranges(const Lexer *self, uint32_t *count) {
  *count = self->included_range_count;
  return self->included_ranges;
}

#undef LOG