29# warning "Lexit debug mode enabled"
35# define INLINE_METHOD __forceinline
37# define INLINE_METHOD inline __attribute__((always_inline))
46 std::string_view debug_str_;
77 std::string_view
str()
const;
102 return a.
type() !=
b;
117 ::operator
delete[](p, std::align_val_t{64});
140 std::vector<std::string_view> token_str_debug_;
141 std::vector<std::string_view> token_str_with_whitespace_debug_;
173 void reserve(
const uint32_t count);
226 const bool with_trailing_whitespaces =
false)
const
228 int start_char =
offsets_[int(start)];
230 return str_.substr(start_char, end_char - start_char);
286 return index_ == other.index_;
290 return index_ != other.index_;
294 return index_ < other.index_;
313 return (*
this)[
size_ - 1];
317 inline void tokenize_scalar(uint32_t &__restrict offset,
318 uint32_t &__restrict cursor_begin,
319 uint32_t &__restrict cursor_end,
321 bool &__restrict prev_whitespace,
325 template<
int Size = 0>
327 uint32_t tok_id_base,
332 uint32_t tok_id_base,
344 debug_type_ = &
type();
345 debug_atom_ = &
atom();
385 return {
buf_->
str_.data() + start, size_t(end - start)};
392 return {
buf_->
str_.data() + start, size_t(end - start)};
404 os <<
"type='" << tok.
type() <<
"', ";
405 os <<
"atom=" << tok.
atom() <<
", ";
406 os <<
"str=\"" << tok.
str() <<
"\", ";
unsigned long long int uint64_t
#define assert(assertion)
ccl_device_inline float2 mask(const MaskType mask, const float2 a)
const CharClass char_class_table[128]
std::ostream & operator<<(std::ostream &os, const Token &tok)
void operator()(T *p) const
TokenIt(TokenBuffer *buf, const int index)
std::forward_iterator_tag iterator_category
TokenMut operator*() const
bool operator==(const TokenIt &other) const
bool operator<(const TokenIt &other) const
bool operator!=(const TokenIt &other) const
AlignedArrayPtr< uint32_t > offsets_
void merge_complex_literals(bool bsl_template_angle_bracket_lexing=false)
Merge complex literals such as floats, strings and comments.
TokenBuffer(const std::string_view str, const CharClass char_class_table[128])
void compute_lengths()
Compute small token length for speeding up certain tasks.
void process(const std::string_view str, const CharClass char_class_table[128])
void append(TokenType type, TokenAtom atom, int32_t str_size, int32_t str_size_with_witespaces)
Append a Token at the end of the buffer.
void atomize_words(IdentifierMap &identifiers, const KeywordTable &keywords)
Assign keyword types and atoms for a small set of identifier.
void clear()
Discard all currently held data. Does not reallocate.
std::string_view substr(const Token &start, const Token &end, const bool with_trailing_whitespaces=false) const
Return the substring between the start and end tokens (included).
uint32_t size() const
Return the amount of token inside the buffer.
AlignedArrayPtr< uint8_t > lengths_
void reserve(const uint32_t count)
Allocate backing memory for the given number of tokens and move currently held data.
AlignedArrayPtr< TokenType > types_
TokenMut operator[](int index)
void tokenize(const CharClass char_class_table[128])
Tokenizes the input string by grouping contiguous characters of the same class.
AlignedArrayPtr< uint32_t > offsets_end_
AlignedArrayPtr< TokenAtom > atoms_
Token operator[](int index) const
TokenMut(TokenBuffer *buf, int32_t index)
friend bool operator!=(const Token &a, TokenType b)
friend bool operator==(const Token &a, const Token &b)
bool followed_by_whitespace() const
friend bool operator==(const Token &a, TokenType b)
Token prev(int i=1) const
friend bool operator!=(const Token &a, const Token &b)
const TokenAtom & atom() const
std::string_view str() const
static Token invalid(const TokenBuffer *buf)
Token(const TokenBuffer *buf, int32_t index)
const TokenType & type() const
std::string_view str_with_whitespace() const
Token next(int i=1) const