Blender V5.3
lexit.hh
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2026 Clement Foucault
2 *
3 * SPDX-License-Identifier: MIT */
4
11
12#pragma once
13
14#include <cassert>
15#include <climits>
16#include <cstdint>
17#include <iostream>
18#include <iterator>
19#include <string_view>
20
21#include "identifier.hh"
22#include "types.hh"
23#include "vector.hh"
24
25// #define LEXIT_DEBUG
26
27#ifdef LEXIT_DEBUG
28/* Make it a warning to avoid shipping with it. */
29# warning "Lexit debug mode enabled"
30
31# include <vector>
32#endif
33
34#if defined(_MSC_VER)
35# define INLINE_METHOD __forceinline
36#else
37# define INLINE_METHOD inline __attribute__((always_inline))
38#endif
39
40namespace lexit {
41
42struct TokenBuffer;
43
44struct Token {
45#ifdef LEXIT_DEBUG
46 std::string_view debug_str_;
47 const TokenType *debug_type_;
48 const TokenAtom *debug_atom_;
49#endif
52 /* General purpose flag. */
54
55 Token(const TokenBuffer *buf, int32_t index);
56
57 /* Invalid token that can still be compared with other tokens. */
58 static Token invalid(const TokenBuffer *buf);
59
60 explicit operator int32_t() const
61 {
62 return index_;
63 }
64
65 bool is_valid() const
66 {
67 return type() != EndOfFile;
68 }
69 bool is_invalid() const
70 {
71 return type() == EndOfFile;
72 }
73
74 const TokenType &type() const;
75 const TokenAtom &atom() const;
76
77 std::string_view str() const;
78 std::string_view str_with_whitespace() const;
79
80 bool followed_by_whitespace() const;
81
82 Token next(int i = 1) const;
83 Token prev(int i = 1) const;
84
85 friend bool operator==(const Token &a, const Token &b)
86 {
87 assert(a.buf_ == b.buf_);
88 return a.index_ == b.index_;
89 }
90 friend bool operator!=(const Token &a, const Token &b)
91 {
92 assert(a.buf_ == b.buf_);
93 return a.index_ != b.index_;
94 }
95
96 friend bool operator==(const Token &a, TokenType b)
97 {
98 return a.type() == b;
99 }
100 friend bool operator!=(const Token &a, TokenType b)
101 {
102 return a.type() != b;
103 }
104};
105
106/* Same as Token but allow type assignment. */
107struct TokenMut : public Token {
109
110 TokenType &type();
111 TokenAtom &atom();
112};
113
114template<typename T> struct AlignedDeleter {
115 void operator()(T *p) const
116 {
117 ::operator delete[](p, std::align_val_t{64});
118 }
119};
120
122 /* Input string. */
123 std::string_view str_;
124 /* Type of each token. */
126 /* Starting character index of each token. */
128 /* Original character index of each token before whitespace merging. */
130 /* Length in characters of each token. A value of 127 means the real size is over 126. */
132 /* Unique id for identifiers (Words). Externally set (optional). */
134 /* Number of tokens inside the buffer excluding the terminating EndOfFile token. */
135 uint32_t size_ = 0;
136 /* Number of tokens that can be contained. */
137 uint32_t allocated_size_ = 0;
138
139#ifdef LEXIT_DEBUG
140 std::vector<std::string_view> token_str_debug_;
141 std::vector<std::string_view> token_str_with_whitespace_debug_;
142#endif
143
144 TokenBuffer() = default;
145
146 TokenBuffer(const std::string_view str, const CharClass char_class_table[128])
147 {
149 }
150
151 /*
152 * The given string lifetime should outlive the #TokenBuffer. No copy is done.
153 */
154 void process(const std::string_view str, const CharClass char_class_table[128])
155 {
156 assert(str.size() < UINT_MAX);
157 str_ = str;
158 clear();
161 }
162
166 void clear();
167
173 void reserve(const uint32_t count);
174
191 void tokenize(const CharClass char_class_table[128]);
192
199 void merge_complex_literals(bool bsl_template_angle_bracket_lexing = false);
200
204 void atomize_words(IdentifierMap &identifiers, const KeywordTable &keywords);
205
209 void compute_lengths();
210
214 uint32_t size() const
215 {
216 return size_;
217 }
218
224 std::string_view substr(const Token &start,
225 const Token &end,
226 const bool with_trailing_whitespaces = false) const
227 {
228 int start_char = offsets_[int(start)];
229 int end_char = (!with_trailing_whitespaces) ? offsets_end_[int(end)] : offsets_[int(end) + 1];
230 return str_.substr(start_char, end_char - start_char);
231 }
232
236 void append(TokenType type, TokenAtom atom, int32_t str_size, int32_t str_size_with_witespaces)
237 {
238 reserve(size_ + 1);
239 types_[size_] = type;
240 atoms_[size_] = atom;
241 offsets_[size_ + 1] = offsets_[size_] + str_size_with_witespaces;
242 offsets_end_[size_] = offsets_[size_] + str_size;
243 size_++;
244 }
245
247 {
248 return Token(this, index);
249 }
251 {
252 return TokenMut(this, index);
253 }
254
258 struct TokenIt {
259 using iterator_category = std::forward_iterator_tag;
260
261 private:
262 TokenBuffer *buf_;
263 int32_t index_;
264
265 public:
266 TokenIt(TokenBuffer *buf, const int index) : buf_(buf), index_(index) {}
267
269 {
270 return TokenMut(buf_, index_);
271 }
272
274 {
275 index_++;
276 return *this;
277 }
278
280 {
281 return index_;
282 }
283
284 bool operator==(const TokenIt &other) const
285 {
286 return index_ == other.index_;
287 }
288 bool operator!=(const TokenIt &other) const
289 {
290 return index_ != other.index_;
291 }
292 bool operator<(const TokenIt &other) const
293 {
294 return index_ < other.index_;
295 }
296 };
297
299 {
300 return TokenIt(this, 0);
301 }
303 {
304 return TokenIt(this, size_);
305 }
306
308 {
309 return (*this)[0];
310 }
312 {
313 return (*this)[size_ - 1];
314 }
315
316 private:
317 inline void tokenize_scalar(uint32_t &__restrict offset,
318 uint32_t &__restrict cursor_begin,
319 uint32_t &__restrict cursor_end,
320 CharClass &__restrict prev_char_class,
321 bool &__restrict prev_whitespace,
322 uint32_t end,
323 const CharClass char_class_table[128]);
324
325 template<int Size = 0>
326 INLINE_METHOD void atomize_short_tokens_in_mask(uint64_t mask,
327 uint32_t tok_id_base,
329 const KeywordTable &kw_table);
330
331 INLINE_METHOD void atomize_tokens_in_mask(uint64_t mask,
332 uint32_t tok_id_base,
334 const KeywordTable &kw_table);
335};
336
337inline Token::Token(const TokenBuffer *buf, int32_t index) : buf_(buf)
338{
339 assert(buf_ != nullptr);
340 /* Set to Invalid / EndOfFile token if out of range. */
342#ifdef LEXIT_DEBUG
343 debug_str_ = str_with_whitespace();
344 debug_type_ = &type();
345 debug_atom_ = &atom();
346#endif
347}
348
350{
351 return Token(buf, buf->size_);
352}
353
354inline const TokenType &Token::type() const
355{
356 return buf_->types_[index_];
357}
359{
360 return const_cast<TokenBuffer *>(buf_)->types_[index_];
361}
362
363inline const TokenAtom &Token::atom() const
364{
365 return buf_->atoms_[index_];
366}
368{
369 return const_cast<TokenBuffer *>(buf_)->atoms_[index_];
370}
371
372inline Token Token::next(int i) const
373{
374 return Token(buf_, index_ + i);
375}
376inline Token Token::prev(int i) const
377{
378 return is_valid() ? Token(buf_, index_ - i) : Token(buf_, -1);
379}
380
381inline std::string_view Token::str() const
382{
383 int start = buf_->offsets_[index_];
384 int end = buf_->offsets_end_[index_];
385 return {buf_->str_.data() + start, size_t(end - start)};
386}
387
388inline std::string_view Token::str_with_whitespace() const
389{
390 int start = buf_->offsets_[index_];
391 int end = buf_->offsets_[index_ + 1];
392 return {buf_->str_.data() + start, size_t(end - start)};
393}
394
396{
397 assert(is_valid());
398 return buf_->offsets_end_[index_] != buf_->offsets_[index_ + 1];
399}
400
401inline std::ostream &operator<<(std::ostream &os, const Token &tok)
402{
403 os << "Token(";
404 os << "type='" << tok.type() << "', ";
405 os << "atom=" << tok.atom() << ", ";
406 os << "str=\"" << tok.str() << "\", ";
407 os << "str_with_whitespace=\"" << tok.str_with_whitespace() << "\"";
408 os << ")";
409 return os;
410}
411
412} // namespace lexit
unsigned long long int uint64_t
#define str(s)
#define assert(assertion)
#define index(i)
#define UINT_MAX
Definition hash_md5.cc:46
#define INLINE_METHOD
Definition identifier.hh:19
ccl_device_inline float2 mask(const MaskType mask, const float2 a)
#define T
uint16_t TokenAtom
Definition types.hh:159
TokenType
Definition types.hh:41
@ EndOfFile
Definition types.hh:77
const CharClass char_class_table[128]
Definition tables.hh:12
CharClass
Definition types.hh:20
std::ostream & operator<<(std::ostream &os, const Token &tok)
Definition lexit.hh:401
void operator()(T *p) const
Definition lexit.hh:115
Token iterator.
Definition lexit.hh:258
int32_t index() const
Definition lexit.hh:279
TokenIt(TokenBuffer *buf, const int index)
Definition lexit.hh:266
std::forward_iterator_tag iterator_category
Definition lexit.hh:259
TokenMut operator*() const
Definition lexit.hh:268
bool operator==(const TokenIt &other) const
Definition lexit.hh:284
bool operator<(const TokenIt &other) const
Definition lexit.hh:292
bool operator!=(const TokenIt &other) const
Definition lexit.hh:288
std::string_view str_
Definition lexit.hh:123
AlignedArrayPtr< uint32_t > offsets_
Definition lexit.hh:127
TokenIt end()
Definition lexit.hh:302
void merge_complex_literals(bool bsl_template_angle_bracket_lexing=false)
Merge complex literals such as floats, strings and comments.
Definition lexit.cc:1010
TokenBuffer(const std::string_view str, const CharClass char_class_table[128])
Definition lexit.hh:146
TokenIt begin()
Definition lexit.hh:298
uint32_t allocated_size_
Definition lexit.hh:137
void compute_lengths()
Compute small token length for speeding up certain tasks.
Definition lexit.cc:644
void process(const std::string_view str, const CharClass char_class_table[128])
Definition lexit.hh:154
void append(TokenType type, TokenAtom atom, int32_t str_size, int32_t str_size_with_witespaces)
Append a Token at the end of the buffer.
Definition lexit.hh:236
void atomize_words(IdentifierMap &identifiers, const KeywordTable &keywords)
Assign keyword types and atoms for a small set of identifier.
Definition lexit.cc:671
void clear()
Discard all currently held data. Does not reallocate.
Definition lexit.cc:55
TokenBuffer()=default
std::string_view substr(const Token &start, const Token &end, const bool with_trailing_whitespaces=false) const
Return the substring between the start and end tokens (included).
Definition lexit.hh:224
uint32_t size() const
Return the amount of token inside the buffer.
Definition lexit.hh:214
AlignedArrayPtr< uint8_t > lengths_
Definition lexit.hh:131
void reserve(const uint32_t count)
Allocate backing memory for the given number of tokens and move currently held data.
Definition lexit.cc:60
AlignedArrayPtr< TokenType > types_
Definition lexit.hh:125
TokenMut operator[](int index)
Definition lexit.hh:250
void tokenize(const CharClass char_class_table[128])
Tokenizes the input string by grouping contiguous characters of the same class.
Definition lexit.cc:488
AlignedArrayPtr< uint32_t > offsets_end_
Definition lexit.hh:129
AlignedArrayPtr< TokenAtom > atoms_
Definition lexit.hh:133
uint32_t size_
Definition lexit.hh:135
Token operator[](int index) const
Definition lexit.hh:246
TokenAtom & atom()
Definition lexit.hh:367
TokenMut(TokenBuffer *buf, int32_t index)
Definition lexit.hh:108
TokenType & type()
Definition lexit.hh:358
friend bool operator!=(const Token &a, TokenType b)
Definition lexit.hh:100
friend bool operator==(const Token &a, const Token &b)
Definition lexit.hh:85
bool is_invalid() const
Definition lexit.hh:69
const TokenBuffer * buf_
Definition lexit.hh:50
bool followed_by_whitespace() const
Definition lexit.hh:395
friend bool operator==(const Token &a, TokenType b)
Definition lexit.hh:96
Token prev(int i=1) const
Definition lexit.hh:376
friend bool operator!=(const Token &a, const Token &b)
Definition lexit.hh:90
const TokenAtom & atom() const
Definition lexit.hh:363
std::string_view str() const
Definition lexit.hh:381
int32_t index_
Definition lexit.hh:51
static Token invalid(const TokenBuffer *buf)
Definition lexit.hh:349
Token(const TokenBuffer *buf, int32_t index)
Definition lexit.hh:337
const TokenType & type() const
Definition lexit.hh:354
bool is_valid() const
Definition lexit.hh:65
std::string_view str_with_whitespace() const
Definition lexit.hh:388
Token next(int i=1) const
Definition lexit.hh:372
int32_t flag
Definition lexit.hh:53