28# define always_inline __forceinline
30# define always_inline inline __attribute__((always_inline))
37 std::string_view directive =
"#line ";
39 std::string_view sub_str =
str.substr(0, pos);
40 size_t nearest_line_directive = sub_str.rfind(directive);
41 size_t line_count = 1;
42 if (nearest_line_directive != std::string::npos) {
43 sub_str = sub_str.substr(nearest_line_directive + directive.size());
44 line_count = std::stoll(std::string(sub_str)) - 1;
46 return line_count + std::count(sub_str.begin(), sub_str.end(),
'\n');
51 std::string_view sub_str =
str.substr(0, pos);
52 size_t nearest_line_directive = sub_str.rfind(
'\n');
53 return (nearest_line_directive == std::string::npos) ?
55 (sub_str.size() - nearest_line_directive - 1);
60 std::string_view directive =
"#line ";
61 std::string_view sub_str =
str.substr(0, pos);
63 while (!sub_str.empty()) {
64 size_t nearest_line_directive = sub_str.rfind(directive);
66 if (nearest_line_directive == std::string_view::npos) {
70 size_t line_end = sub_str.find(
'\n', nearest_line_directive);
71 std::string_view directive_line = sub_str.substr(nearest_line_directive,
72 line_end - nearest_line_directive);
74 size_t first_quote = directive_line.find(
'"');
75 if (first_quote != std::string_view::npos) {
76 size_t second_quote = directive_line.find(
'"', first_quote + 1);
77 if (second_quote != std::string_view::npos) {
78 return std::string(directive_line.substr(first_quote + 1, second_quote - first_quote - 1));
82 if (nearest_line_directive == 0) {
85 sub_str = sub_str.substr(0, nearest_line_directive);
92 size_t start =
str.rfind(
'\n', pos);
93 size_t end =
str.find(
'\n', pos);
94 start = (start != std::string::npos) ? start + 1 : 0;
95 return std::string(
str.substr(start, end - start));
107Scope Token::attribute_before()
const
115 return prev.prev().scope();
140 std::string token_filename = tok.filename();
141 std::string full_report = token_filename.empty() ?
default_filename : token_filename;
142 full_report +=
":" + std::to_string(tok.line_number());
143 full_report +=
":" + std::to_string(tok.char_number() + 1);
144 full_report +=
": " + std::string(message);
147 full_report += tok.line_str() +
"\n";
148 if (!tok.
str().empty()) {
149 full_report += std::string(tok.char_number(),
' ') +
"^" +
150 std::string(tok.
str().size() - 1,
'~');
153 err = {std::string(message), full_report};
163 std::string message = reports.front().second;
164 std::string full_report;
166 for (
const auto &[tok, msg] : reports) {
168 full_report +=
"\n" +
err->full_report;
172 err = {message, full_report};
183 full_report +=
":" + std::to_string(row);
184 full_report +=
":" + std::to_string(column + 1);
185 full_report +=
": " + std::string(message) +
"\n";
187 err = {std::string(message), full_report};
191 std::array<CharClass, 128> table;
198 std::array<CharClass, 128> table;
215 return (s[1] ==
'=') ?
Equal : input;
217 return (s[1] ==
'=') ?
NotEqual : input;
219 return (s[1] ==
'|') ?
LogicalOr : input;
223 return (s[1] ==
'<') ?
LShift : ((s[1] ==
'=') ?
LEqual : input);
225 return (s[1] ==
'>') ?
RShift : ((s[1] ==
'=') ?
GEqual : input);
231 return (s[1] ==
'=') ?
AssignMul : input;
233 return (s[1] ==
'=') ?
AssignDiv : input;
242 return (s[1] ==
'<' && s[2] ==
'=') ?
AssignLShift : input;
244 return (s[1] ==
'>' && s[2] ==
'=') ?
AssignRShift : input;
256 return s.size() * (s[0] - s[std::min(1, sz - 1)] * 2 - s.back() * 2);
263 return (s ==
"do") ?
Do :
Word;
265 return (s ==
"if") ?
If :
Word;
267 return (s ==
"for") ?
For :
Word;
321 for (
auto tok : *
this) {
322 switch (tok.type()) {
337 for (
int i = 1; i <
size(); ++i) {
340 if (type ==
'<' || type ==
'>') {
341 Token prev = (*this)[i - 1];
342 const bool preceded_by_space = prev.followed_by_whitespace();
360 for (
int i = 1; i <
size(); ++i) {
362 switch (tok.
type()) {
393 ranges.reserve(predicted_scope_count);
394 types.reserve(predicted_scope_count);
400 ranges.emplace_back(start_tok_id, 1);
401 types.emplace_back(type);
438 update_string_view();
443 return Token(*
this, i);
446void ParserBase::update_string_view()
448 assert(this->scope_types.data() !=
nullptr);
449 assert(!this->scope_types.empty());
450 this->scope_types_str = std::string_view(
reinterpret_cast<char *
>(this->scope_types.data()),
451 this->scope_types.size());
460 if (!all_mutation_ordered) {
471 bool added_trailing_new_line =
false;
472 if (
str_.back() !=
'\n') {
474 added_trailing_new_line =
true;
482 size_t start = mut.src_range.start;
483 size_t end = start + mut.src_range.size;
485 result.append(
str_.data() + offset, start - offset);
487 result.append(mut.replacement);
496 if (added_trailing_new_line) {
size_t line_number(const std::string_view &str, size_t pos)
static always_inline TokenType type_lookup(std::string_view s)
std::string filename(const std::string_view &str, size_t pos)
size_t char_number(const std::string_view &str, size_t pos)
std::string line_str(const std::string_view &str, size_t pos)
constexpr always_inline uint8_t perfect_hash(std::string_view s)
static always_inline TokenType multi_tok_lookup(TokenType input, std::string_view s)
const CharClass char_class_table[128]
std::string default_filename
void report(Token tok, std::string_view message)
std::optional< Error > err
static const std::array< CharClass, 128 > default_char_class_table
void identify_template_tokens()
void reset_template_tokens()
static const std::array< CharClass, 128 > bsl_char_class_table
std::vector< Mutation > mutations_
bool apply_mutations(LexerBase &lexer, const bool all_mutation_ordered=false)
void build_token_to_scope_map()
Token operator[](int i) const
std::vector< IndexRange > scope_ranges
std::vector< int > token_scope
std::vector< ScopeType > types
Item always_inline back() const
void always_inline exit_scope(int end_tok_id)
void always_inline enter_scope(ScopeType type, size_t start_tok_id)
ScopeStack(size_t predicted_scope_count)
bool always_inline empty() const
std::vector< IndexRange > ranges
std::vector< ScopeStack::Item > scopes
Token next(int i=1) const
Token(const lexit::Token &tok)
Token prev(int i=1) const
uint32_t size() const
Return the amount of token inside the buffer.
std::string_view str() const
Token(const TokenBuffer *buf, int32_t index)
const TokenType & type() const
Token next(int i=1) const