Blender V5.3
gpu_shader_preprocessor.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2026 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
8
9#include "BKE_global.hh"
10#include "BLI_bit_vector.hh"
11
13
15#include "gpu_shader_private.hh"
16
20
21namespace blender::gpu {
22
25using namespace lexit;
26
27template<typename IToken> struct TokenRange {
28 IToken begin, end;
29};
30
31/* -------------------------------------------------------------------- */
34
35class Line;
36class Directive;
37
39 private:
45 Vector<std::unique_ptr<std::string>> pasted_tokens_str;
46
47 public:
49 {
50 pasted_tokens_str.append(std::make_unique<std::string>());
51 pasted_tokens_str.last()->resize(1024 * 2);
52 this->reserve(512);
53 this->offsets_[0] = 0;
54 this->offsets_end_[0] = 0;
55 }
56
60 BLI_INLINE_METHOD TokenMut paste_token(const std::string &tok_str,
61 const TokenType type,
62 const bool followed_by_space)
63 {
64 std::string *str_ptr = pasted_tokens_str.last().get();
65 int occupied = str_.length();
66
67 int token_size = tok_str.size();
68
69 while (occupied + token_size > str_ptr->size()) {
70 /* Create new larger buffer and copy current content.
71 * Do not free old string to not invalidate the StringRefs inside the atomization map.
72 * Cost is relatively small. */
73 pasted_tokens_str.append(std::make_unique<std::string>());
74 std::string *new_str_ptr = pasted_tokens_str.last().get();
75 new_str_ptr->resize(str_ptr->size() * 2);
76 std::memcpy(new_str_ptr->data(), str_ptr->data(), str_ptr->size());
77 /* Continue logic with new string buffer. */
78 str_ptr = new_str_ptr;
79 }
80
81 /* Copy token string. */
82 std::memcpy(str_ptr->data() + occupied, tok_str.data(), token_size);
83 /* Update buffer string_ref. */
84 str_ = {str_ptr->data(), size_t(occupied + token_size)};
85 /* Add token to buffer. */
86 append(type, 0, token_size - followed_by_space, token_size);
87
88 return TokenMut(this, size_ - 1);
89 }
90};
91
98 /* Line index to token range. */
100
101 /* Preprocessor directive to line index. */
103
104 public:
109
110 Line line(int index) const;
111 Directive directive(int index) const;
112
114 {
115 return identifier_map.lookup_or_add(tok_str);
116 }
117
119
120 BLI_INLINE_METHOD Token paste_token(const std::string &tok_str,
121 const TokenType type,
122 const bool followed_by_space)
123 {
124 TokenMut tok = pasting_buf.paste_token(tok_str, type, followed_by_space);
125 /* IMPORTANT: The hash function need to store a StringRef of the string. We have to make sure
126 * to feed it the final stored string to avoid referencing freed memory. */
127 tok.atom() = hash(tok.str());
128 return tok;
129 }
130
132 {
134 }
135
136 protected:
137 void lexical_analysis(std::string_view input)
138 {
140 lex_pass();
141 }
142
143 /* Backing buffer for line_offsets. */
145
147
155 {
157
158 std::memset(atoms_.get(), 0, size() * sizeof(TokenAtom));
159
160 /* Reserved 0 atom (invalid). */
161 hash(" ");
162
163 KeywordTable keywords({
174 });
175
176 /* Not keywords, but warm the identifier map with common identifiers. */
183 identifier_map.lookup_or_add("texture");
189
190 atomize_words(identifier_map, keywords);
191
192 /* Create line structure. */
194 line_offsets_buf_[0] = 0;
195 int line_id = 1;
196 for (int i = 0; i < size_; ++i) {
197 line_offsets_buf_[line_id] = i + 1;
198 line_id += types_[i] == NewLine;
199 }
200 line_offsets_buf_.resize(line_id);
201 /* Finish last line. But only do so if it contains at least one character. */
202 if (line_offsets_buf_.last() != size_) {
203 line_offsets_buf_.append(size_);
204 }
205
206 /* Create directive structure. */
208 for (int i = 0; i < line_offsets_buf_.size(); ++i) {
209 if (types_[line_offsets_buf_[i]] == '#') {
210 directive_lines.append(i);
211 }
212 }
213
215 }
216};
217
219
220/* -------------------------------------------------------------------- */
223
224/* Has quite a performance hit in debug. Enable if needed. */
225// #define DEBUG_ID_STRING
226
227class Line {
228 friend Directive;
229 friend AtomicLexer;
230
231 private:
232#ifdef DEBUG_ID_STRING
233 std::string_view debug_str_;
234#endif
235 const AtomicLexer *lex_;
236 int index_;
237
238 Line() : lex_(nullptr), index_(-1) {};
239
240 Line(const AtomicLexer *lex, int index) : lex_(lex), index_(index)
241 {
243#ifdef DEBUG_ID_STRING
244 debug_str_ = str();
245#endif
246 };
247
248 public:
249 static Line invalid()
250 {
251 return Line();
252 }
253
254 explicit operator int() const
255 {
256 return index_;
257 }
258
259 Token first() const
260 {
261 return (*lex_)[lex_->line_offsets[index_].first()];
262 }
263 /* Return the end element. Return the token before \n or \n if line is empty. */
264 Token last() const
265 {
266 blender::IndexRange range = lex_->line_offsets[index_];
267 return (*lex_)[range.size() > 1 ? range.last(1) : range.last()];
268 }
269 /* NOTE: Return the end of line character '\n'. */
270 Token end() const
271 {
272 return (*lex_)[lex_->line_offsets[index_].last()];
273 }
274
276 {
277 IndexRange range = lex_->line_offsets[index_];
278 return lex_->substr((*lex_)[range.first()], (*lex_)[range.last()]);
279 }
280
281 bool is_last() const
282 {
283 return (lex_->line_offsets.size() - 1) == index_;
284 }
285
286 bool is_valid() const
287 {
288 return index_ >= 0 && index_ < lex_->line_offsets.size();
289 }
290
291 /* Return next token. Result in undefined behavior if id is last. */
292 Line next() const
293 {
294 return Line(lex_, index_ + 1);
295 }
296 /* Return previous token. Result in undefined behavior if id is first. */
297 Line prev() const
298 {
299 return Line(lex_, index_ - 1);
300 }
301
302 friend bool operator==(const Line &a, const Line &b)
303 {
304 return a.index_ == b.index_;
305 }
306
308 {
309 return index_;
310 }
311};
312
327
328/* Simple integer identifier with a debug string view.
329 * Allow type safety and function overload. */
330class Directive {
331 friend AtomicLexer;
332
333 private:
334#ifdef DEBUG_ID_STRING
335 std::string_view debug_str_;
336#endif
337 const AtomicLexer *lex_;
338 int index_;
339
340 Directive() : lex_(nullptr), index_(-1) {};
341 Directive(const AtomicLexer *lex, int index) : lex_(lex), index_(index)
342 {
344#ifdef DEBUG_ID_STRING
345 debug_str_ = str();
346#endif
347 };
348
349 public:
350 static Directive invalid()
351 {
352 return Directive();
353 }
354
355 explicit operator int() const
356 {
357 return index_;
358 }
359
360 Line first() const
361 {
362 return lex_->line(lex_->directive_lines[index_]);
363 }
364
365 Line last() const
366 {
367 /* Could be precomputed if becoming a bottleneck. */
368 Line line = first();
369 while (line.last() == Backslash) {
370 line = line.next();
371 }
372 return line;
373 }
374
376 {
377 return DirectiveType(identifier().type());
378 }
379
380 /* Return token defining the directive type (e.g. define, undef, if ...). */
382 {
383 Line line = first();
384 Token hash_tok = line.first();
385 BLI_assert(hash_tok == Hash);
386 Token dir_tok = hash_tok.next();
387 return dir_tok;
388 }
389
390 bool is_valid() const
391 {
392 return index_ >= 0 && index_ < lex_->directive_lines.size();
393 }
394
395 bool is_last() const
396 {
397 return (lex_->directive_lines.size() - 1) == index_;
398 }
399
400 Directive next() const
401 {
402 return Directive(lex_, index_ + 1);
403 }
404 Directive prev() const
405 {
406 return Directive(lex_, index_ - 1);
407 }
408
410 {
411 Line start = first();
412 Line end = last();
413 Token tok_start = start.first();
414 Token tok_end = end.last();
415 return lex_->substr(tok_start, tok_end);
416 }
417
419 {
420 Line start = first();
421 Line end = last();
422 Token tok_start = start.first();
423 Token tok_end = end.end();
424 return lex_->substr(tok_start, tok_end);
425 }
426
427 friend bool operator==(const Directive &a, const Directive &b)
428 {
429 return a.index_ == b.index_;
430 }
431
433 {
434 return index_;
435 }
436};
437
438/* Returns the type of conditional. */
440{
441 dir = dir.next();
442 while (dir.is_valid()) {
443 DirectiveType type = dir.type();
444 if (ELEM(type,
451 {
452 return type;
453 }
454 dir = dir.next();
455 }
456 /* Missing matching #endif. */
457 // TODO exception?
460}
461
463{
464 int stack = 1;
465 while (dir.is_valid()) {
468 stack++;
469 }
470 else if (ELEM(type, DirectiveType::Endif)) {
471 stack--;
472 }
473
474 if (stack == 0) {
475 return dir; /* Endif. */
476 }
477 if (stack == 1 && ELEM(type, DirectiveType::Else, DirectiveType::Elif)) {
478 return dir;
479 }
480 }
482 return Directive::invalid();
483}
484
486{
487 return Line(this, index);
488}
489
491{
492 return Directive(this, index);
493}
494
496 /* Ensure to add a Space after the previous token. */
497 struct Space {};
498 /* Will paste a "1" token in the stream. */
499 struct True {};
500 /* Will paste a "0" token in the stream. */
501 struct False {};
502 /* Concatenate the previous token with the next one. */
503 struct Concatenate {};
504
505 /* Data container. The tokens can be from different lexers.
506 * We can't iterate using tok.next(). */
508
509 private:
510 /* Needed for token pasting. */
511 AtomicLexer &lex_;
512 /* State tracking. */
513 bool concat_next_ = false;
514 /* Cached output buffer to avoid reallocations. */
515 std::string result_buf_;
516
517 public:
518 explicit TokenStream(AtomicLexer &lex) : lex_(lex) {};
519
520 /* Set size to 0. Doesn't reallocate. */
521 void clear()
522 {
523 tokens.clear();
524 }
525
527 {
528 tokens.extend(stream.tokens);
529 return *this;
530 }
531
533 {
534 bool followed_by_space = tok.followed_by_whitespace();
535 if (concat_next_) [[unlikely]] {
536 tok = paste_token(tokens.last().str(), tok.str(), followed_by_space);
537 tok.flag = followed_by_space;
538 tokens.last() = tok;
539 concat_next_ = false;
540 }
541 else {
542 tok.flag = followed_by_space;
543 tokens.append(tok);
544 }
545 return *this;
546 }
547 /* NOTE: Not compatible with concatenation. */
549 {
550 tokens.append(paste_token("1", "", false));
551 return *this;
552 }
553 /* NOTE: Not compatible with concatenation. */
555 {
556 tokens.append(paste_token("0", "", false));
557 return *this;
558 }
559
560 template<typename IToken> TokenStream &operator<<(const TokenRange<IToken> &stream)
561 {
562 for (IToken tok = stream.begin; tok != stream.end; tok = tok.next()) {
563 *this << tok;
564 }
565 /* "Cancel" concatenation in case range is empty. */
566 concat_next_ = false;
567 return *this;
568 }
569
571 {
572 /* Don't concat if there is nothing to concatenate. */
573 concat_next_ = !tokens.is_empty();
574 return *this;
575 }
576
578 {
579 if (!tokens.is_empty()) {
580 tokens.last().flag = true;
581 }
582 return *this;
583 }
584
585 /* TODO(fclem): Remove this. Only there for expansion parser. */
586 std::string str()
587 {
588 result_buf_.clear();
589 result_buf_.reserve(tokens.size() * 7);
590 for (const auto stream_tok : tokens) {
591 result_buf_ += stream_tok.str();
592 if (stream_tok.flag) {
593 result_buf_ += ' ';
594 }
595 }
596 return result_buf_;
597 }
598
599 /* Wrapper to allow the same interface as a Token on a Token pointer. */
600 struct Iterator {
602 const Token *tok;
603
605 Iterator(const Iterator &other) = default;
606
607 bool is_valid() const
608 {
609 return tok != nullptr;
610 }
611
613 {
614 return tok ? tok->type() : Invalid;
615 }
617 {
618 return tok ? tok->atom() : 0;
619 }
620
621 std::string_view str() const
622 {
623 return tok ? tok->str() : "";
624 }
625
627 {
628 return tok ? tok->flag : false;
629 }
630
632 {
633 if (tok == nullptr || tok + 1 >= stream->tokens.end()) {
634 return Iterator(*stream, nullptr);
635 }
636 return Iterator(*stream, tok + 1);
637 }
639 {
640 if (tok == nullptr || tok - 1 < stream->tokens.begin()) {
641 return Iterator(*stream, nullptr);
642 }
643 return Iterator(*stream, tok - 1);
644 }
645
646 friend bool operator==(const Iterator &a, const Iterator &b)
647 {
648 return a.tok == b.tok;
649 }
650 friend bool operator!=(const Iterator &a, const Iterator &b)
651 {
652 return a.tok != b.tok;
653 }
654
655 friend bool operator==(const Iterator &a, TokenType b)
656 {
657 return a.type() == b;
658 }
659 friend bool operator!=(const Iterator &a, TokenType b)
660 {
661 return a.type() != b;
662 }
663
664 /* For iterator compatibility. */
666 {
667 if (tok == nullptr || tok + 1 >= stream->tokens.end()) {
668 tok = nullptr;
669 }
670 else {
671 ++tok;
672 }
673 return *this;
674 }
676 {
677 return *this;
678 }
679 };
680
682 {
683 *this << *it.tok;
684 tokens.last().flag = it.followed_by_whitespace();
685 return *this;
686 }
687
689 {
690 if (tokens.is_empty()) {
691 return end();
692 }
693 return Iterator(*this, tokens.begin());
694 }
695
696 Iterator end() const
697 {
698 return Iterator(*this, nullptr);
699 }
700
701 private:
702 Token paste_token(const StringRef &a, const StringRef &b, bool followed_by_space)
703 {
704 std::string pasted;
705 pasted.reserve(a.size() + b.size() + followed_by_space);
706 pasted.append(a);
707 pasted.append(b);
708 if (followed_by_space) {
709 pasted += ' ';
710 }
711 return lex_.paste_token(pasted, Word, followed_by_space);
712 }
713};
714
716{
717 for (const auto tok : src.tokens) {
718 dst.parse_token(tok);
719 dst << tok.str();
720 if (tok.flag) {
721 dst << " ";
722 }
723 }
724 return dst;
725}
726
728{
729 dst.parse_token_range(range.begin, range.end);
730 dst << range.begin.buf_->substr(range.begin, range.end, true);
731 return dst;
732}
733
735{
736 dst.parse_token(tok);
737 dst << tok.str_with_whitespace();
738 return dst;
739}
740
742
743/* -------------------------------------------------------------------- */
746
760 private:
761 AtomicLexer lex_;
762
763 PruningStream out_stream_;
764
765 /* Cache the expression lexer to avoid memory allocations. */
766 shader::parser::ExpressionParser expression_parser;
767
768 struct TokenStreamPool {
769 /* Reference to lexer only for atom value lookups. */
770 AtomicLexer &lex_;
771 int allocated = 0;
772 int used = 0;
773 std::deque<TokenStream> pool;
774 std::vector<int> free_indices;
775
776 struct Deleter {
777 TokenStreamPool *stack;
778 int index;
779
780 void operator()(TokenStream * /*stream*/)
781 {
782 stack->free_indices.push_back(index);
783 }
784 };
785
786 using Ptr = std::unique_ptr<TokenStream, Deleter>;
787
788 Ptr alloc()
789 {
790 int target_idx;
791 if (free_indices.empty()) {
792 target_idx = pool.size();
793 pool.emplace_back(lex_);
794 }
795 else {
796 target_idx = free_indices.back();
797 free_indices.pop_back();
798 }
799
800 TokenStream &s = pool[target_idx];
801 s.clear();
802 return Ptr(&s, Deleter{this, target_idx});
803 }
804 };
805
806 using TokenStreamPtr = TokenStreamPool::Ptr;
807
808 struct Macro {
809 Directive id;
810 TokenStreamPtr definition;
811 Vector<TokenAtom> params;
812 bool is_function = false;
813 bool contains_concat = false;
814
815 Macro(Directive id) : id(id) {}
816
817 bool is_empty() const
818 {
819 return definition->tokens.is_empty();
820 }
821 };
822
823 /* When evaluating a condition directive inside this stack, disregard the directive and jump to
824 * the matching #endif. */
825 Vector<Directive, 8> jump_stack;
826 /* Own stack to avoid memory allocation during recursive expansion parsing. */
827 TokenStreamPool stream_pool = {lex_};
828 /* Set of visited macros during recursion (blue painting stack). Using a vector for speed. */
829 Vector<Directive> visited_macros;
830
831 /* Maps containing currently active macros. Map their keyword to their definition. */
832 Map<TokenAtom, Directive> defines;
833 /* Cached parsed data for each Macro. Allow lazy parsing. Indexed by Directive. */
834 Vector<std::unique_ptr<Macro>> parsed_macro;
835 /* Allows fast checking of tokens not matching any declared macro. One bit per TokenAtom. */
836 bits::BitVector<> enabled_macros;
837
841
842 /* Next preprocessor directive to evaluate. Might be overwritten by conditional evaluation. */
843 Directive next_directive = Directive::invalid();
844 /* End of the last evaluated directive. Might be overwritten by conditional evaluation.
845 * Used to resume token expansion after this line. */
846 Line last_directive_end = Line::invalid();
847
848 /* Cached keyword identifier. */
849 TokenAtom defined_atom;
850 TokenAtom va_args_atom;
851
852 public:
853 Preprocessor(const std::string_view str)
854 : lex_(str),
855 out_stream_(Token::invalid(&lex_),
856 lex_.hash("return"),
857 lex_.hash("thread"),
858 lex_.hash("device"),
859 /* In order to avoid too many false positive edges in the DCE graph, pass a
860 * list of common symbols that are builtin and shouldn't be considered as a
861 * function. */
863 lex_.hash("layout"),
864 lex_.hash("float"),
865 lex_.hash("float2"),
866 lex_.hash("float3"),
867 lex_.hash("float4"),
868 lex_.hash("float2x2"),
869 lex_.hash("float3x2"),
870 lex_.hash("float4x2"),
871 lex_.hash("float2x3"),
872 lex_.hash("float3x3"),
873 lex_.hash("float4x3"),
874 lex_.hash("float2x4"),
875 lex_.hash("float3x4"),
876 lex_.hash("float4x4"),
877 lex_.hash("packed_float3"),
878 lex_.hash("packed_int3"),
879 lex_.hash("int"),
880 lex_.hash("int2"),
881 lex_.hash("int3"),
882 lex_.hash("int4"),
883 lex_.hash("uint"),
884 lex_.hash("uint2"),
885 lex_.hash("uint3"),
886 lex_.hash("uint4"),
887 lex_.hash("bool"),
888 lex_.hash("bool2"),
889 lex_.hash("bool3"),
890 lex_.hash("bool4"),
891 lex_.hash("int32_t"),
892 lex_.hash("uint32_t"),
893 lex_.hash("bool32_t"),
894 /* Builtin functions. */
895 lex_.hash("any"),
896 lex_.hash("all"),
897 lex_.hash("texelFetch"),
898 lex_.hash("texture"),
899 lex_.hash("textureLod"),
900 lex_.hash("textureSize"),
901 lex_.hash("imageLoad"),
902 lex_.hash("imageStore"),
903 lex_.hash("imageSize"),
904 lex_.hash("clamp"),
905 lex_.hash("mix"),
906 lex_.hash("dot"),
907 lex_.hash("abs"),
908 lex_.hash("min"),
909 lex_.hash("max"),
910 lex_.hash("for"),
911 lex_.hash("if"),
912 lex_.hash("while"),
913 lex_.hash("cos"),
914 lex_.hash("sin"),
915 lex_.hash("tan"),
916 lex_.hash("atan"),
917 lex_.hash("acos"),
918 lex_.hash("asin"),
919 lex_.hash("exp"),
920 lex_.hash("exp2"),
921 lex_.hash("log"),
922 lex_.hash("pow"),
923 lex_.hash("length"),
924 lex_.hash("floor"),
925 lex_.hash("ceil"),
926 lex_.hash("fract"),
927 lex_.hash("sqrt"),
928 lex_.hash("sign"),
929 lex_.hash("return"),
930 lex_.hash("intBitsToFloat"),
931 lex_.hash("floatBitsToInt"),
932 lex_.hash("uintBitsToFloat"),
933 lex_.hash("floatBitsToUint"),
934 }),
935 enabled_macros(lex_.max_atom_value()),
936 defined_atom(lex_.hash("defined")),
937 va_args_atom(lex_.hash("__VA_ARGS__"))
938 {
939 /* From our stats. Should be enough for 100% of our cases. */
940 defines.reserve(1000);
941 /* Ensure one slot for each directive. */
942 parsed_macro.resize(lex_.directive_lines.size());
943 }
944
946 {
947 if (lex_.directive_lines.is_empty()) {
948 out_stream_ << TokenRange<Token>{.begin = lex_.front(), .end = lex_.back()};
949 return;
950 }
951
952 last_directive_end = lex_.line(0);
953 next_directive = lex_.directive(0);
954 /* Expand until the first directive. */
955 if (lex_.line(0) != next_directive.first()) {
956 expand_macros_in_range(lex_.line(0), next_directive.first().prev());
957 }
958
959 while (!next_directive.is_last()) {
960 Directive id = next_directive;
961 /* The next directive might be overwritten by evaluate_directive. Increment before call. */
962 next_directive = id.next();
963 evaluate_directive(id);
964
965 expand_macros_in_range(last_directive_end.next(), next_directive.first().prev());
966 }
967 /* Evaluate last directive without calling next and creating an invalid ID. */
968 evaluate_directive(next_directive);
969
970 if (!last_directive_end.is_last()) {
971 Line last_line = lex_.line(lex_.line_offsets.size() - 1);
972 expand_macros_in_range(last_directive_end.next(), last_line);
973 }
974 }
975
976 void optimize()
977 {
978 Vector<TokenAtom> entry_points;
979 entry_points.append(lex_.hash("main"));
980
981 out_stream_.optimize(entry_points.as_span());
982 }
983
984 std::string result_get()
985 {
986 return out_stream_.str();
987 }
988
989 private:
990 /* Return valid value if hash is a known string. Is full hash lookup + hashing. */
991 TokenAtom get_atom(StringRef str)
992 {
993 return lex_.hash(str);
994 }
995
996 void evaluate_directive(Directive dir)
997 {
998 DirectiveType dir_type = dir.type();
999
1000 /* Note: gets overwritten by conditional processing. */
1001 last_directive_end = dir.last();
1002
1003 switch (dir_type) {
1005 define_macro(dir);
1006 break;
1008 undefine_macro(dir);
1009 break;
1010 case DirectiveType::If:
1015 process_conditional(dir, dir_type);
1016 break;
1018 erase_lines(dir.first(), dir.last());
1019 break;
1021 erase_lines(dir.first(), dir.last());
1022 break;
1024 process_pragma(dir);
1027 out_stream_ << dir.str_with_whitespace();
1028 break;
1029 }
1030 }
1031
1035
1036 TokenAtom blender_atom = lex_.hash("blender");
1037 TokenAtom dce_atom = lex_.hash("dead_code_elimination");
1038 TokenAtom off_atom = lex_.hash("off");
1039 TokenAtom on_atom = lex_.hash("on");
1040
1041 BLI_NOINLINE void process_pragma(Directive dir)
1042 {
1043 Token tok = dir.identifier().next();
1044 if (tok.atom() == blender_atom) {
1045 tok = tok.next();
1046 if (tok.atom() == dce_atom) {
1047 tok = tok.next();
1048 if (tok.atom() == off_atom) {
1049 out_stream_.set_enabled_parsing(false);
1050 }
1051 else if (tok.atom() == on_atom) {
1052 out_stream_.set_enabled_parsing(true);
1053 }
1054 else {
1055 BLI_assert_msg(false, "Invalid dead_code_elimination pragma. Expecting on or off.");
1056 }
1057 }
1058 }
1059 }
1060
1064
1068 BLI_NOINLINE Vector<TokenAtom> parse_macro_params(Token &tok)
1069 {
1070 Vector<TokenAtom> macro_parameters;
1071 while (true) {
1072 /* Continue to the next name. */
1073 tok = tok.next();
1074 if (tok == Backslash && tok.next() == NewLine) {
1075 /* Preprocessor new line. Skip and continue. */
1076 tok = tok.next(2);
1077 continue;
1078 }
1079 if (tok == ParClose) {
1080 break;
1081 }
1082 macro_parameters.append(tok.str() == "..." ? va_args_atom : tok.atom());
1083 /* Continue to the next separator. */
1084 tok = tok.next();
1085 if (tok == Backslash && tok.next() == NewLine) {
1086 /* Preprocessor new line. Skip and continue. */
1087 tok = tok.next(2);
1088 continue;
1089 }
1090 if (tok == ParClose) {
1091 break;
1092 }
1093 }
1094 /* Skip closing parenthesis. */
1095 tok = tok.next();
1096
1097 return macro_parameters;
1098 }
1099
1100 BLI_NOINLINE TokenStreamPtr parse_macro_definition(Token definition_start_tok,
1101 bool &contains_concat)
1102 {
1103 TokenStreamPtr definition = stream_pool.alloc();
1104
1105 Token tok = definition_start_tok;
1106 while (true) {
1107 Token tok_next = tok.next();
1108 if (ELEM(tok, NewLine, lexit::EndOfFile)) {
1109 break;
1110 }
1111 if (tok == Backslash && tok_next == NewLine) {
1112 /* Preprocessor new line. Skip and continue. */
1113 tok = tok.next(2);
1114 /* Still insert a space to avoid merging tokens. */
1115 *definition << TokenStream::Space{};
1116 continue;
1117 }
1118 if (tok == Hash && tok_next == Hash) {
1119 /* Token Pasting operator. Emit a single Hash token. */
1120 *definition << tok;
1121 contains_concat = true;
1122 tok = tok.next(2);
1123 continue;
1124 }
1125 BLI_assert_msg(tok != Hash, "Stringify operator is not supported");
1126 *definition << tok;
1127 tok = tok_next;
1128 }
1129 return definition;
1130 }
1131
1132 BLI_NOINLINE void define_macro(Directive dir)
1133 {
1134 const Token name = dir.identifier().next();
1135 BLI_assert(name == Word);
1136 defines.add_overwrite(name.atom(), dir);
1137 enabled_macros[name.atom()].set(true);
1138 erase_lines(dir.first(), dir.last());
1139 }
1140
1141 BLI_NOINLINE Macro &get_macro(Directive dir)
1142 {
1143 if (parsed_macro[int(dir)] == nullptr) {
1144 const Token name = dir.identifier().next();
1145 const Token after_name = name.next();
1146 Token cursor = after_name;
1147
1148 auto macro = std::make_unique<Macro>(dir);
1149 macro->is_function = (after_name == ParOpen) && !name.followed_by_whitespace();
1150 if (macro->is_function) {
1151 macro->params = parse_macro_params(cursor);
1152 }
1153 macro->definition = parse_macro_definition(cursor, macro->contains_concat);
1154
1155 parsed_macro[int(dir)] = std::move(macro);
1156 }
1157 return *parsed_macro[int(dir)];
1158 }
1159
1160 void undefine_macro(Directive dir)
1161 {
1162 const Token name = dir.identifier().next();
1163 BLI_assert(name == Word);
1164 enabled_macros[name.atom()].set(false);
1165 defines.remove(name.atom());
1166 erase_lines(dir.first(), dir.last());
1167 }
1168
1172
1173 BLI_NOINLINE void process_conditional(const Directive dir, const DirectiveType dir_type)
1174 {
1175 /* If this is part of an already evaluated statement. */
1176 if (!jump_stack.is_empty() && jump_stack.last() == dir) {
1177 jump_stack.pop_last();
1178 /* Find matching endif. */
1179 Directive endif = find_next_matching_conditional(dir);
1180 while (endif.type() != DirectiveType::Endif) {
1181 endif = find_next_matching_conditional(endif);
1182 }
1183 if (endif.is_last()) {
1184 /* Erase everything this and the last directive. */
1185 Line last_before_endif = endif.first().prev();
1186 erase_lines(dir.first(), last_before_endif);
1187 next_directive = endif;
1188 /* Don't expand inside this section. */
1189 last_directive_end = last_before_endif;
1190 }
1191 else {
1192 /* Erase everything between this directive and the #endif (inclusive). */
1193 Line endif_end = endif.last();
1194 erase_lines(dir.first(), endif_end);
1195 /* Evaluate after the endif. */
1196 next_directive = endif.next();
1197 /* Don't expand inside this section. */
1198 last_directive_end = endif_end;
1199 }
1200 return;
1201 }
1202
1203 const Line dir_line_start = dir.first();
1204 const Line dir_line_end = dir.last();
1205 const Token dir_tok = dir.identifier();
1206 /* Evaluate condition. */
1207 const Token cond_start = dir_tok.next();
1208 const Token cond_end = dir_line_end.last();
1209 const bool condition_result = evaluate_condition(dir_type, cond_start, cond_end);
1210
1211 /* Find matching endif or else. */
1212 const Directive next_condition = find_next_matching_conditional(dir);
1213
1214 if (condition_result) {
1215 /* If is followed by else statement. */
1216 DirectiveType next_dir_type = next_condition.type();
1217 if (ELEM(next_dir_type, DirectiveType::Elif, DirectiveType::Else)) {
1218 /* Record a jump statement at the next #else statement to jump & erase to the #endif. */
1219 jump_stack.append(next_condition);
1220 }
1221 /* Erase condition and continue parsing content.
1222 * The #endif will just be erased later. */
1223 erase_lines(dir_line_start, dir_line_end);
1224 }
1225 else {
1226 Line last_before_next_cond = next_condition.first().prev();
1227 /* Erase everything until next condition (this directive included). */
1228 erase_lines(dir_line_start, last_before_next_cond);
1229 /* Jump to next condition. */
1230 next_directive = next_condition;
1231 /* Don't expand inside this section. */
1232 last_directive_end = last_before_next_cond;
1233 }
1234 }
1235
1236 bool evaluate_condition(const DirectiveType dir_type, Token start, Token end)
1237 {
1238 switch (dir_type) {
1240 return true;
1242 return defines.contains(start.atom());
1244 return !defines.contains(start.atom());
1245 case DirectiveType::If:
1247 return evaluate_expression(start, end);
1248 default:
1250 return true;
1251 }
1252 }
1253
1254 bool evaluate_expression(const Token start, const Token end)
1255 {
1256 /* Expand expression into integer ops string. */
1257 TokenStreamPtr expand = expand_expression(start, end);
1258
1259 /* Early out simple cases. */
1260 if (expand->tokens.size() == 1) {
1261 const auto &token = expand->tokens.first();
1262 StringRef str = token.str();
1263 if (str == "0") {
1264 return false;
1265 }
1266 if (str == "1") {
1267 return true;
1268 }
1269 }
1270
1271 try {
1272 /* TODO(fclem): Do not parse again. Simply use the token stream. */
1273 std::string str = expand->str();
1274 expression_parser.lexical_analysis(str);
1275 return expression_parser.eval() != 0;
1276 }
1277 catch (const std::exception &e) {
1278 std::cout << "\"" << lex_.substr(start, end) << "\" > \"" << expand->str() << "\" ";
1279 std::cerr << "Error: " << e.what() << "\n";
1280 return false;
1281 }
1282 }
1283
1287
1288 BLI_NOINLINE void expand_macros_in_range(const Line start_line, const Line end_line)
1289 {
1290 int start = int(start_line.first());
1291 int end = int(end_line.end());
1292 if (start > end) {
1293 return;
1294 }
1295 if (start == end) {
1296 out_stream_ << lex_[start];
1297 return;
1298 }
1299
1300 const Vector<int> &candidates = gather_candidate_in_range(start, end);
1301
1302 Token after_last_emitted = lex_[start];
1303
1304 for (const auto *it = candidates.begin(); it != candidates.end();) {
1305 const Token tok = lex_[*it];
1306 const Directive macro_id = defines.lookup(tok.atom());
1307 /* Emit tokens between the last emitted token and this one. */
1308 if (int(after_last_emitted) < *it) {
1309 out_stream_ << TokenRange<Token>{after_last_emitted, tok.prev()};
1310 }
1311
1312 if (macro_id == Directive::invalid()) {
1313 out_stream_ << tok;
1314 after_last_emitted = tok.next();
1315 ++it;
1316 continue;
1317 }
1318
1319 const Token end_tok = expand_and_replace(tok, macro_id);
1320 after_last_emitted = end_tok.next();
1321 /* Skip candidates already expanded. */
1322 while (it != candidates.end() && *it <= int(end_tok)) {
1323 ++it;
1324 }
1325 }
1326
1327 const Token last_tok = lex_[end];
1328 out_stream_ << TokenRange<Token>{after_last_emitted, last_tok};
1329 }
1330
1331 /* Cached vector to avoid reallocation. */
1332 Vector<int> expand_candidates_;
1333
1334 /* Perform coarse check using small cache table of defined macros.
1335 * Returns a vector of token index that can potentially expand.
1336 * This avoid costly hash table lookups for every token. */
1337 BLI_NOINLINE const Vector<int> &gather_candidate_in_range(int start, int end)
1338 {
1339 expand_candidates_.clear();
1340 expand_candidates_.resize(end - start + 1);
1341 int candidates_count = 0;
1342 /* TODO(fclem): This is the major bottleneck after the actual expansion.
1343 * Can be sped up using SIMD. */
1344 for (int i : blender::IndexRange::from_begin_end(start, end)) {
1345 /* Branchless insertion. Faster than. */
1346 expand_candidates_[candidates_count] = i;
1347 candidates_count += enabled_macros[lex_.atoms_[i]];
1348 }
1349 expand_candidates_.resize(candidates_count);
1350 return expand_candidates_;
1351 }
1352
1353 BLI_NOINLINE Token expand_and_replace(const Token tok, const Directive macro_id)
1354 {
1355 auto [replacement, end] = expand_macro(tok, get_macro(macro_id));
1356 out_stream_ << *replacement;
1357 return end;
1358 }
1359
1360 /* Parse and expand with the current set of macro identifier. */
1361 BLI_NOINLINE TokenStreamPtr parse_and_expand(const TokenStream &tok_stream)
1362 {
1363 auto result = stream_pool.alloc();
1364
1365 for (auto tok = tok_stream.begin(), end = tok_stream.end(); tok != end; ++tok) {
1366 if (tok == Word) {
1367 /* Try to match the token pointed at by cursor with a defined macro. If that happen advance
1368 * the cursor to the end of the macro (in case of functional macro). */
1369 Directive macro_id = defines.lookup_default(tok.atom(), Directive::invalid());
1370 if (macro_id != Directive::invalid()) {
1371 auto [replacement, end] = expand_macro(tok, get_macro(macro_id));
1372 *result << *replacement;
1373 tok = end;
1374 continue;
1375 }
1376 }
1377 *result << tok;
1378 }
1379 return result;
1380 }
1381
1382 template<typename IToken> struct ExpandedResult {
1383 /* Replacement content. */
1384 TokenStreamPtr output;
1385 /* End of range to replace. */
1386 IToken end_of_expansion;
1387 };
1388
1392 template<typename IToken>
1393 BLI_NOINLINE Vector<TokenRange<IToken>> parse_macro_args(const Vector<TokenAtom> &params,
1394 IToken &tok)
1395 {
1396 Vector<TokenRange<IToken>> args;
1397 for (const TokenAtom param_atom : params) {
1398 IToken param_start = tok;
1399 IToken param_end = get_end_of_parameter(param_start);
1400
1401 if (param_atom == va_args_atom) {
1402 /* Seek end of argument list for variadic arguments. */
1403 param_end = get_end_of_parameter(param_start, true);
1404 }
1405
1406 args.append({param_start.next(), param_end});
1407 /* Continue to the next separator. */
1408 tok = param_end;
1409 if (tok == Invalid) {
1410 break;
1411 }
1412 }
1413 /* No parameter case. */
1414 if (params.is_empty()) {
1415 /* Continue to end parenthesis. */
1416 tok = tok.next();
1417 }
1418 if (tok == Invalid) {
1419 /* Error: missing closing parenthesis. */
1420 /* Cancel expansion. */
1421 return {};
1422 }
1423 if (tok != ParClose) {
1424 /* Error: too many arguments provided to function-like macro invocation. */
1425 /* Cancel expansion. */
1426 return {};
1427 }
1428 return args;
1429 }
1430
1431 template<typename IToken>
1432 BLI_NOINLINE TokenStreamPtr expand_macro_args(const Macro &macro,
1433 const Vector<TokenRange<IToken>> &macro_args)
1434 {
1435 TokenStreamPtr ts = stream_pool.alloc();
1436 for (const auto &def_tok : *macro.definition) {
1437 if (def_tok.type() == Hash) {
1438 *ts << TokenStream::Concatenate{};
1439 continue;
1440 }
1441 if (def_tok.type() == Word && !macro_args.is_empty()) {
1442 /* Lookup macro arguments. */
1443 int arg_id = macro.params.first_index_of_try(def_tok.atom());
1444 if (arg_id != -1) {
1445 const TokenRange<IToken> &macro_value = macro_args[arg_id];
1446
1447 if (def_tok.prev() == Hash || def_tok.next() == Hash) {
1448 /* Token pasting. Do not expand now. */
1449 *ts << macro_value;
1450 }
1451 else {
1452 /* Expand argument. Can expand to the same macro (finite recursion). */
1453 TokenStreamPtr stream = stream_pool.alloc();
1454 *stream << macro_value;
1455 *ts << *parse_and_expand(*stream);
1456 }
1457
1458 if (def_tok.followed_by_whitespace()) {
1459 /* Don't lose whitespace after macro token. */
1460 *ts << TokenStream::Space{};
1461 }
1462 continue;
1463 }
1464 }
1465 *ts << def_tok;
1466 }
1467 return ts;
1468 }
1469
1470 template<typename IToken>
1471 BLI_NOINLINE ExpandedResult<IToken> expand_macro(const IToken expanded_tok, const Macro &macro)
1472 {
1473 if (visited_macros.contains(macro.id)) {
1474 /* Recursion. Do not expand. */
1475 /* Currently still replace by the original token (noop).
1476 * Would be better to not bypass replacement altogether. */
1477 TokenStreamPtr expanded = stream_pool.alloc();
1478 *expanded << expanded_tok;
1479 return {std::move(expanded), expanded_tok};
1480 }
1481
1482 IToken end_of_expansion = expanded_tok;
1483
1484 Vector<TokenRange<IToken>> fn_arguments;
1485 if (macro.is_function) {
1486 IToken tok = expanded_tok;
1487 tok = tok.next();
1488 if (tok != ParOpen) {
1489 /* Macro doesn't have parameters. It should not expand. */
1490 /* Currently still replace by the original token (noop).
1491 * Would be better to bypass replacement altogether. */
1492 TokenStreamPtr expanded = stream_pool.alloc();
1493 *expanded << expanded_tok;
1494 return {std::move(expanded), expanded_tok};
1495 }
1496
1497 fn_arguments = parse_macro_args(macro.params, tok);
1498 /* Make sure to replace the whole call. */
1499 end_of_expansion = tok;
1500 }
1501
1502 if (macro.is_empty()) {
1503 /* Empty definition. */
1504 TokenStreamPtr expanded = stream_pool.alloc();
1505 return {std::move(expanded), end_of_expansion};
1506 }
1507
1508 TokenStreamPtr result;
1509
1510 if (!macro.is_function && !macro.contains_concat) {
1511 /* Fast Path. */
1512 visited_macros.append(macro.id);
1513 result = parse_and_expand(*macro.definition);
1514 visited_macros.pop_last();
1515 }
1516 else {
1517 TokenStreamPtr expanded = expand_macro_args(macro, fn_arguments);
1518 /* Add to the set to avoid infinite recursion. */
1519 visited_macros.append(macro.id);
1520 result = parse_and_expand(*expanded);
1521 visited_macros.pop_last();
1522 }
1523
1524 if (end_of_expansion.followed_by_whitespace()) {
1525 *result << TokenStream::Space{};
1526 }
1527
1528 return {std::move(result), end_of_expansion};
1529 }
1530
1531 /* Expand token range for condition evaluation (e.g. '#if'). */
1532 TokenStreamPtr expand_expression(const Token start, const Token end)
1533 {
1534 TokenStreamPtr result = stream_pool.alloc();
1535
1536 Token tok = start;
1537 while (true) {
1538 BLI_assert(tok.is_valid());
1539 TokenAtom tok_atom = tok == Word ? tok.atom() : TokenAtom(0);
1540
1541 Directive macro_id = defines.lookup_default(tok.atom(), Directive::invalid());
1542
1543 if (tok_atom == TokenAtom(0)) {
1544 /* Non word. */
1545 *result << lex_[int(tok)];
1546 }
1547 else if (macro_id != Directive::invalid()) {
1548 Macro &macro = get_macro(macro_id);
1549 auto [replacement, macro_end] = expand_macro(Token(lex_[int(tok)]), macro);
1550 *result << *replacement;
1551 tok = lex_[int(macro_end)];
1552 }
1553 else if (tok_atom == defined_atom) {
1554 /* Parenthesis or space */
1555 tok = tok.next();
1556 const bool is_function = (tok == ParOpen);
1557 /* Token to search. */
1558 if (is_function) {
1559 tok = tok.next();
1560 }
1561 else {
1562 BLI_assert(tok == Word);
1563 }
1564 if (defines.contains(tok.atom())) {
1565 *result << TokenStream::True{};
1566 }
1567 else {
1568 *result << TokenStream::False{};
1569 }
1570 if (is_function) {
1571 /* End parenthesis. */
1572 tok = tok.next();
1573 }
1574 }
1575 else {
1576 /* Substitution failure. */
1577 *result << lex_[int(tok)];
1578 }
1579 if (tok == end) {
1580 break;
1581 }
1582 tok = skip_directive_newlines(tok.next());
1583 }
1584
1585 return result;
1586 }
1587
1591
1592 void erase_lines(Line start, Line end)
1593 {
1594 if (int(end) > int(start)) {
1595 out_stream_ << new_lines(start, end);
1596 }
1597 out_stream_ << end.end().str_with_whitespace();
1598 }
1599
1600 /* Buffer of newlines since the StringRef must stay valid until result string is built.
1601 * Since this is only to replace content, we cannot have more lines than there is.
1602 * Avoid reallocation and persistent storage logic for a few KB of memory. */
1603 std::string new_lines_buf = std::string(lex_.line_offsets.size(), '\n');
1604
1605 /* Return a string with the amount of newline character between line_start and line_end. */
1606 StringRef new_lines(Line line_start, Line line_end)
1607 {
1608 return StringRef(new_lines_buf).substr(0, int(line_end) - int(line_start));
1609 }
1610
1611 Token skip_directive_newlines(Token tok)
1612 {
1613 while (tok == Backslash && tok.next() == NewLine) {
1614 tok = tok.next().next();
1615 }
1616 return tok;
1617 }
1618
1623 template<typename IToken>
1624 static IToken get_end_of_parameter(IToken tok, bool skip_to_end = false)
1625 {
1626 /* Avoid matching comma inside parameter function calls. */
1627 int stack = 1;
1628 tok = tok.next();
1629 while (tok.is_valid()) {
1630 if (tok == ParOpen) {
1631 stack++;
1632 }
1633 else if (tok == ParClose) {
1634 stack--;
1635 }
1636 if (stack == 0) {
1637 return tok;
1638 }
1639 if (stack == 1 && tok == Comma && !skip_to_end) {
1640 return tok;
1641 }
1642 tok = tok.next();
1643 }
1644 return tok;
1645 }
1646};
1647
1649
1650/* -------------------------------------------------------------------- */
1653
1654std::string Shader::run_preprocessor(StringRef source, bool no_dead_code_elimination)
1655{
1656 BLI_assert_msg(source.find("//") == std::string::npos && source.find("/*") == std::string::npos,
1657 "Input source to the preprocessor should have no comments.");
1658
1660 return source;
1661 }
1662
1663 Preprocessor processor(source);
1664 processor.preprocess();
1665 if (!no_dead_code_elimination) {
1666 processor.optimize();
1667 }
1668 return processor.result_get();
1669}
1670
1672
1673} // namespace blender::gpu
#define ATTR_FALLTHROUGH
#define BLI_assert_unreachable()
#define BLI_INLINE_METHOD
#define BLI_assert(a)
#define BLI_NOINLINE
#define ELEM(...)
#define BLI_assert_msg(a, msg)
constexpr int64_t first() const
constexpr int64_t last(const int64_t n=0) const
constexpr StringRef()
static constexpr IndexRange from_begin_end(const int64_t begin, const int64_t end)
constexpr int64_t last(const int64_t n=0) const
constexpr int64_t size() const
constexpr int64_t size() const
constexpr int64_t find(char c, int64_t pos=0) const
friend bool operator==(const Directive &a, const Directive &b)
friend bool operator==(const Line &a, const Line &b)
static std::string run_preprocessor(StringRef source, bool no_dead_code_elimination)
#define str(s)
shader::parser::LexerBase LexerBase
PruningStream & operator<<(PruningStream &dst, const TokenStream &src)
static Directive find_next_matching_conditional(Directive dir)
shader::parser::NullParser NullParser
static DirectiveType increment_to_next_conditional(Directive &dir)
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
unsigned __int64 uint64_t
Definition BKE_text.h:91
@ G_DEBUG_GPU_SHADER_NO_PREPROCESSOR
Global G
Definition blender.cc:57
ATTR_WARN_UNUSED_RESULT const BMVert const BMEdge * e
uint16_t TokenAtom
Definition types.hh:159
TokenType
Definition types.hh:41
@ ParOpen
Definition types.hh:53
@ EndOfFile
Definition types.hh:77
@ Hash
Definition types.hh:49
@ ParClose
Definition types.hh:54
@ Else
Definition types.hh:98
@ Pragma
Definition types.hh:113
@ If
Definition types.hh:97
@ Endif
Definition types.hh:100
@ NewLine
Definition types.hh:46
@ Ifndef
Definition types.hh:102
@ Define
Definition types.hh:108
@ Elif
Definition types.hh:99
@ Ifdef
Definition types.hh:101
@ Line
Definition types.hh:111
@ Word
Definition types.hh:43
@ Invalid
Definition types.hh:42
@ Undef
Definition types.hh:122
@ Comma
Definition types.hh:66
@ Backslash
Definition types.hh:75
const CharClass char_class_table[128]
Definition tables.hh:12
#define hash
Definition noise_c.cc:156
blender::OffsetIndices< int > line_offsets
BLI_INLINE_METHOD Token paste_token(const std::string &tok_str, const TokenType type, const bool followed_by_space)
Directive directive(int index) const
void lexical_analysis(std::string_view input)
BLI_INLINE_METHOD TokenAtom hash(StringRef tok_str)
Preprocessor(const std::string_view str)
BLI_NOINLINE void parse_token(const Token tok)
BLI_NOINLINE void parse_token_range(const Token start, const Token end)
BLI_INLINE_METHOD TokenMut paste_token(const std::string &tok_str, const TokenType type, const bool followed_by_space)
friend bool operator!=(const Iterator &a, const Iterator &b)
Iterator(const Iterator &other)=default
Iterator(const TokenStream &stream, const Token *tok)
friend bool operator!=(const Iterator &a, TokenType b)
friend bool operator==(const Iterator &a, const Iterator &b)
friend bool operator==(const Iterator &a, TokenType b)
TokenStream & operator<<(const Iterator &it)
TokenStream & operator<<(Token tok)
TokenStream & operator<<(const Concatenate)
TokenStream & operator<<(const TokenRange< IToken > &stream)
TokenStream & operator<<(const TokenStream &stream)
Keyword make_keyword(std::string_view str, TokenType type)
INLINE_METHOD TokenAtom lookup_or_add(StringT str)
TokenAtom max_atom_value() const
void reserve(int token_count)
std::string_view str_
Definition lexit.hh:123
AlignedArrayPtr< uint32_t > offsets_
Definition lexit.hh:127
void process(const std::string_view str, const CharClass char_class_table[128])
Definition lexit.hh:154
void append(TokenType type, TokenAtom atom, int32_t str_size, int32_t str_size_with_witespaces)
Append a Token at the end of the buffer.
Definition lexit.hh:236
void atomize_words(IdentifierMap &identifiers, const KeywordTable &keywords)
Assign keyword types and atoms for a small set of identifier.
Definition lexit.cc:671
uint32_t size() const
Return the amount of token inside the buffer.
Definition lexit.hh:214
void reserve(const uint32_t count)
Allocate backing memory for the given number of tokens and move currently held data.
Definition lexit.cc:60
AlignedArrayPtr< TokenType > types_
Definition lexit.hh:125
AlignedArrayPtr< uint32_t > offsets_end_
Definition lexit.hh:129
AlignedArrayPtr< TokenAtom > atoms_
Definition lexit.hh:133
uint32_t size_
Definition lexit.hh:135
TokenAtom & atom()
Definition lexit.hh:367
TokenMut(TokenBuffer *buf, int32_t index)
Definition lexit.hh:108
bool followed_by_whitespace() const
Definition lexit.hh:395
Token prev(int i=1) const
Definition lexit.hh:376
const TokenAtom & atom() const
Definition lexit.hh:363
std::string_view str() const
Definition lexit.hh:381
Token(const TokenBuffer *buf, int32_t index)
Definition lexit.hh:337
const TokenType & type() const
Definition lexit.hh:354
bool is_valid() const
Definition lexit.hh:65
std::string_view str_with_whitespace() const
Definition lexit.hh:388
Token next(int i=1) const
Definition lexit.hh:372
int32_t flag
Definition lexit.hh:53
int size() const
Definition vector.hh:130
void reserve(int new_size)
Definition vector.hh:78
void resize(int new_size)
Definition vector.hh:94