Blender V5.3
token_stream.hh
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2025 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
9
10#pragma once
11
12#include "ast.hh"
13#include "enums.hh"
14#include "utils.hh"
15
16#include <array>
17
19
20struct Token;
21struct Scope;
22
27 static const std::array<CharClass, 128> bsl_char_class_table;
28 static const std::array<CharClass, 128> default_char_class_table;
29
31 std::string_view token_types_str() const
32 {
33 return std::string_view((const char *)types_.get(), size_);
34 }
35
36 /* Change words into keyword (ex: `if`, `struct`, `template`). */
37 void identify_keywords();
38 /* Change angle bracket tokens into template tokens if they match template condition. */
40 /* Undo the changes from identify_template_tokens. */
42};
43
48 static void lexical_analysis(LexerBase &lex, std::string_view input)
49 {
51 }
52};
53
57struct FullLexer {
58 static void lexical_analysis(LexerBase &lex, std::string_view input)
59 {
61 lex.merge_complex_literals(true);
63 }
64};
65
66struct ParserBase;
67
68struct ScopeLinks {
69 /* All in scope indices. */
70 int parent_ = -1;
71 int prev_ = -1;
72 int next_ = -1;
73 int child_first_ = -1;
74 int child_last_ = -1;
75};
76
82 /* A monotonically increasing ID for each file included inside the compilation unit.
83 * A file is only assigned an ID before parsing symbols (after parsing includes).
84 * Used to order tokens from different included file. */
85 int include_id = 0;
86
88 std::string_view scope_types_str;
89
90 /* --- Structure of Array style data for scopes. --- */
91
93 std::vector<ScopeType> scope_types;
95 std::vector<IndexRange> scope_ranges;
97 std::vector<ScopeLinks> scope_links;
99 std::vector<int> token_scope;
100
101 /* --- Abstract Syntax Tree. --- */
102
110 std::vector<ast::NodeData> ast_nodes;
111
112 /* Return the i'th token. */
113 Token operator[](int i) const;
114
115 void build_scope_tree(ErrorHandler &err_handler);
117 void parse_bsl(ErrorHandler &err_handler);
118
119 ast::LocalScope root() const;
120 void print_ast() const;
121
122 private:
123 void update_string_view();
124};
125
126/* Don't do anything. No access to scopes is allowed. */
128 static void semantic_analysis(ParserBase &parser, ErrorHandler & /*err_handler*/)
129 {
130 parser.scope_types = {};
131 parser.scope_ranges = {};
132 }
133};
134
135/* Do not parse. Creates a single global scope containing all tokens. */
137 static void semantic_analysis(ParserBase &parser, ErrorHandler & /*err_handler*/)
138 {
139 parser.scope_types = {ScopeType::Global};
140 parser.scope_ranges = {IndexRange(0, parser.size())};
141 parser.build_token_to_scope_map();
142 }
143};
144
147 {
148 parser.build_scope_tree(err_handler);
149 parser.build_token_to_scope_map();
150 }
151};
152
153template<typename LexerFn, typename ParserFn> struct Parser : ParserBase {
154 void lexical_analysis(std::string_view input)
155 {
156 LexerFn::lexical_analysis(*this, input);
157 }
158
160 {
161 ParserFn::semantic_analysis(*this, err_handler);
162 }
163};
164
165} // namespace blender::gpu::shader::parser
constexpr IndexRange()=default
BMesh const char void * data
static void semantic_analysis(ParserBase &parser, ErrorHandler &)
static void lexical_analysis(LexerBase &lex, std::string_view input)
static void semantic_analysis(ParserBase &parser, ErrorHandler &err_handler)
std::string_view token_types_str() const
static const std::array< CharClass, 128 > default_char_class_table
static const std::array< CharClass, 128 > bsl_char_class_table
static void semantic_analysis(ParserBase &parser, ErrorHandler &)
void build_scope_tree(ErrorHandler &err_handler)
Definition grammar.cc:1125
std::vector< ScopeLinks > scope_links
std::vector< ast::NodeData > ast_nodes
std::vector< IndexRange > scope_ranges
void parse_bsl(ErrorHandler &err_handler)
void semantic_analysis(ErrorHandler &err_handler)
void lexical_analysis(std::string_view input)
static void lexical_analysis(LexerBase &lex, std::string_view input)
void merge_complex_literals(bool bsl_template_angle_bracket_lexing=false)
Merge complex literals such as floats, strings and comments.
Definition lexit.cc:1010
void process(const std::string_view str, const CharClass char_class_table[128])
Definition lexit.hh:154
AlignedArrayPtr< TokenType > types_
Definition lexit.hh:125
uint32_t size_
Definition lexit.hh:135