Blender V5.3
token.hh
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2025 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
9
10#pragma once
11
12#include "token_stream.hh"
13
15
16struct Scope;
17
23 Token(const lexit::Token &tok) : lexit::Token(tok) {}
24
26 /* Create an invalid token. */
27 Token(const ParserBase &data) : lexit::Token(&data, -1) {}
28
29 Token prev(int i = 1) const
30 {
31 return static_cast<const lexit::Token *>(this)->prev(i);
32 }
33 Token next(int i = 1) const
34 {
35 return static_cast<const lexit::Token *>(this)->next(i);
36 }
37
39 {
40 Token tok = this->next();
41 while (tok.is_valid() && tok != type) {
42 tok = tok.next();
43 }
44 return tok;
45 }
46
47 /* Return start of namespace identifier if the token is part of one. */
49 {
50 if (*this != Word) {
51 return *this;
52 }
53 /* Scan back identifier that could contain namespaces. */
54 Token tok = *this;
55 while (tok.is_valid()) {
56 if (tok.prev() == ':' && tok.prev(2) == ':') {
57 tok = tok.prev(3);
58 }
59 else {
60 return tok;
61 }
62 }
63 return tok;
64 }
65
66 /* For a word, return the name containing the prefix namespaces if present. */
67 std::string full_symbol_name() const
68 {
69 size_t start = namespace_start().str_index_start();
70 size_t end = str_index_last_no_whitespace();
71 return std::string(buf_->str_.substr(start, end - start + 1));
72 }
73
74 /* Returns the scope that contains this token. */
75 Scope scope() const;
76
77 size_t str_index_start() const
78 {
79 return buf_->offsets_[index_];
80 }
81
82 size_t str_index_last() const
83 {
84 return buf_->offsets_[index_ + 1] - 1;
85 }
86
88 {
89 return buf_->offsets_end_[index_] - 1;
90 }
91
92 /* Index of the first character of the line this token is. */
93 size_t line_start() const
94 {
95 size_t pos = buf_->str_.rfind('\n', str_index_start());
96 return (pos == std::string::npos) ? 0 : (pos + 1);
97 }
98
99 /* Index of the last character of the line this token is, excluding `\n`. */
100 size_t line_end() const
101 {
102 size_t pos = buf_->str_.find('\n', str_index_start());
103 return (pos == std::string::npos) ? (buf_->str_.size() - 1) : (pos - 1);
104 }
105
106 std::string_view str_with_whitespace() const
107 {
108 if (is_invalid()) {
109 return "";
110 }
111 return static_cast<const lexit::Token *>(this)->str_with_whitespace();
112 }
113
114 std::string_view str() const
115 {
116 if (is_invalid()) {
117 return "";
118 }
119 return static_cast<const lexit::Token *>(this)->str();
120 }
121
122 /* Return the line number this token is found at. Take into account the #line directives.
123 * If `at_end` is true, return the line number after this token. */
124 size_t line_number(bool at_end = false) const
125 {
126 if (is_invalid()) {
127 return 0;
128 }
130 int line_num = parser::line_number(buf_->str_, index);
131 /* Add the last char (not counted by line_number). */
132 return line_num + int(at_end && buf_->str_[index] == '\n');
133 }
134
135 /* Return the offset to the start of the line. */
136 size_t char_number() const
137 {
138 if (is_invalid()) {
139 return 0;
140 }
142 }
143
144 /* Return the name of the file containing the token. */
145 std::string filename() const
146 {
147 if (is_invalid()) {
148 return "";
149 }
151 }
152
153 /* Return the line the token is at. */
154 std::string line_str() const
155 {
157 }
158
160 {
161 if (is_invalid()) {
162 return Invalid;
163 }
164 return TokenType(buf_->types_[index_]);
165 }
166
167 /* Return the attribute scope before this token if it exists. */
168 Scope attribute_before() const;
169 /* Return the attribute scope after this token if it exists. */
170 Scope attribute_after() const;
171
173 {
174 return this->type() == type;
175 }
177 {
178 return !(*this == type);
179 }
180 bool operator==(char type) const
181 {
182 return *this == TokenType(type);
183 }
184 bool operator!=(char type) const
185 {
186 return *this != TokenType(type);
187 }
188};
189
190} // namespace blender::gpu::shader::parser
191
192namespace lexit {
193
194static inline std::string to_str(TokenType type)
195{
196 switch (type) {
197 case Word:
198 return "Word";
199 case Number:
200 return "Number";
201 case Default:
202 return "default";
203 case TemplateOpen:
204 return "<";
205 case TemplateClose:
206 return ">";
207 case NewLine:
208 return "NewLine";
209 case LogicalAnd:
210 return "&&";
211 case Break:
212 return "break";
213 case Const:
214 return "const";
215 case Constexpr:
216 return "constexpr";
217 case Do:
218 return "do";
219 case Decrement:
220 return "decrement";
221 case NotEqual:
222 return "!=";
223 case Equal:
224 return "==";
225 case LShift:
226 return "<<";
227 case RShift:
228 return ">>";
229 case For:
230 return "for";
231 case While:
232 return "while";
233 case LogicalOr:
234 return "||";
235 case GEqual:
236 return ">=";
237 case Switch:
238 return "switch";
239 case Case:
240 return "case";
241 case If:
242 return "if";
243 case Else:
244 return "else";
245 case Elif:
246 return "elif";
247 case Endif:
248 return "endif";
249 case Ifdef:
250 return "ifdef";
251 case Ifndef:
252 return "ifndef";
253 case Inline:
254 return "inline";
255 case LEqual:
256 return "<=";
257 case Static:
258 return "static";
259 case Enum:
260 return "enum";
261 case Namespace:
262 return "namespace";
263 case Define:
264 return "define";
265 case Union:
266 return "union";
267 case Continue:
268 return "continue";
269 case Line:
270 return "line";
271 case Increment:
272 return "++";
273 case Pragma:
274 return "pragma";
275 case DoubleHash:
276 return "##";
277 case Return:
278 return "return";
279 case Struct:
280 return "struct";
281 case Class:
282 return "class";
283 case Template:
284 return "template";
285 case Typename:
286 return "typename";
287 case This:
288 return "this";
289 case Using:
290 return "using";
291 case Undef:
292 return "undef";
293 case Private:
294 return "private";
295 case Public:
296 return "public";
297 case Invalid:
298 return "EOF";
299 default:
300 return std::string(1, char(type));
301 }
302}
303
304} // namespace lexit
size_t line_number(const std::string_view &str, size_t pos)
std::string filename(const std::string_view &str, size_t pos)
size_t char_number(const std::string_view &str, size_t pos)
std::string line_str(const std::string_view &str, size_t pos)
__int64 int64_t
Definition BKE_text.h:90
BMesh const char void * data
static std::string to_str(TokenType type)
Definition token.hh:194
TokenType
Definition types.hh:41
@ RShift
Definition types.hh:126
@ Default
Definition types.hh:129
@ While
Definition types.hh:92
@ Private
Definition types.hh:123
@ For
Definition types.hh:91
@ LogicalOr
Definition types.hh:93
@ Return
Definition types.hh:115
@ Do
Definition types.hh:87
@ Static
Definition types.hh:105
@ Enum
Definition types.hh:106
@ Struct
Definition types.hh:117
@ Union
Definition types.hh:109
@ LogicalAnd
Definition types.hh:81
@ Switch
Definition types.hh:95
@ Case
Definition types.hh:96
@ Else
Definition types.hh:98
@ LEqual
Definition types.hh:104
@ Namespace
Definition types.hh:107
@ TemplateOpen
Definition types.hh:130
@ Break
Definition types.hh:83
@ Pragma
Definition types.hh:113
@ Continue
Definition types.hh:110
@ TemplateClose
Definition types.hh:131
@ Using
Definition types.hh:121
@ Decrement
Definition types.hh:88
@ Increment
Definition types.hh:112
@ Constexpr
Definition types.hh:86
@ NotEqual
Definition types.hh:89
@ If
Definition types.hh:97
@ Endif
Definition types.hh:100
@ Const
Definition types.hh:85
@ NewLine
Definition types.hh:46
@ Ifndef
Definition types.hh:102
@ Number
Definition types.hh:44
@ Equal
Definition types.hh:90
@ Define
Definition types.hh:108
@ Elif
Definition types.hh:99
@ LShift
Definition types.hh:125
@ Inline
Definition types.hh:103
@ Ifdef
Definition types.hh:101
@ Line
Definition types.hh:111
@ Word
Definition types.hh:43
@ This
Definition types.hh:120
@ Typename
Definition types.hh:84
@ Invalid
Definition types.hh:42
@ Public
Definition types.hh:124
@ GEqual
Definition types.hh:94
@ Undef
Definition types.hh:122
@ Template
Definition types.hh:119
@ DoubleHash
Definition types.hh:114
@ Class
Definition types.hh:118
std::string line_str() const
Definition token.hh:154
bool operator==(TokenType type) const
Definition token.hh:172
size_t str_index_last_no_whitespace() const
Definition token.hh:87
bool operator!=(TokenType type) const
Definition token.hh:176
bool operator!=(char type) const
Definition token.hh:184
std::string filename() const
Definition token.hh:145
Token next(int i=1) const
Definition token.hh:33
bool operator==(char type) const
Definition token.hh:180
Token(const ParserBase &data)
Definition token.hh:27
std::string_view str_with_whitespace() const
Definition token.hh:106
std::string full_symbol_name() const
Definition token.hh:67
Token find_next(TokenType type) const
Definition token.hh:38
Token(const lexit::Token &tok)
Definition token.hh:23
std::string_view str() const
Definition token.hh:114
Token(const ParserBase &data, int64_t index)
Definition token.hh:25
size_t line_number(bool at_end=false) const
Definition token.hh:124
Token prev(int i=1) const
Definition token.hh:29
std::string_view str_
Definition lexit.hh:123
AlignedArrayPtr< uint32_t > offsets_
Definition lexit.hh:127
AlignedArrayPtr< TokenType > types_
Definition lexit.hh:125
AlignedArrayPtr< uint32_t > offsets_end_
Definition lexit.hh:129
bool is_invalid() const
Definition lexit.hh:69
const TokenBuffer * buf_
Definition lexit.hh:50
Token prev(int i=1) const
Definition lexit.hh:376
int32_t index_
Definition lexit.hh:51
bool is_valid() const
Definition lexit.hh:65
Token next(int i=1) const
Definition lexit.hh:372