Blender V5.3
intermediate.hh
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2025 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
29
30#pragma once
31
32#include "scope.hh"
33#include "token.hh"
34#include "token_stream.hh"
35#include "utils.hh"
36
37#include <algorithm>
38#include <cassert>
39#include <iostream>
40
42
44 std::string str_;
45
46 struct Mutation {
47 /* Range of the original string to replace. */
49 /* The replacement string. */
50 std::string replacement;
51
58
59 /* Define operator in order to sort the mutation by starting position.
60 * Otherwise, applying them in one pass will not work. */
61 friend bool operator<(const Mutation &a, const Mutation &b)
62 {
63 return a.src_range.start < b.src_range.start;
64 }
65 };
66 std::vector<Mutation> mutations_;
67
68 MutableString(const std::string_view input) : str_(input) {}
69
70 /* Disable copy construction and assignment. */
71 MutableString(const MutableString &other) = delete;
72 MutableString &operator=(const MutableString &other) = delete;
73 /* Explicitly enable default construction, move construction and move assignment. */
74 MutableString(MutableString &&other) = default;
76
77 /* Access internal string without applying pending mutations. */
78 std::string substr_range_inclusive(size_t start, size_t end)
79 {
80 return str_.substr(start, end - start + 1);
81 }
82 /* Access internal string without applying pending mutations. */
83 std::string substr_range_inclusive(Token start, Token end)
84 {
85 return substr_range_inclusive(start.str_index_start(), end.str_index_last());
86 }
87
88 /* Access internal string without applying pending mutations. */
89 std::string_view substr_range_inclusive_view(size_t start, size_t end)
90 {
91 return std::string_view(str_).substr(start, end - start + 1);
92 }
93 /* Access internal string without applying pending mutations. */
94 std::string_view substr_range_inclusive_view(Token start, Token end)
95 {
96 return substr_range_inclusive_view(start.str_index_start(), end.str_index_last());
97 }
98
100
102 {
103 std::cerr << last_substitution_failure << std::endl;
104 return false;
105 }
106
107 /* Replace everything from `from` to `to` (inclusive).
108 * Return true on success. */
109 bool replace_try(size_t from, size_t to, const std::string &replacement)
110 {
111 IndexRange range = IndexRange(from, to + 1 - from);
112 for (const Mutation &mut : mutations_) {
113 if (mut.src_range.overlaps(range)) {
114#ifndef NDEBUG
115 last_substitution_failure = "Trying to substitute \n\"" +
116 str_.substr(range.start, range.size) + "\"\nwith \n\"" +
117 replacement + "\"\nbut it overlaps with substituting \n\"" +
118 str_.substr(mut.src_range.start, mut.src_range.size) +
119 "\"\nwith \n\"" + mut.replacement + "\"";
120#endif
121 return false;
122 }
123 }
124 mutations_.emplace_back(range, replacement);
125 return true;
126 }
127 /* Replace everything from `from` to `to` (inclusive).
128 * Return true on success. */
130 Token to,
131 const std::string &replacement,
132 bool keep_trailing_whitespaces = false)
133 {
134 if (keep_trailing_whitespaces) {
135 return replace_try(from.str_index_start(), to.str_index_last_no_whitespace(), replacement);
136 }
137 return replace_try(from.str_index_start(), to.str_index_last(), replacement);
138 }
139
140 /* Replace everything from `from` to `to` (inclusive). */
141 void replace(size_t from, size_t to, const std::string &replacement)
142 {
143#ifndef NDEBUG
144 bool success = replace_try(from, to, replacement);
145 assert(success || report_failure());
146 (void)success;
147#else
148 /* No check in release. */
149 IndexRange range = IndexRange(from, to + 1 - from);
150 mutations_.emplace_back(range, replacement);
151#endif
152 }
153 /* Replace everything from `from` to `to` (inclusive). */
154 void replace(Token from,
155 Token to,
156 const std::string &replacement,
157 bool keep_trailing_whitespaces = false)
158 {
159 if (keep_trailing_whitespaces) {
160 replace(from.str_index_start(), to.str_index_last_no_whitespace(), replacement);
161 }
162 else {
163 replace(from.str_index_start(), to.str_index_last(), replacement);
164 }
165 }
166
167 /* Replace token by string. */
168 void replace(Token tok, const std::string &replacement, bool keep_trailing_whitespaces = false)
169 {
170 if (keep_trailing_whitespaces) {
171 replace(tok.str_index_start(), tok.str_index_last_no_whitespace(), replacement);
172 }
173 else {
174 replace(tok.str_index_start(), tok.str_index_last(), replacement);
175 }
176 }
177 /* Replace Scope by string. */
178 void replace(Scope scope, const std::string &replacement, bool keep_trailing_whitespaces = false)
179 {
180 if (keep_trailing_whitespaces) {
181 replace(scope.front().str_index_start(),
182 scope.back().str_index_last_no_whitespace(),
183 replacement);
184 }
185 else {
186 replace(scope.front(), scope.back(), replacement);
187 }
188 }
189 /* Replace Scope by string. */
191 const std::string &replacement,
192 bool keep_trailing_whitespaces = false)
193 {
194 if (keep_trailing_whitespaces) {
195 replace(Token(node.front()).str_index_start(),
196 Token(node.back()).str_index_last_no_whitespace(),
197 replacement);
198 }
199 else {
200 replace(node.front(), node.back(), replacement);
201 }
202 }
203
204 /* Replace the content from `from` to `to` (inclusive) by whitespaces without changing
205 * line count and keep the remaining indentation spaces. */
206 bool erase_try(size_t from, size_t to)
207 {
208 IndexRange range = IndexRange(from, to + 1 - from);
209 std::string content = str_.substr(range.start, range.size);
210 size_t lines = std::count(content.begin(), content.end(), '\n');
211 size_t spaces = content.find_last_of("\n");
212 if (spaces != std::string::npos) {
213 spaces = content.length() - (spaces + 1);
214 }
215 else {
216 spaces = content.length();
217 }
218 return replace_try(from, to, std::string(lines, '\n') + std::string(spaces, ' '));
219 }
220 void erase(size_t from, size_t to)
221 {
222 bool result = erase_try(from, to);
224 (void)result;
225 }
226 /* Replace the content from `from` to `to` (inclusive) by whitespaces without changing
227 * line count and keep the remaining indentation spaces. */
228 bool erase_try(Token from, Token to)
229 {
230 if (from.is_invalid() || to.is_invalid()) {
231 return true;
232 }
233 assert(from.index_ <= to.index_);
234 return erase_try(from.str_index_start(), to.str_index_last());
235 }
236 void erase(Token from, Token to)
237 {
238 bool result = erase_try(from, to);
240 (void)result;
241 }
242 /* Replace the content from `from` to `to` (inclusive) by whitespaces without changing
243 * line count and keep the remaining indentation spaces. */
244 bool erase_try(Token tok)
245 {
246 if (tok.is_invalid()) {
247 return true;
248 }
249 return erase_try(tok, tok);
250 }
251 void erase(Token tok)
252 {
253 bool result = erase_try(tok);
255 (void)result;
256 }
257 /* Replace the content of the scope by whitespaces without changing
258 * line count and keep the remaining indentation spaces. */
259 bool erase_try(Scope scope)
260 {
261 return erase_try(scope.front(), scope.back());
262 }
263 void erase(Scope scope)
264 {
265 bool result = erase_try(scope);
267 (void)result;
268 }
269 /* Replace the content of the scope by whitespaces without changing
270 * line count and keep the remaining indentation spaces. */
272 {
273 return erase_try(node.front(), node.back());
274 }
276 {
277 bool result = erase_try(node);
279 (void)result;
280 }
281
282 /* If prepend is true, will prepend the new content to the list of modifications.
283 * With this enabled, in case of overlapping mutation, the last one added will be first. */
284 void insert_before(size_t at, const std::string &content, bool prepend = false)
285 {
286 IndexRange range = IndexRange(at, 0);
287 if (prepend) {
288 mutations_.insert(mutations_.begin(), {range, content});
289 }
290 else {
291 mutations_.emplace_back(range, content);
292 }
293 }
294 void insert_before(Token at, const std::string &content, bool prepend = false)
295 {
296 insert_before(at.str_index_start(), content, prepend);
297 }
298
299 void insert_after(size_t at, const std::string &content)
300 {
301 IndexRange range = IndexRange(at + 1, 0);
302 mutations_.emplace_back(range, content);
303 }
304 void insert_after(Token at, const std::string &content)
305 {
306 insert_after(at.str_index_last(), content);
307 }
308
309 void insert_line_number(size_t at, int line, std::string_view filename = "")
310 {
311 std::string str = "\n#line " + std::to_string(line);
312 if (!filename.empty()) {
313 str = str + " \"" + std::string(filename) + "\"";
314 }
315 insert_after(at, str + "\n");
316 }
317 void insert_line_number(Token at, int line, std::string_view filename = "")
318 {
319 insert_line_number(at.str_index_last(), line, filename);
320 }
321
322 /* Insert a preprocessor directive after the given token.
323 * This also insert a line directive to keep correct error reporting. */
324 void insert_directive(Token at, const std::string directive)
325 {
326 insert_after(at, "\n" + directive + "\n");
327 insert_line_number(at, at.line_number(true));
328 size_t line_break = str_.find_last_of("\n", at.str_index_last() + 1);
329 size_t spaces = at.str_index_last() - line_break;
330 insert_after(at, std::string(spaces, ' '));
331 }
332
333 /* Return true if any mutation was applied.
334 * Update lexer string view if needed. */
335 bool apply_mutations(LexerBase &lexer, const bool all_mutation_ordered = false);
336
337 /* Get internal string. Does not apply pending mutation. */
338 const std::string &str()
339 {
340 return str_;
341 }
342
343 /* For testing. */
344 std::string serialize_mutations() const
345 {
346 std::string out;
347 for (const Mutation &mut : mutations_) {
348 out += "Replace ";
349 out += std::to_string(mut.src_range.start);
350 out += " - ";
351 out += std::to_string(mut.src_range.size);
352 out += " \"";
353 out += str_.substr(mut.src_range.start, mut.src_range.size);
354 out += "\" by \"";
355 out += mut.replacement;
356 out += "\"\n";
357 }
358 return out;
359 }
360};
361
362inline std::ostream &operator<<(std::ostream &out, const std::vector<int> &v)
363{
364 if (!v.empty()) {
365 out << '[';
366 for (auto val : v) {
367 out << val << ',';
368 }
369 out << "\b]";
370 }
371 return out;
372}
373
374class ParserException : public std::exception {};
375
376/* Structure holding an intermediate form of the source code.
377 * It is made for fast traversal and mutation of source code. */
378template<typename LexerFn, typename ParserFn>
379struct IntermediateForm : MutableString, Parser<LexerFn, ParserFn> {
380 protected:
382
383 public:
385
386 IntermediateForm(const std::string_view input, ErrorHandler &report_error)
388 {
390 }
391
396
397 void set_str(const std::string_view input)
398 {
399 str_ = input;
401 }
402
403 /* Main access operator. Returns the root scope (aka global scope). */
405 {
406 if (this->scope_types.empty()) {
407 return Scope(*this);
408 }
409 return Scope(*this, 0);
410 }
411
412 /* Return true if any mutation was applied. */
413 bool only_apply_mutations(const bool all_mutation_ordered = false)
414 {
415 return static_cast<MutableString *>(this)->apply_mutations(*this, all_mutation_ordered);
416 }
417
418 /* Apply pending mutation and parse the resulting string.
419 * Return true if any mutation was applied. */
420 bool apply_mutations(const bool all_mutation_ordered = false)
421 {
422 bool applied = only_apply_mutations(all_mutation_ordered);
423 if (applied) {
425 }
426 return applied;
427 }
428
429 /* Apply mutations if any and get resulting string. */
430 const std::string &result_get(const bool all_mutation_ordered = false)
431 {
432 only_apply_mutations(all_mutation_ordered);
433 return str_;
434 }
435
437 {
438 this->lexical_analysis(str_);
440 switch (language) {
441 case Language::BSL:
442 this->parse_bsl(report_error);
443 break;
444 default:
445 break;
446 }
447 if (report_error.err.has_value()) {
448 throw ParserException();
449 }
450 }
451
453 {
454 std::cout << "Input: \n" << str_ << " \nEnd of Input\n" << std::endl;
455 std::cout << "Token Types: \"" << this->token_types_str() << "\"" << std::endl;
456 std::cout << "Token scopes: \"" << this->token_scope << "\"" << std::endl;
457 std::cout << "Scope Types: \"" << this->scope_types_str << "\"" << std::endl;
458 }
459
461 {
462 for (auto tok : *this) {
463 std::cout << "id:" << int(tok) << " start:" << this->offsets_[int(tok)]
464 << " end:" << this->offsets_end_[int(tok)] << " type:" << tok.type()
465 << " scope:" << this->token_scope[int(tok)] << "("
466 << this->scope_types_str[this->token_scope[int(tok)]] << ")"
467 << " atom:" << tok.atom() << " str:\"" << tok.str() << "\""
468 << " followed_by_whitespace:" << tok.followed_by_whitespace() << "\n";
469 }
470 }
471};
472
473} // namespace blender::gpu::shader::parser
constexpr IndexRange()=default
std::string filename(const std::string_view &str, size_t pos)
std::ostream & operator<<(std::ostream &out, const std::vector< int > &v)
ATTR_WARN_UNUSED_RESULT const BMVert * v
IntermediateForm(const std::string_view input, ErrorHandler &report_error)
bool only_apply_mutations(const bool all_mutation_ordered=false)
const std::string & result_get(const bool all_mutation_ordered=false)
void set_str(const std::string_view input)
std::string_view token_types_str() const
Mutation(IndexRange src_range, std::string replacement)
friend bool operator<(const Mutation &a, const Mutation &b)
void insert_before(size_t at, const std::string &content, bool prepend=false)
std::string substr_range_inclusive(Token start, Token end)
bool erase_try(size_t from, size_t to)
std::string_view substr_range_inclusive_view(Token start, Token end)
void insert_directive(Token at, const std::string directive)
MutableString(MutableString &&other)=default
void insert_before(Token at, const std::string &content, bool prepend=false)
bool replace_try(Token from, Token to, const std::string &replacement, bool keep_trailing_whitespaces=false)
MutableString(const MutableString &other)=delete
void insert_line_number(size_t at, int line, std::string_view filename="")
void replace(size_t from, size_t to, const std::string &replacement)
void insert_after(size_t at, const std::string &content)
MutableString & operator=(const MutableString &other)=delete
void replace(Scope scope, const std::string &replacement, bool keep_trailing_whitespaces=false)
MutableString(const std::string_view input)
void replace(Token tok, const std::string &replacement, bool keep_trailing_whitespaces=false)
void insert_line_number(Token at, int line, std::string_view filename="")
void replace(Token from, Token to, const std::string &replacement, bool keep_trailing_whitespaces=false)
bool apply_mutations(LexerBase &lexer, const bool all_mutation_ordered=false)
std::string_view substr_range_inclusive_view(size_t start, size_t end)
bool replace_try(size_t from, size_t to, const std::string &replacement)
MutableString & operator=(MutableString &&other)=default
void insert_after(Token at, const std::string &content)
void replace(ast::Node node, const std::string &replacement, bool keep_trailing_whitespaces=false)
std::string substr_range_inclusive(size_t start, size_t end)
void parse_bsl(ErrorHandler &err_handler)
void semantic_analysis(ErrorHandler &err_handler)
void lexical_analysis(std::string_view input)
AlignedArrayPtr< uint32_t > offsets_
Definition lexit.hh:127
AlignedArrayPtr< uint32_t > offsets_end_
Definition lexit.hh:129
bool is_invalid() const
Definition lexit.hh:69
int32_t index_
Definition lexit.hh:51
Token(const TokenBuffer *buf, int32_t index)
Definition lexit.hh:337