Blender V5.3
processor.hh
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2024 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
8
9#pragma once
10
11#include "intermediate.hh"
12#include "metadata.hh"
13
14#include "bsl/symbol_table.hh"
15
16namespace blender::gpu::shader {
17
18static inline Language language_from_filename(const std::string &filename)
19{
20 if (filename.ends_with(".msl")) {
21 return Language::MSL;
22 }
23 if (filename.ends_with(".glsl")) {
24 return Language::GLSL;
25 }
26 if (filename.ends_with(".bsl.hh")) {
27 return Language::BSL;
28 }
29 if (filename.ends_with("_infos.hh")) {
30 return Language::INFO;
31 }
32 if (filename.ends_with(".hh")) {
33 return Language::CPP;
34 }
35 return Language::UNKNOWN;
36}
37
43 public:
48 using Tokens = std::vector<parser::Token>;
49
50 /* Cannot use `__` because of some compilers complaining about reserved symbols. */
51 static constexpr const char *namespace_separator = "_";
52 /* Add a prefix to all member functions so that they are not clashing with local variables. */
53 static constexpr const char *method_call_prefix = "_";
54 static constexpr const char *linted_struct_suffix = "_host_shared_";
55 static constexpr const char *uniform_struct_suffix = "uniform_";
56
57 private:
58 const std::string source_;
59 const std::string filepath_;
60 const std::string filename;
61 metadata::Source metadata_;
62 /* List of file that can be used in include directives. */
63 const std::vector<std::string> file_list_;
64
65 Language language_;
66
67 parser::ErrorHandler error_handler = {
68 .default_filename = filepath_.substr(filepath_.find_last_of('/') + 1)};
69
70 void report_error(parser::ast::Node node, const std::string &message)
71 {
72 error_handler.report(node.front(), message);
73 }
74
75 void report_error(Token tok, const std::string &message)
76 {
77 error_handler.report(tok, message);
78 }
79
80 void report_error(int row, int column, const std::string &line, const std::string &message)
81 {
82 error_handler.report(row, column, line, message);
83 }
84
85 struct SourceManager {
86 private:
87 std::vector<std::unique_ptr<Parser>> parsers_;
88
89 int include_id_ = 0;
90
91 public:
92 Parser &new_source(ErrorHandler &error_handler)
93 {
94 return *parsers_.emplace_back(std::make_unique<Parser>(error_handler));
95 }
96
97 /* To be called after all own include of a file have been processed. */
98 int include_id_get()
99 {
100 return include_id_++;
101 }
102 };
103
104 public:
105 SourceProcessor(const std::string &source,
106 const std::string &filepath,
107 Language language,
108 const std::vector<std::string> &file_list)
109 : source_(source),
110 filepath_(filepath),
111 filename(filepath_.substr(filepath_.find_last_of('/') + 1)),
112 file_list_(file_list),
113 language_(language)
114 {
115 }
116
117 struct Result {
118 /* Resulting Intermediate Language source. */
119 std::string source;
120 /* Parsed metadata. */
122
123 std::optional<parser::ErrorHandler::Error> error;
124 };
125
126 /* Convert to intermediate language. Also outputs metadata.
127 * symbols_set is the set of namespace symbols from external files / dependencies. */
128 Result convert(metadata::Source external_sources_symbols = {});
129
130 /* Lightweight parsing. Only Source::dependencies and Source::symbol_table are populated. */
133 bsl::SymbolTable &symbols,
134 std::vector<std::string> &visited_files);
135
136 /* Return the input string with comments removed. */
137 std::string remove_comments()
138 {
139 return remove_comments(source_);
140 }
141
142 /* String hash are outputted inside GLSL and needs to fit 32 bits. */
143 static uint32_t hash_string(const std::string &str)
144 {
145 uint64_t hash_64 = metadata::hash(str);
146 uint32_t hash_32 = uint32_t(hash_64 ^ (hash_64 >> 32));
147 return hash_32;
148 }
149
150 private:
151 Result convert_bsl();
152 Result convert_info();
153 Result convert_glsl();
154 Result convert_msl();
155 Result convert_bsl_legacy(metadata::Source external_sources_symbols);
156
157 /* --- Cleanup --- */
158
160 std::string remove_comments(const std::string &str);
162 /* Lower preprocessor directives containing `GPU_SHADER`.
163 * Avoid processing code that is not destined to be shader code and could contain unsupported
164 * syntax. */
165 std::string disabled_code_mutation(const std::string &str);
166 void disabled_code_mutation(Parser &parser, bool new_bsl_compiler = false);
167 /* Remove trailing white spaces. */
168 template<typename ParserT> void cleanup_whitespace(ParserT &parser, bool do_leading = false);
169 /* Successive mutations can introduce a lot of unneeded line directives. */
170 void cleanup_line_directives(Parser &parser);
171 /* Successive mutations can introduce a lot of unneeded blank lines. */
172 void cleanup_empty_lines(Parser &parser);
173
174 /* --- Parsing --- */
175
176 /* Parse defines in order to output them with the create infos.
177 * This allow the create infos to use shared defines values. */
178 void parse_defines(Parser &parser);
179 /* Populates metadata::symbol_table by scanning all namespaces.
180 * Does not parse global symbols. */
181 void parse_local_symbols(Parser &parser);
182 /* Legacy create info parsing and removing. */
183 void parse_legacy_create_info(Parser &parser);
184 /* Populates metadata::dependencies by scanning include directives. */
185 void parse_includes(Parser &parser);
186 /* Parse special pragma. */
187 void parse_pragma_runtime_generated(Parser &parser);
189 void parse_library_functions(Parser &parser);
190 void parse_library_functions_ast(Parser &parser);
191 /* Populate metadata::builtins by scanning source for keywords. Can trigger false positive.
192 * This is mostly legacy path as most builtin should be explicitly defined inside the BSL entry
193 * points. */
194 void parse_builtins(const std::string &str, const std::string &filename, bool pure_glsl = false);
195 void parse_builtins(Parser &parser, const std::string &filename);
196 void parse_draw_debug(Parser &parser, const std::string &filename);
197
198 /* Legacy shared variable support. */
199 std::string threadgroup_variables_parse_and_remove(const std::string &str);
200 void threadgroup_variables_parse_and_remove(Parser &parser);
201
202 void scan_external_symbols(SourceManager &sources,
203 bsl::SymbolTable &symbols,
204 std::vector<std::string> &visited_files);
205
206 /* --- Linting --- */
207
208 /* Make sure `if`, `else`, `for` statements are followed by braces. */
209 void lint_unbraced_statements(Parser &parser);
210 /* Lint for BSL reserved tokens. */
211 void lint_reserved_tokens(Parser &parser);
212 /* Lint for valid BSL attributes. */
213 void lint_attributes(Parser &parser);
214 void lint_attributes_ast(Parser &parser);
215 /* Assume formatted source with our code style. Cannot be applied to python shaders. */
216 void lint_global_scope_constants(Parser &parser);
217 void lint_global_scope_constants_ast(Parser &parser);
218 /* Search for constructor definition in active code. These are not supported. */
219 void lint_constructors(Parser &parser);
220 /* Forward declaration of types are not supported and makes no sense in a shader program where
221 * there is no pointers. */
222 void lint_forward_declared_structs(Parser &parser);
223
224 /* --- Lowering --- */
225
226 /* Remove `maybe_unused` attribute. */
227 void lower_noop_attributes(Parser &parser);
228 /* Lower parameters that have no name (invalid in GLSL). */
229 void lower_namesless_parameters(Parser &parser);
230 void lower_namesless_parameters_ast(Parser &parser);
235 void lower_template_dependent_names(Parser &parser);
236 /* Lower template definition and instantiation by doing simple copy paste + argument
237 * substitution. */
238 void lower_templates(Parser &parser);
239 void lower_template_calls(Parser &parser);
240 void lower_template_calls_ast(Parser &parser);
241 void lower_template_specialization(Parser &parser);
242 /* Ensures pragma once is present in headers to comply to our include semantic. */
243 void lint_pragma_once(Parser &parser, const std::string &filename);
244 /* Unroll loops by copy pasting content. */
245 void lower_loop_unroll(Parser &parser);
246 /* Convert if statements marked as static to preprocessor #if statements. */
247 void lower_static_branch(Parser &parser);
248 /* Lower namespaces by adding namespace prefix to all the contained structs and functions. */
249 void lower_namespaces(Parser &parser);
250 void lower_bsl_to_il(Parser &parser, bsl::SymbolTable &symbols);
267 void lower_using(Parser &parser);
268 /* Example: `A::B` --> `A_B` */
269 void lower_scope_resolution_operators(Parser &parser);
270 /* Remove preprocessor directives unsupported by target shading languages.
271 * Examples `#includes`, `#pragma once`. */
272 void lower_preprocessor(Parser &parser);
273 void lower_preprocessor_ast(Parser &parser);
274 /* Support for BLI swizzle syntax.
275 * Examples `a.xy()` --> `a.xy`. */
276 void lower_swizzle_methods(Parser &parser);
277 void lower_swizzle_methods_ast(Parser &parser);
278 /* Support for binary literals.
279 * Examples `0b1001` --> `0x9`. */
280 void lower_binary_literals(Parser &parser);
281 /* Change printf calls to "recursive" call to implementation functions.
282 * This allows to emulate the variadic arguments of printf. */
283 void lower_printf(Parser &parser);
284 /* Turn assert into a printf. */
285 void lower_assert(Parser &parser, const std::string &filename);
286 void lower_assert_ast(Parser &parser, const std::string &filename);
287 /* Parse SRT and interfaces, remove their attributes and create init function for SRT structs. */
288 void lower_resource_table(Parser &parser);
289 /* Examples `string_t s = "a" "b"` --> `string_t s = "ab"`. */
290 void lower_strings_sequences(Parser &parser);
291 void lower_strings_sequences_ast(Parser &parser);
292 /* Replace string literals by their hash and store the original string in the file metadata. */
293 void lower_strings(Parser &parser);
294 /* `class` -> `struct` */
295 void lower_classes(Parser &parser);
296 /* Create default initializer (empty brace) for all classes. */
297 void lower_default_constructors(Parser &parser);
298 /* Make all members of a class to be referenced using `this->`. */
299 void lower_implicit_member(Parser &parser);
300 /* Move all method definition outside of struct definition blocks. */
301 void lower_method_definitions(Parser &parser);
302 void lower_this_keyword(Parser &parser);
303 /* Add padding member to empty structs. */
304 void lower_empty_struct(Parser &parser);
305 /* Transform `a.fn(b)` into `fn(a, b)`. */
306 void lower_method_calls(Parser &parser, bool with_prefix = true);
307 void lower_constructors(Parser &parser);
308 /* Transform `auto [a, b] = fn()` into `S _tmp = fn(); a = _tmp.A; b = _tmp.B;`. */
309 void lower_structured_bindings(Parser &parser);
310 /* Parse, convert to create infos, and erase declaration. */
311 void lower_pipeline_definition(Parser &parser, const std::string &filename);
312 /* Remove `[vertex|fragment|compute]` function attribute and add appropriate guards. */
313 void lower_stage_function(Parser &parser);
314 /* Add #ifdef directive around functions using SRT arguments.
315 * Need to run after `lower_entry_points_signature`. */
316 void lower_srt_arguments(Parser &parser);
317 /* Add ifdefs guards around scopes using resource accessors. */
318 void lower_resource_access_functions(Parser &parser);
319 /* Lower enums to constants. */
320 void lower_enums(Parser &parser);
321 void lower_enums_ast(Parser &parser);
322 /* Merge attribute scopes. They are equivalent in the C++ standard.
323 * This allow to simplify parsing later on.
324 * `[[a]] [[b]]` > `[[a, b]]` */
325 void lower_attribute_sequences(Parser &parser);
326 void lower_attribute_sequences_ast(Parser &parser);
327 /* Lint host shared structure for padding and alignment.
328 * Remove the [[host_shared]] attribute. */
329 void lower_host_shared_structures(Parser &parser);
330 /* Remove noop keywords that makes subsequent lowering passes more complicated. */
331 void lower_noop_keywords(Parser &parser);
332 void lower_noop_keywords_ast(Parser &parser);
333 /* Example: `int a[] = {1,2,};` --> `int a[] = {1,2 };` */
334 void lower_trailing_comma_in_list(Parser &parser);
335 void lower_trailing_comma_in_list_ast(Parser &parser);
336 /* Allow easier parsing of struct member declaration.
337 * Example: `int a, b;` --> `int a; int b;` */
338 void lower_comma_separated_declarations(Parser &parser);
339 /* Example: `return {1, 2};` --> `T tmp = T{1, 2}; return tmp;`. */
340 void lower_implicit_return_types(Parser &parser);
341 void lower_implicit_return_types_ast(Parser &parser);
342 /* Example: `int a{1};` --> `int a = int{1};`. */
343 void lower_initializer_implicit_types(Parser &parser);
344 void lower_initializer_implicit_types_ast(Parser &parser);
345 /* Example: `T a{.a=1};` --> `T a; a.a=1;`. */
346 void lower_designated_initializers(Parser &parser);
347 /* Support for **full** aggregate initialization.
348 * They are converted to default constructor for GLSL. */
349 void lower_aggregate_initializers(Parser &parser);
350 void lower_aggregate_initializers_ast(Parser &parser);
351 /* Auto detect array length, and lower to GLSL compatible syntax.
352 * TODO(fclem): GLSL 4.3 already supports initializer list. So port the old GLSL syntax to
353 * initializer list instead. */
354 void lower_array_initializations(Parser &parser);
359 void lower_function_default_arguments(Parser &parser);
365 void lower_method_forward_declaration(Parser &parser);
366 /* Limited union implementation. Create getters and setters to a raw data struct. */
367 void lower_unions(Parser &parser);
375 void lower_union_accessor_templates(Parser &parser);
376 void lower_union_accessor_templates_ast(Parser &parser);
377 void lower_union_setters(Parser &parser);
383 void lower_bitfield_setters(Parser &parser);
393 void lower_srt_accessor_templates(Parser &parser);
394 void lower_srt_accessor_templates_ast(Parser &parser);
395 /* Add `srt_access` around all member access of SRT variables.
396 * Need to run before local reference mutations. */
397 void lower_srt_member_access(Parser &parser);
398 /* Parse entry point definitions and mutating all parameter usage to global resources. */
399 void lower_entry_points(Parser &parser);
400 /* Removes entry point arguments to make it compatible with the legacy code.
401 * Has to run after mutation related to function arguments. */
402 void lower_entry_points_signature(Parser &parser);
403 /* To be run after `lower_reference_arguments()`. */
404 void lower_reference_variables(Parser &parser);
405 /* To be run before `argument_decorator_macro_injection()`. */
406 void lower_reference_arguments(Parser &parser);
407 /* Example: `out float var[2]` > `_ref(float, var)[2]` */
408 void lower_argument_qualifiers(Parser &parser);
409 /* Example: `textureGather(t,c,1)` > `textureGather1(t,c)` */
410 void lower_gather_component(Parser &parser);
411 /* Lower test expect clauses to SSBO assignments. */
412 void lower_tests(Parser &parser, const std::string &prefix = "");
413 /* Lower resource pragmas to macros (breaks BSL parsing). */
414 void lower_resource_macro_placeholder_ast(Parser &parser);
415
416 /* --- Legacy passes for GLSL --- */
417
418 /* Example: `out float var[2]` > `out float _out_sta var _out_end[2]` */
419 std::string argument_decorator_macro_injection(const std::string &str);
420 /* Example: `= float[2](0.0, 0.0)` > `= ARRAY_T(float) ARRAY_V(0.0, 0.0)` */
421 std::string array_constructor_macro_injection(const std::string &str);
422 /* Used to make GLSL matrix constructor compatible with MSL in pyGPU shaders.
423 * This syntax is not supported in blender's own shaders. */
424 std::string matrix_constructor_mutation(const std::string &str);
425
426 /* --- Utilities --- */
427
428 /* Parse subscript scope with single integer literal and return the literal value.
429 * Return the fallback value in any case of non-literal value, or failed conversion. */
430 int static_array_size(const Scope &array, int fallback_value);
431
432 /* Process struct declaration and instantiate it in this file. */
433 void process_template_struct(metadata::TemplateDefinition &template_def,
435 /* Process templated function (or class method) declaration and instantiate it in this file. */
436 void process_template_function(metadata::TemplateDefinition &template_def,
438 /* If method, the end token of the template inside the struct. */
439 const Token method_end);
440
441 void lower_pre_template(Parser &parser);
442
443 void lower_template_instantiation(
444 Parser &parser,
445 /* If method, the end token of the template inside the struct. */
446 const Token method_end,
447 const Token &inst_start,
448 const Scope &inst_args,
449 const metadata::TemplateDefinition template_def,
450 const Token &symbol_name,
451 const std::vector<std::string> &arg_list,
452 const std::string &fn_decl,
453 const bool all_template_args_in_function_signature);
454
456 Token template_tok,
457 bool is_method,
458 Scope ns_scope,
459 const std::string &filepath);
460
461 void parse_namespace_symbols(SourceProcessor::Parser &parser,
462 Scope ns,
464 const std::string &filepath);
465
466 std::string template_full_specified_name(metadata::TemplateDefinition &template_def);
467
468 public:
469 /* Check for existence of preprocessor pragma in file. */
470 static bool has_pragma(Parser &parser, std::string_view pragma_str);
471
473 static std::string strip_whitespace(const std::string &str);
474
475 /* Example: `<float, 1>` > `TfloatT1` */
476 static std::string template_arguments_mangle(const Scope template_args);
477
478 /* Create placeholder for GLSL declarations generated by the GPU backends (VK/GL). */
479 static std::string get_create_info_placeholder(const std::string &name);
480
481 /* Make a scope only active based on the given condition using `#if` preprocessor directives.
482 * Processor contained return statements by returning 0 if scope is disabled.
483 * fn_type can be invalid token if scope is not a function scope. */
485 Scope scope,
486 const std::string &condition,
487 Token fn_type);
488
489 /* Return `#line 1 filename\n`. */
490 static std::string line_directive_prefix(const std::string &filename);
491};
492
493} // namespace blender::gpu::shader
static std::string strip_whitespace(const std::string &str)
static bool has_pragma(Parser &parser, std::string_view pragma_str)
Definition processor.cc:831
parser::IntermediateForm< parser::FullLexer, parser::FullParser > Parser
Definition processor.hh:45
parser::ErrorHandler ErrorHandler
Definition processor.hh:44
SourceProcessor(const std::string &source, const std::string &filepath, Language language, const std::vector< std::string > &file_list)
Definition processor.hh:105
metadata::Source parse_include_and_symbols()
Definition processor.cc:416
static uint32_t hash_string(const std::string &str)
Definition processor.hh:143
static constexpr const char * namespace_separator
Definition processor.hh:51
std::vector< parser::Token > Tokens
Definition processor.hh:48
static constexpr const char * linted_struct_suffix
Definition processor.hh:54
static std::string get_create_info_placeholder(const std::string &name)
Definition processor.cc:660
metadata::Source parse_include_and_symbols(SourceManager &sources, bsl::SymbolTable &symbols, std::vector< std::string > &visited_files)
static std::string line_directive_prefix(const std::string &filename)
static constexpr const char * method_call_prefix
Definition processor.hh:53
static constexpr const char * uniform_struct_suffix
Definition processor.hh:55
Result convert(metadata::Source external_sources_symbols={})
Definition processor.cc:387
static void guarded_scope_mutation(Parser &parser, Scope scope, const std::string &condition, Token fn_type)
static std::string template_arguments_mangle(const Scope template_args)
Definition template.cc:20
#define str(s)
#define condition(cond)
std::string filename(const std::string_view &str, size_t pos)
static Language language_from_filename(const std::string &filename)
Definition processor.hh:18
unsigned __int64 uint64_t
Definition BKE_text.h:91
unsigned int uint32_t
Definition BKE_text.h:81
std::optional< parser::ErrorHandler::Error > error
Definition processor.hh:123
void report(Token tok, std::string_view message)