Blender V5.3
template.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2024 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
8
9#include <algorithm>
10#include <unordered_map>
11
12#include "intermediate.hh"
13#include "processor.hh"
14
15namespace blender::gpu::shader {
16using namespace std;
17using namespace shader::parser;
18using namespace metadata;
19
21{
22 string args_concat;
23 template_args.foreach_scope(ScopeType::TemplateArg, [&](const Scope &scope) {
24 string str;
25 if (scope[1] == '<') {
26 str = string(scope[0].str()) + template_arguments_mangle(scope[1].scope());
27 }
28 else {
29 str = scope.str();
30 }
31 /* In order to support negative integer literals. Replace minus sign by underscore. */
32 replace(str.begin(), str.end(), '-', '_');
33 args_concat += 'T' + str;
34 });
35 return args_concat;
36}
37
38string SourceProcessor::template_full_specified_name(metadata::TemplateDefinition &template_def)
39{
40 SourceProcessor::Parser name_parser(template_def.name_space + template_def.identifier,
41 error_handler);
42 lower_scope_resolution_operators(name_parser);
43 return name_parser.result_get();
44}
45
47 vector<string> &arg_list,
48 const Scope fn_args,
49 bool &all_template_args_in_function_signature,
50 ErrorHandler &error_handler)
51{
52 const Token type = arg.front();
53 const Token name = type.str() == "enum" ? type.next().next() : type.next();
54 const string_view name_str = name.str();
55 const string_view type_str = type.str();
56
57 arg_list.emplace_back(name_str);
58
59 if (type_str == "typename") {
60 bool found = false;
61 /* Search argument list for type-names. If type-name matches, the template argument is
62 * present inside the function signature. */
63 fn_args.foreach_match("AA", [&](const vector<Token> &tokens) {
64 if (tokens[0].str() == name_str) {
65 found = true;
66 }
67 });
68 fn_args.foreach_match("A&A", [&](const vector<Token> &tokens) {
69 if (tokens[0].str() == name_str) {
70 found = true;
71 }
72 });
73 all_template_args_in_function_signature &= found;
74 }
75 else if (type_str == "enum" || type_str == "bool") {
76 /* Values cannot be resolved using type deduction. */
77 all_template_args_in_function_signature = false;
78 }
79 else if (type_str == "int" || type_str == "uint" || type_str == "char" || type_str == "uchar" ||
80 type_str == "short" || type_str == "ushort")
81 {
82 /* Values cannot be resolved using type deduction. */
83 all_template_args_in_function_signature = false;
84 }
85 else {
86 error_handler.report(type, "Invalid template argument type");
87 }
88}
89
90void SourceProcessor::lower_template_instantiation(
92 /* If method, the end token of the template inside the struct. */
93 const Token method_end,
94 const Token &inst_start,
95 const Scope &inst_args,
96 const metadata::TemplateDefinition template_def,
97 const Token &symbol_name,
98 const vector<string> &arg_list,
99 const string &fn_decl,
100 const bool all_template_args_in_function_signature)
101{
102 /* Note that we do not use the full path as file identifier. So all names are unique. */
103 string instance_filename = filepath_.substr(filepath_.find_last_of('/') + 1);
104 string template_filename = template_def.filepath.substr(template_def.filepath.find_last_of('/') +
105 1);
106 /* Avoid adding noise in the source file if instance is inside the same file as declaration. */
107 if (instance_filename == template_filename) {
108 instance_filename = "";
109 template_filename = "";
110 }
111
112 /* Parse template values. */
113 vector<pair<string, string>> arg_name_value_pairs;
114 {
115 int arg_count = 0;
116 inst_args.foreach_scope(ScopeType::TemplateArg, [&](const Scope &arg) {
117 if (arg_count < arg_list.size()) {
118 arg_name_value_pairs.emplace_back(arg_list[arg_count], arg.str());
119 }
120 arg_count++;
121 });
122 if (arg_count != arg_list.size()) {
123 report_error(inst_args.front(), "Invalid amount of argument in template instantiation.");
124 return;
125 }
126 }
127
128 /* Specialize template content. */
129 string instance_content;
130 {
131 SourceProcessor::Parser instance_parser(fn_decl, error_handler);
132
133 /* Inject namespace around definition for symbols namespaces resolution. */
134 if (template_def.name_space.empty()) {
135 instance_parser.insert_before(instance_parser.front(), "\n");
136 instance_parser.insert_after(instance_parser.back(), "\n");
137 }
138 else {
139 /* Remove suffix "::". */
140 string ns_name(template_def.name_space.substr(0, template_def.name_space.size() - 2));
141
142 if (template_def.is_method) {
143 size_t split = ns_name.rfind("::");
144 string struct_name;
145
146 if (split == string::npos) {
147 struct_name = ns_name;
148 ns_name = "";
149 }
150 else {
151 struct_name = ns_name.substr(split + 2);
152 ns_name = ns_name.substr(0, split);
153 }
154 if (!ns_name.empty()) {
155 instance_parser.insert_before(instance_parser.front(), "namespace " + ns_name + " {");
156 }
157 instance_parser.insert_before(instance_parser.front(), "struct " + struct_name + " {\n");
158 instance_parser.insert_after(instance_parser.back(), "\n};");
159 if (!ns_name.empty()) {
160 instance_parser.insert_after(instance_parser.back(), "}");
161 }
162 instance_parser.insert_after(instance_parser.back(), "\n");
163 }
164 else {
165 instance_parser.insert_before(instance_parser.front(), "namespace " + ns_name + " {\n");
166 instance_parser.insert_after(instance_parser.back(), "\n}\n");
167 }
168 }
169 /* Insert line directive. Important for symbol namespace resolution and error logging.
170 * Not using insert_line_number because it uses insert_after. */
171 string line_str = "\n#line " + std::to_string(template_def.definition_line);
172 if (!template_filename.empty()) {
173 line_str += " \"" + template_filename + '\"';
174 }
175 instance_parser.insert_before(instance_parser.front(), line_str + "\n");
176
177 instance_parser().foreach_token(Word, [&](const Token &word) {
178 string_view token_str = word.str();
179
180 /* Replace each parameter appearance inside the instance. */
181 for (const auto &arg_name_value : arg_name_value_pairs) {
182 if (token_str == arg_name_value.first) {
183 instance_parser.replace(word, arg_name_value.second, true);
184 }
185 }
186 /* Append template args after unspecified struct typename references.
187 * `A func(A<T> b) {}` > `A<T> func(A<T> b) {}`. */
188 if (template_def.is_struct && word.next() != AngleOpen && token_str == symbol_name.str()) {
189 instance_parser.insert_after(word.str_index_last_no_whitespace(),
191 }
192 });
193
194 /* Position of symbol name declaration. */
195 size_t symbol_name_pos = instance_parser.str().find(" " + string(symbol_name.str()));
196
197 /* Append template args after function name if needed. This is required by BSL specification.
198 * `void func() {}` > `void func<a, 1>() {}`. */
199 if (!template_def.is_struct && !all_template_args_in_function_signature) {
200 instance_parser.insert_after(symbol_name_pos + symbol_name.str().size(),
202 }
203 /* Append namespace to symbol name because the appended mangled arguments (above) make
204 * namespace resolution impossible. Methods do not need it because they are instantiated inside
205 * their struct. They will get the correct namespace prefix (if they are static) later on. */
206 if (!template_def.is_method) {
207 instance_parser.insert_after(symbol_name_pos, string(template_def.name_space));
208 }
209
210 instance_parser.apply_mutations();
211
212 lower_pre_template(instance_parser);
213
214 /* Paste template content in place of instantiation. */
215 instance_content = instance_parser.result_get();
216 /* Remove added first line from the injected namespace. */
217 instance_content = instance_content.substr(instance_content.find_first_of('\n') + 1);
218 if (template_def.is_method) {
219 /* Remove added last line from the injected struct + namespace. */
220 instance_content = instance_content.substr(
221 0, instance_content.find_last_of('\n', instance_content.size() - 2) + 1);
222 }
223 }
224
225 const Token inst_end = inst_start.find_next(SemiColon);
226 /* Method are put back in their classes. */
227 const Token insert_at = template_def.is_method ? method_end : inst_end;
228
229 /* Insert instantiation content. Instance line directive was already added. */
230 parser.insert_after(insert_at, instance_content);
231 parser.insert_line_number(insert_at, insert_at.line_number(true), instance_filename);
232}
233
234void SourceProcessor::lower_template_dependent_names(Parser &parser)
235{
236 parser().foreach_match("tA<..>", [&](const Tokens &toks) {
237 if (toks[0].prev() == '.' || (toks[0].prev().prev() == '-' && toks[0].prev() == '>')) {
238 parser.erase(toks[0]);
239 }
240 });
241}
242
243void SourceProcessor::lower_pre_template(Parser &parser)
244{
245 /* Lower noop attributes after linting them. */
246 lower_noop_attributes(parser);
247 /* Lint and remove C++ accessor templates before lowering template. */
248 lower_srt_accessor_templates(parser);
249 lower_union_accessor_templates(parser);
250 /* Lower namespaces. */
251 lower_using(parser);
252 lower_namespaces(parser);
253 lower_scope_resolution_operators(parser);
254
255 lower_template_calls(parser);
256 lower_template_specialization(parser);
257}
258
259/* Mangle template parameter into the symbol name. */
260void SourceProcessor::lower_template_calls(Parser &parser)
261{
262 /* Process templated function calls first to avoid matching them later. */
263 parser().foreach_match("A<..>(..)", [&](const vector<Token> &tokens) {
264 const Scope template_args = tokens[1].scope();
265 template_args.foreach_match("A<..>", [&parser](const vector<Token> &tokens) {
266 parser.replace(tokens[1].scope(), template_arguments_mangle(tokens[1].scope()), true);
267 });
268 });
269 /* Likewise, process templated struct method definitions. */
270 parser().foreach_match("A<..>A<", [&](const vector<Token> &tokens) {
271 parser.replace(tokens[1].scope(), template_arguments_mangle(tokens[1].scope()), true);
272 });
273
274 parser.apply_mutations();
275}
276
277void SourceProcessor::lower_template_specialization(Parser &parser)
278{
279 auto process_specialization = [&](const Token specialization_start, const Scope template_args) {
280 parser.erase(specialization_start, specialization_start.next().next());
281 parser.replace(template_args, template_arguments_mangle(template_args), true);
282 };
283
284 parser().foreach_match("t<>AA<", [&](const vector<Token> &tokens) {
285 process_specialization(tokens[0], tokens[5].scope());
286 });
287
288 parser().foreach_match("t<>sA<..>", [&](const vector<Token> &tokens) {
289 process_specialization(tokens[0], tokens[5].scope());
290 });
291
292 parser.apply_mutations();
293}
294
295void SourceProcessor::process_template_struct(metadata::TemplateDefinition &template_def,
297{
298 struct DefinitionParser {
299 SourceProcessor::Parser def_parser;
300 const Scope template_scope = def_parser[1].scope();
301 const Token struct_start = template_scope.back().next();
302 const Token struct_name = struct_start.next();
303 const Scope struct_body = struct_name.next().scope();
304 const Token struct_end = struct_body.back().next();
305 const string struct_decl = def_parser.substr_range_inclusive(struct_start, struct_end);
306 bool all_template_args_in_function_signature = false;
307 vector<string> arg_list;
308
309 /* Parse template declaration. */
310 DefinitionParser(metadata::TemplateDefinition &template_def, ErrorHandler report_error)
311 : def_parser(template_def.definition, report_error)
312 {
313 template_scope.foreach_scope(ScopeType::TemplateArg, [&](Scope arg) {
315 arg_list,
316 Scope(def_parser),
317 all_template_args_in_function_signature,
318 report_error);
319 });
320 }
321 };
322
323 /* Only parse if there is an instantiation. */
324 unique_ptr<DefinitionParser> def_parser;
325 /* Since we already lowered the namespaces in main parser, we need to search for the namespace
326 * resolved symbol name. */
327 const string full_specified_name = template_full_specified_name(template_def);
328 /* Replace instantiations. */
329 parser().foreach_match("tsA<", [&](const vector<Token> &tokens) {
330 if (full_specified_name != tokens[2].str()) {
331 return;
332 }
333
334 if (!def_parser) {
335 def_parser = make_unique<DefinitionParser>(template_def, error_handler);
336 }
337
338 lower_template_instantiation(parser,
339 Token::invalid(&parser),
340 tokens[0],
341 tokens[3].scope(),
342 template_def,
343 def_parser->struct_name,
344 def_parser->arg_list,
345 def_parser->struct_decl,
346 def_parser->all_template_args_in_function_signature);
347 });
348}
349
350void SourceProcessor::process_template_function(
351 metadata::TemplateDefinition &template_def,
353 /* If method, the end token of the template inside the struct. */
354 const Token method_end)
355{
356 struct DefinitionParser {
357 SourceProcessor::Parser def_parser;
358 const Scope template_scope = def_parser[1].scope();
359 const Token fn_start = template_scope.back().next();
360 /* Skip attributes */
361 const Token after_attr = fn_start == SquareOpen ? fn_start.scope().back().next() : fn_start;
362 const Scope fn_args = after_attr.find_next(ParOpen).scope();
363 const Token fn_name = fn_args.front().prev();
364 const Token fn_end = fn_args.back().find_next(BracketOpen).scope().back();
365 const string fn_decl = def_parser.substr_range_inclusive(fn_start, fn_end);
366
367 bool all_template_args_in_function_signature = true;
368 vector<string> arg_list;
369
370 /* Parse template declaration. */
371 DefinitionParser(metadata::TemplateDefinition &template_def, ErrorHandler report_error)
372 : def_parser(template_def.definition, report_error)
373 {
374 assert(fn_start.is_valid() && fn_name.is_valid() && fn_args.is_valid() &&
375 template_scope.is_valid() && fn_end.is_valid());
376
377 template_scope.foreach_scope(ScopeType::TemplateArg, [&](Scope arg) {
379 arg, arg_list, fn_args, all_template_args_in_function_signature, report_error);
380 });
381 }
382 };
383
384 /* Only parse if there is an instantiation. */
385 unique_ptr<DefinitionParser> def_parser;
386 /* Since we already lowered the namespaces in main parser, we need to search for the namespace
387 * resolved symbol name. */
388 const string full_specified_name = template_full_specified_name(template_def);
389 /* Replace instantiations. */
390 parser().foreach_match("tAA<", [&](const vector<Token> &tokens) {
391 if (full_specified_name != tokens[2].str()) {
392 return;
393 }
394
395 if (!def_parser) {
396 def_parser = make_unique<DefinitionParser>(template_def, error_handler);
397 }
398
399 lower_template_instantiation(parser,
400 method_end,
401 tokens[0],
402 tokens[3].scope(),
403 template_def,
404 def_parser->fn_name,
405 def_parser->arg_list,
406 def_parser->fn_decl,
407 def_parser->all_template_args_in_function_signature);
408 });
409}
410
411void SourceProcessor::lower_templates(Parser &parser)
412{
413 /* Lint missing template arguments in instantiation and specialization.
414 * This is required by the BSL spec in order to simplify implementation. */
415 auto lint_explicit = [&](const Token symbol_name) {
416 if (symbol_name.next().scope().type() != parser::ScopeType::Template) {
417 report_error(
418 symbol_name,
419 "Template instantiation and specialization require explicit template arguments");
420 }
421 };
422 parser().foreach_match("t<>AA", [&](const vector<Token> &toks) { lint_explicit(toks[4]); });
423 parser().foreach_match("t<>A<..>A", [&](const vector<Token> &toks) { lint_explicit(toks[8]); });
424 parser().foreach_match("tAA", [&](const vector<Token> &toks) { lint_explicit(toks[2]); });
425 parser().foreach_match("tA<..>A", [&](const vector<Token> &toks) { lint_explicit(toks[6]); });
426
427 /* Delete templated struct and function definitions (not methods! see later).
428 * They were already parsed by `SourceProcessor::parse_namespace_symbols`. */
429 bool error = false;
430 parser().foreach_match("t<..>", [&](const vector<Token> &toks) {
431 /* Only process global templates, not methods. */
432 if (toks[0].scope() != parser()) {
433 return;
434 }
435 /* Default arguments are not supported. */
436 toks[1].scope().foreach_token(Assign, [&](Token tok) {
437 report_error(tok, "Default arguments are not supported inside template declaration");
438 error = true;
439 });
440
441 Token end = toks[0].find_next(BracketOpen).scope().back();
442 if (toks[4].next() == Struct) {
443 /* Capture end semicolon. */
444 end = end.next();
445 }
446 /* This can fail as it might try to erase templated method inside templated struct. */
447 parser.erase_try(toks[0], end);
448 });
449
450 parser.apply_mutations();
451
452 /* Deduplicate symbols (can happen because the main file is parsed twice). */
453 unordered_map<string, TemplateDefinition> unique_symbols;
454 for (const auto &symbol : metadata_.template_definitions) {
455 unique_symbols.try_emplace(symbol.name_space + symbol.identifier, symbol);
456 }
457
458 /* Process struct first, so methods can instantiate inside them. */
459 for (auto [_, template_def] : unique_symbols) {
460 if (template_def.is_struct) {
461 process_template_struct(template_def, parser);
462 }
463 }
464 parser.apply_mutations();
465
466 /* Then process methods. We can only process methods if their struct exists in the same file.
467 * This holds true for instantiated struct templates. */
468 parser().foreach_struct([&](Token, Scope, Token, Scope body) {
469 body.foreach_match("t<..>", [&](const vector<Token> &toks) {
470 /* Since this can be an instantiated struct, we need to make sure to instantiate its own
471 * methods. Hence the need to parse the definition. */
472 TemplateDefinition template_def = parse_template_definition(
473 parser, toks[0], true, toks[0].scope(), filepath_);
474 /* Insertion point of the instantiation. */
475 Token method_end = toks[0].find_next(BracketOpen).scope().back();
476
477 process_template_function(template_def, parser, method_end);
478 /* Delete definitions. */
479 parser.erase(toks[0], method_end);
480 });
481 });
482
483 /* For each template definition, process all instantiation. */
484 for (auto [_, template_def] : unique_symbols) {
485 if (!template_def.is_struct && !template_def.is_method) {
486 process_template_function(template_def, parser, Token::invalid(&parser));
487 }
488 }
489 parser.apply_mutations();
490
491 /* Remove template instantiation afterward. */
492 parser().foreach_token(Template, [&](const Token &tok) {
493 if (tok.next() == '<') {
494 report_error(tok, "Invalid template definition");
495 }
496 else {
497 parser.erase(tok, tok.find_next(SemiColon));
498 }
499 });
500
501 parser.apply_mutations();
502
503 /* Process calls to templated types or functions. */
504 parser().foreach_match("A<..>", [&](const vector<Token> &tokens) {
505 parser.replace(tokens[1].scope(), template_arguments_mangle(tokens[1].scope()), true);
506 });
507
508 parser.apply_mutations();
509}
510
511} // namespace blender::gpu::shader
parser::IntermediateForm< parser::FullLexer, parser::FullParser > Parser
Definition processor.hh:45
std::vector< parser::Token > Tokens
Definition processor.hh:48
static std::string template_arguments_mangle(const Scope template_args)
Definition template.cc:20
#define str(s)
#define assert(assertion)
std::string line_str(const std::string_view &str, size_t pos)
static void parse_template_definition_args(const Scope arg, vector< string > &arg_list, const Scope fn_args, bool &all_template_args_in_function_signature, ErrorHandler &error_handler)
Definition template.cc:46
void split(const std::string &s, const char delim, std::vector< std::string > &tokens)
Definition abc_util.cc:68
SymEdge< T > * prev(const SymEdge< T > *se)
void report_error(const StringRefNull error)
@ BracketOpen
Definition types.hh:55
@ Struct
Definition types.hh:117
@ SquareOpen
Definition types.hh:57
@ SemiColon
Definition types.hh:62
@ Assign
Definition types.hh:61
@ AngleOpen
Definition types.hh:59
@ Word
Definition types.hh:43
@ Template
Definition types.hh:119
void report(Token tok, std::string_view message)
void insert_line_number(size_t at, int line, std::string_view filename="")
void insert_after(size_t at, const std::string &content)
std::string_view str() const
Definition scope.hh:194
void foreach_match(const std::string &pattern, CallbackFn callback) const
Definition scope.hh:255
void foreach_scope(ScopeType type, CallbackFn callback) const
Definition scope.hh:339
Token prev(int i=1) const
Definition lexit.hh:376
std::string_view str() const
Definition lexit.hh:381
static Token invalid(const TokenBuffer *buf)
Definition lexit.hh:349
const TokenType & type() const
Definition lexit.hh:354
Token next(int i=1) const
Definition lexit.hh:372