15#include <unordered_map>
16#include <unordered_set>
27using namespace shader::parser;
29using namespace shader::parser::ast;
35 string str = this->source_;
39 IntermediateForm<SimpleLexer, DummyParser> parser(error_handler);
44 cleanup_whitespace(parser);
45 str = parser.result_get();
46 str = threadgroup_variables_parse_and_remove(
str);
48 catch (ParserException & ) {
50 return {parser.result_get(), metadata_, error_handler.err};
53 parse_builtins(
str, filename,
true);
55 str = matrix_constructor_mutation(
str);
57 str = argument_decorator_macro_injection(
str);
58 str = array_constructor_macro_injection(
str);
60 return {
str, metadata_, error_handler.err};
67 string str = this->source_;
72 IntermediateForm<SimpleLexer, DummyParser> parser(
str, error_handler);
74 cleanup_whitespace(parser);
75 str = parser.result_get();
78 str = threadgroup_variables_parse_and_remove(
str);
80 Parser parser(error_handler);
84 parse_pragma_runtime_generated(parser);
85 parse_includes(parser);
86 lower_preprocessor(parser);
87 str = parser.result_get();
89 catch (ParserException & ) {
91 return {parser.result_get(), metadata_, error_handler.err};
94 str = argument_decorator_macro_injection(
str);
95 str = array_constructor_macro_injection(
str);
97 return {
str, metadata_, error_handler.err};
106 metadata_.symbol_table.insert(metadata_.symbol_table.end(),
107 external_sources_symbols.symbol_table.begin(),
108 external_sources_symbols.symbol_table.end());
109 metadata_.template_definitions.insert(metadata_.template_definitions.end(),
110 external_sources_symbols.template_definitions.begin(),
111 external_sources_symbols.template_definitions.end());
114 for (
auto &symbol : metadata_.symbol_table) {
115 symbol.definition_line = 0;
120 Parser parser(error_handler);
124 disabled_code_mutation(parser);
126 threadgroup_variables_parse_and_remove(parser);
127 parse_builtins(parser, filename);
129 lint_pragma_once(parser, filename);
130 parse_pragma_runtime_generated(parser);
131 parse_includes(parser);
132 parse_defines(parser);
133 parse_library_functions(parser);
135 lower_preprocessor(parser);
137 parser.apply_mutations();
140 if (parser.str().find(
"\n#pragma no_processing") != string::npos) {
141 cleanup_whitespace(parser);
149 lower_namesless_parameters(parser);
150 lower_attribute_sequences(parser);
151 lower_strings_sequences(parser);
152 lower_swizzle_methods(parser);
153 lower_binary_literals(parser);
154 lower_classes(parser);
155 lower_noop_keywords(parser);
156 lower_trailing_comma_in_list(parser);
157 lower_comma_separated_declarations(parser);
158 lower_assert(parser, filename);
160 lower_implicit_member(parser);
162 parser.apply_mutations();
164 parse_local_symbols(parser);
167 lint_unbraced_statements(parser);
168 lint_reserved_tokens(parser);
169 lint_attributes(parser);
170 lint_global_scope_constants(parser);
171 lint_constructors(parser);
172 lint_forward_declared_structs(parser);
175 lower_pre_template(parser);
177 lower_templates(parser);
179 lower_unions(parser);
180 lower_host_shared_structures(parser);
184 lower_entry_points(parser);
185 lower_pipeline_definition(parser, filename);
186 lower_resource_table(parser);
187 lower_resource_access_functions(parser);
189 lower_default_constructors(parser);
190 lower_function_default_arguments(parser);
191 lower_method_definitions(parser);
192 lower_method_calls(parser);
193 lower_empty_struct(parser);
195 lower_srt_member_access(parser);
196 lower_srt_arguments(parser);
197 lower_entry_points_signature(parser);
198 lower_stage_function(parser);
200 lower_strings(parser);
201 lower_printf(parser);
203 lower_implicit_return_types(parser);
204 lower_initializer_implicit_types(parser);
205 lower_designated_initializers(parser);
206 lower_aggregate_initializers(parser);
207 lower_array_initializations(parser);
208 lower_scope_resolution_operators(parser);
209 lower_structured_bindings(parser);
212 lower_reference_arguments(parser);
213 lower_reference_variables(parser);
215 lower_static_branch(parser);
217 lower_loop_unroll(parser);
221 lower_argument_qualifiers(parser);
222 lower_gather_component(parser);
225 cleanup_whitespace(parser);
226 cleanup_empty_lines(parser);
227 cleanup_line_directives(parser);
229 str = parser.result_get();
231 catch (ParserException & ) {
233 return {parser.result_get(), metadata_, error_handler.err};
237 return {
str, metadata_, error_handler.err};
248 str =
"#define BSL_530\n" +
str +
"\n#undef BSL_530\n";
250 SourceManager sources;
253 Parser &builtin_parser = sources.new_source(error_handler);
255 builtin_parser.set_str(
"#line 1 \"builtin\"\n#error This should not be emitted\n");
256 builtin_parser.include_id = sources.include_id_get();
258 bsl::SymbolTable symbols(builtin_parser);
260 Parser &parser = sources.new_source(error_handler);
266 disabled_code_mutation(parser,
true);
268 lint_pragma_once(parser, filename);
269 parse_pragma_runtime_generated(parser);
270 parse_includes(parser);
271 parse_defines(parser);
272 lower_tests(parser,
"srt.");
274 parser.only_apply_mutations();
276 scan_external_symbols(sources, symbols, metadata_.visited_files);
278 parser.include_id = sources.include_id_get();
281 parser.parse(error_handler);
283 parse_draw_debug(parser, filename);
284 parse_library_functions_ast(parser);
285 lower_preprocessor_ast(parser);
291 lower_namesless_parameters_ast(parser);
292 lower_attribute_sequences_ast(parser);
293 lower_strings_sequences(parser);
294 lower_swizzle_methods_ast(parser);
295 lower_binary_literals(parser);
296 lower_noop_keywords_ast(parser);
297 lower_trailing_comma_in_list_ast(parser);
298 lower_assert_ast(parser, filename);
299 lower_this_keyword(parser);
300 parser.apply_mutations();
303 lower_strings(parser);
304 lower_printf(parser);
307 lint_reserved_tokens(parser);
308 lint_attributes_ast(parser);
310 lower_srt_accessor_templates_ast(parser);
311 lower_union_accessor_templates_ast(parser);
313 symbols.parse(parser.root(), error_handler);
315 lower_bsl_to_il(parser, symbols);
317 lower_pipeline_definition(parser, filename);
319 lower_method_forward_declaration(parser);
320 lower_union_setters(parser);
321 lower_bitfield_setters(parser);
322 lower_method_calls(parser,
false);
324 lower_resource_macro_placeholder_ast(parser);
325 lower_constructors(parser);
328 parser.apply_mutations();
331 lower_reference_arguments(parser);
332 lower_argument_qualifiers(parser);
333 lower_gather_component(parser);
336 cleanup_whitespace(parser,
true);
337 cleanup_empty_lines(parser);
338 cleanup_line_directives(parser);
340 str = parser.result_get();
342 catch (ParserException & ) {
344 return {parser.result_get(), metadata_, error_handler.err};
346 return {
str, metadata_, error_handler.err};
355 Parser parser(error_handler);
359 disabled_code_mutation(parser);
361 threadgroup_variables_parse_and_remove(parser);
362 parse_builtins(parser, filename);
364 lint_pragma_once(parser, filename);
365 parse_pragma_runtime_generated(parser);
366 parse_includes(parser);
367 parse_defines(parser);
368 parse_legacy_create_info(parser);
370 lower_preprocessor(parser);
373 cleanup_whitespace(parser);
374 cleanup_empty_lines(parser);
375 cleanup_line_directives(parser);
377 str = parser.result_get();
379 catch (ParserException & ) {
381 return {parser.result_get(), metadata_, error_handler.err};
384 return {
str, metadata_, error_handler.err};
391 return convert_info();
395 return (filename.starts_with(
"eevee_") || filename.starts_with(
"draw_")) ?
397 convert_bsl_legacy(external_sources_symbols);
399 return convert_bsl();
401 return convert_bsl_legacy(external_sources_symbols);
403 return convert_msl();
405 return convert_glsl();
412 report_error(0, 0,
"",
"Unknown file type");
413 return {
"", metadata_, error_handler.err};
425 disabled_code_mutation(
parser);
426 parse_pragma_runtime_generated(
parser);
431 lower_preprocessor(
parser);
439 lower_namesless_parameters(
parser);
440 lower_attribute_sequences(
parser);
441 lower_strings_sequences(
parser);
442 lower_swizzle_methods(
parser);
444 lower_noop_keywords(
parser);
445 lower_trailing_comma_in_list(
parser);
446 lower_comma_separated_declarations(
parser);
447 lower_assert(
parser, filename);
449 lower_implicit_member(
parser);
453 parse_local_symbols(
parser);
463void SourceProcessor::scan_external_symbols(SourceManager &sources,
465 vector<string> &visited_files)
469 for (
const auto &
filename : file_list_) {
477 report_error(0, 0,
"",
"Error: Included file not found " + dep);
481 if (ranges::find(visited_files, file) == visited_files.end()) {
482 visited_files.emplace_back(file);
484 ifstream input_file(file);
486 report_error(0, 0,
"",
"Error: Could not open file " + file);
487 throw ParserException();
491 buffer << input_file.rdbuf();
493 Language language = language_from_filename(file);
496 processor.parse_include_and_symbols(sources, symbols, visited_files);
499 if (processor.error_handler.err) {
500 this->error_handler.err = processor.error_handler.err;
501 throw ParserException();
508 bsl::SymbolTable &symbols,
509 vector<string> &visited_files)
515 Parser &parser = sources.new_source(error_handler);
518 disabled_code_mutation(parser,
true);
519 parse_pragma_runtime_generated(parser);
520 parse_includes(parser);
527 parser().foreach_match(
"A(A)", [&](
Tokens toks) {
528 string_view fn_name = toks[0].str();
529 if (fn_name ==
"SHADER_LIBRARY_CREATE_INFO" || fn_name ==
"VERTEX_SHADER_CREATE_INFO" ||
530 fn_name ==
"FRAGMENT_SHADER_CREATE_INFO" || fn_name ==
"COMPUTE_SHADER_CREATE_INFO")
532 parser.erase(toks.front(), toks.back().next() ==
';' ? toks.back().next() : toks.back());
537 parser.apply_mutations();
539 lower_preprocessor(parser);
542 parser.only_apply_mutations();
543 parser.parse(error_handler);
545 lower_namesless_parameters_ast(parser);
546 lower_attribute_sequences_ast(parser);
548 parser.apply_mutations();
550 scan_external_symbols(sources, symbols, visited_files);
552 parser.include_id = sources.include_id_get();
554 lower_srt_accessor_templates_ast(parser);
555 lower_union_accessor_templates_ast(parser);
557 symbols.
parse(parser.root(), error_handler);
559 catch (ParserException & ) {
569 string out_str =
str;
572 size_t start, end = 0;
573 while ((start = out_str.find(
"/*", end)) != string::npos) {
574 end = out_str.find(
"*/", start + 2);
575 if (end == string::npos) {
578 for (
size_t i = start; i < end + 2; ++i) {
579 if (out_str[i] !=
'\n') {
585 if (end == string::npos) {
589 "Malformed multi-line comment.");
595 size_t start, end = 0;
596 while ((start = out_str.find(
"//", end)) != string::npos) {
597 end = out_str.find(
'\n', start + 2);
598 if (end == string::npos) {
599 end = out_str.size();
601 for (
size_t i = start; i < end; ++i) {
611 parser().foreach_token(TokenType::Comment, [&](
Token tok) { parser.erase(tok); });
616template<
typename ParserT>
617void SourceProcessor::cleanup_whitespace(ParserT &
parser,
bool do_leading)
619 const string &
str = parser.str();
624 size_t first_char =
str.find_first_not_of(
" \n");
625 if (first_char != 0 && first_char != string::npos) {
626 parser.replace(0, first_char - 1,
"");
630 size_t last_whitespace = -1;
631 while ((last_whitespace =
str.find(
" \n", last_whitespace + 1)) != string::npos) {
632 size_t first_not_whitespace =
str.find_last_not_of(
" ", last_whitespace);
633 if (first_not_whitespace == string::npos) {
634 first_not_whitespace = 0;
636 parser.replace(first_not_whitespace + 1, last_whitespace,
"");
638 parser.apply_mutations();
645 parser().foreach_match<
true>(
"#A", [&](
const vector<Token> &tokens) {
646 if (tokens[1].
str() ==
"define") {
647 if (tokens[1].next().
str().starts_with(
"gather_")) {
652 metadata_.create_infos_defines.emplace_back(tokens[1].next().scope().str_with_whitespace());
654 if (tokens[1].
str() ==
"undef") {
655 metadata_.create_infos_defines.emplace_back(tokens[1].next().scope().str_with_whitespace());
663 placeholder +=
"#ifdef CREATE_INFO_RES_PASS_" + name +
"\n";
664 placeholder +=
"CREATE_INFO_RES_PASS_" + name +
"\n";
665 placeholder +=
"#endif\n";
666 placeholder +=
"#ifdef CREATE_INFO_RES_BATCH_" + name +
"\n";
667 placeholder +=
"CREATE_INFO_RES_BATCH_" + name +
"\n";
668 placeholder +=
"#endif\n";
669 placeholder +=
"#ifdef CREATE_INFO_RES_GEOMETRY_" + name +
"\n";
670 placeholder +=
"CREATE_INFO_RES_GEOMETRY_" + name +
"\n";
671 placeholder +=
"#endif\n";
672 placeholder +=
"#ifdef CREATE_INFO_RES_SHARED_VARS_" + name +
"\n";
673 placeholder +=
"CREATE_INFO_RES_SHARED_VARS_" + name +
"\n";
674 placeholder +=
"#endif\n";
679void SourceProcessor::parse_legacy_create_info(
Parser &
parser)
681 parser().foreach_scope(ScopeType::Attributes, [&](
const Scope attrs) {
687 if (type !=
Word || struct_keyword !=
Struct) {
691 parser.insert_line_number(struct_keyword.str_index_start() - 1, struct_keyword.line_number());
694 parser().foreach_match(
"A(..)", [&](
const vector<Token> &tokens) {
695 if (tokens[0].
str() ==
"CREATE_INFO_VARIANT") {
696 const string variant_name(tokens[1].scope().front().next().
str());
699 const string variant_decl = parser.substr_range_inclusive(tokens.front(), tokens.back());
705 if (tokens[0].
str() ==
"GPU_SHADER_CREATE_INFO") {
706 const string variant_name(tokens[1].scope().front().next().
str());
707 metadata_.create_infos.emplace_back(variant_name);
709 const size_t start_end = tokens.back().str_index_last();
710 const string end_tok =
"GPU_SHADER_CREATE_END()";
711 const size_t end_pos = parser.str().find(end_tok, start_end);
712 if (end_pos == string::npos) {
713 report_error(tokens[0],
"Missing create info end.");
717 const string variant_decl = parser.substr_range_inclusive(tokens.front().str_index_start(),
718 end_pos + end_tok.size());
719 metadata_.create_infos_declarations.emplace_back(variant_decl);
721 parser.replace(tokens.front().str_index_start(),
722 end_pos + end_tok.size(),
726 if (tokens[0].
str() ==
"GPU_SHADER_NAMED_INTERFACE_INFO") {
727 const size_t start_end = tokens.back().str_index_last();
728 const string end_str =
"GPU_SHADER_NAMED_INTERFACE_END(";
729 size_t end_pos = parser.str().find(end_str, start_end);
730 if (end_pos == string::npos) {
731 report_error(tokens[0],
"Missing create info end.");
735 end_pos = parser.str().find(
')', end_pos);
736 if (end_pos == string::npos) {
737 report_error(tokens[0],
"Missing parenthesis at info end.");
741 const string variant_decl = parser.substr_range_inclusive(tokens.front().str_index_start(),
743 metadata_.create_infos_declarations.emplace_back(variant_decl);
745 parser.erase(tokens.front().str_index_start(), end_pos);
748 if (tokens[0].
str() ==
"GPU_SHADER_INTERFACE_INFO") {
749 const size_t start_end = tokens.back().str_index_last();
750 const string end_str =
"GPU_SHADER_INTERFACE_END()";
751 size_t end_pos = parser.str().find(end_str, start_end);
752 if (end_pos == string::npos) {
753 report_error(tokens[0],
"Missing create info end.");
756 const string variant_decl = parser.substr_range_inclusive(tokens.front().str_index_start(),
757 end_pos + end_str.size());
758 metadata_.create_infos_declarations.emplace_back(variant_decl);
760 parser.erase(tokens.front().str_index_start(), end_pos + end_str.size());
765 parser.apply_mutations();
771 std::string_view
str = tok.
str();
772 if (
str.length() < 2) {
775 return str.substr(1,
str.length() - 2);
780 parser().foreach_match<
true>(
"#A\"", [&](
const vector<Token> &tokens) {
781 if (tokens[1].
str() !=
"include") {
786 if (dependency_name.find(
"defines.hh") != string::npos ||
789 (dependency_name.ends_with(
".bsl.hh") &&
filename.ends_with(
".bsl.hh") &&
790 ((dependency_name.starts_with(
"eevee_") &&
filename.starts_with(
"eevee_")) ||
791 (dependency_name.starts_with(
"overlay_") &&
filename.starts_with(
"overlay_")))))
795 if (dependency_name.ends_with(
".bsl.hh")) {
803 if (dependency_name ==
"BLI_utildefines_variadic.hh") {
805 parser.erase(tokens.front(), tokens.back());
808 if (dependency_name ==
"gpu_shader_compat.hh") {
810 parser.erase(tokens.front(), tokens.back());
813 if (dependency_name.find(
"gpu_shader_create_info.hh") != string::npos) {
815 parser.erase(tokens.front(), tokens.back());
819 if (dependency_name.find(
"infos/") != string::npos) {
820 dependency_name = dependency_name.substr(6);
823 if (dependency_name == filename) {
824 report_error(tokens[2],
"Recursive include");
826 metadata_.dependencies.emplace_back(dependency_name);
828 parser.apply_mutations();
836 if (tok.scope().
type() == ScopeType::Preprocessor && tok.
next(1).
str() ==
"pragma" &&
837 tok.
next(2).
str() == pragma_str)
845void SourceProcessor::parse_pragma_runtime_generated(
Parser &
parser)
854 if (
filename.find(
"_lib.") == string::npos &&
filename.find(
".hh") == string::npos) {
858 report_error(parser[0],
"Header files must contain #pragma once directive.");
862void SourceProcessor::lower_namesless_parameters(
Parser &
parser)
865 if (tok.scope().type() != ScopeType::FunctionArgs) {
868 if (tok.prev(2).str().starts_with(
"Pipeline")) {
872 if (tok.prev() !=
'>' && tok.prev(2) !=
Word && tok.prev(2) !=
'>') {
876 tok.scope().foreach_scope(ScopeType::FunctionArg, [&](
Scope arg) {
878 Token back = arg.back();
879 while (back ==
']') {
880 back = back.scope().front().prev();
882 const bool is_single_token = back.str_index_start() == arg.front().str_index_start();
883 if (is_single_token || back.prev() == TokenType::Const || back ==
'&' || back ==
'>') {
885 parser.replace(back.str_index_last_no_whitespace() + 1,
886 back.str_index_last(),
887 " _" + std::to_string(i++));
893void SourceProcessor::lower_namesless_parameters_ast(
Parser &
parser)
895 for (FuncDecl fn : parser.root().descendants_of_type<FuncDecl>()) {
897 for (FuncArg arg : fn.arguments().children_of_type<FuncArg>()) {
898 if (!arg.identifier().is_valid()) {
899 bool is_ref = arg.is_reference();
901 Token arg_back(is_ref ? arg.declarator().reference().back() :
902 (arr.is_valid() ? arr.front().prev() : arg.back()));
904 parser.replace(arg_back.str_index_last_no_whitespace() + 1,
905 arg_back.str_index_last(),
906 " _" + std::to_string(i++));
912void SourceProcessor::disabled_code_mutation(
Parser &
parser,
bool new_bsl_compiler)
914 auto process_disabled_scope = [&](
Token start_tok) {
916 string end_str = string(start_tok.str_with_whitespace()) +
"endif";
917 size_t scope_end = parser.str().find(end_str, start_tok.str_index_start());
918 if (scope_end == string::npos) {
919 report_error(start_tok,
"Couldn't find end of disabled scope.");
923 string else_str = string(start_tok.str_with_whitespace()) +
"el";
924 size_t scope_else = parser.str().find(else_str, start_tok.str_index_start());
925 if (scope_else != string::npos && scope_else < scope_end) {
927 parser.erase(start_tok.line_end() + 1, scope_else - 1);
931 parser.erase(start_tok.str_index_start(), scope_end + end_str.size());
935 auto process_enabled_scope = [&](
Token start_tok) {
937 string end_str = string(start_tok.str_with_whitespace()) +
"endif";
938 size_t scope_end = parser.str().find(end_str, start_tok.str_index_start());
939 if (scope_end == string::npos) {
940 report_error(start_tok,
"Couldn't find end of enabled scope.");
945 size_t code_start = start_tok.line_end() + 1;
948 string else_str = string(start_tok.str_with_whitespace()) +
"el";
949 size_t scope_else = parser.str().find(else_str, start_tok.str_index_start());
952 parser.erase(start_tok.str_index_start(), code_start - 1);
954 if (scope_else != string::npos && scope_else < scope_end) {
956 parser.erase(scope_else, scope_end + end_str.size());
960 parser.erase(scope_end, scope_end + end_str.size());
964 parser().foreach_match<
true>(
"#AA", [&](
const vector<Token> &tokens) {
965 if (tokens[1].
str() !=
"ifndef") {
968 if (tokens[2].
str() ==
"GPU_SHADER" ||
969 (new_bsl_compiler ? tokens[2].
str() ==
"BSL_530" :
false))
971 process_disabled_scope(tokens[0]);
974 parser().foreach_match<
true>(
"#i!A(A)", [&](
const vector<Token> &tokens) {
975 if (tokens[1].
str() !=
"if" || tokens[3].
str() !=
"defined") {
978 if (tokens[5].
str() ==
"GPU_SHADER" ||
979 (new_bsl_compiler ? tokens[5].
str() ==
"BSL_530" :
false))
981 process_disabled_scope(tokens[0]);
984 parser().foreach_match<
true>(
"#i1", [&](
const vector<Token> &tokens) {
985 if (tokens[1].
str() ==
"if") {
986 if (tokens[2].
str() ==
"0") {
987 process_disabled_scope(tokens[0]);
989 else if (tokens[2].
str() ==
"1") {
990 process_enabled_scope(tokens[0]);
995 parser.apply_mutations();
998string SourceProcessor::disabled_code_mutation(
const string &
str)
1001 disabled_code_mutation(parser);
1002 return parser.result_get();
1005void SourceProcessor::lower_preprocessor(
Parser &
parser)
1009 parser().foreach_match<
true>(
"#A", [&](
const vector<Token> &tokens) {
1010 if (tokens[1].
str() ==
"pragma") {
1011 Token next = tokens[1].next();
1012 if (next.str() ==
"once") {
1013 parser.erase(tokens.front(), next);
1015 else if (next.str() ==
"runtime_generated") {
1016 parser.erase(tokens.front(), next);
1019 else if (tokens[1].
str() ==
"include" && tokens[1].next() ==
String) {
1020 parser.erase(tokens.front(), tokens[1].next());
1023 parser.apply_mutations();
1026void SourceProcessor::lower_preprocessor_ast(
Parser &
parser)
1029 for (Preprocessor directive : parser.root().descendants_of_type<Preprocessor>()) {
1031 if (type.str() ==
"pragma") {
1033 if (pragma.str() ==
"once") {
1034 parser.erase(directive);
1036 else if (pragma.str() ==
"runtime_generated") {
1037 parser.erase(directive);
1040 else if (type.str() ==
"include" && type.next() ==
String) {
1041 parser.erase(directive);
1044 parser.apply_mutations();
1048void SourceProcessor::lower_swizzle_methods(
Parser &
parser)
1052 parser().foreach_match(
".A()", [&](
const vector<Token> &tokens) {
1053 string_view method_name(tokens[1].
str());
1054 if (method_name.length() > 1 && method_name.length() <= 4 &&
1055 (method_name.find_first_not_of(
"xyzw") == string::npos ||
1056 method_name.find_first_not_of(
"rgba") == string::npos))
1060 parser.erase(tokens[2], tokens[3]);
1065void SourceProcessor::lower_swizzle_methods_ast(
Parser &
parser)
1069 for (FuncCall call : parser.root().descendants_of_type<FuncCall>()) {
1071 if (call.front().prev() !=
Dot || !params.is_empty()) {
1075 string_view method_name = call.identifier().str();
1076 if (method_name.length() > 1 && method_name.length() <= 4 &&
1077 (method_name.find_first_not_of(
"xyzw") == string::npos ||
1078 method_name.find_first_not_of(
"rgba") == string::npos))
1081 parser.erase(params);
1087void SourceProcessor::lower_binary_literals(
Parser &
parser)
1089 parser().foreach_token(
Number, [&](
const Token tok) {
1090 string_view
str = tok.str();
1091 if (
str.starts_with(
"0b") ||
str.starts_with(
"0B")) {
1092 int64_t value = std::stoll(
string(
str.substr(2)),
nullptr, 2);
1093 parser.replace(tok.str_index_start(),
1094 tok.str_index_last_no_whitespace(),
1095 std::to_string(value) + (
str.ends_with(
"u") ?
"u" :
""));
1100void SourceProcessor::threadgroup_variables_parse_and_remove(
Parser &
parser)
1103 if (shared_tok.str() ==
"shared") {
1104 metadata_.shared_variables.push_back(
1105 {string(type.str()), parser.substr_range_inclusive(name, decl_end.prev())});
1107 parser.erase(shared_tok, decl_end);
1110 parser().foreach_match(
"AAA", [&](
const vector<Token> &tokens) {
1112 process_shared_var(tokens[0], tokens[1], tokens[2], end);
1114 parser.apply_mutations();
1117string SourceProcessor::threadgroup_variables_parse_and_remove(
const string &
str)
1119 IntermediateForm<FullLexer, DummyParser> parser(
str, error_handler);
1121 if (shared_tok.str() ==
"shared") {
1122 metadata_.shared_variables.push_back(
1123 {string(type.str()), parser.substr_range_inclusive(name, decl_end.prev())});
1125 parser.erase(shared_tok, decl_end);
1128 parser().foreach_match(
"AAA", [&](
const vector<Token> &tokens) {
1129 process_shared_var(tokens[0], tokens[1], tokens[2], tokens[2].find_next(
lexit::SemiColon));
1131 parser.apply_mutations();
1132 return parser.result_get();
1135void SourceProcessor::parse_library_functions(
Parser &
parser)
1137 using namespace metadata;
1139 parser().foreach_function(
1141 Token first_tok = is_static ? fn_type.
prev() : fn_type;
1142 Scope attributes = first_tok.attribute_before();
1143 if (!attributes.contains(
"node")) {
1146 if (fn_type.str() !=
"void") {
1147 report_error(fn_type,
"Expected void return type for node function");
1150 if (fn_args.token_count() <= 3) {
1151 report_error(fn_type,
"Expected at least one argument for node function");
1155 fn.name = fn_name.
str();
1157 fn_args.foreach_scope(ScopeType::FunctionArg, [&](
Scope arg) {
1159 Token curr = arg.front();
1162 curr = curr.scope().back().next();
1165 if (curr.str() ==
"const") {
1169 string qualifier =
"in";
1170 if (curr.str() ==
"in") {
1174 else if (curr.str() ==
"out") {
1179 Token type_tok = curr;
1180 string type = string(curr.str());
1191 if (type ==
"ShadingData" || type ==
"KernelGlobals") {
1199 switch (qualifier_enum) {
1205 report_error(arg.front(),
"Unknown qualifier '" + qualifier +
"'");
1209 switch (type_enum) {
1230 report_error(type_tok,
"Invalid type for node function '" + type +
"'");
1234 fn.arguments.emplace_back(ArgumentFormat{qualifier_enum, type_enum});
1237 metadata_.functions.emplace_back(fn);
1240 if (error_handler.err.has_value()) {
1241 throw ParserException();
1245void SourceProcessor::parse_library_functions_ast(
Parser &
parser)
1247 using namespace metadata;
1248 for (FuncDecl func : parser.root().children_of_type<FuncDecl>()) {
1249 if (!func.attributes().contains_attr(
"node")) {
1252 if (func.return_type().str() !=
"void") {
1253 report_error(func.return_type(),
"Expected void return type for node function");
1256 if (func.arguments().is_empty()) {
1257 report_error(func.identifier(),
"Expected at least one argument for node function");
1262 fn.name = func.identifier().str();
1264 for (FuncArg arg : func.arguments().children_of_type<FuncArg>()) {
1265 if (arg.declarator().array().is_valid()) {
1266 report_error(arg.declarator().array(),
1267 "Array arguments are not supported in node functions.");
1270 Type type =
Type(
hash(
string(arg.type().identifier().str())));
1272 if (arg.is_reference() && !arg.is_const()) {
1273 qualifier = Qualifier::out;
1276 qualifier = Qualifier::in;
1279 if (type == Type::KernelGlobals || type == Type::ShadingData) {
1281 qualifier = Qualifier::in;
1290 case Type::float3x3:
1291 case Type::float4x4:
1297 case Type::sampler1DArray:
1298 case Type::sampler2DArray:
1299 case Type::sampler2D:
1300 case Type::sampler3D:
1302 case Type::KernelGlobals:
1303 case Type::ShadingData:
1306 report_error(arg.type().identifier(),
1307 "Invalid type for node function '" +
string(arg.type().identifier().str()) +
1311 fn.arguments.emplace_back(qualifier, type);
1313 metadata_.functions.emplace_back(fn);
1316 if (error_handler.err.has_value()) {
1317 throw ParserException();
1323 const bool skip_drw_debug = filename ==
"draw_debug_draw.bsl.hh" ||
1324 filename ==
"draw_debug_infos.hh" ||
1325 filename ==
"draw_debug_draw_display.bsl.hh" ||
1326 filename ==
"draw_shader_shared.hh";
1328 if (skip_drw_debug) {
1332 for (
auto fn : parser.root().descendants_of_type<FuncCall>()) {
1333 if (fn.identifier().str().starts_with(
"drw_debug_")) {
1334 metadata_.builtins.emplace_back(Builtin::drw_debug);
1340void SourceProcessor::parse_builtins(
const string &
str,
const string &
filename,
bool pure_glsl)
1342 const bool skip_drw_debug = filename ==
"draw_debug_draw.bsl.hh" ||
1343 filename ==
"draw_debug_infos.hh" ||
1344 filename ==
"draw_debug_draw_display.bsl.hh" ||
1345 filename ==
"draw_shader_shared.hh";
1346 using namespace metadata;
1348 vector<string> tokens = {
1350 "gl_FragStencilRefARB",
1352 "gl_GlobalInvocationID",
1353 "gpu_InstanceIndex",
1356 "gl_LocalInvocationID",
1357 "gl_LocalInvocationIndex",
1369 tokens.emplace_back(
"gl_ClipDistance");
1373 tokens.emplace_back(
"drw_debug_");
1374#ifdef WITH_GPU_SHADER_ASSERT
1375 tokens.emplace_back(
"assert");
1379 for (
auto &token : tokens) {
1380 if (skip_drw_debug && token ==
"drw_debug_") {
1383 if (
str.find(token) != string::npos) {
1384 metadata_.builtins.emplace_back(
Builtin(
hash(token)));
1391 parser.apply_mutations();
1392 parse_builtins(parser.str(), filename);
1397void SourceProcessor::lower_empty_struct(
Parser &
parser)
1401 body.foreach_declaration(
1404 if (decl_count == 0) {
1405 parser.insert_before(body.back(),
"int _pad;");
1409 parser.apply_mutations();
1415 using namespace metadata;
1417 auto process_compilation_constants = [&](
Token tok) {
1418 string create_info_decl;
1420 while (tok ==
',') {
1423 if (scope.token_count() == 2) {
1424 report_error(struct_name,
1425 "Empty brace constructor is an error in Pipeline declaration. "
1426 "Either remove it or add compilation constant values to it.");
1428 auto process_constant = [&](
const vector<Token> &toks) {
1429 create_info_decl +=
"COMPILATION_CONSTANT(";
1430 create_info_decl += (toks[3] ==
Number) ?
1431 ((toks[3].
str().back() ==
'u') ?
"uint" :
"int") :
1433 create_info_decl +=
", " + string(toks[1].
str());
1434 create_info_decl +=
", " + string(toks[3].
str());
1435 create_info_decl +=
")\n";
1438 scope.foreach_match(
".A=1", process_constant);
1439 tok = scope.back().next();
1442 return create_info_decl;
1445 auto validate_fn_name = [&](
Token fn_name) {
1446 if (fn_name ==
'&') {
1447 report_error(fn_name,
"Double function reference, remove '&'");
1449 else if (fn_name !=
Word) {
1450 report_error(fn_name,
"Expected function name");
1455 auto process_graphic_pipeline = [&](
Token pipeline_name,
Scope params) {
1456 Token vertex_fn = validate_fn_name(params[1]);
1457 Token fragment_fn = validate_fn_name(params[3]);
1459 string create_info_decl;
1460 create_info_decl +=
"GPU_SHADER_CREATE_INFO(" + string(pipeline_name.str()) +
")\n";
1461 create_info_decl +=
"GRAPHIC_SOURCE(\"" + filename +
"\")\n";
1462 create_info_decl +=
"VERTEX_FUNCTION(\"" + string(vertex_fn.str()) +
"\")\n";
1463 create_info_decl +=
"FRAGMENT_FUNCTION(\"" + string(fragment_fn.str()) +
"\")\n";
1464 create_info_decl +=
"ADDITIONAL_INFO(" + string(vertex_fn.str()) +
"_infos_)\n";
1465 create_info_decl +=
"ADDITIONAL_INFO(" + string(fragment_fn.str()) +
"_infos_)\n";
1466 create_info_decl += process_compilation_constants(params[4]);
1467 create_info_decl +=
"DO_STATIC_COMPILATION()\n";
1468 create_info_decl +=
"GPU_SHADER_CREATE_END()\n";
1470 metadata_.create_infos_declarations.emplace_back(create_info_decl);
1473 auto process_compute_pipeline = [&](
Token pipeline_name,
Scope params) {
1474 Token compute_fn = validate_fn_name(params[1]);
1476 string create_info_decl;
1477 create_info_decl +=
"GPU_SHADER_CREATE_INFO(" + string(pipeline_name.str()) +
")\n";
1478 create_info_decl +=
"COMPUTE_SOURCE(\"" + filename +
"\")\n";
1479 create_info_decl +=
"COMPUTE_FUNCTION(\"" + string(compute_fn.str()) +
"\")\n";
1480 create_info_decl +=
"ADDITIONAL_INFO(" + string(compute_fn.str()) +
"_infos_)\n";
1481 create_info_decl += process_compilation_constants(params[2]);
1482 create_info_decl +=
"DO_STATIC_COMPILATION()\n";
1483 create_info_decl +=
"GPU_SHADER_CREATE_END()\n";
1485 metadata_.create_infos_declarations.emplace_back(create_info_decl);
1488 parser().foreach_match(
"AA(", [&](
const vector<Token> &tokens) {
1489 Scope parameters = tokens[2].scope();
1490 if (tokens[0].
str() ==
"PipelineGraphic") {
1491 process_graphic_pipeline(tokens[1], parameters);
1492 parser.erase(tokens.front(), parameters.back().next());
1494 else if (tokens[0].
str() ==
"PipelineCompute") {
1495 process_compute_pipeline(tokens[1], parameters);
1496 parser.erase(tokens.front(), parameters.back().next());
1501void SourceProcessor::lower_stage_function(
Parser &
parser)
1503 parser().foreach_function([&](
bool is_static,
Token fn_type,
Token,
Scope,
bool,
Scope fn_body) {
1505 if (attr_tok.is_invalid() || attr_tok !=
']' || attr_tok.prev() !=
']') {
1509 if (attributes.type() != ScopeType::Attributes) {
1513 parser.erase(attributes.scope());
1517 const string_view attr = attr_tok.str();
1518 if (attr ==
"vertex") {
1521 else if (attr ==
"fragment") {
1524 else if (attr ==
"compute") {
1535 parser.apply_mutations();
1545 string guard_start =
"#if " +
condition;
1547 if (fn_type.
is_valid() && fn_type.
str() !=
"void") {
1548 string type(fn_type.
str());
1549 bool is_trivial =
false;
1550 if (type ==
"float" || type ==
"float2" || type ==
"float3" || type ==
"float4" ||
1552 type ==
"int" || type ==
"int2" || type ==
"int3" || type ==
"int4" ||
1554 type ==
"uint" || type ==
"uint2" || type ==
"uint3" || type ==
"uint4" ||
1556 type ==
"float2x2" || type ==
"float2x3" || type ==
"float2x4" ||
1558 type ==
"float3x2" || type ==
"float3x3" || type ==
"float3x4" ||
1560 type ==
"float4x2" || type ==
"float4x3" || type ==
"float4x4" || type ==
"bool")
1564 guard_else +=
"#else\n";
1565 guard_else += line_start;
1566 guard_else +=
" return " + type + (is_trivial ?
"(0)" :
"{}") +
";\n";
1568 string guard_end =
"#endif";
1570 parser.insert_directive(scope.
front(), guard_start);
1571 parser.insert_directive(scope.
back().
prev(), guard_else + guard_end);
1576void SourceProcessor::lower_host_shared_structures(
Parser &
parser)
1586 bool is_shared =
false;
1588 if (attr.
str() ==
"host_shared") {
1597 if (comma.is_valid() && comma.scope() == body) {
1600 "comma declaration is not supported in shared struct, expand to multiple definition");
1604 bool is_std140_compatible =
true;
1605 bool has_vec3 =
false;
1611 unordered_map<string, Type> sizeof_types = {
1614 {
"float4", {16, 16}},
1615 {
"float2x4", {16 * 2, 16}},
1616 {
"float3x4", {16 * 3, 16}},
1617 {
"float4x4", {16 * 4, 16}},
1618 {
"bool32_t", {4, 4}},
1624 {
"uint4", {16, 16}},
1625 {
"string_t", {4, 4}},
1626 {
"packed_float3", {12, 16}},
1627 {
"packed_int3", {12, 16}},
1628 {
"packed_uint3", {12, 16}},
1633 string_view type_str(type.str());
1635 if (type_str.find(
"char") != string::npos || type_str.find(
"short") != string::npos ||
1636 type_str.find(
"half") != string::npos)
1638 report_error(type,
"Small types are forbidden in shader interfaces.");
1640 else if (type_str ==
"float3") {
1641 report_error(type,
"use packed_float3 instead of float3 in shared structure");
1643 else if (type_str ==
"uint3") {
1644 report_error(type,
"use packed_uint3 instead of uint3 in shared structure");
1646 else if (type_str ==
"int3") {
1647 report_error(type,
"use packed_int3 instead of int3 in shared structure");
1649 else if (type_str ==
"bool") {
1650 report_error(type,
"bool is not allowed in shared structure, use bool32_t");
1652 else if (type_str ==
"float4x3") {
1653 report_error(type,
"float4x3 is not allowed in shared structure");
1655 else if (type_str ==
"float3x3") {
1656 report_error(type,
"float3x3 is not allowed in shared structure");
1658 else if (type_str ==
"float2x3") {
1659 report_error(type,
"float2x3 is not allowed in shared structure");
1661 else if (type_str ==
"float4x2") {
1662 report_error(type,
"float4x2 is not allowed in shared structure");
1664 else if (type_str ==
"float3x2") {
1665 report_error(type,
"float3x2 is not allowed in shared structure");
1667 else if (type_str ==
"float2x2") {
1668 report_error(type,
"float2x2 is not allowed in shared structure");
1671 auto sz = sizeof_types.find(
string(type_str));
1673 Type type_info{16, 16};
1674 if (sz != sizeof_types.end()) {
1675 type_info = sz->second;
1677 else if (type.prev() ==
Enum) {
1680 parser.erase(type.prev());
1685 else if (type.prev() ==
Struct) {
1687 type_info = {16, 16};
1689 parser.erase(type.prev());
1695 report_error(type,
"Unknown type, add 'enum' or 'struct' keyword before the type name");
1699 if (type_info.size == 12) {
1703 size_t align = type_info.alignment - (offset % type_info.alignment);
1704 if (align != type_info.alignment) {
1705 string err =
"Misaligned member, missing " +
to_string(align) +
" padding bytes";
1706 report_error(type, err);
1709 size_t array_size = 1;
1710 if (array.is_valid()) {
1711 if (array_size > 1 && type_info.size < 16) {
1713 is_std140_compatible =
false;
1717 array_size = static_array_size(array, 4);
1720 offset += type_info.size * array_size;
1726 is_std140_compatible = ((offset % 16) == 0);
1728 else if (offset % 16 != 0) {
1729 string err =
"Alignment issue, missing " +
to_string(16 - (offset % 16)) +
" padding bytes";
1730 report_error(struct_name, err);
1735 string(struct_name.
str()) +
"\n";
1736 if (is_std140_compatible) {
1740 parser.insert_directive(struct_keyword.
prev(), directive);
1742 parser.apply_mutations();
1745void SourceProcessor::lint_unbraced_statements(
Parser &
parser)
1747 auto check_statement = [&](
const Tokens &toks) {
1748 Token end_tok = toks.back();
1749 if (end_tok.next() ==
If || end_tok.scope().type() == ScopeType::Preprocessor) {
1752 if (end_tok.next() ==
'[' && end_tok.next().next() ==
'[') {
1753 end_tok = end_tok.next().scope().back();
1755 if (end_tok.next() !=
'{') {
1756 report_error(end_tok,
"Missing curly braces after flow control statement.");
1760 parser().foreach_match(
"i(..)", check_statement);
1761 parser().foreach_match(
"I", check_statement);
1762 parser().foreach_match(
"f(..)", check_statement);
1763 parser().foreach_match(
"F(..)", check_statement);
1766void SourceProcessor::lint_reserved_tokens(
Parser &
parser)
1768 unordered_set<string> reserved_symbols = {
1769 "vec2",
"vec3",
"vec4",
"mat2x2",
"mat2x3",
"mat2x4",
"mat3x2",
1770 "mat3x3",
"mat3x4",
"mat4x2",
"mat4x3",
"mat4x4",
"mat2",
"mat3",
1771 "mat4",
"ivec2",
"ivec3",
"ivec4",
"uvec2",
"uvec3",
"uvec4",
1772 "bvec2",
"bvec3",
"bvec4",
"common",
"partition",
"active",
"typedef",
1773 "packed",
"resource",
"goto",
"noinline",
"extern",
"external",
"interface",
1774 "long",
"fixed",
"unsigned",
"superp",
"input",
"output",
"hvec2",
1775 "hvec3",
"hvec4",
"fvec2",
"fvec3",
"fvec4",
"sample",
"sampler3DRect",
1776 "filter",
"cast",
"row_major",
"inout",
1779 parser().foreach_token(
Word, [&](
Token tok) {
1780 if (reserved_symbols.contains(
string(tok.str()))) {
1781 string err = string(tok.str()) +
" is a reserved token";
1782 report_error(tok, err);
1787void SourceProcessor::lower_tests(
Parser &
parser,
const string &prefix)
1790 if (type.str() !=
"void") {
1795 fn_body.foreach_match(
"A(A,A){..}", [&](
Tokens toks) {
1796 if (toks[0].
str() !=
"TEST") {
1800 parser.erase(toks[0], toks[5]);
1802 if (toks[0].
str().starts_with(
"EXPECT_")) {
1804 parser.insert_before(toks[0], prefix +
"out_test[" +
to_string(
id) +
"] = ");
1805 parser.insert_after(toks[4],
1806 "; " + prefix +
"out_test[" +
to_string(
id) +
1814 parser.apply_mutations();
1817void SourceProcessor::lower_noop_keywords(
Parser &
parser)
1820 parser().foreach_token(
Inline, [&](
Token tok) { parser.erase(tok); });
1824 ScopeType scope_type = tok.scope().type();
1825 if (scope_type != ScopeType::Struct && scope_type != ScopeType::Preprocessor) {
1833 auto process_access = [&](
Token tok) {
1834 if (tok.next() ==
':') {
1835 parser.erase(tok, tok.next());
1838 report_error(tok,
"Expecting colon ':' after access specifier");
1841 parser().foreach_token(
Private, process_access);
1842 parser().foreach_token(
Public, process_access);
1844 lower_template_dependent_names(parser);
1847void SourceProcessor::lower_noop_keywords_ast(
Parser &
parser)
1850 parser().foreach_token(
Inline, [&](
Token tok) { parser.erase(tok); });
1852 for (AccessSpecifier
node : parser.root().descendants_of_type<AccessSpecifier>()) {
1856 for (TemplateExplicit
node : parser.root().descendants_of_type<TemplateExplicit>()) {
1857 parser.erase(
node.front());
1860 for (IdType type : parser.root().descendants_of_type<IdType>()) {
1861 Token tok = type.identifier().front().
prev();
1868void SourceProcessor::lower_trailing_comma_in_list(
Parser &
parser)
1870 parser().foreach_match(
",}", [&](
const Tokens &t) { parser.erase(t[0]); });
1873void SourceProcessor::lower_trailing_comma_in_list_ast(
Parser &
parser)
1875 for (InitializerList decl : parser.root().descendants_of_type<InitializerList>()) {
1876 if (decl.back().prev() ==
',') {
1877 parser.erase(decl.back().prev());
1884void SourceProcessor::lower_comma_separated_declarations(
Parser &
parser)
1886 auto process_decl = [&](
const Tokens &t) {
1887 if (t[0].scope().type() != ScopeType::Struct) {
1890 string type(t[0].
str());
1892 while (comma ==
',' || comma ==
'[') {
1894 comma = comma.scope().back().next();
1897 parser.replace(comma,
";" + type,
true);
1898 comma = comma.next().next();
1902 parser().foreach_match(
"AA,", [&](
const Tokens &t) { process_decl(t); });
1903 parser().foreach_match(
"AA[..],", [&](
const Tokens &t) { process_decl(t); });
1906void SourceProcessor::lower_implicit_return_types(
Parser &
parser)
1909 fn_body.foreach_match(
"rA?{..};", [&](
Tokens toks) {
1911 if (list.front().next() ==
'.') {
1914 parser.insert_before(toks[0],
1915 "{" +
string(type.str()) +
" _tmp = " +
string(type.str()) +
1916 string(list.str()) +
"; ");
1917 const Token start = toks[1].
is_valid() ? toks[1] : list.front();
1918 parser.replace(start, list.back(),
"_tmp;}");
1920 else if (toks[1].is_invalid()) {
1922 parser.insert_after(toks[0],
string(type.str()));
1928void SourceProcessor::lower_implicit_return_types_ast(
Parser &
parser)
1930 for (FuncDecl func : parser.root().descendants_of_type<FuncDecl>()) {
1931 for (ReturnStmt stmt : func.body().descendants_of_type<ReturnStmt>()) {
1932 Expr expr = stmt.expression();
1933 if (!expr.is_valid()) {
1937 Node
node = expr.child_first();
1938 if (
node == NodeType::InitializerList) {
1941 else if (
node == NodeType::Constructor) {
1942 list =
node.child_first();
1948 const string type_str(func.return_type().str());
1949 if (list.child_first() == NodeType::DesignatedInitializer) {
1953 stmt,
"{" + type_str +
" _tmp" +
string(list.str()) +
"; return _tmp;}",
true);
1957 parser.insert_before(list.front(), type_str);
1963void SourceProcessor::lower_initializer_implicit_types(
Parser &
parser)
1965 auto process_scope = [&](
Scope s) {
1967 s.foreach_match(
"AA{..}",
1968 [&](
Tokens t) { parser.insert_before(t[2],
" = " +
string(t[0].
str())); });
1970 s.foreach_match(
"AA={..}", [&](
Tokens t) { parser.insert_before(t[3],
string(t[0].
str())); });
1973 parser().foreach_scope(ScopeType::FunctionArg, process_scope);
1974 parser().foreach_scope(ScopeType::Function, process_scope);
1975 parser.apply_mutations();
1978void SourceProcessor::lower_initializer_implicit_types_ast(
Parser &
parser)
1980 for (VarDecl decl : parser.root().descendants_of_type<VarDecl>()) {
1981 for (Declarator var : decl.children_of_type<Declarator>()) {
1983 if (init_list.is_valid()) {
1985 parser.insert_before(init_list.front(),
" = " +
string(decl.type().str()));
1990 if (assign.is_valid()) {
1992 if (init_list.is_valid()) {
1994 parser.insert_before(init_list.front(),
string(decl.type().str()));
2001 parser.apply_mutations();
2004void SourceProcessor::lower_designated_initializers(
Parser &
parser)
2007 parser().foreach_match(
"A{.A=", [&](
Tokens t) {
2009 report_error(t[0],
"Designated initializers are only supported in assignments");
2014 if (nested_aggregate_end != t[3]) {
2015 Token nested_aggregate_start = nested_aggregate_end.scope().front();
2016 if (nested_aggregate_start.prev() !=
Word) {
2017 report_error(nested_aggregate_start,
"Nested anonymous aggregate is not supported");
2025 parser.insert_before(assign_tok,
";");
2026 parser.erase(assign_tok, t[1]);
2028 if (t[0].scope() != aggregate) {
2029 report_error(t[0],
"Nested initializer lists are not supported");
2032 parser.insert_before(t[0],
string(var.str()));
2033 Token value_end = t[2].scope().back();
2034 parser.insert_after(value_end,
";");
2035 if (value_end.next() ==
',') {
2036 parser.erase(value_end.next());
2039 parser.erase(aggregate.back(), aggregate.back().next());
2044 parser.apply_mutations();
2049void SourceProcessor::lower_aggregate_initializers(
Parser &
parser)
2051 unordered_set<string> builtin_types = {
2052 "float2",
"float3",
"float4",
"float2x2",
"float2x3",
"float2x4",
2053 "float3x2",
"float3x3",
"float3x4",
"float4x2",
"float4x3",
"float4x4",
2054 "float2x2",
"float3x3",
"float4x4",
"int2",
"int3",
"int4",
2055 "uint2",
"uint3",
"uint4",
"bool2",
"bool3",
"bool4",
2060 parser().foreach_match(
"A{..}", [&](
Tokens t) {
2064 if (builtin_types.contains(
string(t[0].str()))) {
2066 "Aggregate is error prone for built-in vector and matrix types, use "
2067 "constructors instead");
2069 if (t[1].scope().token_count() == 2) {
2071 parser.insert_after(t.front(),
"_ctor_");
2072 parser.replace(t[1], t[4],
"()");
2077 if (nested_aggregate_end != t[4]) {
2078 Token nested_aggregate_start = nested_aggregate_end.scope().front();
2079 if (nested_aggregate_start.prev() !=
Word) {
2080 report_error(nested_aggregate_start,
"Nested anonymous aggregate is not supported");
2083 parser.insert_before(t[0],
"_ctor(");
2084 parser.insert_before(t[1],
")");
2086 if (t[4].
prev() ==
',') {
2087 parser.erase(t[4].
prev());
2089 parser.insert_before(t[4],
" _rotc()");
2094 }
while (parser.apply_mutations());
2099void SourceProcessor::lower_aggregate_initializers_ast(
Parser &
parser)
2101 unordered_set<string> builtin_types = {
2102 "float2",
"float3",
"float4",
"float2x2",
"float2x3",
"float2x4",
2103 "float3x2",
"float3x3",
"float3x4",
"float4x2",
"float4x3",
"float4x4",
2104 "float2x2",
"float3x3",
"float4x4",
"int2",
"int3",
"int4",
2105 "uint2",
"uint3",
"uint4",
"bool2",
"bool3",
"bool4",
2109 for (InitializerList list : parser.root().descendants_of_type<InitializerList>()) {
2110 IdType type(list.prev());
2111 if (!type.is_valid()) {
2115 if (builtin_types.contains(
string(type.str()))) {
2116 report_error(type.front(),
2117 "Aggregate is error prone for built-in vector and matrix types, use "
2118 "constructors instead");
2121 if (list.is_empty()) {
2122 parser.insert_after(type.back(),
"_ctor_");
2123 parser.replace(list,
"()",
true);
2127 for (InitializerList nested_list : list.descendants_of_type<InitializerList>()) {
2128 if (!
IdType(nested_list.prev()).is_valid()) {
2129 report_error(nested_list.front(),
"Nested anonymous aggregate is not supported");
2133 parser.insert_before(type.front(),
"_ctor(");
2134 parser.insert_after(type.back(),
",");
2135 parser.erase(list.front());
2136 if (list.back().prev() ==
',') {
2137 parser.erase(list.back().prev());
2139 parser.insert_before(list.back(),
" _rotc()");
2140 parser.erase(list.back());
2143 parser.apply_mutations();
2149void SourceProcessor::lower_array_initializations(
Parser &
parser)
2151 parser().foreach_match(
"AA[..]={..};", [&](vector<Token> toks) {
2152 const Token type_tok = toks[0];
2153 const Token name_tok = toks[1];
2154 const Scope array_scope = toks[2].scope();
2155 const Scope list_scope = toks[7].scope();
2158 int array_scope_tok_len = array_scope.token_count();
2159 if (array_scope_tok_len == 2) {
2160 int comma_count = 0;
2161 list_scope.foreach_token(
Comma, [&](
Token t) {
2162 if (t.scope() == list_scope) {
2166 const int list_len = (comma_count > 0) ? comma_count + 1 : 0;
2167 if (list_len == 0) {
2168 report_error(name_tok,
"Array size must be greater than zero.");
2170 parser.insert_after(array_scope[0],
to_string(list_len));
2172 else if (array_scope_tok_len == 3 && array_scope[1] ==
Number) {
2173 if (stol(
string(array_scope[1].
str())) == 0) {
2174 report_error(name_tok,
"Array size must be greater than zero.");
2180 if (tok != list_scope.front()) {
2181 report_error(name_tok,
"Nested initializer list is not supported.");
2186 parser.insert_before(list_scope.front(),
"ARRAY_T(" +
string(type_tok.str()) +
") ARRAY_V(");
2187 parser.insert_after(list_scope.back(),
")");
2188 parser.erase(list_scope.front());
2189 parser.erase(list_scope.back());
2190 if (list_scope.back().prev() ==
',') {
2191 parser.erase(list_scope.back().prev());
2194 parser.apply_mutations();
2199 return str.substr(0,
str.find_last_not_of(
" \n") + 1);
2206void SourceProcessor::lower_function_default_arguments(
Parser &
parser)
2208 parser().foreach_function([&](
const bool is_static,
2212 const bool fn_const,
2218 const bool has_non_void_return_type = fn_type.
str() !=
"void";
2219 const bool is_method = fn_type.scope().
type() == ScopeType::Struct;
2226 struct_name = fn_type.scope().front().
prev().
str();
2229 vector<string> fn_overloads;
2233 const char *comma = (args_decl.empty() ?
"" :
", ");
2234 if (
equal.is_invalid()) {
2236 args_names += comma + string(arg.
back().
str());
2239 string arg_name(
equal.prev().str());
2240 string value = parser.substr_range_inclusive(
equal.next(), arg.
back());
2241 string decl = parser.substr_range_inclusive(arg.
front(),
equal.prev());
2243 string fn_call = string(fn_name.
str()) +
'(' + args_names + comma + value +
");";
2249 fn_call =
"this->" + fn_call;
2252 if (has_non_void_return_type) {
2253 fn_call =
"return " + fn_call;
2256 overload += string(fn_type.
str()) +
" ";
2257 overload += string(fn_name.
str()) +
'(' + args_decl +
")" +
2258 string(fn_const ?
" const" :
"") +
"\n";
2260 overload +=
"#line " +
to_string(fn_type.line_number()) +
"\n";
2261 overload +=
" " + fn_call +
"\n}\n";
2262 fn_overloads.emplace_back(overload);
2265 args_names += comma + arg_name;
2267 parser.erase(
equal.scope());
2270 size_t end_of_fn_char = fn_body.
back().line_end() + 1;
2272 for (
auto it = fn_overloads.rbegin(); it != fn_overloads.rend(); ++it) {
2273 parser.insert_line_number(end_of_fn_char, fn_type.line_number());
2274 parser.insert_after(end_of_fn_char, *it);
2276 parser.insert_line_number(end_of_fn_char, fn_body.
back().line_number() + 1);
2279 parser.apply_mutations();
2283void SourceProcessor::cleanup_line_directives(
Parser &
parser)
2285 parser().foreach_match<
true>(
"#A1", [&](vector<Token> toks) {
2286 if (toks[1].
str() !=
"line") {
2289 if (toks[2].next() ==
String) {
2294 if (toks.back().next() ==
'#' && toks.back().next().next() ==
Word &&
2295 toks.back().next().next().next() ==
Number)
2297 parser.replace(toks[0].line_start(), toks[0].line_end() + 1,
"");
2300 parser.apply_mutations();
2302 parser().foreach_match<
true>(
"#A1#A", [&](vector<Token> toks) {
2303 if (toks[1].
str() !=
"line") {
2307 if (toks.back().next() ==
'#' && toks.back().next().next() ==
Word &&
2308 toks.back().next().next().next() ==
Number)
2310 parser.replace(toks[0].line_start(), toks[0].line_end() + 1,
"");
2313 parser.apply_mutations();
2315 parser().foreach_match<
true>(
"#A1", [&](vector<Token> toks) {
2316 if (toks[1].
str() !=
"line") {
2319 if (toks[2].next() ==
String) {
2323 int line = toks[0].line_number();
2324 int value = stol(
string(toks[2].
str()));
2327 Token next = toks[2].next();
2329 if (
prev.line_number() == value) {
2333 while (peek.is_valid() && peek.str_with_whitespace().find_first_of(
'\n') == string::npos) {
2334 first_on_prev_line = peek;
2335 peek = first_on_prev_line.prev();
2339 bool is_prev_directive = (first_on_prev_line.is_valid() && first_on_prev_line ==
'#');
2342 if (!is_prev_directive) {
2343 int prev_end_col =
prev.char_number() +
prev.str().length();
2344 int next_start_col = next.char_number();
2346 if (prev_end_col < next_start_col) {
2347 int spaces_needed = next_start_col - prev_end_col;
2348 parser.replace(
prev.str_index_last_no_whitespace() + 1,
2349 next.str_index_start() - 1,
2350 std::string(spaces_needed,
' '));
2356 if (line == value) {
2357 parser.replace(toks[0].line_start(), toks[0].line_end() + 1,
"");
2360 if (line == value - 1) {
2361 parser.replace(toks[0].line_start(), toks[0].line_end(),
"");
2364 if (line == value - 2) {
2365 parser.replace(toks[0].line_start(), toks[0].line_end(),
"\n");
2368 parser.apply_mutations();
2372void SourceProcessor::cleanup_empty_lines(
Parser &
parser)
2374 const string &
str = parser.str();
2377 size_t sequence_start = 0;
2378 size_t sequence_end = -1;
2379 while ((sequence_start =
str.find(
"\n\n\n", sequence_end + 1)) != string::npos) {
2380 sequence_end =
str.find_first_not_of(
"\n", sequence_start);
2381 if (sequence_end == string::npos) {
2385 parser.replace(sequence_start + 2, sequence_end - 1,
"#line " +
to_string(line) +
"\n");
2387 parser.apply_mutations();
2390 size_t sequence_start = 0;
2391 size_t sequence_end = -1;
2392 while ((sequence_end =
str.find(
"\n\n#line ", sequence_end + 1)) != string::npos) {
2393 sequence_start =
str.find_last_not_of(
"\n", sequence_end) + 1;
2394 if (sequence_start == string::npos) {
2397 parser.replace(sequence_start, sequence_end,
"");
2399 parser.apply_mutations();
2405string SourceProcessor::matrix_constructor_mutation(
const string &
str)
2407 if (
str.find(
"mat") == string::npos) {
2411 IntermediateForm<FullLexer, DummyParser> parser(
str, error_handler);
2413 if (t.prev() ==
Word) {
2415 string_view fn_name_str = fn_name.
str();
2416 if (fn_name_str.size() == 4) {
2418 if (fn_name_str ==
"mat2") {
2419 parser.replace(fn_name,
"__mat2x2",
true);
2421 else if (fn_name_str ==
"mat3") {
2422 parser.replace(fn_name,
"__mat3x3",
true);
2424 else if (fn_name_str ==
"mat4") {
2425 parser.replace(fn_name,
"__mat4x4",
true);
2428 else if (fn_name_str.size() == 6) {
2429 if (fn_name_str ==
"mat2x2" || fn_name_str ==
"mat3x3" || fn_name_str ==
"mat4x4") {
2433 parser.insert_before(fn_name,
"__");
2438 return parser.result_get();
2442void SourceProcessor::lower_reference_arguments(
Parser &
parser)
2445 if (type.prev() == TokenType::Const) {
2446 parser.replace(type.prev(), last_tok,
string(type.str()) +
" " +
string(arg_name.str()));
2449 parser.replace(type, last_tok,
"inout " +
string(type.str()) +
" " +
string(arg_name.str()));
2453 parser().foreach_scope(ScopeType::FunctionArgs, [&](
const Scope scope) {
2454 scope.foreach_match(
2455 "A(&A)", [&](
const vector<Token> toks) { add_mutation(toks[0], toks[3], toks[4]); });
2456 scope.foreach_match(
2457 "A&A", [&](
const vector<Token> toks) { add_mutation(toks[0], toks[2], toks[2]); });
2458 scope.foreach_match(
2459 "A&T", [&](
const vector<Token> toks) { add_mutation(toks[0], toks[2], toks[2]); });
2461 parser.apply_mutations();
2465void SourceProcessor::lower_reference_variables(
Parser &
parser)
2468 fn_scope.foreach_match(
"c?A&A=", [&](
const vector<Token> &tokens) {
2469 const Token name = tokens[4];
2470 const Scope assignment = tokens[5].scope();
2472 Token decl_start = tokens[0].is_valid() ? tokens[0] : tokens[2];
2474 decl_start = (decl_start.prev() ==
']') ? decl_start.prev().scope().front() : decl_start;
2476 const Token decl_end = assignment.back().
next();
2480 report_error(token,
"Reference definitions cannot have side effects.");
2483 report_error(token,
"Reference definitions cannot have side effects.");
2485 assignment.foreach_token(
ParOpen, [&](
const Token token) {
2486 string_view fn_name = token.prev().str();
2487 if ((fn_name !=
"specialization_constant_get") && (fn_name !=
"push_constant_get") &&
2488 (fn_name !=
"interface_get") && (fn_name !=
"resource_table_get") &&
2489 (fn_name !=
"attribute_get") && (fn_name !=
"buffer_get") &&
2490 (fn_name !=
"srt_access") && (fn_name !=
"sampler_get") && (fn_name !=
"image_get"))
2492 report_error(token,
"Reference definitions cannot contain function calls.");
2495 assignment.foreach_scope(ScopeType::Subscript, [&](
const Scope subscript) {
2496 if (subscript.token_count() != 3) {
2497 report_error(subscript.front(),
2498 "Array subscript inside reference declaration must be a single variable or "
2499 "a constant, not an expression.");
2503 const Token index_var = subscript[1];
2505 if (index_var ==
Number) {
2511 bool is_const =
false;
2512 bool is_ref =
false;
2513 bool is_found =
false;
2515 auto process_decl = [&](
const vector<Token> &tokens) {
2516 if (tokens[5].str_index_start() < index_var.str_index_start() &&
2517 tokens[5].str() == index_var.str())
2519 is_const = tokens[0].is_valid();
2520 is_ref = tokens[3].is_valid();
2524 fn_args.foreach_match(
"c?A&?A", [&](
const vector<Token> &toks) { process_decl(toks); });
2525 fn_scope.foreach_match(
"c?A&?A", [&](
const vector<Token> &toks) { process_decl(toks); });
2528 report_error(index_var,
2529 "Cannot locate array subscript variable declaration. "
2530 "If it is a global variable, assign it to a temporary const variable for "
2531 "indexing inside the reference.");
2535 report_error(index_var,
"Array subscript variable must be declared as const qualified.");
2539 report_error(index_var,
"Array subscript variable must not be declared as reference.");
2544 string definition = parser.substr_range_inclusive(assignment[1], assignment.back());
2548 parser.erase(decl_start, decl_end);
2550 name.scope().foreach_token(
Word, [&](
const Token token) {
2552 if (error || token.prev() ==
'.' || token.next() ==
'(') {
2555 if (token.str_index_start() > decl_end.str_index_last() && token.str() == name.str()) {
2556 if (token.prev() ==
'&' && token.next() ==
'=') {
2557 report_error(token,
"Local reference shadowing is not allowed.");
2561 parser.replace(token, definition);
2567 parser.apply_mutations();
2569 parser().foreach_match(
"c?A&A=", [&](
const vector<Token> &tokens) {
2570 report_error(tokens[4],
"Reference is defined inside a global or unterminated scope.");
2574void SourceProcessor::lower_argument_qualifiers(
Parser &
parser)
2576 parser().foreach_match(
"AAA", [&](
const Tokens &toks) {
2577 if (toks[0].scope().type() == ScopeType::Preprocessor) {
2581 if (toks[0].
str() ==
"inout" || toks[0].
str() ==
"out") {
2582 parser.replace(toks[0],
"_ref(");
2583 parser.insert_after(toks[1],
",");
2584 parser.insert_after(toks[2],
")");
2587 parser.apply_mutations();
2590void SourceProcessor::lower_gather_component(
Parser &
parser)
2592 parser().foreach_match(
"A(..)", [&](
const Tokens &toks) {
2593 if (toks[0].scope().type() == ScopeType::Preprocessor) {
2599 if (toks[0].
str() ==
"textureGather" && component ==
Number && component.prev() ==
Comma) {
2600 parser.insert_after(toks[0],
string(component.str()));
2601 parser.erase(component.prev(), component);
2604 parser.apply_mutations();
2607string SourceProcessor::argument_decorator_macro_injection(
const string &
str)
2609 IntermediateForm<FullLexer, DummyParser> parser(
str, error_handler);
2611 parser().foreach_match(
"AAA", [&](
const Tokens &t) {
2612 string_view qualifier = t[0].str();
2613 if (qualifier ==
"out" || qualifier ==
"inout" || qualifier ==
"in" || qualifier ==
"shared") {
2614 parser.insert_after(t[1],
" _" +
string(qualifier) +
"_sta ");
2615 parser.insert_after(t[2],
" _" +
string(qualifier) +
"_end ");
2618 return parser.result_get();
2621string SourceProcessor::array_constructor_macro_injection(
const string &
str)
2623 IntermediateForm<FullLexer, DummyParser> parser(
str, error_handler);
2624 parser().foreach_match(
"=A[", [&](
const Tokens toks) {
2625 Token array_len_start = toks.back();
2627 if (array_len_end.is_valid()) {
2628 Token type = toks[1];
2629 Token array_start = array_len_end.
next();
2630 if (array_start ==
'(') {
2631 parser.insert_before(type,
" ARRAY_T(");
2632 parser.replace(array_len_start, array_len_end,
") ");
2633 parser.insert_before(array_start,
"ARRAY_V");
2637 return parser.result_get();
2641void SourceProcessor::lint_global_scope_constants(
Parser &
parser)
2644 parser().foreach_match(
"cAA=", [&](
const vector<Token> &tokens) {
2645 if (tokens[0].scope().type() == ScopeType::Global) {
2648 "Global scope constant expression found. These get allocated per-thread in MSL. "
2649 "Use Macro's or uniforms instead.");
2654void SourceProcessor::lint_global_scope_constants_ast(
Parser &
parser)
2657 for (VarDecl decl : parser.root().children_of_type<VarDecl>()) {
2658 if (decl.is_const()) {
2661 "Global scope constant expression found. These get allocated per-thread in MSL. "
2662 "Use Macro's or uniforms instead.");
2667int SourceProcessor::static_array_size(
const Scope &array,
int fallback_value)
2669 if (array.token_count() == 3 && array[1] ==
Number) {
2671 return stol(
string(array[1].
str()));
2673 catch (invalid_argument
const & ) {
2674 report_error(array.front(),
"Invalid array size, expecting integer literal");
2677 return fallback_value;
2684 return "#line 1 \"" + filename +
"\"\n";
std::string remove_comments()
static std::string strip_whitespace(const std::string &str)
static bool has_pragma(Parser &parser, std::string_view pragma_str)
parser::IntermediateForm< parser::FullLexer, parser::FullParser > Parser
SourceProcessor(const std::string &source, const std::string &filepath, Language language, const std::vector< std::string > &file_list)
metadata::Source parse_include_and_symbols()
static constexpr const char * namespace_separator
std::vector< parser::Token > Tokens
static constexpr const char * linted_struct_suffix
static std::string get_create_info_placeholder(const std::string &name)
static std::string line_directive_prefix(const std::string &filename)
static constexpr const char * uniform_struct_suffix
Result convert(metadata::Source external_sources_symbols={})
static void guarded_scope_mutation(Parser &parser, Scope scope, const std::string &condition, Token fn_type)
VecBase< bool, D > equal(VecOp< T, D >, VecOp< T, D >) RET
int size(const AssetLibraryReference *library_reference)
size_t line_number(const std::string_view &str, size_t pos)
std::string filename(const std::string_view &str, size_t pos)
size_t char_number(const std::string_view &str, size_t pos)
std::string line_str(const std::string_view &str, size_t pos)
static std::string_view str_view_exclusive(Token tok)
const char * to_string(ShaderStage stage)
SymEdge< T > * prev(const SymEdge< T > *se)
bool contains_token(const char token_type) const
std::string_view str_with_whitespace() const
void foreach_declaration(Callback callback) const
Token find_token(const char token_type) const
std::string_view str() const
void foreach_match(const std::string &pattern, CallbackFn callback) const
void foreach_attribute(CallbackFn callback) const
void foreach_scope(ScopeType type, CallbackFn callback) const
void parse(ast::LocalScope node, ErrorHandler &err_handler)
Token prev(int i=1) const
std::string_view str() const
Token(const TokenBuffer *buf, int32_t index)
const TokenType & type() const
Token next(int i=1) const