768 struct TokenStreamPool {
773 std::deque<TokenStream> pool;
774 std::vector<int> free_indices;
786 using Ptr = std::unique_ptr<TokenStream, Deleter>;
791 if (free_indices.empty()) {
792 target_idx = pool.size();
793 pool.emplace_back(lex_);
796 target_idx = free_indices.back();
797 free_indices.pop_back();
802 return Ptr(&s, Deleter{
this, target_idx});
806 using TokenStreamPtr = TokenStreamPool::Ptr;
810 TokenStreamPtr definition;
811 Vector<TokenAtom> params;
812 bool is_function =
false;
813 bool contains_concat =
false;
815 Macro(Directive
id) : id(id) {}
817 bool is_empty()
const
819 return definition->tokens.is_empty();
825 Vector<Directive, 8> jump_stack;
827 TokenStreamPool stream_pool = {lex_};
829 Vector<Directive> visited_macros;
832 Map<TokenAtom, Directive> defines;
834 Vector<std::unique_ptr<Macro>> parsed_macro;
836 bits::BitVector<> enabled_macros;
855 out_stream_(
Token::invalid(&lex_),
868 lex_.hash(
"float2x2"),
869 lex_.hash(
"float3x2"),
870 lex_.hash(
"float4x2"),
871 lex_.hash(
"float2x3"),
872 lex_.hash(
"float3x3"),
873 lex_.hash(
"float4x3"),
874 lex_.hash(
"float2x4"),
875 lex_.hash(
"float3x4"),
876 lex_.hash(
"float4x4"),
877 lex_.hash(
"packed_float3"),
878 lex_.hash(
"packed_int3"),
891 lex_.hash(
"int32_t"),
892 lex_.hash(
"uint32_t"),
893 lex_.hash(
"bool32_t"),
897 lex_.hash(
"texelFetch"),
898 lex_.hash(
"texture"),
899 lex_.hash(
"textureLod"),
900 lex_.hash(
"textureSize"),
901 lex_.hash(
"imageLoad"),
902 lex_.hash(
"imageStore"),
903 lex_.hash(
"imageSize"),
930 lex_.hash(
"intBitsToFloat"),
931 lex_.hash(
"floatBitsToInt"),
932 lex_.hash(
"uintBitsToFloat"),
933 lex_.hash(
"floatBitsToUint"),
936 defined_atom(lex_.
hash(
"defined")),
937 va_args_atom(lex_.
hash(
"__VA_ARGS__"))
940 defines.reserve(1000);
947 if (lex_.directive_lines.is_empty()) {
948 out_stream_ << TokenRange<Token>{.begin = lex_.front(), .end = lex_.back()};
952 last_directive_end = lex_.line(0);
953 next_directive = lex_.directive(0);
955 if (lex_.line(0) != next_directive.first()) {
956 expand_macros_in_range(lex_.line(0), next_directive.first().prev());
959 while (!next_directive.is_last()) {
962 next_directive =
id.
next();
963 evaluate_directive(
id);
965 expand_macros_in_range(last_directive_end.next(), next_directive.first().prev());
968 evaluate_directive(next_directive);
970 if (!last_directive_end.is_last()) {
971 Line last_line = lex_.line(lex_.line_offsets.size() - 1);
972 expand_macros_in_range(last_directive_end.next(), last_line);
979 entry_points.append(lex_.hash(
"main"));
981 out_stream_.optimize(entry_points.as_span());
986 return out_stream_.str();
1001 last_directive_end = dir.
last();
1008 undefine_macro(dir);
1015 process_conditional(dir, dir_type);
1024 process_pragma(dir);
1037 TokenAtom dce_atom = lex_.hash(
"dead_code_elimination");
1044 if (tok.
atom() == blender_atom) {
1046 if (tok.
atom() == dce_atom) {
1048 if (tok.
atom() == off_atom) {
1049 out_stream_.set_enabled_parsing(
false);
1051 else if (tok.
atom() == on_atom) {
1052 out_stream_.set_enabled_parsing(
true);
1055 BLI_assert_msg(
false,
"Invalid dead_code_elimination pragma. Expecting on or off.");
1070 Vector<TokenAtom> macro_parameters;
1082 macro_parameters.append(tok.
str() ==
"..." ? va_args_atom : tok.
atom());
1097 return macro_parameters;
1101 bool &contains_concat)
1103 TokenStreamPtr definition = stream_pool.alloc();
1105 Token tok = definition_start_tok;
1115 *definition << TokenStream::Space{};
1118 if (tok ==
Hash && tok_next ==
Hash) {
1121 contains_concat =
true;
1134 const Token name = dir.identifier().
next();
1136 defines.add_overwrite(name.
atom(), dir);
1137 enabled_macros[name.
atom()].set(
true);
1138 erase_lines(dir.first(), dir.last());
1143 if (parsed_macro[
int(dir)] ==
nullptr) {
1144 const Token name = dir.identifier().
next();
1146 Token cursor = after_name;
1148 auto macro = std::make_unique<Macro>(dir);
1150 if (macro->is_function) {
1151 macro->params = parse_macro_params(cursor);
1153 macro->definition = parse_macro_definition(cursor, macro->contains_concat);
1155 parsed_macro[int(dir)] = std::move(macro);
1157 return *parsed_macro[int(dir)];
1160 void undefine_macro(Directive dir)
1162 const Token name = dir.identifier().
next();
1164 enabled_macros[name.
atom()].set(
false);
1165 defines.remove(name.
atom());
1166 erase_lines(dir.first(), dir.last());
1176 if (!jump_stack.is_empty() && jump_stack.last() == dir) {
1177 jump_stack.pop_last();
1183 if (endif.is_last()) {
1185 Line last_before_endif = endif.first().prev();
1186 erase_lines(dir.first(), last_before_endif);
1187 next_directive = endif;
1189 last_directive_end = last_before_endif;
1193 Line endif_end = endif.last();
1194 erase_lines(dir.first(), endif_end);
1196 next_directive = endif.next();
1198 last_directive_end = endif_end;
1203 const Line dir_line_start = dir.first();
1204 const Line dir_line_end = dir.last();
1205 const Token dir_tok = dir.identifier();
1207 const Token cond_start = dir_tok.
next();
1208 const Token cond_end = dir_line_end.last();
1209 const bool condition_result = evaluate_condition(dir_type, cond_start, cond_end);
1214 if (condition_result) {
1219 jump_stack.append(next_condition);
1223 erase_lines(dir_line_start, dir_line_end);
1226 Line last_before_next_cond = next_condition.first().prev();
1228 erase_lines(dir_line_start, last_before_next_cond);
1230 next_directive = next_condition;
1232 last_directive_end = last_before_next_cond;
1242 return defines.contains(start.
atom());
1244 return !defines.contains(start.
atom());
1247 return evaluate_expression(start, end);
1254 bool evaluate_expression(
const Token start,
const Token end)
1257 TokenStreamPtr expand = expand_expression(start, end);
1260 if (expand->tokens.size() == 1) {
1261 const auto &token = expand->tokens.first();
1262 StringRef
str = token.str();
1273 std::string
str = expand->str();
1274 expression_parser.lexical_analysis(
str);
1275 return expression_parser.eval() != 0;
1277 catch (
const std::exception &
e) {
1278 std::cout <<
"\"" << lex_.substr(start, end) <<
"\" > \"" << expand->str() <<
"\" ";
1279 std::cerr <<
"Error: " <<
e.what() <<
"\n";
1290 int start = int(start_line.first());
1291 int end = int(end_line.end());
1296 out_stream_ << lex_[start];
1300 const Vector<int> &candidates = gather_candidate_in_range(start, end);
1302 Token after_last_emitted = lex_[start];
1304 for (
const auto *it = candidates.begin(); it != candidates.
end();) {
1305 const Token tok = lex_[*it];
1306 const Directive macro_id = defines.lookup(tok.
atom());
1308 if (
int(after_last_emitted) < *it) {
1309 out_stream_ << TokenRange<Token>{after_last_emitted, tok.
prev()};
1314 after_last_emitted = tok.
next();
1319 const Token end_tok = expand_and_replace(tok, macro_id);
1320 after_last_emitted = end_tok.
next();
1322 while (it != candidates.
end() && *it <=
int(end_tok)) {
1327 const Token last_tok = lex_[end];
1328 out_stream_ << TokenRange<Token>{after_last_emitted, last_tok};
1332 Vector<int> expand_candidates_;
1337 BLI_NOINLINE const Vector<int> &gather_candidate_in_range(
int start,
int end)
1339 expand_candidates_.clear();
1340 expand_candidates_.resize(end - start + 1);
1341 int candidates_count = 0;
1346 expand_candidates_[candidates_count] = i;
1347 candidates_count += enabled_macros[lex_.atoms_[i]];
1349 expand_candidates_.resize(candidates_count);
1350 return expand_candidates_;
1355 auto [replacement, end] = expand_macro(tok, get_macro(macro_id));
1356 out_stream_ << *replacement;
1361 BLI_NOINLINE TokenStreamPtr parse_and_expand(
const TokenStream &tok_stream)
1363 auto result = stream_pool.alloc();
1365 for (
auto tok = tok_stream.begin(), end = tok_stream.end(); tok != end; ++tok) {
1371 auto [replacement, end] = expand_macro(tok, get_macro(macro_id));
1382 template<
typename IToken>
struct ExpandedResult {
1384 TokenStreamPtr output;
1386 IToken end_of_expansion;
1392 template<
typename IToken>
1393 BLI_NOINLINE Vector<TokenRange<IToken>> parse_macro_args(
const Vector<TokenAtom> ¶ms,
1396 Vector<TokenRange<IToken>> args;
1397 for (
const TokenAtom param_atom : params) {
1398 IToken param_start = tok;
1399 IToken param_end = get_end_of_parameter(param_start);
1401 if (param_atom == va_args_atom) {
1403 param_end = get_end_of_parameter(param_start,
true);
1406 args.append({param_start.next(), param_end});
1414 if (params.is_empty()) {
1431 template<
typename IToken>
1432 BLI_NOINLINE TokenStreamPtr expand_macro_args(
const Macro ¯o,
1435 TokenStreamPtr ts = stream_pool.alloc();
1436 for (
const auto &def_tok : *macro.definition) {
1437 if (def_tok.type() ==
Hash) {
1438 *ts << TokenStream::Concatenate{};
1441 if (def_tok.type() ==
Word && !macro_args.is_empty()) {
1443 int arg_id = macro.params.first_index_of_try(def_tok.atom());
1447 if (def_tok.prev() ==
Hash || def_tok.next() ==
Hash) {
1453 TokenStreamPtr stream = stream_pool.alloc();
1454 *stream << macro_value;
1455 *ts << *parse_and_expand(*stream);
1458 if (def_tok.followed_by_whitespace()) {
1460 *ts << TokenStream::Space{};
1470 template<
typename IToken>
1471 BLI_NOINLINE ExpandedResult<IToken> expand_macro(
const IToken expanded_tok,
const Macro ¯o)
1473 if (visited_macros.contains(macro.id)) {
1477 TokenStreamPtr expanded = stream_pool.alloc();
1478 *expanded << expanded_tok;
1479 return {std::move(expanded), expanded_tok};
1482 IToken end_of_expansion = expanded_tok;
1484 Vector<TokenRange<IToken>> fn_arguments;
1485 if (macro.is_function) {
1486 IToken tok = expanded_tok;
1492 TokenStreamPtr expanded = stream_pool.alloc();
1493 *expanded << expanded_tok;
1494 return {std::move(expanded), expanded_tok};
1497 fn_arguments = parse_macro_args(macro.params, tok);
1499 end_of_expansion = tok;
1502 if (macro.is_empty()) {
1504 TokenStreamPtr expanded = stream_pool.alloc();
1505 return {std::move(expanded), end_of_expansion};
1510 if (!macro.is_function && !macro.contains_concat) {
1512 visited_macros.append(macro.id);
1513 result = parse_and_expand(*macro.definition);
1514 visited_macros.pop_last();
1517 TokenStreamPtr expanded = expand_macro_args(macro, fn_arguments);
1519 visited_macros.append(macro.id);
1520 result = parse_and_expand(*expanded);
1521 visited_macros.pop_last();
1524 if (end_of_expansion.followed_by_whitespace()) {
1525 *
result << TokenStream::Space{};
1528 return {std::move(
result), end_of_expansion};
1532 TokenStreamPtr expand_expression(
const Token start,
const Token end)
1534 TokenStreamPtr
result = stream_pool.alloc();
1545 *
result << lex_[int(tok)];
1548 Macro ¯o = get_macro(macro_id);
1549 auto [replacement, macro_end] = expand_macro(
Token(lex_[
int(tok)]), macro);
1551 tok = lex_[int(macro_end)];
1553 else if (tok_atom == defined_atom) {
1556 const bool is_function = (tok ==
ParOpen);
1564 if (defines.contains(tok.
atom())) {
1565 *
result << TokenStream::True{};
1568 *
result << TokenStream::False{};
1577 *
result << lex_[int(tok)];
1582 tok = skip_directive_newlines(tok.
next());
1592 void erase_lines(
Line start,
Line end)
1594 if (
int(end) >
int(start)) {
1595 out_stream_ << new_lines(start, end);
1603 std::string new_lines_buf = std::string(lex_.line_offsets.size(),
'\n');
1606 StringRef new_lines(
Line line_start,
Line line_end)
1608 return StringRef(new_lines_buf).substr(0,
int(line_end) -
int(line_start));
1623 template<
typename IToken>
1624 static IToken get_end_of_parameter(IToken tok,
bool skip_to_end =
false)
1629 while (tok.is_valid()) {
1639 if (stack == 1 && tok ==
Comma && !skip_to_end) {