|
Blender V5.3
|
#include <lexit.hh>
Inherited by blender::gpu::AtomicLexer, blender::gpu::TokenPastingBuffer, and blender::gpu::shader::parser::LexerBase.
Classes | |
| struct | TokenIt |
| Token iterator. More... | |
Public Member Functions | |
| TokenBuffer ()=default | |
| TokenBuffer (const std::string_view str, const CharClass char_class_table[128]) | |
| void | process (const std::string_view str, const CharClass char_class_table[128]) |
| void | clear () |
| Discard all currently held data. Does not reallocate. | |
| void | reserve (const uint32_t count) |
| Allocate backing memory for the given number of tokens and move currently held data. | |
| void | tokenize (const CharClass char_class_table[128]) |
| Tokenizes the input string by grouping contiguous characters of the same class. | |
| void | merge_complex_literals (bool bsl_template_angle_bracket_lexing=false) |
| Merge complex literals such as floats, strings and comments. | |
| void | atomize_words (IdentifierMap &identifiers, const KeywordTable &keywords) |
| Assign keyword types and atoms for a small set of identifier. | |
| void | compute_lengths () |
| Compute small token length for speeding up certain tasks. | |
| uint32_t | size () const |
| Return the amount of token inside the buffer. | |
| std::string_view | substr (const Token &start, const Token &end, const bool with_trailing_whitespaces=false) const |
| Return the substring between the start and end tokens (included). | |
| void | append (TokenType type, TokenAtom atom, int32_t str_size, int32_t str_size_with_witespaces) |
| Append a Token at the end of the buffer. | |
| Token | operator[] (int index) const |
| TokenMut | operator[] (int index) |
| TokenIt | begin () |
| TokenIt | end () |
| Token | front () |
| Token | back () |
Public Attributes | |
| std::string_view | str_ |
| AlignedArrayPtr< TokenType > | types_ |
| AlignedArrayPtr< uint32_t > | offsets_ |
| AlignedArrayPtr< uint32_t > | offsets_end_ |
| AlignedArrayPtr< uint8_t > | lengths_ |
| AlignedArrayPtr< TokenAtom > | atoms_ |
| uint32_t | size_ = 0 |
| uint32_t | allocated_size_ = 0 |
|
default |
Referenced by lexit::TokenBuffer::TokenIt::TokenIt().
|
inline |
Definition at line 146 of file lexit.hh.
References lexit::char_class_table, process(), and str.
|
inline |
Append a Token at the end of the buffer.
Definition at line 236 of file lexit.hh.
References atoms_, offsets_, offsets_end_, reserve(), size_, and types_.
Referenced by blender::gpu::TokenPastingBuffer::paste_token().
| void lexit::TokenBuffer::atomize_words | ( | IdentifierMap & | identifiers, |
| const KeywordTable & | keywords ) |
Assign keyword types and atoms for a small set of identifier.
Definition at line 671 of file lexit.cc.
References lexit::divide_ceil(), end(), lengths_, lexit::Vector< T >::resize(), size(), size_, types_, and lexit::Word.
Referenced by blender::gpu::AtomicLexer::lex_pass().
|
inline |
Definition at line 311 of file lexit.hh.
References size_.
Referenced by blender::gpu::shader::parser::ScopeParser::translation_unit().
|
inline |
Definition at line 298 of file lexit.hh.
References lexit::TokenBuffer::TokenIt::TokenIt().
| void lexit::TokenBuffer::clear | ( | ) |
| void lexit::TokenBuffer::compute_lengths | ( | ) |
Compute small token length for speeding up certain tasks.
Definition at line 644 of file lexit.cc.
References lengths_, offsets_, offsets_end_, lexit::select(), and size_.
Referenced by process().
|
inline |
Definition at line 302 of file lexit.hh.
References size_, and lexit::TokenBuffer::TokenIt::TokenIt().
Referenced by atomize_words(), substr(), and tokenize().
| void lexit::TokenBuffer::merge_complex_literals | ( | bool | bsl_template_angle_bracket_lexing = false | ) |
Merge complex literals such as floats, strings and comments.
| bsl_template_angle_bracket_lexing | If true, will parse angle bracket without a whitespace before or after as a template bracket. |
Definition at line 1010 of file lexit.cc.
References assert, lexit::EndOfFile, lexit::AlignedArrayPtr< T >::get(), lexit::GThan, lexit::lex_angle(), lexit::lex_comment(), lexit::lex_float(), lexit::lex_string(), lexit::LThan, lexit::Number, offsets_, offsets_end_, size_, lexit::Slash, str_, lexit::String, and types_.
Referenced by blender::gpu::shader::parser::ExpressionLexer::lexical_analysis(), and blender::gpu::shader::parser::FullLexer::lexical_analysis().
|
inline |
Definition at line 250 of file lexit.hh.
References index, and lexit::TokenMut::TokenMut().
|
inline |
Definition at line 246 of file lexit.hh.
References index, and lexit::Token::Token().
|
inline |
Definition at line 154 of file lexit.hh.
References assert, lexit::char_class_table, clear(), compute_lengths(), str, str_, tokenize(), and UINT_MAX.
Referenced by blender::gpu::AtomicLexer::lexical_analysis(), blender::gpu::shader::parser::ExpressionLexer::lexical_analysis(), blender::gpu::shader::parser::FullLexer::lexical_analysis(), blender::gpu::shader::parser::SimpleLexer::lexical_analysis(), and TokenBuffer().
| void lexit::TokenBuffer::reserve | ( | const uint32_t | count | ) |
Allocate backing memory for the given number of tokens and move currently held data.
Does nothing if allocation is already large enough.
Definition at line 60 of file lexit.cc.
References allocated_size_, atoms_, lengths_, offsets_, offsets_end_, lexit::realloc_aligned_array(), size_, and types_.
Referenced by append(), tokenize(), and blender::gpu::TokenPastingBuffer::TokenPastingBuffer().
|
inline |
Return the amount of token inside the buffer.
Definition at line 214 of file lexit.hh.
References size_.
Referenced by atomize_words(), blender::gpu::shader::parser::ParserBase::build_scope_tree(), blender::gpu::shader::parser::ParserBase::build_token_to_scope_map(), blender::gpu::shader::parser::LexerBase::identify_template_tokens(), blender::gpu::AtomicLexer::lex_pass(), and blender::gpu::shader::parser::LexerBase::reset_template_tokens().
|
inline |
Return the substring between the start and end tokens (included).
| with_trailing_whitespaces | If true, include the trailing whitespaces. |
Definition at line 224 of file lexit.hh.
References end(), offsets_, offsets_end_, and str_.
Referenced by blender::gpu::shader::parser::ast::Node::str(), bsl::CodegenContext::StaticConditionStack::SubCondition::str(), and blender::gpu::shader::parser::ast::Node::str_full().
| void lexit::TokenBuffer::tokenize | ( | const CharClass | char_class_table[128] | ) |
Tokenizes the input string by grouping contiguous characters of the same class.
This function iterates through the input string and identifies "runs" of characters that map to the same CharClass. For each new group, it records the type and the starting byte offset into the result arrays.
Only characters with the #CanMerge flag are merged together. Characters with a class greater than #ClassToTypeThreshold will just be assigned their class as TokenType. Otherwise, the first character of the token will be used as TokenType.
If the input string contains characters that are not inside the ASCII range, the result of the operation is undefined and might cause segmentation fault.
| char_class_table | A lookup table mapping ASCII values (0-127) to an 8-bit CharClass. |
The goal of this function is to build the offset and type buffers describing the tokens. The type of the token is decided by the character class of its first character. This is a simple lookup from the char_class_table combined with a select. It is then merged with the preceding character if the classes are compatible (see CharClass::CanMerge).
The offsets define the boundaries of the tokens. If whitespaces are treated as tokens, one offset is emitted for the beginning of each token.
str: i n t a = 0 ; EndOfFile emit: 1 0 0 0 1 1 1 1 1 1 offsets: 0 4 6 7 8 9
If whitespaces are merged with preceding tokens, one offset is emitted for the beginning of each token and at either the start of the next token or the start of the next whitespace.
str: i n t a = 0 ; EndOfFile emit start: 1 0 0 0 1 1 1 1 0 1 emit end: 1 0 0 1 0 1 1 1 1 1 offsets start: 0 4 5 6 7 9 offsets end: 3 5 6 7 8 9
Definition at line 488 of file lexit.cc.
References assert, lexit::CanMerge, lexit::char_class_table, lexit::ClassToTypeThreshold, end(), lexit::EndOfFile, is_zero(), lexit::None, offsets_, offsets_end_, popcount(), reserve(), lexit::select(), size_, str, str_, types_, and lexit::WhiteSpace.
Referenced by process().
| uint32_t lexit::TokenBuffer::allocated_size_ = 0 |
| AlignedArrayPtr<TokenAtom> lexit::TokenBuffer::atoms_ |
Definition at line 133 of file lexit.hh.
Referenced by append(), lexit::Token::atom(), blender::gpu::AtomicLexer::lex_pass(), and reserve().
| AlignedArrayPtr<uint8_t> lexit::TokenBuffer::lengths_ |
Definition at line 131 of file lexit.hh.
Referenced by atomize_words(), compute_lengths(), and reserve().
| AlignedArrayPtr<uint32_t> lexit::TokenBuffer::offsets_ |
Definition at line 127 of file lexit.hh.
Referenced by append(), compute_lengths(), blender::gpu::shader::parser::IntermediateForm< parser::FullLexer, parser::FullParser >::debug_print_tokens(), lexit::Token::followed_by_whitespace(), merge_complex_literals(), reserve(), lexit::Token::str(), blender::gpu::shader::parser::Token::str_index_last(), blender::gpu::shader::parser::Token::str_index_start(), lexit::Token::str_with_whitespace(), substr(), tokenize(), and blender::gpu::TokenPastingBuffer::TokenPastingBuffer().
| AlignedArrayPtr<uint32_t> lexit::TokenBuffer::offsets_end_ |
Definition at line 129 of file lexit.hh.
Referenced by append(), compute_lengths(), blender::gpu::shader::parser::IntermediateForm< parser::FullLexer, parser::FullParser >::debug_print_tokens(), lexit::Token::followed_by_whitespace(), merge_complex_literals(), reserve(), lexit::Token::str(), blender::gpu::shader::parser::Token::str_index_last_no_whitespace(), substr(), tokenize(), and blender::gpu::TokenPastingBuffer::TokenPastingBuffer().
| uint32_t lexit::TokenBuffer::size_ = 0 |
Definition at line 135 of file lexit.hh.
Referenced by append(), atomize_words(), back(), clear(), compute_lengths(), end(), blender::gpu::shader::parser::ExpressionParser::eval(), lexit::Token::invalid(), blender::gpu::AtomicLexer::lex_pass(), merge_complex_literals(), blender::gpu::TokenPastingBuffer::paste_token(), reserve(), size(), lexit::Token::Token(), blender::gpu::shader::parser::LexerBase::token_types_str(), and tokenize().
| std::string_view lexit::TokenBuffer::str_ |
Definition at line 123 of file lexit.hh.
Referenced by blender::gpu::shader::parser::MutableString::apply_mutations(), blender::gpu::shader::parser::Token::char_number(), blender::gpu::shader::parser::Token::filename(), blender::gpu::shader::parser::Token::full_symbol_name(), blender::gpu::shader::parser::Token::line_end(), blender::gpu::shader::parser::Token::line_number(), blender::gpu::shader::parser::Token::line_start(), blender::gpu::shader::parser::Token::line_str(), merge_complex_literals(), blender::gpu::TokenPastingBuffer::paste_token(), process(), lexit::Token::str(), lexit::Token::str_with_whitespace(), substr(), and tokenize().
| AlignedArrayPtr<TokenType> lexit::TokenBuffer::types_ |
Definition at line 125 of file lexit.hh.
Referenced by append(), atomize_words(), blender::gpu::AtomicLexer::lex_pass(), merge_complex_literals(), blender::gpu::shader::parser::ScopeParser::preprocessor(), bsl::BSLParser::preprocessor_directive(), reserve(), blender::gpu::shader::parser::LexerBase::token_types_str(), tokenize(), blender::gpu::shader::parser::Token::type(), and lexit::Token::type().