Blender V5.3
lexit Namespace Reference

Classes

class  AlignedArrayPtr
struct  AlignedDeleter
struct  IdentifierMap
struct  Keyword
struct  KeywordTable
struct  PaddedString
struct  Token
struct  TokenBuffer
struct  TokenMut
struct  Vector

Typedefs

using TokenAtom = uint16_t

Enumerations

enum class  CharClass : uint8_t {
  None = 0 , Separator = (1 << 1) , MultiTok = (1 << 2) , WhiteSpace = (1 << 3) ,
  Alpha = 'A' , Numeric = '1' , CanMerge = Alpha | Numeric | MultiTok | WhiteSpace , ClassToTypeThreshold = Numeric - 1
}
enum  TokenType : uint8_t {
  Invalid = 0 , Word = TokenType(CharClass::Alpha) , Number = TokenType(CharClass::Numeric) , NewLine = '\n' ,
  Space = ' ' , Dot = '.' , Hash = '#' , Ampersand = '&' ,
  DoubleQuote = '"' , SingleQuote = '\'' , ParOpen = '(' , ParClose = ')' ,
  BracketOpen = '{' , BracketClose = '}' , SquareOpen = '[' , SquareClose = ']' ,
  AngleOpen = '<' , AngleClose = '>' , Assign = '=' , SemiColon = ';' ,
  Question = '?' , Not = '!' , Colon = ':' , Comma = ',' ,
  Star = '*' , Plus = '+' , Minus = '-' , Divide = '/' ,
  Tilde = '~' , Caret = '^' , Pipe = '|' , Percent = '' ,
  Backslash = '\\' , EndOfFile = '\0' , LogicalAnd = 'a' , Break = 'b' ,
  Typename = 'B' , Const = 'c' , Constexpr = 'C' , Do = 'd' ,
  Decrement = 'D' , NotEqual = 'e' , Equal = 'E' , For = 'f' ,
  While = 'F' , LogicalOr = 'g' , GEqual = 'G' , Switch = 'h' ,
  Case = 'H' , If = 'i' , Else = 'I' , Elif = 'j' ,
  Endif = 'J' , Ifdef = 'k' , Ifndef = 'K' , Inline = 'l' ,
  LEqual = 'L' , Static = 'm' , Enum = 'M' , Namespace = 'n' ,
  Define = 'N' , Union = 'o' , Continue = 'O' , Line = 'p' ,
  Increment = 'P' , Pragma = 'q' , DoubleHash = 'Q' , Return = 'r' ,
  Struct = 's' , Class = 'S' , Template = 't' , This = 'T' ,
  Using = 'u' , Undef = 'U' , Private = 'v' , Public = 'V' ,
  LShift = 'w' , RShift = 'W' , Default = 'y' , TemplateOpen = 'Y' ,
  TemplateClose = 'z' , Comment = 'Z' , AssignAdd = '0' , AssignSub = '2' ,
  AssignMul = '3' , AssignDiv = '4' , AssignLShift = '5' , AssignRShift = '6' ,
  Multiply = Star , And = Ampersand , Or = Pipe , Xor = Caret ,
  GThan = AngleClose , LThan = AngleOpen , BitwiseNot = Tilde , Modulo = Percent ,
  String = DoubleQuote , Slash = Divide
}

Functions

static int builtin_ctzll (uint64_t a)
static uint32_t divide_ceil (uint32_t a, uint32_t b)
template<typename T>
void realloc_aligned_array (AlignedArrayPtr< T > &ptr, size_t elem_count, size_t new_size)
TokenType select (char char_value, char char_class, bool cond)
static void lex_string (const TokenType *types, uint32_t &cursor)
static void lex_float (const std::string_view str, const TokenType *types, const uint32_t *offsets, const uint32_t *offsets_end_, uint32_t &cursor)
static void lex_comment (const std::string_view str, const TokenType *types, const uint32_t *offsets, uint32_t &cursor, TokenType &out_type)
static bool is_whitespace (char c)
static void lex_angle (const std::string_view str, const TokenType *types, const uint32_t *offsets, const uint32_t *offsets_end, uint32_t &cursor, TokenType &out_type)
std::ostream & operator<< (std::ostream &os, const Token &tok)
static std::string to_str (TokenType type)

Variables

static constexpr uint64_t padded_string_masks [8]
const CharClass char_class_table [128]

Typedef Documentation

◆ TokenAtom

using lexit::TokenAtom = uint16_t

Definition at line 159 of file types.hh.

Enumeration Type Documentation

◆ CharClass

enum class lexit::CharClass : uint8_t
strong

Class for each characters inside the ASCII table.

The tokenizer identifies runs of characters with similar classes. A character is grouped with its predecessor if it shares a class. The Separator class is the exception which never group chars together.

Note: The values were chosen to allow fast comparison, masking, and cast to printable TokenType.

Enumerator
None 
Separator 
MultiTok 
WhiteSpace 
Alpha 
Numeric 
CanMerge 
ClassToTypeThreshold 

Definition at line 20 of file types.hh.

◆ TokenType

enum lexit::TokenType : uint8_t
Enumerator
Invalid 
Word 
Number 
NewLine 
Space 
Dot 
Hash 
Ampersand 
DoubleQuote 
SingleQuote 
ParOpen 
ParClose 
BracketOpen 
BracketClose 
SquareOpen 
SquareClose 
AngleOpen 
AngleClose 
Assign 
SemiColon 
Question 
Not 
Colon 
Comma 
Star 
Plus 
Minus 
Divide 
Tilde 
Caret 
Pipe 
Percent 
Backslash 
EndOfFile 
LogicalAnd 
Break 
Typename 
Const 
Constexpr 
Do 
Decrement 
NotEqual 
Equal 
For 
While 
LogicalOr 
GEqual 
Switch 
Case 
If 
Else 
Elif 
Endif 
Ifdef 
Ifndef 
Inline 
LEqual 
Static 
Enum 
Namespace 
Define 
Union 
Continue 
Line 
Increment 
Pragma 
DoubleHash 
Return 
Struct 
Class 
Template 
This 
Using 
Undef 
Private 
Public 
LShift 
RShift 
Default 
TemplateOpen 
TemplateClose 
Comment 
AssignAdd 
AssignSub 
AssignMul 
AssignDiv 
AssignLShift 
AssignRShift 
Multiply 
And 
Or 
Xor 
GThan 
LThan 
BitwiseNot 
Modulo 
String 
Slash 

Definition at line 41 of file types.hh.

Function Documentation

◆ builtin_ctzll()

int lexit::builtin_ctzll ( uint64_t a)
static

Definition at line 27 of file lexit.cc.

References builtin_ctzll().

Referenced by builtin_ctzll().

◆ divide_ceil()

uint32_t lexit::divide_ceil ( uint32_t a,
uint32_t b )
static

Definition at line 38 of file lexit.cc.

References b, and divide_ceil().

Referenced by lexit::TokenBuffer::atomize_words(), and divide_ceil().

◆ is_whitespace()

bool lexit::is_whitespace ( char c)
static

Definition at line 922 of file lexit.cc.

References is_whitespace().

Referenced by is_whitespace(), and lex_angle().

◆ lex_angle()

void lexit::lex_angle ( const std::string_view str,
const TokenType * types,
const uint32_t * offsets,
const uint32_t * offsets_end,
uint32_t & cursor,
TokenType & out_type )
static

◆ lex_comment()

void lexit::lex_comment ( const std::string_view str,
const TokenType * types,
const uint32_t * offsets,
uint32_t & cursor,
TokenType & out_type )
static

Definition at line 881 of file lexit.cc.

References Comment, EndOfFile, lex_comment(), and str.

Referenced by lex_comment(), and lexit::TokenBuffer::merge_complex_literals().

◆ lex_float()

void lexit::lex_float ( const std::string_view str,
const TokenType * types,
const uint32_t * offsets,
const uint32_t * offsets_end_,
uint32_t & cursor )
static

Definition at line 841 of file lexit.cc.

References lex_float(), Number, str, and Word.

Referenced by lex_float(), and lexit::TokenBuffer::merge_complex_literals().

◆ lex_string()

void lexit::lex_string ( const TokenType * types,
uint32_t & cursor )
static

Definition at line 823 of file lexit.cc.

References EndOfFile, lex_string(), and String.

Referenced by lex_string(), and lexit::TokenBuffer::merge_complex_literals().

◆ operator<<()

std::ostream & lexit::operator<< ( std::ostream & os,
const Token & tok )
inline

◆ realloc_aligned_array()

template<typename T>
void lexit::realloc_aligned_array ( AlignedArrayPtr< T > & ptr,
size_t elem_count,
size_t new_size )

◆ select()

◆ to_str()

Variable Documentation

◆ char_class_table

◆ padded_string_masks

uint64_t lexit::padded_string_masks[8]
staticconstexpr
Initial value:
= {
uint64_t(0xFFFFFFFFFFFFFFFF),
uint64_t(0x00000000000000FF),
uint64_t(0x000000000000FFFF),
uint64_t(0x0000000000FFFFFF),
uint64_t(0x00000000FFFFFFFF),
uint64_t(0x000000FFFFFFFFFF),
uint64_t(0x0000FFFFFFFFFFFF),
uint64_t(0x00FFFFFFFFFFFFFF),
}
unsigned long long int uint64_t

Definition at line 24 of file identifier.hh.

Referenced by lexit::PaddedString< Size >::PaddedString().