17# define INLINE_METHOD __forceinline
19# define INLINE_METHOD inline __attribute__((always_inline))
56 std::memcpy(
data, (
const char *)
str.data(),
sizeof(
data));
61 operator std::string_view()
const
102 return std::string_view{(
const char *)
data(),
size} ==
str;
130 static constexpr uint32_t
str_hash(std::string_view s)
132 uint32_t
hash = 5381;
137 return static_cast<uint16_t
>(
hash);
161 id->next = new_index;
176 id.size =
str.size();
180 std::memcpy(
id.data(),
str.data(),
str.size());
194 alignas(64) std::array<TokenType, 64>
map;
200 for (
auto keyword :
vector) {
202 assert(keyword.atom < 128);
204 assert(keyword.atom != 0);
205 map[keyword.atom / 2] = keyword.type;
215 return map[(atom / 2) * (atom < 128)];
unsigned long long int uint64_t
#define assert(assertion)
static constexpr uint64_t padded_string_masks[8]
bool operator==(std::string_view str) const
bool operator==(const PaddedString< Size > &str) const
static constexpr uint32_t str_hash(std::string_view s)
Keyword make_keyword(std::string_view str, TokenType type)
static constexpr uint32_t hash_table_size
TokenAtom add_after(uint32_t hash, std::string_view str, Identifier *id=nullptr)
static constexpr uint32_t hash_table_index_mask
lexit::Vector< Identifier > identifier_buffer
INLINE_METHOD TokenAtom lookup_or_add(StringT str)
std::array< uint16_t, hash_table_size > hash_table
TokenAtom max_atom_value() const
void reserve(int token_count)
TokenType operator[](TokenAtom atom) const
KeywordTable(const std::vector< Keyword > &vector)
std::array< TokenType, 64 > map
Keyword(std::string_view str, TokenType type, TokenAtom atom)
PaddedString(std::string_view str)
void increase_size_by_unchecked(int elem_count)
void reserve(int new_size)