16#if defined(__clang__) || defined(__GNUC__)
17# define count_bits_i(i) __builtin_popcount(i)
18#elif defined(_MSC_VER)
19# define count_bits_i(i) __popcnt(i)
22# define count_bits_i(i) (std::bitset<8>{i}.count())
31 _BitScanForward64(&ctz, a);
34 return __builtin_ctzll(a);
40 return (a +
b - 1) /
b;
47 assert(new_size >= elem_count);
50 std::memcpy(new_ptr.
get(), ptr.
get(), elem_count *
sizeof(
T));
52 ptr = std::move(new_ptr);
73#if defined(USE_NEON) || defined(USE_SSE4_2)
80alignas(16)
static const uint8_t shuffle_table_8[256][8] = {
81 {0, 0, 0, 0, 0, 0, 0, 0},
82 {0, 0, 0, 0, 0, 0, 0, 0},
83 {1, 0, 0, 0, 0, 0, 0, 0},
84 {0, 1, 0, 0, 0, 0, 0, 0},
85 {2, 0, 0, 0, 0, 0, 0, 0},
86 {0, 2, 0, 0, 0, 0, 0, 0},
87 {1, 2, 0, 0, 0, 0, 0, 0},
88 {0, 1, 2, 0, 0, 0, 0, 0},
89 {3, 0, 0, 0, 0, 0, 0, 0},
90 {0, 3, 0, 0, 0, 0, 0, 0},
91 {1, 3, 0, 0, 0, 0, 0, 0},
92 {0, 1, 3, 0, 0, 0, 0, 0},
93 {2, 3, 0, 0, 0, 0, 0, 0},
94 {0, 2, 3, 0, 0, 0, 0, 0},
95 {1, 2, 3, 0, 0, 0, 0, 0},
96 {0, 1, 2, 3, 0, 0, 0, 0},
97 {4, 0, 0, 0, 0, 0, 0, 0},
98 {0, 4, 0, 0, 0, 0, 0, 0},
99 {1, 4, 0, 0, 0, 0, 0, 0},
100 {0, 1, 4, 0, 0, 0, 0, 0},
101 {2, 4, 0, 0, 0, 0, 0, 0},
102 {0, 2, 4, 0, 0, 0, 0, 0},
103 {1, 2, 4, 0, 0, 0, 0, 0},
104 {0, 1, 2, 4, 0, 0, 0, 0},
105 {3, 4, 0, 0, 0, 0, 0, 0},
106 {0, 3, 4, 0, 0, 0, 0, 0},
107 {1, 3, 4, 0, 0, 0, 0, 0},
108 {0, 1, 3, 4, 0, 0, 0, 0},
109 {2, 3, 4, 0, 0, 0, 0, 0},
110 {0, 2, 3, 4, 0, 0, 0, 0},
111 {1, 2, 3, 4, 0, 0, 0, 0},
112 {0, 1, 2, 3, 4, 0, 0, 0},
113 {5, 0, 0, 0, 0, 0, 0, 0},
114 {0, 5, 0, 0, 0, 0, 0, 0},
115 {1, 5, 0, 0, 0, 0, 0, 0},
116 {0, 1, 5, 0, 0, 0, 0, 0},
117 {2, 5, 0, 0, 0, 0, 0, 0},
118 {0, 2, 5, 0, 0, 0, 0, 0},
119 {1, 2, 5, 0, 0, 0, 0, 0},
120 {0, 1, 2, 5, 0, 0, 0, 0},
121 {3, 5, 0, 0, 0, 0, 0, 0},
122 {0, 3, 5, 0, 0, 0, 0, 0},
123 {1, 3, 5, 0, 0, 0, 0, 0},
124 {0, 1, 3, 5, 0, 0, 0, 0},
125 {2, 3, 5, 0, 0, 0, 0, 0},
126 {0, 2, 3, 5, 0, 0, 0, 0},
127 {1, 2, 3, 5, 0, 0, 0, 0},
128 {0, 1, 2, 3, 5, 0, 0, 0},
129 {4, 5, 0, 0, 0, 0, 0, 0},
130 {0, 4, 5, 0, 0, 0, 0, 0},
131 {1, 4, 5, 0, 0, 0, 0, 0},
132 {0, 1, 4, 5, 0, 0, 0, 0},
133 {2, 4, 5, 0, 0, 0, 0, 0},
134 {0, 2, 4, 5, 0, 0, 0, 0},
135 {1, 2, 4, 5, 0, 0, 0, 0},
136 {0, 1, 2, 4, 5, 0, 0, 0},
137 {3, 4, 5, 0, 0, 0, 0, 0},
138 {0, 3, 4, 5, 0, 0, 0, 0},
139 {1, 3, 4, 5, 0, 0, 0, 0},
140 {0, 1, 3, 4, 5, 0, 0, 0},
141 {2, 3, 4, 5, 0, 0, 0, 0},
142 {0, 2, 3, 4, 5, 0, 0, 0},
143 {1, 2, 3, 4, 5, 0, 0, 0},
144 {0, 1, 2, 3, 4, 5, 0, 0},
145 {6, 0, 0, 0, 0, 0, 0, 0},
146 {0, 6, 0, 0, 0, 0, 0, 0},
147 {1, 6, 0, 0, 0, 0, 0, 0},
148 {0, 1, 6, 0, 0, 0, 0, 0},
149 {2, 6, 0, 0, 0, 0, 0, 0},
150 {0, 2, 6, 0, 0, 0, 0, 0},
151 {1, 2, 6, 0, 0, 0, 0, 0},
152 {0, 1, 2, 6, 0, 0, 0, 0},
153 {3, 6, 0, 0, 0, 0, 0, 0},
154 {0, 3, 6, 0, 0, 0, 0, 0},
155 {1, 3, 6, 0, 0, 0, 0, 0},
156 {0, 1, 3, 6, 0, 0, 0, 0},
157 {2, 3, 6, 0, 0, 0, 0, 0},
158 {0, 2, 3, 6, 0, 0, 0, 0},
159 {1, 2, 3, 6, 0, 0, 0, 0},
160 {0, 1, 2, 3, 6, 0, 0, 0},
161 {4, 6, 0, 0, 0, 0, 0, 0},
162 {0, 4, 6, 0, 0, 0, 0, 0},
163 {1, 4, 6, 0, 0, 0, 0, 0},
164 {0, 1, 4, 6, 0, 0, 0, 0},
165 {2, 4, 6, 0, 0, 0, 0, 0},
166 {0, 2, 4, 6, 0, 0, 0, 0},
167 {1, 2, 4, 6, 0, 0, 0, 0},
168 {0, 1, 2, 4, 6, 0, 0, 0},
169 {3, 4, 6, 0, 0, 0, 0, 0},
170 {0, 3, 4, 6, 0, 0, 0, 0},
171 {1, 3, 4, 6, 0, 0, 0, 0},
172 {0, 1, 3, 4, 6, 0, 0, 0},
173 {2, 3, 4, 6, 0, 0, 0, 0},
174 {0, 2, 3, 4, 6, 0, 0, 0},
175 {1, 2, 3, 4, 6, 0, 0, 0},
176 {0, 1, 2, 3, 4, 6, 0, 0},
177 {5, 6, 0, 0, 0, 0, 0, 0},
178 {0, 5, 6, 0, 0, 0, 0, 0},
179 {1, 5, 6, 0, 0, 0, 0, 0},
180 {0, 1, 5, 6, 0, 0, 0, 0},
181 {2, 5, 6, 0, 0, 0, 0, 0},
182 {0, 2, 5, 6, 0, 0, 0, 0},
183 {1, 2, 5, 6, 0, 0, 0, 0},
184 {0, 1, 2, 5, 6, 0, 0, 0},
185 {3, 5, 6, 0, 0, 0, 0, 0},
186 {0, 3, 5, 6, 0, 0, 0, 0},
187 {1, 3, 5, 6, 0, 0, 0, 0},
188 {0, 1, 3, 5, 6, 0, 0, 0},
189 {2, 3, 5, 6, 0, 0, 0, 0},
190 {0, 2, 3, 5, 6, 0, 0, 0},
191 {1, 2, 3, 5, 6, 0, 0, 0},
192 {0, 1, 2, 3, 5, 6, 0, 0},
193 {4, 5, 6, 0, 0, 0, 0, 0},
194 {0, 4, 5, 6, 0, 0, 0, 0},
195 {1, 4, 5, 6, 0, 0, 0, 0},
196 {0, 1, 4, 5, 6, 0, 0, 0},
197 {2, 4, 5, 6, 0, 0, 0, 0},
198 {0, 2, 4, 5, 6, 0, 0, 0},
199 {1, 2, 4, 5, 6, 0, 0, 0},
200 {0, 1, 2, 4, 5, 6, 0, 0},
201 {3, 4, 5, 6, 0, 0, 0, 0},
202 {0, 3, 4, 5, 6, 0, 0, 0},
203 {1, 3, 4, 5, 6, 0, 0, 0},
204 {0, 1, 3, 4, 5, 6, 0, 0},
205 {2, 3, 4, 5, 6, 0, 0, 0},
206 {0, 2, 3, 4, 5, 6, 0, 0},
207 {1, 2, 3, 4, 5, 6, 0, 0},
208 {0, 1, 2, 3, 4, 5, 6, 0},
209 {7, 0, 0, 0, 0, 0, 0, 0},
210 {0, 7, 0, 0, 0, 0, 0, 0},
211 {1, 7, 0, 0, 0, 0, 0, 0},
212 {0, 1, 7, 0, 0, 0, 0, 0},
213 {2, 7, 0, 0, 0, 0, 0, 0},
214 {0, 2, 7, 0, 0, 0, 0, 0},
215 {1, 2, 7, 0, 0, 0, 0, 0},
216 {0, 1, 2, 7, 0, 0, 0, 0},
217 {3, 7, 0, 0, 0, 0, 0, 0},
218 {0, 3, 7, 0, 0, 0, 0, 0},
219 {1, 3, 7, 0, 0, 0, 0, 0},
220 {0, 1, 3, 7, 0, 0, 0, 0},
221 {2, 3, 7, 0, 0, 0, 0, 0},
222 {0, 2, 3, 7, 0, 0, 0, 0},
223 {1, 2, 3, 7, 0, 0, 0, 0},
224 {0, 1, 2, 3, 7, 0, 0, 0},
225 {4, 7, 0, 0, 0, 0, 0, 0},
226 {0, 4, 7, 0, 0, 0, 0, 0},
227 {1, 4, 7, 0, 0, 0, 0, 0},
228 {0, 1, 4, 7, 0, 0, 0, 0},
229 {2, 4, 7, 0, 0, 0, 0, 0},
230 {0, 2, 4, 7, 0, 0, 0, 0},
231 {1, 2, 4, 7, 0, 0, 0, 0},
232 {0, 1, 2, 4, 7, 0, 0, 0},
233 {3, 4, 7, 0, 0, 0, 0, 0},
234 {0, 3, 4, 7, 0, 0, 0, 0},
235 {1, 3, 4, 7, 0, 0, 0, 0},
236 {0, 1, 3, 4, 7, 0, 0, 0},
237 {2, 3, 4, 7, 0, 0, 0, 0},
238 {0, 2, 3, 4, 7, 0, 0, 0},
239 {1, 2, 3, 4, 7, 0, 0, 0},
240 {0, 1, 2, 3, 4, 7, 0, 0},
241 {5, 7, 0, 0, 0, 0, 0, 0},
242 {0, 5, 7, 0, 0, 0, 0, 0},
243 {1, 5, 7, 0, 0, 0, 0, 0},
244 {0, 1, 5, 7, 0, 0, 0, 0},
245 {2, 5, 7, 0, 0, 0, 0, 0},
246 {0, 2, 5, 7, 0, 0, 0, 0},
247 {1, 2, 5, 7, 0, 0, 0, 0},
248 {0, 1, 2, 5, 7, 0, 0, 0},
249 {3, 5, 7, 0, 0, 0, 0, 0},
250 {0, 3, 5, 7, 0, 0, 0, 0},
251 {1, 3, 5, 7, 0, 0, 0, 0},
252 {0, 1, 3, 5, 7, 0, 0, 0},
253 {2, 3, 5, 7, 0, 0, 0, 0},
254 {0, 2, 3, 5, 7, 0, 0, 0},
255 {1, 2, 3, 5, 7, 0, 0, 0},
256 {0, 1, 2, 3, 5, 7, 0, 0},
257 {4, 5, 7, 0, 0, 0, 0, 0},
258 {0, 4, 5, 7, 0, 0, 0, 0},
259 {1, 4, 5, 7, 0, 0, 0, 0},
260 {0, 1, 4, 5, 7, 0, 0, 0},
261 {2, 4, 5, 7, 0, 0, 0, 0},
262 {0, 2, 4, 5, 7, 0, 0, 0},
263 {1, 2, 4, 5, 7, 0, 0, 0},
264 {0, 1, 2, 4, 5, 7, 0, 0},
265 {3, 4, 5, 7, 0, 0, 0, 0},
266 {0, 3, 4, 5, 7, 0, 0, 0},
267 {1, 3, 4, 5, 7, 0, 0, 0},
268 {0, 1, 3, 4, 5, 7, 0, 0},
269 {2, 3, 4, 5, 7, 0, 0, 0},
270 {0, 2, 3, 4, 5, 7, 0, 0},
271 {1, 2, 3, 4, 5, 7, 0, 0},
272 {0, 1, 2, 3, 4, 5, 7, 0},
273 {6, 7, 0, 0, 0, 0, 0, 0},
274 {0, 6, 7, 0, 0, 0, 0, 0},
275 {1, 6, 7, 0, 0, 0, 0, 0},
276 {0, 1, 6, 7, 0, 0, 0, 0},
277 {2, 6, 7, 0, 0, 0, 0, 0},
278 {0, 2, 6, 7, 0, 0, 0, 0},
279 {1, 2, 6, 7, 0, 0, 0, 0},
280 {0, 1, 2, 6, 7, 0, 0, 0},
281 {3, 6, 7, 0, 0, 0, 0, 0},
282 {0, 3, 6, 7, 0, 0, 0, 0},
283 {1, 3, 6, 7, 0, 0, 0, 0},
284 {0, 1, 3, 6, 7, 0, 0, 0},
285 {2, 3, 6, 7, 0, 0, 0, 0},
286 {0, 2, 3, 6, 7, 0, 0, 0},
287 {1, 2, 3, 6, 7, 0, 0, 0},
288 {0, 1, 2, 3, 6, 7, 0, 0},
289 {4, 6, 7, 0, 0, 0, 0, 0},
290 {0, 4, 6, 7, 0, 0, 0, 0},
291 {1, 4, 6, 7, 0, 0, 0, 0},
292 {0, 1, 4, 6, 7, 0, 0, 0},
293 {2, 4, 6, 7, 0, 0, 0, 0},
294 {0, 2, 4, 6, 7, 0, 0, 0},
295 {1, 2, 4, 6, 7, 0, 0, 0},
296 {0, 1, 2, 4, 6, 7, 0, 0},
297 {3, 4, 6, 7, 0, 0, 0, 0},
298 {0, 3, 4, 6, 7, 0, 0, 0},
299 {1, 3, 4, 6, 7, 0, 0, 0},
300 {0, 1, 3, 4, 6, 7, 0, 0},
301 {2, 3, 4, 6, 7, 0, 0, 0},
302 {0, 2, 3, 4, 6, 7, 0, 0},
303 {1, 2, 3, 4, 6, 7, 0, 0},
304 {0, 1, 2, 3, 4, 6, 7, 0},
305 {5, 6, 7, 0, 0, 0, 0, 0},
306 {0, 5, 6, 7, 0, 0, 0, 0},
307 {1, 5, 6, 7, 0, 0, 0, 0},
308 {0, 1, 5, 6, 7, 0, 0, 0},
309 {2, 5, 6, 7, 0, 0, 0, 0},
310 {0, 2, 5, 6, 7, 0, 0, 0},
311 {1, 2, 5, 6, 7, 0, 0, 0},
312 {0, 1, 2, 5, 6, 7, 0, 0},
313 {3, 5, 6, 7, 0, 0, 0, 0},
314 {0, 3, 5, 6, 7, 0, 0, 0},
315 {1, 3, 5, 6, 7, 0, 0, 0},
316 {0, 1, 3, 5, 6, 7, 0, 0},
317 {2, 3, 5, 6, 7, 0, 0, 0},
318 {0, 2, 3, 5, 6, 7, 0, 0},
319 {1, 2, 3, 5, 6, 7, 0, 0},
320 {0, 1, 2, 3, 5, 6, 7, 0},
321 {4, 5, 6, 7, 0, 0, 0, 0},
322 {0, 4, 5, 6, 7, 0, 0, 0},
323 {1, 4, 5, 6, 7, 0, 0, 0},
324 {0, 1, 4, 5, 6, 7, 0, 0},
325 {2, 4, 5, 6, 7, 0, 0, 0},
326 {0, 2, 4, 5, 6, 7, 0, 0},
327 {1, 2, 4, 5, 6, 7, 0, 0},
328 {0, 1, 2, 4, 5, 6, 7, 0},
329 {3, 4, 5, 6, 7, 0, 0, 0},
330 {0, 3, 4, 5, 6, 7, 0, 0},
331 {1, 3, 4, 5, 6, 7, 0, 0},
332 {0, 1, 3, 4, 5, 6, 7, 0},
333 {2, 3, 4, 5, 6, 7, 0, 0},
334 {0, 2, 3, 4, 5, 6, 7, 0},
335 {1, 2, 3, 4, 5, 6, 7, 0},
336 {0, 1, 2, 3, 4, 5, 6, 7},
340alignas(16)
static const uint8_t mask_popcount[256] = {
341 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3,
342 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4,
343 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4,
344 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5,
345 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2,
346 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5,
347 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4,
348 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6,
349 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8};
351template<
int Size>
struct ShuffleIndicesResult {
352 simd::u8_base<Size> indices;
356inline ShuffleIndicesResult<1> shuffle_indices_from_emit_mask(uint16_t emit_mask)
358 const uint8_t emit_mask_lo = emit_mask & 0xFFu;
359 const uint8_t emit_mask_hi = emit_mask >> 8;
360 const uint8_t mask_popcount_lo = mask_popcount[emit_mask_lo];
361 const uint8_t mask_popcount_hi = mask_popcount[emit_mask_hi];
367 alignas(16) uint8_t combined[16];
369 *(
uint64_t *)(combined + mask_popcount_lo) = v1;
371 return {simd::u8x16::load((
const uint8_t *)&combined), mask_popcount_lo + mask_popcount_hi};
374inline ShuffleIndicesResult<4> shuffle_indices_from_emit_mask(
uint64_t emit_mask)
376 const uint8_t emit_mask_0 = (emit_mask >> 0) & 0xFFu;
377 const uint8_t emit_mask_1 = (emit_mask >> 8) & 0xFFu;
378 const uint8_t emit_mask_2 = (emit_mask >> 16) & 0xFFu;
379 const uint8_t emit_mask_3 = (emit_mask >> 24) & 0xFFu;
380 const uint8_t emit_mask_4 = (emit_mask >> 32) & 0xFFu;
381 const uint8_t emit_mask_5 = (emit_mask >> 40) & 0xFFu;
382 const uint8_t emit_mask_6 = (emit_mask >> 48) & 0xFFu;
383 const uint8_t emit_mask_7 = (emit_mask >> 56) & 0xFFu;
384 const uint8_t mask_popcount_0 = mask_popcount[emit_mask_0];
385 const uint8_t mask_popcount_1 = mask_popcount[emit_mask_1];
386 const uint8_t mask_popcount_2 = mask_popcount[emit_mask_2];
387 const uint8_t mask_popcount_3 = mask_popcount[emit_mask_3];
388 const uint8_t mask_popcount_4 = mask_popcount[emit_mask_4];
389 const uint8_t mask_popcount_5 = mask_popcount[emit_mask_5];
390 const uint8_t mask_popcount_6 = mask_popcount[emit_mask_6];
391 const uint8_t mask_popcount_7 = mask_popcount[emit_mask_7];
403 alignas(64) uint8_t combined[64];
414 return {simd::u8x64::load((
const uint8_t *)&combined),
popcount};
420 return TokenType((cond) ? char_class : char_value);
423inline void TokenBuffer::tokenize_scalar(uint32_t &__restrict offset,
424 uint32_t &__restrict cursor_begin,
425 uint32_t &__restrict cursor_end,
427 bool &__restrict prev_whitespace,
431 for (; offset <
end; offset += 1) {
432 const char c =
str_[offset];
438 types_[cursor_begin] = curr_tok_type;
447 const bool emit = (uint8_t(curr_char_class) & uint8_t(prev_char_class) &
449 prev_char_class = curr_char_class;
460 const bool emit_ws = emit && curr_ws;
462 const bool emit_start = emit && !curr_ws;
464 const bool follow_non_ws = emit_start && !prev_whitespace;
466 const bool emit_end = emit_ws || follow_non_ws;
468 prev_whitespace = curr_ws;
472 int start =
offsets_[cursor_begin - 1];
474 token_str_with_whitespace_debug_.emplace_back(
str_.data() + start,
end - start);
479 token_str_debug_.emplace_back(
str_.data() + start,
end - start);
483 cursor_begin += emit_start;
484 cursor_end += emit_end;
524 bool prev_whitespace;
535 uint32_t offset = 1, cursor = 1, cursor_end = prev_whitespace ? 1 : 0;
540 size_t index_at_align = ((stride - (uintptr_t(
str_.data() + 1) & (stride - 1))) & (stride - 1)) +
542 tokenize_scalar(offset,
547 str_.size() > index_at_align ? index_at_align :
str_.size(),
550#if defined(USE_NEON) || defined(USE_SSE4_2)
551 using namespace lexit::simd;
552 const uint8_t *
str = (
const uint8_t *)
str_.data();
553 const u8x128_table char_to_class = u8x128_table::load((
const uint8_t *)
char_class_table);
555 for (; offset + stride <=
str_.size(); offset += stride) {
557 const std::string_view char_simd{
str_.data() + offset, size_t(stride)};
559 const u8x64 c = u8x64::load(
str + offset);
560 const u8x64 curr_char_class = char_to_class[c];
562 const u8x64 prev_char_class = shift_lanes_right<1>(curr_char_class, uint8_t(prev_value));
566 const u8x64 curr_tok_type =
select(c, curr_char_class, curr_char_class > to_type_threshold);
567 const u8x64 emit =
is_zero(curr_char_class & prev_char_class & can_merge);
569 prev_value =
CharClass(curr_char_class.last());
572 const u8x64 curr_non_ws = ~curr_ws;
573 const u8x64 emit_ws = emit & curr_ws;
574 const u8x64 emit_start = emit & curr_non_ws;
575 const u8x64 prev_non_ws = shift_lanes_right<1>(curr_non_ws, prev_whitespace ? 0x0 : 0xFF);
576 const u8x64 follow_non_ws = emit_start & prev_non_ws;
577 const u8x64 emit_end = emit_ws | follow_non_ws;
579 prev_whitespace = !curr_non_ws.last();
580 uint64_t emit_end_mask = movemask(emit_end);
581 uint64_t emit_start_mask = movemask(emit_start);
587 auto [shuffle,
popcount] = shuffle_indices_from_emit_mask(emit_start_mask);
589 const u8x64 data_packed = u8x64_table(curr_tok_type)[shuffle];
591 data_packed.store_unaligned((uint8_t *)&
types_[cursor]);
594 (u32x16(shuffle.lane(0)) + offset).store_unaligned(&
offsets_[cursor + 0]);
595 (u32x16(shuffle.lane(1)) + offset).store_unaligned(&
offsets_[cursor + 16]);
596 (u32x16(shuffle.lane(2)) + offset).store_unaligned(&
offsets_[cursor + 32]);
597 (u32x16(shuffle.lane(3)) + offset).store_unaligned(&
offsets_[cursor + 48]);
599 for (
int i = cursor; i < cursor +
popcount; i++) {
602 token_str_with_whitespace_debug_.emplace_back(
str_.data() + start,
end - start);
608 auto [shuffle,
popcount] = shuffle_indices_from_emit_mask(emit_end_mask);
611 (u32x16(shuffle.lane(0)) + offset).store_unaligned(&
offsets_end_[cursor_end + 0]);
612 (u32x16(shuffle.lane(1)) + offset).store_unaligned(&
offsets_end_[cursor_end + 16]);
613 (u32x16(shuffle.lane(2)) + offset).store_unaligned(&
offsets_end_[cursor_end + 32]);
614 (u32x16(shuffle.lane(3)) + offset).store_unaligned(&
offsets_end_[cursor_end + 48]);
616 for (
int i = cursor_end; i < cursor_end +
popcount; i++) {
619 token_str_debug_.emplace_back(
str_.data() + start,
end - start);
624 assert(cursor_end == cursor || cursor_end + 1 == cursor);
628 assert(cursor_end == cursor || cursor_end + 1 == cursor);
634 assert(cursor_end == cursor || cursor_end + 1 == cursor);
647#if defined(USE_NEON) || defined(USE_SSE4_2)
648 using namespace simd;
649 static constexpr int stride = 64;
650 for (; tok_id + stride <=
size_; tok_id += stride) {
651 const u32x64 str_start = u32x64::load(&
offsets_[tok_id]);
652 const u32x64 str_end = u32x64::load(&
offsets_end_[tok_id]);
653 const u32x64 str_size_32 = str_end - str_start;
654 const u8x64 str_large = u8x64(str_size_32 > 127);
656 const u8x64 str_size =
select(u8x64(str_size_32), u8x64(127), str_large);
661 for (; tok_id <
size_; ++tok_id) {
662 const uint32_t str_start =
offsets_[tok_id];
664 const uint32_t str_size_32 = str_end - str_start;
666 const uint8_t str_size =
select(uint8_t(str_size_32), 127, str_size_32 > 127);
673 static constexpr int stride = 64;
676 uint64_t mask8, mask16, mask24, mask32;
686 uint32_t chunk_id = 0;
688#if defined(USE_NEON) || defined(USE_SSE4_2)
689 using namespace simd;
690 for (; tok_id + stride <=
size_; tok_id += stride, ++chunk_id) {
692 const u8x64 type = u8x64::load((uint8_t *)&
types_[tok_id]);
700 small.mask8 = is_word & less_8;
701 small.mask16 = is_word & less_16 & ~less_8;
702 small.mask24 = is_word & less_24 & ~less_16;
703 small.mask32 = is_word & less_32 & ~less_24;
704 masks_small_id[chunk_id] = small;
705 masks_large_id[chunk_id] = is_word & ~less_32;
708 for (; tok_id <
size_; tok_id += stride, ++chunk_id) {
709 uint64_t is_word = 0, less_8 = 0, less_16 = 0, less_24 = 0, less_32 = 0;
711 for (
int i = 0; i < stride && tok_id + i <
size_; ++i) {
724 small.mask8 = is_word & less_8;
725 small.mask16 = is_word & less_16 & ~less_8;
726 small.mask24 = is_word & less_24 & ~less_16;
727 small.mask32 = is_word & less_32 & ~less_24;
728 masks_small_id[chunk_id] = small;
729 masks_large_id[chunk_id] = is_word & ~less_32;
739 for (; chunk < end_safe; ++chunk) {
740 const Masks small = masks_small_id[chunk];
741 atomize_short_tokens_in_mask<1>(small.mask8, chunk * stride, identifiers, keywords);
742 atomize_short_tokens_in_mask<2>(small.mask16, chunk * stride, identifiers, keywords);
743 atomize_short_tokens_in_mask<3>(small.mask24, chunk * stride, identifiers, keywords);
744 atomize_short_tokens_in_mask<4>(small.mask32, chunk * stride, identifiers, keywords);
745 atomize_tokens_in_mask(masks_large_id[chunk], chunk * stride, identifiers, keywords);
747 for (; chunk <
end; ++chunk) {
748 const Masks small = masks_small_id[chunk];
749 atomize_tokens_in_mask(small.mask8, chunk * stride, identifiers, keywords);
750 atomize_tokens_in_mask(small.mask16, chunk * stride, identifiers, keywords);
751 atomize_tokens_in_mask(small.mask24, chunk * stride, identifiers, keywords);
752 atomize_tokens_in_mask(small.mask32, chunk * stride, identifiers, keywords);
753 atomize_tokens_in_mask(masks_large_id[chunk], chunk * stride, identifiers, keywords);
759 uint32_t tok_id_base,
763 if (
mask == 0) [[likely]] {
768 const int tok_id = tok_id_base +
index;
770 const int str_start =
offsets_[tok_id];
772 const std::string_view
str = {
str_.data() + str_start, size_t(str_size)};
776 types_[tok_id] = kw_table[atom];
784 uint32_t tok_id_base,
790 const int tok_id = tok_id_base +
index;
792 const int str_start =
offsets_[tok_id];
793 const int str_size =
lengths_[tok_id];
795 const std::string_view
str = {
str_.data() + str_start, size_t(str_size)};
796 const PaddedString<Size> padded_str(
str);
798 const TokenAtom atom = id_map.lookup_or_add(padded_str);
800 types_[tok_id] = kw_table[atom];
806template void TokenBuffer::atomize_short_tokens_in_mask<1>(
uint64_t,
810template void TokenBuffer::atomize_short_tokens_in_mask<2>(
uint64_t,
814template void TokenBuffer::atomize_short_tokens_in_mask<3>(
uint64_t,
818template void TokenBuffer::atomize_short_tokens_in_mask<4>(
uint64_t,
843 const uint32_t *offsets,
844 const uint32_t *offsets_end_,
848 const uint32_t *offset = offsets + cursor;
849 const uint32_t *offset_end_ = offsets_end_ + cursor;
852 const bool followed_by_whitespace = offset_end_[0] != offset[1];
853 if (followed_by_whitespace) {
864 if ((*type ==
'+' || *type ==
'-') &&
str[*offset - 1] !=
'e') {
867 if (!(*type ==
Word || *type ==
Number || *type ==
'.' || *type ==
'+' || *type ==
'-')) {
871 const bool followed_by_whitespace = offset_end_[0] != offset[1];
872 if (followed_by_whitespace) {
883 const uint32_t *offsets,
887 const uint32_t start = offsets[cursor];
889 const char c =
str[start + 1];
894 end_pos =
str.find(
'\n', start + 2);
898 end_pos =
str.find(
"*/", start + 2);
900 end_pos += (end_pos != std::string::npos) ? 1 : 0;
910 const uint32_t tok_start = offsets[cursor];
912 if (tok_start > end_pos) {
924 return (c ==
' ' || c ==
'\t' || c ==
'\n' || c ==
'\r' || c ==
'\v' || c ==
'\f');
929 const uint32_t *offsets,
930 const uint32_t *offsets_end,
935 if (
static_cast<const TokenType *
>(&out_type) == types) {
942 bool preceded_by_space =
false;
943 const uint32_t start = offsets[cursor];
949 if (out_type ==
LThan) {
950 if (!preceded_by_space && prev !=
AngleOpen) {
958 const bool not_followed_by_whitespace = offsets_end[cursor] == offsets[cursor + 1];
959 if (types[cursor + 1] ==
LThan && not_followed_by_whitespace) {
964 const bool not_followed_by_whitespace = offsets_end[cursor] == offsets[cursor + 1];
965 if (types[cursor + 1] ==
Assign && not_followed_by_whitespace) {
970 else if (out_type ==
GThan) {
971 if (!preceded_by_space && prev !=
Minus
982 uint32_t curr = offsets[cursor];
984 if (
str[curr] !=
'>') {
994 const bool not_followed_by_whitespace = offsets_end[cursor] == offsets[cursor + 1];
995 if (types[cursor + 1] ==
GThan && not_followed_by_whitespace) {
1000 const bool not_followed_by_whitespace = offsets_end[cursor] == offsets[cursor + 1];
1001 if (types[cursor + 1] ==
Assign && not_followed_by_whitespace) {
1019 for (uint32_t i = 0; i <
size_; i++, out_type++, out_offset++, out_offset_end++) {
1021 const uint32_t offset = in_offsets[i];
1022 const uint32_t offset_end = in_offset_end[i];
1024 *out_offset = offset;
1025 *out_offset_end = offset_end;
1039 if (bsl_template_angle_bracket_lexing) {
1040 lex_angle(
str_, in_types, in_offsets, in_offset_end, i, *out_type);
1047 *out_offset_end = in_offset_end[i];
1050 assert(in_types <= out_type);
1051 assert(out_type - in_types < 0xFFFFFFFFu);
1052 size_ = out_type - in_types;
unsigned long long int uint64_t
#define assert(assertion)
ccl_device_inline uint popcount(const uint x)
ccl_device_inline float2 mask(const MaskType mask, const float2 a)
ATTR_WARN_UNUSED_RESULT const BMVert * v2
static void lex_comment(const std::string_view str, const TokenType *types, const uint32_t *offsets, uint32_t &cursor, TokenType &out_type)
static void lex_float(const std::string_view str, const TokenType *types, const uint32_t *offsets, const uint32_t *offsets_end_, uint32_t &cursor)
void realloc_aligned_array(AlignedArrayPtr< T > &ptr, size_t elem_count, size_t new_size)
static void lex_angle(const std::string_view str, const TokenType *types, const uint32_t *offsets, const uint32_t *offsets_end, uint32_t &cursor, TokenType &out_type)
static uint32_t divide_ceil(uint32_t a, uint32_t b)
const CharClass char_class_table[128]
static void lex_string(const TokenType *types, uint32_t &cursor)
static bool is_whitespace(char c)
TokenType select(char char_value, char char_class, bool cond)
static int builtin_ctzll(uint64_t a)
INLINE_METHOD TokenAtom lookup_or_add(StringT str)
AlignedArrayPtr< uint32_t > offsets_
void merge_complex_literals(bool bsl_template_angle_bracket_lexing=false)
Merge complex literals such as floats, strings and comments.
void compute_lengths()
Compute small token length for speeding up certain tasks.
void atomize_words(IdentifierMap &identifiers, const KeywordTable &keywords)
Assign keyword types and atoms for a small set of identifier.
void clear()
Discard all currently held data. Does not reallocate.
uint32_t size() const
Return the amount of token inside the buffer.
AlignedArrayPtr< uint8_t > lengths_
void reserve(const uint32_t count)
Allocate backing memory for the given number of tokens and move currently held data.
AlignedArrayPtr< TokenType > types_
void tokenize(const CharClass char_class_table[128])
Tokenizes the input string by grouping contiguous characters of the same class.
AlignedArrayPtr< uint32_t > offsets_end_
AlignedArrayPtr< TokenAtom > atoms_
void resize(int new_size)