Blender V5.3
string_utf8.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 1999 Tom Tromey
2 * SPDX-FileCopyrightText: 2000 Red Hat, Inc. All rights reserved.
3 * SPDX-FileCopyrightText: 2011 Blender Authors
4 *
5 * SPDX-License-Identifier: GPL-2.0-or-later
6 *
7 * Code from `gutf8.c` by Tom Tromey & Red Hat, Inc. */
8
12
13#include <algorithm>
14#include <cstdio>
15#include <cstdlib>
16#include <cstring>
17#include <cwchar>
18#include <cwctype>
19#include <wcwidth.h>
20
21#include "BLI_utildefines.hh"
22
23#include "BLI_string.hh" /* #BLI_string_debug_size. */
24#include "BLI_string_utf8.hh" /* own include */
25#ifdef WIN32
26# include "utfconv.hh"
27#endif
28#ifdef __GNUC__
29# pragma GCC diagnostic error "-Wsign-conversion"
30#endif
31
32#include "BLI_strict_flags.hh" /* IWYU pragma: keep. Keep last. */
33
34namespace blender {
35
36static size_t str_utf8_truncate_at_size_unchecked(char *str, const size_t str_size);
37
38/* -------------------------------------------------------------------- */
59
61{
62 if (c >= 192) [[unlikely]] {
63 if ((c & 0xe0) == 0xc0) {
64 return 2;
65 }
66 if ((c & 0xf0) == 0xe0) {
67 return 3;
68 }
69 if ((c & 0xf8) == 0xf0) {
70 return 4;
71 }
72 if ((c & 0xfc) == 0xf8) {
73 return 5;
74 }
75 if ((c & 0xfe) == 0xfc) {
76 return 6;
77 }
78 }
79 return 1;
80}
81
83{
84 if (c < 128) {
85 return 1;
86 }
87 if ((c & 0xe0) == 0xc0) {
88 return 2;
89 }
90 if ((c & 0xf0) == 0xe0) {
91 return 3;
92 }
93 if ((c & 0xf8) == 0xf0) {
94 return 4;
95 }
96 if ((c & 0xfc) == 0xf8) {
97 return 5;
98 }
99 if ((c & 0xfe) == 0xfc) {
100 return 6;
101 }
102 return -1;
103}
104
106{
107 /* Originally from GLIB `UTF8_COMPUTE` macro. */
108 if (c < 128) {
109 *r_mask = 0x7f;
110 return 1;
111 }
112 if ((c & 0xe0) == 0xc0) {
113 *r_mask = 0x1f;
114 return 2;
115 }
116 if ((c & 0xf0) == 0xe0) {
117 *r_mask = 0x0f;
118 return 3;
119 }
120 if ((c & 0xf8) == 0xf0) {
121 *r_mask = 0x07;
122 return 4;
123 }
124 if ((c & 0xfc) == 0xf8) {
125 *r_mask = 0x03;
126 return 5;
127 }
128 if ((c & 0xfe) == 0xfc) {
129 *r_mask = 0x01;
130 return 6;
131 }
132 return -1;
133}
134
138BLI_INLINE uint utf8_char_decode(const char *p, const char mask, const int len, const uint err)
139{
140 /* Originally from GLIB `UTF8_GET` macro, added an 'err' argument. */
141 uint result = p[0] & mask;
142 for (int count = 1; count < len; count++) {
143 if ((p[count] & 0xc0) != 0x80) {
144 return err;
145 }
146 result <<= 6;
147 result |= p[count] & 0x3f;
148 }
149 return result;
150}
151
153
154ptrdiff_t BLI_str_utf8_invalid_byte(const char *str, size_t str_len)
155{
156 /* NOTE(@ideasman42): from libswish3, originally called `u8_isvalid()`,
157 * modified to return the index of the bad character (byte index not UTF).
158 * http://svn.swish-e.org/libswish3/trunk/src/libswish3/utf8.c r3044.
159 *
160 * Comment from code in: `libswish3`.
161 * Based on the `valid_utf8` routine from the PCRE library by Philip Hazel
162 *
163 * length is in bytes, since without knowing whether the string is valid
164 * it's hard to know how many characters there are! */
165
166 const uchar *p, *perr, *pend = reinterpret_cast<const uchar *>(str) + str_len;
167 uchar c;
168 int ab;
169
170 for (p = reinterpret_cast<const uchar *>(str); p < pend; p++, str_len--) {
171 c = *p;
172 perr = p; /* Erroneous char is always the first of an invalid UTF8 sequence... */
173 if (ELEM(c, 0xfe, 0xff, 0x00)) {
174 /* Those three values are not allowed in UTF8 string. */
175 goto utf8_error;
176 }
177 if (c < 128) {
178 continue;
179 }
180 if ((c & 0xc0) != 0xc0) {
181 goto utf8_error;
182 }
183
184 /* Note that since we always increase p (and decrease length) by one byte in main loop,
185 * we only add/subtract extra UTF8 bytes in code below
186 * (ab number, aka number of bytes remaining in the UTF8 sequence after the initial one). */
187 ab = utf8_char_compute_skip(c) - 1;
188 if (str_len <= size_t(ab)) {
189 goto utf8_error;
190 }
191
192 /* Check top bits in the second byte */
193 p++;
194 str_len--;
195 if ((*p & 0xc0) != 0x80) {
196 goto utf8_error;
197 }
198
199 /* Check for overlong sequences for each different length */
200 switch (ab) {
201 case 1:
202 /* Check for: `XX00 000X`. */
203 if ((c & 0x3e) == 0) {
204 goto utf8_error;
205 }
206 continue; /* We know there aren't any more bytes to check */
207
208 case 2:
209 /* Check for: `1110 0000, XX0X XXXX`. */
210 if (c == 0xe0 && (*p & 0x20) == 0) {
211 goto utf8_error;
212 }
213 /* Some special cases, see section 5 of UTF8 decoder stress-test by Markus Kuhn
214 * (https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt). */
215 /* From section 5.1 (and 5.2) */
216 if (c == 0xed) {
217 if (*p == 0xa0 && *(p + 1) == 0x80) {
218 goto utf8_error;
219 }
220 if (*p == 0xad && *(p + 1) == 0xbf) {
221 goto utf8_error;
222 }
223 if (*p == 0xae && *(p + 1) == 0x80) {
224 goto utf8_error;
225 }
226 if (*p == 0xaf && *(p + 1) == 0xbf) {
227 goto utf8_error;
228 }
229 if (*p == 0xb0 && *(p + 1) == 0x80) {
230 goto utf8_error;
231 }
232 if (*p == 0xbe && *(p + 1) == 0x80) {
233 goto utf8_error;
234 }
235 if (*p == 0xbf && *(p + 1) == 0xbf) {
236 goto utf8_error;
237 }
238 }
239 /* From section 5.3 */
240 if (c == 0xef) {
241 if (*p == 0xbf && *(p + 1) == 0xbe) {
242 goto utf8_error;
243 }
244 if (*p == 0xbf && *(p + 1) == 0xbf) {
245 goto utf8_error;
246 }
247 }
248 break;
249
250 case 3:
251 /* Check for: `1111 0000, XX00 XXXX`. */
252 if (c == 0xf0 && (*p & 0x30) == 0) {
253 goto utf8_error;
254 }
255 break;
256
257 case 4:
258 /* Check for `1111 1000, XX00 0XXX`. */
259 if (c == 0xf8 && (*p & 0x38) == 0) {
260 goto utf8_error;
261 }
262 break;
263
264 case 5:
265 /* Check for: `1111 1100, XX00 00XX`. */
266 if (c == 0xfc && (*p & 0x3c) == 0) {
267 goto utf8_error;
268 }
269 break;
270 }
271
272 /* Check for valid bytes after the 2nd, if any; all must start 10. */
273 while (--ab > 0) {
274 p++;
275 str_len--;
276 if ((*p & 0xc0) != 0x80) {
277 goto utf8_error;
278 }
279 }
280 }
281
282 return -1;
283
284utf8_error:
285
286 return (reinterpret_cast<const char *>(perr) - static_cast<const char *>(str));
287}
288
289int BLI_str_utf8_invalid_strip(char *str, size_t str_len)
290{
291 ptrdiff_t bad_char;
292 int tot = 0;
293
294 BLI_assert(str[str_len] == '\0');
295
296 while ((bad_char = BLI_str_utf8_invalid_byte(str, str_len)) != -1) {
297 str += bad_char;
298 str_len -= size_t(bad_char + 1);
299
300 if (str_len == 0) {
301 /* last character bad, strip it */
302 *str = '\0';
303 tot++;
304 break;
305 }
306 /* strip, keep looking */
307 memmove(str, str + 1, str_len + 1); /* +1 for null char! */
308 tot++;
309 }
310
311 return tot;
312}
313
314int BLI_str_utf8_invalid_substitute(char *str, size_t str_len, const char substitute)
315{
316 BLI_assert(substitute);
317 ptrdiff_t bad_char;
318 int tot = 0;
319
320 BLI_assert(str[str_len] == '\0');
321
322 while ((bad_char = BLI_str_utf8_invalid_byte(str, str_len)) != -1) {
323 str[bad_char] = substitute;
324 bad_char += 1; /* Step over the bad character. */
325 str += bad_char;
326 str_len -= size_t(bad_char);
327 tot++;
328 }
329
330 return tot;
331}
332
334 const size_t str_len,
335 const char substitute,
336 char *buf,
337 const size_t buf_maxncpy)
338{
339 BLI_assert(str[str_len] == '\0');
340 const ptrdiff_t bad_char = BLI_str_utf8_invalid_byte(str, str_len);
341 if (bad_char == -1) [[likely]] {
342 return str;
343 }
344 BLI_assert(bad_char >= 0);
345
346 /* In the case a bad character is outside the buffer limit,
347 * simply perform a truncating UTF8 copy into the buffer and return that. */
348 if (size_t(bad_char) >= buf_maxncpy) [[unlikely]] {
349 BLI_strncpy_utf8(buf, str, buf_maxncpy);
350 return buf;
351 }
352
353 size_t buf_len;
354 if (str_len < buf_maxncpy) {
355 memcpy(buf, str, str_len + 1);
356 buf_len = str_len;
357 }
358 else {
359 buf_len = BLI_strncpy_rlen(buf, str, buf_maxncpy);
360 }
361
362 /* Skip the good characters. */
363 BLI_str_utf8_invalid_substitute(buf + bad_char, buf_len - size_t(bad_char), substitute);
364 return buf;
365}
366
380BLI_INLINE char *str_utf8_copy_max_bytes_impl(char *dst, const char *src, size_t dst_maxncpy)
381{
382 /* Cast to `uint8_t` is a no-op, quiets array subscript of type `char` warning.
383 * No need to check `src` points to a nil byte as this will return from the switch statement. */
384 size_t utf8_size;
385 while ((utf8_size = size_t(utf8_char_compute_skip(*src))) <= dst_maxncpy) {
386 dst_maxncpy -= utf8_size;
387 /* Prefer more compact block. */
388 /* NOLINTBEGIN: bugprone-assignment-in-if-condition */
389 /* clang-format off */
390 switch (utf8_size) {
391 case 6: if (!(*dst = *src++)) [[unlikely]] { return dst; } dst++; ATTR_FALLTHROUGH;
392 case 5: if (!(*dst = *src++)) [[unlikely]] { return dst; } dst++; ATTR_FALLTHROUGH;
393 case 4: if (!(*dst = *src++)) [[unlikely]] { return dst; } dst++; ATTR_FALLTHROUGH;
394 case 3: if (!(*dst = *src++)) [[unlikely]] { return dst; } dst++; ATTR_FALLTHROUGH;
395 case 2: if (!(*dst = *src++)) [[unlikely]] { return dst; } dst++; ATTR_FALLTHROUGH;
396 case 1: if (!(*dst = *src++)) [[unlikely]] { return dst; } dst++;
397 }
398 /* clang-format on */
399 /* NOLINTEND: bugprone-assignment-in-if-condition */
400 }
401 return dst;
402}
403
404char *BLI_strncpy_utf8(char *__restrict dst, const char *__restrict src, size_t dst_maxncpy)
405{
406 BLI_assert(dst_maxncpy != 0);
407 BLI_string_debug_size(dst, dst_maxncpy);
408
409 char *dst_end = str_utf8_copy_max_bytes_impl(dst, src, dst_maxncpy - 1);
410 *dst_end = '\0';
411 return dst;
412}
413
414size_t BLI_strncpy_utf8_rlen(char *__restrict dst, const char *__restrict src, size_t dst_maxncpy)
415{
416 BLI_assert(dst_maxncpy != 0);
417 BLI_string_debug_size(dst, dst_maxncpy);
418
419 char *r_dst = dst;
420 dst = str_utf8_copy_max_bytes_impl(dst, src, dst_maxncpy - 1);
421 *dst = '\0';
422
423 return size_t(dst - r_dst);
424}
425
426size_t BLI_strncpy_utf8_rlen_unterminated(char *__restrict dst,
427 const char *__restrict src,
428 size_t dst_maxncpy)
429{
430 BLI_string_debug_size(dst, dst_maxncpy);
431
432 char *r_dst = dst;
433 dst = str_utf8_copy_max_bytes_impl(dst, src, dst_maxncpy);
434
435 return size_t(dst - r_dst);
436}
437
438/* -------------------------------------------------------------------- */
439/* wchar_t / UTF8 functions */
440
441size_t BLI_strncpy_wchar_as_utf8(char *__restrict dst,
442 const wchar_t *__restrict src,
443 const size_t dst_maxncpy)
444{
445 BLI_assert(dst_maxncpy != 0);
446 BLI_string_debug_size(dst, dst_maxncpy);
447
448 const size_t dst_len_max = dst_maxncpy - 1;
449 size_t len = 0;
450 while (*src && len < dst_len_max) {
451 len += BLI_str_utf8_from_unicode(uint(*src++), dst + len, dst_len_max - len);
452 }
453 dst[len] = '\0';
454 /* Return the correct length when part of the final byte did not fit into the string. */
455 while ((len > 0) && UNLIKELY(dst[len - 1] == '\0')) {
456 len--;
457 }
458 return len;
459}
460
461size_t BLI_wstrlen_utf8(const wchar_t *src)
462{
463 size_t len = 0;
464
465 while (*src) {
467 }
468
469 return len;
470}
471
472size_t BLI_strlen_utf8_ex(const char *strc, size_t *r_len_bytes)
473{
474 size_t len = 0;
475 const char *strc_orig = strc;
476
477 while (*strc) {
478 int step = BLI_str_utf8_size_safe(strc);
479
480 /* Detect null bytes within multi-byte sequences.
481 * This matches the behavior of #BLI_strncpy_utf8 for incomplete byte sequences. */
482 for (int i = 1; i < step; i++) {
483 if (strc[i] == '\0') [[unlikely]] {
484 step = i;
485 break;
486 }
487 }
488
489 strc += step;
490 len++;
491 }
492
493 *r_len_bytes = size_t(strc - strc_orig);
494 return len;
495}
496
497size_t BLI_strlen_utf8(const char *strc)
498{
499 size_t len_bytes;
500 return BLI_strlen_utf8_ex(strc, &len_bytes);
501}
502
503size_t BLI_strnlen_utf8_ex(const char *strc, const size_t strc_maxlen, size_t *r_len_bytes)
504{
505 size_t len = 0;
506 const char *strc_orig = strc;
507 const char *strc_end = strc + strc_maxlen;
508
509 while ((strc < strc_end) && *strc) {
510 int step = BLI_str_utf8_size_safe(strc);
511 if (strc + step > strc_end) {
512 break;
513 }
514
515 /* Detect null bytes within multi-byte sequences.
516 * This matches the behavior of #BLI_strncpy_utf8 for incomplete byte sequences. */
517 for (int i = 1; i < step; i++) {
518 if (strc[i] == '\0') [[unlikely]] {
519 step = i;
520 break;
521 }
522 }
523 strc += step;
524 len++;
525 }
526
527 *r_len_bytes = size_t(strc - strc_orig);
528 return len;
529}
530
531size_t BLI_strnlen_utf8(const char *strc, const size_t strc_maxlen)
532{
533 size_t len_bytes;
534 return BLI_strnlen_utf8_ex(strc, strc_maxlen, &len_bytes);
535}
536
537size_t BLI_strncpy_wchar_from_utf8(wchar_t *__restrict dst_w,
538 const char *__restrict src_c,
539 const size_t dst_w_maxncpy)
540{
541#ifdef WIN32
542 BLI_string_debug_size(dst_w, dst_w_maxncpy);
543 conv_utf_8_to_16(src_c, dst_w, dst_w_maxncpy);
544 /* NOTE: it would be more efficient to calculate the length as part of #conv_utf_8_to_16. */
545 return wcslen(dst_w);
546#else
547 return BLI_str_utf8_as_utf32(reinterpret_cast<char32_t *>(dst_w), src_c, dst_w_maxncpy);
548#endif
549}
550
551/* End wchar_t / UTF8 functions. */
552/* -------------------------------------------------------------------- */
553
554size_t BLI_vsnprintf_utf8(char *__restrict dst,
555 size_t dst_maxncpy,
556 const char *__restrict format,
557 va_list arg)
558{
559 /* NOTE: a clone of #BLI_vsnprintf that trims the end. */
560 BLI_string_debug_size(dst, dst_maxncpy);
561
562 BLI_assert(dst != nullptr);
563 BLI_assert(dst_maxncpy > 0);
564 BLI_assert(format != nullptr);
565
566 const size_t n = size_t(vsnprintf(dst, dst_maxncpy, format, arg));
567 if (n < dst_maxncpy) {
568 dst[n] = '\0';
569 }
570 else {
571 str_utf8_truncate_at_size_unchecked(dst, dst_maxncpy);
572 }
573
574 return n;
575}
576
577size_t BLI_vsnprintf_utf8_rlen(char *__restrict dst,
578 size_t dst_maxncpy,
579 const char *__restrict format,
580 va_list arg)
581{
582 BLI_string_debug_size(dst, dst_maxncpy);
583
584 BLI_assert(dst != nullptr);
585 BLI_assert(dst_maxncpy > 0);
586 BLI_assert(format != nullptr);
587
588 size_t n = size_t(vsnprintf(dst, dst_maxncpy, format, arg));
589 if (n < dst_maxncpy) {
590 dst[n] = '\0';
591 }
592 else {
593 n = str_utf8_truncate_at_size_unchecked(dst, dst_maxncpy);
594 }
595 return n;
596}
597
598size_t BLI_snprintf_utf8(char *__restrict dst,
599 size_t dst_maxncpy,
600 const char *__restrict format,
601 ...)
602{
603 BLI_string_debug_size(dst, dst_maxncpy);
604
605 va_list arg;
606 va_start(arg, format);
607 const size_t n = BLI_vsnprintf_utf8(dst, dst_maxncpy, format, arg);
608 va_end(arg);
609
610 return n;
611}
612
613size_t BLI_snprintf_utf8_rlen(char *__restrict dst,
614 size_t dst_maxncpy,
615 const char *__restrict format,
616 ...)
617{
618 BLI_string_debug_size(dst, dst_maxncpy);
619
620 va_list arg;
621 va_start(arg, format);
622 const size_t n = BLI_vsnprintf_utf8_rlen(dst, dst_maxncpy, format, arg);
623 va_end(arg);
624
625 return n;
626}
627
628int BLI_wcwidth_or_error(char32_t ucs)
629{
630 /* Treat private use areas (icon fonts), symbols, and emoticons as double-width. */
631 if (ucs >= 0xf0000 || (ucs >= 0xe000 && ucs < 0xf8ff) || (ucs >= 0x1f300 && ucs < 0x1fbff)) {
632 return 2;
633 }
634 return mk_wcwidth(ucs);
635}
636
637int BLI_wcwidth_safe(char32_t ucs)
638{
639 const int columns = BLI_wcwidth_or_error(ucs);
640 if (columns >= 0) {
641 return columns;
642 }
643 return 1;
644}
645
646int BLI_wcswidth_or_error(const char32_t *pwcs, size_t n)
647{
648 return mk_wcswidth(pwcs, n);
649}
650
652{
654 if (unicode == BLI_UTF8_ERR) {
655 return -1;
656 }
657
658 return BLI_wcwidth_or_error(char32_t(unicode));
659}
660
662{
664 if (unicode == BLI_UTF8_ERR) {
665 return 1;
666 }
667
668 return BLI_wcwidth_safe(char32_t(unicode));
669}
670
671/* -------------------------------------------------------------------- */
680
681char32_t BLI_str_utf32_char_to_upper(const char32_t wc)
682{
683 if (wc < U'\xFF') { /* Latin. */
684 if ((wc <= U'z' && wc >= U'a') || (wc <= U'\xF6' && wc >= U'\xE0') ||
685 /* Correct but the first case is know, only check the second */
686 // (wc <= U'\xFE' && wc >= U'\xF8')
687 (wc >= U'\xF8'))
688 {
689 return wc - 32;
690 }
691 return wc;
692 }
693
694 if ((wc <= U'\x137' && wc >= U'\x101') || (wc <= U'\x1E95' && wc >= U'\x1E01')) {
695 /* Latin Extended. */
696 return (wc & 1) ? wc - 1 : wc;
697 }
698 if ((wc <= U'\x586' && wc >= U'\x561') || (wc <= U'\x10F5' && wc >= U'\x10D0')) {
699 /* Armenian and Georgian */
700 return wc - 48;
701 }
702 if (wc <= U'\x24E9' && wc >= U'\x24D0') { /* Enclosed Numerals. */
703 return wc - 26;
704 }
705 if (wc <= U'\xFF5A' && wc >= U'\xFF41') { /* Full-width Forms. */
706 return wc - 32;
707 }
708
709 /* There are only three remaining ranges that contain capitalization. */
710 if (!(wc <= U'\x0292' && wc >= U'\x00FF') && !(wc <= U'\x04F9' && wc >= U'\x03AC') &&
711 !(wc <= U'\x1FE1' && wc >= U'\x1E01'))
712 {
713 return wc;
714 }
715
716 static const char32_t from[] =
717 U"\x00FF\x013A\x013C\x013E\x0140\x0142\x0144\x0146\x0148\x014B\x014D\x014F\x0151\x0153\x0155"
718 U"\x0157\x0159\x015B\x015D\x015F\x0161\x0163\x0165\x0167\x0169\x016B\x016D\x016F\x0171\x0173"
719 U"\x0175\x0177\x017A\x017C\x017E\x0183\x0185\x0188\x018C\x0192\x0199\x01A1\x01A3\x01A5\x01A8"
720 U"\x01AD\x01B0\x01B4\x01B6\x01B9\x01BD\x01C6\x01C9\x01CC\x01CE\x01D0\x01D2\x01D4\x01D6\x01D8"
721 U"\x01DA\x01DC\x01DF\x01E1\x01E3\x01E5\x01E7\x01E9\x01EB\x01ED\x01EF\x01F3\x01F5\x01FB\x01FD"
722 U"\x01FF\x0201\x0203\x0205\x0207\x0209\x020B\x020D\x020F\x0211\x0213\x0215\x0217\x0253\x0254"
723 U"\x0257\x0258\x0259\x025B\x0260\x0263\x0268\x0269\x026F\x0272\x0275\x0283\x0288\x028A\x028B"
724 U"\x0292\x03AC\x03AD\x03AE\x03AF\x03B1\x03B2\x03B3\x03B4\x03B5\x03B6\x03B7\x03B8\x03B9\x03BA"
725 U"\x03BB\x03BC\x03BD\x03BE\x03BF\x03C0\x03C1\x03C3\x03C4\x03C5\x03C6\x03C7\x03C8\x03C9\x03CA"
726 U"\x03CB\x03CC\x03CD\x03CE\x03E3\x03E5\x03E7\x03E9\x03EB\x03ED\x03EF\x0430\x0431\x0432\x0433"
727 U"\x0434\x0435\x0436\x0437\x0438\x0439\x043A\x043B\x043C\x043D\x043E\x043F\x0440\x0441\x0442"
728 U"\x0443\x0444\x0445\x0446\x0447\x0448\x0449\x044A\x044B\x044C\x044D\x044E\x044F\x0451\x0452"
729 U"\x0453\x0454\x0455\x0456\x0457\x0458\x0459\x045A\x045B\x045C\x045E\x045F\x0461\x0463\x0465"
730 U"\x0467\x0469\x046B\x046D\x046F\x0471\x0473\x0475\x0477\x0479\x047B\x047D\x047F\x0481\x0491"
731 U"\x0493\x0495\x0497\x0499\x049B\x049D\x049F\x04A1\x04A3\x04A5\x04A7\x04A9\x04AB\x04AD\x04AF"
732 U"\x04B1\x04B3\x04B5\x04B7\x04B9\x04BB\x04BD\x04BF\x04C2\x04C4\x04C8\x04CC\x04D1\x04D3\x04D5"
733 U"\x04D7\x04D9\x04DB\x04DD\x04DF\x04E1\x04E3\x04E5\x04E7\x04E9\x04EB\x04EF\x04F1\x04F3\x04F5"
734 U"\x04F9\x1EA1\x1EA3\x1EA5\x1EA7\x1EA9\x1EAB\x1EAD\x1EAF\x1EB1\x1EB3\x1EB5\x1EB7\x1EB9\x1EBB"
735 U"\x1EBD\x1EBF\x1EC1\x1EC3\x1EC5\x1EC7\x1EC9\x1ECB\x1ECD\x1ECF\x1ED1\x1ED3\x1ED5\x1ED7\x1ED9"
736 U"\x1EDB\x1EDD\x1EDF\x1EE1\x1EE3\x1EE5\x1EE7\x1EE9\x1EEB\x1EED\x1EEF\x1EF1\x1EF3\x1EF5\x1EF7"
737 U"\x1EF9\x1F00\x1F01\x1F02\x1F03\x1F04\x1F05\x1F06\x1F07\x1F10\x1F11\x1F12\x1F13\x1F14\x1F15"
738 U"\x1F20\x1F21\x1F22\x1F23\x1F24\x1F25\x1F26\x1F27\x1F30\x1F31\x1F32\x1F33\x1F34\x1F35\x1F36"
739 U"\x1F37\x1F40\x1F41\x1F42\x1F43\x1F44\x1F45\x1F51\x1F53\x1F55\x1F57\x1F60\x1F61\x1F62\x1F63"
740 U"\x1F64\x1F65\x1F66\x1F67\x1F80\x1F81\x1F82\x1F83\x1F84\x1F85\x1F86\x1F87\x1F90\x1F91\x1F92"
741 U"\x1F93\x1F94\x1F95\x1F96\x1F97\x1FA0\x1FA1\x1FA2\x1FA3\x1FA4\x1FA5\x1FA6\x1FA7\x1FB0\x1FB1"
742 U"\x1FD0\x1FD1\x1FE0\x1FE1";
743 static const char32_t to[] =
744 U"\x0178\x0139\x013B\x013D\x013F\x0141\x0143\x0145\x0147\x014A\x014C\x014E\x0150\x0152\x0154"
745 U"\x0156\x0158\x015A\x015C\x015E\x0160\x0162\x0164\x0166\x0168\x016A\x016C\x016E\x0170\x0172"
746 U"\x0174\x0176\x0179\x017B\x017D\x0182\x0184\x0187\x018B\x0191\x0198\x01A0\x01A2\x01A4\x01A7"
747 U"\x01AC\x01AF\x01B3\x01B5\x01B8\x01BC\x01C4\x01C7\x01CA\x01CD\x01CF\x01D1\x01D3\x01D5\x01D7"
748 U"\x01D9\x01DB\x01DE\x01E0\x01E2\x01E4\x01E6\x01E8\x01EA\x01EC\x01EE\x01F1\x01F4\x01FA\x01FC"
749 U"\x01FE\x0200\x0202\x0204\x0206\x0208\x020A\x020C\x020E\x0210\x0212\x0214\x0216\x0181\x0186"
750 U"\x018A\x018E\x018F\x0190\x0193\x0194\x0197\x0196\x019C\x019D\x019F\x01A9\x01AE\x01B1\x01B2"
751 U"\x01B7\x0386\x0388\x0389\x038A\x0391\x0392\x0393\x0394\x0395\x0396\x0397\x0398\x0399\x039A"
752 U"\x039B\x039C\x039D\x039E\x039F\x03A0\x03A1\x03A3\x03A4\x03A5\x03A6\x03A7\x03A8\x03A9\x03AA"
753 U"\x03AB\x038C\x038E\x038F\x03E2\x03E4\x03E6\x03E8\x03EA\x03EC\x03EE\x0410\x0411\x0412\x0413"
754 U"\x0414\x0415\x0416\x0417\x0418\x0419\x041A\x041B\x041C\x041D\x041E\x041F\x0420\x0421\x0422"
755 U"\x0423\x0424\x0425\x0426\x0427\x0428\x0429\x042A\x042B\x042C\x042D\x042E\x042F\x0401\x0402"
756 U"\x0403\x0404\x0405\x0406\x0407\x0408\x0409\x040A\x040B\x040C\x040E\x040F\x0460\x0462\x0464"
757 U"\x0466\x0468\x046A\x046C\x046E\x0470\x0472\x0474\x0476\x0478\x047A\x047C\x047E\x0480\x0490"
758 U"\x0492\x0494\x0496\x0498\x049A\x049C\x049E\x04A0\x04A2\x04A4\x04A6\x04A8\x04AA\x04AC\x04AE"
759 U"\x04B0\x04B2\x04B4\x04B6\x04B8\x04BA\x04BC\x04BE\x04C1\x04C3\x04C7\x04CB\x04D0\x04D2\x04D4"
760 U"\x04D6\x04D8\x04DA\x04DC\x04DE\x04E0\x04E2\x04E4\x04E6\x04E8\x04EA\x04EE\x04F0\x04F2\x04F4"
761 U"\x04F8\x1EA0\x1EA2\x1EA4\x1EA6\x1EA8\x1EAA\x1EAC\x1EAE\x1EB0\x1EB2\x1EB4\x1EB6\x1EB8\x1EBA"
762 U"\x1EBC\x1EBE\x1EC0\x1EC2\x1EC4\x1EC6\x1EC8\x1ECA\x1ECC\x1ECE\x1ED0\x1ED2\x1ED4\x1ED6\x1ED8"
763 U"\x1EDA\x1EDC\x1EDE\x1EE0\x1EE2\x1EE4\x1EE6\x1EE8\x1EEA\x1EEC\x1EEE\x1EF0\x1EF2\x1EF4\x1EF6"
764 U"\x1EF8\x1F08\x1F09\x1F0A\x1F0B\x1F0C\x1F0D\x1F0E\x1F0F\x1F18\x1F19\x1F1A\x1F1B\x1F1C\x1F1D"
765 U"\x1F28\x1F29\x1F2A\x1F2B\x1F2C\x1F2D\x1F2E\x1F2F\x1F38\x1F39\x1F3A\x1F3B\x1F3C\x1F3D\x1F3E"
766 U"\x1F3F\x1F48\x1F49\x1F4A\x1F4B\x1F4C\x1F4D\x1F59\x1F5B\x1F5D\x1F5F\x1F68\x1F69\x1F6A\x1F6B"
767 U"\x1F6C\x1F6D\x1F6E\x1F6F\x1F88\x1F89\x1F8A\x1F8B\x1F8C\x1F8D\x1F8E\x1F8F\x1F98\x1F99\x1F9A"
768 U"\x1F9B\x1F9C\x1F9D\x1F9E\x1F9F\x1FA8\x1FA9\x1FAA\x1FAB\x1FAC\x1FAD\x1FAE\x1FAF\x1FB8\x1FB9"
769 U"\x1FD8\x1FD9\x1FE8\x1FE9";
770
771 if (wc >= from[0] && wc <= from[ARRAY_SIZE(from) - 2]) {
772 /* Binary search since these are sorted. */
773 size_t min = 0;
774 size_t max = ARRAY_SIZE(from) - 2;
775 while (max >= min) {
776 const size_t mid = (min + max) / 2;
777 if (wc > from[mid]) {
778 min = mid + 1;
779 }
780 else if (wc < from[mid]) {
781 max = mid - 1;
782 }
783 else {
784 return to[mid];
785 }
786 }
787 }
788
789 return wc;
790}
791
792char32_t BLI_str_utf32_char_to_lower(const char32_t wc)
793{
794 if (wc < U'\xD8') { /* Latin. */
795 if ((wc <= U'Z' && wc >= U'A') || (wc <= U'\xD6' && wc >= U'\xC0')) {
796 return wc + 32;
797 }
798 return wc;
799 }
800 if ((wc <= U'\x136' && wc >= U'\x100') || (wc <= U'\x1E94' && wc >= U'\x1E00')) {
801 /* Latin Extended. */
802 return (wc % 2 == 0) ? wc + 1 : wc;
803 }
804 if ((wc <= U'\x556' && wc >= U'\x531') || (wc <= U'\x10C5' && wc >= U'\x10A0')) {
805 /* Armenian and Georgian. */
806 return wc + 48;
807 }
808 if (wc <= U'\x24CF' && wc >= U'\x24B6') { /* Enclosed Numerals. */
809 return wc + 26;
810 }
811 if (wc <= U'\xFF3A' && wc >= U'\xFF21') { /* Full-width Forms. */
812 return wc + 32;
813 }
814
815 /* There are only three remaining ranges that contain capitalization. */
816 if (!(wc <= U'\x0216' && wc >= U'\x00D8') && !(wc <= U'\x04F8' && wc >= U'\x0386') &&
817 !(wc <= U'\x1FE9' && wc >= U'\x1E00'))
818 {
819 return wc;
820 }
821
822 static const char32_t from[] =
823 U"\x00D8\x00D9\x00DA\x00DB\x00DC\x00DD\x00DE\x0139\x013B\x013D\x013F\x0141\x0143\x0145\x0147"
824 U"\x014A\x014C\x014E\x0150\x0152\x0154\x0156\x0158\x015A\x015C\x015E\x0160\x0162\x0164\x0166"
825 U"\x0168\x016A\x016C\x016E\x0170\x0172\x0174\x0176\x0178\x0179\x017B\x017D\x0181\x0182\x0184"
826 U"\x0186\x0187\x018A\x018B\x018E\x018F\x0190\x0191\x0193\x0194\x0196\x0197\x0198\x019C\x019D"
827 U"\x019F\x01A0\x01A2\x01A4\x01A7\x01A9\x01AC\x01AE\x01AF\x01B1\x01B2\x01B3\x01B5\x01B7\x01B8"
828 U"\x01BC\x01C4\x01C5\x01C7\x01C8\x01CA\x01CB\x01CD\x01CF\x01D1\x01D3\x01D5\x01D7\x01D9\x01DB"
829 U"\x01DE\x01E0\x01E2\x01E4\x01E6\x01E8\x01EA\x01EC\x01EE\x01F1\x01F4\x01FA\x01FC\x01FE\x0200"
830 U"\x0202\x0204\x0206\x0208\x020A\x020C\x020E\x0210\x0212\x0214\x0216\x0386\x0388\x0389\x038A"
831 U"\x038C\x038E\x038F\x0391\x0392\x0393\x0394\x0395\x0396\x0397\x0398\x0399\x039A\x039B\x039C"
832 U"\x039D\x039E\x039F\x03A0\x03A1\x03A3\x03A4\x03A5\x03A6\x03A7\x03A8\x03A9\x03AA\x03AB\x03E2"
833 U"\x03E4\x03E6\x03E8\x03EA\x03EC\x03EE\x0401\x0402\x0403\x0404\x0405\x0406\x0407\x0408\x0409"
834 U"\x040A\x040B\x040C\x040E\x040F\x0410\x0411\x0412\x0413\x0414\x0415\x0416\x0417\x0418\x0419"
835 U"\x041A\x041B\x041C\x041D\x041E\x041F\x0420\x0421\x0422\x0423\x0424\x0425\x0426\x0427\x0428"
836 U"\x0429\x042A\x042B\x042C\x042D\x042E\x042F\x0460\x0462\x0464\x0466\x0468\x046A\x046C\x046E"
837 U"\x0470\x0472\x0474\x0476\x0478\x047A\x047C\x047E\x0480\x0490\x0492\x0494\x0496\x0498\x049A"
838 U"\x049C\x049E\x04A0\x04A2\x04A4\x04A6\x04A8\x04AA\x04AC\x04AE\x04B0\x04B2\x04B4\x04B6\x04B8"
839 U"\x04BA\x04BC\x04BE\x04C1\x04C3\x04C7\x04CB\x04D0\x04D2\x04D4\x04D6\x04D8\x04DA\x04DC\x04DE"
840 U"\x04E0\x04E2\x04E4\x04E6\x04E8\x04EA\x04EE\x04F0\x04F2\x04F4\x04F8\x1EA0\x1EA2\x1EA4\x1EA6"
841 U"\x1EA8\x1EAA\x1EAC\x1EAE\x1EB0\x1EB2\x1EB4\x1EB6\x1EB8\x1EBA\x1EBC\x1EBE\x1EC0\x1EC2\x1EC4"
842 U"\x1EC6\x1EC8\x1ECA\x1ECC\x1ECE\x1ED0\x1ED2\x1ED4\x1ED6\x1ED8\x1EDA\x1EDC\x1EDE\x1EE0\x1EE2"
843 U"\x1EE4\x1EE6\x1EE8\x1EEA\x1EEC\x1EEE\x1EF0\x1EF2\x1EF4\x1EF6\x1EF8\x1F08\x1F09\x1F0A\x1F0B"
844 U"\x1F0C\x1F0D\x1F0E\x1F0F\x1F18\x1F19\x1F1A\x1F1B\x1F1C\x1F1D\x1F28\x1F29\x1F2A\x1F2B\x1F2C"
845 U"\x1F2D\x1F2E\x1F2F\x1F38\x1F39\x1F3A\x1F3B\x1F3C\x1F3D\x1F3E\x1F3F\x1F48\x1F49\x1F4A\x1F4B"
846 U"\x1F4C\x1F4D\x1F59\x1F5B\x1F5D\x1F5F\x1F68\x1F69\x1F6A\x1F6B\x1F6C\x1F6D\x1F6E\x1F6F\x1F88"
847 U"\x1F89\x1F8A\x1F8B\x1F8C\x1F8D\x1F8E\x1F8F\x1F98\x1F99\x1F9A\x1F9B\x1F9C\x1F9D\x1F9E\x1F9F"
848 U"\x1FA8\x1FA9\x1FAA\x1FAB\x1FAC\x1FAD\x1FAE\x1FAF\x1FB8\x1FB9\x1FD8\x1FD9\x1FE8\x1FE9";
849 static const char32_t to[] =
850 U"\x00F8\x00F9\x00FA\x00FB\x00FC\x00FD\x00FE\x013A\x013C\x013E\x0140\x0142\x0144\x0146\x0148"
851 U"\x014B\x014D\x014F\x0151\x0153\x0155\x0157\x0159\x015B\x015D\x015F\x0161\x0163\x0165\x0167"
852 U"\x0169\x016B\x016D\x016F\x0171\x0173\x0175\x0177\x00FF\x017A\x017C\x017E\x0253\x0183\x0185"
853 U"\x0254\x0188\x0257\x018C\x0258\x0259\x025B\x0192\x0260\x0263\x0269\x0268\x0199\x026f\x0272"
854 U"\x0275\x01A1\x01A3\x01A5\x01A8\x0283\x01AD\x0288\x01B0\x028A\x028B\x01B4\x01B6\x0292\x01B9"
855 U"\x01BD\x01C6\x01C6\x01C9\x01C9\x01CC\x01CC\x01CE\x01D0\x01D2\x01D4\x01D6\x01D8\x01DA\x01DC"
856 U"\x01DF\x01E1\x01E3\x01E5\x01E7\x01E9\x01EB\x01ED\x01EF\x01F3\x01F5\x01FB\x01FD\x01FF\x0201"
857 U"\x0203\x0205\x0207\x0209\x020B\x020D\x020F\x0211\x0213\x0215\x0217\x03AC\x03AD\x03AE\x03AF"
858 U"\x03CC\x03CD\x03CE\x03B1\x03B2\x03B3\x03B4\x03B5\x03B6\x03B7\x03B8\x03B9\x03BA\x03BB\x03BC"
859 U"\x03BD\x03BE\x03BF\x03C0\x03C1\x03C3\x03C4\x03C5\x03C6\x03C7\x03C8\x03C9\x03CA\x03CB\x03E3"
860 U"\x03E5\x03E7\x03E9\x03EB\x03ED\x03EF\x0451\x0452\x0453\x0454\x0455\x0456\x0457\x0458\x0459"
861 U"\x045A\x045B\x045C\x045E\x045F\x0430\x0431\x0432\x0433\x0434\x0435\x0436\x0437\x0438\x0439"
862 U"\x043A\x043B\x043C\x043D\x043E\x043F\x0440\x0441\x0442\x0443\x0444\x0445\x0446\x0447\x0448"
863 U"\x0449\x044A\x044B\x044C\x044D\x044E\x044F\x0461\x0463\x0465\x0467\x0469\x046B\x046D\x046F"
864 U"\x0471\x0473\x0475\x0477\x0479\x047B\x047D\x047F\x0481\x0491\x0493\x0495\x0497\x0499\x049B"
865 U"\x049D\x049F\x04A1\x04A3\x04A5\x04A7\x04A9\x04AB\x04AD\x04AF\x04B1\x04B3\x04B5\x04B7\x04B9"
866 U"\x04BB\x04BD\x04BF\x04C2\x04C4\x04C8\x04CC\x04D1\x04D3\x04D5\x04D7\x04D9\x04DB\x04DD\x04DF"
867 U"\x04E1\x04E3\x04E5\x04E7\x04E9\x04EB\x04EF\x04F1\x04F3\x04F5\x04F9\x1EA1\x1EA3\x1EA5\x1EA7"
868 U"\x1EA9\x1EAB\x1EAD\x1EAF\x1EB1\x1EB3\x1EB5\x1EB7\x1EB9\x1EBB\x1EBD\x1EBF\x1EC1\x1EC3\x1EC5"
869 U"\x1EC7\x1EC9\x1ECB\x1ECD\x1ECF\x1ED1\x1ED3\x1ED5\x1ED7\x1ED9\x1EDB\x1EDD\x1EDF\x1EE1\x1EE3"
870 U"\x1EE5\x1EE7\x1EE9\x1EEB\x1EED\x1EEF\x1EF1\x1EF3\x1EF5\x1EF7\x1EF9\x1F00\x1F01\x1F02\x1F03"
871 U"\x1F04\x1F05\x1F06\x1F07\x1F10\x1F11\x1F12\x1F13\x1F14\x1F15\x1F20\x1F21\x1F22\x1F23\x1F24"
872 U"\x1F25\x1F26\x1F27\x1F30\x1F31\x1F32\x1F33\x1F34\x1F35\x1F36\x1F37\x1F40\x1F41\x1F42\x1F43"
873 U"\x1F44\x1F45\x1F51\x1F53\x1F55\x1F57\x1F60\x1F61\x1F62\x1F63\x1F64\x1F65\x1F66\x1F67\x1F80"
874 U"\x1F81\x1F82\x1F83\x1F84\x1F85\x1F86\x1F87\x1F90\x1F91\x1F92\x1F93\x1F94\x1F95\x1F96\x1F97"
875 U"\x1FA0\x1FA1\x1FA2\x1FA3\x1FA4\x1FA5\x1FA6\x1FA7\x1FB0\x1FB1\x1FD0\x1FD1\x1FE0\x1FE1";
876
877 if (wc >= from[0] && wc <= from[ARRAY_SIZE(from) - 2]) {
878 /* Binary search since these are sorted. */
879 size_t min = 0;
880 size_t max = ARRAY_SIZE(from) - 2;
881 while (max >= min) {
882 const size_t mid = (min + max) / 2;
883 if (wc > from[mid]) {
884 min = mid + 1;
885 }
886 else if (wc < from[mid]) {
887 max = mid - 1;
888 }
889 else {
890 return to[mid];
891 }
892 }
893 }
894
895 return wc;
896}
897
899
900/* -------------------------------------------------------------------- */
906
908{
909 /* Invisible (and so can be removed at end of wrapped line) spacing characters
910 * according to the Unicode Line Breaking Algorithm (Standard Annex #14). Note
911 * to always ignore U+200B (zero-width space) and U+2060 (word joiner). */
912 return ELEM(codepoint,
913 ' ', /* Space. */
914 0x1680, /* Ogham space mark. */
915 0x2000, /* En quad. */
916 0x2001, /* Em quad. */
917 0x2002, /* En space. */
918 0x2003, /* Em space. */
919 0x2004, /* Three-per-em space. */
920 0x2005, /* Four-per-em space. */
921 0x2006, /* Six-per-em space. */
922 0x2008, /* Punctuation space. */
923 0x2009, /* Thin space. */
924 0x200A, /* Hair space. */
925 0x205F, /* Medium mathematical space. */
926 0x3000); /* Ideographic space. */
927}
928
929bool BLI_str_utf32_char_is_optional_break_after(char32_t codepoint, char32_t codepoint_prev)
930{
931 /* Subset of the characters that are line breaking opportunities
932 * according to the Unicode Line Breaking Algorithm (Standard Annex #14).
933 * Can be expanded but please no rules that differ by language. */
934
935 /* Punctuation. Backslash can be used as path separator */
936 if (ELEM(codepoint, '\\', '_')) {
937 return true;
938 }
939
940 /* Do not break on solidus if previous is a number. */
941 if (codepoint == '/' && !(codepoint_prev >= '0' && codepoint_prev <= '9')) {
942 return true;
943 }
944
945 /* Do not break on dash, hyphen, em dash if previous is space */
946 if (ELEM(codepoint, '-', 0x2010, 0x2014) &&
948 {
949 return true;
950 }
951
952 if ((codepoint >= 0x2E80 && codepoint <= 0x2FFF) || /* CJK, Kangxi Radicals. */
953 (codepoint >= 0x3040 && codepoint <= 0x309F) || /* Hiragana (except small characters). */
954 (codepoint >= 0x30A2 && codepoint <= 0x30FA) || /* Katakana (except small characters). */
955 (codepoint >= 0x3400 && codepoint <= 0x4DBF) || /* CJK Unified Ideographs Extension A. */
956 (codepoint >= 0x4E00 && codepoint <= 0x9FFF) || /* CJK Unified Ideographs. */
957 (codepoint >= 0x3040 && codepoint <= 0x309F) || /* CJK Unified Ideographs. */
958 (codepoint >= 0x3130 && codepoint <= 0x318F)) /* Hangul Compatibility Jamo. */
959 {
960 return true;
961 }
962
963 if (ELEM(codepoint, 0x0F0D, 0x0F0B)) {
964 return true; /* Tibetan shad mark and intersyllabic tsheg. */
965 }
966
967 return false;
968}
969
970bool BLI_str_utf32_char_is_optional_break_before(char32_t codepoint, char32_t codepoint_prev)
971{
972 /* Do not break on any of these if a space follows. */
974 return false;
975 }
976
977 /* Infix Numeric Separators. Allow break on these if not numbers afterward. */
978 if (ELEM(codepoint_prev,
979 ',', /* Comma. */
980 ':', /* Colon. */
981 ';', /* Semicolon. */
982 0x037E, /* Greek question mark. */
983 0x0589, /* Armenian full stop. */
984 0x060C, /* Arabic comma. */
985 0x060D, /* Arabic date separator. */
986 0x07F8, /* N'Ko comma. */
987 0x2044) /* Fraction slash. */
988 && !(codepoint >= '0' && codepoint <= '9'))
989 {
990 return true;
991 }
992
993 /* Break on full stop only if not followed by another, or by a number. */
994 if (codepoint_prev == '.' && codepoint != '.' && !(codepoint >= '0' && codepoint <= '9')) {
995 return true;
996 }
997
998 /* Close punctuation. */
999 if (ELEM(codepoint_prev,
1000 0x3001, /* Ideographic comma. */
1001 0x3002, /* Ideographic full stop. */
1002 0xFE10, /* Presentation form for vertical ideographic comma. */
1003 0xFE11, /* Presentation form for vertical ideographic full stop. */
1004 0xFE12, /* Presentation form for vertical ideographic colon. */
1005 0xFE50, /* Small comma. */
1006 0xFE52, /* Small full stop. */
1007 0xFF0C, /* Full-width comma. */
1008 0xFF0E, /* Full-width full stop. */
1009 0XFF61, /* Half-width ideographic full stop. */
1010 0Xff64)) /* Half-width ideographic comma. */
1011 {
1012 return true;
1013 }
1014
1015 /* Exclamation/Interrogation. */
1016 if (ELEM(codepoint_prev,
1017 '!', /* Exclamation mark. */
1018 '?', /* Question mark. */
1019 0x05C6, /* Hebrew punctuation `maqaf`. */
1020 0x061B, /* Arabic semicolon. */
1021 0x061E, /* Arabic triple dot. */
1022 0x061F, /* Arabic question mark. */
1023 0x06D4, /* Arabic full stop. */
1024 0x07F9, /* N'Ko question mark. */
1025 0x0F0D, /* Tibetan shad mark. */
1026 0xFF01, /* Full-width exclamation mark. */
1027 0xff1f)) /* full-width question mark. */
1028 {
1029 return true;
1030 }
1031
1032 return false;
1033}
1034
1036{
1037 /* Characters marking the end of sentences according to Unicode Text Segmentation
1038 * (Standard Annex #29), Sentence Break Property (ATerm, STerm).
1039 * Only the characters available in Blender are matched. */
1040
1041 return (ELEM(codepoint,
1042 0x002E, /* Full stop. */
1043 0x2024, /* One dot leader. */
1044 0xFE52, /* Small full stop. */
1045 0xFF0E, /* Full-width full stop. */
1046 0x0021, /* Exclamation mark. */
1047 0x003F, /* Question mark. */
1048 0x0589, /* Armenian full stop. */
1049 0x061F, /* Arabic question mark. */
1050 0x06D4, /* Arabic full stop. */
1051 0x0964, /* Devanagari danda. */
1052 0x0965, /* Devanagari double danda. */
1053 0x104A, /* Myanmar sign little section. */
1054 0x104B, /* Myanmar sign section. */
1055 0x1362, /* Ethiopic full stop. */
1056 0x1367, /* Ethiopic question mark. */
1057 0x1368) || /* Ethiopic paragraph separator. */
1058 ELEM(codepoint,
1059 0x17D4, /* Khmer sign khan. */
1060 0x17D5, /* Khmer sign bariyoosan. */
1061 0x203C, /* Double exclamation mark. */
1062 0x203D, /* Interrobang. */
1063 0x2047, /* Double question mark. */
1064 0x2048, /* Question exclamation mark. */
1065 0x2049, /* Exclamation question mark. */
1066 0x3002, /* Ideographic full stop. */
1067 0xA9C8, /* Javanese pada lingsa. */
1068 0xA9C9, /* Javanese pada lungsi. */
1069 0xFE12, /* Presentation form for vertical ideographic full stop. */
1070 0xFE15, /* Presentation form for vertical exclamation mark. */
1071 0xFE16, /* Presentation form for vertical question mark. */
1072 0xFE56, /* Small question mark. */
1073 0xFE57, /* Small exclamation mark. */
1074 0xFF01) || /* Full-width exclamation mark. */
1075 ELEM(codepoint,
1076 0xFF1F, /* Full-width question mark. */
1077 0xFF61)); /* Half-width ideographic full stop. */
1078}
1079 /* -------------------------------------------------------------------- */
1081
1083{
1085}
1086
1087int BLI_str_utf8_size_safe(const char *p)
1088{
1089 return utf8_char_compute_skip(*p);
1090}
1091
1093{
1094 /* Originally `g_utf8_get_char` in GLIB. */
1095
1096 const uchar c = uchar(*p);
1097
1098 char mask = 0;
1100 if (len == -1) [[unlikely]] {
1101 return BLI_UTF8_ERR;
1102 }
1104}
1105
1107{
1109 if (result == BLI_UTF8_ERR) [[unlikely]] {
1110 return *p;
1111 }
1112 return result;
1113}
1114
1116 const size_t p_len,
1117 size_t *__restrict index)
1118{
1119 const uchar c = uchar(*(p += *index));
1120
1121 BLI_assert(*index < p_len);
1122 BLI_assert(c != '\0');
1123
1124 char mask = 0;
1126 if (UNLIKELY(len == -1) || (*index + size_t(len) > p_len)) {
1127 return BLI_UTF8_ERR;
1128 }
1129
1131 if (result == BLI_UTF8_ERR) [[unlikely]] {
1132 return BLI_UTF8_ERR;
1133 }
1134 *index += size_t(len);
1135 BLI_assert(*index <= p_len);
1136 return result;
1137}
1138
1140 const size_t p_len,
1141 size_t *__restrict index)
1142{
1144 if (result == BLI_UTF8_ERR) [[unlikely]] {
1145 result = uint(p[*index]);
1146 *index += 1;
1147 }
1148 BLI_assert(*index <= p_len);
1149 return result;
1150}
1151
1152/* was g_unichar_to_utf8 */
1153
1154#define UTF8_VARS_FROM_CHAR32(Char, First, Len) \
1155 if (Char < 0x80) { \
1156 First = 0; \
1157 Len = 1; \
1158 } \
1159 else if (Char < 0x800) { \
1160 First = 0xc0; \
1161 Len = 2; \
1162 } \
1163 else if (Char < 0x10000) { \
1164 First = 0xe0; \
1165 Len = 3; \
1166 } \
1167 else if (Char < 0x200000) { \
1168 First = 0xf0; \
1169 Len = 4; \
1170 } \
1171 else if (Char < 0x4000000) { \
1172 First = 0xf8; \
1173 Len = 5; \
1174 } \
1175 else { \
1176 First = 0xfc; \
1177 Len = 6; \
1178 } \
1179 (void)0
1180
1182{
1183 /* If this gets modified, also update the copy in g_string_insert_unichar() */
1184 uint len = 0;
1185 uint first;
1186
1187 UTF8_VARS_FROM_CHAR32(c, first, len);
1188 (void)first;
1189
1190 return len;
1191}
1192
1193size_t BLI_str_utf8_from_unicode(uint c, char *dst, const size_t dst_maxncpy)
1194
1195{
1196 BLI_string_debug_size(dst, dst_maxncpy);
1197
1198 /* If this gets modified, also update the copy in g_string_insert_unichar() */
1199 uint len = 0;
1200 uint first;
1201
1202 UTF8_VARS_FROM_CHAR32(c, first, len);
1203
1204 if (dst_maxncpy < len) [[unlikely]] {
1205 /* Null terminate instead of writing a partial byte. */
1206 memset(dst, 0x0, dst_maxncpy);
1207 return dst_maxncpy;
1208 }
1209
1210 for (uint i = len - 1; i > 0; i--) {
1211 dst[i] = char((c & 0x3f) | 0x80);
1212 c >>= 6;
1213 }
1214 dst[0] = char(c | first);
1215
1216 return len;
1217}
1218
1219size_t BLI_str_utf8_as_utf32(char32_t *__restrict dst_w,
1220 const char *__restrict src_c,
1221 const size_t dst_w_maxncpy)
1222{
1223 BLI_assert(dst_w_maxncpy != 0);
1224 BLI_string_debug_size(dst_w, dst_w_maxncpy);
1225
1226 const size_t maxlen = dst_w_maxncpy - 1;
1227 size_t len = 0;
1228
1229 const size_t src_c_len = strlen(src_c);
1230 const char *src_c_end = src_c + src_c_len;
1231 size_t index = 0;
1232 while ((index < src_c_len) && (len != maxlen)) {
1233 const uint unicode = BLI_str_utf8_as_unicode_step_or_error(src_c, src_c_len, &index);
1234 if (unicode != BLI_UTF8_ERR) {
1235 *dst_w = unicode;
1236 }
1237 else {
1238 *dst_w = '?';
1239 const char *src_c_next = BLI_str_find_next_char_utf8(src_c + index, src_c_end);
1240 index = size_t(src_c_next - src_c);
1241 }
1242 dst_w++;
1243 len++;
1244 }
1245
1246 *dst_w = 0;
1247
1248 return len;
1249}
1250
1251size_t BLI_str_utf32_as_utf8(char *__restrict dst,
1252 const char32_t *__restrict src,
1253 const size_t dst_maxncpy)
1254{
1255 BLI_assert(dst_maxncpy != 0);
1256 BLI_string_debug_size(dst, dst_maxncpy);
1257
1258 const size_t dst_len_max = dst_maxncpy - 1;
1259 size_t len = 0;
1260 while (*src && len < dst_len_max) {
1261 len += BLI_str_utf8_from_unicode(uint(*src++), dst + len, dst_len_max - len);
1262 }
1263 dst[len] = '\0';
1264 /* Return the correct length when part of the final byte did not fit into the string. */
1265 while ((len > 0) && UNLIKELY(dst[len - 1] == '\0')) {
1266 len--;
1267 }
1268 return len;
1269}
1270
1271size_t BLI_str_utf32_as_utf8_len_ex(const char32_t *src, const size_t src_maxlen)
1272{
1273 size_t len = 0;
1274 const char32_t *src_end = src + src_maxlen;
1275
1276 while ((src < src_end) && *src) {
1278 }
1279
1280 return len;
1281}
1282
1283size_t BLI_str_utf32_as_utf8_len(const char32_t *src)
1284{
1285 size_t len = 0;
1286
1287 while (*src) {
1289 }
1290
1291 return len;
1292}
1293
1294const char *BLI_str_find_prev_char_utf8(const char *p, const char *str_start)
1295{
1296 /* Originally `g_utf8_find_prev_char` in GLIB. */
1297
1298 BLI_assert(p >= str_start);
1299 if (str_start < p) {
1300 for (--p; p > str_start; p--) {
1301 if ((*p & 0xc0) != 0x80) {
1302 return const_cast<char *>(p);
1303 }
1304 }
1305 }
1306 return p;
1307}
1308
1309const char *BLI_str_find_next_char_utf8(const char *p, const char *str_end)
1310{
1311 /* Originally `g_utf8_find_next_char` in GLIB. */
1312
1313 BLI_assert(p <= str_end);
1314 if ((p < str_end) && (*p != '\0')) {
1315 for (++p; p < str_end && (*p & 0xc0) == 0x80; p++) {
1316 /* do nothing */
1317 }
1318 }
1319 return p;
1320}
1321
1322size_t BLI_str_partition_utf8(const char *str,
1323 const uint delim[],
1324 const char **r_sep,
1325 const char **r_suf)
1326{
1327 return BLI_str_partition_ex_utf8(str, nullptr, delim, r_sep, r_suf, false);
1328}
1329
1330size_t BLI_str_rpartition_utf8(const char *str,
1331 const uint delim[],
1332 const char **r_sep,
1333 const char **r_suf)
1334{
1335 return BLI_str_partition_ex_utf8(str, nullptr, delim, r_sep, r_suf, true);
1336}
1337
1339 const char *end,
1340 const uint delim[],
1341 const char **r_sep,
1342 const char **r_suf,
1343 const bool from_right)
1344{
1345 const size_t str_len = end ? size_t(end - str) : strlen(str);
1346 if (end == nullptr) {
1347 end = str + str_len;
1348 }
1349
1350 /* Note that here, we assume end points to a valid UTF8 char! */
1352
1353 char *suf = const_cast<char *>(str + str_len);
1354 size_t index = 0;
1355 for (char *sep = const_cast<char *>(from_right ? BLI_str_find_prev_char_utf8(end, str) : str);
1356 from_right ? ((sep != nullptr) && (sep < end)) : ((sep < end) && (*sep != '\0'));
1357 sep = const_cast<char *>(
1358 from_right ? (str != sep ? BLI_str_find_prev_char_utf8(sep, str) : nullptr) :
1359 str + index))
1360 {
1361 size_t index_ofs = 0;
1362 const uint c = BLI_str_utf8_as_unicode_step_or_error(sep, size_t(end - sep), &index_ofs);
1363 if (c == BLI_UTF8_ERR) [[unlikely]] {
1364 break;
1365 }
1366 index += index_ofs;
1367
1368 for (const uint *d = delim; *d != '\0'; d++) {
1369 if (*d == c) {
1370 /* `suf` is already correct in case from_right is true. */
1371 *r_sep = sep;
1372 *r_suf = from_right ? suf : const_cast<char *>(str + index);
1373 return size_t(sep - str);
1374 }
1375 }
1376
1377 suf = sep; /* Useful in 'from_right' case! */
1378 }
1379
1380 *r_suf = *r_sep = nullptr;
1381 return str_len;
1382}
1383
1387static size_t str_utf8_truncate_at_size_unchecked(char *str, const size_t str_size)
1388{
1389 BLI_assert(str_size > 0);
1390 BLI_assert(!std::memchr(str, '\0', str_size - 1));
1391 size_t str_len_trim;
1392 BLI_strnlen_utf8_ex(str, str_size - 1, &str_len_trim);
1393 str[str_len_trim] = '\0';
1394 return str_len_trim;
1395}
1396
1397bool BLI_str_utf8_truncate_at_size(char *str, const size_t str_size)
1398{
1399 BLI_assert(str_size > 0);
1400 if (std::memchr(str, '\0', str_size)) {
1401 return false;
1402 }
1403
1405 return true;
1406}
1407
1408/* -------------------------------------------------------------------- */
1417
1418int BLI_str_utf8_offset_to_index(const char *str, const size_t str_len, const int offset_target)
1419{
1420 BLI_assert(offset_target >= 0);
1421 const size_t offset_target_as_size = size_t(offset_target);
1422 size_t offset = 0;
1423 int index = 0;
1424 /* Note that `offset != offset_target_as_size` works for valid UTF8 strings. */
1425 while ((offset < str_len) && (offset < offset_target_as_size)) {
1426 /* Use instead of #BLI_str_utf8_size_safe to match behavior when limiting the string length. */
1427 const uint code = BLI_str_utf8_as_unicode_step_safe(str, str_len, &offset);
1428 UNUSED_VARS(code);
1429 index++;
1430 BLI_assert(offset <= offset_target_as_size); /* See DOXY section comment. */
1431 }
1432 return index;
1433}
1434
1435int BLI_str_utf8_offset_from_index(const char *str, const size_t str_len, const int index_target)
1436{
1437 BLI_assert(index_target >= 0);
1438 size_t offset = 0;
1439 int index = 0;
1440 while ((offset < str_len) && (index < index_target)) {
1441 /* Use instead of #BLI_str_utf8_size_safe to match behavior when limiting the string length. */
1442 const uint code = BLI_str_utf8_as_unicode_step_safe(str, str_len, &offset);
1443 UNUSED_VARS(code);
1444 index++;
1445 }
1446 return int(offset);
1447}
1448
1449int BLI_str_utf8_offset_to_column(const char *str, const size_t str_len, const int offset_target)
1450{
1451 BLI_assert(offset_target >= 0);
1452 const size_t offset_target_clamp = std::min(size_t(offset_target), str_len);
1453 size_t offset = 0;
1454 int column = 0;
1455 while (offset < offset_target_clamp) {
1456 const uint code = BLI_str_utf8_as_unicode_step_safe(str, str_len, &offset);
1457 column += BLI_wcwidth_safe(code);
1458 BLI_assert(offset <= size_t(offset_target)); /* See DOXY section comment. */
1459 }
1460 return column;
1461}
1462
1463int BLI_str_utf8_offset_from_column(const char *str, const size_t str_len, const int column_target)
1464{
1465 size_t offset = 0, offset_next = 0;
1466 int column = 0;
1467 while ((offset < str_len) && (column < column_target)) {
1468 const uint code = BLI_str_utf8_as_unicode_step_safe(str, str_len, &offset_next);
1469 column += BLI_wcwidth_safe(code);
1470 if (column > column_target) {
1471 break;
1472 }
1473 offset = offset_next;
1474 }
1475 return int(offset);
1476}
1477
1479 const size_t str_len,
1480 const int offset_target,
1481 const int tab_width)
1482{
1483 BLI_assert(offset_target >= 0);
1484 const size_t offset_target_clamp = std::min(size_t(offset_target), str_len);
1485 size_t offset = 0;
1486 int column = 0;
1487 while (offset < offset_target_clamp) {
1488 const uint code = BLI_str_utf8_as_unicode_step_safe(str, str_len, &offset);
1489 /* The following line is the only change compared with #BLI_str_utf8_offset_to_column. */
1490 column += (code == '\t') ? (tab_width - (column % tab_width)) : BLI_wcwidth_safe(code);
1491 BLI_assert(offset <= size_t(offset_target)); /* See DOXY section comment. */
1492 }
1493 return column;
1494}
1495
1497 const size_t str_len,
1498 const int column_target,
1499 const int tab_width)
1500{
1501 size_t offset = 0, offset_next = 0;
1502 int column = 0;
1503 while ((offset < str_len) && (column < column_target)) {
1504 const uint code = BLI_str_utf8_as_unicode_step_safe(str, str_len, &offset_next);
1505 /* The following line is the only change compared with #BLI_str_utf8_offset_from_column. */
1506 column += (code == '\t') ? (tab_width - (column % tab_width)) : BLI_wcwidth_safe(code);
1507 if (column > column_target) {
1508 break;
1509 }
1510 offset = offset_next;
1511 }
1512 return int(offset);
1513}
1514
1515int BLI_str_utf8_column_count(const char *str, size_t str_len)
1516{
1517 return BLI_str_utf8_offset_to_column(str, str_len, int(str_len));
1518}
1519
1521
1522} // namespace blender
#define ATTR_FALLTHROUGH
#define BLI_string_debug_size(str, str_maxncpy)
#define BLI_UTF8_ERR
#define UNUSED_VARS(...)
#define BLI_INLINE
#define UNLIKELY(x)
#define ELEM(...)
btScalar max
#define str(s)
VecBase< float, D > step(VecOp< float, D >, VecOp< float, D >) RET
ccl_device_inline dual1 len(const ccl_private dual< T > &a)
Definition math_dual.h:263
BMesh Iterators.
const char * BLI_str_utf8_invalid_substitute_if_needed(const char *str, size_t str_len, const char substitute, char *buf, const size_t buf_maxncpy) ATTR_NONNULL(1
UserDef U
Definition blender.cc:58
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
size_t size_t size_t BLI_snprintf_utf8(char *__restrict dst, size_t dst_maxncpy, const char *__restrict format,...) ATTR_NONNULL(1
size_t size_t size_t BLI_str_utf32_as_utf8_len_ex(const char32_t *src, size_t src_maxlen) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
bool BLI_str_utf32_char_is_breaking_space(char32_t codepoint)
size_t BLI_str_utf8_from_unicode(unsigned int c, char *dst, size_t dst_maxncpy) ATTR_NONNULL(2)
size_t size_t BLI_strncpy_wchar_from_utf8(wchar_t *__restrict dst_w, const char *__restrict src_c, size_t dst_w_maxncpy) ATTR_NONNULL(1
ptrdiff_t BLI_str_utf8_invalid_byte(const char *str, size_t str_len) ATTR_NONNULL(1)
unsigned int uint
Definition BKE_text.h:55
int BLI_wcswidth_or_error(const char32_t *pwcs, size_t n) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
unsigned int BLI_str_utf8_as_unicode_safe(const char *p) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
bool BLI_str_utf32_char_is_terminal_punctuation(char32_t codepoint)
int BLI_str_utf8_char_width_safe(const char *p) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
size_t size_t BLI_strnlen_utf8(const char *strc, size_t strc_maxlen) ATTR_NONNULL(1) ATTR_WARN_UNUSED_RESULT
char size_t BLI_strncpy_utf8_rlen(char *__restrict dst, const char *__restrict src, size_t dst_maxncpy) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1
char * BLI_strncpy_utf8(char *__restrict dst, const char *__restrict src, size_t dst_maxncpy) ATTR_NONNULL(1
size_t BLI_str_utf8_as_utf32(char32_t *__restrict dst_w, const char *__restrict src_c, size_t dst_w_maxncpy) ATTR_NONNULL(1
size_t size_t size_t bool BLI_str_utf8_truncate_at_size(char *str, const size_t str_size)
unsigned int BLI_str_utf8_as_unicode_step_safe(const char *__restrict p, size_t p_len, size_t *__restrict index) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1
static size_t str_utf8_truncate_at_size_unchecked(char *str, const size_t str_size)
size_t size_t BLI_vsnprintf_utf8_rlen(char *__restrict dst, size_t dst_maxncpy, const char *__restrict format, va_list arg) ATTR_PRINTF_FORMAT(3
size_t size_t BLI_str_utf32_as_utf8(char *__restrict dst, const char32_t *__restrict src, size_t dst_maxncpy) ATTR_NONNULL(1
int BLI_str_utf8_offset_to_column(const char *str, size_t str_len, int offset_target) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
size_t size_t BLI_str_rpartition_utf8(const char *str, const unsigned int delim[], const char **r_sep, const char **r_suf) ATTR_NONNULL(1
size_t BLI_str_partition_utf8(const char *str, const unsigned int delim[], const char **r_sep, const char **r_suf) ATTR_NONNULL(1
BLI_INLINE int utf8_char_compute_skip(const char c)
unsigned char uchar
Definition BKE_text.h:58
unsigned int unsigned int BLI_str_utf8_as_unicode_step_or_error(const char *__restrict p, size_t p_len, size_t *__restrict index) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1
int BLI_str_utf8_invalid_substitute(char *str, size_t str_len, const char substitute) ATTR_NONNULL(1)
bool BLI_str_utf32_char_is_optional_break_after(char32_t codepoint, char32_t codepoint_prev)
bool BLI_str_utf32_char_is_optional_break_before(char32_t codepoint, char32_t codepoint_prev)
size_t BLI_strncpy_wchar_as_utf8(char *__restrict dst, const wchar_t *__restrict src, size_t dst_maxncpy) ATTR_NONNULL(1
int BLI_str_utf8_char_width_or_error(const char *p) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
int BLI_str_utf8_column_count(const char *str, size_t str_len) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
const char const char size_t BLI_wstrlen_utf8(const wchar_t *src) ATTR_NONNULL(1) ATTR_WARN_UNUSED_RESULT
char char size_t BLI_strncpy_rlen(char *__restrict dst, const char *__restrict src, size_t dst_maxncpy) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1
const char const char * BLI_str_find_next_char_utf8(const char *p, const char *str_end) ATTR_WARN_UNUSED_RESULT ATTR_RETURNS_NONNULL ATTR_NONNULL(1
constexpr size_t ARRAY_SIZE(const T(&arg)[N]) noexcept
int BLI_str_utf8_offset_to_column_with_tabs(const char *str, size_t str_len, int offset_target, int tab_width) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
int BLI_str_utf8_size_safe(const char *p) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
const char int BLI_str_utf8_size_or_error(const char *p) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
BLI_INLINE char * str_utf8_copy_max_bytes_impl(char *dst, const char *src, size_t dst_maxncpy)
size_t BLI_vsnprintf_utf8(char *__restrict dst, size_t dst_maxncpy, const char *__restrict format, va_list arg) ATTR_PRINTF_FORMAT(3
int BLI_str_utf8_offset_from_index(const char *str, size_t str_len, int index_target) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
const char * BLI_str_find_prev_char_utf8(const char *p, const char *str_start) ATTR_WARN_UNUSED_RESULT ATTR_RETURNS_NONNULL ATTR_NONNULL(1
int BLI_str_utf8_offset_from_column(const char *str, size_t str_len, int column_target) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
char32_t BLI_str_utf32_char_to_lower(char32_t wc)
size_t size_t size_t BLI_str_partition_ex_utf8(const char *str, const char *end, const unsigned int delim[], const char **r_sep, const char **r_suf, bool from_right) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1
char size_t size_t BLI_strncpy_utf8_rlen_unterminated(char *__restrict dst, const char *__restrict src, size_t dst_maxncpy)
int BLI_wcwidth_safe(char32_t ucs) ATTR_WARN_UNUSED_RESULT
unsigned int BLI_str_utf8_as_unicode_or_error(const char *p) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
unsigned int unsigned int size_t BLI_str_utf8_from_unicode_len(unsigned int c) ATTR_WARN_UNUSED_RESULT
int BLI_str_utf8_offset_to_index(const char *str, size_t str_len, int offset_target) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
size_t BLI_strlen_utf8_ex(const char *strc, size_t *r_len_bytes) ATTR_NONNULL(1
BLI_INLINE uint utf8_char_decode(const char *p, const char mask, const int len, const uint err)
size_t BLI_strlen_utf8(const char *strc) ATTR_NONNULL(1) ATTR_WARN_UNUSED_RESULT
size_t BLI_str_utf32_as_utf8_len(const char32_t *src) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
size_t BLI_snprintf_utf8_rlen(char *__restrict dst, size_t dst_maxncpy, const char *__restrict format,...) ATTR_NONNULL(1
int BLI_str_utf8_invalid_strip(char *str, size_t str_len) ATTR_NONNULL(1)
char32_t BLI_str_utf32_char_to_upper(char32_t wc)
int BLI_str_utf8_offset_from_column_with_tabs(const char *str, size_t str_len, int column_target, int tab_width) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
size_t BLI_strnlen_utf8_ex(const char *strc, size_t strc_maxlen, size_t *r_len_bytes) ATTR_NONNULL(1
BLI_INLINE int utf8_char_compute_skip_or_error_with_mask(const char c, char *r_mask)
BLI_INLINE int utf8_char_compute_skip_or_error(const char c)
size_t size_t int BLI_wcwidth_or_error(char32_t ucs) ATTR_WARN_UNUSED_RESULT
#define min(a, b)
Definition sort.cc:38
#define UTF8_VARS_FROM_CHAR32(Char, First, Len)
int conv_utf_8_to_16(const char *in8, wchar_t *out16, size_t size16)
Definition utfconv.cc:182