Blender V5.3
string_utf8.cc File Reference
#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <cwchar>
#include <cwctype>
#include <wcwidth.h>
#include "BLI_utildefines.hh"
#include "BLI_string.hh"
#include "BLI_string_utf8.hh"
#include "BLI_strict_flags.hh"

Go to the source code of this file.

Namespaces

namespace  blender
 BMesh Iterators.

Macros

#define UTF8_VARS_FROM_CHAR32(Char, First, Len)

Functions

static size_t blender::str_utf8_truncate_at_size_unchecked (char *str, const size_t str_size)
ptrdiff_t blender::BLI_str_utf8_invalid_byte (const char *str, size_t str_len) ATTR_NONNULL(1)
int blender::BLI_str_utf8_invalid_strip (char *str, size_t str_len) ATTR_NONNULL(1)
int blender::BLI_str_utf8_invalid_substitute (char *str, size_t str_len, const char substitute) ATTR_NONNULL(1)
const char * blender::BLI_str_utf8_invalid_substitute_if_needed (const char *str, size_t str_len, const char substitute, char *buf, const size_t buf_maxncpy) ATTR_NONNULL(1
BLI_INLINE char * blender::str_utf8_copy_max_bytes_impl (char *dst, const char *src, size_t dst_maxncpy)
char * blender::BLI_strncpy_utf8 (char *__restrict dst, const char *__restrict src, size_t dst_maxncpy) ATTR_NONNULL(1
char size_t blender::BLI_strncpy_utf8_rlen (char *__restrict dst, const char *__restrict src, size_t dst_maxncpy) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1
char size_t size_t blender::BLI_strncpy_utf8_rlen_unterminated (char *__restrict dst, const char *__restrict src, size_t dst_maxncpy)
size_t blender::BLI_strncpy_wchar_as_utf8 (char *__restrict dst, const wchar_t *__restrict src, size_t dst_maxncpy) ATTR_NONNULL(1
const char const char size_t blender::BLI_wstrlen_utf8 (const wchar_t *src) ATTR_NONNULL(1) ATTR_WARN_UNUSED_RESULT
size_t blender::BLI_strlen_utf8_ex (const char *strc, size_t *r_len_bytes) ATTR_NONNULL(1
size_t blender::BLI_strlen_utf8 (const char *strc) ATTR_NONNULL(1) ATTR_WARN_UNUSED_RESULT
size_t blender::BLI_strnlen_utf8_ex (const char *strc, size_t strc_maxlen, size_t *r_len_bytes) ATTR_NONNULL(1
size_t size_t blender::BLI_strnlen_utf8 (const char *strc, size_t strc_maxlen) ATTR_NONNULL(1) ATTR_WARN_UNUSED_RESULT
size_t size_t blender::BLI_strncpy_wchar_from_utf8 (wchar_t *__restrict dst_w, const char *__restrict src_c, size_t dst_w_maxncpy) ATTR_NONNULL(1
size_t blender::BLI_vsnprintf_utf8 (char *__restrict dst, size_t dst_maxncpy, const char *__restrict format, va_list arg) ATTR_PRINTF_FORMAT(3
size_t size_t blender::BLI_vsnprintf_utf8_rlen (char *__restrict dst, size_t dst_maxncpy, const char *__restrict format, va_list arg) ATTR_PRINTF_FORMAT(3
size_t size_t size_t blender::BLI_snprintf_utf8 (char *__restrict dst, size_t dst_maxncpy, const char *__restrict format,...) ATTR_NONNULL(1
size_t blender::BLI_snprintf_utf8_rlen (char *__restrict dst, size_t dst_maxncpy, const char *__restrict format,...) ATTR_NONNULL(1
size_t size_t int blender::BLI_wcwidth_or_error (char32_t ucs) ATTR_WARN_UNUSED_RESULT
int blender::BLI_wcwidth_safe (char32_t ucs) ATTR_WARN_UNUSED_RESULT
int blender::BLI_wcswidth_or_error (const char32_t *pwcs, size_t n) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
int blender::BLI_str_utf8_char_width_or_error (const char *p) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
int blender::BLI_str_utf8_char_width_safe (const char *p) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
const char int blender::BLI_str_utf8_size_or_error (const char *p) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
int blender::BLI_str_utf8_size_safe (const char *p) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
unsigned int blender::BLI_str_utf8_as_unicode_or_error (const char *p) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
unsigned int blender::BLI_str_utf8_as_unicode_safe (const char *p) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
unsigned int unsigned int blender::BLI_str_utf8_as_unicode_step_or_error (const char *__restrict p, size_t p_len, size_t *__restrict index) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1
unsigned int blender::BLI_str_utf8_as_unicode_step_safe (const char *__restrict p, size_t p_len, size_t *__restrict index) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1
unsigned int unsigned int size_t blender::BLI_str_utf8_from_unicode_len (unsigned int c) ATTR_WARN_UNUSED_RESULT
size_t blender::BLI_str_utf8_from_unicode (unsigned int c, char *dst, size_t dst_maxncpy) ATTR_NONNULL(2)
size_t blender::BLI_str_utf8_as_utf32 (char32_t *__restrict dst_w, const char *__restrict src_c, size_t dst_w_maxncpy) ATTR_NONNULL(1
size_t size_t blender::BLI_str_utf32_as_utf8 (char *__restrict dst, const char32_t *__restrict src, size_t dst_maxncpy) ATTR_NONNULL(1
size_t size_t size_t blender::BLI_str_utf32_as_utf8_len_ex (const char32_t *src, size_t src_maxlen) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
size_t blender::BLI_str_utf32_as_utf8_len (const char32_t *src) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
const char * blender::BLI_str_find_prev_char_utf8 (const char *p, const char *str_start) ATTR_WARN_UNUSED_RESULT ATTR_RETURNS_NONNULL ATTR_NONNULL(1
const char const char * blender::BLI_str_find_next_char_utf8 (const char *p, const char *str_end) ATTR_WARN_UNUSED_RESULT ATTR_RETURNS_NONNULL ATTR_NONNULL(1
size_t blender::BLI_str_partition_utf8 (const char *str, const unsigned int delim[], const char **r_sep, const char **r_suf) ATTR_NONNULL(1
size_t size_t blender::BLI_str_rpartition_utf8 (const char *str, const unsigned int delim[], const char **r_sep, const char **r_suf) ATTR_NONNULL(1
size_t size_t size_t blender::BLI_str_partition_ex_utf8 (const char *str, const char *end, const unsigned int delim[], const char **r_sep, const char **r_suf, bool from_right) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1
size_t size_t size_t bool blender::BLI_str_utf8_truncate_at_size (char *str, const size_t str_size)
UTF8 Character Decoding (Skip & Mask Lookup)

Derived from GLIB gutf8.c.

Ranges (zero based, inclusive):

  • 000..127: 1 byte.
  • 128..191: invalid.
  • 192..223: 2 bytes.
  • 224..239: 3 bytes.
  • 240..247: 4 bytes.
  • 248..251: 4 bytes.
  • 252..253: 4 bytes.
  • 254..255: invalid.

Invalid values fall back to 1 byte or -1 (for an error value).

Note
From testing string copying via BLI_strncpy_utf8 with large (multi-megabyte) strings, using a function instead of a lookup-table is between 2 & 3 times faster.
BLI_INLINE int blender::utf8_char_compute_skip (const char c)
BLI_INLINE int blender::utf8_char_compute_skip_or_error (const char c)
BLI_INLINE int blender::utf8_char_compute_skip_or_error_with_mask (const char c, char *r_mask)
BLI_INLINE uint blender::utf8_char_decode (const char *p, const char mask, const int len, const uint err)
UTF32 Case Conversion
Warning
the lower/uppercase form of some characters use multiple characters. These cases are not accounted for by this conversion function. A common example is the German eszett / scharfes. Supporting such cases would have to operate on a character array, with support for resizing. (for reference - Python's upper/lower functions support this).
char32_t blender::BLI_str_utf32_char_to_upper (char32_t wc)
char32_t blender::BLI_str_utf32_char_to_lower (char32_t wc)
UTF32 Text Boundary Analysis

Helper functions to help locating linguistic boundaries, like word, sentence, and paragraph boundaries.

bool blender::BLI_str_utf32_char_is_breaking_space (char32_t codepoint)
bool blender::BLI_str_utf32_char_is_optional_break_after (char32_t codepoint, char32_t codepoint_prev)
bool blender::BLI_str_utf32_char_is_optional_break_before (char32_t codepoint, char32_t codepoint_prev)
bool blender::BLI_str_utf32_char_is_terminal_punctuation (char32_t codepoint)
Offset Conversion in Strings
Note
Regarding the assertion: BLI_assert(offset <= offset_target) The offset_target is likely in the middle of a UTF8 byte-sequence. Most likely the offset passed in is incorrect, although it may be impractical to avoid this happening in the case of invalid UTF8 byte sequences. If the assert is impractical to avoid, it could be demoted to a warning.
int blender::BLI_str_utf8_offset_to_index (const char *str, size_t str_len, int offset_target) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
int blender::BLI_str_utf8_offset_from_index (const char *str, size_t str_len, int index_target) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
int blender::BLI_str_utf8_offset_to_column (const char *str, size_t str_len, int offset_target) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
int blender::BLI_str_utf8_offset_from_column (const char *str, size_t str_len, int column_target) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
int blender::BLI_str_utf8_offset_to_column_with_tabs (const char *str, size_t str_len, int offset_target, int tab_width) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
int blender::BLI_str_utf8_offset_from_column_with_tabs (const char *str, size_t str_len, int column_target, int tab_width) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
int blender::BLI_str_utf8_column_count (const char *str, size_t str_len) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)

Macro Definition Documentation

◆ UTF8_VARS_FROM_CHAR32

#define UTF8_VARS_FROM_CHAR32 ( Char,
First,
Len )
Value:
if (Char < 0x80) { \
First = 0; \
Len = 1; \
} \
else if (Char < 0x800) { \
First = 0xc0; \
Len = 2; \
} \
else if (Char < 0x10000) { \
First = 0xe0; \
Len = 3; \
} \
else if (Char < 0x200000) { \
First = 0xf0; \
Len = 4; \
} \
else if (Char < 0x4000000) { \
First = 0xf8; \
Len = 5; \
} \
else { \
First = 0xfc; \
Len = 6; \
} \
(void)0

Definition at line 1154 of file string_utf8.cc.

Referenced by blender::BLI_str_utf8_from_unicode(), and blender::BLI_str_utf8_from_unicode_len().