Blender V5.3
BLI_string_utf8_test.cc File Reference
#include "testing/testing.h"
#include "BLI_rand_c.hh"
#include "BLI_string.hh"
#include "BLI_string_cursor_utf8.hh"
#include "BLI_string_utf8.hh"
#include "BLI_utildefines.hh"

Go to the source code of this file.

Namespaces

namespace  blender
 BMesh Iterators.

Macros

#define STR_MB_ALPHA_1   "\x41"
#define STR_MB_ALPHA_2   "\xc2\xaa"
#define STR_MB_ALPHA_3   "\xe0\xa0\x80"
#define STR_MB_ALPHA_4   "\xf0\x90\x80\x80"
#define STR_MB_ALPHA_5   "\xf8\x80\x80\x80\x80"
#define STR_MB_ALPHA_6   "\xfc\x80\x80\x80\x80\x80"
#define TEST_SIMPLE(src_chars, expected_error_count, expected_str)
#define ARRAY_ARG(...)
#define EXPECT_BYTE_OFFSET(truncate_ofs, expect_nchars)
#define STRNCPY_UTF8_ASCII(...)
#define STRNCPY_UTF8_ASCII_TRUNCATE(maxncpy, ...)
#define STRNCPY_UTF8_TRUNCATE(byte_size, ...)
#define STRNCPY_UTF8_TRUNC_EXPECT(src, dst_expect, dst_maxncpy)
#define STRNCPY_UTF8_TERMINATE_EARLY(byte_size, ...)
#define STR_UTF32_AS_UTF8_TRUNC_EXPECT(src, dst_expect, dst_maxncpy)
#define STRNCPY_WCHAR_AS_UTF8_TRUNC_EXPECT(src, dst_expect, dst_maxncpy)
#define SNPRINTF_UTF8_ASCII(...)
#define SNPRINTF_UTF8_TERMINATE_EARLY(byte_size, ...)
#define SNPRINTF_UTF8_TRUNC_EXPECT(src, dst_expect, dst_maxncpy)

Functions

Test #BLI_strnlen_utf8
 blender::TEST (string, StringNLenUTF8)
 blender::TEST (string, StringNLenUTF8_NoNil)
 blender::TEST (string, StringNLenUTF8_Incomplete)
Test #BLI_strncpy_utf8
 blender::TEST (string, StrCopyUTF8_ASCII)
 blender::TEST (string, StrCopyUTF8_ASCII_Truncate)
 blender::TEST (string, StrCopyUTF8_TruncateEncoding)
 blender::TEST (string, StrCopyUTF8_TruncateEncodingMulti)
 blender::TEST (string, StrCopyUTF8_TerminateEncodingEarly)
Test #BLI_str_utf32_as_utf8
 blender::TEST (string, StrUTF32AsUTF8_Truncate)
Test #BLI_strncpy_wchar_as_utf8
 blender::TEST (string, StrCopyWCharAsUTF8_Truncate)
Test #BLI_snprintf_utf8
 blender::TEST (string, StrPrintfUTF8_ASCII)
 blender::TEST (string, StrPrintfUTF8_TerminateEncodingEarly)
 blender::TEST (string, StrPrintfUTF8_TruncateEncodingMulti)
Test #BLI_str_utf8_offset_from_index
 blender::TEST (string, Utf8OffsetFromIndex_ClampedIndex)
Test #BLI_str_find_prev_char_utf8
 blender::TEST (string, StrFindPrevCharUTF8)
 blender::TEST (string, StrFindPrevCharUTF8_ContinuationOnly)
Test #BLI_str_utf8_as_unicode_step_safe
static size_t blender::utf8_as_char32 (const char *str, const char str_len, char32_t *r_result)
template<size_t Size, size_t SizeWithPadding>
void blender::utf8_as_char32_test_compare_with_pad_bytes (const char utf8_src[Size])
template<size_t Size>
void blender::utf8_as_char32_test_compare (const char utf8_src[Size])
template<size_t Size>
void blender::utf8_as_char32_test_at_buffer_size ()
 blender::TEST (string, Utf8AsUnicodeStep)
Test #BLI_str_cursor_step_next_utf32_empty
 blender::TEST (string, StrCursorStepNextUtf32Empty)
Test #BLI_str_cursor_step_next_utf32_single
 blender::TEST (string, StrCursorStepNextUtf32Single)
Test #BLI_str_cursor_step_next_utf32_simple
 blender::TEST (string, StrCursorStepNextUtf32Simple)
Test #BLI_str_cursor_step_next_utf32_allcombining
 blender::TEST (string, StrCursorStepNextUtf32AllCombining)
Test #BLI_str_cursor_step_next_utf32_complex
 blender::TEST (string, StrCursorStepNextUtf32Complex)
Test #BLI_str_cursor_step_next_utf32_invalid
 blender::TEST (string, StrCursorStepNextUtf32Invalid)
Test #BLI_str_cursor_step_prev_utf32_empty
 blender::TEST (string, StrCursorStepPrevUtf32Empty)
Test #BLI_str_cursor_step_prev_utf32_single
 blender::TEST (string, StrCursorStepPrevUtf32Single)
Test #BLI_str_cursor_step_prev_utf32_simple
 blender::TEST (string, StrCursorStepPrevUtf32Simple)
Test #BLI_str_cursor_step_prev_utf32_allcombining
 blender::TEST (string, StrCursorStepPrevUtf32AllCombining)
Test #BLI_str_cursor_step_prev_utf32_complex
 blender::TEST (string, StrCursorStepPrevUtf32Complex)
Test #BLI_str_cursor_step_prev_utf32_invalid
 blender::TEST (string, StrCursorStepPrevUtf32Invalid)
Test #BLI_str_cursor_step_next_utf8_empty
 blender::TEST (string, StrCursorStepNextUtf8Empty)
Test #BLI_str_cursor_step_next_utf8_single
 blender::TEST (string, StrCursorStepNextUtf8Single)
Test #BLI_str_cursor_step_next_utf8_simple
 blender::TEST (string, StrCursorStepNextUtf8Simple)
Test #BLI_str_cursor_step_next_utf8_allcombining
 blender::TEST (string, StrCursorStepNextUtf8AllCombining)
Test #BLI_str_cursor_step_next_utf8_complex
 blender::TEST (string, StrCursorStepNextUtf8AllComplex)
Test #BLI_str_cursor_step_next_utf8_invalid
 blender::TEST (string, StrCursorStepNextUtf8Invalid)
Test #BLI_str_cursor_step_prev_utf8_empty
 blender::TEST (string, StrCursorStepPrevUtf8Empty)
Test #BLI_str_cursor_step_prev_utf8_single
 blender::TEST (string, StrCursorStepPrevUtf8Single)
 blender::TEST (string, StrCursorStepPrevUtf8Simple)
Test #BLI_str_cursor_step_prev_utf8_allcombining
 blender::TEST (string, StrCursorStepPrevUtf8AllCombining)
Test #BLI_str_cursor_step_prev_utf8_complex
 blender::TEST (string, StrCursorStepPrevUtf8Complex)
Test #BLI_str_cursor_step_prev_utf8_invalid
 blender::TEST (string, StrCursorStepPrevUtf8Invalid)

Test #BLI_str_utf8_invalid_strip

static const char * blender::utf8_invalid_tests [][3]
 blender::TEST (string, Utf8InvalidBytesStrip)
 blender::TEST (string, Utf8InvalidBytesSubstitute)
 blender::TEST (string, Utf8InvalidBytesSubstitutePatterns)

Macro Definition Documentation

◆ ARRAY_ARG

#define ARRAY_ARG ( ...)
Value:
__VA_ARGS__

Referenced by blender::TEST().

◆ EXPECT_BYTE_OFFSET

#define EXPECT_BYTE_OFFSET ( truncate_ofs,
expect_nchars )
Value:
{ \
size_t buf_ofs = 0; \
STRNCPY(buf, ref_str); \
buf[truncate_ofs] = '\0'; \
EXPECT_EQ(BLI_strnlen_utf8_ex(buf, ref_str_len, &buf_ofs), expect_nchars); \
EXPECT_EQ(buf_ofs, truncate_ofs); \
}

Referenced by blender::TEST().

◆ SNPRINTF_UTF8_ASCII

#define SNPRINTF_UTF8_ASCII ( ...)
Value:
{ \
const char src[] = {__VA_ARGS__, 0}; \
char dst[sizeof(src)]; \
memset(dst, 0xff, sizeof(dst)); \
SNPRINTF_UTF8(dst, "%s", src); \
EXPECT_EQ(strlen(dst), sizeof(dst) - 1); \
EXPECT_STREQ(dst, src); \
}

Referenced by blender::TEST().

◆ SNPRINTF_UTF8_TERMINATE_EARLY

#define SNPRINTF_UTF8_TERMINATE_EARLY ( byte_size,
... )
Value:
{ \
char src[] = {__VA_ARGS__, 0}; \
EXPECT_EQ(BLI_str_utf8_size_or_error(src), byte_size); \
char dst[sizeof(src)]; \
memset(dst, 0xff, sizeof(dst)); \
SNPRINTF_UTF8(dst, "%s", src); \
EXPECT_EQ(strlen(dst), sizeof(dst) - 1); \
EXPECT_STREQ(dst, src); \
for (int i = sizeof(dst) - 1; i > 1; i--) { \
src[i] = '\0'; \
memset(dst, 0xff, sizeof(dst)); \
const int dst_copied = SNPRINTF_UTF8_RLEN(dst, "%s", src); \
EXPECT_STREQ(dst, src); \
EXPECT_EQ(strlen(dst), i); \
EXPECT_EQ(dst_copied, i); \
} \
}
#define SNPRINTF_UTF8_RLEN(dst, format,...)

Referenced by blender::TEST().

◆ SNPRINTF_UTF8_TRUNC_EXPECT

#define SNPRINTF_UTF8_TRUNC_EXPECT ( src,
dst_expect,
dst_maxncpy )
Value:
{ \
char dst[dst_maxncpy + 1]; \
dst[dst_maxncpy] = 0xff; \
size_t len = BLI_snprintf_utf8_rlen(dst, dst_maxncpy, "%s", src); \
EXPECT_EQ(len, strlen(dst)); \
EXPECT_STREQ(dst, dst_expect); \
EXPECT_EQ(dst[dst_maxncpy], 0xff); \
}
ccl_device_inline dual1 len(const ccl_private dual< T > &a)
Definition math_dual.h:263

Referenced by blender::TEST().

◆ STR_MB_ALPHA_1

#define STR_MB_ALPHA_1   "\x41"

Multi byte defies, use when the exact character isn't important, and when proper handling of multi-byte sequences is needed.

Define the first text character found in 1-4 bytes. We might want to generate other string tables too.

unicode_max = 0x110000
bytes_max = 4
sample_chars = dict()
for i in range(unicode_max):
ch = chr(i)
try:
byte = ch.encode('utf-8')
byte_len = len(byte)
except UnicodeEncodeError:
continue
if byte_len not in sample_chars:
# Not essential but useful if the character is printable.
if ch.isalpha():
sample_chars[byte_len] = byte
if byte_len == bytes_max:
break
assert tuple(sorted(sample_chars.keys())) == tuple(range(1, bytes_max + 1))
for i in range(1, bytes_max + 1):
byte = sample_chars[i]
print("#define STR_MB_ALPHA_%d \"" % i, end="")
for x in byte:
print("\\x%02x" % x, end="")
print("\"")

Definition at line 54 of file BLI_string_utf8_test.cc.

Referenced by blender::TEST(), blender::TEST(), blender::TEST(), blender::TEST(), blender::TEST(), blender::TEST(), and blender::TEST().

◆ STR_MB_ALPHA_2

#define STR_MB_ALPHA_2   "\xc2\xaa"

◆ STR_MB_ALPHA_3

#define STR_MB_ALPHA_3   "\xe0\xa0\x80"

◆ STR_MB_ALPHA_4

#define STR_MB_ALPHA_4   "\xf0\x90\x80\x80"

◆ STR_MB_ALPHA_5

#define STR_MB_ALPHA_5   "\xf8\x80\x80\x80\x80"

Definition at line 63 of file BLI_string_utf8_test.cc.

Referenced by blender::TEST(), blender::TEST(), and blender::TEST().

◆ STR_MB_ALPHA_6

#define STR_MB_ALPHA_6   "\xfc\x80\x80\x80\x80\x80"

Definition at line 64 of file BLI_string_utf8_test.cc.

Referenced by blender::TEST(), blender::TEST(), blender::TEST(), and blender::TEST().

◆ STR_UTF32_AS_UTF8_TRUNC_EXPECT

#define STR_UTF32_AS_UTF8_TRUNC_EXPECT ( src,
dst_expect,
dst_maxncpy )
Value:
{ \
char dst[dst_maxncpy + 1]; \
dst[dst_maxncpy] = 0xff; \
size_t len = BLI_str_utf32_as_utf8(dst, src, dst_maxncpy); \
EXPECT_EQ(len, strlen(dst)); \
EXPECT_STREQ(dst, dst_expect); \
EXPECT_EQ(dst[dst_maxncpy], 0xff); \
}

Referenced by blender::TEST().

◆ STRNCPY_UTF8_ASCII

#define STRNCPY_UTF8_ASCII ( ...)
Value:
{ \
const char src[] = {__VA_ARGS__, 0}; \
char dst[sizeof(src)]; \
memset(dst, 0xff, sizeof(dst)); \
STRNCPY_UTF8(dst, src); \
EXPECT_EQ(strlen(dst), sizeof(dst) - 1); \
EXPECT_STREQ(dst, src); \
}

Referenced by blender::TEST().

◆ STRNCPY_UTF8_ASCII_TRUNCATE

#define STRNCPY_UTF8_ASCII_TRUNCATE ( maxncpy,
... )
Value:
{ \
char src[] = {__VA_ARGS__}; \
char dst[sizeof(src)]; \
memset(dst, 0xff, sizeof(dst)); \
BLI_strncpy_utf8(dst, src, maxncpy); \
int len_expect = std::min<int>(sizeof(src), maxncpy) - 1; \
src[len_expect] = '\0'; \
EXPECT_EQ(strlen(dst), len_expect); \
EXPECT_STREQ(dst, src); \
}

Referenced by blender::TEST().

◆ STRNCPY_UTF8_TERMINATE_EARLY

#define STRNCPY_UTF8_TERMINATE_EARLY ( byte_size,
... )
Value:
{ \
char src[] = {__VA_ARGS__, 0}; \
EXPECT_EQ(BLI_str_utf8_size_or_error(src), byte_size); \
char dst[sizeof(src)]; \
memset(dst, 0xff, sizeof(dst)); \
STRNCPY_UTF8(dst, src); \
EXPECT_EQ(strlen(dst), sizeof(dst) - 1); \
EXPECT_STREQ(dst, src); \
for (int i = sizeof(dst) - 1; i > 1; i--) { \
src[i] = '\0'; \
memset(dst, 0xff, sizeof(dst)); \
const int dst_copied = STRNCPY_UTF8_RLEN(dst, src); \
EXPECT_STREQ(dst, src); \
EXPECT_EQ(strlen(dst), i); \
EXPECT_EQ(dst_copied, i); \
} \
}
#define STRNCPY_UTF8_RLEN(dst, src)

Referenced by blender::TEST().

◆ STRNCPY_UTF8_TRUNC_EXPECT

#define STRNCPY_UTF8_TRUNC_EXPECT ( src,
dst_expect,
dst_maxncpy )
Value:
{ \
char dst[dst_maxncpy + 1]; \
dst[dst_maxncpy] = 0xff; \
size_t len = BLI_strncpy_utf8_rlen(dst, src, dst_maxncpy); \
EXPECT_EQ(len, strlen(dst)); \
EXPECT_STREQ(dst, dst_expect); \
EXPECT_EQ(dst[dst_maxncpy], 0xff); \
}

Referenced by blender::TEST().

◆ STRNCPY_UTF8_TRUNCATE

#define STRNCPY_UTF8_TRUNCATE ( byte_size,
... )
Value:
{ \
const char src[] = {__VA_ARGS__, 0}; \
EXPECT_EQ(BLI_str_utf8_size_or_error(src), byte_size); \
char dst[sizeof(src)]; \
memset(dst, 0xff, sizeof(dst)); \
STRNCPY_UTF8(dst, src); \
EXPECT_EQ(strlen(dst), sizeof(dst) - 1); \
EXPECT_STREQ(dst, src); \
BLI_strncpy_utf8(dst, src, sizeof(dst) - 1); \
EXPECT_STREQ(dst, ""); \
}

Referenced by blender::TEST().

◆ STRNCPY_WCHAR_AS_UTF8_TRUNC_EXPECT

#define STRNCPY_WCHAR_AS_UTF8_TRUNC_EXPECT ( src,
dst_expect,
dst_maxncpy )
Value:
{ \
char dst[dst_maxncpy + 1]; \
dst[dst_maxncpy] = 0xff; \
size_t len = BLI_strncpy_wchar_as_utf8(dst, src, dst_maxncpy); \
EXPECT_EQ(len, strlen(dst)); \
EXPECT_STREQ(dst, dst_expect); \
EXPECT_EQ(dst[dst_maxncpy], 0xff); \
}

Referenced by blender::TEST().

◆ TEST_SIMPLE

#define TEST_SIMPLE ( src_chars,
expected_error_count,
expected_str )
Value:
{ \
char buff[] = src_chars; \
EXPECT_EQ(BLI_str_utf8_invalid_substitute(buff, strlen(buff), '?'), expected_error_count); \
EXPECT_STREQ(buff, expected_str); \
} \
((void)0)

Referenced by blender::TEST().