Blender V5.3
BLI_string_utf8_test.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2023 Blender Authors
2 *
3 * SPDX-License-Identifier: Apache-2.0 */
4
5#include "testing/testing.h"
6
7#include "BLI_rand_c.hh"
8#include "BLI_string.hh"
10#include "BLI_string_utf8.hh"
11#include "BLI_utildefines.hh"
12
13namespace blender {
14
15/* Note that 'common' UTF8 variants of string functions (like copy, etc.) are tested in
16 * `BLI_string_test.cc` However, tests below are specific UTF8 conformance ones,
17 * and since they eat quite their share of lines, they deserved their own file. */
18
53
54#define STR_MB_ALPHA_1 "\x41"
55#define STR_MB_ALPHA_2 "\xc2\xaa"
56#define STR_MB_ALPHA_3 "\xe0\xa0\x80"
57#define STR_MB_ALPHA_4 "\xf0\x90\x80\x80"
58
59/* These don't decode into valid code-points and wont work in all UTF8 functions.
60 * Use them for functions which support up to #BLI_UTF8_MAX, where failure to test
61 * 5 & 6 byte sequences would cause test coverage to be incomplete.
62 * See https://stackoverflow.com/a/35027139 for details. */
63#define STR_MB_ALPHA_5 "\xf8\x80\x80\x80\x80"
64#define STR_MB_ALPHA_6 "\xfc\x80\x80\x80\x80\x80"
65
66/* -------------------------------------------------------------------- */
69
70/* Breaking strings is confusing here, prefer over-long lines. */
71/* clang-format off */
72
73/* Each test is made of a 79 bytes (80 with null char) string to test, expected string result after
74 * stripping invalid UTF8 bytes, and a single-byte string encoded with expected number of errors.
75 *
76 * Based on UTF8 decoder stress-test (https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt)
77 * by Markus Kuhn <http://www.cl.cam.ac.uk/~mgk25/> - 2015-08-28 - CC BY 4.0
78 */
79static const char *utf8_invalid_tests[][3] = {
80/* 1 Some correct UTF-8 text. */
81 {"You should see the Greek word 'kosme': \"\xce\xba\xe1\xbd\xb9\xcf\x83\xce\xbc\xce\xb5\" |",
82 "You should see the Greek word 'kosme': \"\xce\xba\xe1\xbd\xb9\xcf\x83\xce\xbc\xce\xb5\" |", "\x00"},
83
84/* 2 Boundary condition test cases
85 * Note that those will pass for us, those are not erroneous unicode code points
86 * (aside from \x00, which is only valid as string terminator).
87 * 2.1 First possible sequence of a certain length */
88 {"2.1.1 1 byte (U-00000000): \"\x00\" |",
89 "2.1.1 1 byte (U-00000000): \"\" |", "\x01"},
90 {"2.1.2 2 bytes (U-00000080): \"\xc2\x80\" |",
91 "2.1.2 2 bytes (U-00000080): \"\xc2\x80\" |", "\x00"},
92 {"2.1.3 3 bytes (U-00000800): \"\xe0\xa0\x80\" |",
93 "2.1.3 3 bytes (U-00000800): \"\xe0\xa0\x80\" |", "\x00"},
94 {"2.1.4 4 bytes (U-00010000): \"\xf0\x90\x80\x80\" |",
95 "2.1.4 4 bytes (U-00010000): \"\xf0\x90\x80\x80\" |", "\x00"},
96 {"2.1.5 5 bytes (U-00200000): \"\xf8\x88\x80\x80\x80\" |",
97 "2.1.5 5 bytes (U-00200000): \"\xf8\x88\x80\x80\x80\" |", "\x00"},
98 {"2.1.6 6 bytes (U-04000000): \"\xfc\x84\x80\x80\x80\x80\" |",
99 "2.1.6 6 bytes (U-04000000): \"\xfc\x84\x80\x80\x80\x80\" |", "\x00"},
100/* 2.2 Last possible sequence of a certain length */
101 {"2.2.1 1 byte (U-0000007F): \"\x7f\" |",
102 "2.2.1 1 byte (U-0000007F): \"\x7f\" |", "\x00"},
103 {"2.2.2 2 bytes (U-000007FF): \"\xdf\xbf\" |",
104 "2.2.2 2 bytes (U-000007FF): \"\xdf\xbf\" |", "\x00"},
105 {"2.2.3 3 bytes (U-0000FFFF): \"\xef\xbf\xbf\" |",
106 "2.2.3 3 bytes (U-0000FFFF): \"\" |", "\x03"}, /* matches one of 5.3 sequences... */
107 {"2.2.4 4 bytes (U-001FFFFF): \"\xf7\xbf\xbf\xbf\" |",
108 "2.2.4 4 bytes (U-001FFFFF): \"\xf7\xbf\xbf\xbf\" |", "\x00"},
109 {"2.2.5 5 bytes (U-03FFFFFF): \"\xfb\xbf\xbf\xbf\xbf\" |",
110 "2.2.5 5 bytes (U-03FFFFFF): \"\xfb\xbf\xbf\xbf\xbf\" |", "\x00"},
111 {"2.2.6 6 bytes (U-7FFFFFFF): \"\xfd\xbf\xbf\xbf\xbf\xbf\" |",
112 "2.2.6 6 bytes (U-7FFFFFFF): \"\xfd\xbf\xbf\xbf\xbf\xbf\" |", "\x00"},
113/* 2.3 Other boundary conditions */
114 {"2.3.1 U-0000D7FF = ed 9f bf = \"\xed\x9f\xbf\" |",
115 "2.3.1 U-0000D7FF = ed 9f bf = \"\xed\x9f\xbf\" |", "\x00"},
116 {"2.3.2 U-0000E000 = ee 80 80 = \"\xee\x80\x80\" |",
117 "2.3.2 U-0000E000 = ee 80 80 = \"\xee\x80\x80\" |", "\x00"},
118 {"2.3.3 U-0000FFFD = ef bf bd = \"\xef\xbf\xbd\" |",
119 "2.3.3 U-0000FFFD = ef bf bd = \"\xef\xbf\xbd\" |", "\x00"},
120 {"2.3.4 U-0010FFFF = f4 8f bf bf = \"\xf4\x8f\xbf\xbf\" |",
121 "2.3.4 U-0010FFFF = f4 8f bf bf = \"\xf4\x8f\xbf\xbf\" |", "\x00"},
122 {"2.3.5 U-00110000 = f4 90 80 80 = \"\xf4\x90\x80\x80\" |",
123 "2.3.5 U-00110000 = f4 90 80 80 = \"\xf4\x90\x80\x80\" |", "\x00"},
124
125/* 3 Malformed sequences
126 * 3.1 Unexpected continuation bytes
127 * Each unexpected continuation byte should be separately signaled as a malformed sequence of its own. */
128 {"3.1.1 First continuation byte 0x80: \"\x80\" |",
129 "3.1.1 First continuation byte 0x80: \"\" |", "\x01"},
130 {"3.1.2 Last continuation byte 0xbf: \"\xbf\" |",
131 "3.1.2 Last continuation byte 0xbf: \"\" |", "\x01"},
132 {"3.1.3 2 continuation bytes: \"\x80\xbf\" |",
133 "3.1.3 2 continuation bytes: \"\" |", "\x02"},
134 {"3.1.4 3 continuation bytes: \"\x80\xbf\x80\" |",
135 "3.1.4 3 continuation bytes: \"\" |", "\x03"},
136 {"3.1.5 4 continuation bytes: \"\x80\xbf\x80\xbf\" |",
137 "3.1.5 4 continuation bytes: \"\" |", "\x04"},
138 {"3.1.6 5 continuation bytes: \"\x80\xbf\x80\xbf\x80\" |",
139 "3.1.6 5 continuation bytes: \"\" |", "\x05"},
140 {"3.1.7 6 continuation bytes: \"\x80\xbf\x80\xbf\x80\xbf\" |",
141 "3.1.7 6 continuation bytes: \"\" |", "\x06"},
142 {"3.1.8 7 continuation bytes: \"\x80\xbf\x80\xbf\x80\xbf\x80\" |",
143 "3.1.8 7 continuation bytes: \"\" |", "\x07"},
144/* 3.1.9 Sequence of all 64 possible continuation bytes (0x80-0xbf): | */
145 {"3.1.9 \"\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
146 "\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
147 "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
148 "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\" |",
149 "3.1.9 \"\" |", "\x40"}, /* NOLINT: modernize-raw-string-literal. */
150/* 3.2 Lonely start characters
151 * 3.2.1 All 32 first bytes of 2-byte sequences (0xc0-0xdf), each followed by a space character: */
152 {"3.2.1 \"\xc0 \xc1 \xc2 \xc3 \xc4 \xc5 \xc6 \xc7 \xc8 \xc9 \xca \xcb \xcc \xcd \xce \xcf "
153 "\xd0 \xd1 \xd2 \xd3 \xd4 \xd5 \xd6 \xd7 \xd8 \xd9 \xda \xdb \xdc \xdd \xde \xdf \" |",
154 "3.2.1 \" \" |", "\x20"}, /* NOLINT: modernize-raw-string-literal. */
155/* 3.2.2 All 16 first bytes of 3-byte sequences (0xe0-0xef), each followed by a space character: */
156 {"3.2.2 \"\xe0 \xe1 \xe2 \xe3 \xe4 \xe5 \xe6 \xe7 \xe8 \xe9 \xea \xeb \xec \xed \xee \xef \" |",
157 "3.2.2 \" \" |", "\x10"},
158/* 3.2.3 All 8 first bytes of 4-byte sequences (0xf0-0xf7), each followed by a space character: */
159 {"3.2.3 \"\xf0 \xf1 \xf2 \xf3 \xf4 \xf5 \xf6 \xf7 \" |",
160 "3.2.3 \" \" |", "\x08"},
161/* 3.2.4 All 4 first bytes of 5-byte sequences (0xf8-0xfb), each followed by a space character: */
162 {"3.2.4 \"\xf8 \xf9 \xfa \xfb \" |",
163 "3.2.4 \" \" |", "\x04"},
164/* 3.2.5 All 2 first bytes of 6-byte sequences (0xfc-0xfd), each followed by a space character: */
165 {"3.2.4 \"\xfc \xfd \" |",
166 "3.2.4 \" \" |", "\x02"},
167/* 3.3 Sequences with last continuation byte missing
168 * All bytes of an incomplete sequence should be signaled as a single malformed sequence,
169 * i.e., you should see only a single replacement character in each of the next 10 tests.
170 * (Characters as in section 2) */
171 {"3.3.1 2-byte sequence with last byte missing (U+0000): \"\xc0\" |",
172 "3.3.1 2-byte sequence with last byte missing (U+0000): \"\" |", "\x01"},
173 {"3.3.2 3-byte sequence with last byte missing (U+0000): \"\xe0\x80\" |",
174 "3.3.2 3-byte sequence with last byte missing (U+0000): \"\" |", "\x02"},
175 {"3.3.3 4-byte sequence with last byte missing (U+0000): \"\xf0\x80\x80\" |",
176 "3.3.3 4-byte sequence with last byte missing (U+0000): \"\" |", "\x03"},
177 {"3.3.4 5-byte sequence with last byte missing (U+0000): \"\xf8\x80\x80\x80\" |",
178 "3.3.4 5-byte sequence with last byte missing (U+0000): \"\" |", "\x04"},
179 {"3.3.5 6-byte sequence with last byte missing (U+0000): \"\xfc\x80\x80\x80\x80\" |",
180 "3.3.5 6-byte sequence with last byte missing (U+0000): \"\" |", "\x05"},
181 {"3.3.6 2-byte sequence with last byte missing (U-000007FF): \"\xdf\" |",
182 "3.3.6 2-byte sequence with last byte missing (U-000007FF): \"\" |", "\x01"},
183 {"3.3.7 3-byte sequence with last byte missing (U-0000FFFF): \"\xef\xbf\" |",
184 "3.3.7 3-byte sequence with last byte missing (U-0000FFFF): \"\" |", "\x02"},
185 {"3.3.8 4-byte sequence with last byte missing (U-001FFFFF): \"\xf7\xbf\xbf\" |",
186 "3.3.8 4-byte sequence with last byte missing (U-001FFFFF): \"\" |", "\x03"},
187 {"3.3.9 5-byte sequence with last byte missing (U-03FFFFFF): \"\xfb\xbf\xbf\xbf\" |",
188 "3.3.9 5-byte sequence with last byte missing (U-03FFFFFF): \"\" |", "\x04"},
189 {"3.3.10 6-byte sequence with last byte missing (U-7FFFFFFF): \"\xfd\xbf\xbf\xbf\xbf\" |",
190 "3.3.10 6-byte sequence with last byte missing (U-7FFFFFFF): \"\" |", "\x05"},
191/* 3.4 Concatenation of incomplete sequences
192 * All the 10 sequences of 3.3 concatenated, you should see 10 malformed sequences being signaled: */
193 {"3.4 \"\xc0\xe0\x80\xf0\x80\x80\xf8\x80\x80\x80\xfc\x80\x80\x80\x80"
194 "\xdf\xef\xbf\xf7\xbf\xbf\xfb\xbf\xbf\xbf\xfd\xbf\xbf\xbf\xbf\""
195 " |",
196 "3.4 \"\" |", "\x1e"},
197/* 3.5 Impossible bytes
198 * The following two bytes cannot appear in a correct UTF-8 string */
199 {"3.5.1 fe = \"\xfe\" |",
200 "3.5.1 fe = \"\" |", "\x01"},
201 {"3.5.2 ff = \"\xff\" |",
202 "3.5.2 ff = \"\" |", "\x01"},
203 {"3.5.3 fe fe ff ff = \"\xfe\xfe\xff\xff\" |",
204 "3.5.3 fe fe ff ff = \"\" |", "\x04"},
205
206/* 4 Overlong sequences
207 * The following sequences are not malformed according to the letter of the Unicode 2.0 standard.
208 * However, they are longer then necessary and a correct UTF-8 encoder is not allowed to produce them.
209 * A "safe UTF-8 decoder" should reject them just like malformed sequences for two reasons:
210 * (1) It helps to debug applications if overlong sequences are not treated as valid representations
211 * of characters, because this helps to spot problems more quickly. (2) Overlong sequences provide
212 * alternative representations of characters, that could maliciously be used to bypass filters that check
213 * only for ASCII characters. For instance, a 2-byte encoded line feed (LF) would not be caught by a
214 * line counter that counts only 0x0a bytes, but it would still be processed as a line feed by an unsafe
215 * UTF-8 decoder later in the pipeline. From a security point of view, ASCII compatibility of UTF-8
216 * sequences means also, that ASCII characters are *only* allowed to be represented by ASCII bytes
217 * in the range 0x00-0x7f. To ensure this aspect of ASCII compatibility, use only "safe UTF-8 decoders"
218 * that reject overlong UTF-8 sequences for which a shorter encoding exists.
219 *
220 * 4.1 Examples of an overlong ASCII character
221 * With a safe UTF-8 decoder, all of the following five overlong representations of the ASCII character
222 * slash ("/") should be rejected like a malformed UTF-8 sequence, for instance by substituting it with
223 * a replacement character. If you see a slash below, you do not have a safe UTF-8 decoder! */
224 {"4.1.1 U+002F = c0 af = \"\xc0\xaf\" |",
225 "4.1.1 U+002F = c0 af = \"\" |", "\x02"},
226 {"4.1.2 U+002F = e0 80 af = \"\xe0\x80\xaf\" |",
227 "4.1.2 U+002F = e0 80 af = \"\" |", "\x03"},
228 {"4.1.3 U+002F = f0 80 80 af = \"\xf0\x80\x80\xaf\" |",
229 "4.1.3 U+002F = f0 80 80 af = \"\" |", "\x04"},
230 {"4.1.4 U+002F = f8 80 80 80 af = \"\xf8\x80\x80\x80\xaf\" |",
231 "4.1.4 U+002F = f8 80 80 80 af = \"\" |", "\x05"},
232 {"4.1.5 U+002F = fc 80 80 80 80 af = \"\xfc\x80\x80\x80\x80\xaf\" |",
233 "4.1.5 U+002F = fc 80 80 80 80 af = \"\" |", "\x06"},
234/* 4.2 Maximum overlong sequences
235 * Below you see the highest Unicode value that is still resulting in an overlong sequence if represented
236 * with the given number of bytes. This is a boundary test for safe UTF-8 decoders. All five characters
237 * should be rejected like malformed UTF-8 sequences. */
238 {"4.2.1 U-0000007F = c1 bf = \"\xc1\xbf\" |",
239 "4.2.1 U-0000007F = c1 bf = \"\" |", "\x02"},
240 {"4.2.2 U-000007FF = e0 9f bf = \"\xe0\x9f\xbf\" |",
241 "4.2.2 U-000007FF = e0 9f bf = \"\" |", "\x03"},
242 {"4.2.3 U-0000FFFF = f0 8f bf bf = \"\xf0\x8f\xbf\xbf\" |",
243 "4.2.3 U-0000FFFF = f0 8f bf bf = \"\" |", "\x04"},
244 {"4.2.4 U-001FFFFF = f8 87 bf bf bf = \"\xf8\x87\xbf\xbf\xbf\" |",
245 "4.2.4 U-001FFFFF = f8 87 bf bf bf = \"\" |", "\x05"},
246 {"4.2.5 U+0000 = fc 83 bf bf bf bf = \"\xfc\x83\xbf\xbf\xbf\xbf\" |",
247 "4.2.5 U+0000 = fc 83 bf bf bf bf = \"\" |", "\x06"},
248/* 4.3 Overlong representation of the NUL character
249 * The following five sequences should also be rejected like malformed UTF-8 sequences and should not be
250 * treated like the ASCII NUL character. */
251 {"4.3.1 U+0000 = c0 80 = \"\xc0\x80\" |",
252 "4.3.1 U+0000 = c0 80 = \"\" |", "\x02"},
253 {"4.3.2 U+0000 = e0 80 80 = \"\xe0\x80\x80\" |",
254 "4.3.2 U+0000 = e0 80 80 = \"\" |", "\x03"},
255 {"4.3.3 U+0000 = f0 80 80 80 = \"\xf0\x80\x80\x80\" |",
256 "4.3.3 U+0000 = f0 80 80 80 = \"\" |", "\x04"},
257 {"4.3.4 U+0000 = f8 80 80 80 80 = \"\xf8\x80\x80\x80\x80\" |",
258 "4.3.4 U+0000 = f8 80 80 80 80 = \"\" |", "\x05"},
259 {"4.3.5 U+0000 = fc 80 80 80 80 80 = \"\xfc\x80\x80\x80\x80\x80\" |",
260 "4.3.5 U+0000 = fc 80 80 80 80 80 = \"\" |", "\x06"},
261
262/* 5 Illegal code positions
263 * The following UTF-8 sequences should be rejected like malformed sequences, because they never represent
264 * valid ISO 10646 characters and a UTF-8 decoder that accepts them might introduce security problems
265 * comparable to overlong UTF-8 sequences.
266 * 5.1 Single UTF-16 surrogates */
267 {"5.1.1 U+D800 = ed a0 80 = \"\xed\xa0\x80\" |",
268 "5.1.1 U+D800 = ed a0 80 = \"\" |", "\x03"},
269 {"5.1.2 U+DB7F = ed ad bf = \"\xed\xad\xbf\" |",
270 "5.1.2 U+DB7F = ed ad bf = \"\" |", "\x03"},
271 {"5.1.3 U+DB80 = ed ae 80 = \"\xed\xae\x80\" |",
272 "5.1.3 U+DB80 = ed ae 80 = \"\" |", "\x03"},
273 {"5.1.4 U+DBFF = ed af bf = \"\xed\xaf\xbf\" |",
274 "5.1.4 U+DBFF = ed af bf = \"\" |", "\x03"},
275 {"5.1.5 U+DC00 = ed b0 80 = \"\xed\xb0\x80\" |",
276 "5.1.5 U+DC00 = ed b0 80 = \"\" |", "\x03"},
277 {"5.1.6 U+DF80 = ed be 80 = \"\xed\xbe\x80\" |",
278 "5.1.6 U+DF80 = ed be 80 = \"\" |", "\x03"},
279 {"5.1.7 U+DFFF = ed bf bf = \"\xed\xbf\xbf\" |",
280 "5.1.7 U+DFFF = ed bf bf = \"\" |", "\x03"},
281/* 5.2 Paired UTF-16 surrogates */
282 {"5.2.1 U+D800 U+DC00 = ed a0 80 ed b0 80 = \"\xed\xa0\x80\xed\xb0\x80\" |",
283 "5.2.1 U+D800 U+DC00 = ed a0 80 ed b0 80 = \"\" |", "\x06"},
284 {"5.2.2 U+D800 U+DFFF = ed a0 80 ed bf bf = \"\xed\xa0\x80\xed\xbf\xbf\" |",
285 "5.2.2 U+D800 U+DFFF = ed a0 80 ed bf bf = \"\" |", "\x06"},
286 {"5.2.3 U+DB7F U+DC00 = ed ad bf ed b0 80 = \"\xed\xad\xbf\xed\xb0\x80\" |",
287 "5.2.3 U+DB7F U+DC00 = ed ad bf ed b0 80 = \"\" |", "\x06"},
288 {"5.2.4 U+DB7F U+DFFF = ed ad bf ed bf bf = \"\xed\xad\xbf\xed\xbf\xbf\" |",
289 "5.2.4 U+DB7F U+DFFF = ed ad bf ed bf bf = \"\" |", "\x06"},
290 {"5.2.5 U+DB80 U+DC00 = ed ae 80 ed b0 80 = \"\xed\xae\x80\xed\xb0\x80\" |",
291 "5.2.5 U+DB80 U+DC00 = ed ae 80 ed b0 80 = \"\" |", "\x06"},
292 {"5.2.6 U+DB80 U+DFFF = ed ae 80 ed bf bf = \"\xed\xae\x80\xed\xbf\xbf\" |",
293 "5.2.6 U+DB80 U+DFFF = ed ae 80 ed bf bf = \"\" |", "\x06"},
294 {"5.2.7 U+DBFF U+DC00 = ed af bf ed b0 80 = \"\xed\xaf\xbf\xed\xb0\x80\" |",
295 "5.2.7 U+DBFF U+DC00 = ed af bf ed b0 80 = \"\" |", "\x06"},
296 {"5.2.8 U+DBFF U+DFFF = ed af bf ed bf bf = \"\xed\xaf\xbf\xed\xbf\xbf\" |",
297 "5.2.8 U+DBFF U+DFFF = ed af bf ed bf bf = \"\" |", "\x06"},
298/* 5.3 Non-character code positions
299 * The following "non-characters" are "reserved for internal use" by applications, and according to older versions
300 * of the Unicode Standard "should never be interchanged". Unicode Corrigendum #9 dropped the latter restriction.
301 * Nevertheless, their presence in incoming UTF-8 data can remain a potential security risk, depending
302 * on what use is made of these codes subsequently. Examples of such internal use:
303 * - Some file APIs with 16-bit characters may use the integer value -1 = U+FFFF to signal
304 * an end-of-file (EOF) or error condition.
305 * - In some UTF-16 receivers, code point U+FFFE might trigger a byte-swap operation
306 * (to convert between UTF-16LE and UTF-16BE).
307 * With such internal use of non-characters, it may be desirable and safer to block those code points in
308 * UTF-8 decoders, as they should never occur legitimately in incoming UTF-8 data, and could trigger
309 * unsafe behavior in subsequent processing.
310 *
311 * Particularly problematic non-characters in 16-bit applications: */
312 {"5.3.1 U+FFFE = ef bf be = \"\xef\xbf\xbe\" |",
313 "5.3.1 U+FFFE = ef bf be = \"\" |", "\x03"},
314 {"5.3.2 U+FFFF = ef bf bf = \"\xef\xbf\xbf\" |",
315 "5.3.2 U+FFFF = ef bf bf = \"\" |", "\x03"},
316 /* For now, we ignore those, they do not seem to be crucial anyway... */
317/* 5.3.3 U+FDD0 .. U+FDEF
318 * 5.3.4 U+nFFFE U+nFFFF (for n = 1..10) */
319 {nullptr, nullptr, nullptr},
320};
321/* clang-format on */
322
323/* BLI_str_utf8_invalid_strip (and indirectly, BLI_str_utf8_invalid_byte). */
324TEST(string, Utf8InvalidBytesStrip)
325{
326 for (int i = 0; utf8_invalid_tests[i][0] != nullptr; i++) {
327 const char *tst = utf8_invalid_tests[i][0];
328 const char *tst_stripped = utf8_invalid_tests[i][1];
329 const int errors_num = int(utf8_invalid_tests[i][2][0]);
330
331 char buff[80];
332 memcpy(buff, tst, sizeof(buff));
333
334 const int errors_found_num = BLI_str_utf8_invalid_strip(buff, sizeof(buff) - 1);
335
336 printf("[%02d] -> [%02d] \"%s\" -> \"%s\"\n", errors_num, errors_found_num, tst, buff);
337 EXPECT_EQ(errors_found_num, errors_num);
338 EXPECT_STREQ(buff, tst_stripped);
339 }
340}
341
342/* BLI_str_utf8_invalid_substitute (and indirectly, BLI_str_utf8_invalid_byte). */
343TEST(string, Utf8InvalidBytesSubstitute)
344{
345 for (int i = 0; utf8_invalid_tests[i][0] != nullptr; i++) {
346 const char *tst = utf8_invalid_tests[i][0];
347 const int errors_num = int(utf8_invalid_tests[i][2][0]);
348
349 char buff[80];
350 memcpy(buff, tst, sizeof(buff));
351
352 const int errors_found_num = BLI_str_utf8_invalid_substitute(buff, sizeof(buff) - 1, '?');
353
354 EXPECT_EQ(errors_found_num, errors_num);
355 EXPECT_EQ(BLI_str_utf8_invalid_byte(buff, sizeof(buff) - 1), -1);
356 EXPECT_EQ(strlen(buff), sizeof(buff) - 1);
357 }
358}
359
360TEST(string, Utf8InvalidBytesSubstitutePatterns)
361{
362#define TEST_SIMPLE(src_chars, expected_error_count, expected_str) \
363 { \
364 char buff[] = src_chars; \
365 EXPECT_EQ(BLI_str_utf8_invalid_substitute(buff, strlen(buff), '?'), expected_error_count); \
366 EXPECT_STREQ(buff, expected_str); \
367 } \
368 ((void)0)
369
370#define ARRAY_ARG(...) __VA_ARGS__
371
372 /* Empty string. */
373 TEST_SIMPLE(ARRAY_ARG({0x0}), 0, "");
374 /* One good. */
375 TEST_SIMPLE(ARRAY_ARG({'A', 0x0}), 0, "A");
376 /* One bad. */
377 TEST_SIMPLE(ARRAY_ARG({0xff, 0x0}), 1, "?");
378
379 /* Additional patterns. */
380 TEST_SIMPLE(ARRAY_ARG({0xe0, 0xef, 0x0}), 2, "??");
381 TEST_SIMPLE(ARRAY_ARG({0xe0, 'A', 0x0}), 1, "?A");
382 TEST_SIMPLE(ARRAY_ARG({'A', 0xef, 0x0}), 1, "A?");
383 TEST_SIMPLE(ARRAY_ARG({0xe0, 'A', 0xed, 0x0}), 2, "?A?");
384
385#undef ARRAY_ARG
386#undef TEST_SIMPLE
387}
388
390
391/* -------------------------------------------------------------------- */
394
395TEST(string, StringNLenUTF8)
396{
397 EXPECT_EQ(BLI_strnlen_utf8("", 0), 0);
398 EXPECT_EQ(BLI_strnlen_utf8("", 1), 0);
400}
401
402TEST(string, StringNLenUTF8_NoNil)
403{
404 /* Only fails with ASAN, reading past the end doesn't change the result. */
405 const char str[] = {'A', 'B', 'C'};
406 size_t len_bytes;
407 EXPECT_EQ(BLI_strnlen_utf8_ex(str, sizeof(str), &len_bytes), 3);
408 EXPECT_EQ(len_bytes, 3);
409}
410
411TEST(string, StringNLenUTF8_Incomplete)
412{
413 const char *ref_str =
415 char buf[22];
416 const size_t ref_str_len = 21;
417
418#define EXPECT_BYTE_OFFSET(truncate_ofs, expect_nchars) \
419 { \
420 size_t buf_ofs = 0; \
421 STRNCPY(buf, ref_str); \
422 buf[truncate_ofs] = '\0'; \
423 EXPECT_EQ(BLI_strnlen_utf8_ex(buf, ref_str_len, &buf_ofs), expect_nchars); \
424 EXPECT_EQ(buf_ofs, truncate_ofs); \
425 }
426
427 EXPECT_BYTE_OFFSET(0, 0);
428 EXPECT_BYTE_OFFSET(1, 1);
429
430 EXPECT_BYTE_OFFSET(2, 2);
431 EXPECT_BYTE_OFFSET(3, 2);
432
433 EXPECT_BYTE_OFFSET(4, 3);
434 EXPECT_BYTE_OFFSET(5, 3);
435 EXPECT_BYTE_OFFSET(6, 3);
436
437 EXPECT_BYTE_OFFSET(7, 4);
438 EXPECT_BYTE_OFFSET(8, 4);
439 EXPECT_BYTE_OFFSET(9, 4);
440 EXPECT_BYTE_OFFSET(10, 4);
441
442 EXPECT_BYTE_OFFSET(11, 5);
443 EXPECT_BYTE_OFFSET(12, 5);
444 EXPECT_BYTE_OFFSET(13, 5);
445 EXPECT_BYTE_OFFSET(14, 5);
446 EXPECT_BYTE_OFFSET(15, 5);
447
448 EXPECT_BYTE_OFFSET(16, 6);
449 EXPECT_BYTE_OFFSET(17, 6);
450 EXPECT_BYTE_OFFSET(18, 6);
451 EXPECT_BYTE_OFFSET(19, 6);
452 EXPECT_BYTE_OFFSET(20, 6);
453 EXPECT_BYTE_OFFSET(21, 6);
454
455#undef EXPECT_BYTE_OFFSET
456}
457
459
460/* -------------------------------------------------------------------- */
463
464TEST(string, StrCopyUTF8_ASCII)
465{
466#define STRNCPY_UTF8_ASCII(...) \
467 { \
468 const char src[] = {__VA_ARGS__, 0}; \
469 char dst[sizeof(src)]; \
470 memset(dst, 0xff, sizeof(dst)); \
471 STRNCPY_UTF8(dst, src); \
472 EXPECT_EQ(strlen(dst), sizeof(dst) - 1); \
473 EXPECT_STREQ(dst, src); \
474 }
475
477 STRNCPY_UTF8_ASCII('a', 'b', 'c');
478
479#undef STRNCPY_UTF8_ASCII
480}
481
482TEST(string, StrCopyUTF8_ASCII_Truncate)
483{
484#define STRNCPY_UTF8_ASCII_TRUNCATE(maxncpy, ...) \
485 { \
486 char src[] = {__VA_ARGS__}; \
487 char dst[sizeof(src)]; \
488 memset(dst, 0xff, sizeof(dst)); \
489 BLI_strncpy_utf8(dst, src, maxncpy); \
490 int len_expect = std::min<int>(sizeof(src), maxncpy) - 1; \
491 src[len_expect] = '\0'; /* To be able to use `EXPECT_STREQ`. */ \
492 EXPECT_EQ(strlen(dst), len_expect); \
493 EXPECT_STREQ(dst, src); \
494 }
495
497 STRNCPY_UTF8_ASCII_TRUNCATE(3, 'A', 'A', 'A', 'A');
498
499#undef STRNCPY_UTF8_ASCII_TRUNCATE
500}
501
502TEST(string, StrCopyUTF8_TruncateEncoding)
503{
504 /* Ensure copying one byte less than the code-point results in it being ignored entirely. */
505#define STRNCPY_UTF8_TRUNCATE(byte_size, ...) \
506 { \
507 const char src[] = {__VA_ARGS__, 0}; \
508 EXPECT_EQ(BLI_str_utf8_size_or_error(src), byte_size); \
509 char dst[sizeof(src)]; \
510 memset(dst, 0xff, sizeof(dst)); \
511 STRNCPY_UTF8(dst, src); \
512 EXPECT_EQ(strlen(dst), sizeof(dst) - 1); \
513 EXPECT_STREQ(dst, src); \
514 BLI_strncpy_utf8(dst, src, sizeof(dst) - 1); \
515 EXPECT_STREQ(dst, ""); \
516 }
517
518 STRNCPY_UTF8_TRUNCATE(6, 252, 1, 1, 1, 1, 1);
519 STRNCPY_UTF8_TRUNCATE(5, 248, 1, 1, 1, 1);
520 STRNCPY_UTF8_TRUNCATE(4, 240, 1, 1, 1);
521 STRNCPY_UTF8_TRUNCATE(3, 224, 1, 1);
522 STRNCPY_UTF8_TRUNCATE(2, 192, 1);
524
525#undef STRNCPY_UTF8_TRUNCATE
526}
527
528TEST(string, StrCopyUTF8_TruncateEncodingMulti)
529{
530#define STRNCPY_UTF8_TRUNC_EXPECT(src, dst_expect, dst_maxncpy) \
531 { \
532 char dst[dst_maxncpy + 1]; \
533 dst[dst_maxncpy] = 0xff; \
534 size_t len = BLI_strncpy_utf8_rlen(dst, src, dst_maxncpy); \
535 EXPECT_EQ(len, strlen(dst)); \
536 EXPECT_STREQ(dst, dst_expect); \
537 EXPECT_EQ(dst[dst_maxncpy], 0xff); \
538 }
539
540 /* Single characters. */
543
547
552
558
565
573
574 /* Multiple characters. */
580
588
598
610
624
625#undef STRNCPY_UTF8_TRUNC_EXPECT
626}
627
628TEST(string, StrCopyUTF8_TerminateEncodingEarly)
629{
630 /* A UTF8 sequence that has a null byte before the sequence ends.
631 * Ensure the UTF8 sequence does not step over the null byte. */
632#define STRNCPY_UTF8_TERMINATE_EARLY(byte_size, ...) \
633 { \
634 char src[] = {__VA_ARGS__, 0}; \
635 EXPECT_EQ(BLI_str_utf8_size_or_error(src), byte_size); \
636 char dst[sizeof(src)]; \
637 memset(dst, 0xff, sizeof(dst)); \
638 STRNCPY_UTF8(dst, src); \
639 EXPECT_EQ(strlen(dst), sizeof(dst) - 1); \
640 EXPECT_STREQ(dst, src); \
641 for (int i = sizeof(dst) - 1; i > 1; i--) { \
642 src[i] = '\0'; \
643 memset(dst, 0xff, sizeof(dst)); \
644 const int dst_copied = STRNCPY_UTF8_RLEN(dst, src); \
645 EXPECT_STREQ(dst, src); \
646 EXPECT_EQ(strlen(dst), i); \
647 EXPECT_EQ(dst_copied, i); \
648 } \
649 }
650
651 STRNCPY_UTF8_TERMINATE_EARLY(6, 252, 1, 1, 1, 1, 1);
652 STRNCPY_UTF8_TERMINATE_EARLY(5, 248, 1, 1, 1, 1);
653 STRNCPY_UTF8_TERMINATE_EARLY(4, 240, 1, 1, 1);
654 STRNCPY_UTF8_TERMINATE_EARLY(3, 224, 1, 1);
657
658#undef STRNCPY_UTF8_TERMINATE_EARLY
659}
660
662
663/* -------------------------------------------------------------------- */
666
667TEST(string, StrUTF32AsUTF8_Truncate)
668{
669#define STR_UTF32_AS_UTF8_TRUNC_EXPECT(src, dst_expect, dst_maxncpy) \
670 { \
671 char dst[dst_maxncpy + 1]; \
672 dst[dst_maxncpy] = 0xff; \
673 size_t len = BLI_str_utf32_as_utf8(dst, src, dst_maxncpy); \
674 EXPECT_EQ(len, strlen(dst)); \
675 EXPECT_STREQ(dst, dst_expect); \
676 EXPECT_EQ(dst[dst_maxncpy], 0xff); \
677 }
678
682
684 STR_UTF32_AS_UTF8_TRUNC_EXPECT(U"\xaa", "", 2);
685 STR_UTF32_AS_UTF8_TRUNC_EXPECT(U"\xaa", "", 1);
688
690 STR_UTF32_AS_UTF8_TRUNC_EXPECT(U"\x800", "", 3);
693
695 STR_UTF32_AS_UTF8_TRUNC_EXPECT(U"\x10000", "", 4);
698
699#undef STR_UTF32_AS_UTF8_TRUNC_EXPECT
700}
701
703
704/* -------------------------------------------------------------------- */
707
708TEST(string, StrCopyWCharAsUTF8_Truncate)
709{
710 /* Characters above `U+FFFF` are not tested as `wchar_t` is 16-bit on WIN32. */
711#define STRNCPY_WCHAR_AS_UTF8_TRUNC_EXPECT(src, dst_expect, dst_maxncpy) \
712 { \
713 char dst[dst_maxncpy + 1]; \
714 dst[dst_maxncpy] = 0xff; \
715 size_t len = BLI_strncpy_wchar_as_utf8(dst, src, dst_maxncpy); \
716 EXPECT_EQ(len, strlen(dst)); \
717 EXPECT_STREQ(dst, dst_expect); \
718 EXPECT_EQ(dst[dst_maxncpy], 0xff); \
719 }
720
724
729
734
735#undef STRNCPY_WCHAR_AS_UTF8_TRUNC_EXPECT
736}
737
739
740/* -------------------------------------------------------------------- */
743
744TEST(string, StrPrintfUTF8_ASCII)
745{
746#define SNPRINTF_UTF8_ASCII(...) \
747 { \
748 const char src[] = {__VA_ARGS__, 0}; \
749 char dst[sizeof(src)]; \
750 memset(dst, 0xff, sizeof(dst)); \
751 SNPRINTF_UTF8(dst, "%s", src); \
752 EXPECT_EQ(strlen(dst), sizeof(dst) - 1); \
753 EXPECT_STREQ(dst, src); \
754 }
755
757 SNPRINTF_UTF8_ASCII('a', 'b', 'c');
758
759#undef SNPRINTF_UTF8_ASCII
760}
761
762TEST(string, StrPrintfUTF8_TerminateEncodingEarly)
763{
764 /* A UTF8 sequence that has a null byte before the sequence ends.
765 * Ensure the UTF8 sequence does not step over the null byte. */
766#define SNPRINTF_UTF8_TERMINATE_EARLY(byte_size, ...) \
767 { \
768 char src[] = {__VA_ARGS__, 0}; \
769 EXPECT_EQ(BLI_str_utf8_size_or_error(src), byte_size); \
770 char dst[sizeof(src)]; \
771 memset(dst, 0xff, sizeof(dst)); \
772 SNPRINTF_UTF8(dst, "%s", src); \
773 EXPECT_EQ(strlen(dst), sizeof(dst) - 1); \
774 EXPECT_STREQ(dst, src); \
775 for (int i = sizeof(dst) - 1; i > 1; i--) { \
776 src[i] = '\0'; \
777 memset(dst, 0xff, sizeof(dst)); \
778 const int dst_copied = SNPRINTF_UTF8_RLEN(dst, "%s", src); \
779 EXPECT_STREQ(dst, src); \
780 EXPECT_EQ(strlen(dst), i); \
781 EXPECT_EQ(dst_copied, i); \
782 } \
783 }
784
785 SNPRINTF_UTF8_TERMINATE_EARLY(6, 252, 1, 1, 1, 1, 1);
786 SNPRINTF_UTF8_TERMINATE_EARLY(5, 248, 1, 1, 1, 1);
787 SNPRINTF_UTF8_TERMINATE_EARLY(4, 240, 1, 1, 1);
788 SNPRINTF_UTF8_TERMINATE_EARLY(3, 224, 1, 1);
791
792#undef STRNCPY_UTF8_TERMINATE_EARLY
793}
794
795TEST(string, StrPrintfUTF8_TruncateEncodingMulti)
796{
797#define SNPRINTF_UTF8_TRUNC_EXPECT(src, dst_expect, dst_maxncpy) \
798 { \
799 char dst[dst_maxncpy + 1]; \
800 dst[dst_maxncpy] = 0xff; \
801 size_t len = BLI_snprintf_utf8_rlen(dst, dst_maxncpy, "%s", src); \
802 EXPECT_EQ(len, strlen(dst)); \
803 EXPECT_STREQ(dst, dst_expect); \
804 EXPECT_EQ(dst[dst_maxncpy], 0xff); \
805 }
806
807 /* Single characters. */
810
814
819
825
832
840
841#undef STRNCPY_UTF8_TRUNC_EXPECT
842}
843
845
846/* -------------------------------------------------------------------- */
849
850TEST(string, Utf8OffsetFromIndex_ClampedIndex)
851{
852 /* Ensure an index that exceeds the number of multi-byte characters in the
853 * string has the same behavior as an index which is clamped by the number of code-points. */
854 const char *test_strings[] = {
855 "",
856 "TEST",
863 };
864 for (int i = 0; i < ARRAY_SIZE(test_strings); i++) {
865 const char *str = test_strings[i];
866 const size_t str_len = strlen(str);
867 const int str_len_utf8 = BLI_strlen_utf8(str);
868
869 const int str_offset = BLI_str_utf8_offset_from_index(str, str_len, str_len_utf8);
870 EXPECT_EQ(BLI_str_utf8_offset_from_index(str, str_len, str_len_utf8 + 1), str_offset);
871 EXPECT_EQ(BLI_str_utf8_offset_from_index(str, str_len, str_len_utf8 + 10), str_offset);
872 }
873}
874
876
877/* -------------------------------------------------------------------- */
880
891
892TEST(string, StrFindPrevCharUTF8_ContinuationOnly)
893{
894 /* Invalid UTF8, starts with continuation bytes. */
895 const char str[] = {char(0x80), char(0x80), 'A', 0};
898}
899
901
902/* -------------------------------------------------------------------- */
905
906static size_t utf8_as_char32(const char *str, const char str_len, char32_t *r_result)
907{
908 size_t i = 0, result_len = 0;
909 while ((i < str_len) && (str[i] != '\0')) {
910 r_result[result_len++] = BLI_str_utf8_as_unicode_step_safe(str, str_len, &i);
911 }
912 return i;
913}
914
915template<size_t Size, size_t SizeWithPadding>
916void utf8_as_char32_test_compare_with_pad_bytes(const char utf8_src[Size])
917{
918 char utf8_src_with_pad[SizeWithPadding] = {0};
919
920 memcpy(utf8_src_with_pad, utf8_src, Size);
921
922 char32_t unicode_dst_a[Size], unicode_dst_b[Size];
923
924 memset(unicode_dst_a, 0xff, sizeof(unicode_dst_a));
925 const size_t index_a = utf8_as_char32(utf8_src, Size, unicode_dst_a);
926
927 /* Test with padded and un-padded size,
928 * to ensure that extra available space doesn't yield a different result. */
929 for (int pass = 0; pass < 2; pass++) {
930 memset(unicode_dst_b, 0xff, sizeof(unicode_dst_b));
931 const size_t index_b = utf8_as_char32(
932 utf8_src_with_pad, pass ? Size : SizeWithPadding, unicode_dst_b);
933
934 /* Check the resulting content matches. */
935 EXPECT_EQ_ARRAY(unicode_dst_a, unicode_dst_b, Size);
936 /* Check the index of the source strings match. */
937 EXPECT_EQ(index_a, index_b);
938 }
939}
940
941template<size_t Size> void utf8_as_char32_test_compare(const char utf8_src[Size])
942{
943 /* Note that 7 is a little arbitrary,
944 * chosen since it's the maximum length of multi-byte character + 1
945 * to account for any errors that read past null bytes. */
948}
949
950template<size_t Size> void utf8_as_char32_test_at_buffer_size()
951{
952 char utf8_src[Size];
953
954 /* Test uniform bytes, also with offsets ascending & descending. */
955 for (int i = 0; i <= 0xff; i++) {
956 memset(utf8_src, i, sizeof(utf8_src));
958
959 /* Offset trailing bytes up and down in steps of 1, 2, 4 .. etc. */
960 if (Size > 1) {
961 for (int mul = 1; mul < 256; mul *= 2) {
962 for (int ofs = 1; ofs < int(Size); ofs++) {
963 utf8_src[ofs] = char(i + (ofs * mul));
964 }
966
967 for (int ofs = 1; ofs < int(Size); ofs++) {
968 utf8_src[ofs] = char(i - (ofs * mul));
969 }
971 }
972 }
973 }
974
975 /* Random bytes. */
976 RNG *rng = BLI_rng_new(1);
977 for (int i = 0; i < 256; i++) {
978 BLI_rng_get_char_n(rng, utf8_src, sizeof(utf8_src));
980 }
981 BLI_rng_free(rng);
982}
983
1001
1003
1004/* -------------------------------------------------------------------- */
1007
1008TEST(string, StrCursorStepNextUtf32Empty)
1009{
1010 const char32_t empty[] = U"";
1011 const size_t len = 0;
1012 int pos = 0;
1014 pos = 1;
1016}
1017
1019
1020/* -------------------------------------------------------------------- */
1023
1024TEST(string, StrCursorStepNextUtf32Single)
1025
1026{
1027 const char32_t single[] = U"0";
1028 const size_t len = 1;
1029 int pos = 0;
1030 EXPECT_TRUE(BLI_str_cursor_step_next_utf32(single, len, &pos) && pos == 1);
1032}
1033
1035
1036/* -------------------------------------------------------------------- */
1039
1040TEST(string, StrCursorStepNextUtf32Simple)
1041{
1042 const char32_t simple[] = U"012";
1043 const size_t len = 3;
1044 int pos = 0;
1045 EXPECT_TRUE(BLI_str_cursor_step_next_utf32(simple, len, &pos) && pos == 1);
1046 EXPECT_TRUE(BLI_str_cursor_step_next_utf32(simple, len, &pos) && pos == 2);
1048 EXPECT_TRUE(BLI_str_cursor_step_next_utf32(simple, len, &pos) && pos == 3);
1050}
1051
1053
1054/* -------------------------------------------------------------------- */
1057
1058TEST(string, StrCursorStepNextUtf32AllCombining)
1059{
1060 const char32_t allcombining[] = U"\u0300\u0300\u0300";
1061 const size_t len = 3;
1062 int pos = 0;
1063 EXPECT_TRUE(BLI_str_cursor_step_next_utf32(allcombining, len, &pos) && pos == 3);
1064 pos = 1;
1065 EXPECT_TRUE(BLI_str_cursor_step_next_utf32(allcombining, len, &pos) && pos == 3);
1066 pos = 2;
1067 EXPECT_TRUE(BLI_str_cursor_step_next_utf32(allcombining, len, &pos) && pos == 3);
1068 pos = 3;
1069 EXPECT_FALSE(BLI_str_cursor_step_next_utf32(allcombining, len, &pos));
1070}
1071
1073
1074/* -------------------------------------------------------------------- */
1077
1078TEST(string, StrCursorStepNextUtf32Complex)
1079{
1080 /* Combining character, "A", two combining characters, "B". */
1081 const char32_t complex[] = U"\u0300\u0041\u0300\u0320\u0042";
1082 const size_t len = 5;
1083 int pos = 0;
1085 pos = 1;
1087 pos = 2;
1089 pos = 3;
1091 pos = 4;
1093 pos = 5;
1095}
1096
1098
1099/* -------------------------------------------------------------------- */
1102
1103TEST(string, StrCursorStepNextUtf32Invalid)
1104{
1105 /* Latin1 "À", tab, carriage return, linefeed, separated by combining characters. */
1106 const char32_t invalid[] = U"\u00C0\u0300\u0009\u0300\u000D\u0300\u000A\u0300";
1107 const size_t len = 8;
1108 int pos = 0;
1109 EXPECT_TRUE(BLI_str_cursor_step_next_utf32(invalid, len, &pos) && pos == 2);
1110 pos = 1;
1111 EXPECT_TRUE(BLI_str_cursor_step_next_utf32(invalid, len, &pos) && pos == 2);
1112 pos = 2;
1113 EXPECT_TRUE(BLI_str_cursor_step_next_utf32(invalid, len, &pos) && pos == 4);
1114 pos = 3;
1115 EXPECT_TRUE(BLI_str_cursor_step_next_utf32(invalid, len, &pos) && pos == 4);
1116 pos = 4;
1117 EXPECT_TRUE(BLI_str_cursor_step_next_utf32(invalid, len, &pos) && pos == 6);
1118 pos = 5;
1119 EXPECT_TRUE(BLI_str_cursor_step_next_utf32(invalid, len, &pos) && pos == 6);
1120 pos = 6;
1121 EXPECT_TRUE(BLI_str_cursor_step_next_utf32(invalid, len, &pos) && pos == 8);
1122 pos = 7;
1123 EXPECT_TRUE(BLI_str_cursor_step_next_utf32(invalid, len, &pos) && pos == 8);
1124 pos = 8;
1126}
1127
1129
1130/* -------------------------------------------------------------------- */
1133
1134TEST(string, StrCursorStepPrevUtf32Empty)
1135{
1136 const char32_t empty[] = U"";
1137 const size_t len = 0;
1138 int pos = 0;
1140}
1141
1143
1144/* -------------------------------------------------------------------- */
1147
1148TEST(string, StrCursorStepPrevUtf32Single)
1149{
1150 const char32_t single[] = U"0";
1151 const size_t len = 1;
1152 int pos = 1;
1153 EXPECT_TRUE(BLI_str_cursor_step_prev_utf32(single, len, &pos) && pos == 0);
1155}
1156
1158
1159/* -------------------------------------------------------------------- */
1162
1163TEST(string, StrCursorStepPrevUtf32Simple)
1164{
1165 const char32_t simple[] = U"012";
1166 const size_t len = 3;
1167 int pos = 3;
1168 EXPECT_TRUE(BLI_str_cursor_step_prev_utf32(simple, len, &pos) && pos == 2);
1169 EXPECT_TRUE(BLI_str_cursor_step_prev_utf32(simple, len, &pos) && pos == 1);
1170 EXPECT_TRUE(BLI_str_cursor_step_prev_utf32(simple, len, &pos) && pos == 0);
1172}
1173
1175
1176/* -------------------------------------------------------------------- */
1179
1180TEST(string, StrCursorStepPrevUtf32AllCombining)
1181{
1182 const char32_t allcombining[] = U"\u0300\u0300\u0300";
1183 const size_t len = 3;
1184 int pos = 3;
1185 EXPECT_TRUE(BLI_str_cursor_step_prev_utf32(allcombining, len, &pos) && pos == 0);
1186 pos = 2;
1187 EXPECT_TRUE(BLI_str_cursor_step_prev_utf32(allcombining, len, &pos) && pos == 0);
1188 pos = 1;
1189 EXPECT_TRUE(BLI_str_cursor_step_prev_utf32(allcombining, len, &pos) && pos == 0);
1190 pos = 0;
1191 EXPECT_FALSE(BLI_str_cursor_step_prev_utf32(allcombining, len, &pos));
1192}
1193
1195
1196/* -------------------------------------------------------------------- */
1199
1200TEST(string, StrCursorStepPrevUtf32Complex)
1201{
1202 /* Combining character, "A", two combining characters, "B". */
1203 const char32_t complex[] = U"\u0300\u0041\u0300\u0320\u0042";
1204 const size_t len = 5;
1205 int pos = 5;
1207 pos = 4;
1209 pos = 3;
1211 pos = 2;
1213 pos = 1;
1215 pos = 0;
1217}
1218
1220
1221/* -------------------------------------------------------------------- */
1224
1225TEST(string, StrCursorStepPrevUtf32Invalid)
1226{
1227 /* Latin1 "À", tab, carriage return, linefeed, separated by combining characters. */
1228 const char32_t invalid[] = U"\u00C0\u0300\u0009\u0300\u000D\u0300\u000A\u0300";
1229 const size_t len = 8;
1230 int pos = 8;
1231 EXPECT_TRUE(BLI_str_cursor_step_prev_utf32(invalid, len, &pos) && pos == 6);
1232 pos = 7;
1233 EXPECT_TRUE(BLI_str_cursor_step_prev_utf32(invalid, len, &pos) && pos == 6);
1234 pos = 6;
1235 EXPECT_TRUE(BLI_str_cursor_step_prev_utf32(invalid, len, &pos) && pos == 4);
1236 pos = 5;
1237 EXPECT_TRUE(BLI_str_cursor_step_prev_utf32(invalid, len, &pos) && pos == 4);
1238 pos = 4;
1239 EXPECT_TRUE(BLI_str_cursor_step_prev_utf32(invalid, len, &pos) && pos == 2);
1240 pos = 3;
1241 EXPECT_TRUE(BLI_str_cursor_step_prev_utf32(invalid, len, &pos) && pos == 2);
1242 pos = 2;
1243 EXPECT_TRUE(BLI_str_cursor_step_prev_utf32(invalid, len, &pos) && pos == 0);
1244 pos = 1;
1245 EXPECT_TRUE(BLI_str_cursor_step_prev_utf32(invalid, len, &pos) && pos == 0);
1246 pos = 0;
1248}
1249
1251
1252/* -------------------------------------------------------------------- */
1255TEST(string, StrCursorStepNextUtf8Empty)
1256{
1257 const char empty[] = "";
1258 const size_t len = 0;
1259 int pos = 0;
1261 pos = 1;
1263}
1264
1266
1267/* -------------------------------------------------------------------- */
1270TEST(string, StrCursorStepNextUtf8Single)
1271{
1272 const char single[] = "0";
1273 const size_t len = 1;
1274 int pos = 0;
1275 EXPECT_TRUE(BLI_str_cursor_step_next_utf8(single, len, &pos) && pos == 1);
1277}
1278
1280
1281/* -------------------------------------------------------------------- */
1284
1285TEST(string, StrCursorStepNextUtf8Simple)
1286{
1287 const char simple[] = "012";
1288 const size_t len = 3;
1289 int pos = 0;
1290 EXPECT_TRUE(BLI_str_cursor_step_next_utf8(simple, len, &pos) && pos == 1);
1291 EXPECT_TRUE(BLI_str_cursor_step_next_utf8(simple, len, &pos) && pos == 2);
1293 EXPECT_TRUE(BLI_str_cursor_step_next_utf8(simple, len, &pos) && pos == 3);
1295}
1296
1298
1299/* -------------------------------------------------------------------- */
1302
1303TEST(string, StrCursorStepNextUtf8AllCombining)
1304{
1305 const char allcombining[] = "\xCC\x80\xCC\x80\xCC\x80";
1306 const size_t len = 6;
1307 int pos = 0;
1308 EXPECT_TRUE(BLI_str_cursor_step_next_utf8(allcombining, len, &pos) && pos == 6);
1309 pos = 1;
1310 EXPECT_TRUE(BLI_str_cursor_step_next_utf8(allcombining, len, &pos) && pos == 6);
1311 pos = 2;
1312 EXPECT_TRUE(BLI_str_cursor_step_next_utf8(allcombining, len, &pos) && pos == 6);
1313 pos = 3;
1314 EXPECT_TRUE(BLI_str_cursor_step_next_utf8(allcombining, len, &pos) && pos == 6);
1315 pos = 4;
1316 EXPECT_TRUE(BLI_str_cursor_step_next_utf8(allcombining, len, &pos) && pos == 6);
1317 pos = 5;
1318 EXPECT_TRUE(BLI_str_cursor_step_next_utf8(allcombining, len, &pos) && pos == 6);
1319 pos = 6;
1320 EXPECT_FALSE(BLI_str_cursor_step_next_utf8(allcombining, len, &pos));
1321}
1322
1324
1325/* -------------------------------------------------------------------- */
1328
1329TEST(string, StrCursorStepNextUtf8AllComplex)
1330{
1331 /* Combining character, "A", "©", two combining characters, "B". */
1332 const char complex[] = "\xCC\x80\x41\xC2\xA9\xCC\x80\xCC\xA0\x42";
1333 const size_t len = 10;
1334 int pos = 0;
1336 pos = 1;
1338 pos = 2;
1340 pos = 3;
1342 pos = 4;
1344 pos = 5;
1346 pos = 6;
1348 pos = 7;
1350 pos = 8;
1352 pos = 9;
1354 pos = 10;
1356}
1357
1359
1360/* -------------------------------------------------------------------- */
1363
1364TEST(string, StrCursorStepNextUtf8Invalid)
1365{
1366 /* Latin1 "À", combining, tab, carriage return, linefeed, combining. */
1367 const char invalid[] = "\xC0\xCC\x80\x09\x0D\x0A\xCC\x80";
1368 const size_t len = 8;
1369 int pos = 0;
1370 EXPECT_EQ(BLI_str_cursor_step_next_utf8(invalid, len, &pos) ? pos : -1, 3);
1371 pos = 1;
1372 EXPECT_EQ(BLI_str_cursor_step_next_utf8(invalid, len, &pos) ? pos : -1, 3);
1373 pos = 2;
1374 EXPECT_EQ(BLI_str_cursor_step_next_utf8(invalid, len, &pos) ? pos : -1, 3);
1375 pos = 3;
1376 EXPECT_EQ(BLI_str_cursor_step_next_utf8(invalid, len, &pos) ? pos : -1, 4);
1377 pos = 4;
1378 EXPECT_EQ(BLI_str_cursor_step_next_utf8(invalid, len, &pos) ? pos : -1, 5);
1379 pos = 5;
1380 EXPECT_EQ(BLI_str_cursor_step_next_utf8(invalid, len, &pos) ? pos : -1, 8);
1381 pos = 6;
1382 EXPECT_EQ(BLI_str_cursor_step_next_utf8(invalid, len, &pos) ? pos : -1, 8);
1383 pos = 7;
1384 EXPECT_EQ(BLI_str_cursor_step_next_utf8(invalid, len, &pos) ? pos : -1, 8);
1385 pos = 8;
1387}
1388
1390
1391/* -------------------------------------------------------------------- */
1394
1395TEST(string, StrCursorStepPrevUtf8Empty)
1396{
1397 const char empty[] = "";
1398 const size_t len = 0;
1399 int pos = 0;
1401 pos = 1;
1403}
1404
1406
1407/* -------------------------------------------------------------------- */
1410
1411TEST(string, StrCursorStepPrevUtf8Single)
1412{
1413 const char single[] = "0";
1414 const size_t len = 1;
1415 int pos = 1;
1416 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(single, len, &pos) && pos == 0);
1418}
1419
1421
1422/* -------------------------------------------------------------------- */
1425
1426TEST(string, StrCursorStepPrevUtf8Simple)
1427{
1428 const char simple[] = "012";
1429 const size_t len = 3;
1430 int pos = 3;
1431 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(simple, len, &pos) && pos == 2);
1432 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(simple, len, &pos) && pos == 1);
1433 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(simple, len, &pos) && pos == 0);
1435}
1436
1438
1439/* -------------------------------------------------------------------- */
1442
1443TEST(string, StrCursorStepPrevUtf8AllCombining)
1444{
1445 const char allcombining[] = "\xCC\x80\xCC\x80\xCC\x80";
1446 const size_t len = 6;
1447 int pos = 6;
1448 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(allcombining, len, &pos) && pos == 0);
1449 pos = 5;
1450 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(allcombining, len, &pos) && pos == 0);
1451 pos = 4;
1452 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(allcombining, len, &pos) && pos == 0);
1453 pos = 3;
1454 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(allcombining, len, &pos) && pos == 0);
1455 pos = 2;
1456 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(allcombining, len, &pos) && pos == 0);
1457 pos = 1;
1458 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(allcombining, len, &pos) && pos == 0);
1459 pos = 0;
1460 EXPECT_FALSE(BLI_str_cursor_step_prev_utf8(allcombining, len, &pos));
1461}
1462
1464
1465/* -------------------------------------------------------------------- */
1468
1469TEST(string, StrCursorStepPrevUtf8Complex)
1470{
1471 /* Combining character, "A", "©", two combining characters, "B". */
1472 const char complex[] = "\xCC\x80\x41\xC2\xA9\xCC\x80\xCC\xA0\x42";
1473 const size_t len = 10;
1474 int pos = 10;
1476 pos = 9;
1478 pos = 8;
1480 pos = 7;
1482 pos = 6;
1484 pos = 5;
1486 pos = 4;
1488 pos = 3;
1490 pos = 2;
1492 pos = 1;
1494 pos = 0;
1496}
1497
1499
1500/* -------------------------------------------------------------------- */
1503
1504TEST(string, StrCursorStepPrevUtf8Invalid)
1505{
1506 /* Latin1 "À", combining, tab, carriage return, linefeed, combining. */
1507 const char invalid[] = "\xC0\xCC\x80\x09\x0D\x0A\xCC\x80";
1508 const size_t len = 8;
1509 int pos = 8;
1510 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(invalid, len, &pos) && pos == 5);
1511 pos = 7;
1512 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(invalid, len, &pos) && pos == 5);
1513 pos = 6;
1514 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(invalid, len, &pos) && pos == 5);
1515 pos = 5;
1516 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(invalid, len, &pos) && pos == 4);
1517 pos = 4;
1518 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(invalid, len, &pos) && pos == 3);
1519 pos = 3;
1520 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(invalid, len, &pos) && pos == 0);
1521 pos = 2;
1522 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(invalid, len, &pos) && pos == 0);
1523 pos = 1;
1524 EXPECT_TRUE(BLI_str_cursor_step_prev_utf8(invalid, len, &pos) && pos == 0);
1525 pos = 0;
1527}
1528
1530
1531} // namespace blender
Random number functions.
#define STR_MB_ALPHA_1
#define STR_MB_ALPHA_5
#define SNPRINTF_UTF8_ASCII(...)
#define STRNCPY_UTF8_TRUNCATE(byte_size,...)
#define SNPRINTF_UTF8_TERMINATE_EARLY(byte_size,...)
#define STR_UTF32_AS_UTF8_TRUNC_EXPECT(src, dst_expect, dst_maxncpy)
#define EXPECT_BYTE_OFFSET(truncate_ofs, expect_nchars)
#define STRNCPY_UTF8_ASCII_TRUNCATE(maxncpy,...)
#define STRNCPY_UTF8_ASCII(...)
#define STR_MB_ALPHA_3
#define STRNCPY_UTF8_TERMINATE_EARLY(byte_size,...)
#define STR_MB_ALPHA_2
#define STRNCPY_WCHAR_AS_UTF8_TRUNC_EXPECT(src, dst_expect, dst_maxncpy)
#define SNPRINTF_UTF8_TRUNC_EXPECT(src, dst_expect, dst_maxncpy)
#define STR_MB_ALPHA_6
#define STRNCPY_UTF8_TRUNC_EXPECT(src, dst_expect, dst_maxncpy)
#define ARRAY_ARG(...)
#define STR_MB_ALPHA_4
#define TEST_SIMPLE(src_chars, expected_error_count, expected_str)
#define str(s)
#define EXPECT_FALSE(result)
#define EXPECT_TRUE(result)
ccl_device_inline dual1 len(const ccl_private dual< T > &a)
Definition math_dual.h:263
#define L
BMesh Iterators.
UserDef U
Definition blender.cc:58
static const char * utf8_invalid_tests[][3]
void utf8_as_char32_test_compare(const char utf8_src[Size])
ptrdiff_t BLI_str_utf8_invalid_byte(const char *str, size_t str_len) ATTR_NONNULL(1)
bool BLI_str_cursor_step_next_utf8(const char *str, int str_maxlen, int *pos)
size_t size_t BLI_strnlen_utf8(const char *strc, size_t strc_maxlen) ATTR_NONNULL(1) ATTR_WARN_UNUSED_RESULT
static size_t utf8_as_char32(const char *str, const char str_len, char32_t *r_result)
unsigned int BLI_str_utf8_as_unicode_step_safe(const char *__restrict p, size_t p_len, size_t *__restrict index) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1
struct RNG * BLI_rng_new(unsigned int seed)
Definition rand.cc:41
TEST(array_store, Nop)
void utf8_as_char32_test_compare_with_pad_bytes(const char utf8_src[Size])
void BLI_rng_get_char_n(RNG *rng, char *bytes, size_t bytes_len) ATTR_NONNULL(1
Definition rand.cc:70
int BLI_str_utf8_invalid_substitute(char *str, size_t str_len, const char substitute) ATTR_NONNULL(1)
constexpr size_t ARRAY_SIZE(const T(&arg)[N]) noexcept
bool BLI_str_cursor_step_prev_utf8(const char *str, int str_maxlen, int *pos)
void BLI_rng_free(struct RNG *rng) ATTR_NONNULL(1)
Definition rand.cc:55
EXPECT_EQ(BLI_expr_pylike_eval(expr, nullptr, 0, &result), EXPR_PYLIKE_INVALID)
int BLI_str_utf8_offset_from_index(const char *str, size_t str_len, int index_target) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
const char * BLI_str_find_prev_char_utf8(const char *p, const char *str_start) ATTR_WARN_UNUSED_RESULT ATTR_RETURNS_NONNULL ATTR_NONNULL(1
bool BLI_str_cursor_step_next_utf32(const char32_t *str, int str_maxlen, int *pos)
size_t BLI_strlen_utf8(const char *strc) ATTR_NONNULL(1) ATTR_WARN_UNUSED_RESULT
void utf8_as_char32_test_at_buffer_size()
bool BLI_str_cursor_step_prev_utf32(const char32_t *str, int str_maxlen, int *pos)
int BLI_str_utf8_invalid_strip(char *str, size_t str_len) ATTR_NONNULL(1)
size_t BLI_strnlen_utf8_ex(const char *strc, size_t strc_maxlen, size_t *r_len_bytes) ATTR_NONNULL(1