static int utf8_is_valid(const unsigned char *buf, size_t bufsize) {
unsigned int i = 0;
while (i < bufsize) {
unsigned char c = buf[i];
if (c > 127) {
if (!U8_IS_LEAD(c)) {
return 0;
}
i++;
unsigned int char_length;
char_length = U8_COUNT_TRAIL_BYTES(c);
if (char_length > (bufsize - i) - 1) {
return 0;
}
while (char_length > 0) {
assert(i < bufsize);
if (!U8_IS_TRAIL(buf[i])) {
return 0;
}
char_length--;
i++;
}
continue;
}
i++;
}
return utf8_iterate_codepoints(buf, bufsize,
&utf8_codepoint_is_categorized);
}
static void TestCodeUnitValues()
{
static const uint8_t codeunit[]={0x00, 0x65, 0x7e, 0x7f, 0xc0, 0xc4, 0xf0, 0xfd, 0x80, 0x81, 0xbc, 0xbe,};
int16_t i;
for(i=0; i<sizeof(codeunit)/sizeof(codeunit[0]); i++){
uint8_t c=codeunit[i];
log_verbose("Testing code unit value of %x\n", c);
if(i<4){
if(!UTF8_IS_SINGLE(c) || UTF8_IS_LEAD(c) || UTF8_IS_TRAIL(c) || !U8_IS_SINGLE(c) || U8_IS_LEAD(c) || U8_IS_TRAIL(c)){
log_err("ERROR: 0x%02x is a single byte but results in single: %c lead: %c trail: %c\n",
c, UTF8_IS_SINGLE(c) ? 'y' : 'n', UTF8_IS_LEAD(c) ? 'y' : 'n', UTF8_IS_TRAIL(c) ? 'y' : 'n');
}
} else if(i< 8){
if(!UTF8_IS_LEAD(c) || UTF8_IS_SINGLE(c) || UTF8_IS_TRAIL(c) || !U8_IS_LEAD(c) || U8_IS_SINGLE(c) || U8_IS_TRAIL(c)){
log_err("ERROR: 0x%02x is a lead byte but results in single: %c lead: %c trail: %c\n",
c, UTF8_IS_SINGLE(c) ? 'y' : 'n', UTF8_IS_LEAD(c) ? 'y' : 'n', UTF8_IS_TRAIL(c) ? 'y' : 'n');
}
} else if(i< 12){
if(!UTF8_IS_TRAIL(c) || UTF8_IS_SINGLE(c) || UTF8_IS_LEAD(c) || !U8_IS_TRAIL(c) || U8_IS_SINGLE(c) || U8_IS_LEAD(c)){
log_err("ERROR: 0x%02x is a trail byte but results in single: %c lead: %c trail: %c\n",
c, UTF8_IS_SINGLE(c) ? 'y' : 'n', UTF8_IS_LEAD(c) ? 'y' : 'n', UTF8_IS_TRAIL(c) ? 'y' : 'n');
}
}
}
}
/*
* Handle the non-inline part of the U8_NEXT() and U8_NEXT_FFFD() macros
* and their obsolete sibling UTF8_NEXT_CHAR_SAFE().
*
* U8_NEXT() supports NUL-terminated strings indicated via length<0.
*
* The "strict" parameter controls the error behavior:
* <0 "Safe" behavior of U8_NEXT():
* -1: All illegal byte sequences yield U_SENTINEL=-1.
* -2: Same as -1, except for lenient treatment of surrogate code points as legal.
* Some implementations use this for roundtripping of
* Unicode 16-bit strings that are not well-formed UTF-16, that is, they
* contain unpaired surrogates.
* -3: All illegal byte sequences yield U+FFFD.
* 0 Obsolete "safe" behavior of UTF8_NEXT_CHAR_SAFE(..., FALSE):
* All illegal byte sequences yield a positive code point such that this
* result code point would be encoded with the same number of bytes as
* the illegal sequence.
* >0 Obsolete "strict" behavior of UTF8_NEXT_CHAR_SAFE(..., TRUE):
* Same as the obsolete "safe" behavior, but non-characters are also treated
* like illegal sequences.
*
* Note that a UBool is the same as an int8_t.
*/
U_CAPI UChar32 U_EXPORT2
utf8_nextCharSafeBody(const uint8_t *s, int32_t *pi, int32_t length, UChar32 c, UBool strict) {
int32_t i=*pi;
uint8_t count=U8_COUNT_TRAIL_BYTES(c);
U_ASSERT(count <= 5); /* U8_COUNT_TRAIL_BYTES returns value 0...5 */
if(i+count<=length || length<0) {
uint8_t trail;
U8_MASK_LEAD_BYTE(c, count);
/* support NUL-terminated strings: do not read beyond the first non-trail byte */
switch(count) {
/* each branch falls through to the next one */
case 0:
/* count==0 for illegally leading trail bytes and the illegal bytes 0xfe and 0xff */
case 5:
case 4:
/* count>=4 is always illegal: no more than 3 trail bytes in Unicode's UTF-8 */
break;
case 3:
trail=s[i++]-0x80;
c=(c<<6)|trail;
/* c>=0x110 would result in code point>0x10ffff, outside Unicode */
if(c>=0x110 || trail>0x3f) { break; }
case 2:
trail=s[i++]-0x80;
c=(c<<6)|trail;
/*
* test for a surrogate d800..dfff unless we are lenient:
* before the last (c<<6), a surrogate is c=360..37f
*/
if(((c&0xffe0)==0x360 && strict!=-2) || trail>0x3f) { break; }
case 1:
trail=s[i++]-0x80;
c=(c<<6)|trail;
if(trail>0x3f) { break; }
/* correct sequence - all trail bytes have (b7..b6)==(10) */
if(c>=utf8_minLegal[count] &&
/* strict: forbid non-characters like U+fffe */
(strict<=0 || !U_IS_UNICODE_NONCHAR(c))) {
*pi=i;
return c;
}
/* no default branch to optimize switch() - all values are covered */
}
} else {
/* too few bytes left */
count=length-i;
}
/* error handling */
i=*pi;
while(count>0 && U8_IS_TRAIL(s[i])) {
++i;
--count;
}
c=errorValue(i-*pi, strict);
*pi=i;
return c;
}
