// Missing glyphs run constructor. Core Text will not generate a run of missing glyphs, instead falling back on
// glyphs from LastResort. We want to use the primary font's missing glyph in order to match the fast text code path.
CoreTextController::CoreTextRun::CoreTextRun(const SimpleFontData* fontData, const UChar* characters, unsigned stringLocation, size_t stringLength, bool ltr)
: m_fontData(fontData)
, m_characters(characters)
, m_stringLocation(stringLocation)
, m_stringLength(stringLength)
{
Vector<CFIndex, 16> indices;
unsigned r = 0;
while (r < stringLength) {
indices.append(r);
if (U_IS_SURROGATE(characters[r])) {
ASSERT(r + 1 < stringLength);
ASSERT(U_IS_SURROGATE_LEAD(characters[r]));
ASSERT(U_IS_TRAIL(characters[r + 1]));
r += 2;
} else
r++;
}
m_glyphCount = indices.size();
if (!ltr) {
for (unsigned r = 0, end = m_glyphCount - 1; r < m_glyphCount / 2; ++r, --end)
std::swap(indices[r], indices[end]);
}
m_indicesData.adoptCF(CFDataCreateMutable(kCFAllocatorDefault, m_glyphCount * sizeof(CFIndex)));
CFDataAppendBytes(m_indicesData.get(), reinterpret_cast<const UInt8*>(indices.data()), m_glyphCount * sizeof(CFIndex));
m_indices = reinterpret_cast<const CFIndex*>(CFDataGetBytePtr(m_indicesData.get()));
}
static void TestAppend() {
static const UChar32 codePoints[]={
0x61, 0xdf, 0x901, 0x3040,
0xac00, 0xd800, 0xdbff, 0xdcde,
0xdffd, 0xe000, 0xffff, 0x10000,
0x12345, 0xe0021, 0x10ffff, 0x110000,
0x234567, 0x7fffffff, -1, -1000,
0, 0x400
};
static const UChar expectUnsafe[]={
0x61, 0xdf, 0x901, 0x3040,
0xac00, 0xd800, 0xdbff, 0xdcde,
0xdffd, 0xe000, 0xffff, 0xd800, 0xdc00,
0xd848, 0xdf45, 0xdb40, 0xdc21, 0xdbff, 0xdfff, /* not 0x110000 */
/* none from this line */
0, 0x400
}, expectSafe[]={
0x61, 0xdf, 0x901, 0x3040,
0xac00, 0xd800, 0xdbff, 0xdcde,
0xdffd, 0xe000, 0xffff, 0xd800, 0xdc00,
0xd848, 0xdf45, 0xdb40, 0xdc21, 0xdbff, 0xdfff, /* not 0x110000 */
/* none from this line */
0, 0x400
};
UChar buffer[100];
UChar32 c;
int32_t i, length;
UBool isError, expectIsError, wrongIsError;
length=0;
for(i=0; i<LENGTHOF(codePoints); ++i) {
c=codePoints[i];
if(c<0 || 0x10ffff<c) {
continue; /* skip non-code points for U16_APPEND_UNSAFE */
}
U16_APPEND_UNSAFE(buffer, length, c);
}
if(length!=LENGTHOF(expectUnsafe) || 0!=memcmp(buffer, expectUnsafe, length*U_SIZEOF_UCHAR)) {
log_err("U16_APPEND_UNSAFE did not generate the expected output\n");
}
length=0;
wrongIsError=FALSE;
for(i=0; i<LENGTHOF(codePoints); ++i) {
c=codePoints[i];
expectIsError= c<0 || 0x10ffff<c || U_IS_SURROGATE(c);
isError=FALSE;
U16_APPEND(buffer, length, LENGTHOF(buffer), c, isError);
wrongIsError|= isError!=expectIsError;
}
if(wrongIsError) {
log_err("U16_APPEND did not set isError correctly\n");
}
if(length!=LENGTHOF(expectSafe) || 0!=memcmp(buffer, expectSafe, length*U_SIZEOF_UCHAR)) {
log_err("U16_APPEND did not generate the expected output\n");
}
}
void Normalizer2DataBuilder::setRoundTripMapping(UChar32 c, const UnicodeString &m) {
if(U_IS_SURROGATE(c)) {
fprintf(stderr,
"error in gennorm2 phase %d: "
"illegal round-trip mapping from surrogate code point U+%04lX\n",
(int)phase, (long)c);
exit(U_INVALID_FORMAT_ERROR);
}
if(!isWellFormed(m)) {
fprintf(stderr,
"error in gennorm2 phase %d: "
"illegal round-trip mapping from U+%04lX to malformed string\n",
(int)phase, (long)c);
exit(U_INVALID_FORMAT_ERROR);
}
int32_t numCP=u_countChar32(m.getBuffer(), m.length());
if(numCP!=2) {
fprintf(stderr,
"error in gennorm2 phase %d: "
"illegal round-trip mapping from U+%04lX to %d!=2 code points\n",
(int)phase, (long)c, (int)numCP);
exit(U_INVALID_FORMAT_ERROR);
}
Norm *p=checkNormForMapping(createNorm(c), c);
p->mapping=new UnicodeString(m);
p->mappingType=Norm::ROUND_TRIP;
p->mappingCP=U_SENTINEL;
}
// Missing glyphs run constructor. Core Text will not generate a run of missing glyphs, instead falling back on
// glyphs from LastResort. We want to use the primary font's missing glyph in order to match the fast text code path.
void ComplexTextController::ComplexTextRun::createTextRunFromFontDataCoreText(bool ltr)
{
m_coreTextIndicesVector.reserveInitialCapacity(m_stringLength);
unsigned r = 0;
while (r < m_stringLength) {
m_coreTextIndicesVector.uncheckedAppend(r);
if (U_IS_SURROGATE(m_characters[r])) {
ASSERT(r + 1 < m_stringLength);
ASSERT(U_IS_SURROGATE_LEAD(m_characters[r]));
ASSERT(U_IS_TRAIL(m_characters[r + 1]));
r += 2;
} else
r++;
}
m_glyphCount = m_coreTextIndicesVector.size();
if (!ltr) {
for (unsigned r = 0, end = m_glyphCount - 1; r < m_glyphCount / 2; ++r, --end)
std::swap(m_coreTextIndicesVector[r], m_coreTextIndicesVector[end]);
}
m_coreTextIndices = m_coreTextIndicesVector.data();
// Synthesize a run of missing glyphs.
m_glyphsVector.fill(0, m_glyphCount);
m_glyphs = m_glyphsVector.data();
m_advancesVector.fill(CGSizeMake(m_fontData->widthForGlyph(0), 0), m_glyphCount);
m_advances = m_advancesVector.data();
}
static void TestAppend() {
static const UChar32 codePoints[]={
0x61, 0xdf, 0x901, 0x3040,
0xac00, 0xd800, 0xdbff, 0xdcde,
0xdffd, 0xe000, 0xffff, 0x10000,
0x12345, 0xe0021, 0x10ffff, 0x110000,
0x234567, 0x7fffffff, -1, -1000,
0, 0x400
};
static const uint8_t expectUnsafe[]={
0x61, 0xc3, 0x9f, 0xe0, 0xa4, 0x81, 0xe3, 0x81, 0x80,
0xea, 0xb0, 0x80, 0xed, 0xa0, 0x80, 0xed, 0xaf, 0xbf, 0xed, 0xb3, 0x9e,
0xed, 0xbf, 0xbd, 0xee, 0x80, 0x80, 0xef, 0xbf, 0xbf, 0xf0, 0x90, 0x80, 0x80,
0xf0, 0x92, 0x8d, 0x85, 0xf3, 0xa0, 0x80, 0xa1, 0xf4, 0x8f, 0xbf, 0xbf, /* not 0x110000 */
/* none from this line */
0, 0xd0, 0x80
}, expectSafe[]={
0x61, 0xc3, 0x9f, 0xe0, 0xa4, 0x81, 0xe3, 0x81, 0x80,
0xea, 0xb0, 0x80, /* no surrogates */
/* no surrogates */ 0xee, 0x80, 0x80, 0xef, 0xbf, 0xbf, 0xf0, 0x90, 0x80, 0x80,
0xf0, 0x92, 0x8d, 0x85, 0xf3, 0xa0, 0x80, 0xa1, 0xf4, 0x8f, 0xbf, 0xbf, /* not 0x110000 */
/* none from this line */
0, 0xd0, 0x80
};
uint8_t buffer[100];
UChar32 c;
int32_t i, length;
UBool isError, expectIsError, wrongIsError;
length=0;
for(i=0; i<LENGTHOF(codePoints); ++i) {
c=codePoints[i];
if(c<0 || 0x10ffff<c) {
continue; /* skip non-code points for U8_APPEND_UNSAFE */
}
U8_APPEND_UNSAFE(buffer, length, c);
}
if(length!=LENGTHOF(expectUnsafe) || 0!=memcmp(buffer, expectUnsafe, length)) {
log_err("U8_APPEND_UNSAFE did not generate the expected output\n");
}
length=0;
wrongIsError=FALSE;
for(i=0; i<LENGTHOF(codePoints); ++i) {
c=codePoints[i];
expectIsError= c<0 || 0x10ffff<c || U_IS_SURROGATE(c);
isError=FALSE;
U8_APPEND(buffer, length, LENGTHOF(buffer), c, isError);
wrongIsError|= isError!=expectIsError;
}
if(wrongIsError) {
log_err("U8_APPEND did not set isError correctly\n");
}
if(length!=LENGTHOF(expectSafe) || 0!=memcmp(buffer, expectSafe, length)) {
log_err("U8_APPEND did not generate the expected output\n");
}
}
ConversionResult convertUTF8ToUTF16(
const char** sourceStart, const char* sourceEnd,
UChar** targetStart, UChar* targetEnd, bool strict)
{
ConversionResult result = conversionOK;
const char* source = *sourceStart;
UChar* target = *targetStart;
while (source < sourceEnd) {
int utf8SequenceLength = inlineUTF8SequenceLength(*source);
if (sourceEnd - source < utf8SequenceLength) {
result = sourceExhausted;
break;
}
// Do this check whether lenient or strict
if (!isLegalUTF8(reinterpret_cast<const unsigned char*>(source), utf8SequenceLength)) {
result = sourceIllegal;
break;
}
UChar32 character = readUTF8Sequence(source, utf8SequenceLength);
if (target >= targetEnd) {
source -= utf8SequenceLength; // Back up source pointer!
result = targetExhausted;
break;
}
if (U_IS_BMP(character)) {
// UTF-16 surrogate values are illegal in UTF-32
if (U_IS_SURROGATE(character)) {
if (strict) {
source -= utf8SequenceLength; // return to the illegal value itself
result = sourceIllegal;
break;
} else
*target++ = replacementCharacter;
} else
*target++ = character; // normal case
} else if (U_IS_SUPPLEMENTARY(character)) {
// target is a character in range 0xFFFF - 0x10FFFF
if (target + 1 >= targetEnd) {
source -= utf8SequenceLength; // Back up source pointer!
result = targetExhausted;
break;
}
*target++ = U16_LEAD(character);
*target++ = U16_TRAIL(character);
} else {
if (strict) {
source -= utf8SequenceLength; // return to the start
result = sourceIllegal;
break; // Bail out; shouldn't continue
} else
*target++ = replacementCharacter;
}
}
*sourceStart = source;
*targetStart = target;
return result;
}
static UChar32 legalEntityFor(UChar32 value)
{
if (value <= 0 || value > UCHAR_MAX_VALUE || U_IS_SURROGATE(value))
return replacementCharacter;
if ((value & ~0x1F) != 0x80)
return value;
return windowsLatin1ExtensionArray[value - 0x80];
}
static inline UChar* appendCharacter(UChar* destination, int character)
{
ASSERT(character != nonCharacter);
ASSERT(!U_IS_SURROGATE(character));
if (U_IS_BMP(character))
*destination++ = character;
else {
*destination++ = U16_LEAD(character);
*destination++ = U16_TRAIL(character);
}
return destination;
}
unsigned calculateStringHashAndLengthFromUTF8MaskingTop8Bits(const char* data, const char* dataEnd, unsigned& dataLength, unsigned& utf16Length)
{
if (!data)
return 0;
StringHasher stringHasher;
dataLength = 0;
utf16Length = 0;
while (data < dataEnd || (!dataEnd && *data)) {
if (isASCII(*data)) {
stringHasher.addCharacter(*data++);
dataLength++;
utf16Length++;
continue;
}
int utf8SequenceLength = inlineUTF8SequenceLengthNonASCII(*data);
dataLength += utf8SequenceLength;
if (!dataEnd) {
for (int i = 1; i < utf8SequenceLength; ++i) {
if (!data[i])
return 0;
}
} else if (dataEnd - data < utf8SequenceLength) {
return 0;
}
if (!isLegalUTF8(reinterpret_cast<const unsigned char*>(data), utf8SequenceLength))
return 0;
UChar32 character = readUTF8Sequence(data, utf8SequenceLength);
ASSERT(!isASCII(character));
if (U_IS_BMP(character)) {
// UTF-16 surrogate values are illegal in UTF-32
if (U_IS_SURROGATE(character))
return 0;
stringHasher.addCharacter(static_cast<UChar>(character)); // normal case
utf16Length++;
} else if (U_IS_SUPPLEMENTARY(character)) {
stringHasher.addCharacters(static_cast<UChar>(U16_LEAD(character)), static_cast<UChar>(U16_TRAIL(character)));
utf16Length += 2;
} else {
return 0;
}
}
return stringHasher.hashWithTop8BitsMasked();
}
// Check if there's any unpaird surrogate code point.
// Non-character code points are not checked.
static bool isValidUTF16(const String& s) {
if (s.is8Bit())
return true;
const UChar* ustr = s.characters16();
size_t length = s.length();
size_t position = 0;
while (position < length) {
UChar32 character;
U16_NEXT(ustr, position, length, character);
if (U_IS_SURROGATE(character))
return false;
}
return true;
}
//
// Check if a UTF16 string is valid according to the UTF16 standard
// Specifically, check that we don't have any invalid surrogate pairs
// If the string is valid, we return true.
// If not, we set invalidIndex to the index of the first invalid char index
// and return false
// If the invalid char is a lead surrogate pair, we return its index
// Otherwise, we treat the char before as the invalid one and return index - 1
// This function has defined behavior only for null-terminated strings.
// If the string is not null terminated, the behavior is undefined (likely hang)
//
static bool IsUtf16StringValid(const UChar* str, size_t length, size_t* invalidIndex)
{
Assert(invalidIndex != nullptr);
*invalidIndex = -1;
size_t i = 0;
for (;;)
{
// Iterate through the UTF16-LE string
// If we are at the end of the null terminated string, return true
// since the string is valid
// If not, check if the codepoint we have is a surrogate code unit.
// If it is, the string is malformed since U16_NEXT would have returned
// is the full codepoint if both code units in the surrogate pair were present
UChar32 c;
U16_NEXT(str, i, length, c);
if (c == 0)
{
return true;
}
if (U_IS_SURROGATE(c))
{
if (U16_IS_LEAD(c))
{
*invalidIndex = i;
}
else
{
Assert(i > 0);
*invalidIndex = i - 1;
}
return false;
}
if (i >= length)
{
return true;
}
}
}
unsigned calculateStringHashFromUTF8(const char* data, const char* dataEnd, unsigned& utf16Length)
{
if (!data)
return 0;
WTF::StringHasher stringHasher;
utf16Length = 0;
while (data < dataEnd) {
if (isASCII(*data)) {
stringHasher.addCharacter(*data++);
utf16Length++;
continue;
}
int utf8SequenceLength = inlineUTF8SequenceLengthNonASCII(*data);
if (dataEnd - data < utf8SequenceLength)
return false;
if (!isLegalUTF8(reinterpret_cast<const unsigned char*>(data), utf8SequenceLength))
return 0;
UChar32 character = readUTF8Sequence(data, utf8SequenceLength);
ASSERT(!isASCII(character));
if (U_IS_BMP(character)) {
// UTF-16 surrogate values are illegal in UTF-32
if (U_IS_SURROGATE(character))
return 0;
stringHasher.addCharacter(static_cast<UChar>(character)); // normal case
utf16Length++;
} else if (U_IS_SUPPLEMENTARY(character)) {
stringHasher.addCharacters(static_cast<UChar>(U16_LEAD(character)),
static_cast<UChar>(U16_TRAIL(character)));
utf16Length += 2;
} else
return 0;
}
return stringHasher.hash();
}
ALWAYS_INLINE bool equalWithUTF8Internal(const CharType* a, const CharType* aEnd, const char* b, const char* bEnd)
{
while (b < bEnd) {
if (isASCII(*b)) {
if (*a++ != *b++)
return false;
continue;
}
int utf8SequenceLength = inlineUTF8SequenceLengthNonASCII(*b);
if (bEnd - b < utf8SequenceLength)
return false;
if (!isLegalUTF8(reinterpret_cast<const unsigned char*>(b), utf8SequenceLength))
return 0;
UChar32 character = readUTF8Sequence(b, utf8SequenceLength);
ASSERT(!isASCII(character));
if (U_IS_BMP(character)) {
// UTF-16 surrogate values are illegal in UTF-32
if (U_IS_SURROGATE(character))
return false;
if (*a++ != character)
return false;
} else if (U_IS_SUPPLEMENTARY(character)) {
if (*a++ != U16_LEAD(character))
return false;
if (*a++ != U16_TRAIL(character))
return false;
} else {
return false;
}
}
return a == aEnd;
}
static bool shouldUseCoreText(UChar* buffer, unsigned bufferLength, const SimpleFontData* fontData)
{
if (fontData->platformData().isCompositeFontReference())
return true;
// CoreText doesn't have vertical glyphs of surrogate pair characters.
// Therefore, we should not use CoreText, but this always returns horizontal glyphs.
// FIXME: We should use vertical glyphs. https://code.google.com/p/chromium/issues/detail?id=340173
if (bufferLength >= 2 && U_IS_SURROGATE(buffer[0]) && fontData->hasVerticalGlyphs()) {
ASSERT(U_IS_SURROGATE_LEAD(buffer[0]));
ASSERT(U_IS_TRAIL(buffer[1]));
return false;
}
if (fontData->platformData().widthVariant() != RegularWidth || fontData->hasVerticalGlyphs()) {
// Ideographs don't have a vertical variant or width variants.
for (unsigned i = 0; i < bufferLength; ++i) {
if (!Character::isCJKIdeograph(buffer[i]))
return true;
}
}
return false;
}
/* internal function */
U_CFUNC int32_t
u_strcmpFold(const UChar *s1, int32_t length1,
const UChar *s2, int32_t length2,
uint32_t options,
UErrorCode *pErrorCode) {
const UCaseProps *csp;
/* current-level start/limit - s1/s2 as current */
const UChar *start1, *start2, *limit1, *limit2;
/* case folding variables */
const UChar *p;
int32_t length;
/* stacks of previous-level start/current/limit */
CmpEquivLevel stack1[2], stack2[2];
/* case folding buffers, only use current-level start/limit */
UChar fold1[UCASE_MAX_STRING_LENGTH+1], fold2[UCASE_MAX_STRING_LENGTH+1];
/* track which is the current level per string */
int32_t level1, level2;
/* current code units, and code points for lookups */
UChar32 c1, c2, cp1, cp2;
/* no argument error checking because this itself is not an API */
/*
* assume that at least the option U_COMPARE_IGNORE_CASE is set
* otherwise this function would have to behave exactly as uprv_strCompare()
*/
csp=ucase_getSingleton();
if(U_FAILURE(*pErrorCode)) {
return 0;
}
/* initialize */
start1=s1;
if(length1==-1) {
limit1=NULL;
} else {
limit1=s1+length1;
}
start2=s2;
if(length2==-1) {
limit2=NULL;
} else {
limit2=s2+length2;
}
level1=level2=0;
c1=c2=-1;
/* comparison loop */
for(;;) {
/*
* here a code unit value of -1 means "get another code unit"
* below it will mean "this source is finished"
*/
if(c1<0) {
/* get next code unit from string 1, post-increment */
for(;;) {
if(s1==limit1 || ((c1=*s1)==0 && (limit1==NULL || (options&_STRNCMP_STYLE)))) {
if(level1==0) {
c1=-1;
break;
}
} else {
++s1;
break;
}
/* reached end of level buffer, pop one level */
do {
--level1;
start1=stack1[level1].start;
} while(start1==NULL);
s1=stack1[level1].s;
limit1=stack1[level1].limit;
}
}
if(c2<0) {
/* get next code unit from string 2, post-increment */
for(;;) {
if(s2==limit2 || ((c2=*s2)==0 && (limit2==NULL || (options&_STRNCMP_STYLE)))) {
if(level2==0) {
c2=-1;
break;
}
} else {
++s2;
break;
}
/* reached end of level buffer, pop one level */
do {
--level2;
start2=stack2[level2].start;
} while(start2==NULL);
s2=stack2[level2].s;
limit2=stack2[level2].limit;
}
}
/*
* compare c1 and c2
* either variable c1, c2 is -1 only if the corresponding string is finished
*/
if(c1==c2) {
if(c1<0) {
return 0; /* c1==c2==-1 indicating end of strings */
}
c1=c2=-1; /* make us fetch new code units */
continue;
} else if(c1<0) {
return -1; /* string 1 ends before string 2 */
} else if(c2<0) {
return 1; /* string 2 ends before string 1 */
}
/* c1!=c2 && c1>=0 && c2>=0 */
/* get complete code points for c1, c2 for lookups if either is a surrogate */
cp1=c1;
if(U_IS_SURROGATE(c1)) {
UChar c;
if(U_IS_SURROGATE_LEAD(c1)) {
if(s1!=limit1 && U16_IS_TRAIL(c=*s1)) {
/* advance ++s1; only below if cp1 decomposes/case-folds */
cp1=U16_GET_SUPPLEMENTARY(c1, c);
}
} else /* isTrail(c1) */ {
if(start1<=(s1-2) && U16_IS_LEAD(c=*(s1-2))) {
cp1=U16_GET_SUPPLEMENTARY(c, c1);
}
}
}
cp2=c2;
if(U_IS_SURROGATE(c2)) {
UChar c;
if(U_IS_SURROGATE_LEAD(c2)) {
if(s2!=limit2 && U16_IS_TRAIL(c=*s2)) {
/* advance ++s2; only below if cp2 decomposes/case-folds */
cp2=U16_GET_SUPPLEMENTARY(c2, c);
}
} else /* isTrail(c2) */ {
if(start2<=(s2-2) && U16_IS_LEAD(c=*(s2-2))) {
cp2=U16_GET_SUPPLEMENTARY(c, c2);
}
}
}
/*
* go down one level for each string
* continue with the main loop as soon as there is a real change
*/
if( level1==0 &&
(length=ucase_toFullFolding(csp, (UChar32)cp1, &p, options))>=0
) {
/* cp1 case-folds to the code point "length" or to p[length] */
if(U_IS_SURROGATE(c1)) {
if(U_IS_SURROGATE_LEAD(c1)) {
/* advance beyond source surrogate pair if it case-folds */
++s1;
} else /* isTrail(c1) */ {
/*
* we got a supplementary code point when hitting its trail surrogate,
* therefore the lead surrogate must have been the same as in the other string;
* compare this decomposition with the lead surrogate in the other string
* remember that this simulates bulk text replacement:
* the decomposition would replace the entire code point
*/
--s2;
c2=*(s2-1);
}
}
/* push current level pointers */
stack1[0].start=start1;
stack1[0].s=s1;
stack1[0].limit=limit1;
++level1;
/* copy the folding result to fold1[] */
if(length<=UCASE_MAX_STRING_LENGTH) {
u_memcpy(fold1, p, length);
} else {
int32_t i=0;
U16_APPEND_UNSAFE(fold1, i, length);
length=i;
}
/* set next level pointers to case folding */
start1=s1=fold1;
limit1=fold1+length;
/* get ready to read from decomposition, continue with loop */
c1=-1;
continue;
}
if( level2==0 &&
(length=ucase_toFullFolding(csp, (UChar32)cp2, &p, options))>=0
) {
/* cp2 case-folds to the code point "length" or to p[length] */
if(U_IS_SURROGATE(c2)) {
if(U_IS_SURROGATE_LEAD(c2)) {
/* advance beyond source surrogate pair if it case-folds */
++s2;
} else /* isTrail(c2) */ {
/*
* we got a supplementary code point when hitting its trail surrogate,
* therefore the lead surrogate must have been the same as in the other string;
* compare this decomposition with the lead surrogate in the other string
* remember that this simulates bulk text replacement:
* the decomposition would replace the entire code point
*/
--s1;
c1=*(s1-1);
}
}
/* push current level pointers */
stack2[0].start=start2;
stack2[0].s=s2;
stack2[0].limit=limit2;
++level2;
/* copy the folding result to fold2[] */
if(length<=UCASE_MAX_STRING_LENGTH) {
u_memcpy(fold2, p, length);
} else {
int32_t i=0;
U16_APPEND_UNSAFE(fold2, i, length);
length=i;
}
/* set next level pointers to case folding */
start2=s2=fold2;
limit2=fold2+length;
/* get ready to read from decomposition, continue with loop */
c2=-1;
continue;
}
/*
* no decomposition/case folding, max level for both sides:
* return difference result
*
* code point order comparison must not just return cp1-cp2
* because when single surrogates are present then the surrogate pairs
* that formed cp1 and cp2 may be from different string indexes
*
* example: { d800 d800 dc01 } vs. { d800 dc00 }, compare at second code units
* c1=d800 cp1=10001 c2=dc00 cp2=10000
* cp1-cp2>0 but c1-c2<0 and in fact in UTF-32 it is { d800 10001 } < { 10000 }
*
* therefore, use same fix-up as in ustring.c/uprv_strCompare()
* except: uprv_strCompare() fetches c=*s while this functions fetches c=*s++
* so we have slightly different pointer/start/limit comparisons here
*/
if(c1>=0xd800 && c2>=0xd800 && (options&U_COMPARE_CODE_POINT_ORDER)) {
/* subtract 0x2800 from BMP code points to make them smaller than supplementary ones */
if(
(c1<=0xdbff && s1!=limit1 && U16_IS_TRAIL(*s1)) ||
(U16_IS_TRAIL(c1) && start1!=(s1-1) && U16_IS_LEAD(*(s1-2)))
) {
/* part of a surrogate pair, leave >=d800 */
} else {
/* BMP code point - may be surrogate code point - make <d800 */
c1-=0x2800;
}
if(
(c2<=0xdbff && s2!=limit2 && U16_IS_TRAIL(*s2)) ||
(U16_IS_TRAIL(c2) && start2!=(s2-1) && U16_IS_LEAD(*(s2-2)))
) {
/* part of a surrogate pair, leave >=d800 */
} else {
/* BMP code point - may be surrogate code point - make <d800 */
c2-=0x2800;
}
}
return c1-c2;
}
}
static void
TestSurrogates() {
static const uint8_t b[]={
0xc3, 0x9f, /* 00DF */
0xed, 0x9f, 0xbf, /* D7FF */
0xed, 0xa0, 0x81, /* D801 */
0xed, 0xbf, 0xbe, /* DFFE */
0xee, 0x80, 0x80, /* E000 */
0xf0, 0x97, 0xbf, 0xbe /* 17FFE */
};
static const UChar32 cp[]={
0xdf, 0xd7ff, 0xd801, 0xdffe, 0xe000, 0x17ffe
};
UChar32 cu, cs, cl;
int32_t i, j, k, iu, is, il, length;
k=0; /* index into cp[] */
length=LENGTHOF(b);
for(i=0; i<length;) {
j=i;
U8_NEXT_UNSAFE(b, j, cu);
iu=j;
j=i;
U8_NEXT(b, j, length, cs);
is=j;
j=i;
L8_NEXT(b, j, length, cl);
il=j;
if(cu!=cp[k]) {
log_err("U8_NEXT_UNSAFE(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cu, (long)cp[k]);
}
/* U8_NEXT() returns <0 for surrogate code points */
if(U_IS_SURROGATE(cu) ? cs>=0 : cs!=cu) {
log_err("U8_NEXT(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cs, (long)cu);
}
/* L8_NEXT() returns surrogate code points like U8_NEXT_UNSAFE() */
if(cl!=cu) {
log_err("L8_NEXT(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cl, (long)cu);
}
if(is!=iu || il!=iu) {
log_err("U8_NEXT(b[%ld]) or L8_NEXT(b[%ld]) did not advance the index correctly\n", (long)i, (long)i);
}
++k; /* next code point */
i=iu; /* advance by one UTF-8 sequence */
}
while(i>0) {
--k; /* previous code point */
j=i;
U8_PREV_UNSAFE(b, j, cu);
iu=j;
j=i;
U8_PREV(b, 0, j, cs);
is=j;
j=i;
L8_PREV(b, 0, j, cl);
il=j;
if(cu!=cp[k]) {
log_err("U8_PREV_UNSAFE(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cu, (long)cp[k]);
}
/* U8_PREV() returns <0 for surrogate code points */
if(U_IS_SURROGATE(cu) ? cs>=0 : cs!=cu) {
log_err("U8_PREV(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cs, (long)cu);
}
/* L8_PREV() returns surrogate code points like U8_PREV_UNSAFE() */
if(cl!=cu) {
log_err("L8_PREV(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cl, (long)cu);
}
if(is!=iu || il !=iu) {
log_err("U8_PREV(b[%ld]) or L8_PREV(b[%ld]) did not advance the index correctly\n", (long)i, (long)i);
}
i=iu; /* go back by one UTF-8 sequence */
}
}
U_CFUNC int32_t
u_strFromPunycode(const UChar *src, int32_t srcLength,
UChar *dest, int32_t destCapacity,
UBool *caseFlags,
UErrorCode *pErrorCode) {
int32_t n, destLength, i, bias, basicLength, j, in, oldi, w, k, digit, t,
destCPCount, firstSupplementaryIndex, cpLength;
UChar b;
/* argument checking */
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
return 0;
}
if(src==NULL || srcLength<-1 || (dest==NULL && destCapacity!=0)) {
*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
return 0;
}
if(srcLength==-1) {
srcLength=u_strlen(src);
}
/*
* Handle the basic code points:
* Let basicLength be the number of input code points
* before the last delimiter, or 0 if there is none,
* then copy the first basicLength code points to the output.
*
* The two following loops iterate backward.
*/
for(j=srcLength; j>0;) {
if(src[--j]==DELIMITER) {
break;
}
}
destLength=basicLength=destCPCount=j;
U_ASSERT(destLength>=0);
while(j>0) {
b=src[--j];
if(!IS_BASIC(b)) {
*pErrorCode=U_INVALID_CHAR_FOUND;
return 0;
}
if(j<destCapacity) {
dest[j]=(UChar)b;
if(caseFlags!=NULL) {
caseFlags[j]=IS_BASIC_UPPERCASE(b);
}
}
}
/* Initialize the state: */
n=INITIAL_N;
i=0;
bias=INITIAL_BIAS;
firstSupplementaryIndex=1000000000;
/*
* Main decoding loop:
* Start just after the last delimiter if any
* basic code points were copied; start at the beginning otherwise.
*/
for(in=basicLength>0 ? basicLength+1 : 0; in<srcLength; /* no op */) {
/*
* in is the index of the next character to be consumed, and
* destCPCount is the number of code points in the output array.
*
* Decode a generalized variable-length integer into delta,
* which gets added to i. The overflow checking is easier
* if we increase i as we go, then subtract off its starting
* value at the end to obtain delta.
*/
for(oldi=i, w=1, k=BASE; /* no condition */; k+=BASE) {
if(in>=srcLength) {
*pErrorCode=U_ILLEGAL_CHAR_FOUND;
return 0;
}
digit=basicToDigit[(uint8_t)src[in++]];
if(digit<0) {
*pErrorCode=U_INVALID_CHAR_FOUND;
return 0;
}
if(digit>(0x7fffffff-i)/w) {
/* integer overflow */
*pErrorCode=U_ILLEGAL_CHAR_FOUND;
return 0;
}
i+=digit*w;
/** RAM: comment out the old code for conformance with draft-ietf-idn-punycode-03.txt
t=k-bias;
if(t<TMIN) {
t=TMIN;
} else if(t>TMAX) {
t=TMAX;
}
*/
t=k-bias;
if(t<TMIN) {
t=TMIN;
} else if(k>=(bias+TMAX)) {
t=TMAX;
}
if(digit<t) {
break;
}
if(w>0x7fffffff/(BASE-t)) {
/* integer overflow */
*pErrorCode=U_ILLEGAL_CHAR_FOUND;
return 0;
}
w*=BASE-t;
}
/*
* Modification from sample code:
* Increments destCPCount here,
* where needed instead of in for() loop tail.
*/
++destCPCount;
bias=adaptBias(i-oldi, destCPCount, (UBool)(oldi==0));
/*
* i was supposed to wrap around from (incremented) destCPCount to 0,
* incrementing n each time, so we'll fix that now:
*/
if(i/destCPCount>(0x7fffffff-n)) {
/* integer overflow */
*pErrorCode=U_ILLEGAL_CHAR_FOUND;
return 0;
}
n+=i/destCPCount;
i%=destCPCount;
/* not needed for Punycode: */
/* if (decode_digit(n) <= BASE) return punycode_invalid_input; */
if(n>0x10ffff || U_IS_SURROGATE(n)) {
/* Unicode code point overflow */
*pErrorCode=U_ILLEGAL_CHAR_FOUND;
return 0;
}
/* Insert n at position i of the output: */
cpLength=U16_LENGTH(n);
if(dest!=NULL && ((destLength+cpLength)<=destCapacity)) {
int32_t codeUnitIndex;
/*
* Handle indexes when supplementary code points are present.
*
* In almost all cases, there will be only BMP code points before i
* and even in the entire string.
* This is handled with the same efficiency as with UTF-32.
*
* Only the rare cases with supplementary code points are handled
* more slowly - but not too bad since this is an insertion anyway.
*/
if(i<=firstSupplementaryIndex) {
codeUnitIndex=i;
if(cpLength>1) {
firstSupplementaryIndex=codeUnitIndex;
} else {
++firstSupplementaryIndex;
}
} else {
codeUnitIndex=firstSupplementaryIndex;
U16_FWD_N(dest, codeUnitIndex, destLength, i-codeUnitIndex);
}
/* use the UChar index codeUnitIndex instead of the code point index i */
if(codeUnitIndex<destLength) {
uprv_memmove(dest+codeUnitIndex+cpLength,
dest+codeUnitIndex,
(destLength-codeUnitIndex)*U_SIZEOF_UCHAR);
if(caseFlags!=NULL) {
uprv_memmove(caseFlags+codeUnitIndex+cpLength,
caseFlags+codeUnitIndex,
destLength-codeUnitIndex);
}
}
if(cpLength==1) {
/* BMP, insert one code unit */
dest[codeUnitIndex]=(UChar)n;
} else {
/* supplementary character, insert two code units */
dest[codeUnitIndex]=U16_LEAD(n);
dest[codeUnitIndex+1]=U16_TRAIL(n);
}
if(caseFlags!=NULL) {
/* Case of last character determines uppercase flag: */
caseFlags[codeUnitIndex]=IS_BASIC_UPPERCASE(src[in-1]);
if(cpLength==2) {
caseFlags[codeUnitIndex+1]=FALSE;
}
}
}
destLength+=cpLength;
U_ASSERT(destLength>=0);
++i;
}
return u_terminateUChars(dest, destCapacity, destLength, pErrorCode);
}
/* parse a mapping line; must not be empty */
U_CAPI UBool U_EXPORT2
ucm_parseMappingLine(UCMapping *m,
UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
uint8_t bytes[UCNV_EXT_MAX_BYTES],
const char *line) {
const char *s;
char *end;
UChar32 cp;
int32_t u16Length;
int8_t uLen, bLen, f;
s=line;
uLen=bLen=0;
/* parse code points */
for(;;) {
/* skip an optional plus sign */
if(uLen>0 && *s=='+') {
++s;
}
if(*s!='<') {
break;
}
if( s[1]!='U' ||
(cp=(UChar32)uprv_strtoul(s+2, &end, 16), end)==s+2 ||
*end!='>'
) {
fprintf(stderr, "ucm error: Unicode code point must be formatted as <UXXXX> (1..6 hex digits) - \"%s\"\n", line);
return FALSE;
}
if((uint32_t)cp>0x10ffff || U_IS_SURROGATE(cp)) {
fprintf(stderr, "ucm error: Unicode code point must be 0..d7ff or e000..10ffff - \"%s\"\n", line);
return FALSE;
}
if(uLen==UCNV_EXT_MAX_UCHARS) {
fprintf(stderr, "ucm error: too many code points on \"%s\"\n", line);
return FALSE;
}
codePoints[uLen++]=cp;
s=end+1;
}
if(uLen==0) {
fprintf(stderr, "ucm error: no Unicode code points on \"%s\"\n", line);
return FALSE;
} else if(uLen==1) {
m->u=codePoints[0];
} else {
UErrorCode errorCode=U_ZERO_ERROR;
u_strFromUTF32(NULL, 0, &u16Length, codePoints, uLen, &errorCode);
if( (U_FAILURE(errorCode) && errorCode!=U_BUFFER_OVERFLOW_ERROR) ||
u16Length>UCNV_EXT_MAX_UCHARS
) {
fprintf(stderr, "ucm error: too many UChars on \"%s\"\n", line);
return FALSE;
}
}
s=u_skipWhitespace(s);
/* parse bytes */
bLen=ucm_parseBytes(bytes, line, &s);
if(bLen<0) {
return FALSE;
} else if(bLen==0) {
fprintf(stderr, "ucm error: no bytes on \"%s\"\n", line);
return FALSE;
} else if(bLen<=4) {
uprv_memcpy(m->b.bytes, bytes, bLen);
}
/* skip everything until the fallback indicator, even the start of a comment */
for(;;) {
if(*s==0) {
f=-1; /* no fallback indicator */
break;
} else if(*s=='|') {
f=(int8_t)(s[1]-'0');
if((uint8_t)f>4) {
fprintf(stderr, "ucm error: fallback indicator must be |0..|4 - \"%s\"\n", line);
return FALSE;
}
break;
}
++s;
}
m->uLen=uLen;
m->bLen=bLen;
m->f=f;
return TRUE;
}
U_CAPI void U_EXPORT2
ucm_addMapping(UCMTable *table,
UCMapping *m,
UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
uint8_t bytes[UCNV_EXT_MAX_BYTES]) {
UCMapping *tm;
UChar32 c;
int32_t idx;
if(table->mappingsLength>=table->mappingsCapacity) {
/* make the mappings array larger */
if(table->mappingsCapacity==0) {
table->mappingsCapacity=1000;
} else {
table->mappingsCapacity*=10;
}
table->mappings=(UCMapping *)uprv_realloc(table->mappings,
table->mappingsCapacity*sizeof(UCMapping));
if(table->mappings==NULL) {
fprintf(stderr, "ucm error: unable to allocate %d UCMappings\n",
(int)table->mappingsCapacity);
exit(U_MEMORY_ALLOCATION_ERROR);
}
if(table->reverseMap!=NULL) {
/* the reverseMap must be reallocated in a new sort */
uprv_free(table->reverseMap);
table->reverseMap=NULL;
}
}
if(m->uLen>1 && table->codePointsCapacity==0) {
table->codePointsCapacity=10000;
table->codePoints=(UChar32 *)uprv_malloc(table->codePointsCapacity*4);
if(table->codePoints==NULL) {
fprintf(stderr, "ucm error: unable to allocate %d UChar32s\n",
(int)table->codePointsCapacity);
exit(U_MEMORY_ALLOCATION_ERROR);
}
}
if(m->bLen>4 && table->bytesCapacity==0) {
table->bytesCapacity=10000;
table->bytes=(uint8_t *)uprv_malloc(table->bytesCapacity);
if(table->bytes==NULL) {
fprintf(stderr, "ucm error: unable to allocate %d bytes\n",
(int)table->bytesCapacity);
exit(U_MEMORY_ALLOCATION_ERROR);
}
}
if(m->uLen>1) {
idx=table->codePointsLength;
table->codePointsLength+=m->uLen;
if(table->codePointsLength>table->codePointsCapacity) {
fprintf(stderr, "ucm error: too many code points in multiple-code point mappings\n");
exit(U_MEMORY_ALLOCATION_ERROR);
}
uprv_memcpy(table->codePoints+idx, codePoints, (size_t)m->uLen*4);
m->u=idx;
}
if(m->bLen>4) {
idx=table->bytesLength;
table->bytesLength+=m->bLen;
if(table->bytesLength>table->bytesCapacity) {
fprintf(stderr, "ucm error: too many bytes in mappings with >4 charset bytes\n");
exit(U_MEMORY_ALLOCATION_ERROR);
}
uprv_memcpy(table->bytes+idx, bytes, m->bLen);
m->b.idx=idx;
}
/* set unicodeMask */
for(idx=0; idx<m->uLen; ++idx) {
c=codePoints[idx];
if(c>=0x10000) {
table->unicodeMask|=UCNV_HAS_SUPPLEMENTARY; /* there are supplementary code points */
} else if(U_IS_SURROGATE(c)) {
table->unicodeMask|=UCNV_HAS_SURROGATES; /* there are surrogate code points */
}
}
/* set flagsType */
if(m->f<0) {
table->flagsType|=UCM_FLAGS_IMPLICIT;
} else {
table->flagsType|=UCM_FLAGS_EXPLICIT;
}
tm=table->mappings+table->mappingsLength++;
uprv_memcpy(tm, m, sizeof(UCMapping));
table->isSorted=FALSE;
}
