// Missing glyphs run constructor. Core Text will not generate a run of missing glyphs, instead falling back on // glyphs from LastResort. We want to use the primary font's missing glyph in order to match the fast text code path. CoreTextController::CoreTextRun::CoreTextRun(const SimpleFontData* fontData, const UChar* characters, unsigned stringLocation, size_t stringLength, bool ltr) : m_fontData(fontData) , m_characters(characters) , m_stringLocation(stringLocation) , m_stringLength(stringLength) { Vector<CFIndex, 16> indices; unsigned r = 0; while (r < stringLength) { indices.append(r); if (U_IS_SURROGATE(characters[r])) { ASSERT(r + 1 < stringLength); ASSERT(U_IS_SURROGATE_LEAD(characters[r])); ASSERT(U_IS_TRAIL(characters[r + 1])); r += 2; } else r++; } m_glyphCount = indices.size(); if (!ltr) { for (unsigned r = 0, end = m_glyphCount - 1; r < m_glyphCount / 2; ++r, --end) std::swap(indices[r], indices[end]); } m_indicesData.adoptCF(CFDataCreateMutable(kCFAllocatorDefault, m_glyphCount * sizeof(CFIndex))); CFDataAppendBytes(m_indicesData.get(), reinterpret_cast<const UInt8*>(indices.data()), m_glyphCount * sizeof(CFIndex)); m_indices = reinterpret_cast<const CFIndex*>(CFDataGetBytePtr(m_indicesData.get())); }
static void TestAppend() { static const UChar32 codePoints[]={ 0x61, 0xdf, 0x901, 0x3040, 0xac00, 0xd800, 0xdbff, 0xdcde, 0xdffd, 0xe000, 0xffff, 0x10000, 0x12345, 0xe0021, 0x10ffff, 0x110000, 0x234567, 0x7fffffff, -1, -1000, 0, 0x400 }; static const UChar expectUnsafe[]={ 0x61, 0xdf, 0x901, 0x3040, 0xac00, 0xd800, 0xdbff, 0xdcde, 0xdffd, 0xe000, 0xffff, 0xd800, 0xdc00, 0xd848, 0xdf45, 0xdb40, 0xdc21, 0xdbff, 0xdfff, /* not 0x110000 */ /* none from this line */ 0, 0x400 }, expectSafe[]={ 0x61, 0xdf, 0x901, 0x3040, 0xac00, 0xd800, 0xdbff, 0xdcde, 0xdffd, 0xe000, 0xffff, 0xd800, 0xdc00, 0xd848, 0xdf45, 0xdb40, 0xdc21, 0xdbff, 0xdfff, /* not 0x110000 */ /* none from this line */ 0, 0x400 }; UChar buffer[100]; UChar32 c; int32_t i, length; UBool isError, expectIsError, wrongIsError; length=0; for(i=0; i<LENGTHOF(codePoints); ++i) { c=codePoints[i]; if(c<0 || 0x10ffff<c) { continue; /* skip non-code points for U16_APPEND_UNSAFE */ } U16_APPEND_UNSAFE(buffer, length, c); } if(length!=LENGTHOF(expectUnsafe) || 0!=memcmp(buffer, expectUnsafe, length*U_SIZEOF_UCHAR)) { log_err("U16_APPEND_UNSAFE did not generate the expected output\n"); } length=0; wrongIsError=FALSE; for(i=0; i<LENGTHOF(codePoints); ++i) { c=codePoints[i]; expectIsError= c<0 || 0x10ffff<c || U_IS_SURROGATE(c); isError=FALSE; U16_APPEND(buffer, length, LENGTHOF(buffer), c, isError); wrongIsError|= isError!=expectIsError; } if(wrongIsError) { log_err("U16_APPEND did not set isError correctly\n"); } if(length!=LENGTHOF(expectSafe) || 0!=memcmp(buffer, expectSafe, length*U_SIZEOF_UCHAR)) { log_err("U16_APPEND did not generate the expected output\n"); } }
void Normalizer2DataBuilder::setRoundTripMapping(UChar32 c, const UnicodeString &m) { if(U_IS_SURROGATE(c)) { fprintf(stderr, "error in gennorm2 phase %d: " "illegal round-trip mapping from surrogate code point U+%04lX\n", (int)phase, (long)c); exit(U_INVALID_FORMAT_ERROR); } if(!isWellFormed(m)) { fprintf(stderr, "error in gennorm2 phase %d: " "illegal round-trip mapping from U+%04lX to malformed string\n", (int)phase, (long)c); exit(U_INVALID_FORMAT_ERROR); } int32_t numCP=u_countChar32(m.getBuffer(), m.length()); if(numCP!=2) { fprintf(stderr, "error in gennorm2 phase %d: " "illegal round-trip mapping from U+%04lX to %d!=2 code points\n", (int)phase, (long)c, (int)numCP); exit(U_INVALID_FORMAT_ERROR); } Norm *p=checkNormForMapping(createNorm(c), c); p->mapping=new UnicodeString(m); p->mappingType=Norm::ROUND_TRIP; p->mappingCP=U_SENTINEL; }
// Missing glyphs run constructor. Core Text will not generate a run of missing glyphs, instead falling back on // glyphs from LastResort. We want to use the primary font's missing glyph in order to match the fast text code path. void ComplexTextController::ComplexTextRun::createTextRunFromFontDataCoreText(bool ltr) { m_coreTextIndicesVector.reserveInitialCapacity(m_stringLength); unsigned r = 0; while (r < m_stringLength) { m_coreTextIndicesVector.uncheckedAppend(r); if (U_IS_SURROGATE(m_characters[r])) { ASSERT(r + 1 < m_stringLength); ASSERT(U_IS_SURROGATE_LEAD(m_characters[r])); ASSERT(U_IS_TRAIL(m_characters[r + 1])); r += 2; } else r++; } m_glyphCount = m_coreTextIndicesVector.size(); if (!ltr) { for (unsigned r = 0, end = m_glyphCount - 1; r < m_glyphCount / 2; ++r, --end) std::swap(m_coreTextIndicesVector[r], m_coreTextIndicesVector[end]); } m_coreTextIndices = m_coreTextIndicesVector.data(); // Synthesize a run of missing glyphs. m_glyphsVector.fill(0, m_glyphCount); m_glyphs = m_glyphsVector.data(); m_advancesVector.fill(CGSizeMake(m_fontData->widthForGlyph(0), 0), m_glyphCount); m_advances = m_advancesVector.data(); }
static void TestAppend() { static const UChar32 codePoints[]={ 0x61, 0xdf, 0x901, 0x3040, 0xac00, 0xd800, 0xdbff, 0xdcde, 0xdffd, 0xe000, 0xffff, 0x10000, 0x12345, 0xe0021, 0x10ffff, 0x110000, 0x234567, 0x7fffffff, -1, -1000, 0, 0x400 }; static const uint8_t expectUnsafe[]={ 0x61, 0xc3, 0x9f, 0xe0, 0xa4, 0x81, 0xe3, 0x81, 0x80, 0xea, 0xb0, 0x80, 0xed, 0xa0, 0x80, 0xed, 0xaf, 0xbf, 0xed, 0xb3, 0x9e, 0xed, 0xbf, 0xbd, 0xee, 0x80, 0x80, 0xef, 0xbf, 0xbf, 0xf0, 0x90, 0x80, 0x80, 0xf0, 0x92, 0x8d, 0x85, 0xf3, 0xa0, 0x80, 0xa1, 0xf4, 0x8f, 0xbf, 0xbf, /* not 0x110000 */ /* none from this line */ 0, 0xd0, 0x80 }, expectSafe[]={ 0x61, 0xc3, 0x9f, 0xe0, 0xa4, 0x81, 0xe3, 0x81, 0x80, 0xea, 0xb0, 0x80, /* no surrogates */ /* no surrogates */ 0xee, 0x80, 0x80, 0xef, 0xbf, 0xbf, 0xf0, 0x90, 0x80, 0x80, 0xf0, 0x92, 0x8d, 0x85, 0xf3, 0xa0, 0x80, 0xa1, 0xf4, 0x8f, 0xbf, 0xbf, /* not 0x110000 */ /* none from this line */ 0, 0xd0, 0x80 }; uint8_t buffer[100]; UChar32 c; int32_t i, length; UBool isError, expectIsError, wrongIsError; length=0; for(i=0; i<LENGTHOF(codePoints); ++i) { c=codePoints[i]; if(c<0 || 0x10ffff<c) { continue; /* skip non-code points for U8_APPEND_UNSAFE */ } U8_APPEND_UNSAFE(buffer, length, c); } if(length!=LENGTHOF(expectUnsafe) || 0!=memcmp(buffer, expectUnsafe, length)) { log_err("U8_APPEND_UNSAFE did not generate the expected output\n"); } length=0; wrongIsError=FALSE; for(i=0; i<LENGTHOF(codePoints); ++i) { c=codePoints[i]; expectIsError= c<0 || 0x10ffff<c || U_IS_SURROGATE(c); isError=FALSE; U8_APPEND(buffer, length, LENGTHOF(buffer), c, isError); wrongIsError|= isError!=expectIsError; } if(wrongIsError) { log_err("U8_APPEND did not set isError correctly\n"); } if(length!=LENGTHOF(expectSafe) || 0!=memcmp(buffer, expectSafe, length)) { log_err("U8_APPEND did not generate the expected output\n"); } }
ConversionResult convertUTF8ToUTF16( const char** sourceStart, const char* sourceEnd, UChar** targetStart, UChar* targetEnd, bool strict) { ConversionResult result = conversionOK; const char* source = *sourceStart; UChar* target = *targetStart; while (source < sourceEnd) { int utf8SequenceLength = inlineUTF8SequenceLength(*source); if (sourceEnd - source < utf8SequenceLength) { result = sourceExhausted; break; } // Do this check whether lenient or strict if (!isLegalUTF8(reinterpret_cast<const unsigned char*>(source), utf8SequenceLength)) { result = sourceIllegal; break; } UChar32 character = readUTF8Sequence(source, utf8SequenceLength); if (target >= targetEnd) { source -= utf8SequenceLength; // Back up source pointer! result = targetExhausted; break; } if (U_IS_BMP(character)) { // UTF-16 surrogate values are illegal in UTF-32 if (U_IS_SURROGATE(character)) { if (strict) { source -= utf8SequenceLength; // return to the illegal value itself result = sourceIllegal; break; } else *target++ = replacementCharacter; } else *target++ = character; // normal case } else if (U_IS_SUPPLEMENTARY(character)) { // target is a character in range 0xFFFF - 0x10FFFF if (target + 1 >= targetEnd) { source -= utf8SequenceLength; // Back up source pointer! result = targetExhausted; break; } *target++ = U16_LEAD(character); *target++ = U16_TRAIL(character); } else { if (strict) { source -= utf8SequenceLength; // return to the start result = sourceIllegal; break; // Bail out; shouldn't continue } else *target++ = replacementCharacter; } } *sourceStart = source; *targetStart = target; return result; }
static UChar32 legalEntityFor(UChar32 value) { if (value <= 0 || value > UCHAR_MAX_VALUE || U_IS_SURROGATE(value)) return replacementCharacter; if ((value & ~0x1F) != 0x80) return value; return windowsLatin1ExtensionArray[value - 0x80]; }
static inline UChar* appendCharacter(UChar* destination, int character) { ASSERT(character != nonCharacter); ASSERT(!U_IS_SURROGATE(character)); if (U_IS_BMP(character)) *destination++ = character; else { *destination++ = U16_LEAD(character); *destination++ = U16_TRAIL(character); } return destination; }
unsigned calculateStringHashAndLengthFromUTF8MaskingTop8Bits(const char* data, const char* dataEnd, unsigned& dataLength, unsigned& utf16Length) { if (!data) return 0; StringHasher stringHasher; dataLength = 0; utf16Length = 0; while (data < dataEnd || (!dataEnd && *data)) { if (isASCII(*data)) { stringHasher.addCharacter(*data++); dataLength++; utf16Length++; continue; } int utf8SequenceLength = inlineUTF8SequenceLengthNonASCII(*data); dataLength += utf8SequenceLength; if (!dataEnd) { for (int i = 1; i < utf8SequenceLength; ++i) { if (!data[i]) return 0; } } else if (dataEnd - data < utf8SequenceLength) { return 0; } if (!isLegalUTF8(reinterpret_cast<const unsigned char*>(data), utf8SequenceLength)) return 0; UChar32 character = readUTF8Sequence(data, utf8SequenceLength); ASSERT(!isASCII(character)); if (U_IS_BMP(character)) { // UTF-16 surrogate values are illegal in UTF-32 if (U_IS_SURROGATE(character)) return 0; stringHasher.addCharacter(static_cast<UChar>(character)); // normal case utf16Length++; } else if (U_IS_SUPPLEMENTARY(character)) { stringHasher.addCharacters(static_cast<UChar>(U16_LEAD(character)), static_cast<UChar>(U16_TRAIL(character))); utf16Length += 2; } else { return 0; } } return stringHasher.hashWithTop8BitsMasked(); }
// Check if there's any unpaird surrogate code point. // Non-character code points are not checked. static bool isValidUTF16(const String& s) { if (s.is8Bit()) return true; const UChar* ustr = s.characters16(); size_t length = s.length(); size_t position = 0; while (position < length) { UChar32 character; U16_NEXT(ustr, position, length, character); if (U_IS_SURROGATE(character)) return false; } return true; }
// // Check if a UTF16 string is valid according to the UTF16 standard // Specifically, check that we don't have any invalid surrogate pairs // If the string is valid, we return true. // If not, we set invalidIndex to the index of the first invalid char index // and return false // If the invalid char is a lead surrogate pair, we return its index // Otherwise, we treat the char before as the invalid one and return index - 1 // This function has defined behavior only for null-terminated strings. // If the string is not null terminated, the behavior is undefined (likely hang) // static bool IsUtf16StringValid(const UChar* str, size_t length, size_t* invalidIndex) { Assert(invalidIndex != nullptr); *invalidIndex = -1; size_t i = 0; for (;;) { // Iterate through the UTF16-LE string // If we are at the end of the null terminated string, return true // since the string is valid // If not, check if the codepoint we have is a surrogate code unit. // If it is, the string is malformed since U16_NEXT would have returned // is the full codepoint if both code units in the surrogate pair were present UChar32 c; U16_NEXT(str, i, length, c); if (c == 0) { return true; } if (U_IS_SURROGATE(c)) { if (U16_IS_LEAD(c)) { *invalidIndex = i; } else { Assert(i > 0); *invalidIndex = i - 1; } return false; } if (i >= length) { return true; } } }
unsigned calculateStringHashFromUTF8(const char* data, const char* dataEnd, unsigned& utf16Length) { if (!data) return 0; WTF::StringHasher stringHasher; utf16Length = 0; while (data < dataEnd) { if (isASCII(*data)) { stringHasher.addCharacter(*data++); utf16Length++; continue; } int utf8SequenceLength = inlineUTF8SequenceLengthNonASCII(*data); if (dataEnd - data < utf8SequenceLength) return false; if (!isLegalUTF8(reinterpret_cast<const unsigned char*>(data), utf8SequenceLength)) return 0; UChar32 character = readUTF8Sequence(data, utf8SequenceLength); ASSERT(!isASCII(character)); if (U_IS_BMP(character)) { // UTF-16 surrogate values are illegal in UTF-32 if (U_IS_SURROGATE(character)) return 0; stringHasher.addCharacter(static_cast<UChar>(character)); // normal case utf16Length++; } else if (U_IS_SUPPLEMENTARY(character)) { stringHasher.addCharacters(static_cast<UChar>(U16_LEAD(character)), static_cast<UChar>(U16_TRAIL(character))); utf16Length += 2; } else return 0; } return stringHasher.hash(); }
ALWAYS_INLINE bool equalWithUTF8Internal(const CharType* a, const CharType* aEnd, const char* b, const char* bEnd) { while (b < bEnd) { if (isASCII(*b)) { if (*a++ != *b++) return false; continue; } int utf8SequenceLength = inlineUTF8SequenceLengthNonASCII(*b); if (bEnd - b < utf8SequenceLength) return false; if (!isLegalUTF8(reinterpret_cast<const unsigned char*>(b), utf8SequenceLength)) return 0; UChar32 character = readUTF8Sequence(b, utf8SequenceLength); ASSERT(!isASCII(character)); if (U_IS_BMP(character)) { // UTF-16 surrogate values are illegal in UTF-32 if (U_IS_SURROGATE(character)) return false; if (*a++ != character) return false; } else if (U_IS_SUPPLEMENTARY(character)) { if (*a++ != U16_LEAD(character)) return false; if (*a++ != U16_TRAIL(character)) return false; } else { return false; } } return a == aEnd; }
static bool shouldUseCoreText(UChar* buffer, unsigned bufferLength, const SimpleFontData* fontData) { if (fontData->platformData().isCompositeFontReference()) return true; // CoreText doesn't have vertical glyphs of surrogate pair characters. // Therefore, we should not use CoreText, but this always returns horizontal glyphs. // FIXME: We should use vertical glyphs. https://code.google.com/p/chromium/issues/detail?id=340173 if (bufferLength >= 2 && U_IS_SURROGATE(buffer[0]) && fontData->hasVerticalGlyphs()) { ASSERT(U_IS_SURROGATE_LEAD(buffer[0])); ASSERT(U_IS_TRAIL(buffer[1])); return false; } if (fontData->platformData().widthVariant() != RegularWidth || fontData->hasVerticalGlyphs()) { // Ideographs don't have a vertical variant or width variants. for (unsigned i = 0; i < bufferLength; ++i) { if (!Character::isCJKIdeograph(buffer[i])) return true; } } return false; }
/* internal function */ U_CFUNC int32_t u_strcmpFold(const UChar *s1, int32_t length1, const UChar *s2, int32_t length2, uint32_t options, UErrorCode *pErrorCode) { const UCaseProps *csp; /* current-level start/limit - s1/s2 as current */ const UChar *start1, *start2, *limit1, *limit2; /* case folding variables */ const UChar *p; int32_t length; /* stacks of previous-level start/current/limit */ CmpEquivLevel stack1[2], stack2[2]; /* case folding buffers, only use current-level start/limit */ UChar fold1[UCASE_MAX_STRING_LENGTH+1], fold2[UCASE_MAX_STRING_LENGTH+1]; /* track which is the current level per string */ int32_t level1, level2; /* current code units, and code points for lookups */ UChar32 c1, c2, cp1, cp2; /* no argument error checking because this itself is not an API */ /* * assume that at least the option U_COMPARE_IGNORE_CASE is set * otherwise this function would have to behave exactly as uprv_strCompare() */ csp=ucase_getSingleton(); if(U_FAILURE(*pErrorCode)) { return 0; } /* initialize */ start1=s1; if(length1==-1) { limit1=NULL; } else { limit1=s1+length1; } start2=s2; if(length2==-1) { limit2=NULL; } else { limit2=s2+length2; } level1=level2=0; c1=c2=-1; /* comparison loop */ for(;;) { /* * here a code unit value of -1 means "get another code unit" * below it will mean "this source is finished" */ if(c1<0) { /* get next code unit from string 1, post-increment */ for(;;) { if(s1==limit1 || ((c1=*s1)==0 && (limit1==NULL || (options&_STRNCMP_STYLE)))) { if(level1==0) { c1=-1; break; } } else { ++s1; break; } /* reached end of level buffer, pop one level */ do { --level1; start1=stack1[level1].start; } while(start1==NULL); s1=stack1[level1].s; limit1=stack1[level1].limit; } } if(c2<0) { /* get next code unit from string 2, post-increment */ for(;;) { if(s2==limit2 || ((c2=*s2)==0 && (limit2==NULL || (options&_STRNCMP_STYLE)))) { if(level2==0) { c2=-1; break; } } else { ++s2; break; } /* reached end of level buffer, pop one level */ do { --level2; start2=stack2[level2].start; } while(start2==NULL); s2=stack2[level2].s; limit2=stack2[level2].limit; } } /* * compare c1 and c2 * either variable c1, c2 is -1 only if the corresponding string is finished */ if(c1==c2) { if(c1<0) { return 0; /* c1==c2==-1 indicating end of strings */ } c1=c2=-1; /* make us fetch new code units */ continue; } else if(c1<0) { return -1; /* string 1 ends before string 2 */ } else if(c2<0) { return 1; /* string 2 ends before string 1 */ } /* c1!=c2 && c1>=0 && c2>=0 */ /* get complete code points for c1, c2 for lookups if either is a surrogate */ cp1=c1; if(U_IS_SURROGATE(c1)) { UChar c; if(U_IS_SURROGATE_LEAD(c1)) { if(s1!=limit1 && U16_IS_TRAIL(c=*s1)) { /* advance ++s1; only below if cp1 decomposes/case-folds */ cp1=U16_GET_SUPPLEMENTARY(c1, c); } } else /* isTrail(c1) */ { if(start1<=(s1-2) && U16_IS_LEAD(c=*(s1-2))) { cp1=U16_GET_SUPPLEMENTARY(c, c1); } } } cp2=c2; if(U_IS_SURROGATE(c2)) { UChar c; if(U_IS_SURROGATE_LEAD(c2)) { if(s2!=limit2 && U16_IS_TRAIL(c=*s2)) { /* advance ++s2; only below if cp2 decomposes/case-folds */ cp2=U16_GET_SUPPLEMENTARY(c2, c); } } else /* isTrail(c2) */ { if(start2<=(s2-2) && U16_IS_LEAD(c=*(s2-2))) { cp2=U16_GET_SUPPLEMENTARY(c, c2); } } } /* * go down one level for each string * continue with the main loop as soon as there is a real change */ if( level1==0 && (length=ucase_toFullFolding(csp, (UChar32)cp1, &p, options))>=0 ) { /* cp1 case-folds to the code point "length" or to p[length] */ if(U_IS_SURROGATE(c1)) { if(U_IS_SURROGATE_LEAD(c1)) { /* advance beyond source surrogate pair if it case-folds */ ++s1; } else /* isTrail(c1) */ { /* * we got a supplementary code point when hitting its trail surrogate, * therefore the lead surrogate must have been the same as in the other string; * compare this decomposition with the lead surrogate in the other string * remember that this simulates bulk text replacement: * the decomposition would replace the entire code point */ --s2; c2=*(s2-1); } } /* push current level pointers */ stack1[0].start=start1; stack1[0].s=s1; stack1[0].limit=limit1; ++level1; /* copy the folding result to fold1[] */ if(length<=UCASE_MAX_STRING_LENGTH) { u_memcpy(fold1, p, length); } else { int32_t i=0; U16_APPEND_UNSAFE(fold1, i, length); length=i; } /* set next level pointers to case folding */ start1=s1=fold1; limit1=fold1+length; /* get ready to read from decomposition, continue with loop */ c1=-1; continue; } if( level2==0 && (length=ucase_toFullFolding(csp, (UChar32)cp2, &p, options))>=0 ) { /* cp2 case-folds to the code point "length" or to p[length] */ if(U_IS_SURROGATE(c2)) { if(U_IS_SURROGATE_LEAD(c2)) { /* advance beyond source surrogate pair if it case-folds */ ++s2; } else /* isTrail(c2) */ { /* * we got a supplementary code point when hitting its trail surrogate, * therefore the lead surrogate must have been the same as in the other string; * compare this decomposition with the lead surrogate in the other string * remember that this simulates bulk text replacement: * the decomposition would replace the entire code point */ --s1; c1=*(s1-1); } } /* push current level pointers */ stack2[0].start=start2; stack2[0].s=s2; stack2[0].limit=limit2; ++level2; /* copy the folding result to fold2[] */ if(length<=UCASE_MAX_STRING_LENGTH) { u_memcpy(fold2, p, length); } else { int32_t i=0; U16_APPEND_UNSAFE(fold2, i, length); length=i; } /* set next level pointers to case folding */ start2=s2=fold2; limit2=fold2+length; /* get ready to read from decomposition, continue with loop */ c2=-1; continue; } /* * no decomposition/case folding, max level for both sides: * return difference result * * code point order comparison must not just return cp1-cp2 * because when single surrogates are present then the surrogate pairs * that formed cp1 and cp2 may be from different string indexes * * example: { d800 d800 dc01 } vs. { d800 dc00 }, compare at second code units * c1=d800 cp1=10001 c2=dc00 cp2=10000 * cp1-cp2>0 but c1-c2<0 and in fact in UTF-32 it is { d800 10001 } < { 10000 } * * therefore, use same fix-up as in ustring.c/uprv_strCompare() * except: uprv_strCompare() fetches c=*s while this functions fetches c=*s++ * so we have slightly different pointer/start/limit comparisons here */ if(c1>=0xd800 && c2>=0xd800 && (options&U_COMPARE_CODE_POINT_ORDER)) { /* subtract 0x2800 from BMP code points to make them smaller than supplementary ones */ if( (c1<=0xdbff && s1!=limit1 && U16_IS_TRAIL(*s1)) || (U16_IS_TRAIL(c1) && start1!=(s1-1) && U16_IS_LEAD(*(s1-2))) ) { /* part of a surrogate pair, leave >=d800 */ } else { /* BMP code point - may be surrogate code point - make <d800 */ c1-=0x2800; } if( (c2<=0xdbff && s2!=limit2 && U16_IS_TRAIL(*s2)) || (U16_IS_TRAIL(c2) && start2!=(s2-1) && U16_IS_LEAD(*(s2-2))) ) { /* part of a surrogate pair, leave >=d800 */ } else { /* BMP code point - may be surrogate code point - make <d800 */ c2-=0x2800; } } return c1-c2; } }
static void TestSurrogates() { static const uint8_t b[]={ 0xc3, 0x9f, /* 00DF */ 0xed, 0x9f, 0xbf, /* D7FF */ 0xed, 0xa0, 0x81, /* D801 */ 0xed, 0xbf, 0xbe, /* DFFE */ 0xee, 0x80, 0x80, /* E000 */ 0xf0, 0x97, 0xbf, 0xbe /* 17FFE */ }; static const UChar32 cp[]={ 0xdf, 0xd7ff, 0xd801, 0xdffe, 0xe000, 0x17ffe }; UChar32 cu, cs, cl; int32_t i, j, k, iu, is, il, length; k=0; /* index into cp[] */ length=LENGTHOF(b); for(i=0; i<length;) { j=i; U8_NEXT_UNSAFE(b, j, cu); iu=j; j=i; U8_NEXT(b, j, length, cs); is=j; j=i; L8_NEXT(b, j, length, cl); il=j; if(cu!=cp[k]) { log_err("U8_NEXT_UNSAFE(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cu, (long)cp[k]); } /* U8_NEXT() returns <0 for surrogate code points */ if(U_IS_SURROGATE(cu) ? cs>=0 : cs!=cu) { log_err("U8_NEXT(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cs, (long)cu); } /* L8_NEXT() returns surrogate code points like U8_NEXT_UNSAFE() */ if(cl!=cu) { log_err("L8_NEXT(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cl, (long)cu); } if(is!=iu || il!=iu) { log_err("U8_NEXT(b[%ld]) or L8_NEXT(b[%ld]) did not advance the index correctly\n", (long)i, (long)i); } ++k; /* next code point */ i=iu; /* advance by one UTF-8 sequence */ } while(i>0) { --k; /* previous code point */ j=i; U8_PREV_UNSAFE(b, j, cu); iu=j; j=i; U8_PREV(b, 0, j, cs); is=j; j=i; L8_PREV(b, 0, j, cl); il=j; if(cu!=cp[k]) { log_err("U8_PREV_UNSAFE(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cu, (long)cp[k]); } /* U8_PREV() returns <0 for surrogate code points */ if(U_IS_SURROGATE(cu) ? cs>=0 : cs!=cu) { log_err("U8_PREV(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cs, (long)cu); } /* L8_PREV() returns surrogate code points like U8_PREV_UNSAFE() */ if(cl!=cu) { log_err("L8_PREV(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cl, (long)cu); } if(is!=iu || il !=iu) { log_err("U8_PREV(b[%ld]) or L8_PREV(b[%ld]) did not advance the index correctly\n", (long)i, (long)i); } i=iu; /* go back by one UTF-8 sequence */ } }
U_CFUNC int32_t u_strFromPunycode(const UChar *src, int32_t srcLength, UChar *dest, int32_t destCapacity, UBool *caseFlags, UErrorCode *pErrorCode) { int32_t n, destLength, i, bias, basicLength, j, in, oldi, w, k, digit, t, destCPCount, firstSupplementaryIndex, cpLength; UChar b; /* argument checking */ if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { return 0; } if(src==NULL || srcLength<-1 || (dest==NULL && destCapacity!=0)) { *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; return 0; } if(srcLength==-1) { srcLength=u_strlen(src); } /* * Handle the basic code points: * Let basicLength be the number of input code points * before the last delimiter, or 0 if there is none, * then copy the first basicLength code points to the output. * * The two following loops iterate backward. */ for(j=srcLength; j>0;) { if(src[--j]==DELIMITER) { break; } } destLength=basicLength=destCPCount=j; U_ASSERT(destLength>=0); while(j>0) { b=src[--j]; if(!IS_BASIC(b)) { *pErrorCode=U_INVALID_CHAR_FOUND; return 0; } if(j<destCapacity) { dest[j]=(UChar)b; if(caseFlags!=NULL) { caseFlags[j]=IS_BASIC_UPPERCASE(b); } } } /* Initialize the state: */ n=INITIAL_N; i=0; bias=INITIAL_BIAS; firstSupplementaryIndex=1000000000; /* * Main decoding loop: * Start just after the last delimiter if any * basic code points were copied; start at the beginning otherwise. */ for(in=basicLength>0 ? basicLength+1 : 0; in<srcLength; /* no op */) { /* * in is the index of the next character to be consumed, and * destCPCount is the number of code points in the output array. * * Decode a generalized variable-length integer into delta, * which gets added to i. The overflow checking is easier * if we increase i as we go, then subtract off its starting * value at the end to obtain delta. */ for(oldi=i, w=1, k=BASE; /* no condition */; k+=BASE) { if(in>=srcLength) { *pErrorCode=U_ILLEGAL_CHAR_FOUND; return 0; } digit=basicToDigit[(uint8_t)src[in++]]; if(digit<0) { *pErrorCode=U_INVALID_CHAR_FOUND; return 0; } if(digit>(0x7fffffff-i)/w) { /* integer overflow */ *pErrorCode=U_ILLEGAL_CHAR_FOUND; return 0; } i+=digit*w; /** RAM: comment out the old code for conformance with draft-ietf-idn-punycode-03.txt t=k-bias; if(t<TMIN) { t=TMIN; } else if(t>TMAX) { t=TMAX; } */ t=k-bias; if(t<TMIN) { t=TMIN; } else if(k>=(bias+TMAX)) { t=TMAX; } if(digit<t) { break; } if(w>0x7fffffff/(BASE-t)) { /* integer overflow */ *pErrorCode=U_ILLEGAL_CHAR_FOUND; return 0; } w*=BASE-t; } /* * Modification from sample code: * Increments destCPCount here, * where needed instead of in for() loop tail. */ ++destCPCount; bias=adaptBias(i-oldi, destCPCount, (UBool)(oldi==0)); /* * i was supposed to wrap around from (incremented) destCPCount to 0, * incrementing n each time, so we'll fix that now: */ if(i/destCPCount>(0x7fffffff-n)) { /* integer overflow */ *pErrorCode=U_ILLEGAL_CHAR_FOUND; return 0; } n+=i/destCPCount; i%=destCPCount; /* not needed for Punycode: */ /* if (decode_digit(n) <= BASE) return punycode_invalid_input; */ if(n>0x10ffff || U_IS_SURROGATE(n)) { /* Unicode code point overflow */ *pErrorCode=U_ILLEGAL_CHAR_FOUND; return 0; } /* Insert n at position i of the output: */ cpLength=U16_LENGTH(n); if(dest!=NULL && ((destLength+cpLength)<=destCapacity)) { int32_t codeUnitIndex; /* * Handle indexes when supplementary code points are present. * * In almost all cases, there will be only BMP code points before i * and even in the entire string. * This is handled with the same efficiency as with UTF-32. * * Only the rare cases with supplementary code points are handled * more slowly - but not too bad since this is an insertion anyway. */ if(i<=firstSupplementaryIndex) { codeUnitIndex=i; if(cpLength>1) { firstSupplementaryIndex=codeUnitIndex; } else { ++firstSupplementaryIndex; } } else { codeUnitIndex=firstSupplementaryIndex; U16_FWD_N(dest, codeUnitIndex, destLength, i-codeUnitIndex); } /* use the UChar index codeUnitIndex instead of the code point index i */ if(codeUnitIndex<destLength) { uprv_memmove(dest+codeUnitIndex+cpLength, dest+codeUnitIndex, (destLength-codeUnitIndex)*U_SIZEOF_UCHAR); if(caseFlags!=NULL) { uprv_memmove(caseFlags+codeUnitIndex+cpLength, caseFlags+codeUnitIndex, destLength-codeUnitIndex); } } if(cpLength==1) { /* BMP, insert one code unit */ dest[codeUnitIndex]=(UChar)n; } else { /* supplementary character, insert two code units */ dest[codeUnitIndex]=U16_LEAD(n); dest[codeUnitIndex+1]=U16_TRAIL(n); } if(caseFlags!=NULL) { /* Case of last character determines uppercase flag: */ caseFlags[codeUnitIndex]=IS_BASIC_UPPERCASE(src[in-1]); if(cpLength==2) { caseFlags[codeUnitIndex+1]=FALSE; } } } destLength+=cpLength; U_ASSERT(destLength>=0); ++i; } return u_terminateUChars(dest, destCapacity, destLength, pErrorCode); }
/* parse a mapping line; must not be empty */ U_CAPI UBool U_EXPORT2 ucm_parseMappingLine(UCMapping *m, UChar32 codePoints[UCNV_EXT_MAX_UCHARS], uint8_t bytes[UCNV_EXT_MAX_BYTES], const char *line) { const char *s; char *end; UChar32 cp; int32_t u16Length; int8_t uLen, bLen, f; s=line; uLen=bLen=0; /* parse code points */ for(;;) { /* skip an optional plus sign */ if(uLen>0 && *s=='+') { ++s; } if(*s!='<') { break; } if( s[1]!='U' || (cp=(UChar32)uprv_strtoul(s+2, &end, 16), end)==s+2 || *end!='>' ) { fprintf(stderr, "ucm error: Unicode code point must be formatted as <UXXXX> (1..6 hex digits) - \"%s\"\n", line); return FALSE; } if((uint32_t)cp>0x10ffff || U_IS_SURROGATE(cp)) { fprintf(stderr, "ucm error: Unicode code point must be 0..d7ff or e000..10ffff - \"%s\"\n", line); return FALSE; } if(uLen==UCNV_EXT_MAX_UCHARS) { fprintf(stderr, "ucm error: too many code points on \"%s\"\n", line); return FALSE; } codePoints[uLen++]=cp; s=end+1; } if(uLen==0) { fprintf(stderr, "ucm error: no Unicode code points on \"%s\"\n", line); return FALSE; } else if(uLen==1) { m->u=codePoints[0]; } else { UErrorCode errorCode=U_ZERO_ERROR; u_strFromUTF32(NULL, 0, &u16Length, codePoints, uLen, &errorCode); if( (U_FAILURE(errorCode) && errorCode!=U_BUFFER_OVERFLOW_ERROR) || u16Length>UCNV_EXT_MAX_UCHARS ) { fprintf(stderr, "ucm error: too many UChars on \"%s\"\n", line); return FALSE; } } s=u_skipWhitespace(s); /* parse bytes */ bLen=ucm_parseBytes(bytes, line, &s); if(bLen<0) { return FALSE; } else if(bLen==0) { fprintf(stderr, "ucm error: no bytes on \"%s\"\n", line); return FALSE; } else if(bLen<=4) { uprv_memcpy(m->b.bytes, bytes, bLen); } /* skip everything until the fallback indicator, even the start of a comment */ for(;;) { if(*s==0) { f=-1; /* no fallback indicator */ break; } else if(*s=='|') { f=(int8_t)(s[1]-'0'); if((uint8_t)f>4) { fprintf(stderr, "ucm error: fallback indicator must be |0..|4 - \"%s\"\n", line); return FALSE; } break; } ++s; } m->uLen=uLen; m->bLen=bLen; m->f=f; return TRUE; }
U_CAPI void U_EXPORT2 ucm_addMapping(UCMTable *table, UCMapping *m, UChar32 codePoints[UCNV_EXT_MAX_UCHARS], uint8_t bytes[UCNV_EXT_MAX_BYTES]) { UCMapping *tm; UChar32 c; int32_t idx; if(table->mappingsLength>=table->mappingsCapacity) { /* make the mappings array larger */ if(table->mappingsCapacity==0) { table->mappingsCapacity=1000; } else { table->mappingsCapacity*=10; } table->mappings=(UCMapping *)uprv_realloc(table->mappings, table->mappingsCapacity*sizeof(UCMapping)); if(table->mappings==NULL) { fprintf(stderr, "ucm error: unable to allocate %d UCMappings\n", (int)table->mappingsCapacity); exit(U_MEMORY_ALLOCATION_ERROR); } if(table->reverseMap!=NULL) { /* the reverseMap must be reallocated in a new sort */ uprv_free(table->reverseMap); table->reverseMap=NULL; } } if(m->uLen>1 && table->codePointsCapacity==0) { table->codePointsCapacity=10000; table->codePoints=(UChar32 *)uprv_malloc(table->codePointsCapacity*4); if(table->codePoints==NULL) { fprintf(stderr, "ucm error: unable to allocate %d UChar32s\n", (int)table->codePointsCapacity); exit(U_MEMORY_ALLOCATION_ERROR); } } if(m->bLen>4 && table->bytesCapacity==0) { table->bytesCapacity=10000; table->bytes=(uint8_t *)uprv_malloc(table->bytesCapacity); if(table->bytes==NULL) { fprintf(stderr, "ucm error: unable to allocate %d bytes\n", (int)table->bytesCapacity); exit(U_MEMORY_ALLOCATION_ERROR); } } if(m->uLen>1) { idx=table->codePointsLength; table->codePointsLength+=m->uLen; if(table->codePointsLength>table->codePointsCapacity) { fprintf(stderr, "ucm error: too many code points in multiple-code point mappings\n"); exit(U_MEMORY_ALLOCATION_ERROR); } uprv_memcpy(table->codePoints+idx, codePoints, (size_t)m->uLen*4); m->u=idx; } if(m->bLen>4) { idx=table->bytesLength; table->bytesLength+=m->bLen; if(table->bytesLength>table->bytesCapacity) { fprintf(stderr, "ucm error: too many bytes in mappings with >4 charset bytes\n"); exit(U_MEMORY_ALLOCATION_ERROR); } uprv_memcpy(table->bytes+idx, bytes, m->bLen); m->b.idx=idx; } /* set unicodeMask */ for(idx=0; idx<m->uLen; ++idx) { c=codePoints[idx]; if(c>=0x10000) { table->unicodeMask|=UCNV_HAS_SUPPLEMENTARY; /* there are supplementary code points */ } else if(U_IS_SURROGATE(c)) { table->unicodeMask|=UCNV_HAS_SURROGATES; /* there are surrogate code points */ } } /* set flagsType */ if(m->f<0) { table->flagsType|=UCM_FLAGS_IMPLICIT; } else { table->flagsType|=UCM_FLAGS_EXPLICIT; } tm=table->mappings+table->mappingsLength++; uprv_memcpy(tm, m, sizeof(UCMapping)); table->isSorted=FALSE; }