void CasePropsBuilder::addClosureMapping(UChar32 src, UChar32 dest, UErrorCode &errorCode) { if(U_FAILURE(errorCode)) { return; } if(beVerbose) { printf("add closure mapping U+%04lx->U+%04lx\n", (unsigned long)src, (unsigned long)dest); } uint32_t value=utrie2_get32(pTrie, src); if((value&UCASE_EXCEPTION)==0) { /* * decode value into p2 (enough for makeException() to work properly), * add the closure mapping, * and set the new exception for src */ value=makeExcProps(src, value, errorCode); utrie2_set32(pTrie, src, value, &errorCode); if(U_FAILURE(errorCode)) { fprintf(stderr, "genprops error: unable to set case mapping values, code: %s\n", u_errorName(errorCode)); return; } } excProps[value>>UGENCASE_EXC_SHIFT]->closure.add(dest); }
/* * get or create a Norm unit; * get or create the intermediate trie entries for it as well */ Norm *Normalizer2DataBuilder::createNorm(UChar32 c) { uint32_t i=utrie2_get32(normTrie, c); if(i!=0) { return norms+i; } else { /* allocate Norm */ Norm *p=allocNorm(); IcuToolErrorCode errorCode("gennorm2/createNorm()"); utrie2_set32(normTrie, c, (uint32_t)(p-norms), errorCode); return p; } }
static UBool U_CALLCONV copyEnumRange(const void *context, UChar32 start, UChar32 end, uint32_t value) { NewTrieAndStatus *nt=(NewTrieAndStatus *)context; if(value!=nt->trie->initialValue) { if(nt->exclusiveLimit) { --end; } if(start==end) { utrie2_set32(nt->trie, start, value, &nt->errorCode); } else { utrie2_setRange32(nt->trie, start, end, value, TRUE, &nt->errorCode); } return U_SUCCESS(nt->errorCode); } else { return TRUE; } }
// Build the Whole Script Confusable data // // TODO: Reorganize. Either get rid of the WSConfusableDataBuilder class, // because everything is local to this one build function anyhow, // OR // break this function into more reasonably sized pieces, with // state in WSConfusableDataBuilder. // void buildWSConfusableData(SpoofImpl *spImpl, const char * confusablesWS, int32_t confusablesWSLen, UParseError *pe, UErrorCode &status) { if (U_FAILURE(status)) { return; } URegularExpression *parseRegexp = NULL; int32_t inputLen = 0; UChar *input = NULL; int32_t lineNum = 0; UVector *scriptSets = NULL; uint32_t rtScriptSetsCount = 2; UTrie2 *anyCaseTrie = NULL; UTrie2 *lowerCaseTrie = NULL; anyCaseTrie = utrie2_open(0, 0, &status); lowerCaseTrie = utrie2_open(0, 0, &status); // The scriptSets vector provides a mapping from TRIE values to the set of scripts. // // Reserved TRIE values: // 0: Code point has no whole script confusables. // 1: Code point is of script Common or Inherited. // These code points do not participate in whole script confusable detection. // (This is logically equivalent to saying that they contain confusables in // all scripts) // // Because Trie values are indexes into the ScriptSets vector, pre-fill // vector positions 0 and 1 to avoid conflicts with the reserved values. scriptSets = new UVector(status); if (scriptSets == NULL) { status = U_MEMORY_ALLOCATION_ERROR; goto cleanup; } scriptSets->addElement((void *)NULL, status); scriptSets->addElement((void *)NULL, status); // Convert the user input data from UTF-8 to UChar (UTF-16) u_strFromUTF8(NULL, 0, &inputLen, confusablesWS, confusablesWSLen, &status); if (status != U_BUFFER_OVERFLOW_ERROR) { goto cleanup; } status = U_ZERO_ERROR; input = static_cast<UChar *>(uprv_malloc((inputLen+1) * sizeof(UChar))); if (input == NULL) { status = U_MEMORY_ALLOCATION_ERROR; goto cleanup; } u_strFromUTF8(input, inputLen+1, NULL, confusablesWS, confusablesWSLen, &status); parseRegexp = uregex_openC(parseExp, 0, NULL, &status); // Zap any Byte Order Mark at the start of input. Changing it to a space is benign // given the syntax of the input. if (*input == 0xfeff) { *input = 0x20; } // Parse the input, one line per iteration of this loop. uregex_setText(parseRegexp, input, inputLen, &status); while (uregex_findNext(parseRegexp, &status)) { lineNum++; UChar line[200]; uregex_group(parseRegexp, 0, line, 200, &status); if (uregex_start(parseRegexp, 1, &status) >= 0) { // this was a blank or comment line. continue; } if (uregex_start(parseRegexp, 8, &status) >= 0) { // input file syntax error. status = U_PARSE_ERROR; goto cleanup; } if (U_FAILURE(status)) { goto cleanup; } // Pick up the start and optional range end code points from the parsed line. UChar32 startCodePoint = SpoofImpl::ScanHex( input, uregex_start(parseRegexp, 2, &status), uregex_end(parseRegexp, 2, &status), status); UChar32 endCodePoint = startCodePoint; if (uregex_start(parseRegexp, 3, &status) >=0) { endCodePoint = SpoofImpl::ScanHex( input, uregex_start(parseRegexp, 3, &status), uregex_end(parseRegexp, 3, &status), status); } // Extract the two script names from the source line. We need these in an 8 bit // default encoding (will be EBCDIC on IBM mainframes) in order to pass them on // to the ICU u_getPropertyValueEnum() function. Ugh. char srcScriptName[20]; char targScriptName[20]; extractGroup(parseRegexp, 4, srcScriptName, sizeof(srcScriptName), status); extractGroup(parseRegexp, 5, targScriptName, sizeof(targScriptName), status); UScriptCode srcScript = static_cast<UScriptCode>(u_getPropertyValueEnum(UCHAR_SCRIPT, srcScriptName)); UScriptCode targScript = static_cast<UScriptCode>(u_getPropertyValueEnum(UCHAR_SCRIPT, targScriptName)); if (U_FAILURE(status)) { goto cleanup; } if (srcScript == USCRIPT_INVALID_CODE || targScript == USCRIPT_INVALID_CODE) { status = U_INVALID_FORMAT_ERROR; goto cleanup; } // select the table - (A) any case or (L) lower case only UTrie2 *table = anyCaseTrie; if (uregex_start(parseRegexp, 7, &status) >= 0) { table = lowerCaseTrie; } // Build the set of scripts containing confusable characters for // the code point(s) specified in this input line. // Sanity check that the script of the source code point is the same // as the source script indicated in the input file. Failure of this check is // an error in the input file. // Include the source script in the set (needed for Mixed Script Confusable detection). // UChar32 cp; for (cp=startCodePoint; cp<=endCodePoint; cp++) { int32_t setIndex = utrie2_get32(table, cp); BuilderScriptSet *bsset = NULL; if (setIndex > 0) { U_ASSERT(setIndex < scriptSets->size()); bsset = static_cast<BuilderScriptSet *>(scriptSets->elementAt(setIndex)); } else { bsset = new BuilderScriptSet(); if (bsset == NULL) { status = U_MEMORY_ALLOCATION_ERROR; goto cleanup; } bsset->codePoint = cp; bsset->trie = table; bsset->sset = new ScriptSet(); setIndex = scriptSets->size(); bsset->index = setIndex; bsset->rindex = 0; if (bsset->sset == NULL) { status = U_MEMORY_ALLOCATION_ERROR; goto cleanup; } scriptSets->addElement(bsset, status); utrie2_set32(table, cp, setIndex, &status); } bsset->sset->Union(targScript); bsset->sset->Union(srcScript); if (U_FAILURE(status)) { goto cleanup; } UScriptCode cpScript = uscript_getScript(cp, &status); if (cpScript != srcScript) { status = U_INVALID_FORMAT_ERROR; goto cleanup; } } } // Eliminate duplicate script sets. At this point we have a separate // script set for every code point that had data in the input file. // // We eliminate underlying ScriptSet objects, not the BuildScriptSets that wrap them // // printf("Number of scriptSets: %d\n", scriptSets->size()); { int32_t duplicateCount = 0; rtScriptSetsCount = 2; for (int32_t outeri=2; outeri<scriptSets->size(); outeri++) { BuilderScriptSet *outerSet = static_cast<BuilderScriptSet *>(scriptSets->elementAt(outeri)); if (outerSet->index != static_cast<uint32_t>(outeri)) { // This set was already identified as a duplicate. // It will not be allocated a position in the runtime array of ScriptSets. continue; } outerSet->rindex = rtScriptSetsCount++; for (int32_t inneri=outeri+1; inneri<scriptSets->size(); inneri++) { BuilderScriptSet *innerSet = static_cast<BuilderScriptSet *>(scriptSets->elementAt(inneri)); if (*(outerSet->sset) == *(innerSet->sset) && outerSet->sset != innerSet->sset) { delete innerSet->sset; innerSet->scriptSetOwned = FALSE; innerSet->sset = outerSet->sset; innerSet->index = outeri; innerSet->rindex = outerSet->rindex; duplicateCount++; } // But this doesn't get all. We need to fix the TRIE. } } // printf("Number of distinct script sets: %d\n", rtScriptSetsCount); } // Update the Trie values to be reflect the run time script indexes (after duplicate merging). // (Trie Values 0 and 1 are reserved, and the corresponding slots in scriptSets // are unused, which is why the loop index starts at 2.) { for (int32_t i=2; i<scriptSets->size(); i++) { BuilderScriptSet *bSet = static_cast<BuilderScriptSet *>(scriptSets->elementAt(i)); if (bSet->rindex != (uint32_t)i) { utrie2_set32(bSet->trie, bSet->codePoint, bSet->rindex, &status); } } } // For code points with script==Common or script==Inherited, // Set the reserved value of 1 into both Tries. These characters do not participate // in Whole Script Confusable detection; this reserved value is the means // by which they are detected. { UnicodeSet ignoreSet; ignoreSet.applyIntPropertyValue(UCHAR_SCRIPT, USCRIPT_COMMON, status); UnicodeSet inheritedSet; inheritedSet.applyIntPropertyValue(UCHAR_SCRIPT, USCRIPT_INHERITED, status); ignoreSet.addAll(inheritedSet); for (int32_t rn=0; rn<ignoreSet.getRangeCount(); rn++) { UChar32 rangeStart = ignoreSet.getRangeStart(rn); UChar32 rangeEnd = ignoreSet.getRangeEnd(rn); utrie2_setRange32(anyCaseTrie, rangeStart, rangeEnd, 1, TRUE, &status); utrie2_setRange32(lowerCaseTrie, rangeStart, rangeEnd, 1, TRUE, &status); } } // Serialize the data to the Spoof Detector { utrie2_freeze(anyCaseTrie, UTRIE2_16_VALUE_BITS, &status); int32_t size = utrie2_serialize(anyCaseTrie, NULL, 0, &status); // printf("Any case Trie size: %d\n", size); if (status != U_BUFFER_OVERFLOW_ERROR) { goto cleanup; } status = U_ZERO_ERROR; spImpl->fSpoofData->fRawData->fAnyCaseTrie = spImpl->fSpoofData->fMemLimit; spImpl->fSpoofData->fRawData->fAnyCaseTrieLength = size; spImpl->fSpoofData->fAnyCaseTrie = anyCaseTrie; void *where = spImpl->fSpoofData->reserveSpace(size, status); utrie2_serialize(anyCaseTrie, where, size, &status); utrie2_freeze(lowerCaseTrie, UTRIE2_16_VALUE_BITS, &status); size = utrie2_serialize(lowerCaseTrie, NULL, 0, &status); // printf("Lower case Trie size: %d\n", size); if (status != U_BUFFER_OVERFLOW_ERROR) { goto cleanup; } status = U_ZERO_ERROR; spImpl->fSpoofData->fRawData->fLowerCaseTrie = spImpl->fSpoofData->fMemLimit; spImpl->fSpoofData->fRawData->fLowerCaseTrieLength = size; spImpl->fSpoofData->fLowerCaseTrie = lowerCaseTrie; where = spImpl->fSpoofData->reserveSpace(size, status); utrie2_serialize(lowerCaseTrie, where, size, &status); spImpl->fSpoofData->fRawData->fScriptSets = spImpl->fSpoofData->fMemLimit; spImpl->fSpoofData->fRawData->fScriptSetsLength = rtScriptSetsCount; ScriptSet *rtScriptSets = static_cast<ScriptSet *> (spImpl->fSpoofData->reserveSpace(rtScriptSetsCount * sizeof(ScriptSet), status)); uint32_t rindex = 2; for (int32_t i=2; i<scriptSets->size(); i++) { BuilderScriptSet *bSet = static_cast<BuilderScriptSet *>(scriptSets->elementAt(i)); if (bSet->rindex < rindex) { // We have already copied this script set to the serialized data. continue; } U_ASSERT(rindex == bSet->rindex); rtScriptSets[rindex] = *bSet->sset; // Assignment of a ScriptSet just copies the bits. rindex++; } } // Open new utrie2s from the serialized data. We don't want to keep the ones // we just built because we would then have two copies of the data, one internal to // the utries that we have already constructed, and one in the serialized data area. // An alternative would be to not pre-serialize the Trie data, but that makes the // spoof detector data different, depending on how the detector was constructed. // It's simpler to keep the data always the same. spImpl->fSpoofData->fAnyCaseTrie = utrie2_openFromSerialized( UTRIE2_16_VALUE_BITS, (const char *)spImpl->fSpoofData->fRawData + spImpl->fSpoofData->fRawData->fAnyCaseTrie, spImpl->fSpoofData->fRawData->fAnyCaseTrieLength, NULL, &status); spImpl->fSpoofData->fLowerCaseTrie = utrie2_openFromSerialized( UTRIE2_16_VALUE_BITS, (const char *)spImpl->fSpoofData->fRawData + spImpl->fSpoofData->fRawData->fLowerCaseTrie, spImpl->fSpoofData->fRawData->fAnyCaseTrieLength, NULL, &status); cleanup: if (U_FAILURE(status)) { pe->line = lineNum; } uregex_close(parseRegexp); uprv_free(input); int32_t i; for (i=0; i<scriptSets->size(); i++) { BuilderScriptSet *bsset = static_cast<BuilderScriptSet *>(scriptSets->elementAt(i)); delete bsset; } delete scriptSets; utrie2_close(anyCaseTrie); utrie2_close(lowerCaseTrie); return; }
U_CAPI UTrie2 * U_EXPORT2 utrie2_open(uint32_t initialValue, uint32_t errorValue, UErrorCode *pErrorCode) { UTrie2 *trie; UNewTrie2 *newTrie; uint32_t *data; int32_t i, j; if(U_FAILURE(*pErrorCode)) { return NULL; } trie=(UTrie2 *)uprv_malloc(sizeof(UTrie2)); newTrie=(UNewTrie2 *)uprv_malloc(sizeof(UNewTrie2)); data=(uint32_t *)uprv_malloc(UNEWTRIE2_INITIAL_DATA_LENGTH*4); if(trie==NULL || newTrie==NULL || data==NULL) { uprv_free(trie); uprv_free(newTrie); uprv_free(data); *pErrorCode=U_MEMORY_ALLOCATION_ERROR; return 0; } uprv_memset(trie, 0, sizeof(UTrie2)); trie->initialValue=initialValue; trie->errorValue=errorValue; trie->highStart=0x110000; trie->newTrie=newTrie; newTrie->data=data; newTrie->dataCapacity=UNEWTRIE2_INITIAL_DATA_LENGTH; newTrie->initialValue=initialValue; newTrie->errorValue=errorValue; newTrie->highStart=0x110000; newTrie->firstFreeBlock=0; /* no free block in the list */ newTrie->isCompacted=FALSE; /* * preallocate and reset * - ASCII * - the bad-UTF-8-data block * - the null data block */ for(i=0; i<0x80; ++i) { newTrie->data[i]=initialValue; } for(; i<0xc0; ++i) { newTrie->data[i]=errorValue; } for(i=UNEWTRIE2_DATA_NULL_OFFSET; i<UNEWTRIE2_DATA_START_OFFSET; ++i) { newTrie->data[i]=initialValue; } newTrie->dataNullOffset=UNEWTRIE2_DATA_NULL_OFFSET; newTrie->dataLength=UNEWTRIE2_DATA_START_OFFSET; /* set the index-2 indexes for the 2=0x80>>UTRIE2_SHIFT_2 ASCII data blocks */ for(i=0, j=0; j<0x80; ++i, j+=UTRIE2_DATA_BLOCK_LENGTH) { newTrie->index2[i]=j; newTrie->map[i]=1; } /* reference counts for the bad-UTF-8-data block */ for(; j<0xc0; ++i, j+=UTRIE2_DATA_BLOCK_LENGTH) { newTrie->map[i]=0; } /* * Reference counts for the null data block: all blocks except for the ASCII blocks. * Plus 1 so that we don't drop this block during compaction. * Plus as many as needed for lead surrogate code points. */ /* i==newTrie->dataNullOffset */ newTrie->map[i++]= (0x110000>>UTRIE2_SHIFT_2)- (0x80>>UTRIE2_SHIFT_2)+ 1+ UTRIE2_LSCP_INDEX_2_LENGTH; j+=UTRIE2_DATA_BLOCK_LENGTH; for(; j<UNEWTRIE2_DATA_START_OFFSET; ++i, j+=UTRIE2_DATA_BLOCK_LENGTH) { newTrie->map[i]=0; } /* * set the remaining indexes in the BMP index-2 block * to the null data block */ for(i=0x80>>UTRIE2_SHIFT_2; i<UTRIE2_INDEX_2_BMP_LENGTH; ++i) { newTrie->index2[i]=UNEWTRIE2_DATA_NULL_OFFSET; } /* * Fill the index gap with impossible values so that compaction * does not overlap other index-2 blocks with the gap. */ for(i=0; i<UNEWTRIE2_INDEX_GAP_LENGTH; ++i) { newTrie->index2[UNEWTRIE2_INDEX_GAP_OFFSET+i]=-1; } /* set the indexes in the null index-2 block */ for(i=0; i<UTRIE2_INDEX_2_BLOCK_LENGTH; ++i) { newTrie->index2[UNEWTRIE2_INDEX_2_NULL_OFFSET+i]=UNEWTRIE2_DATA_NULL_OFFSET; } newTrie->index2NullOffset=UNEWTRIE2_INDEX_2_NULL_OFFSET; newTrie->index2Length=UNEWTRIE2_INDEX_2_START_OFFSET; /* set the index-1 indexes for the linear index-2 block */ for(i=0, j=0; i<UTRIE2_OMITTED_BMP_INDEX_1_LENGTH; ++i, j+=UTRIE2_INDEX_2_BLOCK_LENGTH ) { newTrie->index1[i]=j; } /* set the remaining index-1 indexes to the null index-2 block */ for(; i<UNEWTRIE2_INDEX_1_LENGTH; ++i) { newTrie->index1[i]=UNEWTRIE2_INDEX_2_NULL_OFFSET; } /* * Preallocate and reset data for U+0080..U+07ff, * for 2-byte UTF-8 which will be compacted in 64-blocks * even if UTRIE2_DATA_BLOCK_LENGTH is smaller. */ for(i=0x80; i<0x800; i+=UTRIE2_DATA_BLOCK_LENGTH) { utrie2_set32(trie, i, initialValue, pErrorCode); } return trie; }