/* Almost the same as utrie2_cloneAsThawed() but copies a UTrie and freezes the clone. */ U_CAPI UTrie2 * U_EXPORT2 utrie2_fromUTrie(const UTrie *trie1, uint32_t errorValue, UErrorCode *pErrorCode) { NewTrieAndStatus context; UChar lead; if(U_FAILURE(*pErrorCode)) { return NULL; } if(trie1==NULL) { *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; return NULL; } context.trie=utrie2_open(trie1->initialValue, errorValue, pErrorCode); if(U_FAILURE(*pErrorCode)) { return NULL; } context.exclusiveLimit=TRUE; context.errorCode=*pErrorCode; utrie_enum(trie1, NULL, copyEnumRange, &context); *pErrorCode=context.errorCode; for(lead=0xd800; lead<0xdc00; ++lead) { uint32_t value; if(trie1->data32==NULL) { value=UTRIE_GET16_FROM_LEAD(trie1, lead); } else { value=UTRIE_GET32_FROM_LEAD(trie1, lead); } if(value!=trie1->initialValue) { utrie2_set32ForLeadSurrogateCodeUnit(context.trie, lead, value, pErrorCode); } } if(U_SUCCESS(*pErrorCode)) { utrie2_freeze(context.trie, trie1->data32!=NULL ? UTRIE2_32_VALUE_BITS : UTRIE2_16_VALUE_BITS, pErrorCode); } #ifdef UTRIE2_DEBUG if(U_SUCCESS(*pErrorCode)) { utrie_printLengths(trie1); utrie2_printLengths(context.trie, "fromUTrie"); } #endif if(U_FAILURE(*pErrorCode)) { utrie2_close(context.trie); context.trie=NULL; } return context.trie; }
U_CAPI UTrie2 * U_EXPORT2 utrie2_cloneAsThawed(const UTrie2 *other, UErrorCode *pErrorCode) { NewTrieAndStatus context; UChar lead; if(U_FAILURE(*pErrorCode)) { return NULL; } if(other==NULL || (other->memory==NULL && other->newTrie==NULL)) { *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; return NULL; } if(other->newTrie!=NULL && !other->newTrie->isCompacted) { return utrie2_clone(other, pErrorCode); /* clone an unfrozen trie */ } /* Clone the frozen trie by enumerating it and building a new one. */ context.trie=utrie2_open(other->initialValue, other->errorValue, pErrorCode); if(U_FAILURE(*pErrorCode)) { return NULL; } context.exclusiveLimit=FALSE; context.errorCode=*pErrorCode; utrie2_enum(other, NULL, copyEnumRange, &context); *pErrorCode=context.errorCode; for(lead=0xd800; lead<0xdc00; ++lead) { uint32_t value; if(other->data32==NULL) { value=UTRIE2_GET16_FROM_U16_SINGLE_LEAD(other, lead); } else { value=UTRIE2_GET32_FROM_U16_SINGLE_LEAD(other, lead); } if(value!=other->initialValue) { utrie2_set32ForLeadSurrogateCodeUnit(context.trie, lead, value, pErrorCode); } } if(U_FAILURE(*pErrorCode)) { utrie2_close(context.trie); context.trie=NULL; } return context.trie; }
extern void generateData(const char *dataDir, UBool csource) { static int32_t indexes[UBIDI_IX_TOP]={ UBIDI_IX_TOP }; static uint8_t trieBlock[40000]; static uint8_t jgArray[0x300]; /* at most for U+0600..U+08FF */ const uint32_t *row; UChar32 start, end, prev, jgStart; int32_t i; UNewDataMemory *pData; UNewTrie *pTrie; UErrorCode errorCode=U_ZERO_ERROR; int32_t trieSize; long dataLength; makeMirror(); pTrie=utrie_open(NULL, NULL, 20000, 0, 0, TRUE); if(pTrie==NULL) { fprintf(stderr, "genbidi error: unable to create a UNewTrie\n"); exit(U_MEMORY_ALLOCATION_ERROR); } prev=jgStart=0; for(i=0; (row=upvec_getRow(pv, i, &start, &end))!=NULL && start<UPVEC_FIRST_SPECIAL_CP; ++i) { /* store most values from vector column 0 in the trie */ if(!utrie_setRange32(pTrie, start, end+1, *row, TRUE)) { fprintf(stderr, "genbidi error: unable to set trie value (overflow)\n"); exit(U_BUFFER_OVERFLOW_ERROR); } /* store Joining_Group values from vector column 1 in a simple byte array */ if(row[1]!=0) { if(start<0x600 || 0x8ff<end) { fprintf(stderr, "genbidi error: Joining_Group for out-of-range code points U+%04lx..U+%04lx\n", (long)start, (long)end); exit(U_ILLEGAL_ARGUMENT_ERROR); } if(prev==0) { /* first code point with any value */ prev=jgStart=start; } else { /* add No_Joining_Group for code points between prev and start */ while(prev<start) { jgArray[prev++ -jgStart]=0; } } /* set Joining_Group value for start..end */ while(prev<=end) { jgArray[prev++ -jgStart]=(uint8_t)row[1]; } } } /* finish jgArray, pad to multiple of 4 */ while((prev-jgStart)&3) { jgArray[prev++ -jgStart]=0; } indexes[UBIDI_IX_JG_START]=jgStart; indexes[UBIDI_IX_JG_LIMIT]=prev; trieSize=utrie_serialize(pTrie, trieBlock, sizeof(trieBlock), NULL, TRUE, &errorCode); if(U_FAILURE(errorCode)) { fprintf(stderr, "genbidi error: utrie_serialize failed: %s (length %ld)\n", u_errorName(errorCode), (long)trieSize); exit(errorCode); } indexes[UBIDI_IX_TRIE_SIZE]=trieSize; indexes[UBIDI_IX_MIRROR_LENGTH]=mirrorTop; indexes[UBIDI_IX_LENGTH]= (int32_t)sizeof(indexes)+ trieSize+ 4*mirrorTop+ (prev-jgStart); if(beVerbose) { printf("trie size in bytes: %5d\n", (int)trieSize); printf("size in bytes of mirroring table: %5d\n", (int)(4*mirrorTop)); printf("length of Joining_Group array: %5d (U+%04x..U+%04x)\n", (int)(prev-jgStart), (int)jgStart, (int)(prev-1)); printf("data size: %5d\n", (int)indexes[UBIDI_IX_LENGTH]); } indexes[UBIDI_MAX_VALUES_INDEX]= ((int32_t)U_CHAR_DIRECTION_COUNT-1)| (((int32_t)U_JT_COUNT-1)<<UBIDI_JT_SHIFT)| (((int32_t)U_JG_COUNT-1)<<UBIDI_MAX_JG_SHIFT); if(csource) { /* write .c file for hardcoded data */ UTrie trie={ NULL }; UTrie2 *trie2; FILE *f; utrie_unserialize(&trie, trieBlock, trieSize, &errorCode); if(U_FAILURE(errorCode)) { fprintf( stderr, "genbidi error: failed to utrie_unserialize(ubidi.icu trie) - %s\n", u_errorName(errorCode)); exit(errorCode); } /* use UTrie2 */ dataInfo.formatVersion[0]=2; dataInfo.formatVersion[2]=0; dataInfo.formatVersion[3]=0; trie2=utrie2_fromUTrie(&trie, 0, &errorCode); if(U_FAILURE(errorCode)) { fprintf( stderr, "genbidi error: utrie2_fromUTrie() failed - %s\n", u_errorName(errorCode)); exit(errorCode); } { /* delete lead surrogate code unit values */ UChar lead; trie2=utrie2_cloneAsThawed(trie2, &errorCode); for(lead=0xd800; lead<0xdc00; ++lead) { utrie2_set32ForLeadSurrogateCodeUnit(trie2, lead, trie2->initialValue, &errorCode); } utrie2_freeze(trie2, UTRIE2_16_VALUE_BITS, &errorCode); if(U_FAILURE(errorCode)) { fprintf( stderr, "genbidi error: deleting lead surrogate code unit values failed - %s\n", u_errorName(errorCode)); exit(errorCode); } } f=usrc_create(dataDir, "ubidi_props_data.c"); if(f!=NULL) { usrc_writeArray(f, "static const UVersionInfo ubidi_props_dataVersion={", dataInfo.dataVersion, 8, 4, "};\n\n"); usrc_writeArray(f, "static const int32_t ubidi_props_indexes[UBIDI_IX_TOP]={", indexes, 32, UBIDI_IX_TOP, "};\n\n"); usrc_writeUTrie2Arrays(f, "static const uint16_t ubidi_props_trieIndex[%ld]={\n", NULL, trie2, "\n};\n\n"); usrc_writeArray(f, "static const uint32_t ubidi_props_mirrors[%ld]={\n", mirrors, 32, mirrorTop, "\n};\n\n"); usrc_writeArray(f, "static const uint8_t ubidi_props_jgArray[%ld]={\n", jgArray, 8, prev-jgStart, "\n};\n\n"); fputs( "static const UBiDiProps ubidi_props_singleton={\n" " NULL,\n" " ubidi_props_indexes,\n" " ubidi_props_mirrors,\n" " ubidi_props_jgArray,\n", f); usrc_writeUTrie2Struct(f, " {\n", trie2, "ubidi_props_trieIndex", NULL, " },\n"); usrc_writeArray(f, " { ", dataInfo.formatVersion, 8, 4, " }\n"); fputs("};\n", f); fclose(f); } utrie2_close(trie2); } else { /* write the data */ pData=udata_create(dataDir, UBIDI_DATA_TYPE, UBIDI_DATA_NAME, &dataInfo, haveCopyright ? U_COPYRIGHT_STRING : NULL, &errorCode); if(U_FAILURE(errorCode)) { fprintf(stderr, "genbidi: unable to create data memory, %s\n", u_errorName(errorCode)); exit(errorCode); } udata_writeBlock(pData, indexes, sizeof(indexes)); udata_writeBlock(pData, trieBlock, trieSize); udata_writeBlock(pData, mirrors, 4*mirrorTop); udata_writeBlock(pData, jgArray, prev-jgStart); /* finish up */ dataLength=udata_finish(pData, &errorCode); if(U_FAILURE(errorCode)) { fprintf(stderr, "genbidi: error %d writing the output file\n", errorCode); exit(errorCode); } if(dataLength!=indexes[UBIDI_IX_LENGTH]) { fprintf(stderr, "genbidi: data length %ld != calculated size %d\n", dataLength, (int)indexes[UBIDI_IX_LENGTH]); exit(U_INTERNAL_PROGRAM_ERROR); } } utrie_close(pTrie); upvec_close(pv); }
extern void generateData(const char *dataDir, UBool csource) { static int32_t indexes[UPROPS_INDEX_COUNT]={ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; static uint8_t trieBlock[40000]; static uint8_t additionalProps[120000]; UNewDataMemory *pData; UErrorCode errorCode=U_ZERO_ERROR; uint32_t size = 0; int32_t trieSize, additionalPropsSize, offset; long dataLength; trieSize=utrie_serialize(pTrie, trieBlock, sizeof(trieBlock), NULL, TRUE, &errorCode); if(U_FAILURE(errorCode)) { fprintf(stderr, "error: utrie_serialize failed: %s (length %ld)\n", u_errorName(errorCode), (long)trieSize); exit(errorCode); } offset=sizeof(indexes)/4; /* uint32_t offset to the properties trie */ /* round up trie size to 4-alignment */ trieSize=(trieSize+3)&~3; offset+=trieSize>>2; indexes[UPROPS_PROPS32_INDEX]= /* set indexes to the same offsets for empty */ indexes[UPROPS_EXCEPTIONS_INDEX]= /* structures from the old format version 3 */ indexes[UPROPS_EXCEPTIONS_TOP_INDEX]= /* so that less runtime code has to be changed */ indexes[UPROPS_ADDITIONAL_TRIE_INDEX]=offset; if(beVerbose) { printf("trie size in bytes: %5u\n", (int)trieSize); } if(csource) { /* write .c file for hardcoded data */ UTrie trie={ NULL }; UTrie2 *trie2; FILE *f; utrie_unserialize(&trie, trieBlock, trieSize, &errorCode); if(U_FAILURE(errorCode)) { fprintf( stderr, "genprops error: failed to utrie_unserialize(uprops.icu main trie) - %s\n", u_errorName(errorCode)); exit(errorCode); } /* use UTrie2 */ trie2=utrie2_fromUTrie(&trie, 0, &errorCode); if(U_FAILURE(errorCode)) { fprintf( stderr, "genprops error: utrie2_fromUTrie() failed - %s\n", u_errorName(errorCode)); exit(errorCode); } { /* delete lead surrogate code unit values */ UChar lead; trie2=utrie2_cloneAsThawed(trie2, &errorCode); for(lead=0xd800; lead<0xdc00; ++lead) { utrie2_set32ForLeadSurrogateCodeUnit(trie2, lead, trie2->initialValue, &errorCode); } utrie2_freeze(trie2, UTRIE2_16_VALUE_BITS, &errorCode); if(U_FAILURE(errorCode)) { fprintf( stderr, "genprops error: deleting lead surrogate code unit values failed - %s\n", u_errorName(errorCode)); exit(errorCode); } } f=usrc_create(dataDir, "uchar_props_data.c"); if(f!=NULL) { /* unused usrc_writeArray(f, "static const UVersionInfo formatVersion={", dataInfo.formatVersion, 8, 4, "};\n\n"); */ usrc_writeArray(f, "static const UVersionInfo dataVersion={", dataInfo.dataVersion, 8, 4, "};\n\n"); usrc_writeUTrie2Arrays(f, "static const uint16_t propsTrie_index[%ld]={\n", NULL, trie2, "\n};\n\n"); usrc_writeUTrie2Struct(f, "static const UTrie2 propsTrie={\n", trie2, "propsTrie_index", NULL, "};\n\n"); additionalPropsSize=writeAdditionalData(f, additionalProps, sizeof(additionalProps), indexes); size=4*offset+additionalPropsSize; /* total size of data */ usrc_writeArray(f, "static const int32_t indexes[UPROPS_INDEX_COUNT]={", indexes, 32, UPROPS_INDEX_COUNT, "};\n\n"); fclose(f); } utrie2_close(trie2); } else { /* write the data */ pData=udata_create(dataDir, DATA_TYPE, DATA_NAME, &dataInfo, haveCopyright ? U_COPYRIGHT_STRING : NULL, &errorCode); if(U_FAILURE(errorCode)) { fprintf(stderr, "genprops: unable to create data memory, %s\n", u_errorName(errorCode)); exit(errorCode); } additionalPropsSize=writeAdditionalData(NULL, additionalProps, sizeof(additionalProps), indexes); size=4*offset+additionalPropsSize; /* total size of data */ udata_writeBlock(pData, indexes, sizeof(indexes)); udata_writeBlock(pData, trieBlock, trieSize); udata_writeBlock(pData, additionalProps, additionalPropsSize); /* finish up */ dataLength=udata_finish(pData, &errorCode); if(U_FAILURE(errorCode)) { fprintf(stderr, "genprops: error %d writing the output file\n", errorCode); exit(errorCode); } if(dataLength!=(long)size) { fprintf(stderr, "genprops: data length %ld != calculated size %lu\n", dataLength, (unsigned long)size); exit(U_INTERNAL_PROGRAM_ERROR); } } if(beVerbose) { printf("data size: %6lu\n", (unsigned long)size); } }
U_CFUNC int32_t writeAdditionalData(FILE *f, uint8_t *p, int32_t capacity, int32_t indexes[UPROPS_INDEX_COUNT]) { const uint32_t *pvArray; int32_t pvRows, pvCount; int32_t length; UErrorCode errorCode; pvArray=upvec_getArray(pv, &pvRows, NULL); pvCount=pvRows*UPROPS_VECTOR_WORDS; errorCode=U_ZERO_ERROR; length=utrie_serialize(newTrie, p, capacity, NULL, TRUE, &errorCode); if(U_FAILURE(errorCode)) { fprintf(stderr, "genprops error: unable to serialize trie for additional properties: %s\n", u_errorName(errorCode)); exit(errorCode); } if(p!=NULL) { if(beVerbose) { printf("size in bytes of additional props trie:%5u\n", (int)length); } if(f!=NULL) { UTrie trie={ NULL }; UTrie2 *trie2; utrie_unserialize(&trie, p, length, &errorCode); if(U_FAILURE(errorCode)) { fprintf( stderr, "genprops error: failed to utrie_unserialize(trie for additional properties) - %s\n", u_errorName(errorCode)); exit(errorCode); } /* use UTrie2 */ trie2=utrie2_fromUTrie(&trie, trie.initialValue, &errorCode); if(U_FAILURE(errorCode)) { fprintf( stderr, "genprops error: utrie2_fromUTrie() failed - %s\n", u_errorName(errorCode)); exit(errorCode); } { /* delete lead surrogate code unit values */ UChar lead; trie2=utrie2_cloneAsThawed(trie2, &errorCode); for(lead=0xd800; lead<0xdc00; ++lead) { utrie2_set32ForLeadSurrogateCodeUnit(trie2, lead, trie2->initialValue, &errorCode); } utrie2_freeze(trie2, UTRIE2_16_VALUE_BITS, &errorCode); if(U_FAILURE(errorCode)) { fprintf( stderr, "genbidi error: deleting lead surrogate code unit values failed - %s\n", u_errorName(errorCode)); exit(errorCode); } } usrc_writeUTrie2Arrays(f, "static const uint16_t propsVectorsTrie_index[%ld]={\n", NULL, trie2, "\n};\n\n"); usrc_writeUTrie2Struct(f, "static const UTrie2 propsVectorsTrie={\n", trie2, "propsVectorsTrie_index", NULL, "};\n\n"); utrie2_close(trie2); } p+=length; capacity-=length; /* set indexes */ indexes[UPROPS_ADDITIONAL_VECTORS_INDEX]= indexes[UPROPS_ADDITIONAL_TRIE_INDEX]+length/4; indexes[UPROPS_ADDITIONAL_VECTORS_COLUMNS_INDEX]=UPROPS_VECTOR_WORDS; indexes[UPROPS_RESERVED_INDEX]= indexes[UPROPS_ADDITIONAL_VECTORS_INDEX]+pvCount; indexes[UPROPS_MAX_VALUES_INDEX]= (((int32_t)U_EA_COUNT-1)<<UPROPS_EA_SHIFT)| (((int32_t)UBLOCK_COUNT-1)<<UPROPS_BLOCK_SHIFT)| (((int32_t)USCRIPT_CODE_LIMIT-1)&UPROPS_SCRIPT_MASK); indexes[UPROPS_MAX_VALUES_2_INDEX]= (((int32_t)U_LB_COUNT-1)<<UPROPS_LB_SHIFT)| (((int32_t)U_SB_COUNT-1)<<UPROPS_SB_SHIFT)| (((int32_t)U_WB_COUNT-1)<<UPROPS_WB_SHIFT)| (((int32_t)U_GCB_COUNT-1)<<UPROPS_GCB_SHIFT)| ((int32_t)U_DT_COUNT-1); } if(p!=NULL && (pvCount*4)<=capacity) { if(f!=NULL) { usrc_writeArray(f, "static const uint32_t propsVectors[%ld]={\n", pvArray, 32, pvCount, "};\n\n"); fprintf(f, "static const int32_t countPropsVectors=%ld;\n", (long)pvCount); fprintf(f, "static const int32_t propsVectorsColumns=%ld;\n", (long)indexes[UPROPS_ADDITIONAL_VECTORS_COLUMNS_INDEX]); } else { uprv_memcpy(p, pvArray, pvCount*4); } if(beVerbose) { printf("number of additional props vectors: %5u\n", (int)pvRows); printf("number of 32-bit words per vector: %5u\n", UPROPS_VECTOR_WORDS); } } length+=pvCount*4; return length; }