static uint32_t
getToUnicodeValue(CnvExtData *extData, UCMTable *table, UCMapping *m) {
UChar32 *u32;
UChar *u;
uint32_t value;
int32_t u16Length, ratio;
UErrorCode errorCode;
/* write the Unicode result code point or string index */
if(m->uLen==1) {
u16Length=U16_LENGTH(m->u);
value=(uint32_t)(UCNV_EXT_TO_U_MIN_CODE_POINT+m->u);
} else {
/* the parser enforces m->uLen<=UCNV_EXT_MAX_UCHARS */
/* get the result code point string and its 16-bit string length */
u32=UCM_GET_CODE_POINTS(table, m);
errorCode=U_ZERO_ERROR;
u_strFromUTF32(NULL, 0, &u16Length, u32, m->uLen, &errorCode);
if(U_FAILURE(errorCode) && errorCode!=U_BUFFER_OVERFLOW_ERROR) {
exit(errorCode);
}
/* allocate it and put its length and index into the value */
value=
(((uint32_t)m->uLen+UCNV_EXT_TO_U_LENGTH_OFFSET)<<UCNV_EXT_TO_U_LENGTH_SHIFT)|
((uint32_t)utm_countItems(extData->toUUChars));
u=utm_allocN(extData->toUUChars, u16Length);
/* write the result 16-bit string */
errorCode=U_ZERO_ERROR;
u_strFromUTF32(u, u16Length, NULL, u32, m->uLen, &errorCode);
if(U_FAILURE(errorCode) && errorCode!=U_BUFFER_OVERFLOW_ERROR) {
exit(errorCode);
}
}
if(m->f==0) {
value|=UCNV_EXT_TO_U_ROUNDTRIP_FLAG;
}
/* update statistics */
if(m->bLen>extData->maxInBytes) {
extData->maxInBytes=m->bLen;
}
if(u16Length>extData->maxOutUChars) {
extData->maxOutUChars=u16Length;
}
ratio=(u16Length+(m->bLen-1))/m->bLen;
if(ratio>extData->maxUCharsPerByte) {
extData->maxUCharsPerByte=ratio;
}
return value;
}
Normalizer2DataBuilder::~Normalizer2DataBuilder() {
utrie2_close(normTrie);
int32_t normsLength=utm_countItems(normMem);
for(int32_t i=1; i<normsLength; ++i) {
delete norms[i].mapping;
delete norms[i].rawMapping;
delete norms[i].compositions;
}
utm_close(normMem);
utrie2_close(norm16Trie);
}
/*
* works like generateToUTable(), except that the
* output section consists of two arrays, one for input UChars and one
* for result values
*
* also, fromUTable sections are always stored in a compact form for
* access via binary search
*/
static UBool
generateFromUTable(CnvExtData *extData, UCMTable *table,
int32_t start, int32_t limit, int32_t unitIndex,
uint32_t defaultValue) {
UCMapping *mappings, *m;
int32_t *map;
int32_t i, j, uniqueCount, count, subStart, subLimit;
UChar *uchars;
UChar32 low, high, prev;
UChar *sectionUChars;
uint32_t *sectionValues;
mappings=table->mappings;
map=table->reverseMap;
/* step 1: examine the input units; set low, high, uniqueCount */
m=mappings+map[start];
uchars=(UChar *)UCM_GET_CODE_POINTS(table, m);
low=uchars[unitIndex];
uniqueCount=1;
prev=high=low;
for(i=start+1; i<limit; ++i) {
m=mappings+map[i];
uchars=(UChar *)UCM_GET_CODE_POINTS(table, m);
high=uchars[unitIndex];
if(high!=prev) {
prev=high;
++uniqueCount;
}
}
/* step 2: allocate the section; set count, section */
/* the fromUTable always stores for access via binary search */
count=uniqueCount;
/* allocate the section: 1 entry for the header + count for the items */
sectionUChars=(UChar *)utm_allocN(extData->fromUTableUChars, 1+count);
sectionValues=(uint32_t *)utm_allocN(extData->fromUTableValues, 1+count);
/* write the section header */
*sectionUChars++=(UChar)count;
*sectionValues++=defaultValue;
/* step 3: write temporary section table with subsection starts */
prev=low-1; /* just before low to prevent empty subsections before low */
j=0; /* section table index */
for(i=start; i<limit; ++i) {
m=mappings+map[i];
uchars=(UChar *)UCM_GET_CODE_POINTS(table, m);
high=uchars[unitIndex];
if(high!=prev) {
/* start of a new subsection for unit high */
prev=high;
/* write the entry with the subsection start */
sectionUChars[j]=(UChar)high;
sectionValues[j]=(uint32_t)i;
++j;
}
}
/* assert(j==count) */
/* step 4: recurse and write results */
subLimit=(int32_t)(sectionValues[0]);
for(j=0; j<count; ++j) {
subStart=subLimit;
subLimit= (j+1)<count ? (int32_t)(sectionValues[j+1]) : limit;
/* see if there is exactly one input unit sequence of length unitIndex+1 */
defaultValue=0;
m=mappings+map[subStart];
if(m->uLen==unitIndex+1) {
/* do not include this in generateToUTable() */
++subStart;
if(subStart<subLimit && mappings[map[subStart]].uLen==unitIndex+1) {
/* print error for multiple same-input-sequence mappings */
fprintf(stderr, "error: multiple mappings from same Unicode code points\n");
ucm_printMapping(table, m, stderr);
ucm_printMapping(table, mappings+map[subStart], stderr);
return FALSE;
}
defaultValue=getFromUBytesValue(extData, table, m);
}
if(subStart==subLimit) {
/* write the result for the input sequence ending here */
sectionValues[j]=defaultValue;
} else {
/* write the index to the subsection table */
sectionValues[j]=(uint32_t)utm_countItems(extData->fromUTableValues);
/* recurse */
if(!generateFromUTable(extData, table, subStart, subLimit, unitIndex+1, defaultValue)) {
return FALSE;
}
}
}
return TRUE;
}
static uint32_t
getFromUBytesValue(CnvExtData *extData, UCMTable *table, UCMapping *m) {
uint8_t *bytes, *resultBytes;
uint32_t value;
int32_t u16Length, ratio;
if(m->f==2) {
/*
* no mapping, <subchar1> preferred
*
* no need to count in statistics because the subchars are already
* counted for maxOutBytes and maxBytesPerUChar in UConverterStaticData,
* and this non-mapping does not count for maxInUChars which are always
* trivially at least two if counting unmappable supplementary code points
*/
return UCNV_EXT_FROM_U_SUBCHAR1;
}
bytes=UCM_GET_BYTES(table, m);
value=0;
switch(m->bLen) {
/* 1..3: store the bytes in the value word */
case 3:
value=((uint32_t)*bytes++)<<16;
case 2:
value|=((uint32_t)*bytes++)<<8;
case 1:
value|=*bytes;
break;
default:
/* the parser enforces m->bLen<=UCNV_EXT_MAX_BYTES */
/* store the bytes in fromUBytes[] and the index in the value word */
value=(uint32_t)utm_countItems(extData->fromUBytes);
resultBytes=utm_allocN(extData->fromUBytes, m->bLen);
uprv_memcpy(resultBytes, bytes, m->bLen);
break;
}
value|=(uint32_t)m->bLen<<UCNV_EXT_FROM_U_LENGTH_SHIFT;
if(m->f==0) {
value|=UCNV_EXT_FROM_U_ROUNDTRIP_FLAG;
}
/* calculate the real UTF-16 length (see recoding in prepareFromUMappings()) */
if(m->uLen==1) {
u16Length=U16_LENGTH(m->u);
} else {
u16Length=U16_LENGTH(UCM_GET_CODE_POINTS(table, m)[0])+(m->uLen-2);
}
/* update statistics */
if(u16Length>extData->maxInUChars) {
extData->maxInUChars=u16Length;
}
if(m->bLen>extData->maxOutBytes) {
extData->maxOutBytes=m->bLen;
}
ratio=(m->bLen+(u16Length-1))/u16Length;
if(ratio>extData->maxBytesPerUChar) {
extData->maxBytesPerUChar=ratio;
}
return value;
}
/*
* Recursive toUTable generator core function.
* Preconditions:
* - start<limit (There is at least one mapping.)
* - The mappings are sorted lexically. (Access is through the reverseMap.)
* - All mappings between start and limit have input sequences that share
* the same prefix of unitIndex length, and therefore all of these sequences
* are at least unitIndex+1 long.
* - There are only relevant mappings available through the reverseMap,
* see reduceToUMappings().
*
* One function invocation generates one section table.
*
* Steps:
* 1. Count the number of unique unit values and get the low/high unit values
* that occur at unitIndex.
* 2. Allocate the section table with possible optimization for linear access.
* 3. Write temporary version of the section table with start indexes of
* subsections, each corresponding to one unit value at unitIndex.
* 4. Iterate through the table once more, and depending on the subsection length:
* 0: write 0 as a result value (unused byte in linear-access section table)
* >0: if there is one mapping with an input unit sequence of unitIndex+1
* then defaultValue=compute the mapping result for this whole sequence
* else defaultValue=0
*
* recurse into the subsection
*/
static UBool
generateToUTable(CnvExtData *extData, UCMTable *table,
int32_t start, int32_t limit, int32_t unitIndex,
uint32_t defaultValue) {
UCMapping *mappings, *m;
int32_t *map;
int32_t i, j, uniqueCount, count, subStart, subLimit;
uint8_t *bytes;
int32_t low, high, prev;
uint32_t *section;
mappings=table->mappings;
map=table->reverseMap;
/* step 1: examine the input units; set low, high, uniqueCount */
m=mappings+map[start];
bytes=UCM_GET_BYTES(table, m);
low=bytes[unitIndex];
uniqueCount=1;
prev=high=low;
for(i=start+1; i<limit; ++i) {
m=mappings+map[i];
bytes=UCM_GET_BYTES(table, m);
high=bytes[unitIndex];
if(high!=prev) {
prev=high;
++uniqueCount;
}
}
/* step 2: allocate the section; set count, section */
count=(high-low)+1;
if(unitIndex==0 || uniqueCount>=(3*count)/4) {
/*
* for the root table and for fairly full tables:
* allocate for direct, linear array access
* by keeping count, to write an entry for each unit value
* from low to high
*/
} else {
count=uniqueCount;
}
/* allocate the section: 1 entry for the header + count for the items */
section=(uint32_t *)utm_allocN(extData->toUTable, 1+count);
/* write the section header */
*section++=((uint32_t)count<<UCNV_EXT_TO_U_BYTE_SHIFT)|defaultValue;
/* step 3: write temporary section table with subsection starts */
prev=low-1; /* just before low to prevent empty subsections before low */
j=0; /* section table index */
for(i=start; i<limit; ++i) {
m=mappings+map[i];
bytes=UCM_GET_BYTES(table, m);
high=bytes[unitIndex];
if(high!=prev) {
/* start of a new subsection for unit high */
if(count>uniqueCount) {
/* write empty subsections for unused units in a linear table */
while(++prev<high) {
section[j++]=((uint32_t)prev<<UCNV_EXT_TO_U_BYTE_SHIFT)|(uint32_t)i;
}
} else {
prev=high;
}
/* write the entry with the subsection start */
section[j++]=((uint32_t)high<<UCNV_EXT_TO_U_BYTE_SHIFT)|(uint32_t)i;
}
}
/* assert(j==count) */
/* step 4: recurse and write results */
subLimit=UCNV_EXT_TO_U_GET_VALUE(section[0]);
for(j=0; j<count; ++j) {
subStart=subLimit;
subLimit= (j+1)<count ? UCNV_EXT_TO_U_GET_VALUE(section[j+1]) : limit;
/* remove the subStart temporary value */
section[j]&=~UCNV_EXT_TO_U_VALUE_MASK;
if(subStart==subLimit) {
/* leave the value zero: empty subsection for unused unit in a linear table */
continue;
}
/* see if there is exactly one input unit sequence of length unitIndex+1 */
defaultValue=0;
m=mappings+map[subStart];
if(m->bLen==unitIndex+1) {
/* do not include this in generateToUTable() */
++subStart;
if(subStart<subLimit && mappings[map[subStart]].bLen==unitIndex+1) {
/* print error for multiple same-input-sequence mappings */
fprintf(stderr, "error: multiple mappings from same bytes\n");
ucm_printMapping(table, m, stderr);
ucm_printMapping(table, mappings+map[subStart], stderr);
return FALSE;
}
defaultValue=getToUnicodeValue(extData, table, m);
}
if(subStart==subLimit) {
/* write the result for the input sequence ending here */
section[j]|=defaultValue;
} else {
/* write the index to the subsection table */
section[j]|=(uint32_t)utm_countItems(extData->toUTable);
/* recurse */
if(!generateToUTable(extData, table, subStart, subLimit, unitIndex+1, defaultValue)) {
return FALSE;
}
}
}
return TRUE;
}
static uint32_t
CnvExtWrite(NewConverter *cnvData, const UConverterStaticData *staticData,
UNewDataMemory *pData, int32_t tableType) {
CnvExtData *extData=(CnvExtData *)cnvData;
int32_t length, top, headerSize;
int32_t indexes[UCNV_EXT_INDEXES_MIN_LENGTH]={ 0 };
if(tableType&TABLE_BASE) {
headerSize=0;
} else {
_MBCSHeader header={ { 0, 0, 0, 0 }, 0, 0, 0, 0, 0, 0, 0 };
/* write the header and base table name for an extension-only table */
length=(int32_t)uprv_strlen(extData->ucm->baseName)+1;
while(length&3) {
/* add padding */
extData->ucm->baseName[length++]=0;
}
headerSize=sizeof(header)+length;
/* fill the header */
header.version[0]=4;
header.version[1]=2;
header.flags=(uint32_t)((headerSize<<8)|MBCS_OUTPUT_EXT_ONLY);
/* write the header and the base table name */
udata_writeBlock(pData, &header, sizeof(header));
udata_writeBlock(pData, extData->ucm->baseName, length);
}
/* fill indexes[] - offsets/indexes are in units of the target array */
top=0;
indexes[UCNV_EXT_INDEXES_LENGTH]=length=UCNV_EXT_INDEXES_MIN_LENGTH;
top+=length*4;
indexes[UCNV_EXT_TO_U_INDEX]=top;
indexes[UCNV_EXT_TO_U_LENGTH]=length=utm_countItems(extData->toUTable);
top+=length*4;
indexes[UCNV_EXT_TO_U_UCHARS_INDEX]=top;
indexes[UCNV_EXT_TO_U_UCHARS_LENGTH]=length=utm_countItems(extData->toUUChars);
top+=length*2;
indexes[UCNV_EXT_FROM_U_UCHARS_INDEX]=top;
length=utm_countItems(extData->fromUTableUChars);
top+=length*2;
if(top&3) {
/* add padding */
*((UChar *)utm_alloc(extData->fromUTableUChars))=0;
*((uint32_t *)utm_alloc(extData->fromUTableValues))=0;
++length;
top+=2;
}
indexes[UCNV_EXT_FROM_U_LENGTH]=length;
indexes[UCNV_EXT_FROM_U_VALUES_INDEX]=top;
top+=length*4;
indexes[UCNV_EXT_FROM_U_BYTES_INDEX]=top;
length=utm_countItems(extData->fromUBytes);
top+=length;
if(top&1) {
/* add padding */
*((uint8_t *)utm_alloc(extData->fromUBytes))=0;
++length;
++top;
}
indexes[UCNV_EXT_FROM_U_BYTES_LENGTH]=length;
indexes[UCNV_EXT_FROM_U_STAGE_12_INDEX]=top;
indexes[UCNV_EXT_FROM_U_STAGE_1_LENGTH]=length=extData->stage1Top;
indexes[UCNV_EXT_FROM_U_STAGE_12_LENGTH]=length+=extData->stage2Top;
top+=length*2;
indexes[UCNV_EXT_FROM_U_STAGE_3_INDEX]=top;
length=extData->stage3Top;
top+=length*2;
if(top&3) {
/* add padding */
extData->stage3[extData->stage3Top++]=0;
++length;
top+=2;
}
indexes[UCNV_EXT_FROM_U_STAGE_3_LENGTH]=length;
indexes[UCNV_EXT_FROM_U_STAGE_3B_INDEX]=top;
indexes[UCNV_EXT_FROM_U_STAGE_3B_LENGTH]=length=extData->stage3bTop;
top+=length*4;
indexes[UCNV_EXT_SIZE]=top;
/* statistics */
indexes[UCNV_EXT_COUNT_BYTES]=
(extData->maxInBytes<<16)|
(extData->maxOutBytes<<8)|
extData->maxBytesPerUChar;
indexes[UCNV_EXT_COUNT_UCHARS]=
(extData->maxInUChars<<16)|
(extData->maxOutUChars<<8)|
extData->maxUCharsPerByte;
indexes[UCNV_EXT_FLAGS]=extData->ucm->ext->unicodeMask;
/* write the extension data */
udata_writeBlock(pData, indexes, sizeof(indexes));
udata_writeBlock(pData, utm_getStart(extData->toUTable), indexes[UCNV_EXT_TO_U_LENGTH]*4);
udata_writeBlock(pData, utm_getStart(extData->toUUChars), indexes[UCNV_EXT_TO_U_UCHARS_LENGTH]*2);
udata_writeBlock(pData, utm_getStart(extData->fromUTableUChars), indexes[UCNV_EXT_FROM_U_LENGTH]*2);
udata_writeBlock(pData, utm_getStart(extData->fromUTableValues), indexes[UCNV_EXT_FROM_U_LENGTH]*4);
udata_writeBlock(pData, utm_getStart(extData->fromUBytes), indexes[UCNV_EXT_FROM_U_BYTES_LENGTH]);
udata_writeBlock(pData, extData->stage1, extData->stage1Top*2);
udata_writeBlock(pData, extData->stage2, extData->stage2Top*2);
udata_writeBlock(pData, extData->stage3, extData->stage3Top*2);
udata_writeBlock(pData, extData->stage3b, extData->stage3bTop*4);
#if 0
{
int32_t i, j;
length=extData->stage1Top;
printf("\nstage1[%x]:\n", length);
for(i=0; i<length; ++i) {
if(extData->stage1[i]!=length) {
printf("stage1[%04x]=%04x\n", i, extData->stage1[i]);
}
}
j=length;
length=extData->stage2Top;
printf("\nstage2[%x]:\n", length);
for(i=0; i<length; ++j, ++i) {
if(extData->stage2[i]!=0) {
printf("stage12[%04x]=%04x\n", j, extData->stage2[i]);
}
}
length=extData->stage3Top;
printf("\nstage3[%x]:\n", length);
for(i=0; i<length; ++i) {
if(extData->stage3[i]!=0) {
printf("stage3[%04x]=%04x\n", i, extData->stage3[i]);
}
}
length=extData->stage3bTop;
printf("\nstage3b[%x]:\n", length);
for(i=0; i<length; ++i) {
if(extData->stage3b[i]!=0) {
printf("stage3b[%04x]=%08x\n", i, extData->stage3b[i]);
}
}
}
#endif
if(VERBOSE) {
printf("size of extension data: %ld\n", (long)top);
}
/* return the number of bytes that should have been written */
return (uint32_t)(headerSize+top);
}