void
PNamesBuilderImpl::writeCSourceFile(const char *path, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
FILE *f=usrc_create(path, "propname_data.h",
"icu/tools/unicode/c/genprops/pnamesbuilder.cpp");
if(f==NULL) {
errorCode=U_FILE_ACCESS_ERROR;
return;
}
fputs("#ifndef INCLUDED_FROM_PROPNAME_CPP\n"
"# error This file must be #included from propname.cpp only.\n"
"#endif\n\n", f);
fputs("U_NAMESPACE_BEGIN\n\n", f);
usrc_writeArray(f, "const int32_t PropNameData::indexes[%ld]={",
indexes, 32, PropNameData::IX_COUNT,
"};\n\n");
usrc_writeArray(f, "const int32_t PropNameData::valueMaps[%ld]={\n",
valueMaps.getBuffer(), 32, valueMaps.size(),
"\n};\n\n");
usrc_writeArray(f, "const uint8_t PropNameData::bytesTries[%ld]={\n",
bytesTries.data(), 8, bytesTries.length(),
"\n};\n\n");
usrc_writeArrayOfMostlyInvChars(
f, "const char PropNameData::nameGroups[%ld]={\n",
nameGroups.data(), nameGroups.length(),
"\n};\n\n");
fputs("U_NAMESPACE_END\n", f);
fclose(f);
}
U_CAPI void U_EXPORT2
usrc_writeUTrie2Arrays(FILE *f,
const char *indexPrefix, const char *data32Prefix,
const UTrie2 *pTrie,
const char *postfix) {
if(pTrie->data32==NULL) {
/* 16-bit trie */
usrc_writeArray(f, indexPrefix, pTrie->index, 16, pTrie->indexLength+pTrie->dataLength, postfix);
} else {
/* 32-bit trie */
usrc_writeArray(f, indexPrefix, pTrie->index, 16, pTrie->indexLength, postfix);
usrc_writeArray(f, data32Prefix, pTrie->data32, 32, pTrie->dataLength, postfix);
}
}
void
BiDiPropsBuilder::writeCSourceFile(const char *path, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
FILE *f=usrc_create(path, "ubidi_props_data.h",
"icu/tools/unicode/c/genprops/bidipropsbuilder.cpp");
if(f==NULL) {
errorCode=U_FILE_ACCESS_ERROR;
return;
}
fputs("#ifndef INCLUDED_FROM_UBIDI_PROPS_C\n"
"# error This file must be #included from ubidi_props.c only.\n"
"#endif\n\n", f);
usrc_writeArray(f,
"static const UVersionInfo ubidi_props_dataVersion={",
dataInfo.dataVersion, 8, 4,
"};\n\n");
usrc_writeArray(f,
"static const int32_t ubidi_props_indexes[UBIDI_IX_TOP]={",
indexes, 32, UBIDI_IX_TOP,
"};\n\n");
usrc_writeUTrie2Arrays(f,
"static const uint16_t ubidi_props_trieIndex[%ld]={\n", NULL,
pTrie,
"\n};\n\n");
usrc_writeArray(f,
"static const uint32_t ubidi_props_mirrors[%ld]={\n",
mirrors, 32, mirrorTop,
"\n};\n\n");
UChar32 jgStart=indexes[UBIDI_IX_JG_START];
UChar32 jgLimit=indexes[UBIDI_IX_JG_LIMIT];
usrc_writeArray(f,
"static const uint8_t ubidi_props_jgArray[%ld]={\n",
jgArray+(jgStart-MIN_JG_START), 8, jgLimit-jgStart,
"\n};\n\n");
UChar32 jgStart2=indexes[UBIDI_IX_JG_START2];
UChar32 jgLimit2=indexes[UBIDI_IX_JG_LIMIT2];
usrc_writeArray(f,
"static const uint8_t ubidi_props_jgArray2[%ld]={\n",
jgArray2+(jgStart2-MIN_JG_START2), 8, jgLimit2-jgStart2,
"\n};\n\n");
fputs(
"static const UBiDiProps ubidi_props_singleton={\n"
" NULL,\n"
" ubidi_props_indexes,\n"
" ubidi_props_mirrors,\n"
" ubidi_props_jgArray,\n"
" ubidi_props_jgArray2,\n",
f);
usrc_writeUTrie2Struct(f,
" {\n",
pTrie, "ubidi_props_trieIndex", NULL,
" },\n");
usrc_writeArray(f, " { ", dataInfo.formatVersion, 8, 4, " }\n");
fputs("};\n", f);
fclose(f);
}
extern void
generateData(const char *dataDir, UBool csource) {
static int32_t indexes[UBIDI_IX_TOP]={
UBIDI_IX_TOP
};
static uint8_t trieBlock[40000];
static uint8_t jgArray[0x300]; /* at most for U+0600..U+08FF */
const uint32_t *row;
UChar32 start, end, prev, jgStart;
int32_t i;
UNewDataMemory *pData;
UNewTrie *pTrie;
UErrorCode errorCode=U_ZERO_ERROR;
int32_t trieSize;
long dataLength;
makeMirror();
pTrie=utrie_open(NULL, NULL, 20000, 0, 0, TRUE);
if(pTrie==NULL) {
fprintf(stderr, "genbidi error: unable to create a UNewTrie\n");
exit(U_MEMORY_ALLOCATION_ERROR);
}
prev=jgStart=0;
for(i=0; (row=upvec_getRow(pv, i, &start, &end))!=NULL && start<UPVEC_FIRST_SPECIAL_CP; ++i) {
/* store most values from vector column 0 in the trie */
if(!utrie_setRange32(pTrie, start, end+1, *row, TRUE)) {
fprintf(stderr, "genbidi error: unable to set trie value (overflow)\n");
exit(U_BUFFER_OVERFLOW_ERROR);
}
/* store Joining_Group values from vector column 1 in a simple byte array */
if(row[1]!=0) {
if(start<0x600 || 0x8ff<end) {
fprintf(stderr, "genbidi error: Joining_Group for out-of-range code points U+%04lx..U+%04lx\n",
(long)start, (long)end);
exit(U_ILLEGAL_ARGUMENT_ERROR);
}
if(prev==0) {
/* first code point with any value */
prev=jgStart=start;
} else {
/* add No_Joining_Group for code points between prev and start */
while(prev<start) {
jgArray[prev++ -jgStart]=0;
}
}
/* set Joining_Group value for start..end */
while(prev<=end) {
jgArray[prev++ -jgStart]=(uint8_t)row[1];
}
}
}
/* finish jgArray, pad to multiple of 4 */
while((prev-jgStart)&3) {
jgArray[prev++ -jgStart]=0;
}
indexes[UBIDI_IX_JG_START]=jgStart;
indexes[UBIDI_IX_JG_LIMIT]=prev;
trieSize=utrie_serialize(pTrie, trieBlock, sizeof(trieBlock), NULL, TRUE, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "genbidi error: utrie_serialize failed: %s (length %ld)\n", u_errorName(errorCode), (long)trieSize);
exit(errorCode);
}
indexes[UBIDI_IX_TRIE_SIZE]=trieSize;
indexes[UBIDI_IX_MIRROR_LENGTH]=mirrorTop;
indexes[UBIDI_IX_LENGTH]=
(int32_t)sizeof(indexes)+
trieSize+
4*mirrorTop+
(prev-jgStart);
if(beVerbose) {
printf("trie size in bytes: %5d\n", (int)trieSize);
printf("size in bytes of mirroring table: %5d\n", (int)(4*mirrorTop));
printf("length of Joining_Group array: %5d (U+%04x..U+%04x)\n", (int)(prev-jgStart), (int)jgStart, (int)(prev-1));
printf("data size: %5d\n", (int)indexes[UBIDI_IX_LENGTH]);
}
indexes[UBIDI_MAX_VALUES_INDEX]=
((int32_t)U_CHAR_DIRECTION_COUNT-1)|
(((int32_t)U_JT_COUNT-1)<<UBIDI_JT_SHIFT)|
(((int32_t)U_JG_COUNT-1)<<UBIDI_MAX_JG_SHIFT);
if(csource) {
/* write .c file for hardcoded data */
UTrie trie={ NULL };
UTrie2 *trie2;
FILE *f;
utrie_unserialize(&trie, trieBlock, trieSize, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(
stderr,
"genbidi error: failed to utrie_unserialize(ubidi.icu trie) - %s\n",
u_errorName(errorCode));
exit(errorCode);
}
/* use UTrie2 */
dataInfo.formatVersion[0]=2;
dataInfo.formatVersion[2]=0;
dataInfo.formatVersion[3]=0;
trie2=utrie2_fromUTrie(&trie, 0, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(
stderr,
"genbidi error: utrie2_fromUTrie() failed - %s\n",
u_errorName(errorCode));
exit(errorCode);
}
{
/* delete lead surrogate code unit values */
UChar lead;
trie2=utrie2_cloneAsThawed(trie2, &errorCode);
for(lead=0xd800; lead<0xdc00; ++lead) {
utrie2_set32ForLeadSurrogateCodeUnit(trie2, lead, trie2->initialValue, &errorCode);
}
utrie2_freeze(trie2, UTRIE2_16_VALUE_BITS, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(
stderr,
"genbidi error: deleting lead surrogate code unit values failed - %s\n",
u_errorName(errorCode));
exit(errorCode);
}
}
f=usrc_create(dataDir, "ubidi_props_data.c");
if(f!=NULL) {
usrc_writeArray(f,
"static const UVersionInfo ubidi_props_dataVersion={",
dataInfo.dataVersion, 8, 4,
"};\n\n");
usrc_writeArray(f,
"static const int32_t ubidi_props_indexes[UBIDI_IX_TOP]={",
indexes, 32, UBIDI_IX_TOP,
"};\n\n");
usrc_writeUTrie2Arrays(f,
"static const uint16_t ubidi_props_trieIndex[%ld]={\n", NULL,
trie2,
"\n};\n\n");
usrc_writeArray(f,
"static const uint32_t ubidi_props_mirrors[%ld]={\n",
mirrors, 32, mirrorTop,
"\n};\n\n");
usrc_writeArray(f,
"static const uint8_t ubidi_props_jgArray[%ld]={\n",
jgArray, 8, prev-jgStart,
"\n};\n\n");
fputs(
"static const UBiDiProps ubidi_props_singleton={\n"
" NULL,\n"
" ubidi_props_indexes,\n"
" ubidi_props_mirrors,\n"
" ubidi_props_jgArray,\n",
f);
usrc_writeUTrie2Struct(f,
" {\n",
trie2, "ubidi_props_trieIndex", NULL,
" },\n");
usrc_writeArray(f, " { ", dataInfo.formatVersion, 8, 4, " }\n");
fputs("};\n", f);
fclose(f);
}
utrie2_close(trie2);
} else {
/* write the data */
pData=udata_create(dataDir, UBIDI_DATA_TYPE, UBIDI_DATA_NAME, &dataInfo,
haveCopyright ? U_COPYRIGHT_STRING : NULL, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "genbidi: unable to create data memory, %s\n", u_errorName(errorCode));
exit(errorCode);
}
udata_writeBlock(pData, indexes, sizeof(indexes));
udata_writeBlock(pData, trieBlock, trieSize);
udata_writeBlock(pData, mirrors, 4*mirrorTop);
udata_writeBlock(pData, jgArray, prev-jgStart);
/* finish up */
dataLength=udata_finish(pData, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "genbidi: error %d writing the output file\n", errorCode);
exit(errorCode);
}
if(dataLength!=indexes[UBIDI_IX_LENGTH]) {
fprintf(stderr, "genbidi: data length %ld != calculated size %d\n",
dataLength, (int)indexes[UBIDI_IX_LENGTH]);
exit(U_INTERNAL_PROGRAM_ERROR);
}
}
utrie_close(pTrie);
upvec_close(pv);
}
extern void
generateData(const char *dataDir, UBool csource) {
static int32_t indexes[UPROPS_INDEX_COUNT]={
0, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0
};
static uint8_t trieBlock[40000];
static uint8_t additionalProps[120000];
UNewDataMemory *pData;
UErrorCode errorCode=U_ZERO_ERROR;
uint32_t size = 0;
int32_t trieSize, additionalPropsSize, offset;
long dataLength;
trieSize=utrie_serialize(pTrie, trieBlock, sizeof(trieBlock), NULL, TRUE, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "error: utrie_serialize failed: %s (length %ld)\n", u_errorName(errorCode), (long)trieSize);
exit(errorCode);
}
offset=sizeof(indexes)/4; /* uint32_t offset to the properties trie */
/* round up trie size to 4-alignment */
trieSize=(trieSize+3)&~3;
offset+=trieSize>>2;
indexes[UPROPS_PROPS32_INDEX]= /* set indexes to the same offsets for empty */
indexes[UPROPS_EXCEPTIONS_INDEX]= /* structures from the old format version 3 */
indexes[UPROPS_EXCEPTIONS_TOP_INDEX]= /* so that less runtime code has to be changed */
indexes[UPROPS_ADDITIONAL_TRIE_INDEX]=offset;
if(beVerbose) {
printf("trie size in bytes: %5u\n", (int)trieSize);
}
if(csource) {
/* write .c file for hardcoded data */
UTrie trie={ NULL };
UTrie2 *trie2;
FILE *f;
utrie_unserialize(&trie, trieBlock, trieSize, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(
stderr,
"genprops error: failed to utrie_unserialize(uprops.icu main trie) - %s\n",
u_errorName(errorCode));
exit(errorCode);
}
/* use UTrie2 */
trie2=utrie2_fromUTrie(&trie, 0, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(
stderr,
"genprops error: utrie2_fromUTrie() failed - %s\n",
u_errorName(errorCode));
exit(errorCode);
}
{
/* delete lead surrogate code unit values */
UChar lead;
trie2=utrie2_cloneAsThawed(trie2, &errorCode);
for(lead=0xd800; lead<0xdc00; ++lead) {
utrie2_set32ForLeadSurrogateCodeUnit(trie2, lead, trie2->initialValue, &errorCode);
}
utrie2_freeze(trie2, UTRIE2_16_VALUE_BITS, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(
stderr,
"genprops error: deleting lead surrogate code unit values failed - %s\n",
u_errorName(errorCode));
exit(errorCode);
}
}
f=usrc_create(dataDir, "uchar_props_data.c");
if(f!=NULL) {
/* unused
usrc_writeArray(f,
"static const UVersionInfo formatVersion={",
dataInfo.formatVersion, 8, 4,
"};\n\n");
*/
usrc_writeArray(f,
"static const UVersionInfo dataVersion={",
dataInfo.dataVersion, 8, 4,
"};\n\n");
usrc_writeUTrie2Arrays(f,
"static const uint16_t propsTrie_index[%ld]={\n", NULL,
trie2,
"\n};\n\n");
usrc_writeUTrie2Struct(f,
"static const UTrie2 propsTrie={\n",
trie2, "propsTrie_index", NULL,
"};\n\n");
additionalPropsSize=writeAdditionalData(f, additionalProps, sizeof(additionalProps), indexes);
size=4*offset+additionalPropsSize; /* total size of data */
usrc_writeArray(f,
"static const int32_t indexes[UPROPS_INDEX_COUNT]={",
indexes, 32, UPROPS_INDEX_COUNT,
"};\n\n");
fclose(f);
}
utrie2_close(trie2);
} else {
/* write the data */
pData=udata_create(dataDir, DATA_TYPE, DATA_NAME, &dataInfo,
haveCopyright ? U_COPYRIGHT_STRING : NULL, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "genprops: unable to create data memory, %s\n", u_errorName(errorCode));
exit(errorCode);
}
additionalPropsSize=writeAdditionalData(NULL, additionalProps, sizeof(additionalProps), indexes);
size=4*offset+additionalPropsSize; /* total size of data */
udata_writeBlock(pData, indexes, sizeof(indexes));
udata_writeBlock(pData, trieBlock, trieSize);
udata_writeBlock(pData, additionalProps, additionalPropsSize);
/* finish up */
dataLength=udata_finish(pData, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "genprops: error %d writing the output file\n", errorCode);
exit(errorCode);
}
if(dataLength!=(long)size) {
fprintf(stderr, "genprops: data length %ld != calculated size %lu\n",
dataLength, (unsigned long)size);
exit(U_INTERNAL_PROGRAM_ERROR);
}
}
if(beVerbose) {
printf("data size: %6lu\n", (unsigned long)size);
}
}
U_CFUNC int32_t
writeAdditionalData(FILE *f, uint8_t *p, int32_t capacity, int32_t indexes[UPROPS_INDEX_COUNT]) {
const uint32_t *pvArray;
int32_t pvRows, pvCount;
int32_t length;
UErrorCode errorCode;
pvArray=upvec_getArray(pv, &pvRows, NULL);
pvCount=pvRows*UPROPS_VECTOR_WORDS;
errorCode=U_ZERO_ERROR;
length=utrie_serialize(newTrie, p, capacity, NULL, TRUE, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "genprops error: unable to serialize trie for additional properties: %s\n", u_errorName(errorCode));
exit(errorCode);
}
if(p!=NULL) {
if(beVerbose) {
printf("size in bytes of additional props trie:%5u\n", (int)length);
}
if(f!=NULL) {
UTrie trie={ NULL };
UTrie2 *trie2;
utrie_unserialize(&trie, p, length, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(
stderr,
"genprops error: failed to utrie_unserialize(trie for additional properties) - %s\n",
u_errorName(errorCode));
exit(errorCode);
}
/* use UTrie2 */
trie2=utrie2_fromUTrie(&trie, trie.initialValue, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(
stderr,
"genprops error: utrie2_fromUTrie() failed - %s\n",
u_errorName(errorCode));
exit(errorCode);
}
{
/* delete lead surrogate code unit values */
UChar lead;
trie2=utrie2_cloneAsThawed(trie2, &errorCode);
for(lead=0xd800; lead<0xdc00; ++lead) {
utrie2_set32ForLeadSurrogateCodeUnit(trie2, lead, trie2->initialValue, &errorCode);
}
utrie2_freeze(trie2, UTRIE2_16_VALUE_BITS, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(
stderr,
"genbidi error: deleting lead surrogate code unit values failed - %s\n",
u_errorName(errorCode));
exit(errorCode);
}
}
usrc_writeUTrie2Arrays(f,
"static const uint16_t propsVectorsTrie_index[%ld]={\n", NULL,
trie2,
"\n};\n\n");
usrc_writeUTrie2Struct(f,
"static const UTrie2 propsVectorsTrie={\n",
trie2, "propsVectorsTrie_index", NULL,
"};\n\n");
utrie2_close(trie2);
}
p+=length;
capacity-=length;
/* set indexes */
indexes[UPROPS_ADDITIONAL_VECTORS_INDEX]=
indexes[UPROPS_ADDITIONAL_TRIE_INDEX]+length/4;
indexes[UPROPS_ADDITIONAL_VECTORS_COLUMNS_INDEX]=UPROPS_VECTOR_WORDS;
indexes[UPROPS_RESERVED_INDEX]=
indexes[UPROPS_ADDITIONAL_VECTORS_INDEX]+pvCount;
indexes[UPROPS_MAX_VALUES_INDEX]=
(((int32_t)U_EA_COUNT-1)<<UPROPS_EA_SHIFT)|
(((int32_t)UBLOCK_COUNT-1)<<UPROPS_BLOCK_SHIFT)|
(((int32_t)USCRIPT_CODE_LIMIT-1)&UPROPS_SCRIPT_MASK);
indexes[UPROPS_MAX_VALUES_2_INDEX]=
(((int32_t)U_LB_COUNT-1)<<UPROPS_LB_SHIFT)|
(((int32_t)U_SB_COUNT-1)<<UPROPS_SB_SHIFT)|
(((int32_t)U_WB_COUNT-1)<<UPROPS_WB_SHIFT)|
(((int32_t)U_GCB_COUNT-1)<<UPROPS_GCB_SHIFT)|
((int32_t)U_DT_COUNT-1);
}
if(p!=NULL && (pvCount*4)<=capacity) {
if(f!=NULL) {
usrc_writeArray(f,
"static const uint32_t propsVectors[%ld]={\n",
pvArray, 32, pvCount,
"};\n\n");
fprintf(f, "static const int32_t countPropsVectors=%ld;\n", (long)pvCount);
fprintf(f, "static const int32_t propsVectorsColumns=%ld;\n", (long)indexes[UPROPS_ADDITIONAL_VECTORS_COLUMNS_INDEX]);
} else {
uprv_memcpy(p, pvArray, pvCount*4);
}
if(beVerbose) {
printf("number of additional props vectors: %5u\n", (int)pvRows);
printf("number of 32-bit words per vector: %5u\n", UPROPS_VECTOR_WORDS);
}
}
length+=pvCount*4;
return length;
}
