// sets the desired transformation data.
// should be populated from a command line argument
// so far the only acceptable format is offset-<hex constant>
// eventually others (mask-<hex constant>?) may be enabled
// more complex functions may be more difficult
void setTransform(const char *t) {
if (strncmp(t, "offset-", 7) == 0) {
char *end;
unsigned long base = uprv_strtoul(t + 7, &end, 16);
if (end == (t + 7) || *end != 0 || base > 0x10FF80) {
fprintf(stderr, "Syntax for offset value in --transform offset-%s invalid!\n", t + 7);
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
transformType = DictionaryData::TRANSFORM_TYPE_OFFSET;
transformConstant = (UChar32)base;
}
else {
fprintf(stderr, "Invalid transform specified: %s\n", t);
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
}
//----------------------------------------------------------------------------
//
// main for gencfu
//
//----------------------------------------------------------------------------
int main(int argc, char **argv) {
UErrorCode status = U_ZERO_ERROR;
const char *confFileName;
const char *confWSFileName;
const char *outFileName;
const char *outDir = NULL;
const char *copyright = NULL;
//
// Pick up and check the command line arguments,
// using the standard ICU tool utils option handling.
//
U_MAIN_INIT_ARGS(argc, argv);
progName = argv[0];
argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
if(argc<0) {
// Unrecognized option
fprintf(stderr, "error in command line argument \"%s\"\n", argv[-argc]);
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
if(options[0].doesOccur || options[1].doesOccur) {
// -? or -h for help.
usageAndDie(0);
}
if (!(options[3].doesOccur && options[4].doesOccur && options[5].doesOccur)) {
fprintf(stderr, "confusables file, whole script confusables file and output file must all be specified.\n");
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
confFileName = options[3].value;
confWSFileName = options[4].value;
outFileName = options[5].value;
if (options[6].doesOccur) {
u_setDataDirectory(options[6].value);
}
status = U_ZERO_ERROR;
/* Combine the directory with the file name */
if(options[7].doesOccur) {
outDir = options[7].value;
}
if (options[8].doesOccur) {
copyright = U_COPYRIGHT_STRING;
}
#if UCONFIG_NO_REGULAR_EXPRESSIONS || UCONFIG_NO_NORMALIZATION || UCONFIG_NO_FILE_IO
// spoof detection data file parsing is dependent on regular expressions.
// TODO: have the tool return an error status. Requires fixing the ICU data build
// so that it doesn't abort entirely on that error.
UNewDataMemory *pData;
char msg[1024];
/* write message with just the name */
sprintf(msg, "gencfu writes dummy %s because of UCONFIG_NO_REGULAR_EXPRESSIONS and/or UCONFIG_NO_NORMALIZATION and/or UCONFIG_NO_FILE_IO, see uconfig.h", outFileName);
fprintf(stderr, "%s\n", msg);
/* write the dummy data file */
pData = udata_create(outDir, NULL, outFileName, &dummyDataInfo, NULL, &status);
udata_writeBlock(pData, msg, strlen(msg));
udata_finish(pData, &status);
return (int)status;
#else
/* Initialize ICU */
u_init(&status);
if (U_FAILURE(status)) {
fprintf(stderr, "%s: can not initialize ICU. status = %s\n",
argv[0], u_errorName(status));
exit(1);
}
status = U_ZERO_ERROR;
// Read in the confusables source file
int32_t confusablesLen = 0;
const char *confusables = readFile(confFileName, &confusablesLen);
if (confusables == NULL) {
printf("gencfu: error reading file \"%s\"\n", confFileName);
exit(-1);
}
int32_t wsConfusablesLen = 0;
const char *wsConfsables = readFile(confWSFileName, &wsConfusablesLen);
if (wsConfsables == NULL) {
printf("gencfu: error reading file \"%s\"\n", confFileName);
exit(-1);
}
//
// Create the Spoof Detector from the source confusables files.
// This will compile the data.
//
UParseError parseError;
parseError.line = 0;
parseError.offset = 0;
int32_t errType;
USpoofChecker *sc = uspoof_openFromSource(confusables, confusablesLen,
wsConfsables, wsConfusablesLen,
&errType, &parseError, &status);
if (U_FAILURE(status)) {
const char *errFile =
(errType == USPOOF_WHOLE_SCRIPT_CONFUSABLE)? confWSFileName : confFileName;
fprintf(stderr, "gencfu: uspoof_openFromSource error \"%s\" at file %s, line %d, column %d\n",
u_errorName(status), errFile, (int)parseError.line, (int)parseError.offset);
exit(status);
};
//
// Get the compiled rule data from the USpoofChecker.
//
uint32_t outDataSize;
uint8_t *outData;
outDataSize = uspoof_serialize(sc, NULL, 0, &status);
if (status != U_BUFFER_OVERFLOW_ERROR) {
fprintf(stderr, "gencfu: uspoof_serialize() returned %s\n", u_errorName(status));
exit(status);
}
status = U_ZERO_ERROR;
outData = new uint8_t[outDataSize];
uspoof_serialize(sc, outData, outDataSize, &status);
// Copy the data format version numbers from the spoof data header into the UDataMemory header.
uprv_memcpy(dh.info.formatVersion,
reinterpret_cast<SpoofDataHeader *>(outData)->fFormatVersion,
sizeof(dh.info.formatVersion));
//
// Create the output file
//
size_t bytesWritten;
UNewDataMemory *pData;
pData = udata_create(outDir, NULL, outFileName, &(dh.info), copyright, &status);
if(U_FAILURE(status)) {
fprintf(stderr, "gencfu: Could not open output file \"%s\", \"%s\"\n",
outFileName, u_errorName(status));
exit(status);
}
// Write the data itself.
udata_writeBlock(pData, outData, outDataSize);
// finish up
bytesWritten = udata_finish(pData, &status);
if(U_FAILURE(status)) {
fprintf(stderr, "gencfu: Error %d writing the output file\n", status);
exit(status);
}
if (bytesWritten != outDataSize) {
fprintf(stderr, "gencfu: Error writing to output file \"%s\"\n", outFileName);
exit(-1);
}
uspoof_close(sc);
delete [] outData;
delete [] confusables;
delete [] wsConfsables;
u_cleanup();
printf("gencfu: tool completed successfully.\n");
return 0;
#endif // UCONFIG_NO_REGULAR_EXPRESSIONS
}
//----------------------------------------------------------------------------
//
// main for genctd
//
//----------------------------------------------------------------------------
int main(int argc, char **argv) {
UErrorCode status = U_ZERO_ERROR;
const char *wordFileName;
const char *outFileName;
const char *outDir = NULL;
const char *copyright = NULL;
//
// Pick up and check the command line arguments,
// using the standard ICU tool utils option handling.
//
U_MAIN_INIT_ARGS(argc, argv);
progName = argv[0];
argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
if(argc<0) {
// Unrecognized option
fprintf(stderr, "error in command line argument \"%s\"\n", argv[-argc]);
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
if(options[0].doesOccur || options[1].doesOccur) {
// -? or -h for help.
usageAndDie(0);
}
if (!options[3].doesOccur || argc < 2) {
fprintf(stderr, "input and output file must both be specified.\n");
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
outFileName = options[3].value;
wordFileName = argv[1];
if (options[4].doesOccur) {
u_setDataDirectory(options[4].value);
}
status = U_ZERO_ERROR;
/* Combine the directory with the file name */
if(options[5].doesOccur) {
outDir = options[5].value;
}
if (options[6].doesOccur) {
copyright = U_COPYRIGHT_STRING;
}
#if UCONFIG_NO_BREAK_ITERATION || UCONFIG_NO_FILE_IO
UNewDataMemory *pData;
char msg[1024];
/* write message with just the name */
sprintf(msg, "genctd writes dummy %s because of UCONFIG_NO_BREAK_ITERATION and/or UCONFIG_NO_FILE_IO, see uconfig.h", outFileName);
fprintf(stderr, "%s\n", msg);
/* write the dummy data file */
pData = udata_create(outDir, NULL, outFileName, &dummyDataInfo, NULL, &status);
udata_writeBlock(pData, msg, strlen(msg));
udata_finish(pData, &status);
return (int)status;
#else
/* Initialize ICU */
u_init(&status);
if (U_FAILURE(status)) {
fprintf(stderr, "%s: can not initialize ICU. status = %s\n",
argv[0], u_errorName(status));
exit(1);
}
status = U_ZERO_ERROR;
//
// Read in the dictionary source file
//
long result;
long wordFileSize;
FILE *file;
char *wordBufferC;
MutableTrieDictionary *mtd = NULL;
file = fopen(wordFileName, "rb");
if( file == 0 ) { //cannot find file
//create 1-line dummy file: ie 1 char, 1 value
UNewDataMemory *pData;
char msg[1024];
/* write message with just the name */
sprintf(msg, "%s not found, genctd writes dummy %s", wordFileName, outFileName);
fprintf(stderr, "%s\n", msg);
UChar c = 0x0020;
mtd = new MutableTrieDictionary(c, status, TRUE);
mtd->addWord(&c, 1, status, 1);
} else { //read words in from input file
fseek(file, 0, SEEK_END);
wordFileSize = ftell(file);
fseek(file, 0, SEEK_SET);
wordBufferC = new char[wordFileSize+10];
result = (long)fread(wordBufferC, 1, wordFileSize, file);
if (result != wordFileSize) {
fprintf(stderr, "Error reading file \"%s\"\n", wordFileName);
exit (-1);
}
wordBufferC[wordFileSize]=0;
fclose(file);
//
// Look for a Unicode Signature (BOM) on the word file
//
int32_t signatureLength;
const char * wordSourceC = wordBufferC;
const char* encoding = ucnv_detectUnicodeSignature(
wordSourceC, wordFileSize, &signatureLength, &status);
if (U_FAILURE(status)) {
exit(status);
}
if(encoding!=NULL ){
wordSourceC += signatureLength;
wordFileSize -= signatureLength;
}
//
// Open a converter to take the rule file to UTF-16
//
UConverter* conv;
conv = ucnv_open(encoding, &status);
if (U_FAILURE(status)) {
fprintf(stderr, "ucnv_open: ICU Error \"%s\"\n", u_errorName(status));
exit(status);
}
//
// Convert the words to UChar.
// Preflight first to determine required buffer size.
//
uint32_t destCap = ucnv_toUChars(conv,
NULL, // dest,
0, // destCapacity,
wordSourceC,
wordFileSize,
&status);
if (status != U_BUFFER_OVERFLOW_ERROR) {
fprintf(stderr, "ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status));
exit(status);
};
status = U_ZERO_ERROR;
UChar *wordSourceU = new UChar[destCap+1];
ucnv_toUChars(conv,
wordSourceU, // dest,
destCap+1,
wordSourceC,
wordFileSize,
&status);
if (U_FAILURE(status)) {
fprintf(stderr, "ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status));
exit(status);
};
ucnv_close(conv);
// Get rid of the original file buffer
delete[] wordBufferC;
// Create a MutableTrieDictionary, and loop through all the lines, inserting
// words.
// First, pick a median character.
UChar *current = wordSourceU + (destCap/2);
UChar uc = *current++;
UnicodeSet breaks;
breaks.add(0x000A); // Line Feed
breaks.add(0x000D); // Carriage Return
breaks.add(0x2028); // Line Separator
breaks.add(0x2029); // Paragraph Separator
do {
// Look for line break
while (uc && !breaks.contains(uc)) {
uc = *current++;
}
// Now skip to first non-line-break
while (uc && breaks.contains(uc)) {
uc = *current++;
}
}
while (uc && (breaks.contains(uc) || u_isspace(uc)));
mtd = new MutableTrieDictionary(uc, status);
if (U_FAILURE(status)) {
fprintf(stderr, "new MutableTrieDictionary: ICU Error \"%s\"\n", u_errorName(status));
exit(status);
}
// Now add the words. Words are non-space characters at the beginning of
// lines, and must be at least one UChar. If a word has an associated value,
// the value should follow the word on the same line after a tab character.
current = wordSourceU;
UChar *candidate = current;
uc = *current++;
int32_t length = 0;
int count = 0;
while (uc) {
while (uc && !u_isspace(uc)) {
++length;
uc = *current++;
}
UnicodeString valueString;
UChar candidateValue;
if(uc == 0x0009){ //separator is a tab char, read in number after space
while (uc && u_isspace(uc)) {
uc = *current++;
}
while (uc && !u_isspace(uc)) {
valueString.append(uc);
uc = *current++;
}
}
if (length > 0) {
count++;
if(valueString.length() > 0){
mtd->setValued(TRUE);
uint32_t value = 0;
char* s = new char[valueString.length()];
valueString.extract(0,valueString.length(), s, valueString.length());
int n = sscanf(s, "%ud", &value);
U_ASSERT(n == 1);
U_ASSERT(value >= 0);
mtd->addWord(candidate, length, status, (uint16_t)value);
delete[] s;
} else {
mtd->addWord(candidate, length, status);
}
if (U_FAILURE(status)) {
fprintf(stderr, "MutableTrieDictionary::addWord: ICU Error \"%s\" at line %d in input file\n",
u_errorName(status), count);
exit(status);
}
}
// Find beginning of next line
while (uc && !breaks.contains(uc)) {
uc = *current++;
}
// Find next non-line-breaking character
while (uc && breaks.contains(uc)) {
uc = *current++;
}
candidate = current-1;
length = 0;
}
// Get rid of the Unicode text buffer
delete[] wordSourceU;
}
// Now, create a CompactTrieDictionary from the mutable dictionary
CompactTrieDictionary *ctd = new CompactTrieDictionary(*mtd, status);
if (U_FAILURE(status)) {
fprintf(stderr, "new CompactTrieDictionary: ICU Error \"%s\"\n", u_errorName(status));
exit(status);
}
// Get rid of the MutableTrieDictionary
delete mtd;
//
// Get the binary data from the dictionary.
//
uint32_t outDataSize = ctd->dataSize();
const uint8_t *outData = (const uint8_t *)ctd->data();
//
// Create the output file
//
size_t bytesWritten;
UNewDataMemory *pData;
pData = udata_create(outDir, NULL, outFileName, &(dh.info), copyright, &status);
if(U_FAILURE(status)) {
fprintf(stderr, "genctd: Could not open output file \"%s\", \"%s\"\n",
outFileName, u_errorName(status));
exit(status);
}
// Write the data itself.
udata_writeBlock(pData, outData, outDataSize);
// finish up
bytesWritten = udata_finish(pData, &status);
if(U_FAILURE(status)) {
fprintf(stderr, "genctd: error \"%s\" writing the output file\n", u_errorName(status));
exit(status);
}
if (bytesWritten != outDataSize) {
fprintf(stderr, "Error writing to output file \"%s\"\n", outFileName);
exit(-1);
}
// Get rid of the CompactTrieDictionary
delete ctd;
u_cleanup();
printf("genctd: tool completed successfully.\n");
return 0;
#endif /* #if !UCONFIG_NO_BREAK_ITERATION */
}
//----------------------------------------------------------------------------
//
// main for gendict
//
//----------------------------------------------------------------------------
int main(int argc, char **argv) {
//
// Pick up and check the command line arguments,
// using the standard ICU tool utils option handling.
//
U_MAIN_INIT_ARGS(argc, argv);
progName = argv[0];
argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
if(argc<0) {
// Unrecognized option
fprintf(stderr, "error in command line argument \"%s\"\n", argv[-argc]);
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
if(options[ARG_HELP].doesOccur || options[ARG_QMARK].doesOccur) {
// -? or -h for help.
usageAndDie(U_ZERO_ERROR);
}
UBool verbose = options[ARG_VERBOSE].doesOccur;
if (argc < 3) {
fprintf(stderr, "input and output file must both be specified.\n");
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
const char *outFileName = argv[2];
const char *wordFileName = argv[1];
startTime = uprv_getRawUTCtime(); // initialize start timer
if (options[ARG_ICUDATADIR].doesOccur) {
u_setDataDirectory(options[ARG_ICUDATADIR].value);
}
const char *copyright = NULL;
if (options[ARG_COPYRIGHT].doesOccur) {
copyright = U_COPYRIGHT_STRING;
}
if (options[ARG_UCHARS].doesOccur == options[ARG_BYTES].doesOccur) {
fprintf(stderr, "you must specify exactly one type of trie to output!\n");
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
UBool isBytesTrie = options[ARG_BYTES].doesOccur;
if (isBytesTrie != options[ARG_TRANSFORM].doesOccur) {
fprintf(stderr, "you must provide a transformation for a bytes trie, and must not provide one for a uchars trie!\n");
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
IcuToolErrorCode status("gendict/main()");
#if UCONFIG_NO_BREAK_ITERATION || UCONFIG_NO_FILE_IO
const char* outDir=NULL;
UNewDataMemory *pData;
char msg[1024];
UErrorCode tempstatus = U_ZERO_ERROR;
/* write message with just the name */ // potential for a buffer overflow here...
sprintf(msg, "gendict writes dummy %s because of UCONFIG_NO_BREAK_ITERATION and/or UCONFIG_NO_FILE_IO, see uconfig.h", outFileName);
fprintf(stderr, "%s\n", msg);
/* write the dummy data file */
pData = udata_create(outDir, NULL, outFileName, &dataInfo, NULL, &tempstatus);
udata_writeBlock(pData, msg, strlen(msg));
udata_finish(pData, &tempstatus);
return (int)tempstatus;
#else
// Read in the dictionary source file
if (verbose) { printf("Opening file %s...\n", wordFileName); }
const char *codepage = "UTF-8";
UCHARBUF *f = ucbuf_open(wordFileName, &codepage, TRUE, FALSE, status);
if (status.isFailure()) {
fprintf(stderr, "error opening input file: ICU Error \"%s\"\n", status.errorName());
exit(status.reset());
}
if (verbose) { printf("Initializing dictionary builder of type %s...\n", (isBytesTrie ? "BytesTrie" : "UCharsTrie")); }
DataDict dict(isBytesTrie, status);
if (status.isFailure()) {
fprintf(stderr, "new DataDict: ICU Error \"%s\"\n", status.errorName());
exit(status.reset());
}
if (options[ARG_TRANSFORM].doesOccur) {
dict.setTransform(options[ARG_TRANSFORM].value);
}
UnicodeString fileLine;
if (verbose) { puts("Adding words to dictionary..."); }
UBool hasValues = FALSE;
UBool hasValuelessContents = FALSE;
int lineCount = 0;
int wordCount = 0;
int minlen = 255;
int maxlen = 0;
UBool isOk = TRUE;
while (readLine(f, fileLine, status)) {
lineCount++;
if (fileLine.isEmpty()) continue;
// Parse word [spaces value].
int32_t keyLen;
for (keyLen = 0; keyLen < fileLine.length() && !u_isspace(fileLine[keyLen]); ++keyLen) {}
if (keyLen == 0) {
fprintf(stderr, "Error: no word on line %i!\n", lineCount);
isOk = FALSE;
continue;
}
int32_t valueStart;
for (valueStart = keyLen;
valueStart < fileLine.length() && u_isspace(fileLine[valueStart]);
++valueStart) {}
if (keyLen < valueStart) {
int32_t valueLength = fileLine.length() - valueStart;
if (valueLength > 15) {
fprintf(stderr, "Error: value too long on line %i!\n", lineCount);
isOk = FALSE;
continue;
}
char s[16];
fileLine.extract(valueStart, valueLength, s, 16, US_INV);
char *end;
unsigned long value = uprv_strtoul(s, &end, 0);
if (end == s || *end != 0 || (int32_t)uprv_strlen(s) != valueLength || value > 0xffffffff) {
fprintf(stderr, "Error: value syntax error or value too large on line %i!\n", lineCount);
isOk = FALSE;
continue;
}
dict.addWord(fileLine.tempSubString(0, keyLen), (int32_t)value, status);
hasValues = TRUE;
wordCount++;
if (keyLen < minlen) minlen = keyLen;
if (keyLen > maxlen) maxlen = keyLen;
} else {
dict.addWord(fileLine.tempSubString(0, keyLen), 0, status);
hasValuelessContents = TRUE;
wordCount++;
if (keyLen < minlen) minlen = keyLen;
if (keyLen > maxlen) maxlen = keyLen;
}
if (status.isFailure()) {
fprintf(stderr, "ICU Error \"%s\": Failed to add word to trie at input line %d in input file\n",
status.errorName(), lineCount);
exit(status.reset());
}
}
if (verbose) { printf("Processed %d lines, added %d words, minlen %d, maxlen %d\n", lineCount, wordCount, minlen, maxlen); }
if (!isOk && status.isSuccess()) {
status.set(U_ILLEGAL_ARGUMENT_ERROR);
}
if (hasValues && hasValuelessContents) {
fprintf(stderr, "warning: file contained both valued and unvalued strings!\n");
}
if (verbose) { printf("Serializing data...isBytesTrie? %d\n", isBytesTrie); }
int32_t outDataSize;
const void *outData;
UnicodeString usp;
if (isBytesTrie) {
StringPiece sp = dict.serializeBytes(status);
outDataSize = sp.size();
outData = sp.data();
} else {
dict.serializeUChars(usp, status);
outDataSize = usp.length() * U_SIZEOF_UCHAR;
outData = usp.getBuffer();
}
if (status.isFailure()) {
fprintf(stderr, "gendict: got failure of type %s while serializing, if U_ILLEGAL_ARGUMENT_ERROR possibly due to duplicate dictionary entries\n", status.errorName());
exit(status.reset());
}
if (verbose) { puts("Opening output file..."); }
UNewDataMemory *pData = udata_create(NULL, NULL, outFileName, &dataInfo, copyright, status);
if (status.isFailure()) {
fprintf(stderr, "gendict: could not open output file \"%s\", \"%s\"\n", outFileName, status.errorName());
exit(status.reset());
}
if (verbose) { puts("Writing to output file..."); }
int32_t indexes[DictionaryData::IX_COUNT] = {
DictionaryData::IX_COUNT * sizeof(int32_t), 0, 0, 0, 0, 0, 0, 0
};
int32_t size = outDataSize + indexes[DictionaryData::IX_STRING_TRIE_OFFSET];
indexes[DictionaryData::IX_RESERVED1_OFFSET] = size;
indexes[DictionaryData::IX_RESERVED2_OFFSET] = size;
indexes[DictionaryData::IX_TOTAL_SIZE] = size;
indexes[DictionaryData::IX_TRIE_TYPE] = isBytesTrie ? DictionaryData::TRIE_TYPE_BYTES : DictionaryData::TRIE_TYPE_UCHARS;
if (hasValues) {
indexes[DictionaryData::IX_TRIE_TYPE] |= DictionaryData::TRIE_HAS_VALUES;
}
indexes[DictionaryData::IX_TRANSFORM] = dict.getTransform();
udata_writeBlock(pData, indexes, sizeof(indexes));
udata_writeBlock(pData, outData, outDataSize);
size_t bytesWritten = udata_finish(pData, status);
if (status.isFailure()) {
fprintf(stderr, "gendict: error \"%s\" writing the output file\n", status.errorName());
exit(status.reset());
}
if (bytesWritten != (size_t)size) {
fprintf(stderr, "Error writing to output file \"%s\"\n", outFileName);
exit(U_INTERNAL_PROGRAM_ERROR);
}
printf("%s: done writing\t%s (%ds).\n", progName, outFileName, elapsedTime());
#ifdef TEST_GENDICT
if (isBytesTrie) {
BytesTrie::Iterator it(outData, outDataSize, status);
while (it.hasNext()) {
it.next(status);
const StringPiece s = it.getString();
int32_t val = it.getValue();
printf("%s -> %i\n", s.data(), val);
}
} else {
UCharsTrie::Iterator it((const UChar *)outData, outDataSize, status);
while (it.hasNext()) {
it.next(status);
const UnicodeString s = it.getString();
int32_t val = it.getValue();
char tmp[1024];
s.extract(0, s.length(), tmp, 1024);
printf("%s -> %i\n", tmp, val);
}
}
#endif
return 0;
#endif /* #if !UCONFIG_NO_BREAK_ITERATION */
}
//----------------------------------------------------------------------------
//
// main for genbrk
//
//----------------------------------------------------------------------------
int main(int argc, char **argv) {
UErrorCode status = U_ZERO_ERROR;
const char *ruleFileName;
const char *outFileName;
const char *outDir = NULL;
const char *copyright = NULL;
//
// Pick up and check the command line arguments,
// using the standard ICU tool utils option handling.
//
U_MAIN_INIT_ARGS(argc, argv);
progName = argv[0];
argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
if(argc<0) {
// Unrecognized option
fprintf(stderr, "error in command line argument \"%s\"\n", argv[-argc]);
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
if(options[0].doesOccur || options[1].doesOccur) {
// -? or -h for help.
usageAndDie(0);
}
if (!(options[3].doesOccur && options[4].doesOccur)) {
fprintf(stderr, "rule file and output file must both be specified.\n");
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
ruleFileName = options[3].value;
outFileName = options[4].value;
if (options[5].doesOccur) {
u_setDataDirectory(options[5].value);
}
/* Initialize ICU */
u_init(&status);
if (U_FAILURE(status)) {
fprintf(stderr, "%s: can not initialize ICU. status = %s\n",
argv[0], u_errorName(status));
exit(1);
}
status = U_ZERO_ERROR;
/* Combine the directory with the file name */
if(options[6].doesOccur) {
outDir = options[6].value;
}
if (options[7].doesOccur) {
copyright = U_COPYRIGHT_STRING;
}
#if UCONFIG_NO_BREAK_ITERATION
UNewDataMemory *pData;
char msg[1024];
/* write message with just the name */
sprintf(msg, "genbrk writes dummy %s because of UCONFIG_NO_BREAK_ITERATION, see uconfig.h", outFileName);
fprintf(stderr, "%s\n", msg);
/* write the dummy data file */
pData = udata_create(outDir, NULL, outFileName, &dummyDataInfo, NULL, &status);
udata_writeBlock(pData, msg, strlen(msg));
udata_finish(pData, &status);
return (int)status;
#else
//
// Read in the rule source file
//
long result;
long ruleFileSize;
FILE *file;
char *ruleBufferC;
file = fopen(ruleFileName, "rb");
if( file == 0 ) {
fprintf(stderr, "Could not open file \"%s\"\n", ruleFileName);
exit(-1);
}
fseek(file, 0, SEEK_END);
ruleFileSize = ftell(file);
fseek(file, 0, SEEK_SET);
ruleBufferC = new char[ruleFileSize+10];
result = (long)fread(ruleBufferC, 1, ruleFileSize, file);
if (result != ruleFileSize) {
fprintf(stderr, "Error reading file \"%s\"\n", ruleFileName);
exit (-1);
}
ruleBufferC[ruleFileSize]=0;
fclose(file);
//
// Look for a Unicode Signature (BOM) on the rule file
//
int32_t signatureLength;
const char * ruleSourceC = ruleBufferC;
const char* encoding = ucnv_detectUnicodeSignature(
ruleSourceC, ruleFileSize, &signatureLength, &status);
if (U_FAILURE(status)) {
exit(status);
}
if(encoding!=NULL ){
ruleSourceC += signatureLength;
ruleFileSize -= signatureLength;
}
//
// Open a converter to take the rule file to UTF-16
//
UConverter* conv;
conv = ucnv_open(encoding, &status);
if (U_FAILURE(status)) {
fprintf(stderr, "ucnv_open: ICU Error \"%s\"\n", u_errorName(status));
exit(status);
}
//
// Convert the rules to UChar.
// Preflight first to determine required buffer size.
//
uint32_t destCap = ucnv_toUChars(conv,
NULL, // dest,
0, // destCapacity,
ruleSourceC,
ruleFileSize,
&status);
if (status != U_BUFFER_OVERFLOW_ERROR) {
fprintf(stderr, "ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status));
exit(status);
};
status = U_ZERO_ERROR;
UChar *ruleSourceU = new UChar[destCap+1];
ucnv_toUChars(conv,
ruleSourceU, // dest,
destCap+1,
ruleSourceC,
ruleFileSize,
&status);
if (U_FAILURE(status)) {
fprintf(stderr, "ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status));
exit(status);
};
ucnv_close(conv);
//
// Put the source rules into a UnicodeString
//
UnicodeString ruleSourceS(FALSE, ruleSourceU, destCap);
//
// Create the break iterator from the rules
// This will compile the rules.
//
UParseError parseError;
parseError.line = 0;
parseError.offset = 0;
RuleBasedBreakIterator *bi = new RuleBasedBreakIterator(ruleSourceS, parseError, status);
if (U_FAILURE(status)) {
fprintf(stderr, "createRuleBasedBreakIterator: ICU Error \"%s\" at line %d, column %d\n",
u_errorName(status), (int)parseError.line, (int)parseError.offset);
exit(status);
};
//
// Get the compiled rule data from the break iterator.
//
uint32_t outDataSize;
const uint8_t *outData;
outData = bi->getBinaryRules(outDataSize);
// Copy the data format version numbers from the RBBI data header into the UDataMemory header.
uprv_memcpy(dh.info.formatVersion, ((RBBIDataHeader *)outData)->fFormatVersion, sizeof(dh.info.formatVersion));
//
// Create the output file
//
size_t bytesWritten;
UNewDataMemory *pData;
pData = udata_create(outDir, NULL, outFileName, &(dh.info), copyright, &status);
if(U_FAILURE(status)) {
fprintf(stderr, "genbrk: Could not open output file \"%s\", \"%s\"\n",
outFileName, u_errorName(status));
exit(status);
}
// Write the data itself.
udata_writeBlock(pData, outData, outDataSize);
// finish up
bytesWritten = udata_finish(pData, &status);
if(U_FAILURE(status)) {
fprintf(stderr, "genbrk: error %d writing the output file\n", status);
exit(status);
}
if (bytesWritten != outDataSize) {
fprintf(stderr, "Error writing to output file \"%s\"\n", outFileName);
exit(-1);
}
delete bi;
delete[] ruleSourceU;
delete[] ruleBufferC;
u_cleanup();
printf("genbrk: tool completed successfully.\n");
return 0;
#endif /* #if !UCONFIG_NO_BREAK_ITERATION */
}
