/*------------------------------------------------------------------------------
*
* computeDirPath given a user-supplied path of an item to be opened,
* compute and return
* - the full directory path to be used
* when opening the file.
* - Pointer to null at end of above returned path
*
* Parameters:
* path: input path. Buffer is not altered.
* pathBuffer: Output buffer. Any contents are overwritten.
*
* Returns:
* Pointer to null termination in returned pathBuffer.
*
* TODO: This works the way ICU historically has, but the
* whole data fallback search path is so complicated that
* proabably almost no one will ever really understand it,
* the potential for confusion is large. (It's not just
* this one function, but the whole scheme.)
*
*------------------------------------------------------------------------------*/
static char *uprv_computeDirPath(const char *path, char *pathBuffer)
{
char *finalSlash; /* Ptr to last dir separator in input path, or null if none. */
int32_t pathLen; /* Length of the returned directory path */
finalSlash = 0;
if (path != 0) {
finalSlash = uprv_strrchr(path, U_FILE_SEP_CHAR);
}
*pathBuffer = 0;
if (finalSlash == 0) {
/* No user-supplied path.
* Copy the ICU_DATA path to the path buffer and return that*/
const char *icuDataDir;
icuDataDir=u_getDataDirectory();
if(icuDataDir!=NULL && *icuDataDir!=0) {
return strcpy_returnEnd(pathBuffer, icuDataDir);
} else {
/* there is no icuDataDir either. Just return the empty pathBuffer. */
return pathBuffer;
}
}
/* User supplied path did contain a directory portion.
* Copy it to the output path buffer */
pathLen = (int32_t)(finalSlash - path + 1);
uprv_memcpy(pathBuffer, path, pathLen);
*(pathBuffer+pathLen) = 0;
return pathBuffer+pathLen;
}
/* These tests do cleanup and reinitialize ICU in the course of their operation.
* The ICU data directory must be preserved across these operations.
* Here is a helper function to assist with that.
*/
static char *safeGetICUDataDirectory() {
const char *dataDir = u_getDataDirectory(); /* Returned string vanashes with u_cleanup */
char *retStr = NULL;
if (dataDir != NULL) {
retStr = (char *)malloc(strlen(dataDir)+1);
strcpy(retStr, dataDir);
}
return retStr;
}
void register_libcore_icu_ICU(JNIEnv* env) {
std::string path;
path = u_getDataDirectory();
path += "/";
path += U_ICUDATA_NAME;
path += ".dat";
#define FAIL_WITH_STRERROR(s) \
ALOGE("Couldn't " s " '%s': %s", path.c_str(), strerror(errno)); \
abort();
#define MAYBE_FAIL_WITH_ICU_ERROR(s) \
if (status != U_ZERO_ERROR) {\
ALOGE("Couldn't initialize ICU (" s "): %s (%s)", u_errorName(status), path.c_str()); \
abort(); \
}
// Open the file and get its length.
ScopedFd fd(open(path.c_str(), O_RDONLY));
if (fd.get() == -1) {
FAIL_WITH_STRERROR("open");
}
struct stat sb;
if (fstat(fd.get(), &sb) == -1) {
FAIL_WITH_STRERROR("stat");
}
// Map it.
void* data = mmap(NULL, sb.st_size, PROT_READ, MAP_SHARED, fd.get(), 0);
if (data == MAP_FAILED) {
FAIL_WITH_STRERROR("mmap");
}
// Tell the kernel that accesses are likely to be random rather than sequential.
if (madvise(data, sb.st_size, MADV_RANDOM) == -1) {
FAIL_WITH_STRERROR("madvise(MADV_RANDOM)");
}
// Tell ICU to use our memory-mapped data.
UErrorCode status = U_ZERO_ERROR;
udata_setCommonData(data, &status);
MAYBE_FAIL_WITH_ICU_ERROR("udata_setCommonData");
// Tell ICU it can *only* use our memory-mapped data.
udata_setFileAccess(UDATA_NO_FILES, &status);
MAYBE_FAIL_WITH_ICU_ERROR("udata_setFileAccess");
// Failures to find the ICU data tend to be somewhat obscure because ICU loads its data on first
// use, which can be anywhere. Force initialization up front so we can report a nice clear error
// and bail.
u_init(&status);
MAYBE_FAIL_WITH_ICU_ERROR("u_init");
jniRegisterNativeMethods(env, "libcore/icu/ICU", gMethods, NELEM(gMethods));
}
void usageAndDie(int retCode) {
printf("Usage: %s [-v] [-options] -o output-file dictionary-file\n", progName);
printf("\tRead in word list and write out compact trie dictionary\n"
"options:\n"
"\t-h or -? or --help this usage text\n"
"\t-V or --version show a version message\n"
"\t-c or --copyright include a copyright notice\n"
"\t-v or --verbose turn on verbose output\n"
"\t-i or --icudatadir directory for locating any needed intermediate data files,\n"
"\t followed by path, defaults to %s\n"
"\t-d or --destdir destination directory, followed by the path\n",
u_getDataDirectory());
exit (retCode);
}
void usageAndDie(int retCode) {
printf("Usage: %s [-v] [-options] -r confusablesRules.txt -w wholeScriptConfusables.txt -o output-file\n", progName);
printf("\tRead in Unicode confusable character definitions and write out the binary data\n"
"options:\n"
"\t-h or -? or --help this usage text\n"
"\t-V or --version show a version message\n"
"\t-c or --copyright include a copyright notice\n"
"\t-v or --verbose turn on verbose output\n"
"\t-i or --icudatadir directory for locating any needed intermediate data files,\n"
"\t followed by path, defaults to %s\n"
"\t-d or --destdir destination directory, followed by the path\n",
u_getDataDirectory());
exit (retCode);
}
static void initMsg(const char *pname) {
static int ps = 0;
if (!ps) {
char dataPath[2048]; /* XXX Sloppy: should be PATH_MAX. */
UErrorCode err = U_ZERO_ERROR;
ps = 1;
/* Set up our static data - if any */
#ifdef UCONVMSG_LINK
udata_setAppData(UCONVMSG, (const void*) uconvmsg_dat, &err);
if (U_FAILURE(err)) {
fprintf(stderr, "%s: warning, problem installing our static resource bundle data uconvmsg: %s - trying anyways.\n",
pname, u_errorName(err));
err = U_ZERO_ERROR; /* It may still fail */
}
#endif
/* Get messages. */
gBundle = u_wmsg_setPath(UCONVMSG, &err);
if (U_FAILURE(err)) {
fprintf(stderr,
"%s: warning: couldn't open bundle %s: %s\n",
pname, UCONVMSG, u_errorName(err));
#ifdef UCONVMSG_LINK
fprintf(stderr,
"%s: setAppData was called, internal data %s failed to load\n",
pname, UCONVMSG);
#endif
err = U_ZERO_ERROR;
/* that was try #1, try again with a path */
uprv_strcpy(dataPath, u_getDataDirectory());
uprv_strcat(dataPath, U_FILE_SEP_STRING);
uprv_strcat(dataPath, UCONVMSG);
gBundle = u_wmsg_setPath(dataPath, &err);
if (U_FAILURE(err)) {
fprintf(stderr,
"%s: warning: still couldn't open bundle %s: %s\n",
pname, dataPath, u_errorName(err));
fprintf(stderr, "%s: warning: messages will not be displayed\n", pname);
}
}
}
}
// prints out the standard usage method describing command line arguments,
// then bails out with the desired exit code
static void usageAndDie(UErrorCode retCode) {
fprintf((U_SUCCESS(retCode) ? stdout : stderr), "Usage: %s -trietype [-options] input-dictionary-file output-file\n", progName);
fprintf((U_SUCCESS(retCode) ? stdout : stderr),
"\tRead in a word list and write out a string trie dictionary\n"
"options:\n"
"\t-h or -? or --help this usage text\n"
"\t-V or --version show a version message\n"
"\t-c or --copyright include a copyright notice\n"
"\t-v or --verbose turn on verbose output\n"
"\t-i or --icudatadir directory for locating any needed intermediate data files,\n" // TODO: figure out if we need this option
"\t followed by path, defaults to %s\n"
"\t--uchars output a UCharsTrie (mutually exclusive with -b!)\n"
"\t--bytes output a BytesTrie (mutually exclusive with -u!)\n"
"\t--transform the kind of transform to use (eg --transform offset-40A3,\n"
"\t which specifies an offset transform with constant 0x40A3)\n",
u_getDataDirectory());
exit(retCode);
}
static int printHelp(int argc, char* argv[]){
/*
* Broken into chucks because the C89 standard says the minimum
* required supported string length is 509 bytes.
*/
fprintf(stderr,
"Usage: %s [-options] [file_name]\n"
"\n"
"Read the files specified and\n"
"create a binary file [package-name]_[bundle-name]." DATA_TYPE " with the StringPrep profile data\n"
"\n",
argv[0]);
fprintf(stderr,
"Options:\n"
"\t-h or -? or --help print this usage text\n"
"\t-v or --verbose verbose output\n"
"\t-c or --copyright include a copyright notice\n");
fprintf(stderr,
"\t-d or --destdir destination directory, followed by the path\n"
"\t-s or --sourcedir source directory of ICU data, followed by the path\n"
"\t-b or --bundle-name generate the ouput data file with the name specified\n"
"\t-i or --icudatadir directory for locating any needed intermediate data files,\n"
"\t followed by path, defaults to %s\n",
u_getDataDirectory());
fprintf(stderr,
"\t-n or --normalize turn on the option for normalization and include mappings\n"
"\t from NormalizationCorrections.txt from the given path,\n"
"\t e.g: /test/icu/source/data/unidata\n");
fprintf(stderr,
"\t-m or --norm-correction use NormalizationCorrections.txt from the given path\n"
"\t when the input file contains a normalization directive.\n"
"\t unlike -n/--normalize, this option does not force the\n"
"\t normalization.\n");
fprintf(stderr,
"\t-k or --check-bidi turn on the option for checking for BiDi in the profile\n"
"\t-u or --unicode version of Unicode to be used with this profile followed by the version\n"
);
return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
}
int
main(int argc,
char* argv[])
{
UErrorCode status = U_ZERO_ERROR;
const char *arg = NULL;
const char *outputDir = NULL; /* NULL = no output directory, use current */
const char *inputDir = NULL;
const char *encoding = "";
int i;
U_MAIN_INIT_ARGS(argc, argv);
argc = u_parseArgs(argc, argv, (int32_t)(sizeof(options)/sizeof(options[0])), options);
/* error handling, printing usage message */
if(argc<0) {
fprintf(stderr, "%s: error in command line argument \"%s\"\n", argv[0], argv[-argc]);
} else if(argc<2) {
argc = -1;
}
if(options[WRITE_POOL_BUNDLE].doesOccur && options[USE_POOL_BUNDLE].doesOccur) {
fprintf(stderr, "%s: cannot combine --writePoolBundle and --usePoolBundle\n", argv[0]);
argc = -1;
}
if(options[FORMAT_VERSION].doesOccur) {
const char *s = options[FORMAT_VERSION].value;
if(uprv_strlen(s) != 1 || (s[0] != '1' && s[0] != '2')) {
fprintf(stderr, "%s: unsupported --formatVersion %s\n", argv[0], s);
argc = -1;
} else if(s[0] == '1' &&
(options[WRITE_POOL_BUNDLE].doesOccur || options[USE_POOL_BUNDLE].doesOccur)
) {
fprintf(stderr, "%s: cannot combine --formatVersion 1 with --writePoolBundle or --usePoolBundle\n", argv[0]);
argc = -1;
} else {
setFormatVersion(s[0] - '0');
}
}
if(options[VERSION].doesOccur) {
fprintf(stderr,
"%s version %s (ICU version %s).\n"
"%s\n",
argv[0], GENRB_VERSION, U_ICU_VERSION, U_COPYRIGHT_STRING);
return U_ZERO_ERROR;
}
if(argc<0 || options[HELP1].doesOccur || options[HELP2].doesOccur) {
/*
* Broken into chunks because the C89 standard says the minimum
* required supported string length is 509 bytes.
*/
fprintf(stderr,
"Usage: %s [OPTIONS] [FILES]\n"
"\tReads the list of resource bundle source files and creates\n"
"\tbinary version of reosurce bundles (.res files)\n",
argv[0]);
fprintf(stderr,
"Options:\n"
"\t-h or -? or --help this usage text\n"
"\t-q or --quiet do not display warnings\n"
"\t-v or --verbose print extra information when processing files\n"
"\t-V or --version prints out version number and exits\n"
"\t-c or --copyright include copyright notice\n");
fprintf(stderr,
"\t-e or --encoding encoding of source files\n"
"\t-d of --destdir destination directory, followed by the path, defaults to %s\n"
"\t-s or --sourcedir source directory for files followed by path, defaults to %s\n"
"\t-i or --icudatadir directory for locating any needed intermediate data files,\n"
"\t followed by path, defaults to %s\n",
u_getDataDirectory(), u_getDataDirectory(), u_getDataDirectory());
fprintf(stderr,
"\t-j or --write-java write a Java ListResourceBundle for ICU4J, followed by optional encoding\n"
"\t defaults to ASCII and \\uXXXX format.\n");
/* This option is deprecated and should not be used ever.
"\t-p or --package-name For ICU4J: package name for writing the ListResourceBundle for ICU4J,\n"
"\t defaults to com.ibm.icu.impl.data\n"); */
fprintf(stderr,
"\t-b or --bundle-name bundle name for writing the ListResourceBundle for ICU4J,\n"
"\t defaults to LocaleElements\n"
"\t-x or --write-xliff write an XLIFF file for the resource bundle. Followed by\n"
"\t an optional output file name.\n"
"\t-k or --strict use pedantic parsing of syntax\n"
/*added by Jing*/
"\t-l or --language for XLIFF: language code compliant with BCP 47.\n");
fprintf(stderr,
"\t-C or --noBinaryCollation do not generate binary collation image;\n"
"\t makes .res file smaller but collator instantiation much slower;\n"
"\t maintains ability to get tailoring rules\n"
"\t-R or --omitCollationRules do not include collation (tailoring) rules;\n"
"\t makes .res file smaller and maintains collator instantiation speed\n"
"\t but tailoring rules will not be available (they are rarely used)\n");
fprintf(stderr,
"\t --formatVersion write a .res file compatible with the requested formatVersion (single digit);\n"
"\t for example, --formatVersion 1\n");
fprintf(stderr,
"\t --writePoolBundle write a pool.res file with all of the keys of all input bundles\n"
"\t --usePoolBundle [path-to-pool.res] point to keys from the pool.res keys pool bundle if they are available there;\n"
"\t makes .res files smaller but dependent on the pool bundle\n"
"\t (--writePoolBundle and --usePoolBundle cannot be combined)\n");
return argc < 0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
}
if(options[VERBOSE].doesOccur) {
setVerbose(TRUE);
}
if(options[QUIET].doesOccur) {
setShowWarning(FALSE);
}
if(options[STRICT].doesOccur) {
setStrict(TRUE);
}
if(options[COPYRIGHT].doesOccur){
setIncludeCopyright(TRUE);
}
if(options[SOURCEDIR].doesOccur) {
inputDir = options[SOURCEDIR].value;
}
if(options[DESTDIR].doesOccur) {
outputDir = options[DESTDIR].value;
}
/* This option is deprecated and should never be used.
if(options[PACKAGE_NAME].doesOccur) {
gPackageName = options[PACKAGE_NAME].value;
if(!strcmp(gPackageName, "ICUDATA"))
{
gPackageName = U_ICUDATA_NAME;
}
if(gPackageName[0] == 0)
{
gPackageName = NULL;
}
}*/
if(options[ENCODING].doesOccur) {
encoding = options[ENCODING].value;
}
if(options[ICUDATADIR].doesOccur) {
u_setDataDirectory(options[ICUDATADIR].value);
}
/* Initialize ICU */
u_init(&status);
if (U_FAILURE(status) && status != U_FILE_ACCESS_ERROR) {
/* Note: u_init() will try to open ICU property data.
* failures here are expected when building ICU from scratch.
* ignore them.
*/
fprintf(stderr, "%s: can not initialize ICU. status = %s\n",
argv[0], u_errorName(status));
exit(1);
}
status = U_ZERO_ERROR;
if(options[WRITE_JAVA].doesOccur) {
write_java = TRUE;
outputEnc = options[WRITE_JAVA].value;
}
if(options[BUNDLE_NAME].doesOccur) {
bundleName = options[BUNDLE_NAME].value;
}
if(options[WRITE_XLIFF].doesOccur) {
write_xliff = TRUE;
if(options[WRITE_XLIFF].value != NULL){
xliffOutputFileName = options[WRITE_XLIFF].value;
}
}
initParser(options[NO_BINARY_COLLATION].doesOccur, options[NO_COLLATION_RULES].doesOccur);
/*added by Jing*/
if(options[LANGUAGE].doesOccur) {
language = options[LANGUAGE].value;
}
if(options[WRITE_POOL_BUNDLE].doesOccur) {
newPoolBundle = bundle_open(NULL, TRUE, &status);
if(U_FAILURE(status)) {
fprintf(stderr, "unable to create an empty bundle for the pool keys: %s\n", u_errorName(status));
return status;
} else {
const char *poolResName = "pool.res";
char *nameWithoutSuffix = uprv_malloc(uprv_strlen(poolResName) + 1);
if (nameWithoutSuffix == NULL) {
fprintf(stderr, "out of memory error\n");
return U_MEMORY_ALLOCATION_ERROR;
}
uprv_strcpy(nameWithoutSuffix, poolResName);
*uprv_strrchr(nameWithoutSuffix, '.') = 0;
newPoolBundle->fLocale = nameWithoutSuffix;
}
}
if(options[USE_POOL_BUNDLE].doesOccur) {
const char *poolResName = "pool.res";
FileStream *poolFile;
int32_t poolFileSize;
int32_t indexLength;
/*
* TODO: Consolidate inputDir/filename handling from main() and processFile()
* into a common function, and use it here as well.
* Try to create toolutil functions for dealing with dir/filenames and
* loading ICU data files without udata_open().
* Share code with icupkg?
* Also, make_res_filename() seems to be unused. Review and remove.
*/
if (options[USE_POOL_BUNDLE].value!=NULL) {
uprv_strcpy(theCurrentFileName, options[USE_POOL_BUNDLE].value);
uprv_strcat(theCurrentFileName, U_FILE_SEP_STRING);
} else if (inputDir) {
uprv_strcpy(theCurrentFileName, inputDir);
uprv_strcat(theCurrentFileName, U_FILE_SEP_STRING);
} else {
*theCurrentFileName = 0;
}
uprv_strcat(theCurrentFileName, poolResName);
poolFile = T_FileStream_open(theCurrentFileName, "rb");
if (poolFile == NULL) {
fprintf(stderr, "unable to open pool bundle file %s\n", theCurrentFileName);
return 1;
}
poolFileSize = T_FileStream_size(poolFile);
if (poolFileSize < 32) {
fprintf(stderr, "the pool bundle file %s is too small\n", theCurrentFileName);
return 1;
}
poolBundle.fBytes = (uint8_t *)uprv_malloc((poolFileSize + 15) & ~15);
if (poolFileSize > 0 && poolBundle.fBytes == NULL) {
fprintf(stderr, "unable to allocate memory for the pool bundle file %s\n", theCurrentFileName);
return U_MEMORY_ALLOCATION_ERROR;
} else {
UDataSwapper *ds;
const DataHeader *header;
int32_t bytesRead = T_FileStream_read(poolFile, poolBundle.fBytes, poolFileSize);
int32_t keysBottom;
if (bytesRead != poolFileSize) {
fprintf(stderr, "unable to read the pool bundle file %s\n", theCurrentFileName);
return 1;
}
/*
* Swap the pool bundle so that a single checked-in file can be used.
* The swapper functions also test that the data looks like
* a well-formed .res file.
*/
ds = udata_openSwapperForInputData(poolBundle.fBytes, bytesRead,
U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, &status);
if (U_FAILURE(status)) {
fprintf(stderr, "udata_openSwapperForInputData(pool bundle %s) failed: %s\n",
theCurrentFileName, u_errorName(status));
return status;
}
ures_swap(ds, poolBundle.fBytes, bytesRead, poolBundle.fBytes, &status);
udata_closeSwapper(ds);
if (U_FAILURE(status)) {
fprintf(stderr, "ures_swap(pool bundle %s) failed: %s\n",
theCurrentFileName, u_errorName(status));
return status;
}
header = (const DataHeader *)poolBundle.fBytes;
if (header->info.formatVersion[0]!=2) {
fprintf(stderr, "invalid format of pool bundle file %s\n", theCurrentFileName);
return U_INVALID_FORMAT_ERROR;
}
poolBundle.fKeys = (const char *)header + header->dataHeader.headerSize;
poolBundle.fIndexes = (const int32_t *)poolBundle.fKeys + 1;
indexLength = poolBundle.fIndexes[URES_INDEX_LENGTH] & 0xff;
if (indexLength <= URES_INDEX_POOL_CHECKSUM) {
fprintf(stderr, "insufficient indexes[] in pool bundle file %s\n", theCurrentFileName);
return U_INVALID_FORMAT_ERROR;
}
keysBottom = (1 + indexLength) * 4;
poolBundle.fKeys += keysBottom;
poolBundle.fKeysLength = (poolBundle.fIndexes[URES_INDEX_KEYS_TOP] * 4) - keysBottom;
poolBundle.fChecksum = poolBundle.fIndexes[URES_INDEX_POOL_CHECKSUM];
}
for (i = 0; i < poolBundle.fKeysLength; ++i) {
if (poolBundle.fKeys[i] == 0) {
++poolBundle.fKeysCount;
}
}
T_FileStream_close(poolFile);
setUsePoolBundle(TRUE);
}
if(options[INCLUDE_UNIHAN_COLL].doesOccur) {
gIncludeUnihanColl = TRUE;
}
if((argc-1)!=1) {
printf("genrb number of files: %d\n", argc - 1);
}
/* generate the binary files */
for(i = 1; i < argc; ++i) {
status = U_ZERO_ERROR;
arg = getLongPathname(argv[i]);
if (inputDir) {
uprv_strcpy(theCurrentFileName, inputDir);
uprv_strcat(theCurrentFileName, U_FILE_SEP_STRING);
} else {
*theCurrentFileName = 0;
}
uprv_strcat(theCurrentFileName, arg);
if (isVerbose()) {
printf("Processing file \"%s\"\n", theCurrentFileName);
}
processFile(arg, encoding, inputDir, outputDir, gPackageName, &status);
}
uprv_free(poolBundle.fBytes);
if(options[WRITE_POOL_BUNDLE].doesOccur) {
char outputFileName[256];
bundle_write(newPoolBundle, outputDir, NULL, outputFileName, sizeof(outputFileName), &status);
bundle_close(newPoolBundle, &status);
if(U_FAILURE(status)) {
fprintf(stderr, "unable to write the pool bundle: %s\n", u_errorName(status));
}
}
/* Dont return warnings as a failure */
if (U_SUCCESS(status)) {
return 0;
}
return status;
}
extern int
main(int argc, char* argv[]) {
char filename[300];
const char *srcDir=NULL, *destDir=NULL, *suffix=NULL;
char *basename=NULL;
UErrorCode errorCode=U_ZERO_ERROR;
U_MAIN_INIT_ARGS(argc, argv);
/* preset then read command line options */
options[DESTDIR].value=u_getDataDirectory();
options[SOURCEDIR].value="";
options[UNICODE_VERSION].value="";
options[ICUDATADIR].value=u_getDataDirectory();
argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
/* error handling, printing usage message */
if(argc<0) {
fprintf(stderr,
"error in command line argument \"%s\"\n",
argv[-argc]);
}
if(argc<0 || options[HELP_H].doesOccur || options[HELP_QUESTION_MARK].doesOccur) {
/*
* Broken into chucks because the C89 standard says the minimum
* required supported string length is 509 bytes.
*/
fprintf(stderr,
"Usage: %s [-options] [suffix]\n"
"\n"
"read the UnicodeData.txt file and other Unicode properties files and\n"
"create a binary file " UBIDI_DATA_NAME "." UBIDI_DATA_TYPE " with the bidi/shaping properties\n"
"\n",
argv[0]);
fprintf(stderr,
"Options:\n"
"\t-h or -? or --help this usage text\n"
"\t-v or --verbose verbose output\n"
"\t-c or --copyright include a copyright notice\n"
"\t-u or --unicode Unicode version, followed by the version like 3.0.0\n"
"\t-C or --csource generate a .c source file rather than the .icu binary\n");
fprintf(stderr,
"\t-d or --destdir destination directory, followed by the path\n"
"\t-s or --sourcedir source directory, followed by the path\n"
"\t-i or --icudatadir directory for locating any needed intermediate data files,\n"
"\t followed by path, defaults to %s\n"
"\tsuffix suffix that is to be appended with a '-'\n"
"\t to the source file basenames before opening;\n"
"\t 'genbidi new' will read UnicodeData-new.txt etc.\n",
u_getDataDirectory());
return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
}
/* get the options values */
beVerbose=options[VERBOSE].doesOccur;
haveCopyright=options[COPYRIGHT].doesOccur;
srcDir=options[SOURCEDIR].value;
destDir=options[DESTDIR].value;
if(argc>=2) {
suffix=argv[1];
} else {
suffix=NULL;
}
if(options[UNICODE_VERSION].doesOccur) {
setUnicodeVersion(options[UNICODE_VERSION].value);
}
/* else use the default dataVersion in store.c */
if (options[ICUDATADIR].doesOccur) {
u_setDataDirectory(options[ICUDATADIR].value);
}
/* prepare the filename beginning with the source dir */
uprv_strcpy(filename, srcDir);
basename=filename+uprv_strlen(filename);
if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
*basename++=U_FILE_SEP_CHAR;
}
/* initialize */
pv=upvec_open(2, &errorCode);
/* process BidiMirroring.txt */
writeUCDFilename(basename, "BidiMirroring", suffix);
parseBidiMirroring(filename, &errorCode);
/* process additional properties files */
*basename=0;
parseBinariesFile(filename, basename, suffix, &propListBinaries, &errorCode);
parseSingleEnumFile(filename, basename, suffix, &jtSingleEnum, &errorCode);
parseSingleEnumFile(filename, basename, suffix, &jgSingleEnum, &errorCode);
/* process UnicodeData.txt */
writeUCDFilename(basename, "UnicodeData", suffix);
parseDB(filename, &errorCode);
/* set proper bidi class for unassigned code points (Cn) */
parseTwoFieldFile(filename, basename, "DerivedBidiClass", suffix, bidiClassLineFn, &errorCode);
/* process parsed data */
if(U_SUCCESS(errorCode)) {
/* write the properties data file */
generateData(destDir, options[CSOURCE].doesOccur);
}
u_cleanup();
return errorCode;
}
int
main(int argc,
char* argv[])
{
UErrorCode status = U_ZERO_ERROR;
const char *arg = NULL;
const char *outputDir = NULL; /* NULL = no output directory, use current */
const char *inputDir = NULL;
const char *encoding = "";
int i;
UBool illegalArg = FALSE;
U_MAIN_INIT_ARGS(argc, argv);
options[JAVA_PACKAGE].value = "com.ibm.icu.impl.data";
options[BUNDLE_NAME].value = "LocaleElements";
argc = u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options);
/* error handling, printing usage message */
if(argc<0) {
fprintf(stderr, "%s: error in command line argument \"%s\"\n", argv[0], argv[-argc]);
illegalArg = TRUE;
} else if(argc<2) {
illegalArg = TRUE;
}
if(options[WRITE_POOL_BUNDLE].doesOccur && options[USE_POOL_BUNDLE].doesOccur) {
fprintf(stderr, "%s: cannot combine --writePoolBundle and --usePoolBundle\n", argv[0]);
illegalArg = TRUE;
}
if(options[FORMAT_VERSION].doesOccur) {
const char *s = options[FORMAT_VERSION].value;
if(uprv_strlen(s) != 1 || (s[0] < '1' && '3' < s[0])) {
fprintf(stderr, "%s: unsupported --formatVersion %s\n", argv[0], s);
illegalArg = TRUE;
} else if(s[0] == '1' &&
(options[WRITE_POOL_BUNDLE].doesOccur || options[USE_POOL_BUNDLE].doesOccur)
) {
fprintf(stderr, "%s: cannot combine --formatVersion 1 with --writePoolBundle or --usePoolBundle\n", argv[0]);
illegalArg = TRUE;
} else {
setFormatVersion(s[0] - '0');
}
}
if((options[JAVA_PACKAGE].doesOccur || options[BUNDLE_NAME].doesOccur) &&
!options[WRITE_JAVA].doesOccur) {
fprintf(stderr,
"%s error: command line argument --java-package or --bundle-name "
"without --write-java\n",
argv[0]);
illegalArg = TRUE;
}
if(options[VERSION].doesOccur) {
fprintf(stderr,
"%s version %s (ICU version %s).\n"
"%s\n",
argv[0], GENRB_VERSION, U_ICU_VERSION, U_COPYRIGHT_STRING);
if(!illegalArg) {
return U_ZERO_ERROR;
}
}
if(illegalArg || options[HELP1].doesOccur || options[HELP2].doesOccur) {
/*
* Broken into chunks because the C89 standard says the minimum
* required supported string length is 509 bytes.
*/
fprintf(stderr,
"Usage: %s [OPTIONS] [FILES]\n"
"\tReads the list of resource bundle source files and creates\n"
"\tbinary version of resource bundles (.res files)\n",
argv[0]);
fprintf(stderr,
"Options:\n"
"\t-h or -? or --help this usage text\n"
"\t-q or --quiet do not display warnings\n"
"\t-v or --verbose print extra information when processing files\n"
"\t-V or --version prints out version number and exits\n"
"\t-c or --copyright include copyright notice\n");
fprintf(stderr,
"\t-e or --encoding encoding of source files\n"
"\t-d of --destdir destination directory, followed by the path, defaults to %s\n"
"\t-s or --sourcedir source directory for files followed by path, defaults to %s\n"
"\t-i or --icudatadir directory for locating any needed intermediate data files,\n"
"\t followed by path, defaults to %s\n",
u_getDataDirectory(), u_getDataDirectory(), u_getDataDirectory());
fprintf(stderr,
"\t-j or --write-java write a Java ListResourceBundle for ICU4J, followed by optional encoding\n"
"\t defaults to ASCII and \\uXXXX format.\n"
"\t --java-package For --write-java: package name for writing the ListResourceBundle,\n"
"\t defaults to com.ibm.icu.impl.data\n");
fprintf(stderr,
"\t-b or --bundle-name For --write-java: root resource bundle name for writing the ListResourceBundle,\n"
"\t defaults to LocaleElements\n"
"\t-x or --write-xliff write an XLIFF file for the resource bundle. Followed by\n"
"\t an optional output file name.\n"
"\t-k or --strict use pedantic parsing of syntax\n"
/*added by Jing*/
"\t-l or --language for XLIFF: language code compliant with BCP 47.\n");
fprintf(stderr,
"\t-C or --noBinaryCollation do not generate binary collation image;\n"
"\t makes .res file smaller but collator instantiation much slower;\n"
"\t maintains ability to get tailoring rules\n"
"\t-R or --omitCollationRules do not include collation (tailoring) rules;\n"
"\t makes .res file smaller and maintains collator instantiation speed\n"
"\t but tailoring rules will not be available (they are rarely used)\n");
fprintf(stderr,
"\t --formatVersion write a .res file compatible with the requested formatVersion (single digit);\n"
"\t for example, --formatVersion 1\n");
fprintf(stderr,
"\t --writePoolBundle write a pool.res file with all of the keys of all input bundles\n"
"\t --usePoolBundle [path-to-pool.res] point to keys from the pool.res keys pool bundle if they are available there;\n"
"\t makes .res files smaller but dependent on the pool bundle\n"
"\t (--writePoolBundle and --usePoolBundle cannot be combined)\n");
return illegalArg ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
}
if(options[VERBOSE].doesOccur) {
setVerbose(TRUE);
}
if(options[QUIET].doesOccur) {
setShowWarning(FALSE);
}
if(options[STRICT].doesOccur) {
setStrict(TRUE);
}
if(options[COPYRIGHT].doesOccur){
setIncludeCopyright(TRUE);
}
if(options[SOURCEDIR].doesOccur) {
inputDir = options[SOURCEDIR].value;
}
if(options[DESTDIR].doesOccur) {
outputDir = options[DESTDIR].value;
}
if(options[ENCODING].doesOccur) {
encoding = options[ENCODING].value;
}
if(options[ICUDATADIR].doesOccur) {
u_setDataDirectory(options[ICUDATADIR].value);
}
/* Initialize ICU */
u_init(&status);
if (U_FAILURE(status) && status != U_FILE_ACCESS_ERROR) {
/* Note: u_init() will try to open ICU property data.
* failures here are expected when building ICU from scratch.
* ignore them.
*/
fprintf(stderr, "%s: can not initialize ICU. status = %s\n",
argv[0], u_errorName(status));
exit(1);
}
status = U_ZERO_ERROR;
if(options[WRITE_JAVA].doesOccur) {
write_java = TRUE;
outputEnc = options[WRITE_JAVA].value;
}
if(options[WRITE_XLIFF].doesOccur) {
write_xliff = TRUE;
if(options[WRITE_XLIFF].value != NULL){
xliffOutputFileName = options[WRITE_XLIFF].value;
}
}
initParser();
/*added by Jing*/
if(options[LANGUAGE].doesOccur) {
language = options[LANGUAGE].value;
}
LocalPointer<SRBRoot> newPoolBundle;
if(options[WRITE_POOL_BUNDLE].doesOccur) {
newPoolBundle.adoptInsteadAndCheckErrorCode(new SRBRoot(NULL, TRUE, status), status);
if(U_FAILURE(status)) {
fprintf(stderr, "unable to create an empty bundle for the pool keys: %s\n", u_errorName(status));
return status;
} else {
const char *poolResName = "pool.res";
char *nameWithoutSuffix = static_cast<char *>(uprv_malloc(uprv_strlen(poolResName) + 1));
if (nameWithoutSuffix == NULL) {
fprintf(stderr, "out of memory error\n");
return U_MEMORY_ALLOCATION_ERROR;
}
uprv_strcpy(nameWithoutSuffix, poolResName);
*uprv_strrchr(nameWithoutSuffix, '.') = 0;
newPoolBundle->fLocale = nameWithoutSuffix;
}
}
if(options[USE_POOL_BUNDLE].doesOccur) {
const char *poolResName = "pool.res";
FileStream *poolFile;
int32_t poolFileSize;
int32_t indexLength;
/*
* TODO: Consolidate inputDir/filename handling from main() and processFile()
* into a common function, and use it here as well.
* Try to create toolutil functions for dealing with dir/filenames and
* loading ICU data files without udata_open().
* Share code with icupkg?
* Also, make_res_filename() seems to be unused. Review and remove.
*/
CharString poolFileName;
if (options[USE_POOL_BUNDLE].value!=NULL) {
poolFileName.append(options[USE_POOL_BUNDLE].value, status);
} else if (inputDir) {
poolFileName.append(inputDir, status);
}
poolFileName.appendPathPart(poolResName, status);
if (U_FAILURE(status)) {
return status;
}
poolFile = T_FileStream_open(poolFileName.data(), "rb");
if (poolFile == NULL) {
fprintf(stderr, "unable to open pool bundle file %s\n", poolFileName.data());
return 1;
}
poolFileSize = T_FileStream_size(poolFile);
if (poolFileSize < 32) {
fprintf(stderr, "the pool bundle file %s is too small\n", poolFileName.data());
return 1;
}
poolBundle.fBytes = new uint8_t[(poolFileSize + 15) & ~15];
if (poolFileSize > 0 && poolBundle.fBytes == NULL) {
fprintf(stderr, "unable to allocate memory for the pool bundle file %s\n", poolFileName.data());
return U_MEMORY_ALLOCATION_ERROR;
}
UDataSwapper *ds;
const DataHeader *header;
int32_t bytesRead = T_FileStream_read(poolFile, poolBundle.fBytes, poolFileSize);
if (bytesRead != poolFileSize) {
fprintf(stderr, "unable to read the pool bundle file %s\n", poolFileName.data());
return 1;
}
/*
* Swap the pool bundle so that a single checked-in file can be used.
* The swapper functions also test that the data looks like
* a well-formed .res file.
*/
ds = udata_openSwapperForInputData(poolBundle.fBytes, bytesRead,
U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, &status);
if (U_FAILURE(status)) {
fprintf(stderr, "udata_openSwapperForInputData(pool bundle %s) failed: %s\n",
poolFileName.data(), u_errorName(status));
return status;
}
ures_swap(ds, poolBundle.fBytes, bytesRead, poolBundle.fBytes, &status);
udata_closeSwapper(ds);
if (U_FAILURE(status)) {
fprintf(stderr, "ures_swap(pool bundle %s) failed: %s\n",
poolFileName.data(), u_errorName(status));
return status;
}
header = (const DataHeader *)poolBundle.fBytes;
if (header->info.formatVersion[0] < 2) {
fprintf(stderr, "invalid format of pool bundle file %s\n", poolFileName.data());
return U_INVALID_FORMAT_ERROR;
}
const int32_t *pRoot = (const int32_t *)(
(const char *)header + header->dataHeader.headerSize);
poolBundle.fIndexes = pRoot + 1;
indexLength = poolBundle.fIndexes[URES_INDEX_LENGTH] & 0xff;
if (indexLength <= URES_INDEX_POOL_CHECKSUM) {
fprintf(stderr, "insufficient indexes[] in pool bundle file %s\n", poolFileName.data());
return U_INVALID_FORMAT_ERROR;
}
int32_t keysBottom = 1 + indexLength;
int32_t keysTop = poolBundle.fIndexes[URES_INDEX_KEYS_TOP];
poolBundle.fKeys = (const char *)(pRoot + keysBottom);
poolBundle.fKeysLength = (keysTop - keysBottom) * 4;
poolBundle.fChecksum = poolBundle.fIndexes[URES_INDEX_POOL_CHECKSUM];
for (i = 0; i < poolBundle.fKeysLength; ++i) {
if (poolBundle.fKeys[i] == 0) {
++poolBundle.fKeysCount;
}
}
// 16BitUnits[] begins with strings-v2.
// The strings-v2 may optionally be terminated by what looks like
// an explicit string length that exceeds the number of remaining 16-bit units.
int32_t stringUnitsLength = (poolBundle.fIndexes[URES_INDEX_16BIT_TOP] - keysTop) * 2;
if (stringUnitsLength >= 2 && getFormatVersion() >= 3) {
poolBundle.fStrings = new PseudoListResource(NULL, status);
if (poolBundle.fStrings == NULL) {
fprintf(stderr, "unable to allocate memory for the pool bundle strings %s\n",
poolFileName.data());
return U_MEMORY_ALLOCATION_ERROR;
}
// The PseudoListResource constructor call did not allocate further memory.
assert(U_SUCCESS(status));
const UChar *p = (const UChar *)(pRoot + keysTop);
int32_t remaining = stringUnitsLength;
do {
int32_t first = *p;
int8_t numCharsForLength;
int32_t length;
if (!U16_IS_TRAIL(first)) {
// NUL-terminated
numCharsForLength = 0;
for (length = 0;
length < remaining && p[length] != 0;
++length) {}
} else if (first < 0xdfef) {
numCharsForLength = 1;
length = first & 0x3ff;
} else if (first < 0xdfff && remaining >= 2) {
numCharsForLength = 2;
length = ((first - 0xdfef) << 16) | p[1];
} else if (first == 0xdfff && remaining >= 3) {
numCharsForLength = 3;
length = ((int32_t)p[1] << 16) | p[2];
} else {
break; // overrun
}
// Check for overrun before changing remaining,
// so that it is always accurate after the loop body.
if ((numCharsForLength + length) >= remaining ||
p[numCharsForLength + length] != 0) {
break; // overrun or explicitly terminated
}
int32_t poolStringIndex = stringUnitsLength - remaining;
// Maximum pool string index when suffix-sharing the last character.
int32_t maxStringIndex = poolStringIndex + numCharsForLength + length - 1;
if (maxStringIndex >= RES_MAX_OFFSET) {
// pool string index overrun
break;
}
p += numCharsForLength;
remaining -= numCharsForLength;
if (length != 0) {
StringResource *sr =
new StringResource(poolStringIndex, numCharsForLength,
p, length, status);
if (sr == NULL) {
fprintf(stderr, "unable to allocate memory for a pool bundle string %s\n",
poolFileName.data());
return U_MEMORY_ALLOCATION_ERROR;
}
poolBundle.fStrings->add(sr);
poolBundle.fStringIndexLimit = maxStringIndex + 1;
// The StringResource constructor did not allocate further memory.
assert(U_SUCCESS(status));
}
p += length + 1;
remaining -= length + 1;
} while (remaining > 0);
if (poolBundle.fStrings->fCount == 0) {
delete poolBundle.fStrings;
poolBundle.fStrings = NULL;
}
}
T_FileStream_close(poolFile);
setUsePoolBundle(TRUE);
if (isVerbose() && poolBundle.fStrings != NULL) {
printf("number of shared strings: %d\n", (int)poolBundle.fStrings->fCount);
int32_t length = poolBundle.fStringIndexLimit + 1; // incl. last NUL
printf("16-bit units for strings: %6d = %6d bytes\n",
(int)length, (int)length * 2);
}
}
if(!options[FORMAT_VERSION].doesOccur && getFormatVersion() == 3 &&
poolBundle.fStrings == NULL &&
!options[WRITE_POOL_BUNDLE].doesOccur) {
// If we just default to formatVersion 3
// but there are no pool bundle strings to share
// and we do not write a pool bundle,
// then write formatVersion 2 which is just as good.
setFormatVersion(2);
}
if(options[INCLUDE_UNIHAN_COLL].doesOccur) {
puts("genrb option --includeUnihanColl ignored: \n"
"CLDR 26/ICU 54 unihan data is small, except\n"
"the ucadata-unihan.icu version of the collation root data\n"
"is about 300kB larger than the ucadata-implicithan.icu version.");
}
if((argc-1)!=1) {
printf("genrb number of files: %d\n", argc - 1);
}
/* generate the binary files */
for(i = 1; i < argc; ++i) {
status = U_ZERO_ERROR;
arg = getLongPathname(argv[i]);
CharString theCurrentFileName;
if (inputDir) {
theCurrentFileName.append(inputDir, status);
}
theCurrentFileName.appendPathPart(arg, status);
if (U_FAILURE(status)) {
break;
}
gCurrentFileName = theCurrentFileName.data();
if (isVerbose()) {
printf("Processing file \"%s\"\n", theCurrentFileName.data());
}
processFile(arg, encoding, inputDir, outputDir, NULL,
newPoolBundle.getAlias(),
options[NO_BINARY_COLLATION].doesOccur, status);
}
poolBundle.close();
if(U_SUCCESS(status) && options[WRITE_POOL_BUNDLE].doesOccur) {
char outputFileName[256];
newPoolBundle->write(outputDir, NULL, outputFileName, sizeof(outputFileName), status);
if(U_FAILURE(status)) {
fprintf(stderr, "unable to write the pool bundle: %s\n", u_errorName(status));
}
}
u_cleanup();
/* Dont return warnings as a failure */
if (U_SUCCESS(status)) {
return 0;
}
return status;
}
U_CAPI void U_EXPORT2
createCommonDataFile(const char *destDir, const char *name, const char *entrypointName, const char *type, const char *source, const char *copyRight,
const char *dataFile, uint32_t max_size, UBool sourceTOC, UBool verbose, char *gencmnFileName) {
static char buffer[4096];
char *line;
char *linePtr;
char *s = NULL;
UErrorCode errorCode=U_ZERO_ERROR;
uint32_t i, fileOffset, basenameOffset, length, nread;
FileStream *in, *file;
line = (char *)uprv_malloc(sizeof(char) * LINE_BUFFER_SIZE);
if (line == NULL) {
fprintf(stderr, "gencmn: unable to allocate memory for line buffer of size %d\n", LINE_BUFFER_SIZE);
exit(U_MEMORY_ALLOCATION_ERROR);
}
linePtr = line;
maxSize = max_size;
if (destDir == NULL) {
destDir = u_getDataDirectory();
}
if (name == NULL) {
name = COMMON_DATA_NAME;
}
if (type == NULL) {
type = DATA_TYPE;
}
if (source == NULL) {
source = ".";
}
if (dataFile == NULL) {
in = T_FileStream_stdin();
} else {
in = T_FileStream_open(dataFile, "r");
if(in == NULL) {
fprintf(stderr, "gencmn: unable to open input file %s\n", dataFile);
exit(U_FILE_ACCESS_ERROR);
}
}
if (verbose) {
if(sourceTOC) {
printf("generating %s_%s.c (table of contents source file)\n", name, type);
} else {
printf("generating %s.%s (common data file with table of contents)\n", name, type);
}
}
/* read the list of files and get their lengths */
while((s != NULL && *s != 0) || (s=T_FileStream_readLine(in, (line=linePtr),
LINE_BUFFER_SIZE))!=NULL) {
/* remove trailing newline characters and parse space separated items */
if (s != NULL && *s != 0) {
line=s;
} else {
s=line;
}
while(*s!=0) {
if(*s==' ') {
*s=0;
++s;
break;
} else if(*s=='\r' || *s=='\n') {
*s=0;
break;
}
++s;
}
/* check for comment */
if (*line == '#') {
continue;
}
/* add the file */
#if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR)
{
char *t;
while((t = uprv_strchr(line,U_FILE_ALT_SEP_CHAR))) {
*t = U_FILE_SEP_CHAR;
}
}
#endif
addFile(getLongPathname(line), name, source, sourceTOC, verbose);
}
uprv_free(linePtr);
if(in!=T_FileStream_stdin()) {
T_FileStream_close(in);
}
if(fileCount==0) {
fprintf(stderr, "gencmn: no files listed in %s\n", dataFile == NULL ? "<stdin>" : dataFile);
return;
}
/* sort the files by basename */
qsort(files, fileCount, sizeof(File), compareFiles);
if(!sourceTOC) {
UNewDataMemory *out;
/* determine the offsets of all basenames and files in this common one */
basenameOffset=4+8*fileCount;
fileOffset=(basenameOffset+(basenameTotal+15))&~0xf;
for(i=0; i<fileCount; ++i) {
files[i].fileOffset=fileOffset;
fileOffset+=(files[i].fileSize+15)&~0xf;
files[i].basenameOffset=basenameOffset;
basenameOffset+=files[i].basenameLength;
}
/* create the output file */
out=udata_create(destDir, type, name,
&dataInfo,
copyRight == NULL ? U_COPYRIGHT_STRING : copyRight,
&errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "gencmn: udata_create(-d %s -n %s -t %s) failed - %s\n",
destDir, name, type,
u_errorName(errorCode));
exit(errorCode);
}
/* write the table of contents */
udata_write32(out, fileCount);
for(i=0; i<fileCount; ++i) {
udata_write32(out, files[i].basenameOffset);
udata_write32(out, files[i].fileOffset);
}
/* write the basenames */
for(i=0; i<fileCount; ++i) {
udata_writeString(out, files[i].basename, files[i].basenameLength);
}
length=4+8*fileCount+basenameTotal;
/* copy the files */
for(i=0; i<fileCount; ++i) {
/* pad to 16-align the next file */
length&=0xf;
if(length!=0) {
udata_writePadding(out, 16-length);
}
if (verbose) {
printf("adding %s (%ld byte%s)\n", files[i].pathname, (long)files[i].fileSize, files[i].fileSize == 1 ? "" : "s");
}
/* copy the next file */
file=T_FileStream_open(files[i].pathname, "rb");
if(file==NULL) {
fprintf(stderr, "gencmn: unable to open listed file %s\n", files[i].pathname);
exit(U_FILE_ACCESS_ERROR);
}
for(nread = 0;;) {
length=T_FileStream_read(file, buffer, sizeof(buffer));
if(length <= 0) {
break;
}
nread += length;
udata_writeBlock(out, buffer, length);
}
T_FileStream_close(file);
length=files[i].fileSize;
if (nread != files[i].fileSize) {
fprintf(stderr, "gencmn: unable to read %s properly (got %ld/%ld byte%s)\n", files[i].pathname, (long)nread, (long)files[i].fileSize, files[i].fileSize == 1 ? "" : "s");
exit(U_FILE_ACCESS_ERROR);
}
}
/* pad to 16-align the last file (cleaner, avoids growing .dat files in icuswap) */
length&=0xf;
if(length!=0) {
udata_writePadding(out, 16-length);
}
/* finish */
udata_finish(out, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "gencmn: udata_finish() failed - %s\n", u_errorName(errorCode));
exit(errorCode);
}
} else {
/* write a .c source file with the table of contents */
char *filename;
FileStream *out;
/* create the output filename */
filename=s=buffer;
uprv_strcpy(filename, destDir);
s=filename+uprv_strlen(filename);
if(s>filename && *(s-1)!=U_FILE_SEP_CHAR) {
*s++=U_FILE_SEP_CHAR;
}
uprv_strcpy(s, name);
if(*(type)!=0) {
s+=uprv_strlen(s);
*s++='_';
uprv_strcpy(s, type);
}
s+=uprv_strlen(s);
uprv_strcpy(s, ".c");
/* open the output file */
out=T_FileStream_open(filename, "w");
if (gencmnFileName != NULL) {
uprv_strcpy(gencmnFileName, filename);
}
if(out==NULL) {
fprintf(stderr, "gencmn: unable to open .c output file %s\n", filename);
exit(U_FILE_ACCESS_ERROR);
}
/* write the source file */
sprintf(buffer,
"/*\n"
" * ICU common data table of contents for %s.%s\n"
" * Automatically generated by icu/source/tools/gencmn/gencmn .\n"
" */\n\n"
"#include \"unicode/utypes.h\"\n"
"#include \"unicode/udata.h\"\n"
"\n"
"/* external symbol declarations for data (%d files) */\n",
name, type, fileCount);
T_FileStream_writeLine(out, buffer);
sprintf(buffer, "extern const char\n %s%s[]", symPrefix?symPrefix:"", files[0].pathname);
T_FileStream_writeLine(out, buffer);
for(i=1; i<fileCount; ++i) {
sprintf(buffer, ",\n %s%s[]", symPrefix?symPrefix:"", files[i].pathname);
T_FileStream_writeLine(out, buffer);
}
T_FileStream_writeLine(out, ";\n\n");
sprintf(
buffer,
"U_EXPORT struct {\n"
" uint16_t headerSize;\n"
" uint8_t magic1, magic2;\n"
" UDataInfo info;\n"
" char padding[%lu];\n"
" uint32_t count, reserved;\n"
" struct {\n"
" const char *name;\n"
" const void *data;\n"
" } toc[%lu];\n"
"} U_EXPORT2 %s_dat = {\n"
" 32, 0xda, 0x27, {\n"
" %lu, 0,\n"
" %u, %u, %u, 0,\n"
" {0x54, 0x6f, 0x43, 0x50},\n"
" {1, 0, 0, 0},\n"
" {0, 0, 0, 0}\n"
" },\n"
" \"\", %lu, 0, {\n",
(unsigned long)32-4-sizeof(UDataInfo),
(unsigned long)fileCount,
entrypointName,
(unsigned long)sizeof(UDataInfo),
U_IS_BIG_ENDIAN,
U_CHARSET_FAMILY,
U_SIZEOF_UCHAR,
(unsigned long)fileCount
);
T_FileStream_writeLine(out, buffer);
sprintf(buffer, " { \"%s\", %s%s }", files[0].basename, symPrefix?symPrefix:"", files[0].pathname);
T_FileStream_writeLine(out, buffer);
for(i=1; i<fileCount; ++i) {
sprintf(buffer, ",\n { \"%s\", %s%s }", files[i].basename, symPrefix?symPrefix:"", files[i].pathname);
T_FileStream_writeLine(out, buffer);
}
T_FileStream_writeLine(out, "\n }\n};\n");
T_FileStream_close(out);
uprv_free(symPrefix);
}
}
extern int
main(int argc, char* argv[]) {
UVersionInfo version;
Options moreOptions={ TRUE, FALSE, TRUE };
UErrorCode errorCode = U_ZERO_ERROR;
U_MAIN_INIT_ARGS(argc, argv);
/* Initialize ICU */
u_init(&errorCode);
if (U_FAILURE(errorCode) && errorCode != U_FILE_ACCESS_ERROR) {
/* Note: u_init() will try to open ICU property data.
* failures here are expected when building ICU from scratch.
* ignore them.
*/
fprintf(stderr, "%s: can not initialize ICU. errorCode = %s\n",
argv[0], u_errorName(errorCode));
exit(1);
}
/* preset then read command line options */
options[DESTDIR].value=u_getDataDirectory();
options[UNICODE].value="4.1";
argc=u_parseArgs(argc, argv, LENGTHOF(options), options);
/* error handling, printing usage message */
if(argc<0) {
fprintf(stderr,
"error in command line argument \"%s\"\n",
argv[-argc]);
} else if(argc<2) {
argc=-1;
}
if(argc<0 || options[HELP_H].doesOccur || options[HELP_QUESTION_MARK].doesOccur) {
/*
* Broken into chucks because the C89 standard says the minimum
* required supported string length is 509 bytes.
*/
fprintf(stderr,
"Usage: %s [-1[+|-]] [-v[+|-]] [-c[+|-]] [filename_ud [filename_na]]\n"
"\n"
"Read the UnicodeData.txt file and \n"
"create a binary file " DATA_NAME "." DATA_TYPE " with the character names\n"
"\n"
"\tfilename_ud absolute path/filename for the UnicodeData.txt file\n"
"\t (default: standard input)\n"
"\tfilename_na absolute path/filename for the NameAliases.txt file\n"
"\t (default: no name aliases)\n"
"\n",
argv[0]);
fprintf(stderr,
"Options:\n"
"\t-h or -? or --help this usage text\n"
"\t-v or --verbose verbose output\n"
"\t-q or --quiet no output\n"
"\t-c or --copyright include a copyright notice\n"
"\t-d or --destdir destination directory, followed by the path\n"
"\t-u or --unicode Unicode version, followed by the version like 3.0.0\n");
fprintf(stderr,
"\t-1 or --unicode1-names store Unicode 1.0 character names\n"
"\t --no-iso-comments do not store ISO comments\n"
"\t --only-iso-comments write ucomment.icu with only ISO comments\n");
return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
}
/* get the options values */
beVerbose=options[VERBOSE].doesOccur;
beQuiet=options[QUIET].doesOccur;
haveCopyright=options[COPYRIGHT].doesOccur;
moreOptions.store10Names=options[UNICODE1_NAMES].doesOccur;
moreOptions.storeISOComments=!options[NO_ISO_COMMENTS].doesOccur;
if(options[ONLY_ISO_COMMENTS].doesOccur) {
moreOptions.storeNames=moreOptions.store10Names=FALSE;
moreOptions.storeISOComments=TRUE;
}
/* set the Unicode version */
u_versionFromString(version, options[UNICODE].value);
uprv_memcpy(dataInfo.dataVersion, version, 4);
ucdVersion=findUnicodeVersion(version);
init();
if(argc>=3) {
parseNameAliases(argv[2], &moreOptions);
}
parseDB(argc>=2 ? argv[1] : "-", &moreOptions);
compress();
generateData(options[DESTDIR].value, &moreOptions);
u_cleanup();
return 0;
}
static UDataMemory *
doOpenChoice(const char *path, const char *type, const char *name,
UDataMemoryIsAcceptable *isAcceptable, void *context,
UErrorCode *pErrorCode)
{
UDataMemory *retVal = NULL;
TinyString tocEntryName; /* entry name in tree format. ex: 'icudt28b/coll/ar.res' */
TinyString tocEntryPath; /* entry name in path format. ex: 'icudt28b\\coll\\ar.res' */
TinyString pkgName;
TinyString treeName;
#if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR) /* '/' vs '\' */
TinyString altSepPath;
#endif
const char *dataPath;
int32_t tocEntrySuffixIndex;
const char *tocEntryPathSuffix;
UErrorCode subErrorCode=U_ZERO_ERROR;
const char *treeChar;
UBool isICUData = FALSE;
/* Is this path ICU data? */
if(path == NULL ||
!strcmp(path, U_ICUDATA_ALIAS) || /* "ICUDATA" */
!uprv_strncmp(path, U_ICUDATA_NAME U_TREE_SEPARATOR_STRING, /* "icudt26e-" */
uprv_strlen(U_ICUDATA_NAME U_TREE_SEPARATOR_STRING)) ||
!uprv_strncmp(path, U_ICUDATA_ALIAS U_TREE_SEPARATOR_STRING, /* "ICUDATA-" */
uprv_strlen(U_ICUDATA_ALIAS U_TREE_SEPARATOR_STRING))) {
isICUData = TRUE;
}
#if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR) /* Windows: try "foo\bar" and "foo/bar" */
/* remap from alternate path char to the main one */
TinyString_init(&altSepPath);
if(path) {
char *p;
if((p=uprv_strchr(path,U_FILE_ALT_SEP_CHAR))) {
TinyString_append(&altSepPath, path);
while((p=uprv_strchr(altSepPath.s,U_FILE_ALT_SEP_CHAR))) {
*p = U_FILE_SEP_CHAR;
}
#if defined (UDATA_DEBUG)
fprintf(stderr, "Changed path from [%s] to [%s]\n", path, altSepPath.s);
#endif
path = altSepPath.s;
}
}
#endif
TinyString_init(&tocEntryName);
TinyString_init(&tocEntryPath);
TinyString_init(&pkgName);
TinyString_init(&treeName);
/* ======= Set up strings */
if(path==NULL) {
TinyString_append(&pkgName, U_ICUDATA_NAME);
} else {
const char *pkg;
const char *first;
pkg = uprv_strrchr(path, U_FILE_SEP_CHAR);
first = uprv_strchr(path, U_FILE_SEP_CHAR);
if(uprv_pathIsAbsolute(path) || (pkg != first)) { /* more than one slash in the path- not a tree name */
/* see if this is an /absolute/path/to/package path */
if(pkg) {
TinyString_append(&pkgName, pkg+1);
} else {
TinyString_append(&pkgName, path);
}
} else {
treeChar = uprv_strchr(path, U_TREE_SEPARATOR);
if(treeChar) {
TinyString_append(&treeName, treeChar+1); /* following '-' */
if(isICUData) {
TinyString_append(&pkgName, U_ICUDATA_NAME);
} else {
TinyString_appendn(&pkgName, path, (int32_t)(treeChar-path));
if (first == NULL) {
/*
This user data has no path, but there is a tree name.
Look up the correct path from the data cache later.
*/
path = pkgName.s;
}
}
} else {
if(isICUData) {
TinyString_append(&pkgName, U_ICUDATA_NAME);
} else {
TinyString_append(&pkgName, path);
}
}
}
}
#ifdef UDATA_DEBUG
fprintf(stderr, " P=%s T=%s\n", pkgName.s, treeName.s);
#endif
/* setting up the entry name and file name
* Make up a full name by appending the type to the supplied
* name, assuming that a type was supplied.
*/
/* prepend the package */
TinyString_append(&tocEntryName, pkgName.s);
TinyString_append(&tocEntryPath, pkgName.s);
tocEntrySuffixIndex = tocEntryName.length;
if(treeName.s[0]) {
TinyString_append(&tocEntryName, U_TREE_ENTRY_SEP_STRING);
TinyString_append(&tocEntryName, treeName.s);
TinyString_append(&tocEntryPath, U_FILE_SEP_STRING);
TinyString_append(&tocEntryPath, treeName.s);
}
TinyString_append(&tocEntryName, U_TREE_ENTRY_SEP_STRING);
TinyString_append(&tocEntryPath, U_FILE_SEP_STRING);
TinyString_append(&tocEntryName, name);
TinyString_append(&tocEntryPath, name);
if(type!=NULL && *type!=0) {
TinyString_append(&tocEntryName, ".");
TinyString_append(&tocEntryName, type);
TinyString_append(&tocEntryPath, ".");
TinyString_append(&tocEntryPath, type);
}
tocEntryPathSuffix = tocEntryPath.s+tocEntrySuffixIndex; /* suffix starts here */
#ifdef UDATA_DEBUG
fprintf(stderr, " tocEntryName = %s\n", tocEntryName.s);
fprintf(stderr, " tocEntryPath = %s\n", tocEntryName.s);
#endif
if(path == NULL) {
path = COMMON_DATA_NAME; /* "icudt26e" */
}
/************************ Begin loop looking for ind. files ***************/
#ifdef UDATA_DEBUG
fprintf(stderr, "IND: inBasename = %s, pkg=%s\n", "(n/a)", packageNameFromPath(path));
#endif
/* End of dealing with a null basename */
dataPath = u_getDataDirectory();
/**** COMMON PACKAGE - only if packages are first. */
if(gDataFileAccess == UDATA_PACKAGES_FIRST) {
#ifdef UDATA_DEBUG
fprintf(stderr, "Trying packages (UDATA_PACKAGES_FIRST)\n");
#endif
/* #2 */
retVal = doLoadFromCommonData(isICUData,
pkgName.s, dataPath, tocEntryPathSuffix, tocEntryName.s,
path, type, name, isAcceptable, context, &subErrorCode, pErrorCode);
if((retVal != NULL) || U_FAILURE(*pErrorCode)) {
goto commonReturn;
}
}
/**** INDIVIDUAL FILES */
if((gDataFileAccess==UDATA_PACKAGES_FIRST) ||
(gDataFileAccess==UDATA_FILES_FIRST)) {
#ifdef UDATA_DEBUG
fprintf(stderr, "Trying individual files\n");
#endif
/* Check to make sure that there is a dataPath to iterate over */
if ((dataPath && *dataPath) || !isICUData) {
retVal = doLoadFromIndividualFiles(pkgName.s, dataPath, tocEntryPathSuffix,
path, type, name, isAcceptable, context, &subErrorCode, pErrorCode);
if((retVal != NULL) || U_FAILURE(*pErrorCode)) {
goto commonReturn;
}
}
}
/**** COMMON PACKAGE */
if((gDataFileAccess==UDATA_ONLY_PACKAGES) ||
(gDataFileAccess==UDATA_FILES_FIRST)) {
#ifdef UDATA_DEBUG
fprintf(stderr, "Trying packages (UDATA_ONLY_PACKAGES || UDATA_FILES_FIRST)\n");
#endif
retVal = doLoadFromCommonData(isICUData,
pkgName.s, dataPath, tocEntryPathSuffix, tocEntryName.s,
path, type, name, isAcceptable, context, &subErrorCode, pErrorCode);
if((retVal != NULL) || U_FAILURE(*pErrorCode)) {
goto commonReturn;
}
}
/* Load from DLL. If we haven't attempted package load, we also haven't had any chance to
try a DLL (static or setCommonData/etc) load.
If we ever have a "UDATA_ONLY_FILES", add it to the or list here. */
if(gDataFileAccess==UDATA_NO_FILES) {
#ifdef UDATA_DEBUG
fprintf(stderr, "Trying common data (UDATA_NO_FILES)\n");
#endif
retVal = doLoadFromCommonData(isICUData,
pkgName.s, "", tocEntryPathSuffix, tocEntryName.s,
path, type, name, isAcceptable, context, &subErrorCode, pErrorCode);
if((retVal != NULL) || U_FAILURE(*pErrorCode)) {
goto commonReturn;
}
}
/* data not found */
if(U_SUCCESS(*pErrorCode)) {
if(U_SUCCESS(subErrorCode)) {
/* file not found */
*pErrorCode=U_FILE_ACCESS_ERROR;
} else {
/* entry point not found or rejected */
*pErrorCode=subErrorCode;
}
}
commonReturn:
TinyString_dt(&tocEntryName);
TinyString_dt(&tocEntryPath);
TinyString_dt(&pkgName);
TinyString_dt(&treeName);
#if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR)
TinyString_dt(&altSepPath);
#endif
return retVal;
}
int genpname::MMain(int argc, char* argv[])
{
int32_t i, j;
UErrorCode status = U_ZERO_ERROR;
u_init(&status);
if (U_FAILURE(status) && status != U_FILE_ACCESS_ERROR) {
fprintf(stderr, "Error: u_init returned %s\n", u_errorName(status));
status = U_ZERO_ERROR;
}
/* preset then read command line options */
options[3].value=u_getDataDirectory();
argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
/* error handling, printing usage message */
if (argc<0) {
fprintf(stderr,
"error in command line argument \"%s\"\n",
argv[-argc]);
}
debug = options[5].doesOccur ? (*options[5].value - '0') : 0;
if (argc!=1 || options[0].doesOccur || options[1].doesOccur ||
debug < 0 || debug > 9) {
fprintf(stderr,
"usage: %s [-options]\n"
"\tcreate " PNAME_DATA_NAME "." PNAME_DATA_TYPE "\n"
"options:\n"
"\t-h or -? or --help this usage text\n"
"\t-v or --verbose turn on verbose output\n"
"\t-c or --copyright include a copyright notice\n"
"\t-d or --destdir destination directory, followed by the path\n"
"\t-D or --debug 0..9 emit debugging messages (if > 0)\n",
argv[0]);
return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
}
/* get the options values */
useCopyright=options[2].doesOccur;
verbose = options[4].doesOccur;
// ------------------------------------------------------------
// Do not sort the string table, instead keep it in data.h order.
// This simplifies data swapping and testing thereof because the string
// table itself need not be sorted during swapping.
// The NameToEnum sorter sorts each such map's string offsets instead.
if (debug>1) {
printf("String pool: %d\n", (int)STRING_COUNT);
for (i=0; i<STRING_COUNT; ++i) {
if (i != 0) {
printf(", ");
}
printf("%s (%d)", STRING_TABLE[i].str, (int)STRING_TABLE[i].index);
}
printf("\n\n");
}
// ------------------------------------------------------------
// Create top-level property indices
PropertyArrayList props(PROPERTY, PROPERTY_COUNT);
int32_t propNameCount;
NameToEnumEntry* propName = createNameIndex(props, propNameCount);
EnumToNameGroupEntry* propEnum = createEnumIndex(props);
// ------------------------------------------------------------
// Create indices for the value list for each enumerated property
// This will have more entries than we need...
EnumToValueEntry* enumToValue = MALLOC(EnumToValueEntry, PROPERTY_COUNT);
int32_t enumToValue_count = 0;
for (i=0, j=0; i<PROPERTY_COUNT; ++i) {
if (PROPERTY[i].valueCount == 0) continue;
AliasArrayList values(PROPERTY[i].valueList,
PROPERTY[i].valueCount);
enumToValue[j].enumValue = PROPERTY[i].enumValue;
enumToValue[j].enumToName = createEnumIndex(values);
enumToValue[j].enumToName_count = PROPERTY[i].valueCount;
enumToValue[j].nameToEnum = createNameIndex(values,
enumToValue[j].nameToEnum_count);
++j;
}
enumToValue_count = j;
uprv_sortArray(enumToValue, enumToValue_count, sizeof(enumToValue[0]),
compareEnumToValueEntry, NULL, FALSE, &status);
// ------------------------------------------------------------
// Build PropertyAliases layout in memory
Builder builder(debug);
builder.buildTopLevelProperties(propName,
propNameCount,
propEnum,
PROPERTY_COUNT);
builder.buildValues(enumToValue,
enumToValue_count);
builder.buildStringPool(STRING_TABLE,
STRING_COUNT,
NAME_GROUP,
NAME_GROUP_COUNT);
builder.fixup();
////////////////////////////////////////////////////////////
// Write the output file
////////////////////////////////////////////////////////////
int32_t wlen = writeDataFile(options[3].value, builder);
if (verbose) {
fprintf(stdout, "Output file: %s.%s, %ld bytes\n",
U_ICUDATA_NAME "_" PNAME_DATA_NAME, PNAME_DATA_TYPE, (long)wlen);
}
return 0; // success
}
int
main(int argc,
char* argv[])
{
UErrorCode status = U_ZERO_ERROR;
const char *arg = NULL;
const char *outputDir = NULL; /* NULL = no output directory, use current */
const char *inputDir = NULL;
const char *encoding = "";
int i;
U_MAIN_INIT_ARGS(argc, argv);
argc = u_parseArgs(argc, argv, (int32_t)(sizeof(options)/sizeof(options[0])), options);
/* error handling, printing usage message */
if(argc<0) {
fprintf(stderr, "%s: error in command line argument \"%s\"\n", argv[0], argv[-argc]);
} else if(argc<2) {
argc = -1;
}
if(options[VERSION].doesOccur) {
fprintf(stderr,
"%s version %s (ICU version %s).\n"
"%s\n",
argv[0], GENRB_VERSION, U_ICU_VERSION, U_COPYRIGHT_STRING);
return U_ZERO_ERROR;
}
if(argc<0 || options[HELP1].doesOccur || options[HELP2].doesOccur) {
/*
* Broken into chucks because the C89 standard says the minimum
* required supported string length is 509 bytes.
*/
fprintf(stderr,
"Usage: %s [OPTIONS] [FILES]\n"
"\tReads the list of resource bundle source files and creates\n"
"\tbinary version of reosurce bundles (.res files)\n",
argv[0]);
fprintf(stderr,
"Options:\n"
"\t-h or -? or --help this usage text\n"
"\t-q or --quiet do not display warnings\n"
"\t-v or --verbose print extra information when processing files\n"
"\t-V or --version prints out version number and exits\n"
"\t-c or --copyright include copyright notice\n");
fprintf(stderr,
"\t-e or --encoding encoding of source files\n"
"\t-d of --destdir destination directory, followed by the path, defaults to %s\n"
"\t-s or --sourcedir source directory for files followed by path, defaults to %s\n"
"\t-i or --icudatadir directory for locating any needed intermediate data files,\n"
"\t followed by path, defaults to %s\n",
u_getDataDirectory(), u_getDataDirectory(), u_getDataDirectory());
fprintf(stderr,
"\t-j or --write-java write a Java ListResourceBundle for ICU4J, followed by optional encoding\n"
"\t defaults to ASCII and \\uXXXX format.\n"
"\t-p or --package-name For ICU4J: package name for writing the ListResourceBundle for ICU4J,\n"
"\t defaults to com.ibm.icu.impl.data\n");
fprintf(stderr,
"\t-b or --bundle-name bundle name for writing the ListResourceBundle for ICU4J,\n"
"\t defaults to LocaleElements\n"
"\t-x or --write-xliff write a XLIFF file for the resource bundle. Followed by an optional output file name.\n"
"\t-k or --strict use pedantic parsing of syntax\n"
/*added by Jing*/
"\t-l or --language For XLIFF: language code compliant with ISO 639.\n");
return argc < 0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
}
if(options[VERBOSE].doesOccur) {
setVerbose(TRUE);
}
if(options[QUIET].doesOccur) {
setShowWarning(FALSE);
}
if(options[STRICT].doesOccur) {
setStrict(TRUE);
}
if(options[COPYRIGHT].doesOccur){
setIncludeCopyright(TRUE);
}
if(options[SOURCEDIR].doesOccur) {
inputDir = options[SOURCEDIR].value;
}
if(options[DESTDIR].doesOccur) {
outputDir = options[DESTDIR].value;
}
if(options[PACKAGE_NAME].doesOccur) {
gPackageName = options[PACKAGE_NAME].value;
if(!strcmp(gPackageName, "ICUDATA"))
{
gPackageName = U_ICUDATA_NAME;
}
if(gPackageName[0] == 0)
{
gPackageName = NULL;
}
}
if(options[ENCODING].doesOccur) {
encoding = options[ENCODING].value;
}
if(options[ICUDATADIR].doesOccur) {
u_setDataDirectory(options[ICUDATADIR].value);
}
/* Initialize ICU */
u_init(&status);
if (U_FAILURE(status) && status != U_FILE_ACCESS_ERROR) {
/* Note: u_init() will try to open ICU property data.
* failures here are expected when building ICU from scratch.
* ignore them.
*/
fprintf(stderr, "%s: can not initialize ICU. status = %s\n",
argv[0], u_errorName(status));
exit(1);
}
status = U_ZERO_ERROR;
if(options[WRITE_JAVA].doesOccur) {
write_java = TRUE;
outputEnc = options[WRITE_JAVA].value;
}
if(options[BUNDLE_NAME].doesOccur) {
bundleName = options[BUNDLE_NAME].value;
}
if(options[WRITE_XLIFF].doesOccur) {
write_xliff = TRUE;
if(options[WRITE_XLIFF].value != NULL){
xliffOutputFileName = options[WRITE_XLIFF].value;
}
}
if(options[NO_BINARY_COLLATION].doesOccur) {
initParser(FALSE);
} else {
initParser(TRUE);
}
/*added by Jing*/
if(options[LANGUAGE].doesOccur) {
language = options[LANGUAGE].value;
}
/* generate the binary files */
for(i = 1; i < argc; ++i) {
status = U_ZERO_ERROR;
arg = getLongPathname(argv[i]);
if (inputDir) {
uprv_strcpy(theCurrentFileName, inputDir);
uprv_strcat(theCurrentFileName, U_FILE_SEP_STRING);
} else {
*theCurrentFileName = 0;
}
uprv_strcat(theCurrentFileName, arg);
if (isVerbose()) {
printf("Processing file \"%s\"\n", theCurrentFileName);
}
processFile(arg, encoding, inputDir, outputDir, gPackageName, &status);
}
/* Dont return warnings as a failure */
if (! U_FAILURE(status)) {
return 0;
}
return status;
}
extern int
main(int argc, char* argv[]) {
#if !UCONFIG_NO_IDNA
char* filename = NULL;
#endif
const char *srcDir=NULL, *destDir=NULL, *icuUniDataDir=NULL;
const char *bundleName=NULL, *inputFileName = NULL;
char *basename=NULL;
int32_t sprepOptions = 0;
UErrorCode errorCode=U_ZERO_ERROR;
U_MAIN_INIT_ARGS(argc, argv);
/* preset then read command line options */
options[DESTDIR].value=u_getDataDirectory();
options[SOURCEDIR].value="";
options[UNICODE_VERSION].value="0"; /* don't assume the unicode version */
options[BUNDLE_NAME].value = DATA_NAME;
options[NORMALIZE].value = "";
argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
/* error handling, printing usage message */
if(argc<0) {
fprintf(stderr,
"error in command line argument \"%s\"\n",
argv[-argc]);
}
if(argc<0 || options[HELP].doesOccur || options[HELP_QUESTION_MARK].doesOccur) {
return printHelp(argc, argv);
}
/* get the options values */
beVerbose=options[VERBOSE].doesOccur;
haveCopyright=options[COPYRIGHT].doesOccur;
srcDir=options[SOURCEDIR].value;
destDir=options[DESTDIR].value;
bundleName = options[BUNDLE_NAME].value;
if(options[NORMALIZE].doesOccur) {
icuUniDataDir = options[NORMALIZE].value;
} else {
icuUniDataDir = options[NORM_CORRECTION_DIR].value;
}
if(argc<2) {
/* print the help message */
return printHelp(argc, argv);
} else {
inputFileName = argv[1];
}
if(!options[UNICODE_VERSION].doesOccur){
return printHelp(argc, argv);
}
if(options[ICUDATADIR].doesOccur) {
u_setDataDirectory(options[ICUDATADIR].value);
}
#if UCONFIG_NO_IDNA
fprintf(stderr,
"gensprep writes dummy " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE
" because UCONFIG_NO_IDNA is set, \n"
"see icu/source/common/unicode/uconfig.h\n");
generateData(destDir, bundleName);
#else
setUnicodeVersion(options[UNICODE_VERSION].value);
filename = (char* ) uprv_malloc(uprv_strlen(srcDir) + 300); /* hopefully this should be enough */
/* prepare the filename beginning with the source dir */
if(uprv_strchr(srcDir,U_FILE_SEP_CHAR) == NULL && uprv_strchr(srcDir,U_FILE_ALT_SEP_CHAR) == NULL){
filename[0] = '.';
filename[1] = U_FILE_SEP_CHAR;
uprv_strcpy(filename+2,srcDir);
}else{
uprv_strcpy(filename, srcDir);
}
basename=filename+uprv_strlen(filename);
if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
*basename++=U_FILE_SEP_CHAR;
}
/* initialize */
init();
/* process the file */
uprv_strcpy(basename,inputFileName);
parseMappings(filename,FALSE, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "Could not open file %s for reading. Error: %s \n", filename, u_errorName(errorCode));
return errorCode;
}
if(options[NORMALIZE].doesOccur){ /* this option might be set by @normalize;; in the source file */
/* set up directory for NormalizationCorrections.txt */
uprv_strcpy(filename,icuUniDataDir);
basename=filename+uprv_strlen(filename);
if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
*basename++=U_FILE_SEP_CHAR;
}
*basename++=U_FILE_SEP_CHAR;
uprv_strcpy(basename,NORM_CORRECTIONS_FILE_NAME);
parseNormalizationCorrections(filename,&errorCode);
if(U_FAILURE(errorCode)){
fprintf(stderr,"Could not open file %s for reading \n", filename);
return errorCode;
}
sprepOptions |= _SPREP_NORMALIZATION_ON;
}
if(options[CHECK_BIDI].doesOccur){ /* this option might be set by @check-bidi;; in the source file */
sprepOptions |= _SPREP_CHECK_BIDI_ON;
}
setOptions(sprepOptions);
/* process parsed data */
if(U_SUCCESS(errorCode)) {
/* write the data file */
generateData(destDir, bundleName);
cleanUpData();
}
uprv_free(filename);
u_cleanup();
#endif
return errorCode;
}
int main(int argc, char* argv[])
{
ConvData data;
UErrorCode err = U_ZERO_ERROR, localError;
char outFileName[UCNV_MAX_FULL_FILE_NAME_LENGTH];
const char* destdir, *arg;
size_t destdirlen;
char* dot = NULL, *outBasename;
char cnvName[UCNV_MAX_FULL_FILE_NAME_LENGTH];
char cnvNameWithPkg[UCNV_MAX_FULL_FILE_NAME_LENGTH];
UVersionInfo icuVersion;
UBool printFilename;
err = U_ZERO_ERROR;
U_MAIN_INIT_ARGS(argc, argv);
/* Set up the ICU version number */
u_getVersion(icuVersion);
uprv_memcpy(&dataInfo.dataVersion, &icuVersion, sizeof(UVersionInfo));
/* preset then read command line options */
options[OPT_DESTDIR].value=u_getDataDirectory();
argc=u_parseArgs(argc, argv, LENGTHOF(options), options);
/* error handling, printing usage message */
if(argc<0) {
fprintf(stderr,
"error in command line argument \"%s\"\n",
argv[-argc]);
} else if(argc<2) {
argc=-1;
}
if(argc<0 || options[OPT_HELP_H].doesOccur || options[OPT_HELP_QUESTION_MARK].doesOccur) {
FILE *stdfile=argc<0 ? stderr : stdout;
fprintf(stdfile,
"usage: %s [-options] files...\n"
"\tread .ucm codepage mapping files and write .cnv files\n"
"options:\n"
"\t-h or -? or --help this usage text\n"
"\t-V or --version show a version message\n"
"\t-c or --copyright include a copyright notice\n"
"\t-d or --destdir destination directory, followed by the path\n"
"\t-v or --verbose Turn on verbose output\n",
argv[0]);
fprintf(stdfile,
"\t --small Generate smaller .cnv files. They will be\n"
"\t significantly smaller but may not be compatible with\n"
"\t older versions of ICU and will require heap memory\n"
"\t allocation when loaded.\n"
"\t --ignore-siso-check Use SI/SO other than 0xf/0xe.\n");
return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
}
if(options[OPT_VERSION].doesOccur) {
printf("makeconv version %hu.%hu, ICU tool to read .ucm codepage mapping files and write .cnv files\n",
dataInfo.formatVersion[0], dataInfo.formatVersion[1]);
printf("%s\n", U_COPYRIGHT_STRING);
exit(0);
}
/* get the options values */
haveCopyright = options[OPT_COPYRIGHT].doesOccur;
destdir = options[OPT_DESTDIR].value;
VERBOSE = options[OPT_VERBOSE].doesOccur;
SMALL = options[OPT_SMALL].doesOccur;
if (options[OPT_IGNORE_SISO_CHECK].doesOccur) {
IGNORE_SISO_CHECK = TRUE;
}
if (destdir != NULL && *destdir != 0) {
uprv_strcpy(outFileName, destdir);
destdirlen = uprv_strlen(destdir);
outBasename = outFileName + destdirlen;
if (*(outBasename - 1) != U_FILE_SEP_CHAR) {
*outBasename++ = U_FILE_SEP_CHAR;
++destdirlen;
}
} else {
destdirlen = 0;
outBasename = outFileName;
}
#if DEBUG
{
int i;
printf("makeconv: processing %d files...\n", argc - 1);
for(i=1; i<argc; ++i) {
printf("%s ", argv[i]);
}
printf("\n");
fflush(stdout);
}
#endif
err = U_ZERO_ERROR;
printFilename = (UBool) (argc > 2 || VERBOSE);
for (++argv; --argc; ++argv)
{
arg = getLongPathname(*argv);
/* Check for potential buffer overflow */
if(strlen(arg) > UCNV_MAX_FULL_FILE_NAME_LENGTH)
{
fprintf(stderr, "%s\n", u_errorName(U_BUFFER_OVERFLOW_ERROR));
return U_BUFFER_OVERFLOW_ERROR;
}
/*produces the right destination path for display*/
if (destdirlen != 0)
{
const char *basename;
/* find the last file sepator */
basename = findBasename(arg);
uprv_strcpy(outBasename, basename);
}
else
{
uprv_strcpy(outFileName, arg);
}
/*removes the extension if any is found*/
dot = uprv_strrchr(outBasename, '.');
if (dot)
{
*dot = '\0';
}
/* the basename without extension is the converter name */
uprv_strcpy(cnvName, outBasename);
/*Adds the target extension*/
uprv_strcat(outBasename, CONVERTER_FILE_EXTENSION);
#if DEBUG
printf("makeconv: processing %s ...\n", arg);
fflush(stdout);
#endif
localError = U_ZERO_ERROR;
initConvData(&data);
createConverter(&data, arg, &localError);
if (U_FAILURE(localError))
{
/* if an error is found, print out an error msg and keep going */
fprintf(stderr, "Error creating converter for \"%s\" file for \"%s\" (%s)\n", outFileName, arg,
u_errorName(localError));
if(U_SUCCESS(err)) {
err = localError;
}
}
else
{
/* Insure the static data name matches the file name */
/* Changed to ignore directory and only compare base name
LDH 1/2/08*/
char *p;
p = strrchr(cnvName, U_FILE_SEP_CHAR); /* Find last file separator */
if(p == NULL) /* OK, try alternate */
{
p = strrchr(cnvName, U_FILE_ALT_SEP_CHAR);
if(p == NULL)
{
p=cnvName; /* If no separators, no problem */
}
}
else
{
p++; /* If found separtor, don't include it in compare */
}
if(uprv_stricmp(p,data.staticData.name))
{
fprintf(stderr, "Warning: %s%s claims to be '%s'\n",
cnvName, CONVERTER_FILE_EXTENSION,
data.staticData.name);
}
uprv_strcpy((char*)data.staticData.name, cnvName);
if(!uprv_isInvariantString((char*)data.staticData.name, -1)) {
fprintf(stderr,
"Error: A converter name must contain only invariant characters.\n"
"%s is not a valid converter name.\n",
data.staticData.name);
if(U_SUCCESS(err)) {
err = U_INVALID_TABLE_FORMAT;
}
}
uprv_strcpy(cnvNameWithPkg, cnvName);
localError = U_ZERO_ERROR;
writeConverterData(&data, cnvNameWithPkg, destdir, &localError);
if(U_FAILURE(localError))
{
/* if an error is found, print out an error msg and keep going*/
fprintf(stderr, "Error writing \"%s\" file for \"%s\" (%s)\n", outFileName, arg,
u_errorName(localError));
if(U_SUCCESS(err)) {
err = localError;
}
}
else if (printFilename)
{
puts(outBasename);
}
}
fflush(stdout);
fflush(stderr);
cleanupConvData(&data);
}
return err;
}
void cmd_version(UBool noLoad)
{
UVersionInfo icu;
char str[200];
printf("<ICUINFO>\n");
printf("International Components for Unicode for C/C++\n");
printf("%s\n", U_COPYRIGHT_STRING);
printf("Compiled-Version: %s\n", U_ICU_VERSION);
u_getVersion(icu);
u_versionToString(icu, str);
printf("Runtime-Version: %s\n", str);
printf("Compiled-Unicode-Version: %s\n", U_UNICODE_VERSION);
u_getUnicodeVersion(icu);
u_versionToString(icu, str);
printf("Runtime-Unicode-Version: %s\n", U_UNICODE_VERSION);
printf("Platform: %s\n", getPlatform());
#if defined(U_BUILD)
printf("Build: %s\n", U_BUILD);
#if defined(U_HOST)
if(strcmp(U_BUILD,U_HOST)) {
printf("Host: %s\n", U_HOST);
}
#endif
#endif
#if defined(U_CC)
printf("C compiler: %s\n", U_CC);
#endif
#if defined(U_CXX)
printf("C++ compiler: %s\n", U_CXX);
#endif
#if defined(CYGWINMSVC)
printf("Cygwin: CYGWINMSVC\n");
#endif
printf("ICUDATA: %s\n", U_ICUDATA_NAME);
do_init();
printf("Data Directory: %s\n", u_getDataDirectory());
printf("ICU Initialization returned: %s\n", u_errorName(initStatus));
printf( "Default locale: %s\n", uloc_getDefault());
{
UErrorCode subStatus = U_ZERO_ERROR;
ulocdata_getCLDRVersion(icu, &subStatus);
if(U_SUCCESS(subStatus)) {
u_versionToString(icu, str);
printf("CLDR-Version: %s\n", str);
} else {
printf("CLDR-Version: %s\n", u_errorName(subStatus));
}
}
#if !UCONFIG_NO_CONVERSION
if(noLoad == FALSE)
{
printf("Default converter: %s\n", ucnv_getDefaultName());
}
#endif
#if !UCONFIG_NO_FORMATTING
{
UChar buf[100];
char buf2[100];
UErrorCode subsubStatus= U_ZERO_ERROR;
int32_t len;
len = ucal_getDefaultTimeZone(buf, 100, &subsubStatus);
if(U_SUCCESS(subsubStatus)&&len>0) {
u_UCharsToChars(buf, buf2, len+1);
printf("Default TZ: %s\n", buf2);
} else {
printf("Default TZ: %s\n", u_errorName(subsubStatus));
}
}
{
UErrorCode subStatus = U_ZERO_ERROR;
const char *tzVer = ucal_getTZDataVersion(&subStatus);
if(U_FAILURE(subStatus)) {
tzVer = u_errorName(subStatus);
}
printf("TZ data version: %s\n", tzVer);
}
#endif
#if U_ENABLE_DYLOAD
const char *pluginFile = uplug_getPluginFile();
printf("Plugin file is: %s\n", (pluginFile&&*pluginFile)?pluginFile:"(not set. try setting ICU_PLUGINS to a directory.)");
#else
fprintf(stderr, "Dynamic Loading: is disabled. No plugins will be loaded at start-up.\n");
#endif
printf("</ICUINFO>\n\n");
}
extern int
main(int argc, char* argv[]) {
int i, n;
char pathBuf[512];
FileStream *in;
UNewDataMemory *out;
UErrorCode errorCode=U_ZERO_ERROR;
U_MAIN_INIT_ARGS(argc, argv);
/* preset then read command line options */
options[DESTDIR].value=options[SOURCEDIR].value=u_getDataDirectory();
argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
/* error handling, printing usage message */
if(argc<0) {
fprintf(stderr,
"error in command line argument \"%s\"\n",
argv[-argc]);
}
if(argc<0 || options[HELP1].doesOccur || options[HELP2].doesOccur) {
fprintf(stderr,
"usage: %s [-options] [convrtrs.txt]\n"
"\tread convrtrs.txt and create " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE "\n"
"options:\n"
"\t-h or -? or --help this usage text\n"
"\t-v or --verbose prints out extra information about the alias table\n"
"\t-q or --quiet do not display warnings and progress\n"
"\t-c or --copyright include a copyright notice\n"
"\t-d or --destdir destination directory, followed by the path\n"
"\t-s or --sourcedir source directory, followed by the path\n",
argv[0]);
return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
}
if(options[VERBOSE].doesOccur) {
verbose = TRUE;
}
if(options[QUIET].doesOccur) {
quiet = TRUE;
}
if(argc>=2) {
path=argv[1];
} else {
path=options[SOURCEDIR].value;
if(path!=NULL && *path!=0) {
char *end;
uprv_strcpy(pathBuf, path);
end = uprv_strchr(pathBuf, 0);
if(*(end-1)!=U_FILE_SEP_CHAR) {
*(end++)=U_FILE_SEP_CHAR;
}
uprv_strcpy(end, "convrtrs.txt");
path=pathBuf;
} else {
path = "convrtrs.txt";
}
}
uprv_memset(stringStore, 0, sizeof(stringStore));
uprv_memset(tagStore, 0, sizeof(tagStore));
uprv_memset(converters, 0, sizeof(converters));
uprv_memset(tags, 0, sizeof(tags));
uprv_memset(aliasLists, 0, sizeof(aliasLists));
uprv_memset(knownAliases, 0, sizeof(aliasLists));
in=T_FileStream_open(path, "r");
if(in==NULL) {
fprintf(stderr, "gencnval: unable to open input file %s\n", path);
exit(U_FILE_ACCESS_ERROR);
}
parseFile(in);
T_FileStream_close(in);
/* create the output file */
out=udata_create(options[DESTDIR].value, DATA_TYPE, DATA_NAME, &dataInfo,
options[COPYRIGHT].doesOccur ? U_COPYRIGHT_STRING : NULL, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "gencnval: unable to open output file - error %s\n", u_errorName(errorCode));
exit(errorCode);
}
/* write the table of aliases based on a tag/converter name combination */
writeAliasTable(out);
/* finish */
udata_finish(out, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "gencnval: error finishing output file - %s\n", u_errorName(errorCode));
exit(errorCode);
}
/* clean up tags */
for (i = 0; i < MAX_TAG_COUNT; i++) {
for (n = 0; n < MAX_CONV_COUNT; n++) {
if (tags[i].aliasList[n].aliases!=NULL) {
uprv_free(tags[i].aliasList[n].aliases);
}
}
}
return 0;
}
extern int
main(int argc, char* argv[]) {
UVersionInfo version;
UBool store10Names=FALSE;
UErrorCode errorCode = U_ZERO_ERROR;
U_MAIN_INIT_ARGS(argc, argv);
/* Initialize ICU */
u_init(&errorCode);
if (U_FAILURE(errorCode) && errorCode != U_FILE_ACCESS_ERROR) {
/* Note: u_init() will try to open ICU property data.
* failures here are expected when building ICU from scratch.
* ignore them.
*/
fprintf(stderr, "%s: can not initialize ICU. errorCode = %s\n",
argv[0], u_errorName(errorCode));
exit(1);
}
/* preset then read command line options */
options[5].value=u_getDataDirectory();
options[6].value="4.1";
argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
/* error handling, printing usage message */
if(argc<0) {
fprintf(stderr,
"error in command line argument \"%s\"\n",
argv[-argc]);
} else if(argc<2) {
argc=-1;
}
if(argc<0 || options[0].doesOccur || options[1].doesOccur) {
/*
* Broken into chucks because the C89 standard says the minimum
* required supported string length is 509 bytes.
*/
fprintf(stderr,
"Usage: %s [-1[+|-]] [-v[+|-]] [-c[+|-]] filename\n"
"\n"
"Read the UnicodeData.txt file and \n"
"create a binary file " DATA_NAME "." DATA_TYPE " with the character names\n"
"\n"
"\tfilename absolute path/filename for the Unicode database text file\n"
"\t\t(default: standard input)\n"
"\n",
argv[0]);
fprintf(stderr,
"Options:\n"
"\t-h or -? or --help this usage text\n"
"\t-v or --verbose verbose output\n"
"\t-q or --quiet no output\n"
"\t-c or --copyright include a copyright notice\n"
"\t-d or --destdir destination directory, followed by the path\n"
"\t-u or --unicode Unicode version, followed by the version like 3.0.0\n"
"\t-1 or --unicode1-names store Unicode 1.0 character names\n");
return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
}
/* get the options values */
beVerbose=options[2].doesOccur;
beQuiet=options[3].doesOccur;
haveCopyright=options[4].doesOccur;
store10Names=options[7].doesOccur;
/* set the Unicode version */
u_versionFromString(version, options[6].value);
uprv_memcpy(dataInfo.dataVersion, version, 4);
ucdVersion=findUnicodeVersion(version);
init();
parseDB(argc>=2 ? argv[1] : "-", store10Names);
compress();
generateData(options[5].value);
u_cleanup();
return 0;
}
FileStream *
NormalizerConformanceTest::openNormalizationTestFile(const char *filename) {
char unidataPath[2000];
const char *folder;
FileStream *input;
UErrorCode errorCode;
// look inside ICU_DATA first
folder=u_getDataDirectory();
if(folder!=NULL) {
strcpy(unidataPath, folder);
strcat(unidataPath, "unidata" U_FILE_SEP_STRING);
strcat(unidataPath, filename);
input=T_FileStream_open(unidataPath, "rb");
if(input!=NULL) {
return input;
}
}
// find icu/source/data/unidata relative to the test data
errorCode=U_ZERO_ERROR;
folder=loadTestData(errorCode);
if(U_SUCCESS(errorCode)) {
strcpy(unidataPath, folder);
strcat(unidataPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".."
U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".."
U_FILE_SEP_STRING "data" U_FILE_SEP_STRING "unidata" U_FILE_SEP_STRING);
strcat(unidataPath, filename);
input=T_FileStream_open(unidataPath, "rb");
if(input!=NULL) {
return input;
}
}
// look in icu/source/test/testdata/out/build
errorCode=U_ZERO_ERROR;
folder=loadTestData(errorCode);
if(U_SUCCESS(errorCode)) {
strcpy(unidataPath, folder);
strcat(unidataPath, U_FILE_SEP_STRING);
strcat(unidataPath, filename);
input=T_FileStream_open(unidataPath, "rb");
if(input!=NULL) {
return input;
}
}
// look in icu/source/test/testdata
errorCode=U_ZERO_ERROR;
folder=loadTestData(errorCode);
if(U_SUCCESS(errorCode)) {
strcpy(unidataPath, folder);
strcat(unidataPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING);
strcat(unidataPath, filename);
input=T_FileStream_open(unidataPath, "rb");
if(input!=NULL) {
return input;
}
}
// find icu/source/data/unidata relative to U_TOPSRCDIR
#if defined(U_TOPSRCDIR)
strcpy(unidataPath, U_TOPSRCDIR U_FILE_SEP_STRING "data" U_FILE_SEP_STRING "unidata" U_FILE_SEP_STRING);
strcat(unidataPath, filename);
input=T_FileStream_open(unidataPath, "rb");
if(input!=NULL) {
return input;
}
strcpy(unidataPath, U_TOPSRCDIR U_FILE_SEP_STRING "test" U_FILE_SEP_STRING "testdata" U_FILE_SEP_STRING);
strcat(unidataPath, filename);
input=T_FileStream_open(unidataPath, "rb");
if(input!=NULL) {
return input;
}
#endif
errln("Failed to open %s", filename);
return NULL;
}
extern int
main(int argc, char* argv[]) {
#if !UCONFIG_NO_NORMALIZATION
char filename[300];
#endif
const char *srcDir=NULL, *destDir=NULL, *suffix=NULL;
char *basename=NULL;
UErrorCode errorCode=U_ZERO_ERROR;
U_MAIN_INIT_ARGS(argc, argv);
/* preset then read command line options */
options[4].value=u_getDataDirectory();
options[5].value="";
options[6].value="3.0.0";
argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
/* error handling, printing usage message */
if(argc<0) {
fprintf(stderr,
"error in command line argument \"%s\"\n",
argv[-argc]);
}
if(argc<0 || options[0].doesOccur || options[1].doesOccur) {
/*
* Broken into chucks because the C89 standard says the minimum
* required supported string length is 509 bytes.
*/
fprintf(stderr,
"Usage: %s [-options] [suffix]\n"
"\n"
"Read the UnicodeData.txt file and other Unicode properties files and\n"
"create a binary file " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE " with the normalization data\n"
"\n",
argv[0]);
fprintf(stderr,
"Options:\n"
"\t-h or -? or --help this usage text\n"
"\t-v or --verbose verbose output\n"
"\t-c or --copyright include a copyright notice\n"
"\t-u or --unicode Unicode version, followed by the version like 3.0.0\n");
fprintf(stderr,
"\t-d or --destdir destination directory, followed by the path\n"
"\t-s or --sourcedir source directory, followed by the path\n"
"\tsuffix suffix that is to be appended with a '-'\n"
"\t to the source file basenames before opening;\n"
"\t 'gennorm new' will read UnicodeData-new.txt etc.\n");
return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
}
/* get the options values */
beVerbose=options[2].doesOccur;
haveCopyright=options[3].doesOccur;
srcDir=options[5].value;
destDir=options[4].value;
if(argc>=2) {
suffix=argv[1];
} else {
suffix=NULL;
}
#if UCONFIG_NO_NORMALIZATION
fprintf(stderr,
"gennorm writes a dummy " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE
" because UCONFIG_NO_NORMALIZATION is set, \n"
"see icu/source/common/unicode/uconfig.h\n");
generateData(destDir);
#else
setUnicodeVersion(options[6].value);
/* prepare the filename beginning with the source dir */
uprv_strcpy(filename, srcDir);
basename=filename+uprv_strlen(filename);
if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
*basename++=U_FILE_SEP_CHAR;
}
/* initialize */
init();
/* process DerivedNormalizationProps.txt (name changed for Unicode 3.2, to <=31 characters) */
if(suffix==NULL) {
uprv_strcpy(basename, "DerivedNormalizationProps.txt");
} else {
uprv_strcpy(basename, "DerivedNormalizationProps");
basename[30]='-';
uprv_strcpy(basename+31, suffix);
uprv_strcat(basename+31, ".txt");
}
parseDerivedNormalizationProperties(filename, &errorCode, FALSE);
if(U_FAILURE(errorCode)) {
/* can be only U_FILE_ACCESS_ERROR - try filename from before Unicode 3.2 */
if(suffix==NULL) {
uprv_strcpy(basename, "DerivedNormalizationProperties.txt");
} else {
uprv_strcpy(basename, "DerivedNormalizationProperties");
basename[30]='-';
uprv_strcpy(basename+31, suffix);
uprv_strcat(basename+31, ".txt");
}
parseDerivedNormalizationProperties(filename, &errorCode, TRUE);
}
/* process UnicodeData.txt */
if(suffix==NULL) {
uprv_strcpy(basename, "UnicodeData.txt");
} else {
uprv_strcpy(basename, "UnicodeData");
basename[11]='-';
uprv_strcpy(basename+12, suffix);
uprv_strcat(basename+12, ".txt");
}
parseDB(filename, &errorCode);
/* process parsed data */
if(U_SUCCESS(errorCode)) {
processData();
/* write the properties data file */
generateData(destDir);
cleanUpData();
}
#endif
return errorCode;
}
static void udata_pathiter_init(UDataPathIterator *iter, const char *path, const char *pkg,
const char *item, const char *suffix, UBool doCheckLastFour)
{
#ifdef UDATA_DEBUG
fprintf(stderr, "SUFFIX1=%s PATH=%s\n", suffix, path);
#endif
/** Path **/
if(path == NULL) {
iter->path = u_getDataDirectory();
} else {
iter->path = path;
}
/** Package **/
if(pkg == NULL) {
iter->packageStubLen = 0;
iter->packageStub=iter->packageStubBuf;
iter->packageStub[0] = 0;
} else {
if(uprv_strlen(pkg) + 2 > U_DATA_PATHITER_BUFSIZ) {
iter->packageStub = uprv_malloc(uprv_strlen(pkg)+2);
/* Check for null pointer. */
if (iter->packageStub == NULL) {
return;
}
} else {
iter->packageStub = iter->packageStubBuf;
}
iter->packageStub[0] = U_FILE_SEP_CHAR;
uprv_strcpy(iter->packageStub+1, pkg);
iter->packageStubLen = (int32_t)uprv_strlen(iter->packageStub);
#ifdef UDATA_DEBUG
fprintf(stderr, "STUB=%s [%d]\n", iter->packageStub, iter->packageStubLen);
#endif
}
/** Item **/
iter->basename = findBasename(item);
iter->basenameLen = (int32_t)uprv_strlen(iter->basename);
/** Item path **/
iter->itemPath = iter->itemPathBuf;
if(iter->basename == item) {
iter->itemPath[0] = 0;
iter->nextPath = iter->path;
} else {
int32_t itemPathLen = (int32_t)(iter->basename-item);
if (itemPathLen >= U_DATA_PATHITER_BUFSIZ) {
char *t = (char *)uprv_malloc(itemPathLen+1);
if (t != NULL) {
iter->itemPath = t;
} else {
/* Malloc failed. Ignore the itemPath. */
itemPathLen = 0;
}
}
uprv_strncpy(iter->itemPath, item, itemPathLen);
iter->itemPath[itemPathLen]=0;
iter->nextPath = iter->itemPath;
}
#ifdef UDATA_DEBUG
fprintf(stderr, "SUFFIX=%s [%p]\n", suffix, suffix);
#endif
/** Suffix **/
if(suffix != NULL) {
iter->suffix = suffix;
} else {
iter->suffix = "";
}
iter->checkLastFour = doCheckLastFour;
/* pathBuffer will hold the output path strings returned by the this iterator
* Get an upper bound of possible string size, and make sure that the buffer
* is big enough (sum of length of each piece, 2 extra delimiters, + trailing NULL) */
{
int32_t maxPathLen = (int32_t)(uprv_strlen(iter->path) + uprv_strlen(item) + uprv_strlen(iter->suffix) + iter->packageStubLen + 3);
iter->pathBuffer = iter->pathBufferA;
if (maxPathLen >= U_DATA_PATHITER_BUFSIZ) {
iter->pathBuffer = (char *)uprv_malloc(maxPathLen);
if (iter->pathBuffer == NULL) {
iter->pathBuffer = iter->pathBufferA;
iter->path = "";
}
}
}
#ifdef UDATA_DEBUG
fprintf(stderr, "%p: init %s -> [path=%s], [base=%s], [suff=%s], [itempath=%s], [nextpath=%s], [checklast4=%s]\n",
iter,
item,
iter->path,
iter->basename,
iter->suffix,
iter->itemPath,
iter->nextPath,
iter->checkLastFour?"TRUE":"false");
#endif
}
static UDataMemory *
openCommonData(const char *path, /* Path from OpenChoice? */
UBool isICUData, /* ICU Data true if path == NULL */
UErrorCode *pErrorCode)
{
UDataMemory tData;
UDataPathIterator iter;
const char *pathBuffer;
const char *inBasename;
if (U_FAILURE(*pErrorCode)) {
return NULL;
}
UDataMemory_init(&tData);
/* ??????? TODO revisit this */
if (isICUData) {
/* "mini-cache" for common ICU data */
if(gCommonICUData != NULL) {
return gCommonICUData;
}
tData.pHeader = &U_ICUDATA_ENTRY_POINT.hdr;
udata_checkCommonData(&tData, pErrorCode);
setCommonICUData(&tData, NULL, FALSE, pErrorCode);
return gCommonICUData;
}
/* request is NOT for ICU Data. */
/* Find the base name portion of the supplied path. */
/* inBasename will be left pointing somewhere within the original path string. */
inBasename = findBasename(path);
#ifdef UDATA_DEBUG
fprintf(stderr, "inBasename = %s\n", inBasename);
#endif
if(*inBasename==0) {
/* no basename. This will happen if the original path was a directory name, */
/* like "a/b/c/". (Fallback to separate files will still work.) */
#ifdef UDATA_DEBUG
fprintf(stderr, "ocd: no basename in %s, bailing.\n", path);
#endif
*pErrorCode=U_FILE_ACCESS_ERROR;
return NULL;
}
/* Is the requested common data file already open and cached? */
/* Note that the cache is keyed by the base name only. The rest of the path, */
/* if any, is not considered. */
{
UDataMemory *dataToReturn = udata_findCachedData(inBasename);
if (dataToReturn != NULL) {
return dataToReturn;
}
}
/* Requested item is not in the cache.
* Hunt it down, trying all the path locations
*/
udata_pathiter_init(&iter, u_getDataDirectory(), inBasename, path, ".dat", TRUE);
while((UDataMemory_isLoaded(&tData)==FALSE) &&
(pathBuffer = udata_pathiter_next(&iter)) != NULL)
{
#ifdef UDATA_DEBUG
fprintf(stderr, "ocd: trying path %s - ", pathBuffer);
#endif
uprv_mapFile(&tData, pathBuffer);
#ifdef UDATA_DEBUG
fprintf(stderr, "%s\n", UDataMemory_isLoaded(&tData)?"LOADED":"not loaded");
#endif
}
udata_pathiter_dt(&iter); /* Note: this call may invalidate "pathBuffer" */
#if defined(OS390_STUBDATA) && defined(OS390BATCH)
if (!UDataMemory_isLoaded(&tData)) {
char ourPathBuffer[1024];
/* One more chance, for extendCommonData() */
uprv_strncpy(ourPathBuffer, path, 1019);
ourPathBuffer[1019]=0;
uprv_strcat(ourPathBuffer, ".dat");
uprv_mapFile(&tData, ourPathBuffer);
}
#endif
if (!UDataMemory_isLoaded(&tData)) {
/* no common data */
*pErrorCode=U_FILE_ACCESS_ERROR;
return NULL;
}
/* we have mapped a file, check its header */
udata_checkCommonData(&tData, pErrorCode);
/* Cache the UDataMemory struct for this .dat file,
* so we won't need to hunt it down and map it again next time
* something is needed from it. */
return udata_cacheDataItem(inBasename, &tData, pErrorCode);
}
extern int
main(int argc, char* argv[]) {
static char buffer[4096];
char line[512];
FileStream *in, *file;
char *s;
UErrorCode errorCode=U_ZERO_ERROR;
uint32_t i, fileOffset, basenameOffset, length, nread;
UBool sourceTOC, verbose;
const char *entrypointName = NULL;
U_MAIN_INIT_ARGS(argc, argv);
/* preset then read command line options */
options[4].value=u_getDataDirectory();
options[6].value=COMMON_DATA_NAME;
options[7].value=DATA_TYPE;
options[10].value=".";
argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
/* error handling, printing usage message */
if(argc<0) {
fprintf(stderr,
"error in command line argument \"%s\"\n",
argv[-argc]);
} else if(argc<2) {
argc=-1;
}
if(argc<0 || options[0].doesOccur || options[1].doesOccur) {
FILE *where = argc < 0 ? stderr : stdout;
/*
* Broken into chucks because the C89 standard says the minimum
* required supported string length is 509 bytes.
*/
fprintf(where,
"%csage: %s [ -h, -?, --help ] [ -v, --verbose ] [ -c, --copyright ] [ -C, --comment comment ] [ -d, --destdir dir ] [ -n, --name filename ] [ -t, --type filetype ] [ -S, --source tocfile ] [ -e, --entrypoint name ] maxsize listfile\n", argc < 0 ? 'u' : 'U', *argv);
if (options[0].doesOccur || options[1].doesOccur) {
fprintf(where, "\n"
"Read the list file (default: standard input) and create a common data\n"
"file from specified files. Omit any files larger than maxsize, if maxsize > 0.\n");
fprintf(where, "\n"
"Options:\n"
"\t-h, -?, --help this usage text\n"
"\t-v, --verbose verbose output\n"
"\t-c, --copyright include the ICU copyright notice\n"
"\t-C, --comment comment include a comment string\n"
"\t-d, --destdir dir destination directory\n");
fprintf(where,
"\t-n, --name filename output filename, without .type extension\n"
"\t (default: " COMMON_DATA_NAME ")\n"
"\t-t, --type filetype type of the destination file\n"
"\t (default: \"" DATA_TYPE "\")\n"
"\t-S, --source tocfile write a .c source file with the table of\n"
"\t contents\n"
"\t-e, --entrypoint name override the c entrypoint name\n"
"\t (default: \"<name>_<type>\")\n");
}
return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
}
sourceTOC=options[8].doesOccur;
verbose = options[2].doesOccur;
maxSize=(uint32_t)uprv_strtoul(argv[1], NULL, 0);
if(argc==2) {
in=T_FileStream_stdin();
} else {
in=T_FileStream_open(argv[2], "r");
if(in==NULL) {
fprintf(stderr, "gencmn: unable to open input file %s\n", argv[2]);
exit(U_FILE_ACCESS_ERROR);
}
}
if (verbose) {
if(sourceTOC) {
printf("generating %s_%s.c (table of contents source file)\n", options[6].value, options[7].value);
} else {
printf("generating %s.%s (common data file with table of contents)\n", options[6].value, options[7].value);
}
}
/* read the list of files and get their lengths */
while(T_FileStream_readLine(in, line, sizeof(line))!=NULL) {
/* remove trailing newline characters */
s=line;
while(*s!=0) {
if(*s=='\r' || *s=='\n') {
*s=0;
break;
}
++s;
}
/* check for comment */
if (*line == '#') {
continue;
}
/* add the file */
#if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR)
{
char *t;
while((t = uprv_strchr(line,U_FILE_ALT_SEP_CHAR))) {
*t = U_FILE_SEP_CHAR;
}
}
#endif
addFile(getLongPathname(line), sourceTOC, verbose);
}
if(in!=T_FileStream_stdin()) {
T_FileStream_close(in);
}
if(fileCount==0) {
fprintf(stderr, "gencmn: no files listed in %s\n", argc==2 ? "<stdin>" : argv[2]);
return 0;
}
/* sort the files by basename */
qsort(files, fileCount, sizeof(File), compareFiles);
if(!sourceTOC) {
UNewDataMemory *out;
/* determine the offsets of all basenames and files in this common one */
basenameOffset=4+8*fileCount;
fileOffset=(basenameOffset+(basenameTotal+15))&~0xf;
for(i=0; i<fileCount; ++i) {
files[i].fileOffset=fileOffset;
fileOffset+=(files[i].fileSize+15)&~0xf;
files[i].basenameOffset=basenameOffset;
basenameOffset+=files[i].basenameLength;
}
/* create the output file */
out=udata_create(options[4].value, options[7].value, options[6].value,
&dataInfo,
options[3].doesOccur ? U_COPYRIGHT_STRING : options[5].value,
&errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "gencmn: udata_create(-d %s -n %s -t %s) failed - %s\n",
options[4].value, options[6].value, options[7].value,
u_errorName(errorCode));
exit(errorCode);
}
/* write the table of contents */
udata_write32(out, fileCount);
for(i=0; i<fileCount; ++i) {
udata_write32(out, files[i].basenameOffset);
udata_write32(out, files[i].fileOffset);
}
/* write the basenames */
for(i=0; i<fileCount; ++i) {
udata_writeString(out, files[i].basename, files[i].basenameLength);
}
length=4+8*fileCount+basenameTotal;
/* copy the files */
for(i=0; i<fileCount; ++i) {
/* pad to 16-align the next file */
length&=0xf;
if(length!=0) {
udata_writePadding(out, 16-length);
}
if (verbose) {
printf("adding %s (%ld byte%s)\n", files[i].pathname, (long)files[i].fileSize, files[i].fileSize == 1 ? "" : "s");
}
/* copy the next file */
file=T_FileStream_open(files[i].pathname, "rb");
if(file==NULL) {
fprintf(stderr, "gencmn: unable to open listed file %s\n", files[i].pathname);
exit(U_FILE_ACCESS_ERROR);
}
for(nread = 0;;) {
length=T_FileStream_read(file, buffer, sizeof(buffer));
if(length <= 0) {
break;
}
nread += length;
udata_writeBlock(out, buffer, length);
}
T_FileStream_close(file);
length=files[i].fileSize;
if (nread != files[i].fileSize) {
fprintf(stderr, "gencmn: unable to read %s properly (got %ld/%ld byte%s)\n", files[i].pathname, (long)nread, (long)files[i].fileSize, files[i].fileSize == 1 ? "" : "s");
exit(U_FILE_ACCESS_ERROR);
}
}
/* pad to 16-align the last file (cleaner, avoids growing .dat files in icuswap) */
length&=0xf;
if(length!=0) {
udata_writePadding(out, 16-length);
}
/* finish */
udata_finish(out, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "gencmn: udata_finish() failed - %s\n", u_errorName(errorCode));
exit(errorCode);
}
} else {
/* write a .c source file with the table of contents */
char *filename;
FileStream *out;
/* create the output filename */
filename=s=buffer;
uprv_strcpy(filename, options[4].value);
s=filename+uprv_strlen(filename);
if(s>filename && *(s-1)!=U_FILE_SEP_CHAR) {
*s++=U_FILE_SEP_CHAR;
}
uprv_strcpy(s, options[6].value);
if(*(options[7].value)!=0) {
s+=uprv_strlen(s);
*s++='_';
uprv_strcpy(s, options[7].value);
}
s+=uprv_strlen(s);
uprv_strcpy(s, ".c");
/* open the output file */
out=T_FileStream_open(filename, "w");
if(out==NULL) {
fprintf(stderr, "gencmn: unable to open .c output file %s\n", filename);
exit(U_FILE_ACCESS_ERROR);
}
/* If an entrypoint is specified, use it. */
if(options[9].doesOccur) {
entrypointName = options[9].value;
} else {
entrypointName = options[6].value;
}
/* write the source file */
sprintf(buffer,
"/*\n"
" * ICU common data table of contents for %s.%s ,\n"
" * Automatically generated by icu/source/tools/gencmn/gencmn .\n"
" */\n\n"
"#include \"unicode/utypes.h\"\n"
"#include \"unicode/udata.h\"\n"
"\n"
"/* external symbol declarations for data */\n",
options[6].value, options[7].value);
T_FileStream_writeLine(out, buffer);
sprintf(buffer, "extern const char\n %s%s[]", symPrefix?symPrefix:"", files[0].pathname);
T_FileStream_writeLine(out, buffer);
for(i=1; i<fileCount; ++i) {
sprintf(buffer, ",\n %s%s[]", symPrefix?symPrefix:"", files[i].pathname);
T_FileStream_writeLine(out, buffer);
}
T_FileStream_writeLine(out, ";\n\n");
sprintf(
buffer,
"U_EXPORT struct {\n"
" uint16_t headerSize;\n"
" uint8_t magic1, magic2;\n"
" UDataInfo info;\n"
" char padding[%lu];\n"
" uint32_t count, reserved;\n"
" struct {\n"
" const char *name;\n"
" const void *data;\n"
" } toc[%lu];\n"
"} U_EXPORT2 %s_dat = {\n"
" 32, 0xda, 0x27, {\n"
" %lu, 0,\n"
" %u, %u, %u, 0,\n"
" {0x54, 0x6f, 0x43, 0x50},\n"
" {1, 0, 0, 0},\n"
" {0, 0, 0, 0}\n"
" },\n"
" \"\", %lu, 0, {\n",
(unsigned long)32-4-sizeof(UDataInfo),
(unsigned long)fileCount,
entrypointName,
(unsigned long)sizeof(UDataInfo),
U_IS_BIG_ENDIAN,
U_CHARSET_FAMILY,
U_SIZEOF_UCHAR,
(unsigned long)fileCount
);
T_FileStream_writeLine(out, buffer);
sprintf(buffer, " { \"%s\", %s%s }", files[0].basename, symPrefix?symPrefix:"", files[0].pathname);
T_FileStream_writeLine(out, buffer);
for(i=1; i<fileCount; ++i) {
sprintf(buffer, ",\n { \"%s\", %s%s }", files[i].basename, symPrefix?symPrefix:"", files[i].pathname);
T_FileStream_writeLine(out, buffer);
}
T_FileStream_writeLine(out, "\n }\n};\n");
T_FileStream_close(out);
uprv_free(symPrefix);
}
return 0;
}
