static int32_t pkg_installFileMode(const char *installDir, const char *srcDir, const char *fileListName) {
int32_t result = 0;
char cmd[SMALL_BUFFER_MAX_SIZE] = "";
if (!T_FileStream_file_exists(installDir)) {
UErrorCode status = U_ZERO_ERROR;
uprv_mkdir(installDir, &status);
if (U_FAILURE(status)) {
fprintf(stderr, "Error creating installation directory: %s\n", installDir);
return -1;
}
}
#ifndef U_WINDOWS_WITH_MSVC
char buffer[SMALL_BUFFER_MAX_SIZE] = "";
FileStream *f = T_FileStream_open(fileListName, "r");
if (f != NULL) {
for(;;) {
if (T_FileStream_readLine(f, buffer, SMALL_BUFFER_MAX_SIZE) != NULL) {
/* Remove new line character. */
buffer[uprv_strlen(buffer)-1] = 0;
sprintf(cmd, "%s %s%s%s %s%s%s",
pkgDataFlags[INSTALL_CMD],
srcDir, PKGDATA_FILE_SEP_STRING, buffer,
installDir, PKGDATA_FILE_SEP_STRING, buffer);
result = system(cmd);
if (result != 0) {
fprintf(stderr, "Failed to install data file with command: %s\n", cmd);
break;
}
} else {
if (!T_FileStream_eof(f)) {
fprintf(stderr, "Failed to read line from file: %s\n", fileListName);
result = -1;
}
break;
}
}
T_FileStream_close(f);
} else {
result = -1;
fprintf(stderr, "Unable to open list file: %s\n", fileListName);
}
#else
sprintf(cmd, "%s %s %s %s", WIN_INSTALL_CMD, srcDir, installDir, WIN_INSTALL_CMD_FLAGS);
result = system(cmd);
if (result != 0) {
fprintf(stderr, "Failed to install data file with command: %s\n", cmd);
}
#endif
return result;
}
U_CAPI void U_EXPORT2
ucm_readTable(UCMFile *ucm, FileStream* convFile,
UBool forBase, UCMStates *baseStates,
UErrorCode *pErrorCode) {
char line[500];
char *end;
UBool isOK;
if(U_FAILURE(*pErrorCode)) {
return;
}
isOK=TRUE;
for(;;) {
/* read the next line */
if(!T_FileStream_readLine(convFile, line, sizeof(line))) {
fprintf(stderr, "incomplete charmap section\n");
isOK=FALSE;
break;
}
/* remove CR LF */
end=uprv_strchr(line, 0);
while(line<end && (*(end-1)=='\r' || *(end-1)=='\n')) {
--end;
}
*end=0;
/* ignore empty and comment lines */
if(line[0]==0 || line[0]=='#') {
continue;
}
/* stop at the end of the mapping table */
if(0==uprv_strcmp(line, "END CHARMAP")) {
break;
}
isOK&=ucm_addMappingFromLine(ucm, line, forBase, baseStates);
}
if(!isOK) {
*pErrorCode=U_INVALID_TABLE_FORMAT;
}
}
void NormalizerConformanceTest::TestConformance(FileStream *input, int32_t options) {
enum { BUF_SIZE = 1024 };
char lineBuf[BUF_SIZE];
UnicodeString fields[FIELD_COUNT];
UErrorCode status = U_ZERO_ERROR;
int32_t passCount = 0;
int32_t failCount = 0;
UChar32 c;
if(input==NULL) {
return;
}
// UnicodeSet for all code points that are not mentioned in NormalizationTest.txt
UnicodeSet other(0, 0x10ffff);
int32_t count, countMoreCases = UPRV_LENGTHOF(moreCases);
for (count = 1;;++count) {
if (!T_FileStream_eof(input)) {
T_FileStream_readLine(input, lineBuf, (int32_t)sizeof(lineBuf));
} else {
// once NormalizationTest.txt is finished, use moreCases[]
if(count > countMoreCases) {
count = 0;
} else if(count == countMoreCases) {
// all done
break;
}
uprv_strcpy(lineBuf, moreCases[count]);
}
if (lineBuf[0] == 0 || lineBuf[0] == '\n' || lineBuf[0] == '\r') continue;
// Expect 5 columns of this format:
// 1E0C;1E0C;0044 0323;1E0C;0044 0323; # <comments>
// Parse out the comment.
if (lineBuf[0] == '#') continue;
// Read separator lines starting with '@'
if (lineBuf[0] == '@') {
logln(lineBuf);
continue;
}
// Parse out the fields
if (!hexsplit(lineBuf, ';', fields, FIELD_COUNT)) {
errln((UnicodeString)"Unable to parse line " + count);
break; // Syntax error
}
// Remove a single code point from the "other" UnicodeSet
if(fields[0].length()==fields[0].moveIndex32(0, 1)) {
c=fields[0].char32At(0);
if(0xac20<=c && c<=0xd73f && quick) {
// not an exhaustive test run: skip most Hangul syllables
if(c==0xac20) {
other.remove(0xac20, 0xd73f);
}
continue;
}
other.remove(c);
}
if (checkConformance(fields, lineBuf, options, status)) {
++passCount;
} else {
++failCount;
if(status == U_FILE_ACCESS_ERROR) {
dataerrln("Something is wrong with the normalizer, skipping the rest of the test.");
break;
}
}
if ((count % 1000) == 0) {
logln("Line %d", count);
}
}
T_FileStream_close(input);
/*
* Test that all characters that are not mentioned
* as single code points in column 1
* do not change under any normalization.
*/
// remove U+ffff because that is the end-of-iteration sentinel value
other.remove(0xffff);
for(c=0; c<=0x10ffff; quick ? c+=113 : ++c) {
if(0x30000<=c && c<0xe0000) {
c=0xe0000;
}
if(!other.contains(c)) {
continue;
}
fields[0]=fields[1]=fields[2]=fields[3]=fields[4].setTo(c);
sprintf(lineBuf, "not mentioned code point U+%04lx", (long)c);
if (checkConformance(fields, lineBuf, options, status)) {
++passCount;
} else {
++failCount;
if(status == U_FILE_ACCESS_ERROR) {
dataerrln("Something is wrong with the normalizer, skipping the rest of the test.: %s", u_errorName(status));
break;
}
}
if ((c % 0x1000) == 0) {
logln("Code point U+%04lx", c);
}
}
if (failCount != 0) {
dataerrln((UnicodeString)"Total: " + failCount + " lines/code points failed, " +
passCount + " lines/code points passed");
} else {
logln((UnicodeString)"Total: " + passCount + " lines/code points passed");
}
}
/*
* Opens the given fileName and reads in the information storing the data in flagBuffer.
*/
U_CAPI int32_t U_EXPORT2
parseFlagsFile(const char *fileName, char **flagBuffer, int32_t flagBufferSize, const char ** flagNames, int32_t numOfFlags, UErrorCode *status) {
char* buffer = NULL;
char* tmpFlagBuffer = NULL;
UBool allocateMoreSpace = FALSE;
int32_t idx, i;
int32_t result = 0;
FileStream *f = T_FileStream_open(fileName, "r");
if (f == NULL) {
*status = U_FILE_ACCESS_ERROR;
goto parseFlagsFile_cleanup;
}
buffer = uprv_malloc(sizeof(char) * currentBufferSize);
tmpFlagBuffer = uprv_malloc(sizeof(char) * flagBufferSize);
if (buffer == NULL || tmpFlagBuffer == NULL) {
*status = U_MEMORY_ALLOCATION_ERROR;
goto parseFlagsFile_cleanup;
}
do {
if (allocateMoreSpace) {
allocateMoreSpace = FALSE;
currentBufferSize *= 2;
uprv_free(buffer);
buffer = uprv_malloc(sizeof(char) * currentBufferSize);
if (buffer == NULL) {
*status = U_MEMORY_ALLOCATION_ERROR;
goto parseFlagsFile_cleanup;
}
}
for (i = 0; i < numOfFlags;) {
if (T_FileStream_readLine(f, buffer, currentBufferSize) == NULL) {
/* End of file reached. */
break;
}
if (buffer[0] == '#') {
continue;
}
if (uprv_strlen(buffer) == (currentBufferSize - 1) && buffer[currentBufferSize-2] != '\n') {
/* Allocate more space for buffer if it didnot read the entrire line */
allocateMoreSpace = TRUE;
T_FileStream_rewind(f);
break;
} else {
idx = extractFlag(buffer, currentBufferSize, tmpFlagBuffer, flagBufferSize, flagNames, numOfFlags, status);
if (U_FAILURE(*status)) {
if (*status == U_BUFFER_OVERFLOW_ERROR) {
result = currentBufferSize;
} else {
result = -1;
}
break;
} else {
if (flagNames != NULL) {
if (idx >= 0) {
uprv_strcpy(flagBuffer[idx], tmpFlagBuffer);
} else {
/* No match found. Skip it. */
continue;
}
} else {
uprv_strcpy(flagBuffer[i++], tmpFlagBuffer);
}
}
}
}
} while (allocateMoreSpace && U_SUCCESS(*status));
parseFlagsFile_cleanup:
uprv_free(tmpFlagBuffer);
uprv_free(buffer);
T_FileStream_close(f);
if (U_FAILURE(*status)) {
return -1;
}
if (U_SUCCESS(*status) && result == 0) {
currentBufferSize = DEFAULT_BUFFER_SIZE;
}
return result;
}
static void TestFileStream(void){
int32_t c = 0;
int32_t c1=0;
UErrorCode status = U_ZERO_ERROR;
const char* testdatapath = loadTestData(&status);
char* fileName = (char*) malloc(uprv_strlen(testdatapath) +10);
FileStream* stream = NULL;
/* these should not be closed */
FileStream* pStdin = T_FileStream_stdin();
FileStream* pStdout = T_FileStream_stdout();
FileStream* pStderr = T_FileStream_stderr();
const char* testline = "This is a test line";
int32_t bufLen = (int32_t)strlen(testline)+10;
char* buf = (char*) malloc(bufLen);
int32_t retLen = 0;
if(pStdin==NULL){
log_err("failed to get T_FileStream_stdin()");
}
if(pStdout==NULL){
log_err("failed to get T_FileStream_stdout()");
}
if(pStderr==NULL){
log_err("failed to get T_FileStream_stderr()");
}
uprv_strcpy(fileName,testdatapath);
uprv_strcat(fileName,".dat");
stream = T_FileStream_open(fileName, "r");
if(stream==NULL){
log_data_err("T_FileStream_open failed to open %s\n",fileName);
} else {
if(!T_FileStream_file_exists(fileName)){
log_data_err("T_FileStream_file_exists failed to verify existence of %s \n",fileName);
}
retLen=T_FileStream_read(stream,&c,1);
if(retLen==0){
log_data_err("T_FileStream_read failed to read from %s \n",fileName);
}
retLen=0;
T_FileStream_rewind(stream);
T_FileStream_read(stream,&c1,1);
if(c!=c1){
log_data_err("T_FileStream_rewind failed to rewind %s \n",fileName);
}
T_FileStream_rewind(stream);
c1 = T_FileStream_peek(stream);
if(c!=c1){
log_data_err("T_FileStream_peek failed to peekd %s \n",fileName);
}
c = T_FileStream_getc(stream);
T_FileStream_ungetc(c,stream);
if(c!= T_FileStream_getc(stream)){
log_data_err("T_FileStream_ungetc failed to d %s \n",fileName);
}
if(T_FileStream_size(stream)<=0){
log_data_err("T_FileStream_size failed to d %s \n",fileName);
}
if(T_FileStream_error(stream)){
log_data_err("T_FileStream_error shouldn't have an error %s\n",fileName);
}
if(!T_FileStream_error(NULL)){
log_err("T_FileStream_error didn't get an error %s\n",fileName);
}
T_FileStream_putc(stream, 0x20);
if(!T_FileStream_error(stream)){
/*
Warning
writing to a read-only file may not consistently fail on all platforms
(e.g. HP-UX, FreeBSD, MacOSX)
*/
log_verbose("T_FileStream_error didn't get an error when writing to a readonly file %s\n",fileName);
}
T_FileStream_close(stream);
}
/* test writing function */
stream=NULL;
uprv_strcpy(fileName,testdatapath);
uprv_strcat(fileName,".tmp");
stream = T_FileStream_open(fileName,"w+");
if(stream == NULL){
log_data_err("Could not open %s for writing\n",fileName);
} else {
c= '$';
T_FileStream_putc(stream,c);
T_FileStream_rewind(stream);
if(c != T_FileStream_getc(stream)){
log_data_err("T_FileStream_putc failed %s\n",fileName);
}
T_FileStream_rewind(stream);
T_FileStream_writeLine(stream,testline);
T_FileStream_rewind(stream);
T_FileStream_readLine(stream,buf,bufLen);
if(uprv_strncmp(testline, buf,uprv_strlen(buf))!=0){
log_data_err("T_FileStream_writeLine failed %s\n",fileName);
}
T_FileStream_rewind(stream);
T_FileStream_write(stream,testline,(int32_t)strlen(testline));
T_FileStream_rewind(stream);
retLen = T_FileStream_read(stream, buf, bufLen);
if(uprv_strncmp(testline, buf,retLen)!=0){
log_data_err("T_FileStream_write failed %s\n",fileName);
}
T_FileStream_close(stream);
}
if(!T_FileStream_remove(fileName)){
log_data_err("T_FileStream_remove failed to delete %s\n",fileName);
}
free(fileName);
free(buf);
}
U_CAPI void U_EXPORT2
u_parseDelimitedFile(const char *filename, char delimiter,
char *fields[][2], int32_t fieldCount,
UParseLineFn *lineFn, void *context,
UErrorCode *pErrorCode) {
FileStream *file;
char line[300];
char *start, *limit;
int32_t i, length;
if(U_FAILURE(*pErrorCode)) {
return;
}
if(fields==NULL || lineFn==NULL || fieldCount<=0) {
*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
return;
}
if(filename==NULL || *filename==0 || (*filename=='-' && filename[1]==0)) {
filename=NULL;
file=T_FileStream_stdin();
} else {
file=T_FileStream_open(filename, "r");
}
if(file==NULL) {
*pErrorCode=U_FILE_ACCESS_ERROR;
return;
}
while(T_FileStream_readLine(file, line, sizeof(line))!=NULL) {
/* remove trailing newline characters */
length=(int32_t)(u_rtrim(line)-line);
/*
* detect a line with # @missing:
* start parsing after that, or else from the beginning of the line
* set the default warning for @missing lines
*/
start=(char *)getMissingLimit(line);
if(start==line) {
*pErrorCode=U_ZERO_ERROR;
} else {
*pErrorCode=U_USING_DEFAULT_WARNING;
}
/* skip this line if it is empty or a comment */
if(*start==0 || *start=='#') {
continue;
}
/* remove in-line comments */
limit=uprv_strchr(start, '#');
if(limit!=NULL) {
/* get white space before the pound sign */
while(limit>start && U_IS_INV_WHITESPACE(*(limit-1))) {
--limit;
}
/* truncate the line */
*limit=0;
}
/* skip lines with only whitespace */
if(u_skipWhitespace(start)[0]==0) {
continue;
}
/* for each field, call the corresponding field function */
for(i=0; i<fieldCount; ++i) {
/* set the limit pointer of this field */
limit=start;
while(*limit!=delimiter && *limit!=0) {
++limit;
}
/* set the field start and limit in the fields array */
fields[i][0]=start;
fields[i][1]=limit;
/* set start to the beginning of the next field, if any */
start=limit;
if(*start!=0) {
++start;
} else if(i+1<fieldCount) {
*pErrorCode=U_PARSE_ERROR;
limit=line+length;
i=fieldCount;
break;
}
}
/* error in a field function? */
if(U_FAILURE(*pErrorCode)) {
break;
}
/* call the field function */
lineFn(context, fields, fieldCount, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
break;
}
}
if(filename!=NULL) {
T_FileStream_close(file);
}
}
U_CAPI void U_EXPORT2
u_parseDelimitedFile(const char *filename, char delimiter,
char *fields[][2], int32_t fieldCount,
UParseLineFn *lineFn, void *context,
UErrorCode *pErrorCode) {
FileStream *file;
char line[300];
char *start, *limit;
int32_t i, length;
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
return;
}
if(fields==NULL || lineFn==NULL || fieldCount<=0) {
*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
return;
}
if(filename==NULL || *filename==0 || (*filename=='-' && filename[1]==0)) {
filename=NULL;
file=T_FileStream_stdin();
} else {
file=T_FileStream_open(filename, "r");
}
if(file==NULL) {
*pErrorCode=U_FILE_ACCESS_ERROR;
return;
}
while(T_FileStream_readLine(file, line, sizeof(line))!=NULL) {
length=(int32_t)uprv_strlen(line);
/* remove trailing newline characters */
while(length>0 && (line[length-1]=='\r' || line[length-1]=='\n')) {
line[--length]=0;
}
/* skip this line if it is empty or a comment */
if(line[0]==0 || line[0]=='#') {
continue;
}
/* remove in-line comments */
limit=uprv_strchr(line, '#');
if(limit!=NULL) {
/* get white space before the pound sign */
while(limit>line && (*(limit-1)==' ' || *(limit-1)=='\t')) {
--limit;
}
/* truncate the line */
*limit=0;
}
/* skip lines with only whitespace */
if(u_skipWhitespace(line)[0]==0) {
continue;
}
/* for each field, call the corresponding field function */
start=line;
for(i=0; i<fieldCount; ++i) {
/* set the limit pointer of this field */
limit=start;
while(*limit!=delimiter && *limit!=0) {
++limit;
}
/* set the field start and limit in the fields array */
fields[i][0]=start;
fields[i][1]=limit;
/* set start to the beginning of the next field, if any */
start=limit;
if(*start!=0) {
++start;
} else if(i+1<fieldCount) {
*pErrorCode=U_PARSE_ERROR;
limit=line+length;
i=fieldCount;
break;
}
}
/* error in a field function? */
if(U_FAILURE(*pErrorCode)) {
break;
}
/* call the field function */
lineFn(context, fields, fieldCount, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
break;
}
}
if(filename!=NULL) {
T_FileStream_close(file);
}
}
U_CAPI void U_EXPORT2
createCommonDataFile(const char *destDir, const char *name, const char *entrypointName, const char *type, const char *source, const char *copyRight,
const char *dataFile, uint32_t max_size, UBool sourceTOC, UBool verbose, char *gencmnFileName) {
static char buffer[4096];
char *line;
char *linePtr;
char *s = NULL;
UErrorCode errorCode=U_ZERO_ERROR;
uint32_t i, fileOffset, basenameOffset, length, nread;
FileStream *in, *file;
line = (char *)uprv_malloc(sizeof(char) * LINE_BUFFER_SIZE);
if (line == NULL) {
fprintf(stderr, "gencmn: unable to allocate memory for line buffer of size %d\n", LINE_BUFFER_SIZE);
exit(U_MEMORY_ALLOCATION_ERROR);
}
linePtr = line;
maxSize = max_size;
if (destDir == NULL) {
destDir = u_getDataDirectory();
}
if (name == NULL) {
name = COMMON_DATA_NAME;
}
if (type == NULL) {
type = DATA_TYPE;
}
if (source == NULL) {
source = ".";
}
if (dataFile == NULL) {
in = T_FileStream_stdin();
} else {
in = T_FileStream_open(dataFile, "r");
if(in == NULL) {
fprintf(stderr, "gencmn: unable to open input file %s\n", dataFile);
exit(U_FILE_ACCESS_ERROR);
}
}
if (verbose) {
if(sourceTOC) {
printf("generating %s_%s.c (table of contents source file)\n", name, type);
} else {
printf("generating %s.%s (common data file with table of contents)\n", name, type);
}
}
/* read the list of files and get their lengths */
while((s != NULL && *s != 0) || (s=T_FileStream_readLine(in, (line=linePtr),
LINE_BUFFER_SIZE))!=NULL) {
/* remove trailing newline characters and parse space separated items */
if (s != NULL && *s != 0) {
line=s;
} else {
s=line;
}
while(*s!=0) {
if(*s==' ') {
*s=0;
++s;
break;
} else if(*s=='\r' || *s=='\n') {
*s=0;
break;
}
++s;
}
/* check for comment */
if (*line == '#') {
continue;
}
/* add the file */
#if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR)
{
char *t;
while((t = uprv_strchr(line,U_FILE_ALT_SEP_CHAR))) {
*t = U_FILE_SEP_CHAR;
}
}
#endif
addFile(getLongPathname(line), name, source, sourceTOC, verbose);
}
uprv_free(linePtr);
if(in!=T_FileStream_stdin()) {
T_FileStream_close(in);
}
if(fileCount==0) {
fprintf(stderr, "gencmn: no files listed in %s\n", dataFile == NULL ? "<stdin>" : dataFile);
return;
}
/* sort the files by basename */
qsort(files, fileCount, sizeof(File), compareFiles);
if(!sourceTOC) {
UNewDataMemory *out;
/* determine the offsets of all basenames and files in this common one */
basenameOffset=4+8*fileCount;
fileOffset=(basenameOffset+(basenameTotal+15))&~0xf;
for(i=0; i<fileCount; ++i) {
files[i].fileOffset=fileOffset;
fileOffset+=(files[i].fileSize+15)&~0xf;
files[i].basenameOffset=basenameOffset;
basenameOffset+=files[i].basenameLength;
}
/* create the output file */
out=udata_create(destDir, type, name,
&dataInfo,
copyRight == NULL ? U_COPYRIGHT_STRING : copyRight,
&errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "gencmn: udata_create(-d %s -n %s -t %s) failed - %s\n",
destDir, name, type,
u_errorName(errorCode));
exit(errorCode);
}
/* write the table of contents */
udata_write32(out, fileCount);
for(i=0; i<fileCount; ++i) {
udata_write32(out, files[i].basenameOffset);
udata_write32(out, files[i].fileOffset);
}
/* write the basenames */
for(i=0; i<fileCount; ++i) {
udata_writeString(out, files[i].basename, files[i].basenameLength);
}
length=4+8*fileCount+basenameTotal;
/* copy the files */
for(i=0; i<fileCount; ++i) {
/* pad to 16-align the next file */
length&=0xf;
if(length!=0) {
udata_writePadding(out, 16-length);
}
if (verbose) {
printf("adding %s (%ld byte%s)\n", files[i].pathname, (long)files[i].fileSize, files[i].fileSize == 1 ? "" : "s");
}
/* copy the next file */
file=T_FileStream_open(files[i].pathname, "rb");
if(file==NULL) {
fprintf(stderr, "gencmn: unable to open listed file %s\n", files[i].pathname);
exit(U_FILE_ACCESS_ERROR);
}
for(nread = 0;;) {
length=T_FileStream_read(file, buffer, sizeof(buffer));
if(length <= 0) {
break;
}
nread += length;
udata_writeBlock(out, buffer, length);
}
T_FileStream_close(file);
length=files[i].fileSize;
if (nread != files[i].fileSize) {
fprintf(stderr, "gencmn: unable to read %s properly (got %ld/%ld byte%s)\n", files[i].pathname, (long)nread, (long)files[i].fileSize, files[i].fileSize == 1 ? "" : "s");
exit(U_FILE_ACCESS_ERROR);
}
}
/* pad to 16-align the last file (cleaner, avoids growing .dat files in icuswap) */
length&=0xf;
if(length!=0) {
udata_writePadding(out, 16-length);
}
/* finish */
udata_finish(out, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "gencmn: udata_finish() failed - %s\n", u_errorName(errorCode));
exit(errorCode);
}
} else {
/* write a .c source file with the table of contents */
char *filename;
FileStream *out;
/* create the output filename */
filename=s=buffer;
uprv_strcpy(filename, destDir);
s=filename+uprv_strlen(filename);
if(s>filename && *(s-1)!=U_FILE_SEP_CHAR) {
*s++=U_FILE_SEP_CHAR;
}
uprv_strcpy(s, name);
if(*(type)!=0) {
s+=uprv_strlen(s);
*s++='_';
uprv_strcpy(s, type);
}
s+=uprv_strlen(s);
uprv_strcpy(s, ".c");
/* open the output file */
out=T_FileStream_open(filename, "w");
if (gencmnFileName != NULL) {
uprv_strcpy(gencmnFileName, filename);
}
if(out==NULL) {
fprintf(stderr, "gencmn: unable to open .c output file %s\n", filename);
exit(U_FILE_ACCESS_ERROR);
}
/* write the source file */
sprintf(buffer,
"/*\n"
" * ICU common data table of contents for %s.%s\n"
" * Automatically generated by icu/source/tools/gencmn/gencmn .\n"
" */\n\n"
"#include \"unicode/utypes.h\"\n"
"#include \"unicode/udata.h\"\n"
"\n"
"/* external symbol declarations for data (%d files) */\n",
name, type, fileCount);
T_FileStream_writeLine(out, buffer);
sprintf(buffer, "extern const char\n %s%s[]", symPrefix?symPrefix:"", files[0].pathname);
T_FileStream_writeLine(out, buffer);
for(i=1; i<fileCount; ++i) {
sprintf(buffer, ",\n %s%s[]", symPrefix?symPrefix:"", files[i].pathname);
T_FileStream_writeLine(out, buffer);
}
T_FileStream_writeLine(out, ";\n\n");
sprintf(
buffer,
"U_EXPORT struct {\n"
" uint16_t headerSize;\n"
" uint8_t magic1, magic2;\n"
" UDataInfo info;\n"
" char padding[%lu];\n"
" uint32_t count, reserved;\n"
" struct {\n"
" const char *name;\n"
" const void *data;\n"
" } toc[%lu];\n"
"} U_EXPORT2 %s_dat = {\n"
" 32, 0xda, 0x27, {\n"
" %lu, 0,\n"
" %u, %u, %u, 0,\n"
" {0x54, 0x6f, 0x43, 0x50},\n"
" {1, 0, 0, 0},\n"
" {0, 0, 0, 0}\n"
" },\n"
" \"\", %lu, 0, {\n",
(unsigned long)32-4-sizeof(UDataInfo),
(unsigned long)fileCount,
entrypointName,
(unsigned long)sizeof(UDataInfo),
U_IS_BIG_ENDIAN,
U_CHARSET_FAMILY,
U_SIZEOF_UCHAR,
(unsigned long)fileCount
);
T_FileStream_writeLine(out, buffer);
sprintf(buffer, " { \"%s\", %s%s }", files[0].basename, symPrefix?symPrefix:"", files[0].pathname);
T_FileStream_writeLine(out, buffer);
for(i=1; i<fileCount; ++i) {
sprintf(buffer, ",\n { \"%s\", %s%s }", files[i].basename, symPrefix?symPrefix:"", files[i].pathname);
T_FileStream_writeLine(out, buffer);
}
T_FileStream_writeLine(out, "\n }\n};\n");
T_FileStream_close(out);
uprv_free(symPrefix);
}
}
/* return TRUE if a base table was read, FALSE for an extension table */
static UBool
readFile(ConvData *data, const char* converterName,
UErrorCode *pErrorCode) {
char line[1024];
char *end;
FileStream *convFile;
UCMStates *baseStates;
UBool dataIsBase;
if(U_FAILURE(*pErrorCode)) {
return FALSE;
}
data->ucm=ucm_open();
convFile=T_FileStream_open(converterName, "r");
if(convFile==NULL) {
*pErrorCode=U_FILE_ACCESS_ERROR;
return FALSE;
}
readHeader(data, convFile, converterName, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
return FALSE;
}
if(data->ucm->baseName[0]==0) {
dataIsBase=TRUE;
baseStates=&data->ucm->states;
ucm_processStates(baseStates, IGNORE_SISO_CHECK);
} else {
dataIsBase=FALSE;
baseStates=NULL;
}
/* read the base table */
ucm_readTable(data->ucm, convFile, dataIsBase, baseStates, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
return FALSE;
}
/* read an extension table if there is one */
while(T_FileStream_readLine(convFile, line, sizeof(line))) {
end=uprv_strchr(line, 0);
while(line<end &&
(*(end-1)=='\n' || *(end-1)=='\r' || *(end-1)==' ' || *(end-1)=='\t')) {
--end;
}
*end=0;
if(line[0]=='#' || u_skipWhitespace(line)==end) {
continue; /* ignore empty and comment lines */
}
if(0==uprv_strcmp(line, "CHARMAP")) {
/* read the extension table */
ucm_readTable(data->ucm, convFile, FALSE, baseStates, pErrorCode);
} else {
fprintf(stderr, "unexpected text after the base mapping table\n");
}
break;
}
T_FileStream_close(convFile);
if(data->ucm->base->flagsType==UCM_FLAGS_MIXED || data->ucm->ext->flagsType==UCM_FLAGS_MIXED) {
fprintf(stderr, "error: some entries have the mapping precision (with '|'), some do not\n");
*pErrorCode=U_INVALID_TABLE_FORMAT;
}
return dataIsBase;
}
static void
readHeader(ConvData *data,
FileStream* convFile,
const char* converterName,
UErrorCode *pErrorCode) {
char line[1024];
char *s, *key, *value;
const UConverterStaticData *prototype;
UConverterStaticData *staticData;
if(U_FAILURE(*pErrorCode)) {
return;
}
staticData=&data->staticData;
staticData->platform=UCNV_IBM;
staticData->subCharLen=0;
while(T_FileStream_readLine(convFile, line, sizeof(line))) {
/* basic parsing and handling of state-related items */
if(ucm_parseHeaderLine(data->ucm, line, &key, &value)) {
continue;
}
/* stop at the beginning of the mapping section */
if(uprv_strcmp(line, "CHARMAP")==0) {
break;
}
/* collect the information from the header field, ignore unknown keys */
if(uprv_strcmp(key, "code_set_name")==0) {
if(*value!=0) {
uprv_strcpy((char *)staticData->name, value);
getPlatformAndCCSIDFromName(value, &staticData->platform, &staticData->codepage);
}
} else if(uprv_strcmp(key, "subchar")==0) {
uint8_t bytes[UCNV_EXT_MAX_BYTES];
int8_t length;
s=value;
length=ucm_parseBytes(bytes, line, (const char **)&s);
if(1<=length && length<=4 && *s==0) {
staticData->subCharLen=length;
uprv_memcpy(staticData->subChar, bytes, length);
} else {
fprintf(stderr, "error: illegal <subchar> %s\n", value);
*pErrorCode=U_INVALID_TABLE_FORMAT;
return;
}
} else if(uprv_strcmp(key, "subchar1")==0) {
uint8_t bytes[UCNV_EXT_MAX_BYTES];
s=value;
if(1==ucm_parseBytes(bytes, line, (const char **)&s) && *s==0) {
staticData->subChar1=bytes[0];
} else {
fprintf(stderr, "error: illegal <subchar1> %s\n", value);
*pErrorCode=U_INVALID_TABLE_FORMAT;
return;
}
}
}
/* copy values from the UCMFile to the static data */
staticData->maxBytesPerChar=(int8_t)data->ucm->states.maxCharLength;
staticData->minBytesPerChar=(int8_t)data->ucm->states.minCharLength;
staticData->conversionType=data->ucm->states.conversionType;
if(staticData->conversionType==UCNV_UNSUPPORTED_CONVERTER) {
fprintf(stderr, "ucm error: missing conversion type (<uconv_class>)\n");
*pErrorCode=U_INVALID_TABLE_FORMAT;
return;
}
/*
* Now that we know the type, copy any 'default' values from the table.
* We need not check the type any further because the parser only
* recognizes what we have prototypes for.
*
* For delta (extension-only) tables, copy values from the base file
* instead, see createConverter().
*/
if(data->ucm->baseName[0]==0) {
prototype=ucnv_converterStaticData[staticData->conversionType];
if(prototype!=NULL) {
if(staticData->name[0]==0) {
uprv_strcpy((char *)staticData->name, prototype->name);
}
if(staticData->codepage==0) {
staticData->codepage=prototype->codepage;
}
if(staticData->platform==0) {
staticData->platform=prototype->platform;
}
if(staticData->minBytesPerChar==0) {
staticData->minBytesPerChar=prototype->minBytesPerChar;
}
if(staticData->maxBytesPerChar==0) {
staticData->maxBytesPerChar=prototype->maxBytesPerChar;
}
if(staticData->subCharLen==0) {
staticData->subCharLen=prototype->subCharLen;
if(prototype->subCharLen>0) {
uprv_memcpy(staticData->subChar, prototype->subChar, prototype->subCharLen);
}
}
}
}
if(data->ucm->states.outputType<0) {
data->ucm->states.outputType=(int8_t)data->ucm->states.maxCharLength-1;
}
if( staticData->subChar1!=0 &&
(staticData->minBytesPerChar>1 ||
(staticData->conversionType!=UCNV_MBCS &&
staticData->conversionType!=UCNV_EBCDIC_STATEFUL))
) {
fprintf(stderr, "error: <subchar1> defined for a type other than MBCS or EBCDIC_STATEFUL\n");
*pErrorCode=U_INVALID_TABLE_FORMAT;
}
}
/*
* Opens the given fileName and reads in the information storing the data in flagBuffer.
*/
U_CAPI int32_t U_EXPORT2
parseFlagsFile(const char *fileName, char **flagBuffer, int32_t flagBufferSize, int32_t numOfFlags, UErrorCode *status) {
char* buffer = uprv_malloc(sizeof(char) * currentBufferSize);
UBool allocateMoreSpace = FALSE;
int32_t i;
int32_t result = 0;
FileStream *f = T_FileStream_open(fileName, "r");
if (f == NULL) {
*status = U_FILE_ACCESS_ERROR;
return -1;
}
if (buffer == NULL) {
*status = U_MEMORY_ALLOCATION_ERROR;
return -1;
}
do {
if (allocateMoreSpace) {
allocateMoreSpace = FALSE;
currentBufferSize *= 2;
uprv_free(buffer);
buffer = uprv_malloc(sizeof(char) * currentBufferSize);
if (buffer == NULL) {
*status = U_MEMORY_ALLOCATION_ERROR;
return -1;
}
}
for (i = 0; i < numOfFlags; i++) {
if (T_FileStream_readLine(f, buffer, currentBufferSize) == NULL) {
*status = U_FILE_ACCESS_ERROR;
break;
}
if (uprv_strlen(buffer) == (currentBufferSize - 1) && buffer[currentBufferSize-2] != '\n') {
/* Allocate more space for buffer if it didnot read the entrire line */
allocateMoreSpace = TRUE;
T_FileStream_rewind(f);
break;
} else {
extractFlag(buffer, currentBufferSize, flagBuffer[i], flagBufferSize, status);
if (U_FAILURE(*status)) {
if (*status == U_BUFFER_OVERFLOW_ERROR) {
result = currentBufferSize;
} else {
result = -1;
}
break;
}
}
}
} while (allocateMoreSpace && U_SUCCESS(*status));
uprv_free(buffer);
T_FileStream_close(f);
if (U_SUCCESS(*status) && result == 0) {
currentBufferSize = DEFAULT_BUFFER_SIZE;
}
return result;
}
static void
parseFile(FileStream *in) {
char line[MAX_LINE_SIZE];
char lastLine[MAX_LINE_SIZE];
int32_t lineSize = 0;
int32_t lastLineSize = 0;
UBool validParse = TRUE;
lineNum = 0;
/* Add the empty tag, which is for untagged aliases */
getTagNumber("", 0);
getTagNumber(ALL_TAG_STR, 3);
allocString(&stringBlock, "", 0);
/* read the list of aliases */
while (validParse) {
validParse = FALSE;
/* Read non-empty lines that don't start with a space character. */
while (T_FileStream_readLine(in, lastLine, MAX_LINE_SIZE) != NULL) {
lastLineSize = chomp(lastLine);
if (lineSize == 0 || (lastLineSize > 0 && isspace((int)*lastLine))) {
uprv_strcpy(line + lineSize, lastLine);
lineSize += lastLineSize;
} else if (lineSize > 0) {
validParse = TRUE;
break;
}
lineNum++;
}
if (validParse || lineSize > 0) {
if (isspace((int)*line)) {
fprintf(stderr, "%s:%d: error: cannot start an alias with a space\n", path, lineNum-1);
exit(U_PARSE_ERROR);
} else if (line[0] == '{') {
if (!standardTagsUsed && line[lineSize - 1] != '}') {
fprintf(stderr, "%s:%d: error: alias needs to start with a converter name\n", path, lineNum);
exit(U_PARSE_ERROR);
}
addOfficialTaggedStandards(line, lineSize);
standardTagsUsed = TRUE;
} else {
if (standardTagsUsed) {
parseLine(line);
}
else {
fprintf(stderr, "%s:%d: error: alias table needs to start a list of standard tags\n", path, lineNum);
exit(U_PARSE_ERROR);
}
}
/* Was the last line consumed */
if (lastLineSize > 0) {
uprv_strcpy(line, lastLine);
lineSize = lastLineSize;
}
else {
lineSize = 0;
}
}
lineNum++;
}
}
extern int
main(int argc, char* argv[]) {
static char buffer[4096];
char line[512];
FileStream *in, *file;
char *s;
UErrorCode errorCode=U_ZERO_ERROR;
uint32_t i, fileOffset, basenameOffset, length, nread;
UBool sourceTOC, verbose;
const char *entrypointName = NULL;
U_MAIN_INIT_ARGS(argc, argv);
/* preset then read command line options */
options[4].value=u_getDataDirectory();
options[6].value=COMMON_DATA_NAME;
options[7].value=DATA_TYPE;
options[10].value=".";
argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
/* error handling, printing usage message */
if(argc<0) {
fprintf(stderr,
"error in command line argument \"%s\"\n",
argv[-argc]);
} else if(argc<2) {
argc=-1;
}
if(argc<0 || options[0].doesOccur || options[1].doesOccur) {
FILE *where = argc < 0 ? stderr : stdout;
/*
* Broken into chucks because the C89 standard says the minimum
* required supported string length is 509 bytes.
*/
fprintf(where,
"%csage: %s [ -h, -?, --help ] [ -v, --verbose ] [ -c, --copyright ] [ -C, --comment comment ] [ -d, --destdir dir ] [ -n, --name filename ] [ -t, --type filetype ] [ -S, --source tocfile ] [ -e, --entrypoint name ] maxsize listfile\n", argc < 0 ? 'u' : 'U', *argv);
if (options[0].doesOccur || options[1].doesOccur) {
fprintf(where, "\n"
"Read the list file (default: standard input) and create a common data\n"
"file from specified files. Omit any files larger than maxsize, if maxsize > 0.\n");
fprintf(where, "\n"
"Options:\n"
"\t-h, -?, --help this usage text\n"
"\t-v, --verbose verbose output\n"
"\t-c, --copyright include the ICU copyright notice\n"
"\t-C, --comment comment include a comment string\n"
"\t-d, --destdir dir destination directory\n");
fprintf(where,
"\t-n, --name filename output filename, without .type extension\n"
"\t (default: " COMMON_DATA_NAME ")\n"
"\t-t, --type filetype type of the destination file\n"
"\t (default: \"" DATA_TYPE "\")\n"
"\t-S, --source tocfile write a .c source file with the table of\n"
"\t contents\n"
"\t-e, --entrypoint name override the c entrypoint name\n"
"\t (default: \"<name>_<type>\")\n");
}
return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
}
sourceTOC=options[8].doesOccur;
verbose = options[2].doesOccur;
maxSize=(uint32_t)uprv_strtoul(argv[1], NULL, 0);
if(argc==2) {
in=T_FileStream_stdin();
} else {
in=T_FileStream_open(argv[2], "r");
if(in==NULL) {
fprintf(stderr, "gencmn: unable to open input file %s\n", argv[2]);
exit(U_FILE_ACCESS_ERROR);
}
}
if (verbose) {
if(sourceTOC) {
printf("generating %s_%s.c (table of contents source file)\n", options[6].value, options[7].value);
} else {
printf("generating %s.%s (common data file with table of contents)\n", options[6].value, options[7].value);
}
}
/* read the list of files and get their lengths */
while(T_FileStream_readLine(in, line, sizeof(line))!=NULL) {
/* remove trailing newline characters */
s=line;
while(*s!=0) {
if(*s=='\r' || *s=='\n') {
*s=0;
break;
}
++s;
}
/* check for comment */
if (*line == '#') {
continue;
}
/* add the file */
#if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR)
{
char *t;
while((t = uprv_strchr(line,U_FILE_ALT_SEP_CHAR))) {
*t = U_FILE_SEP_CHAR;
}
}
#endif
addFile(getLongPathname(line), sourceTOC, verbose);
}
if(in!=T_FileStream_stdin()) {
T_FileStream_close(in);
}
if(fileCount==0) {
fprintf(stderr, "gencmn: no files listed in %s\n", argc==2 ? "<stdin>" : argv[2]);
return 0;
}
/* sort the files by basename */
qsort(files, fileCount, sizeof(File), compareFiles);
if(!sourceTOC) {
UNewDataMemory *out;
/* determine the offsets of all basenames and files in this common one */
basenameOffset=4+8*fileCount;
fileOffset=(basenameOffset+(basenameTotal+15))&~0xf;
for(i=0; i<fileCount; ++i) {
files[i].fileOffset=fileOffset;
fileOffset+=(files[i].fileSize+15)&~0xf;
files[i].basenameOffset=basenameOffset;
basenameOffset+=files[i].basenameLength;
}
/* create the output file */
out=udata_create(options[4].value, options[7].value, options[6].value,
&dataInfo,
options[3].doesOccur ? U_COPYRIGHT_STRING : options[5].value,
&errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "gencmn: udata_create(-d %s -n %s -t %s) failed - %s\n",
options[4].value, options[6].value, options[7].value,
u_errorName(errorCode));
exit(errorCode);
}
/* write the table of contents */
udata_write32(out, fileCount);
for(i=0; i<fileCount; ++i) {
udata_write32(out, files[i].basenameOffset);
udata_write32(out, files[i].fileOffset);
}
/* write the basenames */
for(i=0; i<fileCount; ++i) {
udata_writeString(out, files[i].basename, files[i].basenameLength);
}
length=4+8*fileCount+basenameTotal;
/* copy the files */
for(i=0; i<fileCount; ++i) {
/* pad to 16-align the next file */
length&=0xf;
if(length!=0) {
udata_writePadding(out, 16-length);
}
if (verbose) {
printf("adding %s (%ld byte%s)\n", files[i].pathname, (long)files[i].fileSize, files[i].fileSize == 1 ? "" : "s");
}
/* copy the next file */
file=T_FileStream_open(files[i].pathname, "rb");
if(file==NULL) {
fprintf(stderr, "gencmn: unable to open listed file %s\n", files[i].pathname);
exit(U_FILE_ACCESS_ERROR);
}
for(nread = 0;;) {
length=T_FileStream_read(file, buffer, sizeof(buffer));
if(length <= 0) {
break;
}
nread += length;
udata_writeBlock(out, buffer, length);
}
T_FileStream_close(file);
length=files[i].fileSize;
if (nread != files[i].fileSize) {
fprintf(stderr, "gencmn: unable to read %s properly (got %ld/%ld byte%s)\n", files[i].pathname, (long)nread, (long)files[i].fileSize, files[i].fileSize == 1 ? "" : "s");
exit(U_FILE_ACCESS_ERROR);
}
}
/* pad to 16-align the last file (cleaner, avoids growing .dat files in icuswap) */
length&=0xf;
if(length!=0) {
udata_writePadding(out, 16-length);
}
/* finish */
udata_finish(out, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "gencmn: udata_finish() failed - %s\n", u_errorName(errorCode));
exit(errorCode);
}
} else {
/* write a .c source file with the table of contents */
char *filename;
FileStream *out;
/* create the output filename */
filename=s=buffer;
uprv_strcpy(filename, options[4].value);
s=filename+uprv_strlen(filename);
if(s>filename && *(s-1)!=U_FILE_SEP_CHAR) {
*s++=U_FILE_SEP_CHAR;
}
uprv_strcpy(s, options[6].value);
if(*(options[7].value)!=0) {
s+=uprv_strlen(s);
*s++='_';
uprv_strcpy(s, options[7].value);
}
s+=uprv_strlen(s);
uprv_strcpy(s, ".c");
/* open the output file */
out=T_FileStream_open(filename, "w");
if(out==NULL) {
fprintf(stderr, "gencmn: unable to open .c output file %s\n", filename);
exit(U_FILE_ACCESS_ERROR);
}
/* If an entrypoint is specified, use it. */
if(options[9].doesOccur) {
entrypointName = options[9].value;
} else {
entrypointName = options[6].value;
}
/* write the source file */
sprintf(buffer,
"/*\n"
" * ICU common data table of contents for %s.%s ,\n"
" * Automatically generated by icu/source/tools/gencmn/gencmn .\n"
" */\n\n"
"#include \"unicode/utypes.h\"\n"
"#include \"unicode/udata.h\"\n"
"\n"
"/* external symbol declarations for data */\n",
options[6].value, options[7].value);
T_FileStream_writeLine(out, buffer);
sprintf(buffer, "extern const char\n %s%s[]", symPrefix?symPrefix:"", files[0].pathname);
T_FileStream_writeLine(out, buffer);
for(i=1; i<fileCount; ++i) {
sprintf(buffer, ",\n %s%s[]", symPrefix?symPrefix:"", files[i].pathname);
T_FileStream_writeLine(out, buffer);
}
T_FileStream_writeLine(out, ";\n\n");
sprintf(
buffer,
"U_EXPORT struct {\n"
" uint16_t headerSize;\n"
" uint8_t magic1, magic2;\n"
" UDataInfo info;\n"
" char padding[%lu];\n"
" uint32_t count, reserved;\n"
" struct {\n"
" const char *name;\n"
" const void *data;\n"
" } toc[%lu];\n"
"} U_EXPORT2 %s_dat = {\n"
" 32, 0xda, 0x27, {\n"
" %lu, 0,\n"
" %u, %u, %u, 0,\n"
" {0x54, 0x6f, 0x43, 0x50},\n"
" {1, 0, 0, 0},\n"
" {0, 0, 0, 0}\n"
" },\n"
" \"\", %lu, 0, {\n",
(unsigned long)32-4-sizeof(UDataInfo),
(unsigned long)fileCount,
entrypointName,
(unsigned long)sizeof(UDataInfo),
U_IS_BIG_ENDIAN,
U_CHARSET_FAMILY,
U_SIZEOF_UCHAR,
(unsigned long)fileCount
);
T_FileStream_writeLine(out, buffer);
sprintf(buffer, " { \"%s\", %s%s }", files[0].basename, symPrefix?symPrefix:"", files[0].pathname);
T_FileStream_writeLine(out, buffer);
for(i=1; i<fileCount; ++i) {
sprintf(buffer, ",\n { \"%s\", %s%s }", files[i].basename, symPrefix?symPrefix:"", files[i].pathname);
T_FileStream_writeLine(out, buffer);
}
T_FileStream_writeLine(out, "\n }\n};\n");
T_FileStream_close(out);
uprv_free(symPrefix);
}
return 0;
}