extern int main(int argc, const char *argv[]) { UNewDataMemory *pData; UErrorCode errorCode=U_ZERO_ERROR; char stringValue[]={'E', 'X', 'A', 'M', 'P', 'L', 'E', '\0'}; uint16_t intValue=2000; long dataLength; uint32_t size; #ifdef WIN32 char *currdir = _getcwd(NULL, 0); #else char *currdir = getcwd(NULL, 0); #endif pData=udata_create(currdir, DATA_TYPE, DATA_NAME, &dataInfo, U_COPYRIGHT_STRING, &errorCode); if(currdir != NULL) { free(currdir); } if(U_FAILURE(errorCode)) { fprintf(stderr, "Error: unable to create data memory, error %d\n", errorCode); exit(errorCode); } /* write the data to the file */ /* a 16 bit value and a String*/ printf("Writing uint16_t value of %d\n", intValue); udata_write16(pData, intValue); printf("Writing string value of %s\n", stringValue); udata_writeString(pData, stringValue, sizeof(stringValue)); /* finish up */ dataLength=udata_finish(pData, &errorCode); if(U_FAILURE(errorCode)) { fprintf(stderr, "Error: error %d writing the output file\n", errorCode); exit(errorCode); } size=sizeof(stringValue) + sizeof(intValue); if(dataLength!=(long)size) { fprintf(stderr, "Error: data length %ld != calculated size %lu\n", dataLength, size); exit(U_INTERNAL_PROGRAM_ERROR); } return 0; }
static void writeAliasTable(UNewDataMemory *out) { uint32_t i, j; uint32_t uniqueAliasesSize; uint16_t aliasOffset = (uint16_t)(tagBlock.top/sizeof(uint16_t)); uint16_t *aliasArrLists = (uint16_t *)uprv_malloc(tagCount * converterCount * sizeof(uint16_t)); uint16_t *uniqueAliases = (uint16_t *)uprv_malloc(knownAliasesCount * sizeof(uint16_t)); uint16_t *uniqueAliasesToConverter = (uint16_t *)uprv_malloc(knownAliasesCount * sizeof(uint16_t)); qsort(knownAliases, knownAliasesCount, sizeof(knownAliases[0]), compareAliases); uniqueAliasesSize = resolveAliases(uniqueAliases, uniqueAliasesToConverter, aliasOffset); /* Array index starts at 1. aliasLists[0] is the size of the lists section. */ aliasListsSize = 0; /* write the offsets of all the aliases lists in a 2D array, and create the lists. */ for (i = 0; i < tagCount; ++i) { for (j = 0; j < converterCount; ++j) { createOneAliasList(aliasArrLists, i, j, aliasOffset); } } /* Write the size of the TOC */ if (tableOptions.stringNormalizationType == UCNV_IO_UNNORMALIZED) { udata_write32(out, 8); } else { udata_write32(out, 9); } /* Write the sizes of each section */ /* All sizes are the number of uint16_t units, not bytes */ udata_write32(out, converterCount); udata_write32(out, tagCount); udata_write32(out, uniqueAliasesSize); /* list of aliases */ udata_write32(out, uniqueAliasesSize); /* The preresolved form of mapping an untagged the alias to a converter */ udata_write32(out, tagCount * converterCount); udata_write32(out, aliasListsSize + 1); udata_write32(out, sizeof(tableOptions) / sizeof(uint16_t)); udata_write32(out, (tagBlock.top + stringBlock.top) / sizeof(uint16_t)); if (tableOptions.stringNormalizationType != UCNV_IO_UNNORMALIZED) { udata_write32(out, (tagBlock.top + stringBlock.top) / sizeof(uint16_t)); } /* write the table of converters */ /* Think of this as the column headers */ for(i=0; i<converterCount; ++i) { udata_write16(out, (uint16_t)(converters[i].converter + aliasOffset)); } /* write the table of tags */ /* Think of this as the row headers */ for(i=UCNV_NUM_RESERVED_TAGS; i<tagCount; ++i) { udata_write16(out, tags[i].tag); } /* The empty tag is considered the leftover list, and put that at the end of the priority list. */ udata_write16(out, tags[EMPTY_TAG_NUM].tag); udata_write16(out, tags[ALL_TAG_NUM].tag); /* Write the unique list of aliases */ udata_writeBlock(out, uniqueAliases, uniqueAliasesSize * sizeof(uint16_t)); /* Write the unique list of aliases */ udata_writeBlock(out, uniqueAliasesToConverter, uniqueAliasesSize * sizeof(uint16_t)); /* Write the array to the lists */ udata_writeBlock(out, (const void *)(aliasArrLists + (2*converterCount)), (((tagCount - 2) * converterCount) * sizeof(uint16_t))); /* Now write the leftover part of the array for the EMPTY and ALL lists */ udata_writeBlock(out, (const void *)aliasArrLists, (2 * converterCount * sizeof(uint16_t))); /* Offset the next array to make the index start at 1. */ udata_write16(out, 0xDEAD); /* Write the lists */ udata_writeBlock(out, (const void *)aliasLists, aliasListsSize * sizeof(uint16_t)); /* Write any options for the alias table. */ udata_writeBlock(out, (const void *)&tableOptions, sizeof(tableOptions)); /* write the tags strings */ udata_writeString(out, tagBlock.store, tagBlock.top); /* write the aliases strings */ udata_writeString(out, stringBlock.store, stringBlock.top); /* write the normalized aliases strings */ if (tableOptions.stringNormalizationType != UCNV_IO_UNNORMALIZED) { char *normalizedStrings = (char *)uprv_malloc(tagBlock.top + stringBlock.top); createNormalizedAliasStrings(normalizedStrings, tagBlock.store, tagBlock.top); createNormalizedAliasStrings(normalizedStrings + tagBlock.top, stringBlock.store, stringBlock.top); /* Write out the complete normalized array. */ udata_writeString(out, normalizedStrings, tagBlock.top + stringBlock.top); uprv_free(normalizedStrings); } uprv_free(uniqueAliasesToConverter); uprv_free(uniqueAliases); uprv_free(aliasArrLists); }
U_CAPI void U_EXPORT2 createCommonDataFile(const char *destDir, const char *name, const char *entrypointName, const char *type, const char *source, const char *copyRight, const char *dataFile, uint32_t max_size, UBool sourceTOC, UBool verbose, char *gencmnFileName) { static char buffer[4096]; char *line; char *linePtr; char *s = NULL; UErrorCode errorCode=U_ZERO_ERROR; uint32_t i, fileOffset, basenameOffset, length, nread; FileStream *in, *file; line = (char *)uprv_malloc(sizeof(char) * LINE_BUFFER_SIZE); if (line == NULL) { fprintf(stderr, "gencmn: unable to allocate memory for line buffer of size %d\n", LINE_BUFFER_SIZE); exit(U_MEMORY_ALLOCATION_ERROR); } linePtr = line; maxSize = max_size; if (destDir == NULL) { destDir = u_getDataDirectory(); } if (name == NULL) { name = COMMON_DATA_NAME; } if (type == NULL) { type = DATA_TYPE; } if (source == NULL) { source = "."; } if (dataFile == NULL) { in = T_FileStream_stdin(); } else { in = T_FileStream_open(dataFile, "r"); if(in == NULL) { fprintf(stderr, "gencmn: unable to open input file %s\n", dataFile); exit(U_FILE_ACCESS_ERROR); } } if (verbose) { if(sourceTOC) { printf("generating %s_%s.c (table of contents source file)\n", name, type); } else { printf("generating %s.%s (common data file with table of contents)\n", name, type); } } /* read the list of files and get their lengths */ while((s != NULL && *s != 0) || (s=T_FileStream_readLine(in, (line=linePtr), LINE_BUFFER_SIZE))!=NULL) { /* remove trailing newline characters and parse space separated items */ if (s != NULL && *s != 0) { line=s; } else { s=line; } while(*s!=0) { if(*s==' ') { *s=0; ++s; break; } else if(*s=='\r' || *s=='\n') { *s=0; break; } ++s; } /* check for comment */ if (*line == '#') { continue; } /* add the file */ #if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR) { char *t; while((t = uprv_strchr(line,U_FILE_ALT_SEP_CHAR))) { *t = U_FILE_SEP_CHAR; } } #endif addFile(getLongPathname(line), name, source, sourceTOC, verbose); } uprv_free(linePtr); if(in!=T_FileStream_stdin()) { T_FileStream_close(in); } if(fileCount==0) { fprintf(stderr, "gencmn: no files listed in %s\n", dataFile == NULL ? "<stdin>" : dataFile); return; } /* sort the files by basename */ qsort(files, fileCount, sizeof(File), compareFiles); if(!sourceTOC) { UNewDataMemory *out; /* determine the offsets of all basenames and files in this common one */ basenameOffset=4+8*fileCount; fileOffset=(basenameOffset+(basenameTotal+15))&~0xf; for(i=0; i<fileCount; ++i) { files[i].fileOffset=fileOffset; fileOffset+=(files[i].fileSize+15)&~0xf; files[i].basenameOffset=basenameOffset; basenameOffset+=files[i].basenameLength; } /* create the output file */ out=udata_create(destDir, type, name, &dataInfo, copyRight == NULL ? U_COPYRIGHT_STRING : copyRight, &errorCode); if(U_FAILURE(errorCode)) { fprintf(stderr, "gencmn: udata_create(-d %s -n %s -t %s) failed - %s\n", destDir, name, type, u_errorName(errorCode)); exit(errorCode); } /* write the table of contents */ udata_write32(out, fileCount); for(i=0; i<fileCount; ++i) { udata_write32(out, files[i].basenameOffset); udata_write32(out, files[i].fileOffset); } /* write the basenames */ for(i=0; i<fileCount; ++i) { udata_writeString(out, files[i].basename, files[i].basenameLength); } length=4+8*fileCount+basenameTotal; /* copy the files */ for(i=0; i<fileCount; ++i) { /* pad to 16-align the next file */ length&=0xf; if(length!=0) { udata_writePadding(out, 16-length); } if (verbose) { printf("adding %s (%ld byte%s)\n", files[i].pathname, (long)files[i].fileSize, files[i].fileSize == 1 ? "" : "s"); } /* copy the next file */ file=T_FileStream_open(files[i].pathname, "rb"); if(file==NULL) { fprintf(stderr, "gencmn: unable to open listed file %s\n", files[i].pathname); exit(U_FILE_ACCESS_ERROR); } for(nread = 0;;) { length=T_FileStream_read(file, buffer, sizeof(buffer)); if(length <= 0) { break; } nread += length; udata_writeBlock(out, buffer, length); } T_FileStream_close(file); length=files[i].fileSize; if (nread != files[i].fileSize) { fprintf(stderr, "gencmn: unable to read %s properly (got %ld/%ld byte%s)\n", files[i].pathname, (long)nread, (long)files[i].fileSize, files[i].fileSize == 1 ? "" : "s"); exit(U_FILE_ACCESS_ERROR); } } /* pad to 16-align the last file (cleaner, avoids growing .dat files in icuswap) */ length&=0xf; if(length!=0) { udata_writePadding(out, 16-length); } /* finish */ udata_finish(out, &errorCode); if(U_FAILURE(errorCode)) { fprintf(stderr, "gencmn: udata_finish() failed - %s\n", u_errorName(errorCode)); exit(errorCode); } } else { /* write a .c source file with the table of contents */ char *filename; FileStream *out; /* create the output filename */ filename=s=buffer; uprv_strcpy(filename, destDir); s=filename+uprv_strlen(filename); if(s>filename && *(s-1)!=U_FILE_SEP_CHAR) { *s++=U_FILE_SEP_CHAR; } uprv_strcpy(s, name); if(*(type)!=0) { s+=uprv_strlen(s); *s++='_'; uprv_strcpy(s, type); } s+=uprv_strlen(s); uprv_strcpy(s, ".c"); /* open the output file */ out=T_FileStream_open(filename, "w"); if (gencmnFileName != NULL) { uprv_strcpy(gencmnFileName, filename); } if(out==NULL) { fprintf(stderr, "gencmn: unable to open .c output file %s\n", filename); exit(U_FILE_ACCESS_ERROR); } /* write the source file */ sprintf(buffer, "/*\n" " * ICU common data table of contents for %s.%s\n" " * Automatically generated by icu/source/tools/gencmn/gencmn .\n" " */\n\n" "#include \"unicode/utypes.h\"\n" "#include \"unicode/udata.h\"\n" "\n" "/* external symbol declarations for data (%d files) */\n", name, type, fileCount); T_FileStream_writeLine(out, buffer); sprintf(buffer, "extern const char\n %s%s[]", symPrefix?symPrefix:"", files[0].pathname); T_FileStream_writeLine(out, buffer); for(i=1; i<fileCount; ++i) { sprintf(buffer, ",\n %s%s[]", symPrefix?symPrefix:"", files[i].pathname); T_FileStream_writeLine(out, buffer); } T_FileStream_writeLine(out, ";\n\n"); sprintf( buffer, "U_EXPORT struct {\n" " uint16_t headerSize;\n" " uint8_t magic1, magic2;\n" " UDataInfo info;\n" " char padding[%lu];\n" " uint32_t count, reserved;\n" " struct {\n" " const char *name;\n" " const void *data;\n" " } toc[%lu];\n" "} U_EXPORT2 %s_dat = {\n" " 32, 0xda, 0x27, {\n" " %lu, 0,\n" " %u, %u, %u, 0,\n" " {0x54, 0x6f, 0x43, 0x50},\n" " {1, 0, 0, 0},\n" " {0, 0, 0, 0}\n" " },\n" " \"\", %lu, 0, {\n", (unsigned long)32-4-sizeof(UDataInfo), (unsigned long)fileCount, entrypointName, (unsigned long)sizeof(UDataInfo), U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, U_SIZEOF_UCHAR, (unsigned long)fileCount ); T_FileStream_writeLine(out, buffer); sprintf(buffer, " { \"%s\", %s%s }", files[0].basename, symPrefix?symPrefix:"", files[0].pathname); T_FileStream_writeLine(out, buffer); for(i=1; i<fileCount; ++i) { sprintf(buffer, ",\n { \"%s\", %s%s }", files[i].basename, symPrefix?symPrefix:"", files[i].pathname); T_FileStream_writeLine(out, buffer); } T_FileStream_writeLine(out, "\n }\n};\n"); T_FileStream_close(out); uprv_free(symPrefix); } }
extern int main(int argc, char* argv[]) { static char buffer[4096]; char line[512]; FileStream *in, *file; char *s; UErrorCode errorCode=U_ZERO_ERROR; uint32_t i, fileOffset, basenameOffset, length, nread; UBool sourceTOC, verbose; const char *entrypointName = NULL; U_MAIN_INIT_ARGS(argc, argv); /* preset then read command line options */ options[4].value=u_getDataDirectory(); options[6].value=COMMON_DATA_NAME; options[7].value=DATA_TYPE; options[10].value="."; argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options); /* error handling, printing usage message */ if(argc<0) { fprintf(stderr, "error in command line argument \"%s\"\n", argv[-argc]); } else if(argc<2) { argc=-1; } if(argc<0 || options[0].doesOccur || options[1].doesOccur) { FILE *where = argc < 0 ? stderr : stdout; /* * Broken into chucks because the C89 standard says the minimum * required supported string length is 509 bytes. */ fprintf(where, "%csage: %s [ -h, -?, --help ] [ -v, --verbose ] [ -c, --copyright ] [ -C, --comment comment ] [ -d, --destdir dir ] [ -n, --name filename ] [ -t, --type filetype ] [ -S, --source tocfile ] [ -e, --entrypoint name ] maxsize listfile\n", argc < 0 ? 'u' : 'U', *argv); if (options[0].doesOccur || options[1].doesOccur) { fprintf(where, "\n" "Read the list file (default: standard input) and create a common data\n" "file from specified files. Omit any files larger than maxsize, if maxsize > 0.\n"); fprintf(where, "\n" "Options:\n" "\t-h, -?, --help this usage text\n" "\t-v, --verbose verbose output\n" "\t-c, --copyright include the ICU copyright notice\n" "\t-C, --comment comment include a comment string\n" "\t-d, --destdir dir destination directory\n"); fprintf(where, "\t-n, --name filename output filename, without .type extension\n" "\t (default: " COMMON_DATA_NAME ")\n" "\t-t, --type filetype type of the destination file\n" "\t (default: \"" DATA_TYPE "\")\n" "\t-S, --source tocfile write a .c source file with the table of\n" "\t contents\n" "\t-e, --entrypoint name override the c entrypoint name\n" "\t (default: \"<name>_<type>\")\n"); } return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR; } sourceTOC=options[8].doesOccur; verbose = options[2].doesOccur; maxSize=(uint32_t)uprv_strtoul(argv[1], NULL, 0); if(argc==2) { in=T_FileStream_stdin(); } else { in=T_FileStream_open(argv[2], "r"); if(in==NULL) { fprintf(stderr, "gencmn: unable to open input file %s\n", argv[2]); exit(U_FILE_ACCESS_ERROR); } } if (verbose) { if(sourceTOC) { printf("generating %s_%s.c (table of contents source file)\n", options[6].value, options[7].value); } else { printf("generating %s.%s (common data file with table of contents)\n", options[6].value, options[7].value); } } /* read the list of files and get their lengths */ while(T_FileStream_readLine(in, line, sizeof(line))!=NULL) { /* remove trailing newline characters */ s=line; while(*s!=0) { if(*s=='\r' || *s=='\n') { *s=0; break; } ++s; } /* check for comment */ if (*line == '#') { continue; } /* add the file */ #if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR) { char *t; while((t = uprv_strchr(line,U_FILE_ALT_SEP_CHAR))) { *t = U_FILE_SEP_CHAR; } } #endif addFile(getLongPathname(line), sourceTOC, verbose); } if(in!=T_FileStream_stdin()) { T_FileStream_close(in); } if(fileCount==0) { fprintf(stderr, "gencmn: no files listed in %s\n", argc==2 ? "<stdin>" : argv[2]); return 0; } /* sort the files by basename */ qsort(files, fileCount, sizeof(File), compareFiles); if(!sourceTOC) { UNewDataMemory *out; /* determine the offsets of all basenames and files in this common one */ basenameOffset=4+8*fileCount; fileOffset=(basenameOffset+(basenameTotal+15))&~0xf; for(i=0; i<fileCount; ++i) { files[i].fileOffset=fileOffset; fileOffset+=(files[i].fileSize+15)&~0xf; files[i].basenameOffset=basenameOffset; basenameOffset+=files[i].basenameLength; } /* create the output file */ out=udata_create(options[4].value, options[7].value, options[6].value, &dataInfo, options[3].doesOccur ? U_COPYRIGHT_STRING : options[5].value, &errorCode); if(U_FAILURE(errorCode)) { fprintf(stderr, "gencmn: udata_create(-d %s -n %s -t %s) failed - %s\n", options[4].value, options[6].value, options[7].value, u_errorName(errorCode)); exit(errorCode); } /* write the table of contents */ udata_write32(out, fileCount); for(i=0; i<fileCount; ++i) { udata_write32(out, files[i].basenameOffset); udata_write32(out, files[i].fileOffset); } /* write the basenames */ for(i=0; i<fileCount; ++i) { udata_writeString(out, files[i].basename, files[i].basenameLength); } length=4+8*fileCount+basenameTotal; /* copy the files */ for(i=0; i<fileCount; ++i) { /* pad to 16-align the next file */ length&=0xf; if(length!=0) { udata_writePadding(out, 16-length); } if (verbose) { printf("adding %s (%ld byte%s)\n", files[i].pathname, (long)files[i].fileSize, files[i].fileSize == 1 ? "" : "s"); } /* copy the next file */ file=T_FileStream_open(files[i].pathname, "rb"); if(file==NULL) { fprintf(stderr, "gencmn: unable to open listed file %s\n", files[i].pathname); exit(U_FILE_ACCESS_ERROR); } for(nread = 0;;) { length=T_FileStream_read(file, buffer, sizeof(buffer)); if(length <= 0) { break; } nread += length; udata_writeBlock(out, buffer, length); } T_FileStream_close(file); length=files[i].fileSize; if (nread != files[i].fileSize) { fprintf(stderr, "gencmn: unable to read %s properly (got %ld/%ld byte%s)\n", files[i].pathname, (long)nread, (long)files[i].fileSize, files[i].fileSize == 1 ? "" : "s"); exit(U_FILE_ACCESS_ERROR); } } /* pad to 16-align the last file (cleaner, avoids growing .dat files in icuswap) */ length&=0xf; if(length!=0) { udata_writePadding(out, 16-length); } /* finish */ udata_finish(out, &errorCode); if(U_FAILURE(errorCode)) { fprintf(stderr, "gencmn: udata_finish() failed - %s\n", u_errorName(errorCode)); exit(errorCode); } } else { /* write a .c source file with the table of contents */ char *filename; FileStream *out; /* create the output filename */ filename=s=buffer; uprv_strcpy(filename, options[4].value); s=filename+uprv_strlen(filename); if(s>filename && *(s-1)!=U_FILE_SEP_CHAR) { *s++=U_FILE_SEP_CHAR; } uprv_strcpy(s, options[6].value); if(*(options[7].value)!=0) { s+=uprv_strlen(s); *s++='_'; uprv_strcpy(s, options[7].value); } s+=uprv_strlen(s); uprv_strcpy(s, ".c"); /* open the output file */ out=T_FileStream_open(filename, "w"); if(out==NULL) { fprintf(stderr, "gencmn: unable to open .c output file %s\n", filename); exit(U_FILE_ACCESS_ERROR); } /* If an entrypoint is specified, use it. */ if(options[9].doesOccur) { entrypointName = options[9].value; } else { entrypointName = options[6].value; } /* write the source file */ sprintf(buffer, "/*\n" " * ICU common data table of contents for %s.%s ,\n" " * Automatically generated by icu/source/tools/gencmn/gencmn .\n" " */\n\n" "#include \"unicode/utypes.h\"\n" "#include \"unicode/udata.h\"\n" "\n" "/* external symbol declarations for data */\n", options[6].value, options[7].value); T_FileStream_writeLine(out, buffer); sprintf(buffer, "extern const char\n %s%s[]", symPrefix?symPrefix:"", files[0].pathname); T_FileStream_writeLine(out, buffer); for(i=1; i<fileCount; ++i) { sprintf(buffer, ",\n %s%s[]", symPrefix?symPrefix:"", files[i].pathname); T_FileStream_writeLine(out, buffer); } T_FileStream_writeLine(out, ";\n\n"); sprintf( buffer, "U_EXPORT struct {\n" " uint16_t headerSize;\n" " uint8_t magic1, magic2;\n" " UDataInfo info;\n" " char padding[%lu];\n" " uint32_t count, reserved;\n" " struct {\n" " const char *name;\n" " const void *data;\n" " } toc[%lu];\n" "} U_EXPORT2 %s_dat = {\n" " 32, 0xda, 0x27, {\n" " %lu, 0,\n" " %u, %u, %u, 0,\n" " {0x54, 0x6f, 0x43, 0x50},\n" " {1, 0, 0, 0},\n" " {0, 0, 0, 0}\n" " },\n" " \"\", %lu, 0, {\n", (unsigned long)32-4-sizeof(UDataInfo), (unsigned long)fileCount, entrypointName, (unsigned long)sizeof(UDataInfo), U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, U_SIZEOF_UCHAR, (unsigned long)fileCount ); T_FileStream_writeLine(out, buffer); sprintf(buffer, " { \"%s\", %s%s }", files[0].basename, symPrefix?symPrefix:"", files[0].pathname); T_FileStream_writeLine(out, buffer); for(i=1; i<fileCount; ++i) { sprintf(buffer, ",\n { \"%s\", %s%s }", files[i].basename, symPrefix?symPrefix:"", files[i].pathname); T_FileStream_writeLine(out, buffer); } T_FileStream_writeLine(out, "\n }\n};\n"); T_FileStream_close(out); uprv_free(symPrefix); } return 0; }