U_CDECL_END
void U_CALLCONV
CollationRoot::load(UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
LocalPointer<CollationTailoring> t(new CollationTailoring(NULL));
if(t.isNull() || t->isBogus()) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
return;
}
t->memory = udata_openChoice(U_ICUDATA_NAME U_TREE_SEPARATOR_STRING "coll",
"icu", "ucadata",
CollationDataReader::isAcceptable, t->version, &errorCode);
if(U_FAILURE(errorCode)) { return; }
const uint8_t *inBytes = static_cast<const uint8_t *>(udata_getMemory(t->memory));
CollationDataReader::read(NULL, inBytes, udata_getLength(t->memory), *t, errorCode);
if(U_FAILURE(errorCode)) { return; }
ucln_i18n_registerCleanup(UCLN_I18N_COLLATION_ROOT, uprv_collation_root_cleanup);
CollationCacheEntry *entry = new CollationCacheEntry(Locale::getRoot(), t.getAlias());
if(entry != NULL) {
t.orphan(); // The rootSingleton took ownership of the tailoring.
entry->addRef();
rootSingleton = entry;
}
}
void RBBIAPITest::RoundtripRule(const char *dataFile) {
UErrorCode status = U_ZERO_ERROR;
UParseError parseError;
parseError.line = 0;
parseError.offset = 0;
LocalUDataMemoryPointer data(udata_open(U_ICUDATA_BRKITR, "brk", dataFile, &status));
uint32_t length;
const UChar *builtSource;
const uint8_t *rbbiRules;
const uint8_t *builtRules;
if (U_FAILURE(status)) {
errcheckln(status, "Can't open \"%s\" - %s", dataFile, u_errorName(status));
return;
}
builtRules = (const uint8_t *)udata_getMemory(data.getAlias());
builtSource = (const UChar *)(builtRules + ((RBBIDataHeader*)builtRules)->fRuleSource);
RuleBasedBreakIterator *brkItr = new RuleBasedBreakIterator(builtSource, parseError, status);
if (U_FAILURE(status)) {
errln("createRuleBasedBreakIterator: ICU Error \"%s\" at line %d, column %d\n",
u_errorName(status), parseError.line, parseError.offset);
return;
};
rbbiRules = brkItr->getBinaryRules(length);
logln("Comparing \"%s\" len=%d", dataFile, length);
if (memcmp(builtRules, rbbiRules, (int32_t)length) != 0) {
errln("Built rules and rebuilt rules are different %s", dataFile);
return;
}
delete brkItr;
}
/* open uprops.icu */
static void
_openProps(UCharProps *ucp, UErrorCode *pErrorCode) {
const uint32_t *p;
int32_t length;
ucp->propsData=udata_openChoice(NULL, DATA_TYPE, DATA_NAME, isAcceptable, NULL, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
return;
}
ucp->pData32=p=(const uint32_t *)udata_getMemory(ucp->propsData);
/* unserialize the trie; it is directly after the int32_t indexes[UPROPS_INDEX_COUNT] */
length=(int32_t)p[UPROPS_PROPS32_INDEX]*4;
length=utrie_unserialize(&ucp->propsTrie, (const uint8_t *)(p+UPROPS_INDEX_COUNT), length-64, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
return;
}
/* unserialize the properties vectors trie */
length=(int32_t)(p[UPROPS_ADDITIONAL_VECTORS_INDEX]-p[UPROPS_ADDITIONAL_TRIE_INDEX])*4;
if(length>0) {
length=utrie_unserialize(&ucp->propsVectorsTrie, (const uint8_t *)(p+p[UPROPS_ADDITIONAL_TRIE_INDEX]), length, pErrorCode);
}
if(length<=0 || U_FAILURE(*pErrorCode)) {
/*
* length==0:
* Allow the properties vectors trie to be missing -
* also requires propsVectorsColumns=indexes[UPROPS_ADDITIONAL_VECTORS_COLUMNS_INDEX]
* to be zero so that this trie is never accessed.
*/
uprv_memset(&ucp->propsVectorsTrie, 0, sizeof(ucp->propsVectorsTrie));
}
}
U_CDECL_END
/**
* Load the property names data. Caller should check that data is
* not loaded BEFORE calling this function. Returns TRUE if the load
* succeeds.
*/
static UBool _load() {
UErrorCode ec = U_ZERO_ERROR;
UDataMemory* data =
udata_openChoice(0, PNAME_DATA_TYPE, PNAME_DATA_NAME,
isPNameAcceptable, 0, &ec);
if (U_SUCCESS(ec)) {
umtx_lock(NULL);
if (UDATA == NULL) {
UDATA = data;
PNAME = (const PropertyAliases*) udata_getMemory(UDATA);
ucln_common_registerCleanup(UCLN_COMMON_PNAME, pname_cleanup);
data = NULL;
}
umtx_unlock(NULL);
}
if (data) {
udata_close(data);
}
return PNAME!=NULL;
}
static void TestUDataGetMemory() {
UDataMemory *result;
const int32_t *table=NULL;
uint16_t* intValue=0;
UErrorCode status=U_ZERO_ERROR;
const char* name="cnvalias";
const char* type;
const char* name2="test";
const char* testPath = loadTestData(&status);
type="icu";
log_verbose("Testing udata_getMemory() for \"cnvalias.icu\"\n");
result=udata_openChoice(NULL, type, name, isAcceptable1, NULL, &status);
if(U_FAILURE(status)){
log_err("FAIL: udata_openChoice() failed for name=%s, type=%s, \n errorcode=%s\n", name, type, myErrorName(status));
return;
}
table=(const int32_t *)udata_getMemory(result);
/* The alias table may list more converters than what's actually available now. [grhoten] */
if(ucnv_countAvailable() > table[1]) /*???*/
log_err("FAIL: udata_getMemory() failed ucnv_countAvailable returned = %d, expected = %d\n", ucnv_countAvailable(), table[1+2*(*table)]);
udata_close(result);
type="icu";
log_verbose("Testing udata_getMemory for \"test.icu\"()\n");
result=udata_openChoice(testPath, type, name2, isAcceptable3, NULL, &status);
if(U_FAILURE(status)){
log_err("FAIL: udata_openChoice() failed for path=%s name=%s, type=%s, \n errorcode=%s\n", testPath, name2, type, myErrorName(status));
return;
}
intValue=(uint16_t *)udata_getMemory(result);
/*printf("%d ..... %s", *(intValue), intValue+1));*/
if( *intValue != 2000 || strcmp((char*)(intValue+1), "YEAR") != 0 )
log_err("FAIL: udata_getMemory() failed: intValue :- Expected:2000 Got:%d \n\tstringValue:- Expected:YEAR Got:%s\n", *intValue, (intValue+1));
udata_close(result);
}
DictionaryMatcher *
ICULanguageBreakFactory::loadDictionaryMatcherFor(UScriptCode script, int32_t /* brkType */) {
UErrorCode status = U_ZERO_ERROR;
// open root from brkitr tree.
UResourceBundle *b = ures_open(U_ICUDATA_BRKITR, "", &status);
b = ures_getByKeyWithFallback(b, "dictionaries", b, &status);
int32_t dictnlength = 0;
const UChar *dictfname =
ures_getStringByKeyWithFallback(b, uscript_getShortName(script), &dictnlength, &status);
if (U_FAILURE(status)) {
ures_close(b);
return NULL;
}
CharString dictnbuf;
CharString ext;
const UChar *extStart = u_memrchr(dictfname, 0x002e, dictnlength); // last dot
if (extStart != NULL) {
int32_t len = (int32_t)(extStart - dictfname);
ext.appendInvariantChars(UnicodeString(FALSE, extStart + 1, dictnlength - len - 1), status);
dictnlength = len;
}
dictnbuf.appendInvariantChars(UnicodeString(FALSE, dictfname, dictnlength), status);
ures_close(b);
UDataMemory *file = udata_open(U_ICUDATA_BRKITR, ext.data(), dictnbuf.data(), &status);
if (U_SUCCESS(status)) {
// build trie
const uint8_t *data = (const uint8_t *)udata_getMemory(file);
const int32_t *indexes = (const int32_t *)data;
const int32_t offset = indexes[DictionaryData::IX_STRING_TRIE_OFFSET];
const int32_t trieType = indexes[DictionaryData::IX_TRIE_TYPE] & DictionaryData::TRIE_TYPE_MASK;
DictionaryMatcher *m = NULL;
if (trieType == DictionaryData::TRIE_TYPE_BYTES) {
const int32_t transform = indexes[DictionaryData::IX_TRANSFORM];
const char *characters = (const char *)(data + offset);
m = new BytesDictionaryMatcher(characters, transform, file);
}
else if (trieType == DictionaryData::TRIE_TYPE_UCHARS) {
const UChar *characters = (const UChar *)(data + offset);
m = new UCharsDictionaryMatcher(characters, file);
}
if (m == NULL) {
// no matcher exists to take ownership - either we are an invalid
// type or memory allocation failed
udata_close(file);
}
return m;
} else if (dictfname != NULL) {
// we don't have a dictionary matcher.
// returning NULL here will cause us to fail to find a dictionary break engine, as expected
status = U_ZERO_ERROR;
return NULL;
}
return NULL;
}
SpoofData::SpoofData(UDataMemory *udm, UErrorCode &status)
{
reset();
if (U_FAILURE(status)) {
return;
}
fUDM = udm;
// fRawData is non-const because it may be constructed by the data builder.
fRawData = reinterpret_cast<SpoofDataHeader *>(
const_cast<void *>(udata_getMemory(udm)));
validateDataVersion(status);
initPtrs(status);
}
extern int
main(int argc, const char *argv[]) {
UDataMemory *result = NULL;
UErrorCode status=U_ZERO_ERROR;
uint16_t intValue = 0;
char *string = NULL;
uint16_t *intPointer = NULL;
const void *dataMemory = NULL;
char curPathBuffer[1024];
#ifdef WIN32
char *currdir = _getcwd(NULL, 0);
#else
char *currdir = getcwd(NULL, 0);
#endif
/* need to put "current/dir" as path */
strcpy(curPathBuffer, currdir);
result=udata_openChoice(curPathBuffer, DATA_TYPE, DATA_NAME, isAcceptable, NULL, &status);
if(currdir != NULL) {
free(currdir);
}
if(U_FAILURE(status)){
printf("Failed to open data file example.dat in %s with error number %d\n", curPathBuffer, status);
return -1;
}
dataMemory = udata_getMemory(result);
intPointer = (uint16_t *)dataMemory;
printf("Read value %d from data file\n", *intPointer);
string = (char *) (intPointer+1);
printf("Read string %s from data file\n", string);
if(U_SUCCESS(status)){
udata_close(result);
}
return 0;
}
U_CFUNC void
res_load(ResourceData *pResData,
const char *path, const char *name, UErrorCode *errorCode) {
UVersionInfo formatVersion;
uprv_memset(pResData, 0, sizeof(ResourceData));
/* load the ResourceBundle file */
pResData->data=udata_openChoice(path, "res", name, isAcceptable, formatVersion, errorCode);
if(U_FAILURE(*errorCode)) {
return;
}
/* get its memory and initialize *pResData */
res_init(pResData, formatVersion, udata_getMemory(pResData->data), -1, errorCode);
}
U_CDECL_END
/* do not close UCA returned by ucol_initUCA! */
UCollator *
ucol_initUCA(UErrorCode *status) {
if(U_FAILURE(*status)) {
return NULL;
}
UBool needsInit;
UMTX_CHECK(NULL, (_staticUCA == NULL), needsInit);
if(needsInit) {
UDataMemory *result = udata_openChoice(U_ICUDATA_COLL, UCA_DATA_TYPE, UCA_DATA_NAME, isAcceptableUCA, NULL, status);
if(U_SUCCESS(*status)){
UCollator *newUCA = ucol_initCollator((const UCATableHeader *)udata_getMemory(result), NULL, NULL, status);
if(U_SUCCESS(*status)){
// Initalize variables for implicit generation
uprv_uca_initImplicitConstants(status);
umtx_lock(NULL);
if(_staticUCA == NULL) {
UCA_DATA_MEM = result;
_staticUCA = newUCA;
newUCA = NULL;
result = NULL;
}
umtx_unlock(NULL);
ucln_i18n_registerCleanup(UCLN_I18N_UCOL_RES, ucol_res_cleanup);
if(newUCA != NULL) {
ucol_close(newUCA);
udata_close(result);
}
}else{
ucol_close(newUCA);
udata_close(result);
}
}
else {
udata_close(result);
}
}
return _staticUCA;
}
U_CAPI UCaseProps * U_EXPORT2
ucase_open(UErrorCode *pErrorCode) {
UCaseProps cspProto={ NULL }, *csp;
cspProto.mem=udata_openChoice(NULL, UCASE_DATA_TYPE, UCASE_DATA_NAME, isAcceptable, &cspProto, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
return NULL;
}
csp=ucase_openData(
&cspProto,
udata_getMemory(cspProto.mem),
udata_getLength(cspProto.mem),
pErrorCode);
if(U_FAILURE(*pErrorCode)) {
udata_close(cspProto.mem);
return NULL;
} else {
return csp;
}
}
U_CAPI UBiDiProps * U_EXPORT2
ubidi_openProps(UErrorCode *pErrorCode) {
UBiDiProps bdpProto={ NULL }, *bdp;
bdpProto.mem=udata_openChoice(NULL, UBIDI_DATA_TYPE, UBIDI_DATA_NAME, isAcceptable, &bdpProto, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
return NULL;
}
bdp=ubidi_openData(
&bdpProto,
udata_getMemory(bdpProto.mem),
udata_getLength(bdpProto.mem),
pErrorCode);
if(U_FAILURE(*pErrorCode)) {
udata_close(bdpProto.mem);
return NULL;
} else {
return bdp;
}
}
void
LoadedNormalizer2Impl::load(const char *packageName, const char *name, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) {
return;
}
memory=udata_openChoice(packageName, "nrm", name, isAcceptable, this, &errorCode);
if(U_FAILURE(errorCode)) {
return;
}
const uint8_t *inBytes=(const uint8_t *)udata_getMemory(memory);
const int32_t *inIndexes=(const int32_t *)inBytes;
int32_t indexesLength=inIndexes[IX_NORM_TRIE_OFFSET]/4;
if(indexesLength<=IX_MIN_MAYBE_YES) {
errorCode=U_INVALID_FORMAT_ERROR; // Not enough indexes.
return;
}
int32_t offset=inIndexes[IX_NORM_TRIE_OFFSET];
int32_t nextOffset=inIndexes[IX_EXTRA_DATA_OFFSET];
ownedTrie=utrie2_openFromSerialized(UTRIE2_16_VALUE_BITS,
inBytes+offset, nextOffset-offset, NULL,
&errorCode);
if(U_FAILURE(errorCode)) {
return;
}
offset=nextOffset;
nextOffset=inIndexes[IX_SMALL_FCD_OFFSET];
const uint16_t *inExtraData=(const uint16_t *)(inBytes+offset);
// smallFCD: new in formatVersion 2
offset=nextOffset;
const uint8_t *inSmallFCD=inBytes+offset;
init(inIndexes, ownedTrie, inExtraData, inSmallFCD);
}
static void U_CALLCONV initAliasData(UErrorCode &errCode) {
UDataMemory *data;
const uint16_t *table;
const uint32_t *sectionSizes;
uint32_t tableStart;
uint32_t currOffset;
ucln_common_registerCleanup(UCLN_COMMON_UCNV_IO, ucnv_io_cleanup);
U_ASSERT(gAliasData == NULL);
data = udata_openChoice(NULL, DATA_TYPE, DATA_NAME, isAcceptable, NULL, &errCode);
if(U_FAILURE(errCode)) {
return;
}
sectionSizes = (const uint32_t *)udata_getMemory(data);
table = (const uint16_t *)sectionSizes;
tableStart = sectionSizes[0];
if (tableStart < minTocLength) {
errCode = U_INVALID_FORMAT_ERROR;
udata_close(data);
return;
}
gAliasData = data;
gMainTable.converterListSize = sectionSizes[1];
gMainTable.tagListSize = sectionSizes[2];
gMainTable.aliasListSize = sectionSizes[3];
gMainTable.untaggedConvArraySize = sectionSizes[4];
gMainTable.taggedAliasArraySize = sectionSizes[5];
gMainTable.taggedAliasListsSize = sectionSizes[6];
gMainTable.optionTableSize = sectionSizes[7];
gMainTable.stringTableSize = sectionSizes[8];
if (tableStart > 8) {
gMainTable.normalizedStringTableSize = sectionSizes[9];
}
currOffset = tableStart * (sizeof(uint32_t)/sizeof(uint16_t)) + (sizeof(uint32_t)/sizeof(uint16_t));
gMainTable.converterList = table + currOffset;
currOffset += gMainTable.converterListSize;
gMainTable.tagList = table + currOffset;
currOffset += gMainTable.tagListSize;
gMainTable.aliasList = table + currOffset;
currOffset += gMainTable.aliasListSize;
gMainTable.untaggedConvArray = table + currOffset;
currOffset += gMainTable.untaggedConvArraySize;
gMainTable.taggedAliasArray = table + currOffset;
/* aliasLists is a 1's based array, but it has a padding character */
currOffset += gMainTable.taggedAliasArraySize;
gMainTable.taggedAliasLists = table + currOffset;
currOffset += gMainTable.taggedAliasListsSize;
if (gMainTable.optionTableSize > 0
&& ((const UConverterAliasOptions *)(table + currOffset))->stringNormalizationType < UCNV_IO_NORM_TYPE_COUNT)
{
/* Faster table */
gMainTable.optionTable = (const UConverterAliasOptions *)(table + currOffset);
}
else {
/* Smaller table, or I can't handle this normalization mode!
Use the original slower table lookup. */
gMainTable.optionTable = &defaultTableOptions;
}
currOffset += gMainTable.optionTableSize;
gMainTable.stringTable = table + currOffset;
currOffset += gMainTable.stringTableSize;
gMainTable.normalizedStringTable = ((gMainTable.optionTable->stringNormalizationType == UCNV_IO_UNNORMALIZED)
? gMainTable.stringTable : (table + currOffset));
}
static UBool U_CALLCONV
loadData(UStringPrepProfile* profile,
const char* path,
const char* name,
const char* type,
UErrorCode* errorCode) {
/* load Unicode SPREP data from file */
UTrie _sprepTrie={ 0,0,0,0,0,0,0 };
UDataMemory *dataMemory;
const int32_t *p=NULL;
const uint8_t *pb;
UVersionInfo normUnicodeVersion;
int32_t normUniVer, sprepUniVer, normCorrVer;
if(errorCode==NULL || U_FAILURE(*errorCode)) {
return 0;
}
/* open the data outside the mutex block */
//TODO: change the path
dataMemory=udata_openChoice(path, type, name, isSPrepAcceptable, NULL, errorCode);
if(U_FAILURE(*errorCode)) {
return FALSE;
}
p=(const int32_t *)udata_getMemory(dataMemory);
pb=(const uint8_t *)(p+_SPREP_INDEX_TOP);
utrie_unserialize(&_sprepTrie, pb, p[_SPREP_INDEX_TRIE_SIZE], errorCode);
_sprepTrie.getFoldingOffset=getSPrepFoldingOffset;
if(U_FAILURE(*errorCode)) {
udata_close(dataMemory);
return FALSE;
}
/* in the mutex block, set the data for this process */
umtx_lock(usprepMutex());
if(profile->sprepData==NULL) {
profile->sprepData=dataMemory;
dataMemory=NULL;
uprv_memcpy(&profile->indexes, p, sizeof(profile->indexes));
uprv_memcpy(&profile->sprepTrie, &_sprepTrie, sizeof(UTrie));
} else {
p=(const int32_t *)udata_getMemory(profile->sprepData);
}
umtx_unlock(usprepMutex());
/* initialize some variables */
profile->mappingData=(uint16_t *)((uint8_t *)(p+_SPREP_INDEX_TOP)+profile->indexes[_SPREP_INDEX_TRIE_SIZE]);
u_getUnicodeVersion(normUnicodeVersion);
normUniVer = (normUnicodeVersion[0] << 24) + (normUnicodeVersion[1] << 16) +
(normUnicodeVersion[2] << 8 ) + (normUnicodeVersion[3]);
sprepUniVer = (dataVersion[0] << 24) + (dataVersion[1] << 16) +
(dataVersion[2] << 8 ) + (dataVersion[3]);
normCorrVer = profile->indexes[_SPREP_NORM_CORRECTNS_LAST_UNI_VERSION];
if(U_FAILURE(*errorCode)){
udata_close(dataMemory);
return FALSE;
}
if( normUniVer < sprepUniVer && /* the Unicode version of SPREP file must be less than the Unicode Vesion of the normalization data */
normUniVer < normCorrVer && /* the Unicode version of the NormalizationCorrections.txt file should be less than the Unicode Vesion of the normalization data */
((profile->indexes[_SPREP_OPTIONS] & _SPREP_NORMALIZATION_ON) > 0) /* normalization turned on*/
){
*errorCode = U_INVALID_FORMAT_ERROR;
udata_close(dataMemory);
return FALSE;
}
profile->isDataLoaded = TRUE;
/* if a different thread set it first, then close the extra data */
if(dataMemory!=NULL) {
udata_close(dataMemory); /* NULL if it was set correctly */
}
return profile->isDataLoaded;
}
// Try out the RuleBasedBreakIterator constructors that take RBBIDataHeader*
// (these are protected so we access them via a local class RBBIWithProtectedFunctions).
// This is just a sanity check, not a thorough test (e.g. we don't check that the
// first delete actually frees rulesCopy).
void RBBIAPITest::TestCreateFromRBBIData() {
// Get some handy RBBIData
const char *brkName = "word"; // or "sent", "line", "char", etc.
UErrorCode status = U_ZERO_ERROR;
LocalUDataMemoryPointer data(udata_open(U_ICUDATA_BRKITR, "brk", brkName, &status));
if ( U_SUCCESS(status) ) {
const RBBIDataHeader * builtRules = (const RBBIDataHeader *)udata_getMemory(data.getAlias());
uint32_t length = builtRules->fLength;
RBBIWithProtectedFunctions * brkItr;
// Try the memory-adopting constructor, need to copy the data first
RBBIDataHeader * rulesCopy = (RBBIDataHeader *) uprv_malloc(length);
if ( rulesCopy ) {
uprv_memcpy( rulesCopy, builtRules, length );
brkItr = new RBBIWithProtectedFunctions(rulesCopy, status);
if ( U_SUCCESS(status) ) {
delete brkItr; // this should free rulesCopy
} else {
errln("create RuleBasedBreakIterator from RBBIData (adopted): ICU Error \"%s\"\n", u_errorName(status) );
status = U_ZERO_ERROR;// reset for the next test
uprv_free( rulesCopy );
}
}
// Now try the non-adopting constructor
brkItr = new RBBIWithProtectedFunctions(builtRules, RBBIWithProtectedFunctions::kDontAdopt, status);
if ( U_SUCCESS(status) ) {
delete brkItr; // this should NOT attempt to free builtRules
if (builtRules->fLength != length) { // sanity check
errln("create RuleBasedBreakIterator from RBBIData (non-adopted): delete affects data\n" );
}
} else {
errln("create RuleBasedBreakIterator from RBBIData (non-adopted): ICU Error \"%s\"\n", u_errorName(status) );
}
}
// getBinaryRules() and RuleBasedBreakIterator(uint8_t binaryRules, ...)
//
status = U_ZERO_ERROR;
RuleBasedBreakIterator *rb = (RuleBasedBreakIterator *)BreakIterator::createWordInstance(Locale::getEnglish(), status);
if (rb == NULL || U_FAILURE(status)) {
dataerrln("Unable to create BreakIterator::createWordInstance (Locale::getEnglish) - %s", u_errorName(status));
} else {
uint32_t length;
const uint8_t *rules = rb->getBinaryRules(length);
RuleBasedBreakIterator *rb2 = new RuleBasedBreakIterator(rules, length, status);
TEST_ASSERT_SUCCESS(status);
TEST_ASSERT(*rb == *rb2);
UnicodeString words = "one two three ";
rb2->setText(words);
int wordCounter = 0;
while (rb2->next() != UBRK_DONE) {
wordCounter++;
}
TEST_ASSERT(wordCounter == 6);
status = U_ZERO_ERROR;
RuleBasedBreakIterator *rb3 = new RuleBasedBreakIterator(rules, length-1, status);
TEST_ASSERT(status == U_ILLEGAL_ARGUMENT_ERROR);
delete rb;
delete rb2;
delete rb3;
}
}
U_NAMESPACE_BEGIN
/**
* Load the system time zone data from icudata.dll (or its
* equivalent). If this call succeeds, it will return TRUE and
* UDATA_MEMORY will be non-null, and DATA and INDEX_BY_* will be set
* to point into it. If this call fails, either because the data
* could not be opened, or because the ID array could not be
* allocated, then it will return FALSE.
*
* Must be called OUTSIDE mutex.
*/
static UBool loadZoneData() {
// Open a data memory object, to be closed either later in this
// function or in timeZone_cleanup(). Purify (etc.) may
// mistakenly report this as a leak.
UErrorCode status = U_ZERO_ERROR;
UDataMemory* udata = udata_openChoice(0, TZ_DATA_TYPE, TZ_DATA_NAME,
(UDataMemoryIsAcceptable*)isTimeZoneDataAcceptable, 0, &status);
if (U_FAILURE(status)) {
U_ASSERT(udata==0);
return FALSE;
}
U_ASSERT(udata!=0);
TZHeader* tzh = (TZHeader*)udata_getMemory(udata);
U_ASSERT(tzh!=0);
const uint32_t* index_by_id =
(const uint32_t*)((int8_t*)tzh + tzh->nameIndexDelta);
const OffsetIndex* index_by_offset =
(const OffsetIndex*)((int8_t*)tzh + tzh->offsetIndexDelta);
const CountryIndex* index_by_country =
(const CountryIndex*)((int8_t*)tzh + tzh->countryIndexDelta);
// Construct the available IDs array. The ordering
// of this array conforms to the ordering of the
// index by name table.
UnicodeString* zone_ids = new UnicodeString[tzh->count ? tzh->count : 1];
if (zone_ids == 0) {
udata_close(udata);
return FALSE;
}
// Find start of name table, and walk through it
// linearly. If you're wondering why we don't use
// the INDEX_BY_ID, it's because that indexes the
// zone objects, not the name table. The name
// table is unindexed.
const char* name = (const char*)tzh + tzh->nameTableDelta;
int32_t length;
for (uint32_t i=0; i<tzh->count; ++i) {
zone_ids[i] = UnicodeString(name, ""); // invariant converter
length = zone_ids[i].length(); // add a NUL but don't count it so that
zone_ids[i].append((UChar)0); // getBuffer() gets a terminated string
zone_ids[i].truncate(length);
name += uprv_strlen(name) + 1;
}
// Keep mutexed operations as short as possible by doing all
// computations first, then doing pointer copies within the mutex.
umtx_lock(&LOCK);
if (UDATA_MEMORY == 0) {
UDATA_MEMORY = udata;
DATA = tzh;
INDEX_BY_ID = index_by_id;
INDEX_BY_OFFSET = index_by_offset;
INDEX_BY_COUNTRY = index_by_country;
ZONE_IDS = zone_ids;
udata = NULL;
zone_ids = NULL;
}
umtx_unlock(&LOCK);
// If another thread initialized the statics first, then delete
// our unused data.
if (udata != NULL) {
udata_close(udata);
delete[] zone_ids;
}
// Cleanup handles both _GMT and the UDataMemory-based statics
ucln_i18n_registerCleanup();
return TRUE;
}
/**
* Un flatten shared data from a UDATA..
*/
static UConverterSharedData*
ucnv_data_unFlattenClone(UConverterLoadArgs *pArgs, UDataMemory *pData, UErrorCode *status)
{
/* UDataInfo info; -- necessary only if some converters have different formatVersion */
const uint8_t *raw = (const uint8_t *)udata_getMemory(pData);
const UConverterStaticData *source = (const UConverterStaticData *) raw;
UConverterSharedData *data;
UConverterType type = (UConverterType)source->conversionType;
if(U_FAILURE(*status))
return NULL;
if( (uint16_t)type >= UCNV_NUMBER_OF_SUPPORTED_CONVERTER_TYPES ||
converterData[type] == NULL ||
converterData[type]->referenceCounter != 1 ||
source->structSize != sizeof(UConverterStaticData))
{
*status = U_INVALID_TABLE_FORMAT;
return NULL;
}
data = (UConverterSharedData *)uprv_malloc(sizeof(UConverterSharedData));
if(data == NULL) {
*status = U_MEMORY_ALLOCATION_ERROR;
return NULL;
}
/* copy initial values from the static structure for this type */
uprv_memcpy(data, converterData[type], sizeof(UConverterSharedData));
#if 0 /* made UConverterMBCSTable part of UConverterSharedData -- markus 20031107 */
/*
* It would be much more efficient if the table were a direct member, not a pointer.
* However, that would add to the size of all UConverterSharedData objects
* even if they do not use this table (especially algorithmic ones).
* If this changes, then the static templates from converterData[type]
* need more entries.
*
* In principle, it would be cleaner if the load() function below
* allocated the table.
*/
data->table = (UConverterTable *)uprv_malloc(sizeof(UConverterTable));
if(data->table == NULL) {
uprv_free(data);
*status = U_MEMORY_ALLOCATION_ERROR;
return NULL;
}
uprv_memset(data->table, 0, sizeof(UConverterTable));
#endif
data->staticData = source;
data->sharedDataCached = FALSE;
/* fill in fields from the loaded data */
data->dataMemory = (void*)pData; /* for future use */
if(data->impl->load != NULL) {
data->impl->load(data, pArgs, raw + source->structSize, status);
if(U_FAILURE(*status)) {
uprv_free(data->table);
uprv_free(data);
return NULL;
}
}
return data;
}
static void TestErrorConditions(){
UDataMemory *result=NULL;
UErrorCode status=U_ZERO_ERROR;
uint16_t* intValue=0;
static UDataInfo dataInfo={
30, /*sizeof(UDataInfo),*/
0,
U_IS_BIG_ENDIAN,
U_CHARSET_FAMILY,
sizeof(UChar),
0,
{0x54, 0x65, 0x73, 0x74}, /* dataFormat="Test" */
{9, 0, 0, 0}, /* formatVersion */
{4, 0, 0, 0} /* dataVersion */
};
const char* name = "test";
const char* type="icu";
const char *testPath = loadTestData(&status);
status = U_ILLEGAL_ARGUMENT_ERROR;
/*Try udata_open with status != U_ZERO_ERROR*/
log_verbose("Testing udata_open() with status != U_ZERO_ERROR\n");
result=udata_open(testPath, type, name, &status);
if(result != NULL){
log_err("FAIL: udata_open() is supposed to fail for path = %s, name=%s, type=%s, \n errorcode !=U_ZERO_ERROR\n", testPath, name, type);
udata_close(result);
} else {
log_verbose("PASS: udata_open with errorCode != U_ZERO_ERROR failed as expected\n");
}
/*Try udata_open with data name=NULL*/
log_verbose("Testing udata_open() with data name=NULL\n");
status=U_ZERO_ERROR;
result=udata_open(testPath, type, NULL, &status);
if(U_FAILURE(status)){
if(status != U_ILLEGAL_ARGUMENT_ERROR || result != NULL){
log_err("FAIL: udata_open() with name=NULL should return NULL and errocode U_ILLEGAL_ARGUMENT_ERROR, GOT: errorcode=%s\n", myErrorName(status));
}else{
log_verbose("PASS: udata_open with name=NULL failed as expected and errorcode = %s as expected\n", myErrorName(status));
}
}else{
log_err("FAIL: udata_open() with data name=NULL is supposed to fail for path = %s, name=NULL type=%s errorcode=U_ZERO_ERROR \n", testPath, type);
udata_close(result);
}
/*Try udata_openChoice with status != U_ZERO_ERROR*/
log_verbose("Testing udata_openChoice() with status != U_ZERO_ERROR\n");
status=U_ILLEGAL_ARGUMENT_ERROR;
result=udata_openChoice(testPath, type, name, isAcceptable3, NULL, &status);
if(result != NULL){
log_err("FAIL: udata_openChoice() is supposed to fail for path = %s, name=%s, type=%s, \n errorcode != U_ZERO_ERROR\n", testPath, name, type);
udata_close(result);
} else {
log_verbose("PASS: udata_openChoice() with errorCode != U_ZERO_ERROR failed as expected\n");
}
/*Try udata_open with data name=NULL*/
log_verbose("Testing udata_openChoice() with data name=NULL\n");
status=U_ZERO_ERROR;
result=udata_openChoice(testPath, type, NULL, isAcceptable3, NULL, &status);
if(U_FAILURE(status)){
if(status != U_ILLEGAL_ARGUMENT_ERROR || result != NULL){
log_err("FAIL: udata_openChoice() with name=NULL should return NULL and errocode U_ILLEGAL_ARGUMENT_ERROR, GOT: errorcode=%s\n", myErrorName(status));
}else{
log_verbose("PASS: udata_openChoice with name=NULL failed as expected and errorcode = %s as expected\n", myErrorName(status));
}
}else{
log_err("FAIL: udata_openChoice() with data name=NULL is supposed to fail for path = %s, name=NULL type=%s errorcode=U_ZERO_ERROR \n", testPath, type);
udata_close(result);
}
/*Try udata_getMemory with UDataMemory=NULL*/
log_verbose("Testing udata_getMemory with UDataMemory=NULL\n");
intValue=(uint16_t*)udata_getMemory(NULL);
if(intValue != NULL){
log_err("FAIL: udata_getMemory with UDataMemory = NULL is supposed to fail\n");
}
/*Try udata_getInfo with UDataMemory=NULL*/
status=U_ZERO_ERROR;
udata_getInfo(NULL, &dataInfo);
if(dataInfo.size != 0){
log_err("FAIL : udata_getInfo with UDataMemory = NULL us supposed to fail\n");
}
/*Try udata_openChoice with a non existing binary file*/
log_verbose("Testing udata_openChoice() with a non existing binary file\n");
result=udata_openChoice(testPath, "tst", "nonexist", isAcceptable3, NULL, &status);
if(status==U_FILE_ACCESS_ERROR){
log_verbose("Opening udata_openChoice with non-existing file handled correctly.\n");
status=U_ZERO_ERROR;
} else {
log_err("calling udata_open with non-existing file not handled correctly\n. Expected: U_FILE_ACCESS_ERROR, Got: %s\n", myErrorName(status));
if(U_SUCCESS(status)) {
udata_close(result);
}
}
if(result != NULL){
log_err("calling udata_open with non-existing file didn't return a null value\n");
} else {
log_verbose("calling udat_open with non-existing file returned null as expected\n");
}
}
static void
TestSwapCase(UDataMemory *pData, const char *name,
UDataSwapFn *swapFn,
uint8_t *buffer, uint8_t *buffer2) {
UDataSwapper *ds;
const void *inData, *inHeader;
int32_t length, dataLength, length2, headerLength;
UErrorCode errorCode;
UBool inEndian, oppositeEndian;
uint8_t inCharset, oppositeCharset;
inData=udata_getMemory(pData);
/*
* get the data length if possible, to verify that swapping and preflighting
* handles the entire data
*/
dataLength=udata_getLength(pData);
/*
* get the header and its length
* all of the swap implementation functions require the header to be included
*/
inHeader=udata_getRawMemory(pData);
headerLength=(int32_t)((const char *)inData-(const char *)inHeader);
/* first swap to opposite endianness but same charset family */
errorCode=U_ZERO_ERROR;
ds=udata_openSwapperForInputData(inHeader, headerLength,
!U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, &errorCode);
if(U_FAILURE(errorCode)) {
log_err("udata_openSwapperForInputData(%s->!isBig+same charset) failed - %s\n",
name, u_errorName(errorCode));
return;
}
inEndian=ds->inIsBigEndian;
inCharset=ds->inCharset;
oppositeEndian=!inEndian;
oppositeCharset= inCharset==U_ASCII_FAMILY ? U_EBCDIC_FAMILY : U_ASCII_FAMILY;
/* make this test work with data files that are built for a different platform */
if(inEndian!=U_IS_BIG_ENDIAN || inCharset!=U_CHARSET_FAMILY) {
udata_closeSwapper(ds);
ds=udata_openSwapper(inEndian, inCharset, oppositeEndian, inCharset, &errorCode);
if(U_FAILURE(errorCode)) {
log_err("udata_openSwapper(%s->!isBig+same charset) failed - %s\n",
name, u_errorName(errorCode));
return;
}
}
ds->printError=printError;
/* preflight the length */
length=swapFn(ds, inHeader, -1, NULL, &errorCode);
if(U_FAILURE(errorCode)) {
log_err("swapFn(preflight %s->!isBig+same charset) failed - %s\n",
name, u_errorName(errorCode));
udata_closeSwapper(ds);
return;
}
/* compare the preflighted length against the data length */
if(dataLength>=0 && (length+15)<(headerLength+dataLength)) {
log_err("swapFn(preflight %s->!isBig+same charset) length too small: %d < data length %d\n",
name, length, (headerLength+dataLength));
udata_closeSwapper(ds);
return;
}
/* swap, not in-place */
length2=swapFn(ds, inHeader, length, buffer, &errorCode);
udata_closeSwapper(ds);
if(U_FAILURE(errorCode)) {
log_err("swapFn(%s->!isBig+same charset) failed - %s\n",
name, u_errorName(errorCode));
return;
}
/* compare the swap length against the preflighted length */
if(length2!=length) {
log_err("swapFn(%s->!isBig+same charset) length differs from preflighting: %d != preflighted %d\n",
name, length2, length);
return;
}
/* next swap to opposite charset family */
ds=udata_openSwapper(oppositeEndian, inCharset,
oppositeEndian, oppositeCharset,
&errorCode);
if(U_FAILURE(errorCode)) {
log_err("udata_openSwapper(%s->!isBig+other charset) failed - %s\n",
name, u_errorName(errorCode));
return;
}
ds->printError=printError;
/* swap in-place */
length2=swapFn(ds, buffer, length, buffer, &errorCode);
udata_closeSwapper(ds);
if(U_FAILURE(errorCode)) {
log_err("swapFn(%s->!isBig+other charset) failed - %s\n",
name, u_errorName(errorCode));
return;
}
/* compare the swap length against the original length */
if(length2!=length) {
log_err("swapFn(%s->!isBig+other charset) length differs from original: %d != original %d\n",
name, length2, length);
return;
}
/* finally swap to original platform values */
ds=udata_openSwapper(oppositeEndian, oppositeCharset,
inEndian, inCharset,
&errorCode);
if(U_FAILURE(errorCode)) {
log_err("udata_openSwapper(%s->back to original) failed - %s\n",
name, u_errorName(errorCode));
return;
}
ds->printError=printError;
/* swap, not in-place */
length2=swapFn(ds, buffer, length, buffer2, &errorCode);
udata_closeSwapper(ds);
if(U_FAILURE(errorCode)) {
log_err("swapFn(%s->back to original) failed - %s\n",
name, u_errorName(errorCode));
return;
}
/* compare the swap length against the original length */
if(length2!=length) {
log_err("swapFn(%s->back to original) length differs from original: %d != original %d\n",
name, length2, length);
return;
}
/* compare the final contents with the original */
if(0!=uprv_memcmp(inHeader, buffer2, length)) {
const uint8_t *original;
uint8_t diff[8];
int32_t i, j;
log_err("swapFn(%s->back to original) contents differs from original\n",
name);
/* find the first difference */
original=(const uint8_t *)inHeader;
for(i=0; i<length && original[i]==buffer2[i]; ++i) {}
/* find the next byte that is the same */
for(j=i+1; j<length && original[j]!=buffer2[j]; ++j) {}
log_info(" difference at index %d=0x%x, until index %d=0x%x\n", i, i, j, j);
/* round down to the last 4-boundary for better result output */
i&=~3;
log_info("showing bytes from index %d=0x%x (length %d=0x%x):\n", i, i, length, length);
/* print 8 bytes but limit to the buffer contents */
length2=i+sizeof(diff);
if(length2>length) {
length2=length;
}
/* print the original bytes */
uprv_memset(diff, 0, sizeof(diff));
for(j=i; j<length2; ++j) {
diff[j-i]=original[j];
}
log_info(" original: %02x %02x %02x %02x %02x %02x %02x %02x\n",
diff[0], diff[1], diff[2], diff[3], diff[4], diff[5], diff[6], diff[7]);
/* print the swapped bytes */
uprv_memset(diff, 0, sizeof(diff));
for(j=i; j<length2; ++j) {
diff[j-i]=buffer2[j];
}
log_info(" swapped: %02x %02x %02x %02x %02x %02x %02x %02x\n",
diff[0], diff[1], diff[2], diff[3], diff[4], diff[5], diff[6], diff[7]);
}
}
static UBool
haveAliasData(UErrorCode *pErrorCode) {
int needInit;
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
return FALSE;
}
UMTX_CHECK(NULL, (gAliasData==NULL), needInit);
/* load converter alias data from file if necessary */
if (needInit) {
UDataMemory *data;
const uint16_t *table;
const uint32_t *sectionSizes;
uint32_t tableStart;
uint32_t currOffset;
data = udata_openChoice(NULL, DATA_TYPE, DATA_NAME, isAcceptable, NULL, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
return FALSE;
}
sectionSizes = (const uint32_t *)udata_getMemory(data);
table = (const uint16_t *)sectionSizes;
tableStart = sectionSizes[0];
if (tableStart < minTocLength) {
*pErrorCode = U_INVALID_FORMAT_ERROR;
udata_close(data);
return FALSE;
}
umtx_lock(NULL);
if(gAliasData==NULL) {
gMainTable.converterListSize = sectionSizes[1];
gMainTable.tagListSize = sectionSizes[2];
gMainTable.aliasListSize = sectionSizes[3];
gMainTable.untaggedConvArraySize = sectionSizes[4];
gMainTable.taggedAliasArraySize = sectionSizes[5];
gMainTable.taggedAliasListsSize = sectionSizes[6];
gMainTable.optionTableSize = sectionSizes[7];
gMainTable.stringTableSize = sectionSizes[8];
if (tableStart > 8) {
gMainTable.normalizedStringTableSize = sectionSizes[9];
}
currOffset = tableStart * (sizeof(uint32_t)/sizeof(uint16_t)) + (sizeof(uint32_t)/sizeof(uint16_t));
gMainTable.converterList = table + currOffset;
currOffset += gMainTable.converterListSize;
gMainTable.tagList = table + currOffset;
currOffset += gMainTable.tagListSize;
gMainTable.aliasList = table + currOffset;
currOffset += gMainTable.aliasListSize;
gMainTable.untaggedConvArray = table + currOffset;
currOffset += gMainTable.untaggedConvArraySize;
gMainTable.taggedAliasArray = table + currOffset;
/* aliasLists is a 1's based array, but it has a padding character */
currOffset += gMainTable.taggedAliasArraySize;
gMainTable.taggedAliasLists = table + currOffset;
currOffset += gMainTable.taggedAliasListsSize;
if (gMainTable.optionTableSize > 0
&& ((const UConverterAliasOptions *)(table + currOffset))->stringNormalizationType < UCNV_IO_NORM_TYPE_COUNT)
{
/* Faster table */
gMainTable.optionTable = (const UConverterAliasOptions *)(table + currOffset);
}
else {
/* Smaller table, or I can't handle this normalization mode!
Use the original slower table lookup. */
gMainTable.optionTable = &defaultTableOptions;
}
currOffset += gMainTable.optionTableSize;
gMainTable.stringTable = table + currOffset;
currOffset += gMainTable.stringTableSize;
gMainTable.normalizedStringTable = ((gMainTable.optionTable->stringNormalizationType == UCNV_IO_UNNORMALIZED)
? gMainTable.stringTable : (table + currOffset));
ucln_common_registerCleanup(UCLN_COMMON_UCNV_IO, ucnv_io_cleanup);
gAliasData = data;
data=NULL;
}
umtx_unlock(NULL);
/* if a different thread set it first, then close the extra data */
if(data!=NULL) {
udata_close(data); /* NULL if it was set correctly */
}
}
return TRUE;
}
U_CDECL_END
#if !UNORM_HARDCODE_DATA
static int8_t
loadNormData(UErrorCode &errorCode) {
/* load Unicode normalization data from file */
/*
* This lazy intialization with double-checked locking (without mutex protection for
* haveNormData==0) is transiently unsafe under certain circumstances.
* Check the readme and use u_init() if necessary.
*
* While u_init() initializes the main normalization data via this functions,
* it does not do so for exclusion sets (which are fully mutexed).
* This is because
* - there can be many exclusion sets
* - they are rarely used
* - they are not usually used in execution paths that are
* as performance-sensitive as others
* (e.g., IDNA takes more time than unorm_quickCheck() anyway)
*
* TODO: Remove code in support for non-hardcoded data. u_init() is now advertised
* as not being required for thread safety, and we can't reasonably
* revert to requiring it.
*/
if(haveNormData==0) {
UTrie _normTrie={ 0,0,0,0,0,0,0 }, _fcdTrie={ 0,0,0,0,0,0,0 }, _auxTrie={ 0,0,0,0,0,0,0 };
UDataMemory *data;
const int32_t *p=NULL;
const uint8_t *pb;
if(&errorCode==NULL || U_FAILURE(errorCode)) {
return 0;
}
/* open the data outside the mutex block */
data=udata_openChoice(NULL, DATA_TYPE, DATA_NAME, isAcceptable, NULL, &errorCode);
dataErrorCode=errorCode;
if(U_FAILURE(errorCode)) {
return haveNormData=-1;
}
p=(const int32_t *)udata_getMemory(data);
pb=(const uint8_t *)(p+_NORM_INDEX_TOP);
utrie_unserialize(&_normTrie, pb, p[_NORM_INDEX_TRIE_SIZE], &errorCode);
_normTrie.getFoldingOffset=getFoldingNormOffset;
pb+=p[_NORM_INDEX_TRIE_SIZE]+p[_NORM_INDEX_UCHAR_COUNT]*2+p[_NORM_INDEX_COMBINE_DATA_COUNT]*2;
if(p[_NORM_INDEX_FCD_TRIE_SIZE]!=0) {
utrie_unserialize(&_fcdTrie, pb, p[_NORM_INDEX_FCD_TRIE_SIZE], &errorCode);
}
pb+=p[_NORM_INDEX_FCD_TRIE_SIZE];
if(p[_NORM_INDEX_AUX_TRIE_SIZE]!=0) {
utrie_unserialize(&_auxTrie, pb, p[_NORM_INDEX_AUX_TRIE_SIZE], &errorCode);
_auxTrie.getFoldingOffset=getFoldingAuxOffset;
}
if(U_FAILURE(errorCode)) {
dataErrorCode=errorCode;
udata_close(data);
return haveNormData=-1;
}
/* in the mutex block, set the data for this process */
umtx_lock(NULL);
if(normData==NULL) {
normData=data;
data=NULL;
uprv_memcpy(&indexes, p, sizeof(indexes));
uprv_memcpy(&normTrie, &_normTrie, sizeof(UTrie));
uprv_memcpy(&fcdTrie, &_fcdTrie, sizeof(UTrie));
uprv_memcpy(&auxTrie, &_auxTrie, sizeof(UTrie));
} else {
p=(const int32_t *)udata_getMemory(normData);
}
/* initialize some variables */
extraData=(uint16_t *)((uint8_t *)(p+_NORM_INDEX_TOP)+indexes[_NORM_INDEX_TRIE_SIZE]);
combiningTable=extraData+indexes[_NORM_INDEX_UCHAR_COUNT];
formatVersion_2_1=formatVersion[0]>2 || (formatVersion[0]==2 && formatVersion[1]>=1);
formatVersion_2_2=formatVersion[0]>2 || (formatVersion[0]==2 && formatVersion[1]>=2);
if(formatVersion_2_1) {
canonStartSets=combiningTable+
indexes[_NORM_INDEX_COMBINE_DATA_COUNT]+
(indexes[_NORM_INDEX_FCD_TRIE_SIZE]+indexes[_NORM_INDEX_AUX_TRIE_SIZE])/2;
}
haveNormData=1;
ucln_common_registerCleanup(UCLN_COMMON_UNORM, unorm_cleanup);
umtx_unlock(NULL);
/* if a different thread set it first, then close the extra data */
if(data!=NULL) {
udata_close(data); /* NULL if it was set correctly */
}
}
return haveNormData;
}
