// ---------------------------------------------------------------------------
// RangeToken: Getter methods
// ---------------------------------------------------------------------------
RangeToken* RangeToken::getCaseInsensitiveToken(TokenFactory* const tokFactory) {
if (fCaseIToken == 0 && tokFactory) {
bool isNRange = (getTokenType() == T_NRANGE) ? true : false;
RangeToken* lwrToken = tokFactory->createRange(isNRange);
for (unsigned int i = 0; i < fElemCount - 1; i += 2) {
for (XMLInt32 ch = fRanges[i]; ch <= fRanges[i + 1]; ++ch) {
#if defined(XML_USE_ICU_TRANSCODER) || defined (XML_USE_UNICONV390_TRANSCODER)
const XMLInt32 upperCh = u_toupper(ch);
if (upperCh != ch)
{
lwrToken->addRange(upperCh, upperCh);
}
const XMLInt32 lowerCh = u_tolower(ch);
if (lowerCh != ch)
{
lwrToken->addRange(lowerCh, lowerCh);
}
const XMLInt32 titleCh = u_totitle(ch);
if (titleCh != ch && titleCh != upperCh)
{
lwrToken->addRange(titleCh, titleCh);
}
#else
if (ch >= chLatin_A && ch <= chLatin_Z)
{
ch += chLatin_a - chLatin_A;
lwrToken->addRange(ch, ch);
}
else if (ch >= chLatin_a && ch <= chLatin_z)
{
ch -= chLatin_a - chLatin_A;
lwrToken->addRange(ch, ch);
}
#endif
}
}
lwrToken->mergeRanges(this);
lwrToken->compactRanges();
lwrToken->createMap();
fCaseIToken = lwrToken;
}
return fCaseIToken;
}
// ---------------------------------------------------------------------------
// BlockRangeFactory: Range creation methods
// ---------------------------------------------------------------------------
void BlockRangeFactory::buildRanges(RangeTokenMap *rangeTokMap) {
if (fRangesCreated)
return;
if (!fKeywordsInitialized) {
initializeKeywordMap(rangeTokMap);
}
TokenFactory* tokFactory = rangeTokMap->getTokenFactory();
//for performance, once the desired specials and private use are found
//don't need to compareString anymore
bool foundSpecial = false;
bool foundPrivate = false;
for (int i=0; i < BLOCKNAMESIZE; i++) {
RangeToken* tok = tokFactory->createRange();
tok->addRange(blockRanges[i*2], blockRanges[(i*2)+1]);
if (!foundSpecial && XMLString::equals((XMLCh*)fgBlockNames[i] , (XMLCh*) fgBlockIsSpecials)) {
tok->addRange(0xFFF0, 0xFFFD);
foundSpecial = true;
}
if (!foundPrivate && XMLString::equals((XMLCh*)fgBlockNames[i] , (XMLCh*) fgBlockIsPrivateUse)) {
tok->addRange(0xF0000, 0xFFFFD);
tok->addRange(0x100000, 0x10FFFD);
foundPrivate = true;
}
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgBlockNames[i], tok);
tok = (RangeToken*) RangeToken::complementRanges(tok, tokFactory);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgBlockNames[i], tok , true);
}
fRangesCreated = true;
}
// ---------------------------------------------------------------------------
// ASCIIRangeFactory: Range creation methods
// ---------------------------------------------------------------------------
void ASCIIRangeFactory::buildRanges(RangeTokenMap *rangeTokMap) {
if (fRangesCreated)
return;
if (!fKeywordsInitialized) {
initializeKeywordMap(rangeTokMap);
}
TokenFactory* tokFactory = rangeTokMap->getTokenFactory();
// Create space ranges
RangeToken* tok = tokFactory->createRange();
tok->addRange(chHTab, chHTab);
tok->addRange(chLF, chLF);
tok->addRange(chFF, chFF);
tok->addRange(chCR, chCR);
tok->addRange(chSpace, chSpace);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgASCIISpace, tok);
tok = (RangeToken*) RangeToken::complementRanges(tok, tokFactory);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgASCIISpace, tok , true);
// Create digits ranges
tok = tokFactory->createRange();
tok->addRange(chDigit_0, chDigit_9);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgASCIIDigit, tok);
tok = (RangeToken*) RangeToken::complementRanges(tok, tokFactory);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgASCIIDigit, tok , true);
// Create word ranges
tok = tokFactory->createRange();
tok->addRange(chDigit_0, chDigit_9);
tok->addRange(chLatin_A, chLatin_Z);
tok->addRange(chUnderscore, chUnderscore);
tok->addRange(chLatin_a, chLatin_z);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgASCIIWord, tok);
tok = (RangeToken*) RangeToken::complementRanges(tok, tokFactory);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgASCIIWord, tok , true);
// Create xdigit ranges
tok = tokFactory->createRange();
tok->addRange(chDigit_0, chDigit_9);
tok->addRange(chLatin_A, chLatin_F);
tok->addRange(chLatin_a, chLatin_a);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgASCIIXDigit, tok);
tok = (RangeToken*) RangeToken::complementRanges(tok, tokFactory);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgASCIIXDigit, tok , true);
// Create ascii ranges
tok = tokFactory->createRange();
tok->addRange(0x00, 0x7F);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgASCII, tok);
tok = (RangeToken*) RangeToken::complementRanges(tok, tokFactory);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgASCII, tok , true);
fRangesCreated = true;
}
// ---------------------------------------------------------------------------
// XMLRangeFactory: Range creation methods
// ---------------------------------------------------------------------------
void XMLRangeFactory::buildRanges(RangeTokenMap *rangeTokMap) {
if (fRangesCreated)
return;
if (!fKeywordsInitialized) {
initializeKeywordMap(rangeTokMap);
}
TokenFactory* tokFactory = rangeTokMap->getTokenFactory();
// Create space ranges
unsigned int wsTblLen = getTableLen(gWhitespaceChars);
RangeToken* tok = tokFactory->createRange();
XMLInt32* wsRange = (XMLInt32*) XMLPlatformUtils::fgMemoryManager->allocate
(
wsTblLen * sizeof(XMLInt32)
);//new XMLInt32[wsTblLen];
tok->setRangeValues(wsRange, wsTblLen);
setupRange(wsRange, gWhitespaceChars, 0);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgXMLSpace, tok);
tok = (RangeToken*) RangeToken::complementRanges(tok, tokFactory);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgXMLSpace, tok , true);
// Create digits ranges
tok = tokFactory->createRange();
unsigned int digitTblLen = getTableLen(gDigitChars);
XMLInt32* digitRange = (XMLInt32*) XMLPlatformUtils::fgMemoryManager->allocate
(
digitTblLen * sizeof(XMLInt32)
);//new XMLInt32[digitTblLen];
tok->setRangeValues(digitRange, digitTblLen);
setupRange(digitRange, gDigitChars, 0);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgXMLDigit, tok);
tok = (RangeToken*) RangeToken::complementRanges(tok, tokFactory);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgXMLDigit, tok , true);
// Build word ranges
unsigned int baseTblLen = getTableLen(gBaseChars);
unsigned int ideoTblLen = getTableLen(gIdeographicChars);
unsigned int wordRangeLen = baseTblLen + ideoTblLen + digitTblLen;
XMLInt32* wordRange = (XMLInt32*) XMLPlatformUtils::fgMemoryManager->allocate
(
wordRangeLen * sizeof(XMLInt32)
);//new XMLInt32[wordRangeLen];
ArrayJanitor<XMLInt32> janWordRange(wordRange, XMLPlatformUtils::fgMemoryManager);
setupRange(wordRange, gBaseChars, 0);
setupRange(wordRange, gIdeographicChars, baseTblLen);
memcpy(wordRange + baseTblLen + ideoTblLen, digitRange, digitTblLen * sizeof(XMLInt32));
// Create NameChar ranges
tok = tokFactory->createRange();
unsigned int combTblLen = getTableLen(gCombiningChars);
unsigned int extTblLen = getTableLen(gExtenderChars);
unsigned int nameTblLen = wordRangeLen + combTblLen + extTblLen;
XMLInt32* nameRange = (XMLInt32*) XMLPlatformUtils::fgMemoryManager->allocate
(
(nameTblLen + 8) * sizeof(XMLInt32)
);//new XMLInt32[nameTblLen + 8];
tok->setRangeValues(nameRange, nameTblLen + 8);
memcpy(nameRange, wordRange, wordRangeLen * sizeof(XMLInt32));
setupRange(nameRange, gCombiningChars, wordRangeLen);
setupRange(nameRange, gExtenderChars, wordRangeLen + combTblLen);
nameRange[nameTblLen++] = chDash;
nameRange[nameTblLen++] = chDash;
nameRange[nameTblLen++] = chColon;
nameRange[nameTblLen++] = chColon;
nameRange[nameTblLen++] = chPeriod;
nameRange[nameTblLen++] = chPeriod;
nameRange[nameTblLen++] = chUnderscore;
nameRange[nameTblLen++] = chUnderscore;
tok->sortRanges();
tok->compactRanges();
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgXMLNameChar, tok);
tok = (RangeToken*) RangeToken::complementRanges(tok, tokFactory);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgXMLNameChar, tok , true);
// Create initialNameChar ranges
tok = tokFactory->createRange();
unsigned int initialNameTblLen = baseTblLen + ideoTblLen;
XMLInt32* initialNameRange = (XMLInt32*) XMLPlatformUtils::fgMemoryManager->allocate
(
(initialNameTblLen + 4) * sizeof(XMLInt32)
);//new XMLInt32[initialNameTblLen + 4];
tok->setRangeValues(initialNameRange, initialNameTblLen + 4);
memcpy(initialNameRange, wordRange, initialNameTblLen * sizeof(XMLInt32));
initialNameRange[initialNameTblLen++] = chColon;
initialNameRange[initialNameTblLen++] = chColon;
initialNameRange[initialNameTblLen++] = chUnderscore;
initialNameRange[initialNameTblLen++] = chUnderscore;
tok->sortRanges();
tok->compactRanges();
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgXMLInitialNameChar, tok);
tok = (RangeToken*) RangeToken::complementRanges(tok, tokFactory);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgXMLInitialNameChar, tok , true);
// Create word range
tok = tokFactory->createRange();
tok->setRangeValues(wordRange, wordRangeLen);
janWordRange.orphan();
tok->sortRanges();
tok->compactRanges();
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgXMLWord, tok);
tok = (RangeToken*) RangeToken::complementRanges(tok, tokFactory);
// Build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgXMLWord, tok , true);
fRangesCreated = true;
}
/*
* Prepares for matching. This method is called during construction.
*/
void RegularExpression::prepare() {
compile(fTokenTree);
fMinLength = fTokenTree->getMinLength();
fFirstChar = 0;
if (!isSet(fOptions, PROHIBIT_HEAD_CHARACTER_OPTIMIZATION) &&
!isSet(fOptions, XMLSCHEMA_MODE)) {
RangeToken* rangeTok = fTokenFactory->createRange();
int result = fTokenTree->analyzeFirstCharacter(rangeTok, fOptions, fTokenFactory);
if (result == Token::FC_TERMINAL) {
rangeTok->compactRanges();
fFirstChar = rangeTok;
}
rangeTok->createMap();
if (isSet(fOptions, IGNORE_CASE))
{
rangeTok->getCaseInsensitiveToken(fTokenFactory);
}
}
if (fOperations != 0 && fOperations->getNextOp() == 0 &&
(fOperations->getOpType() == Op::O_STRING ||
fOperations->getOpType() == Op::O_CHAR) &&
!isSet(fOptions, IGNORE_CASE) ) {
fFixedStringOnly = true;
if (fOperations->getOpType() == Op::O_STRING) {
fMemoryManager->deallocate(fFixedString);//delete [] fFixedString;
fFixedString = XMLString::replicate(fOperations->getLiteral(), fMemoryManager);
}
else{
XMLInt32 ch = fOperations->getData();
if ( ch >= 0x10000) { // add as constant
fMemoryManager->deallocate(fFixedString);//delete [] fFixedString;
fFixedString = RegxUtil::decomposeToSurrogates(ch, fMemoryManager);
}
else {
XMLCh* dummyStr = (XMLCh*) fMemoryManager->allocate(2 * sizeof(XMLCh));//new XMLCh[2];
dummyStr[0] = (XMLCh) fOperations->getData();
dummyStr[1] = chNull;
fMemoryManager->deallocate(fFixedString);//delete [] fFixedString;
fFixedString = dummyStr;
}
}
fBMPattern = new (fMemoryManager) BMPattern(fFixedString, 256,
isSet(fOptions, IGNORE_CASE), fMemoryManager);
}
else if (!isSet(fOptions, XMLSCHEMA_MODE) &&
!isSet(fOptions, PROHIBIT_FIXED_STRING_OPTIMIZATION) &&
!isSet(fOptions, IGNORE_CASE)) {
int fixedOpts = 0;
Token* tok = fTokenTree->findFixedString(fOptions, fixedOpts);
fMemoryManager->deallocate(fFixedString);//delete [] fFixedString;
fFixedString = (tok == 0) ? 0
: XMLString::replicate(tok->getString(), fMemoryManager);
if (fFixedString != 0 && XMLString::stringLen(fFixedString) < 2) {
fMemoryManager->deallocate(fFixedString);//delete [] fFixedString;
fFixedString = 0;
}
if (fFixedString != 0) {
fBMPattern = new (fMemoryManager) BMPattern(fFixedString, 256,
isSet(fixedOpts, IGNORE_CASE), fMemoryManager);
}
}
}
// ---------------------------------------------------------------------------
// UnicodeRangeFactory: Range creation methods
// ---------------------------------------------------------------------------
void UnicodeRangeFactory::buildRanges(RangeTokenMap *rangeTokMap) {
if (fRangesCreated)
return;
if (!fKeywordsInitialized) {
initializeKeywordMap(rangeTokMap);
}
TokenFactory* tokFactory = rangeTokMap->getTokenFactory();
RangeToken* ranges[UNICATEGSIZE];
RangeToken* tok;
for (int i=0; i < UNICATEGSIZE; i++) {
ranges[i] = tokFactory->createRange();
}
for (int j=0; j < 0x10000; j++) {
unsigned short charType = XMLUniCharacter::getType(j);
ranges[charType]->addRange(j, j);
charType = getUniCategory(charType);
ranges[charType]->addRange(j, j);
}
ranges[XMLUniCharacter::UNASSIGNED]->addRange(0x10000, Token::UTF16_MAX);
for (int k=0; k < UNICATEGSIZE; k++) {
tok = (RangeToken*) RangeToken::complementRanges(ranges[k], tokFactory);
// build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(uniCategNames[k], ranges[k]);
rangeTokMap->setRangeToken(uniCategNames[k], tok , true);
}
// Create all range
tok = tokFactory->createRange();
tok->addRange(0, Token::UTF16_MAX);
// build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgUniAll, tok);
// Create alpha range
tok = tokFactory->createRange();
tok->mergeRanges(ranges[XMLUniCharacter::UPPERCASE_LETTER]);
tok->mergeRanges(ranges[XMLUniCharacter::LOWERCASE_LETTER]);
tok->mergeRanges(ranges[XMLUniCharacter::OTHER_LETTER]);
// build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgUniIsAlpha, tok);
// Create alpha-num range
RangeToken* alnumTok = tokFactory->createRange();
alnumTok->mergeRanges(tok);
alnumTok->mergeRanges(ranges[XMLUniCharacter::DECIMAL_DIGIT_NUMBER]);
// build the internal map.
alnumTok->createMap();
rangeTokMap->setRangeToken(fgUniIsAlnum, alnumTok);
// Create word range
tok = tokFactory->createRange();
tok->mergeRanges(alnumTok);
tok->addRange(chUnderscore, chUnderscore);
// build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgUniIsWord, tok);
tok = (RangeToken*) RangeToken::complementRanges(tok, tokFactory);
// build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgUniIsWord, tok , true);
// Create assigned range
tok = (RangeToken*)RangeToken::complementRanges(
ranges[XMLUniCharacter::UNASSIGNED],
tokFactory,
tokFactory->getMemoryManager());
// build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgUniAssigned,tok);
// Create space range
tok = tokFactory->createRange();
tok->mergeRanges(ranges[XMLUniCharacter::SPACE_SEPARATOR]);
tok->mergeRanges(ranges[XMLUniCharacter::LINE_SEPARATOR]);
//tok->mergeRanges(ranges[XMLUniCharacter::PARAGRAPH_SEPARATOR]);
// build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgUniIsSpace, tok);
tok = (RangeToken*) RangeToken::complementRanges(tok, tokFactory);
// build the internal map.
tok->createMap();
rangeTokMap->setRangeToken(fgUniIsSpace, tok , true);
// build the internal maps.
for (int l=0; l < UNICATEGSIZE; l++) {
ranges[l]->createMap();
}
fRangesCreated = true;
}
// ---------------------------------------------------------------------------
// RangeToken: Getter methods
// ---------------------------------------------------------------------------
RangeToken* RangeToken::getCaseInsensitiveToken(TokenFactory* const tokFactory) {
if (fCaseIToken == 0 && tokFactory && fRanges) {
bool isNRange = (getTokenType() == T_NRANGE) ? true : false;
RangeToken* lwrToken = tokFactory->createRange(isNRange);
#if XERCES_USE_TRANSCODER_ICU && ((U_ICU_VERSION_MAJOR_NUM > 2) || (U_ICU_VERSION_MAJOR_NUM == 2 && U_ICU_VERSION_MINOR_NUM >=4))
UChar* rangeStr=(UChar*)fMemoryManager->allocate(40*fElemCount*sizeof(UChar));
ArrayJanitor<UChar> janRange(rangeStr, fMemoryManager);
int c=0;
rangeStr[c++] = chOpenSquare;
for (unsigned int i = 0; i < fElemCount - 1; i += 2) {
XMLCh buffer[10];
XMLSize_t len, j;
rangeStr[c++] = chBackSlash;
rangeStr[c++] = chLatin_U;
XMLString::binToText(fRanges[i], buffer, 10, 16, fMemoryManager);
len = XMLString::stringLen(buffer);
for(j=0;j<(8-len);j++)
rangeStr[c++] = chDigit_0;
XMLCh* p=buffer;
while(*p)
rangeStr[c++] = *p++;
if(fRanges[i+1]!=fRanges[i])
{
rangeStr[c++] = chDash;
rangeStr[c++] = chBackSlash;
rangeStr[c++] = chLatin_U;
XMLString::binToText(fRanges[i+1], buffer, 10, 16, fMemoryManager);
len = XMLString::stringLen(buffer);
for(j=0;j<(8-len);j++)
rangeStr[c++] = chDigit_0;
p=buffer;
while(*p)
rangeStr[c++] = *p++;
}
}
rangeStr[c++] = chCloseSquare;
rangeStr[c++] = chNull;
UErrorCode ec=U_ZERO_ERROR;
USet* range=uset_openPatternOptions(rangeStr, -1, USET_CASE_INSENSITIVE, &ec);
if(range)
{
ec = U_ZERO_ERROR;
uint32_t cbCount=uset_serialize(range, NULL, 0, &ec);
uint16_t* buffer=(uint16_t*)fMemoryManager->allocate(cbCount*sizeof(uint16_t));
ArrayJanitor<uint16_t> janSet(buffer, fMemoryManager);
ec = U_ZERO_ERROR;
uset_serialize(range, buffer, cbCount, &ec);
USerializedSet serializedSet;
uset_getSerializedSet(&serializedSet, buffer, cbCount);
int32_t nSets=uset_getSerializedRangeCount(&serializedSet);
for(int32_t i=0; i<nSets; i++)
{
UChar32 start, end;
uset_getSerializedRange(&serializedSet, i, &start, &end);
lwrToken->addRange(start, end);
}
// does this release the memory allocated by the set?
uset_setSerializedToOne(&serializedSet, 32);
uset_close(range);
}
#else
unsigned int exceptIndex = 0;
for (unsigned int i = 0; i < fElemCount - 1; i += 2) {
for (XMLInt32 ch = fRanges[i]; ch <= fRanges[i + 1]; ++ch) {
#if XERCES_USE_TRANSCODER_ICU
const XMLInt32 upperCh = u_toupper(ch);
if (upperCh != ch)
{
lwrToken->addRange(upperCh, upperCh);
}
const XMLInt32 lowerCh = u_tolower(ch);
if (lowerCh != ch)
{
lwrToken->addRange(lowerCh, lowerCh);
}
const XMLInt32 titleCh = u_totitle(ch);
if (titleCh != ch && titleCh != upperCh)
{
lwrToken->addRange(titleCh, titleCh);
}
#else
if (ch >= chLatin_A && ch <= chLatin_Z)
{
ch += chLatin_a - chLatin_A;
lwrToken->addRange(ch, ch);
}
else if (ch >= chLatin_a && ch <= chLatin_z)
{
ch -= chLatin_a - chLatin_A;
lwrToken->addRange(ch, ch);
}
#endif
const unsigned int exceptionsSize =
sizeof(s_exceptions) / sizeof(s_exceptions[0]);
// Add any exception chars. These are characters where the the
// case mapping is not symmetric. (Unicode case mappings are not isomorphic...)
while (exceptIndex < exceptionsSize)
{
if (s_exceptions[exceptIndex].baseChar < ch)
{
++exceptIndex;
}
else if (s_exceptions[exceptIndex].baseChar == ch)
{
const XMLInt32 matchingChar =
s_exceptions[exceptIndex].matchingChar;
lwrToken->addRange(
matchingChar,
matchingChar);
++exceptIndex;
}
else
{
break;
}
}
}
}
lwrToken->mergeRanges(this);
#endif
lwrToken->compactRanges();
lwrToken->createMap();
fCaseIToken = lwrToken;
// TODO(dbertoni) This is a temporary hack until we can change the ABI.
// See Jira issue XERCESC-1866 for more details.
// Overload the fCaseIToken data member to be the case-insensitive token
// that's caching the case-insensitive one. We need this because tokens
// have varying lifetimes.
fCaseIToken->setCaseInsensitiveToken(this);
}
return fCaseIToken;
}
