/* ------------------------------------------------------------------------------- Class: CStifSectionParser Method: GotoEndOfLine Description: Goes end of the line. Parameters: TLex& lex: inout: Parsed line. Return Values: TInt: Last item's end position. Errors/Exceptions: None Status: Proposal ------------------------------------------------------------------------------- */ TInt CStifSectionParser::GotoEndOfLine( TLex& lex ) { // End position of the last token(Initialized with current position) TInt lastItemPosition( lex.Offset() ); // LINE BREAK NOTE: // Line break in SOS, WIN: '\r\n' // Line break in UNIX: '\n' do { // Peek next character(10 or '\n' in UNIX style ) if( lex.Peek() == 0x0A ) { lex.Inc(); break; } // Peek next character(13 or '\r' in Symbian OS) if ( lex.Peek() == 0x0D ) { // Increment the lex position lex.Inc(); // Peek next character(10 or '\n' in Symbian OS) if ( lex.Peek() == 0x0A ) { // End of the section is found and increment the lex position lex.Inc(); break; } // 0x0A not found, decrement position lex.UnGet(); } // Peek for tabulator(0x09) and space(0x20) else if ( lex.Peek() == 0x09 || lex.Peek() == 0x20 ) { // Increment the lex position lex.Inc(); continue; } // If white spaces not found take next token lex.NextToken(); lastItemPosition = lex.Offset(); } while ( !lex.Eos() ); return lastItemPosition; }
TInt SkipToNextCommand(TLex& aLex) { if ((aLex.Offset() == 0) && (aLex.Remainder().Find(KPartialCommandStart) == 0)) { return KErrNone; } return SkipTo(aLex, KCommandStart); }
void CScriptFile::FoundNewItemL(const TDesC& aText, TLex& arInput, TInt& arCurrentItemStart, CScriptSection& aSection, CScriptSectionItem*& arCurrentItem) { TPtrC token = arInput.MarkedToken(); ParseAndSetItemValueL(aText, arInput, arCurrentItemStart, arCurrentItem); arInput.SkipSpaceAndMark(); arCurrentItemStart = arInput.Offset(); TPtrC itemEnd(KScriptItemEnd); const TInt length = token.Length() - itemEnd.Length(); arCurrentItem = &aSection.ReplaceItemL(token.Left(length), KNullDesC); }
// ---------------------------------------------------- // CheckIPv6Address() // IPv6address = hexpart [ ":" IPv4address ] // ---------------------------------------------------- // LOCAL_C TBool CheckIPv6Address( TLex& aLex ) { TBool result( CheckIPv6HexPart( aLex ) ); if( aLex.Peek() == '.' ) { while( aLex.Peek() != ':' && aLex.Offset() > 0 ) { aLex.UnGet(); } aLex.Inc(); result = CheckIPv4Address( aLex ); } return result && aLex.Eos(); }
void CCommandInfoFile::ReadDetailsL(TLex& aLex, RFs& aFs, const TDesC& aFileName) { SkipToNextCommand(aLex); // Ignore everything before the first '==' command. while (!aLex.Eos()) { TLexMark mark; aLex.Mark(mark); TPtrC command(NextCommand(aLex)); if (command == KNullDesC) { // Do nothing - we're at the end of the string. } else if (command == KCmndName) { aLex.SkipSpaceAndMark(); aLex.SkipCharacters(); iName.Set(aLex.MarkedToken()); } else if (command == KCmndShortDescription) { iShortDescription.Set(TextToNextCommand(aLex)); } else if (command == KCmndLongDescription) { iLongDescription.Set(TextToNextCommand(aLex)); } else if (command == KCmndSeeAlso) { iSeeAlso.Set(TextToNextCommand(aLex)); } else if (command == KCmndCopyright) { iCopyright.Set(TextToNextCommand(aLex)); } else if (command == KCmndSmokeTest) { // Hmm no easy way to get the line number we're currently on iSmokeTestLineNumber = 1; TLex lex(aLex); lex.Inc(-aLex.Offset()); // Only way to put a TLex back to the beginning! TPtrC preceding = lex.Remainder().Left(aLex.Offset()); const TUint16* ptr = preceding.Ptr(); const TUint16* end = ptr + preceding.Length(); while (ptr != end) { if (*ptr++ == '\n') iSmokeTestLineNumber++; } // At this point iSmokeTestLineNumber points to the "==smoketest" line - add 2 to skip this line and the blank line below it iSmokeTestLineNumber += 2; iSmokeTest.Set(TextToNextCommand(aLex)); } else if (command == KCmndArgument) { ReadArgumentL(aLex, aFileName); } else if (command == KCmndOption) { ReadOptionL(aLex, aFileName); } else if (command == KCmndInclude) { if (iParent == NULL) { iProcessInclude = EFalse; TLex lineLex(LineRemainder(aLex)); TPtrC fileName(NextWord(lineLex)); TFileName2* fullFileName = new(ELeave) TFileName2(aFileName); CleanupStack::PushL(fullFileName); fullFileName->SetNameAndExtL(fileName); ReadFileL(aFs, *fullFileName); CleanupStack::PopAndDestroy(fullFileName); break; } else { // We're a sub-command. Let control return to the root to handle the include. aLex.UnGetToMark(mark); iParent->ProcessInclude(*this); break; } } else if (command == KCmndSubCommand) { if (iParent == NULL) { TLex lineLex(LineRemainder(aLex)); AddSubCommandL(lineLex, aLex, aFs, aFileName); } else { // We're a sub-command. Let control return to the root to handle the next sub-command. aLex.UnGetToMark(mark); iParent->ProcessNewChild(); break; } } else { StaticLeaveIfErr(KErrArgument, _L("Unknown command \"%S\" in \"%S\""), &command, &aFileName); } } }
/** Search fixed start URLs, i.e. URLs without schema (www., wap.). Also finds IPv4 addresses (*.*.*.*). As a special case, supports deprecated hardcoded schematic addresses finding (http://, https://, rtsp://) to make sure deprecated search cases work as they did previously. @param aText Text that will be parsed @param aFindFixedSchemas If true, will find old fixed schematic URLs also @return ETrue if any URL are found else returns EFalse @leave KErrNone, if successful; otherwise one of the other system-wide error codes. @panic ETulPanicDescriptorLength in debug build if item's position and/or length is out of the document's range. */ TBool CTulAddressStringTokenizer::SearchUrlL( const TDesC& aText, const TBool aFindFixedSchemas ) { TLex text = aText; while ( !text.Eos() ) { while( !(text.Eos()) && !IsValidUrlChar( text.Peek() ) ) text.Inc(); text.Mark(); while( !(text.Eos()) && IsValidUrlChar( text.Peek() ) ) text.Inc(); TPtrC tokenPtr = text.MarkedToken(); TBool wasValidUrl = EFalse; if ( aFindFixedSchemas ) // Search for http:// wasValidUrl = ParseUrlL( KHttpUrlAddress, tokenPtr, text.Offset() ); if (aFindFixedSchemas && !wasValidUrl) // Search for https:// wasValidUrl = ParseUrlL( KHttpsUrlAddress, tokenPtr, text.Offset() ); if (aFindFixedSchemas && !wasValidUrl) // Search for rtsp:// wasValidUrl = ParseUrlL( KRtspUrlAddress, tokenPtr, text.Offset() ); if ( !wasValidUrl ) // Search for www. wasValidUrl = ParseUrlL( KWwwUrlAddress, tokenPtr, text.Offset() ); if ( !wasValidUrl ) // Search for wap. wasValidUrl = ParseUrlL( KWapUrlAddress, tokenPtr, text.Offset() ); if ( !wasValidUrl ) // Search for IP-address (xxx.xxx.xxx.xxx) { if ( tokenPtr.Match( KIPAddress ) != KErrNotFound ) { TInt periods = 0; wasValidUrl = ETrue; TBool endWithPunctuation = EFalse; TBool betweenBrackets = EFalse; // First see if token ends with ",",".","!","?",";" or ":" TChar charac = tokenPtr[tokenPtr.Length() - 1]; TChar charac0 = tokenPtr[0]; if ( charac == ',' || charac == '.' || charac == '!' || charac == '?' || charac == ';' || charac == ':' ) { endWithPunctuation = ETrue; } // Or if it starts and ends with brackets or quotation marks else if ( ( charac0 == '(' && charac == ')' ) || ( charac0 == '"' && charac == '"' ) || ( charac0 == '[' && charac == ']' ) || ( charac0 == '<' && charac == '>' ) ) { betweenBrackets = ETrue; } TInt i = 0; TInt tokensEnd = tokenPtr.Length(); if ( endWithPunctuation ) tokensEnd--; else if ( betweenBrackets ) { i = 1; tokensEnd--; } // Take a closer look to see if a valid IP-address TBuf<3> ipPart; TInt numbers = 0; for ( ; i < tokensEnd; i++ ) { if ( !( ((TChar)tokenPtr[i]).IsDigit() || tokenPtr[i] == '.' ) ) { wasValidUrl = EFalse; break; } if ( tokenPtr[i] == '.' ) periods++; else numbers++; if ( numbers > KNumbersInIpAddress || periods > KDotsInIpAddress ) { wasValidUrl = EFalse; break; } if ( ((TChar)tokenPtr[i]).IsDigit() ) { ipPart.Append( tokenPtr[i] ); TBool checkInt = EFalse; if ( i + 1 < tokensEnd ) { if ( tokenPtr[i+1] == '.' ) checkInt = ETrue; } if ( i == tokensEnd - 1 || checkInt ) { TLex val = ipPart; TInt numberInt; TInt error = val.Val( numberInt ); if ( error != KErrNone || numberInt > 255 ) { wasValidUrl = EFalse; break; } numbers = 0; ipPart.Delete( 0, ipPart.Length() ); } } } if ( wasValidUrl && periods == KDotsInIpAddress ) { TInt startPos = text.Offset() - tokenPtr.Length(); TInt length = tokenPtr.Length(); // If there was a punctuation at the end or brackets, let's take it/them away if ( endWithPunctuation || betweenBrackets) { length--; if ( betweenBrackets ) { startPos++; length--; } } __ASSERT_DEBUG( startPos + length <= aText.Length(), Panic(ETulPanicDescriptorLength) ); AddItemL( startPos, length, EFindItemSearchURLBin ); } } } } return (iFoundItems->Count() > 0); }
/** Search algorithm for searching phone numbers @param aText Text that will be parsed @return ETrue if any Phone Number items were found else returns EFalse @leave KErrNone, if successful; otherwise one of the other system-wide error codes. @panic ETulPanicDescriptorLength in debug build if item's position and/or length is out of the document's range. */ TBool CTulAddressStringTokenizer::SearchPhoneNumberL( const TDesC& aText ) { TLexMark startMark; // Points to the start of the found phone number TLexMark endMark; // Points to the end of the found phone number TLexMark mark; const TInt end = aText.Length(); TLex number = aText; while ( !(number.Eos()) ) { TInt numberCount = 0; // How many real numbers (1234567890) TInt bracketsOpen = 0; // How many brackets are currently open TInt brackets = 0; // How many brackets overall TChar charac = number.Peek(); while( (!(IsValidPhoneNumberChar( charac ) || charac == '+' || charac == '(' ) || charac == '-' || charac == '.' || charac == '/') && !(number.Eos()) && number.Offset() < end ) { number.Inc(); charac = number.Peek(); } if ( number.Offset() >= end ) break; if ( number.Peek() == '#' ) { number.Inc(); if (number.Peek() == '.' ) continue; number.UnGet(); } if ( number.Peek() == '+' ) { // '+' has to be followed by a number (not # or * ...) number.Inc(); if ( !(number.Peek().IsDigit()) ) continue; number.UnGet(); } if ( number.Peek() == '(' ) { // '(' has to be followed by valid phone number // character (whitespaces are allowed before) or '+' is a next character number.Inc(); if ( !(number.Peek() == '+') ) { number.Mark(mark); number.SkipSpace(); charac = number.Peek(); if ( !( IsValidPhoneNumberChar(charac) || charac == '+' || charac == '(' ) || charac == '-' || charac == '.' || charac == '/') { number.Inc(); continue; } else { number.UnGetToMark(mark); number.UnGet(); number.Mark(startMark); } } else { number.UnGet(); number.Mark(startMark); number.Inc(); } bracketsOpen++; brackets++; } else number.Mark(startMark); if ( number.Peek().IsDigit() ) // If the character was a number numberCount++; else if ( bracketsOpen > 0 ) { number.Inc(); TChar next = number.Peek(); TInt bracketsOpen2 = bracketsOpen; while( (IsValidPhoneNumberChar( next ) || next.IsSpace() || next == '(' || next == ')' || next == 'p' || next == '+' || next == 'w' ) && !(number.Eos()) && number.Offset() < end) { if ( next == '(' ) bracketsOpen2++; else if ( next == ')' ) bracketsOpen2--; if ( bracketsOpen2 == 0 ) break; number.Inc(); next = number.Peek(); } number.UnGetToMark(startMark); if ( bracketsOpen2 != 0 ) { number.Inc(); continue; } } number.Inc(); while ( number.Peek() == '(' && !(number.Eos()) && bracketsOpen > 0 ) { number.Inc(); bracketsOpen++; } if ( number.Peek() == '+' && bracketsOpen > 0 ) number.Inc(); // a Valid first character has been found. Let's go forward as long as valid characters are found. charac = number.Peek(); while( (IsValidPhoneNumberChar( charac ) || charac.IsSpace() || charac == '(' || charac == ')' || charac == 'p' || charac == 'w' ) && !(number.Eos()) && number.Offset() < end && charac != KCharLinefeed && charac != KCharFormfeed && charac != KCharCarriageReturn && charac != KCharLineSeparator && charac != KCharParagraphSeparator ) { if ( number.Peek() == '(' ) { // '(' can't be the last character in phone number number.Mark(mark); number.Inc(); TChar spaceJump = number.Peek(); while ( !number.Eos() && spaceJump.IsSpace() && spaceJump != KCharLinefeed && spaceJump != KCharFormfeed && spaceJump != KCharCarriageReturn && charac != KCharLineSeparator && spaceJump != KCharParagraphSeparator) { number.Inc(); spaceJump = number.Peek(); } if ( !(IsValidPhoneNumberChar(number.Peek())) && number.Peek() != ')' && number.Peek() != '(' ) { number.UnGetToMark(mark); break; } TChar next = number.Peek(); TInt bracketsOpen2 = bracketsOpen + 1; while( (IsValidPhoneNumberChar( next ) || next.IsSpace() || next == '(' || next == ')' || next == 'p' || next == 'w' ) && !(number.Eos()) && number.Offset() < end) { if ( next == '(' ) bracketsOpen2++; else if ( next == ')' ) bracketsOpen2--; if ( bracketsOpen2 == 0 ) break; number.Inc(); next = number.Peek(); } number.UnGetToMark(mark); if ( bracketsOpen2 != 0 ) break; bracketsOpen++; brackets++; } else if ( number.Peek() == ')' ) { if ( bracketsOpen <= 0 ) // there has to be equal number of brackets break; bracketsOpen--; number.Mark(mark); number.Inc(); if ( number.Peek() == '.' ) // '.' is not allowed after ')' break; number.UnGetToMark(mark); } else if ( number.Peek() == '-' || number.Peek() == 'w' || number.Peek() == 'p' || number.Peek() == '.' || number.Peek() == '/') { // Hyphen mark and 'p' & 'w' chars must be followed by a number TChar last = number.Peek(); number.Mark(mark); number.Inc(); TChar spaceJump = number.Peek(); while ( !number.Eos() && spaceJump.IsSpace() && spaceJump != KCharLinefeed && spaceJump != KCharFormfeed && spaceJump != KCharCarriageReturn && charac != KCharLineSeparator && spaceJump != KCharParagraphSeparator ) { number.Inc(); spaceJump = number.Peek(); } if ( !(number.Peek().IsDigit()) ) { if (last == '.' && number.Peek() == ')' && bracketsOpen > 0 ) continue; else { number.UnGetToMark(mark); break; } } number.UnGetToMark(mark); } else if ( number.Peek().IsDigit() ) numberCount++; number.Inc(); charac = number.Peek(); } // Get rid of whitespaces from the end number.UnGet(); while( number.Peek().IsSpace() && !(number.Eos())) number.UnGet(); number.Inc(); // ------------------------------------ number.Mark(endMark); // If they exist, remove brackets from the beginning and the end number.Mark(mark); // Let's mark where to continue the search TBool endBrackets = ETrue; do { number.UnGet(); if ( number.Peek() == ')' ) { number.UnGetToMark(startMark); if ( number.Peek() == '(' ) { // If there's more than one pair of brackets -> don't strip them. if ( brackets > 1 ) break; number.Inc(); number.Mark(startMark); number.UnGetToMark(endMark); number.UnGet(); number.Mark(endMark); // Get rid of whitespaces and periods from the end and from the beginning number.UnGet(); while ( (number.Peek().IsSpace() || number.Peek() == '.') && number.Offset() > number.MarkedOffset(startMark) ) { // from the end number.UnGet(); } number.Inc(); number.Mark(endMark); number.UnGetToMark(startMark); while ( (number.Peek().IsSpace() || number.Peek() == '.') && number.Offset() < number.MarkedOffset(endMark) ) { // from the beginning number.Inc(); } number.Mark(startMark); number.UnGetToMark(endMark); // ---- } else endBrackets = EFalse; } else endBrackets = EFalse; } while ( endBrackets ); number.UnGetToMark(mark); // ---------------- if ( numberCount <= KFindItemMaxNumbers && numberCount >= iMinNumbers ) { TPtrC tokenPtr = number.MarkedToken(startMark); TInt tokensEnd = tokenPtr.Length(); TInt numbers = 0; TInt partialNumber = 0; TBool wasValidPhoneNumber = ETrue; TInt i = 0; for ( ; i < tokensEnd; i++ ) { if ( tokenPtr[i] == '.' ) partialNumber = 0; else if ( ((TChar)tokenPtr[i]).IsDigit() ) { numbers++; partialNumber++; } if ( ( partialNumber == 1 || partialNumber == 2 ) && i + 1 < tokensEnd ) { if ( tokenPtr[i + 1] == '.' ) wasValidPhoneNumber = EFalse; } } if (!wasValidPhoneNumber && numbers > 6) wasValidPhoneNumber = ETrue; if (wasValidPhoneNumber) { __ASSERT_DEBUG( number.MarkedOffset(startMark) + number.MarkedOffset(endMark) - number.MarkedOffset(startMark) <= aText.Length(), Panic(ETulPanicDescriptorLength) ); AddItemL( number.MarkedOffset(startMark), number.MarkedOffset(endMark) - number.MarkedOffset(startMark), EFindItemSearchPhoneNumberBin ); } } } return (iFoundItems->Count() > 0); }
/** Search algorithm for searching e-mail addresses @param aText Text that will be parsed @return ETrue if any EMail items were found else returns EFalse @leave KErrNone, if successful; otherwise one of the other system-wide error codes. @panic ETulPanicDescriptorLength in debug build if item's position and/or length is out of the document's range. */ TBool CTulAddressStringTokenizer::SearchMailAddressL( const TDesC& aText ) { TInt searchStart = 0; TInt searchResult = 0; const TInt end = aText.Length(); // end of document do { TPtrC segment = aText.Right( end - searchStart ); searchResult = segment.LocateF('@'); if (searchResult != KErrNotFound) { // @ found // There should be valid characters (not a period) before and after the @ character if ( searchResult == 0 // first char || (searchResult >= segment.Length() - 1) // last char || !(IsValidEmailChar(segment[searchResult - 1])) || !(IsValidEmailHostChar(segment[searchResult + 1])) || segment[searchResult - 1] == '.' || segment[searchResult + 1] == '.' ) { searchStart += searchResult + 1; continue; } TBool wasPeriod = EFalse; // To prevent sequential periods // Get TLex from the pointer to get a better API for parsing TLexMark startPos; TLexMark endPos; TLex token = segment; // Go to searchResult and un-get until the beginning of e-mail address is reached token.Inc( searchResult ); token.Mark(); do { token.UnGet(); if ( token.Peek() == '.' ) { // If it was a period if (wasPeriod) // and if the former was also -> break break; else // else mark that this one was a period wasPeriod = ETrue; } else wasPeriod = EFalse; } while (token.Offset() > 0 && IsValidEmailChar(token.Peek())); if (token.Offset() != 0 || !IsValidEmailChar(token.Peek())) token.Inc(); // Get rid of periods from the start of address // Does it have to start with a number or char(abc...). // If it does, the loop should check that it gets rid of all special chars also. while (token.Peek() == '.') token.Inc(); token.Mark( startPos ); // Mark the beginning of address token.UnGetToMark(); wasPeriod = EFalse; do // Go forward until a nonvalid character { token.Inc(); if ( token.Peek() == '.' ) { // If it was a period if ( wasPeriod ) // and if the former was also -> break break; else // else mark that this one was a period wasPeriod = ETrue; } else wasPeriod = EFalse; } while ( !token.Eos() && IsValidEmailHostChar( token.Peek() ) ); // If address ends with a period take it away token.UnGet(); if (token.Peek() != '.') token.Inc(); token.Mark( endPos ); // Mark the beginning of address // Append the found string to the array __ASSERT_DEBUG( searchStart + token.MarkedOffset( startPos ) + token.MarkedOffset( endPos ) - token.MarkedOffset( startPos ) <= aText.Length(), Panic(ETulPanicDescriptorLength) ); AddItemL( searchStart + token.MarkedOffset( startPos ), token.MarkedOffset( endPos ) - token.MarkedOffset( startPos ), EFindItemSearchMailAddressBin); searchStart += token.MarkedOffset( endPos ) + 1; } } while ( searchResult != KErrNotFound && searchStart < end ); return (iFoundItems->Count() > 0); }
/** Search algorithm for searching generic URIs @param aText Text that will be parsed @return ETrue if any generic URI is found else returns EFalse @leave KErrNone, if successful; otherwise one of the other system-wide error codes. */ TBool CTulAddressStringTokenizer::SearchGenericUriL( const TDesC& aText ) { // Detect generic URI within the token const TDesC& schemeStartArray = KURISchemeStartCharacters; const TDesC& schemeBodyArray = KURISchemeBodyCharacters; const TDesC& schemeTerminatorArray = KURISchemeTerminator; const TDesC& URIArray = KURICharacters; TBool wasValidUri = EFalse; TLex text = aText; while ( !text.Eos() ) { // Discard characters until URI scheme terminator is found while( !(text.Eos()) && schemeTerminatorArray.Locate(text.Peek()) == KErrNotFound ) text.Inc(); // if at end of the text, no legit URI found if ( !text.Eos() ) { // Store the schema end offset (+1 to skip ':') TInt schemeEndOffset = text.Offset() + 1; // Scheme must be at least 1 character long at the beginning of the text to be valid if ( text.Offset() > 0 ) { // Un-get last scheme character to begin examination text.UnGet(); // Rewind until beginning of the URI while ( text.Offset() > 0 && schemeBodyArray.Locate(text.Peek().GetLowerCase()) != KErrNotFound ) text.UnGet(); // Now text pointer is at first character of the URI // Do go back through the scheme until a legal beginning character for URI // is found or back to the (schemeEndOffset - 1) i.e. URI scheme terminator while ( schemeStartArray.Locate(text.Peek().GetLowerCase()) == KErrNotFound && (text.Offset() + 1) < schemeEndOffset ) text.Inc(); // check if terminated because a valid start character was found when // scheme terminator was reached. if ( schemeStartArray.Locate(text.Peek().GetLowerCase()) != KErrNotFound ) { // First character is a valid URI char, so the scheme is valid -> // marks the beginning of the array text.Mark(); // fast forward to the end of the scheme while( text.Offset() < schemeEndOffset ) text.Inc(); // Get characters until end of schema while( !(text.Eos()) && URIArray.Locate( text.Peek().GetLowerCase() ) != KErrNotFound ) text.Inc(); // remove certain punctuation from end of the URI, as it is likely // to be part of the surrounding text. text.UnGet(); //special processing for bracket //only remove the end bracket if there is no open bracket in the uri //not counting bracket pairs for efficiency if (text.Peek()!=')' || text.MarkedToken().Locate(TChar('('))!=-1) text.Inc(); text.UnGet(); if ( text.Peek() != '.' && text.Peek() != '?' && text.Peek() != ',') text.Inc(); // URI cannot contain only scheme, so check that pointer was increased // by at least one character if ( schemeEndOffset != text.Offset() ) { // Append found text to item array (it is now known to be // syntactically valid URI as it contains characters after the scheme) AddItemL( text.MarkedOffset(), text.Offset() - text.MarkedOffset(), EFindItemSearchScheme ); wasValidUri = ETrue; } } else // First character of scheme is not legit, fast forward to end of the // scheme anyway to continue search { while( text.Offset() < schemeEndOffset ) text.Inc(); } } else text.Inc(); } } return wasValidUri; }