/**
* Get next token in the current string expr.
* Uses the data in m_expr pointed to by m_e to
* produce m_tokenType and m_token, set m_err in case of an error
*/
void CondParser::getToken()
{
m_tokenType = NOTHING;
m_token.resize(0);
//printf("\tgetToken e:{%c}, ascii=%i, col=%i\n", *e, *e, e-expr);
// skip over whitespaces
while (*m_e == ' ' || *m_e == '\t') // space or tab
{
m_e++;
}
// check for end of expression
if (*m_e=='\0')
{
// token is still empty
m_tokenType = DELIMITER;
return;
}
// check for parentheses
if (*m_e == '(' || *m_e == ')')
{
m_tokenType = DELIMITER;
m_token += *m_e++;
return;
}
// check for operators (delimeters)
if (isDelimiter(*m_e))
{
m_tokenType = DELIMITER;
while (isDelimiter(*m_e))
{
m_token += *m_e++;
}
return;
}
// check for variables
if (isAlpha(*m_e))
{
m_tokenType = VARIABLE;
while (isAlphaNum(*m_e))
{
m_token += *m_e++;
}
return;
}
// something unknown is found, wrong characters -> a syntax error
m_tokenType = UNKNOWN;
while (*m_e)
{
m_token += *m_e++;
}
m_err = QCString("Syntax error in part '")+m_token+"'";
return;
}
char * tokenize(char * & str, const char * delim)
{
if (!str || !*str || !delim)
return nullptr;
// Find the start of the next token
char * pStartToken = str;
while (*pStartToken && isDelimiter(*pStartToken, delim))
++pStartToken;
// Did we hit the end of the string?
if (!*pStartToken)
{
str = pStartToken;
return nullptr;
}
// Find the end of the token
char * pEndToken = pStartToken;
while (*pEndToken && !isDelimiter(*pEndToken, delim))
++pEndToken;
// Did we hit the end of the string?
if (!*pEndToken)
{
str = pEndToken;
return pStartToken;
}
// Terminate the token and update str to the following character
*pEndToken = 0;
str = pEndToken + 1;
return pStartToken;
}
void CdTreeStream::readToDelimiter(std::istream& is, std::string& str)
{
//str.erase();
char ch;
while (is.get(ch) && (!isDelimiter(ch)))
//if (!isspace((unsigned char) ch))
str += ch;
if (isDelimiter(ch))
is.putback(ch);
}
void JsonSerializer::deserializeToken()
{
m_token += m_token_size;
if (m_is_string_token)
{
++m_token;
}
while (m_token < m_data + m_data_size && isDelimiter(*m_token))
{
++m_token;
}
if (*m_token == '/' && m_token < m_data + m_data_size - 1 && m_token[1] == '/')
{
m_token_size = int((m_data + m_data_size) - m_token);
m_is_string_token = false;
}
else if (*m_token == '"')
{
++m_token;
m_is_string_token = true;
const char* token_end = m_token;
while (token_end < m_data + m_data_size && *token_end != '"')
{
++token_end;
}
if (token_end == m_data + m_data_size)
{
ErrorProxy(*this).log() << "Unexpected end of file while looking for \".";
m_token_size = 0;
}
else
{
m_token_size = int(token_end - m_token);
}
}
else if (isSingleCharToken(*m_token))
{
m_is_string_token = false;
m_token_size = 1;
}
else
{
m_is_string_token = false;
const char* token_end = m_token;
while (token_end < m_data + m_data_size && !isDelimiter(*token_end) &&
!isSingleCharToken(*token_end))
{
++token_end;
}
m_token_size = int(token_end - m_token);
}
}
static char*
parseName(FILE *file) {
int ch;
unsigned int i;
char *ret;
char buff[BUFFSIZE];
ch = parseWhiteSpace(file);
if(ch != '/') {
ungetc(ch, file);
return NULL;
}
ch = getc(file);
for(i=0; i<BUFFSIZE && !isWhiteSpace(ch) &&
!isDelimiter(ch) && ch != EOF; ++i) {
buff[i] = ch;
ch = getc(file);
}
ungetc(ch, file);
buff[i++] = '\0';
ret = malloc(sizeof(char)*i);
memcpy(ret, buff, i);
return ret;
}
QString BraceMatcher::insertMatchingBrace(const QTextCursor &cursor,
const QString &text,
const QChar la,
int *skippedChars) const
{
if (text.length() != 1)
return QString();
if (!shouldInsertMatchingText(cursor))
return QString();
const QChar ch = text.at(0);
if (isQuote(ch)) {
if (la != ch)
return QString(ch);
++*skippedChars;
return QString();
}
if (isOpeningBrace(ch))
return QString(m_braceChars[ch]);
if (isDelimiter(ch) || isClosingBrace(ch)) {
if (la == ch)
++*skippedChars;
}
return QString();
}
char* Tokenizer::getToken(char *buf,int len)
{
int ilen;
int itap = 1;
// copy while no delimiter is read
ilen = 0;
while (*current!='\0')
{
if (isDelimiter(*current)){
current++;
itap = 0;
break;
}
*buf = *current;
buf++;
current++;
ilen++;
}
*buf = '\0';
//
if (ilen==0&&*current=='\0'&&itap)
return NULL;
return buf;
}
int main(int argc, char const *argv[])
{
printf("输入一段文章\n");
int count[11]={0}; //count[1] ＝＝ 含有1个字母的单词的数量
int word_cnt = 0; //单词数量
int char_cnt = 0; //单词字母个数
char ch;
while((ch=getchar()) != EOF){
if (isDelimiter(ch))
{
if(char_cnt != 0){
word_cnt++;
if(char_cnt <= 10){
count[char_cnt]++;
}
char_cnt = 0;
}
continue;
}else{
char_cnt++;
}
}
printf("%d ",word_cnt);
for (int i = 1; i < 11; ++i)
{
printf("%d",count[i]);
if(i != 10){
printf(" ");
}
}
return 0;
}
bool BraceMatcher::shouldInsertMatchingText(const QChar lookAhead) const
{
return lookAhead.isSpace()
|| isQuote(lookAhead)
|| isDelimiter(lookAhead)
|| isClosingBrace(lookAhead);
}
string Tokenizer::prevToken(){
if(!mode)
{
if(this->inpString.size()<=0) return "";
this->Token="";
this->skipDelimiters_backward();
while(this->currPos >= this->inpString.begin() && !isDelimiter(*(this->currPos))){
this->Token =this->Token + *(this->currPos);
this->currPos=this->currPos-1;
}
string temp = this->Token;
this->Token="";
for(string::iterator i=temp.end();i>=temp.begin();i--)
this->Token =this->Token + *(i);
}
else
{
this->setMode(0);
this->Token=this->nextToken();
this->setMode(1);
}
return this->Token;
}
string Tokenizer::nextToken(){
if(!mode)
{
if(this->inpString.length()<=0) return "";
this->Token="";
this->skipDelimiters_forward();
while(this->currPos != this->inpString.end() && !isDelimiter(*(this->currPos))){
this->Token =this->Token + *(this->currPos);
this->currPos=this->currPos+1;
}
}
else
{
this->setMode(0);
this->Token=this->prevToken();
this->setMode(1);
}
return this->Token;
}
void WordNgrams::addTokens()
{
// get token string from input file
string & inFileName = getInFileName();
FILE * fp = inFileName.length() > 0 ? fopen( inFileName.c_str(), "r" ) : stdin;
if ( !fp )
{
printf("Can not find file %s, use stdio as input.\n", inFileName.c_str() );
fp = stdin;
}
int count = 0;
char c;
bool isSpecialChar = false;
string token;
token.reserve(256);
while ( ( c = (char) fgetc( fp ) ) != EOF )
{
if ( isDelimiter( c ) || isStopChar ( c ) )
{
if ( !isSpecialChar && token.length() >0 )
{
addToken( token );
token.empty();
++count;
isSpecialChar = true;
}
else
{
isSpecialChar = false;
}
}
else
{
token.append( c );
isSpecialChar = false;
}
}
if ( token.length() > 0 )
{
++count;
addToken( token );
}
// special processing need to be done, if less than NGRAM_N tokens in the whole input text.
if ( count < this->getN() )
{
preParse( count );
}
/* int padding = ngramN - count % ngramN;
for ( int i=0; i< padding; i++)
{
addToken( "_" );
}
*/
fclose( fp );
}
size_t StringBuilder::FirstNotOf( const std::string &_delimiters ) const
{
for( size_t i = 0; i < m_pos; i++ )
{
if(!isDelimiter(m_buffer[i],_delimiters))
return i;
}
return SIZE_T_FAIL;
}
size_t StringBuilder::LastNotOf( const std::string &_delimiters ) const
{
for( int i = (((int)m_pos)-1); i >= 0; i-- ) // This must be an 'int' i can be zero, otherwise an infinate loop is created (size_t cant be <0, int can)
{
if(!isDelimiter(m_buffer[i],_delimiters))
return i;
}
return SIZE_T_FAIL;
}
static void skipDelimiters(LexerCarriage* carriage)
{
while (carriage->posInText < carriage->lexingText.length &&
isDelimiter(carriage->lexingText.pointer[carriage->posInText]))
{
if (carriage->lexingText.pointer[carriage->posInText] == '\n')
++carriage->currentLineNumber;
++carriage->posInText;
}
}
char *readWord(int *endOfInput){
int nCursor, nRead;
char * buff = malloc(BUFF_SIZE);
*endOfInput = FALSE;
if( NULL == buff) {
*endOfInput = TRUE;
free(buff);
return NULL;
}
/* Read from input */
for(nCursor = 0; nCursor < BUFF_SIZE; nCursor++) {
if(read(STDIN_FILENO, (void*) &buff[nCursor], 1) != 1) {
/* EOF detected or an error has occurred while reading input */
*endOfInput = TRUE;
free(buff);
return NULL;
}
if( isDelimiter( buff[nCursor] ) )
break;
}
if(nCursor == BUFF_SIZE) {
/* Discard extra bytes */
do {
nRead = read (STDIN_FILENO, (void*) buff, 1);
} while ( nRead > 0 && ! isDelimiter(*buff) );
if ( !(nRead > 0) )
*endOfInput = TRUE;
free(buff);
return NULL;
}
buff[nCursor] = '\0';
return buff;
}
void Tokenizer::skipDelimiters(const char* delimiters) {
#if DEBUG_TOKENIZER
LOGD("skipDelimiters");
#endif
const char* end = getEnd();
while (mCurrent != end) {
char ch = *mCurrent;
if (ch == '\n' || !isDelimiter(ch, delimiters)) {
break;
}
mCurrent += 1;
}
}
char *Strtok(char *str, const char* delimiters) {
if(str==NULL && endOfstringReached==1) return NULL;
if(str!=NULL) {
mystr=(char*) malloc((strlen(str)+1)*sizeof(char));
strcpy(mystr,str);
curr_pos=mystr;
endOfstringReached=0;
} else {
mystr=curr_pos;
}
while(isDelimiter(*mystr,delimiters)) {
*mystr++;
}
if(*mystr=='\0') {
endOfstringReached=1;
}
char *temp=mystr;
while(!isDelimiter(*temp,delimiters) && *temp!='\0') {
temp++;
}
if(*temp=='\0') {
endOfstringReached=1;
} else {
*temp='\0';
}
if(temp!='\0') curr_pos=temp+1;
return mystr;
}
short SlkToken::getOperatorTokenValue(void)const
{
const char* pOperator = mCurToken;
const char* pLastToken = mLastToken;
int lastIsOperator = TRUE;
if (pLastToken && pLastToken[0]) {
if (isDelimiter(pLastToken[0])) {
lastIsOperator = TRUE;
} else if (isBeginParentheses(pLastToken[0])) {
lastIsOperator = TRUE;
} else {
lastIsOperator = isOperator(pLastToken[0]);
}
}
short val = OP_TOKEN_0_;
if (pOperator && pOperator[0]) {
if ((pOperator[0] == '?' || pOperator[0] == ':') && pOperator[1] == '\0') {
val = OP_TOKEN_1_;
} else if (pOperator[0] == '|' && pOperator[1] == '|') {
val = OP_TOKEN_2_;
} else if (pOperator[0] == '&' && pOperator[1] == '&') {
val = OP_TOKEN_3_;
} else if (pOperator[0] == '|' && pOperator[1] == '\0') {
val = OP_TOKEN_4_;
} else if (pOperator[0] == '^' && pOperator[1] == '\0') {
val = OP_TOKEN_5_;
} else if (pOperator[0] == '&' && pOperator[1] == '\0') {
val = OP_TOKEN_6_;
} else if ((pOperator[0] == '=' || pOperator[0] == '!') && pOperator[1] == '=') {
val = OP_TOKEN_7_;
} else if ((pOperator[0] == '<' || pOperator[0] == '>') && (pOperator[1] == '=' || pOperator[1] == '\0')) {
val = OP_TOKEN_8_;
} else if ((pOperator[0] == '<' && pOperator[1] == '<') || (pOperator[0] == '>' && pOperator[1] == '>') || (pOperator[0] == '>' && pOperator[1] == '>' && pOperator[2] == '>')) {
val = OP_TOKEN_9_;
} else if ((pOperator[0] == '+' || pOperator[0] == '-') && pOperator[1] == '\0') {
if (lastIsOperator)
val = OP_TOKEN_12_;
else
val = OP_TOKEN_10_;
} else if ((pOperator[0] == '*' || pOperator[0] == '/' || pOperator[0] == '%') && pOperator[1] == '\0') {
val = OP_TOKEN_11_;
} else if ((pOperator[0] == '+' && pOperator[1] == '+') || (pOperator[0] == '-' && pOperator[1] == '-') || (pOperator[0] == '~' && pOperator[1] == '\0') || (pOperator[0] == '!' && pOperator[1] == '\0')) {
val = OP_TOKEN_12_;
} else {
val = OP_TOKEN_0_;
}
}
return val;
}
String8 Tokenizer::nextToken(const char* delimiters) {
#if DEBUG_TOKENIZER
LOGD("nextToken");
#endif
const char* end = getEnd();
const char* tokenStart = mCurrent;
while (mCurrent != end) {
char ch = *mCurrent;
if (ch == '\n' || isDelimiter(ch, delimiters)) {
break;
}
mCurrent += 1;
}
return String8(tokenStart, mCurrent - tokenStart);
}
static bool parseUnknown(LexerCarriage* carriage, Token* result)
{
int shift = 0;
while (carriage->posInText + shift < carriage->lexingText.length &&
!isDelimiter(carriage->lexingText.pointer[carriage->posInText + shift]))
{
++shift;
}
result->type = TOK_UNKNOWN;
result->identifierValue = StringSlice{ &carriage->lexingText.pointer[carriage->posInText], shift };
carriage->posInText += shift;
return true;
}
///////////////////////////////////////////////////////////////////////////////
// return the next token
// If cannot find a token anymore, return "".
///////////////////////////////////////////////////////////////////////////////
std::string Tokenizer::next()
{
if(buffer.size() <= 0) return ""; // skip if buffer is empty
token.clear(); // reset token string
this->skipDelimiter(); // skip leading delimiters
// append each char to token string until it meets delimiter
while(currPos != buffer.end() && !isDelimiter(*currPos))
{
token += *currPos;
++currPos;
}
return token;
}
char *TKGetNextToken(TokenizerT *tk) {
int i, b, BBIndex;
BBIndex = 0;
char c;
char* BigBuffer; //to be used for returning tokens
BigBuffer = (char*) malloc(1000);
for(i = 0; i<(strlen(indexPointer)+1); i++){
c = (indexPointer[i]);
b = isDelimiter(c, tk);
if(c == '\0'){
BigBuffer[BBIndex] = '\0';
indexPointer = '\0';
return BigBuffer;
}
if(b == 0){ //not delimiter, add to BigBuffer
BigBuffer[BBIndex]=c;
BBIndex++;
}else{ //is delimiter, return now
indexPointer = &tk->input[i+1];
tk->input = indexPointer;
BigBuffer[BBIndex] = '\0';
return BigBuffer;
}
}
return 0;
}
size_t StringBuilder::XthNotOf( size_t instance, const std::string &_delimiters ) const
{
size_t count = 0;
for( size_t i = 0; i < m_pos; i++ )
{
if(!isDelimiter(m_buffer[i],_delimiters))
{
if( count < instance )
{
count++;
}
else
{
return i;
}
}
}
return SIZE_T_FAIL;
}
void TimeTaggerDialog::open_action() {
QFile fileOriginal(QString(filename).toStdString().data());
if (fileOriginal.open(QIODevice::ReadOnly))
{
text->clear();
QTextStream s(&fileOriginal);
s.setCodec("utf-8");
text->setLayoutDirection(Qt::RightToLeft);
QTextCursor c=text->textCursor();
c.clearSelection();
c.movePosition(QTextCursor::Start,QTextCursor::MoveAnchor);
text->setTextCursor(c);
text->setTextBackgroundColor(Qt::white);
text->setTextColor(Qt::black);
text->setText(s.readAll());
QFile file(QString("%1"+ext).arg(filename).toStdString().data());
if (file.open(QIODevice::ReadOnly))
{
QDataStream out(&file); // we will serialize the data into the file
out >> tags;
file.close();
for (int i=0;i<tags.size();i++) {
int start=tags[i].first;
int end=tags[i].second;
QString all=text->toPlainText();
if (isDelimiter(all.at(end))) {
end--;
tags[i].second=end;
}
QTextCursor c=text->textCursor();
c.setPosition(start,QTextCursor::MoveAnchor);
c.setPosition(end+1,QTextCursor::KeepAnchor);
text->setTextCursor(c);
text->setTextBackgroundColor(Qt::darkYellow);
}
c.clearSelection();
c.setPosition(0);
text->setTextCursor(c);
} else {
///////////////////////////////////////////////////////////////////////////////
// return the next token
// If cannot find a token anymore, return "".
///////////////////////////////////////////////////////////////////////////////
std::string KukaduTokenizer::next() {
++tokenIdx;
if(useLastToken) {
useLastToken = false;
return lastToken;
} else {
if(buffer.size() <= 0) return ""; // skip if buffer is empty
token.clear(); // reset token string
this->skipDelimiter(); // skip leading delimiters
// append each char to token string until it meets delimiter
while(currPos != buffer.size() && !isDelimiter(*(buffer.begin() + currPos))) {
token += *(buffer.begin() + currPos);
++currPos;
}
return (lastToken = token);
}
return "";
}
int main(int argc, char** argv) {
char text[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aliquam eleifend condimentum magna imperdiet tristique. Pellentesque fringilla maximus mi vitae ullamcorper. Fusce mattis neque vel nisi fringilla, vel aliquet libero sagittis. Nullam fringilla at mi at accumsan. Suspendisse ullamcorper auctor enim, eu malesuada felis ornare eu. Morbi sodales nunc eget lacus sollicitudin semper. Donec tristique consequat tellus, ut porttitor felis maximus eget. Fusce consectetur iaculis pharetra. Donec congue volutpat arcu, eget sodales sapien tristique quis. Nullam ut posuere risus, sed pharetra lorem. Cras varius, est eu pulvinar scelerisque, ipsum nisi bibendum sapien, in scelerisque velit risus eget felis. Nam nec porttitor erat. Suspendisse dictum sit amet magna finibus lobortis. Phasellus non sem mollis, ultrices odio non, volutpat ligula. Aenean mollis eros ex, vitae lobortis quam gravida sed.";
size_t length = strlen(text);
bool last_was_delimiter = true;
size_t count = 0;
for (size_t i = 0; i < length; ++i) {
if (isDelimiter(text[i])) {
if (last_was_delimiter == false) {
++count;
last_was_delimiter = true;
}
} else {
last_was_delimiter = false;
}
}
if (last_was_delimiter == false) {
++count;
}
printf("Word count: %i\n", count);
return 0;
}
int INIParser::parse()
{
char ch;
char buffer[100];
int i = 0;
int line = 1;
std::string section = "__NO_SECTION__";
std::string name;
parseState = NO_SECTION;
// open ini file
std::ifstream in(_iniFile.c_str(), std::ios::in | std::ios::binary);
while ( in.get(ch) ) {
switch (ch) {
case '[' :
if (isStartSection())
return line;
if ( i == 0 )
parseState = START_SECTION;
else
return line;
break;
case ']' :
if (isStopSection())
return line;
if (isStartSection() && i > 0) {
buffer[i] = '\0';
section = buffer;
//std::cout << "section => " << buffer << std::endl;
i=0;
} else { // else error , empty section [] or ] not the end of start section
return line;
}
parseState = STOP_SECTION;
break;
case '=' :
if ( isStartSection() || isDelimiter())
return line;
if ( isNoSection() && i == 0 )
return line;
if ((isNewline() || isNoSection()) && i > 0) {
buffer[i] = '\0';
name = buffer;
//std::cout << "key => " << buffer << std::endl;
i=0;
} else if (isNewline() && i == 0)
return line;
parseState = DELIMITER;
break;
case '\n' :
if ( isNewline() && i > 0 )
return line;
if ( isNoSection() && i > 0 )
return line;
if ( isStartSection() )
return line;
if ( isDelimiter() && i > 0 ) {
buffer[i] = '\0';
sections[section][name] = buffer;
//std::cout << "value => " << buffer << std::endl;
i=0;
} else if ( isDelimiter() && i == 0 )
return line;
//if ( parseState != DELIMITER || parseState != STOP_SECTION || parseState != COMMENT )
// return line;
parseState = NEWLINE;
line++;
break;
case ';' :
parseState = COMMENT;
break;
case '#' :
parseState = COMMENT;
break;
case ' ' :
break;
default :
if ( isNewline() || isNoSection() ) {
buffer[i] = ch;
i++;
}
if ( isDelimiter() ) {
buffer[i] = ch;
i++;
}
if ( isStartSection() ) {
buffer[i] = ch;
i++;
}
// else error
break;
}
}
in.close();
return 0;
}
nitf_PluginRegistry_internalLoadDir(nitf_PluginRegistry * reg,
const char *dirName,
nitf_Error * error)
{
const char *name;
size_t sizePath;
nitf_Directory *dir = NULL;
if (!dirName)
{
nitf_Error_initf(error, NITF_CTXT, NITF_ERR_OPENING_FILE,
"Null directory name");
return NITF_FAILURE;
}
dir = nitf_Directory_construct(error);
if (!dir)
{
return NITF_FAILURE;
}
sizePath = strlen(dirName);
if (nitf_Directory_exists(dirName))
{
name = nitf_Directory_findFirstFile(dir, dirName);
if (name)
{
do
{
char *end;
char fullName[NITF_MAX_PATH];
int pathSize = sizePath;
memset(fullName, 0, NITF_MAX_PATH);
memcpy(fullName, dirName, sizePath);
if (!isDelimiter(dirName[pathSize - 1]))
fullName[pathSize++] = DIR_DELIMITER;
memcpy(fullName + pathSize, name, strlen(name));
/* See if we have .so or .dll extensions */
if ((end =
(char *) strstr(name, NITF_DLL_EXTENSION)) != NULL)
{
if (!nitf_PluginRegistry_loadPlugin(fullName, error))
{
#ifdef NITF_DEBUG_PLUGIN_REG
printf("Warning: plugin [%s] failed to load!\n", name);
#endif
}
}
else
{
#ifdef NITF_DEBUG_PLUGIN_REG
printf("Skipping directory [%s]\n", name);
#endif
}
name = nitf_Directory_findNextFile(dir);
}
while (name);
}
else
{
printf("Error: %s\n", NITF_STRERROR(NITF_ERRNO));
}
}
else
{
#ifdef NITF_DEBUG_PLUGIN_REG
fprintf(stdout,
"Could not open plug-in directory '%s'. "
"You may have forgotten to set your NITF_PLUGIN_PATH environment "
"variable : continuing without plugins...\n", dirName);
#endif
}
nitf_Directory_destruct(&dir);
return NITF_SUCCESS;
}
/*!
* Returns true if given character was added as one of character types.
*/
bool BraceMatcher::isKnownChar(const QChar c) const
{
return isQuote(c) || isDelimiter(c) || isOpeningBrace(c) || isClosingBrace(c);
}
