bool textSource::segment(int level ,int sstatus ,bool PrevIsField // True if previous sibling block contains a \field ,charprops CharProps ) { wint_t ch; curr_pos = Ftell(sourceFile);// After parsing a html-tag, seeking to curr_pos brings you back to the position where the parsed sequence started. if(Option.keepEOLsequence) { copyEOLsequence(); // SourceFile is rewinded } do { ch = Getc(sourceFile); end_offset = Ftell(sourceFile); if(curr_pos >= tagendpos) { // We are not inside an HTML-tag. if(flgs.inhtmltag) { flgs.firstafterhtmltag = true; flgs.inhtmltag = false; } // Check whether a well-formed HTML tag is ahead. Returns sourceFile // in same file position. flgs.htmltagcoming = isHTMLtagComing(ch); // assert(new_pos == Ftell(sourceFile)); assert(end_offset == Ftell(sourceFile)); } else if(flgs.htmltagcoming) { // We are leaving an HTML-tag and entering a new one. flgs.inhtmltag = true; flgs.htmltagcoming = false; } /* Scan in advance, checking whether the line to come is a heading and therefore must be preceded with a newline (WritePar will then be set to true.) */ if( ch == '\n' || ch == '\r' || ch == WEOF || ch == 26 ) { flgs.in_fileName = false; heading = isHeading(firsttext,ch,WriteParAfterHeadingOrField); if(!skipSegmentation(firsttext,ch)) { doTheSegmentation(CharProps,true,false); // Bart 20040120. true because: Suppose that end of line is end of segment if(!WriteParAfterHeadingOrField && heading) {// A normal line after a heading has WritePar==false and heading==true WriteParAfterHeadingOrField = true; heading = false; } } if(firsttext.EOL) firsttext.b.LS = 1; } else { updateFlags(ch,flgs); int EOL = firsttext.EOL; bool sentenceEnd = checkSentenceStartDueToBullet(ch); if( sentenceEnd || flgs.htmltagcoming || flgs.inhtmltag || (end_offset - begin_offset > MAXSEGMENTLENGTH && isSpace(ch)) // check for buffer overflow ) { doTheSegmentation(CharProps,false,false); firsttext.b.SD = 1; firsttext.b.LS = 0; } if(isSpace(ch)) { if(EOL) firsttext.b.LS = 1; } else { firsttext.b.LS = 0; firsttext.EOL = 0; // resets SD, CR and LF } } curr_pos = end_offset; } while(ch != WEOF && ch != 26); outputtext->PutHandlingLine('\n',flgs); // 20100106 Flush last line return false; }
int main(void) { int ret = 0; INIT_MACHINE(); reg_t numCount = 1; reg_t upCount = 2; reg_t lowCount = 3; reg_t specialCount = 4; reg_t otherCount = 5; reg_t spaceCount = 6; MOVIM8(numCount, 0); MOVIM8(spaceCount, 0); MOVIM8(upCount, 0); MOVIM8(lowCount, 0); MOVIM8(specialCount, 0); MOVIM8(otherCount, 0); reg_t numString = 7; reg_t upString = 9; reg_t lowString = 11; reg_t spaceString = 13; reg_t specialString = 15; reg_t otherString = 17; reg_t newlineString = 19; MOVIM16(numString, ADDR(0)); MOVIM16(spaceString, ADDR(16)); MOVIM16(upString, ADDR(32)); MOVIM16(lowString, ADDR(48)); MOVIM16(specialString, ADDR(64)); MOVIM16(otherString, ADDR(80)); MOVIM16(newlineString, ADDR(96)); reg_t i = 21; // 123456789012 insertString(numString, "numbers = ", i); insertString(spaceString, "spaces = ", i); insertString(upString, "uppers = ", i); insertString(lowString, "lowers = ", i); insertString(specialString, "special = ", i); insertString(otherString, "other = ", i); //endl MOVIM8(REG16(newlineString), '\n'); MOVIM32(REG16(newlineString) + 1, 0); ret = getChar(i); while (ret != (-1)) { if (isUpLetter(i, i+1)) { INC8(upCount); } else if (isLowLetter(i, i+1)) { INC8(lowCount); } else if (isNumber(i, i+1)) { INC8(numCount); } else if (isSpace(i, i+1)) { #ifdef PATCHED INC8(spaceCount); #else INC32(spaceCount); #endif } else if (isSpecial(i, i+1)) { INC8(specialCount); } else { INC8(otherCount); } printChar(i); ret = getChar(i); } printString(numString, i, i+4); printReg8(numCount, i); printString(newlineString, i, i+4); printString(upString, i, i+4); printReg8(upCount, i); printString(newlineString, i, i+4); printString(lowString, i, i+4); printReg8(lowCount, i); printString(newlineString, i, i+4); printString(spaceString, i, i+4); printReg8(spaceCount, i); printString(newlineString, i, i+4); printString(specialString, i, i+4); printReg8(specialCount, i); printString(newlineString, i, i+4); printString(otherString, i, i+4); printReg8(otherCount, i); printString(newlineString, i, i+4); return (0); }
std::string LoadObj( std::vector<shape_t>& shapes, std::vector<material_t>& materials, // [output] std::istream& inStream, MaterialReader& readMatFn) { std::stringstream err; std::vector<float> v; std::vector<float> vn; std::vector<float> vt; std::vector<std::vector<vertex_index> > faceGroup; std::string name; // material std::map<std::string, int> material_map; std::map<vertex_index, unsigned int> vertexCache; int material = -1; shape_t shape; int maxchars = 8192; // Alloc enough size. std::vector<char> buf(maxchars); // Alloc enough size. while (inStream.peek() != -1) { inStream.getline(&buf[0], maxchars); std::string linebuf(&buf[0]); // Trim newline '\r\n' or '\n' if (linebuf.size() > 0) { if (linebuf[linebuf.size()-1] == '\n') linebuf.erase(linebuf.size()-1); } if (linebuf.size() > 0) { if (linebuf[linebuf.size()-1] == '\r') linebuf.erase(linebuf.size()-1); } // Skip if empty line. if (linebuf.empty()) { continue; } // Skip leading space. const char* token = linebuf.c_str(); token += strspn(token, " \t"); assert(token); if (token[0] == '\0') continue; // empty line if (token[0] == '#') continue; // comment line // vertex if (token[0] == 'v' && isSpace((token[1]))) { token += 2; float x, y, z; parseFloat3(x, y, z, token); v.push_back(x); v.push_back(y); v.push_back(z); continue; } // normal if (token[0] == 'v' && token[1] == 'n' && isSpace((token[2]))) { token += 3; float x, y, z; parseFloat3(x, y, z, token); vn.push_back(x); vn.push_back(y); vn.push_back(z); continue; } // texcoord if (token[0] == 'v' && token[1] == 't' && isSpace((token[2]))) { token += 3; float x, y; parseFloat2(x, y, token); vt.push_back(x); vt.push_back(y); continue; } // face if (token[0] == 'f' && isSpace((token[1]))) { token += 2; token += strspn(token, " \t"); std::vector<vertex_index> face; while (!isNewLine(token[0])) { vertex_index vi = parseTriple(token, v.size() / 3, vn.size() / 3, vt.size() / 2); face.push_back(vi); int n = strspn(token, " \t\r"); token += n; } faceGroup.push_back(face); continue; } // use mtl if ((0 == strncmp(token, "usemtl", 6)) && isSpace((token[6]))) { char namebuf[4096]; token += 7; sscanf(token, "%s", namebuf); faceGroup.clear(); if (material_map.find(namebuf) != material_map.end()) { material = material_map[namebuf]; } else { // { error!! material not found } material = -1; } continue; } // load mtl if ((0 == strncmp(token, "mtllib", 6)) && isSpace((token[6]))) { char namebuf[4096]; token += 7; sscanf(token, "%s", namebuf); std::string err_mtl = readMatFn(namebuf, materials, material_map); if (!err_mtl.empty()) { faceGroup.clear(); // for safety return err_mtl; } continue; } // group name if (token[0] == 'g' && isSpace((token[1]))) { // flush previous face group. bool ret = exportFaceGroupToShape(shape, vertexCache, v, vn, vt, faceGroup, material, name, true); if (ret) { shapes.push_back(shape); } shape = shape_t(); //material = -1; faceGroup.clear(); std::vector<std::string> names; while (!isNewLine(token[0])) { std::string str = parseString(token); names.push_back(str); token += strspn(token, " \t\r"); // skip tag } assert(names.size() > 0); // names[0] must be 'g', so skipt 0th element. if (names.size() > 1) { name = names[1]; } else { name = ""; } continue; } // object name if (token[0] == 'o' && isSpace((token[1]))) { // flush previous face group. bool ret = exportFaceGroupToShape(shape, vertexCache, v, vn, vt, faceGroup, material, name, true); if (ret) { shapes.push_back(shape); } //material = -1; faceGroup.clear(); shape = shape_t(); // @todo { multiple object name? } char namebuf[4096]; token += 2; sscanf(token, "%s", namebuf); name = std::string(namebuf); continue; } // Ignore unknown command. } bool ret = exportFaceGroupToShape(shape, vertexCache, v, vn, vt, faceGroup, material, name, true); if (ret) { shapes.push_back(shape); } faceGroup.clear(); // for safety return err.str(); }
void MapperGui::mouseClicked(gcn::MouseEvent& mouseEvent){ if( mouseEvent.isConsumed() ){ return; }; //click from the map image ( worldImagePlaceHolder ) if( mouseEvent.getSource() == worldImagePlaceHolder ){ //consume mouse event mouseEvent.consume(); //click from the tilesetImage if( world != NULL ){ if( world->empty() ){ return; }; int layerNum = layerToggleBox->getWorkingLayer(); Layer& workingLayer = world->getLayer(layerNum); // tileset select int tileWidth = workingLayer.getTileWidth(); int tileHeight = workingLayer.getTileHeight(); // get the x-y co-ords reletive to the tileset region int col = mouseEvent.getX() - worldImagePlaceHolder->getX(); int row = mouseEvent.getY() - worldImagePlaceHolder->getY(); // if its a multiple of tilewidth/height (ie in the barrier between) if( col % tileWidth == 0){ ++col; }; if( row % tileHeight == 0){ ++row; }; // turn co-ords into row/col col = col/tileWidth; row = row/tileHeight; // calculate the amount of tiles off the screen int trueCol = worldImagePlaceHolder->getX(); int trueRow = worldImagePlaceHolder->getY(); // if its a multiple of tilewidth/height (ie in the barrier between) if( trueCol % tileWidth == 0){ ++col; }; if( trueRow % tileHeight == 0){ ++row; }; // turn co-ords into row/col trueCol = trueCol/tileWidth; trueRow = trueRow/tileHeight; //add the on screen offset to the offsecreen tiles trueCol += col-1; trueRow += row-1; //check if within range if( (trueCol < workingLayer.getWidth()) && (trueRow < workingLayer.getHeight()) ){ //if left click if( mouseEvent.getButton() == gcn::MouseEvent::LEFT ){ //set the tile wher mouse is on map, to selected tile on ts //if the attrib ts is on if( tilesetBox->getSelectedTileset() == tilesetBox->getAttribTileset() ){ //only replace the attibute workingLayer.at(trueCol, trueRow).setAttribute( tilesetBox->getSelectedTile().getAttribute() ); } else { //replace old tile workingLayer.at(trueCol, trueRow) = tilesetBox->getSelectedTile(); } } else if( mouseEvent.getButton() == gcn::MouseEvent::MIDDLE ){ //fill layer, to selected tile on ts //if the attrib ts is on if( tilesetBox->getSelectedTileset() == tilesetBox->getAttribTileset() ){ //only replace the attibute for(int i = 0; i < workingLayer.getHeight(); ++i){ for(int j = 0; j < workingLayer.getWidth(); ++j){ workingLayer.at(j, i).setAttribute( tilesetBox->getSelectedTile().getAttribute() ); } } } else { //replace old tile for(int i = 0; i < workingLayer.getHeight(); ++i){ for(int j = 0; j < workingLayer.getWidth(); ++j){ workingLayer.at(j, i) = tilesetBox->getSelectedTile(); } } } } else if ( mouseEvent.getButton() == gcn::MouseEvent::RIGHT ){ //if the attrib ts is on if( tilesetBox->getSelectedTileset() == tilesetBox->getAttribTileset() ){ //clear attrib workingLayer.at(trueCol, trueRow).setAttribute( MapTile::REGULAR ); } else { //clear the tileset (delete tile) workingLayer.at(trueCol, trueRow).setSource( NULL, 0, 0 ); } } mapTouched = 1; } }//end if event from drop down } else //click from the tilset panel //load tileset button if( mouseEvent.getSource() == tilesetBox->getLoadTilesetButton() ){ mouseEvent.consume(); loadTilesetBox->setVisible( true ); loadTilesetBox->requestFocus(); } else //make tileset button if( mouseEvent.getSource() == tilesetBox->getMakeTilesetButton() ){ mouseEvent.consume(); makeTilesetBox->setVisible( true ); makeTilesetBox->requestFocus(); } else //close tileset button if( mouseEvent.getSource() == tilesetBox->getCloseTilesetButton() ){ mouseEvent.consume(); tilesetDetailsBox->setInfoSource( NULL ); } else //display tileset info if( mouseEvent.getSource() == tilesetBox->getTilesetDetailsButton() ){ mouseEvent.consume(); tilesetDetailsBox->setInfoSource( tilesetBox->getSelectedTileset() ); tilesetDetailsBox->setVisible( true ); tilesetDetailsBox->requestFocus(); } else // from file tool bar //new doc if( mouseEvent.getSource() == fileToolBar->getNewDocButton() ){ mouseEvent.consume(); loadMapBox->setVisible( true ); loadMapBox->requestFocus(); } else //save document if( mouseEvent.getSource() == fileToolBar->getSaveDocButton() ){ mouseEvent.consume(); if( world == NULL ){ alert("Theres no map to save (try the new map button =P)"); return; } if( world->size() == 0 ){ alert("Theres no map to save (try the new map button =P)"); return; } std::string a = world->getFilename() ; if( world->getFilename() == "NULL" || world->getFilename().empty() || isSpace(world->getFilename()) || world->getFilename() == "" ){ filenameBox->setFilename("data/maps/"); filenameBox->setVisible( true ); action = SAVEAS; } else { if( !world->saveMap(world->getFilename()) ){ alert( ("Cant save file: " + world->getFilename()) ); return; } else { alert("Saved"); } } } else // save as if( mouseEvent.getSource() == fileToolBar->getSaveAsButton() ){ mouseEvent.consume(); if( world == NULL ){ alert("Theres no map to save (try the new map button =P)"); return; } if( world->size() == 0 ){ alert("Theres no map to save (try the new map button =P)"); return; } action = SAVEAS; filenameBox->setFilename("data/maps/"); filenameBox->setVisible( true ); } else //open new doc (close current 1st) if( mouseEvent.getSource() == fileToolBar->getOpenDocButton() ){ mouseEvent.consume(); action = OPEN; filenameBox->setFilename("data/maps/"); filenameBox->setVisible( true ); } else //close (no save) and reOpen if( mouseEvent.getSource() == fileToolBar->getReopenDocButton() ){ mouseEvent.consume(); if( world != NULL ){ std::string filename = correctFilepath(world->getFilename()); //delete map notify("Releasing old world from memory"); world->unload(); //clear the tileset array notify("Releasing tilesets"); tilesetBox->freeTilesets(); //reopen notify("Reopening file"); if( !world->loadMap(filename) ){ alert( ("Cant open file: " + filename) ); } unNotify(); mapDetailsBox->setInfoSource( world ); // free old worldImage if( worldImage != NULL ){ SDL_FreeSurface(worldImage); worldImage = NULL; }; mapTouched = 1; action = NOTHING; } } else //close if( mouseEvent.getSource() == fileToolBar->getCloseDocButton() ){ mouseEvent.consume(); //delete map notify("Releasing old world from memory"); if( world != NULL ){ //delete worlds world->unload(); //clear the tileset array notify("Releasing tilesets"); tilesetBox->freeTilesets(); unNotify(); mapDetailsBox->setInfoSource( world ); // free old worldImage if( worldImage != NULL ){ SDL_FreeSurface(worldImage); worldImage = NULL; } mapTouched = 1; action = NOTHING; } } else //get details if( mouseEvent.getSource() == fileToolBar->getDetailsButton() ){ mouseEvent.consume(); mapDetailsBox->setVisible( true ); } }
Token *Lexer::scanNewline() { startToken(); do consume(); while(isNewline() || isSpace()); return finishToken(Token::Newline); }
static int skipWhitespaces (int c) { while (isSpace (c)) c = getcFromInputFile (); return c; }
bool KileScriptDocument::isSpace(int line, int column) { return isSpace(KTextEditor::Cursor(line,column)); }
std::string trimCopy(const std::string& input) { return trimCopyIf(input, isSpace()); }
int config_loadFromBuffer(const char* buf,const int size) { const char* pa = buf; int ind = 0; ConfigEntry obj; config_init(); while(*pa) { { //reset pointer ind = 0; obj.variable[0] = obj.value[0] = '\0'; obj.variableLength = obj.valueLength = 0; //skip head WS while(*pa){ if(isSpace(*pa))pa++;else break;} //load variable while(*pa) { if(*pa == '=') break; else if(*pa == ' ') break; else { if(ind > CONF_ENTRY_BUFFER_Va_SIZE-1) break; { if(! (*pa) ) break; if(*pa == '\r') { ++pa; if(*pa == '\n') ++pa; break; } else if(*pa == '\n') { ++pa; break; } } obj.variable[ind++] = *pa; } ++pa; } obj.variable[ind] = '\0'; obj.variableLength = ind; ind = 0; if( (*pa == '=')) { ++pa; //skip WS after assignment --if exists while(*pa){ if(isSpace(*pa))pa++;else break;} //load value while(*pa) { if(*pa == '\r') { ++pa; if(*pa == '\n') { ++pa; break; } } else if(*pa == '\n') { ++pa; break; } else { if(ind > CONF_ENTRY_BUFFER_Pa_SIZE-1) break; { if(! (*pa) ) break; if(*pa == '\r') { ++pa; if(*pa == '\n') ++pa; break; } else if(*pa == '\n') { ++pa; break; } } obj.value[ind++] = *pa; } ++pa; } obj.value[ind] = '\0'; obj.valueLength = ind; if(obj.variableLength) { //printf("Adding [%s] with data [%s]\n",obj.variable,obj.value); internal_config_push(obj.variable,obj.value,obj.variableLength,obj.valueLength);//config_push(obj.variable,obj.value); } } } } return 0; }
void trimRight(std::string& str) { trimRightIf(str, isSpace()); }
void trim(std::string& str) { trimIf(str, isSpace()); }
void trimLeft(std::string& str) { trimLeftIf(str, isSpace()); }
/* ** Return the length of the token that begins at z[0]. Return ** -1 if the token is (or might be) incomplete. Store the token ** type in *tokenType before returning. */ int sqliteGetToken(const WCHAR *z, int *tokenType, int *skip){ int i; *skip = 0; switch( *z ){ case ' ': case '\t': case '\n': case '\f': for(i=1; isSpace(z[i]); i++){} *tokenType = TK_SPACE; return i; case '-': if( z[1]==0 ) return -1; *tokenType = TK_MINUS; return 1; case '(': *tokenType = TK_LP; return 1; case ')': *tokenType = TK_RP; return 1; case '*': *tokenType = TK_STAR; return 1; case '=': *tokenType = TK_EQ; return 1; case '<': if( z[1]=='=' ){ *tokenType = TK_LE; return 2; }else if( z[1]=='>' ){ *tokenType = TK_NE; return 2; }else{ *tokenType = TK_LT; return 1; } case '>': if( z[1]=='=' ){ *tokenType = TK_GE; return 2; }else{ *tokenType = TK_GT; return 1; } case '!': if( z[1]!='=' ){ *tokenType = TK_ILLEGAL; return 2; }else{ *tokenType = TK_NE; return 2; } case '?': *tokenType = TK_WILDCARD; return 1; case ',': *tokenType = TK_COMMA; return 1; case '`': case '\'': { int delim = z[0]; for(i=1; z[i]; i++){ if( z[i]==delim ) break; } if( z[i] ) i++; if( delim == '`' ) *tokenType = TK_ID; else *tokenType = TK_STRING; return i; } case '.': if( !isDigit(z[1]) ){ *tokenType = TK_DOT; return 1; } /* Fall through */ case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': *tokenType = TK_INTEGER; for(i=1; isDigit(z[i]); i++){} return i; case '[': for(i=1; z[i] && z[i-1]!=']'; i++){} *tokenType = TK_ID; return i; default: if( !isIdChar[*z] ){ break; } for(i=1; isIdChar[z[i]]; i++){} *tokenType = sqliteKeywordCode(z, i); if( *tokenType == TK_ID && z[i] == '`' ) *skip = 1; return i; } *tokenType = TK_ILLEGAL; return 1; }
void TextDataSourceDialog::loadFile(const QString &fileName) { mFileName = fileName; QFile inputFile(mFileName); if (!inputFile.open(QIODevice::ReadOnly | QIODevice::Text)) return; //TODO: Add normal treatment ui->groupBox->setTitle(fileName + " contents"); QTextStream in(&inputFile); int nLines = 0; QMap<QChar, int> symbolsCount; QStringList fileContents; while (!in.atEnd() && nLines < 10) { QString line = in.readLine(); fileContents << line; ui->textBrowser->append("<font color=\"blue\"><b><i>" + QString::number(++nLines) + ":</i></b></font> " + line); for (auto ch : line) { if (symbolsCount.find(ch) == symbolsCount.end()) { symbolsCount[ch] = 0; } symbolsCount[ch]++; } } if (!in.atEnd()) { ui->textBrowser->append("<font color=\"blue\"><b><i> ...</i></b></font> " ); } //TODO: the stuff below is to move scrollBar to the top (doesn't seem to work), need to be fixed ui->textBrowser->verticalScrollBar()->setValue(ui->textBrowser->verticalScrollBar()->minimum()); //determine separator auto isValidSeparator = [] (QChar ch) { return !ch.isLetterOrNumber() && !ch.isSpace() && (ch != sDecimalMark) && (ch != '-') && (ch != '+'); }; int separatorCounter = 0; for (const auto& sym : symbolsCount.keys()) { if (isValidSeparator(sym)) { if (symbolsCount[sym] > separatorCounter) { separatorCounter = symbolsCount[sym]; mFieldSeparator = sym.toLatin1(); } } } ui->fieldSeparatorLineEdit->setText(QString(mFieldSeparator)); auto countPercentageOfLetters = [separator = this->mFieldSeparator] (const QString& str) { int totalNumber = 0; int numberLetters = 0; for (auto i = str.begin(); i != str.end(); ++i) { if (*i != separator && ! i->isSpace()) { totalNumber++; if (i->isLetter()) { numberLetters++; } } } return totalNumber ? (double)numberLetters / totalNumber : 0; }; if (fileContents.size() > 0 && countPercentageOfLetters(fileContents[0]) > 0.5) { ui->namesLineComboBox->setCurrentValue(0); if (fileContents.size() > 1 && countPercentageOfLetters(fileContents[1]) > 0.5) { ui->descriptionLineComboBox->setCurrentValue(1); if (fileContents.size() > 2 && countPercentageOfLetters(fileContents[2]) > 0.5) { ui->unitsLineComboBox->setCurrentValue(2); } } } }
/* The lexer is in charge of reading the file. * Some of sub-lexer (like eatComment) also read file. * lexing is finished when the lexer return Tok_EOF */ static ocamlKeyword lex (lexingState * st) { int retType; /* handling data input here */ while (st->cp == NULL || st->cp[0] == '\0') { st->cp = fileReadLine (); if (st->cp == NULL) return Tok_EOF; } if (isAlpha (*st->cp)) { readIdentifier (st); retType = lookupKeyword (vStringValue (st->name), Lang_Ocaml); if (retType == -1) /* If it's not a keyword */ { return OcaIDENTIFIER; } else { return retType; } } else if (isNum (*st->cp)) return eatNumber (st); else if (isSpace (*st->cp)) { eatWhiteSpace (st); return lex (st); } /* OCaml permit the definition of our own operators * so here we check all the consecuting chars which * are operators to discard them. */ else if (isOperator[*st->cp]) return eatOperator (st); else switch (*st->cp) { case '(': if (st->cp[1] == '*') /* ergl, a comment */ { eatComment (st); return lex (st); } else { st->cp++; return Tok_PARL; } case ')': st->cp++; return Tok_PARR; case '[': st->cp++; return Tok_BRL; case ']': st->cp++; return Tok_BRR; case '{': st->cp++; return Tok_CurlL; case '}': st->cp++; return Tok_CurlR; case '\'': st->cp++; return Tok_Prime; case ',': st->cp++; return Tok_comma; case '=': st->cp++; return Tok_EQ; case ';': st->cp++; return Tok_semi; case '"': eatString (st); return Tok_Val; case '_': st->cp++; return Tok_Val; case '#': st->cp++; return Tok_Sharp; case '\\': st->cp++; return Tok_Backslash; default: st->cp++; break; } /* default return if nothing is recognized, * shouldn't happen, but at least, it will * be handled without destroying the parsing. */ return Tok_Val; }
int DSNLEXER::NextTok() throw( IO_ERROR ) { const char* cur = next; const char* head = cur; prevTok = curTok; if( curTok != DSN_EOF ) { if( cur >= limit ) { L_read: // blank lines are returned as "\n" and will have a len of 1. // EOF will have a len of 0 and so is detectable. int len = readLine(); if( len == 0 ) { cur = start; // after readLine(), since start can change, set cur offset to start curTok = DSN_EOF; goto exit; } cur = start; // after readLine() since start can change. // skip leading whitespace while( cur<limit && isSpace(*cur) ) ++cur; // If the first non-blank character is #, this line is a comment. // Comments cannot follow any other token on the same line. if( cur<limit && *cur=='#' ) { if( commentsAreTokens ) { // save the entire line, including new line as the current token. // the '#' character may not be at offset zero. curText = start; // entire line is the token cur = start; // ensure a good curOffset below curTok = DSN_COMMENT; head = limit; // do a readLine() on next call in here. goto exit; } else goto L_read; } } else { // skip leading whitespace while( cur<limit && isSpace(*cur) ) ++cur; } if( cur >= limit ) goto L_read; // switching the string_quote character if( prevTok == DSN_STRING_QUOTE ) { static const wxString errtxt( _("String delimiter must be a single character of ', \", or $")); char cc = *cur; switch( cc ) { case '\'': case '$': case '"': break; default: THROW_PARSE_ERROR( errtxt, CurSource(), CurLine(), CurLineNumber(), CurOffset() ); } curText = cc; head = cur+1; if( head<limit && *head!=')' && *head!='(' && !isSpace(*head) ) { THROW_PARSE_ERROR( errtxt, CurSource(), CurLine(), CurLineNumber(), CurOffset() ); } curTok = DSN_QUOTE_DEF; goto exit; } if( *cur == '(' ) { curText = *cur; curTok = DSN_LEFT; head = cur+1; goto exit; } if( *cur == ')' ) { curText = *cur; curTok = DSN_RIGHT; head = cur+1; goto exit; } /* get the dash out of a <pin_reference> which is embedded for example like: U2-14 or "U2"-"14" This is detectable by a non-space immediately preceeding the dash. */ if( *cur == '-' && cur>start && !isSpace( cur[-1] ) ) { curText = '-'; curTok = DSN_DASH; head = cur+1; goto exit; } // handle DSN_NUMBER if( strchr( "+-.0123456789", *cur ) ) { head = cur+1; while( head<limit && strchr( ".0123456789", *head ) ) ++head; if( (head<limit && isSpace(*head)) || *head==')' || *head=='(' || head==limit ) { curText.clear(); curText.append( cur, head ); curTok = DSN_NUMBER; goto exit; } // else it was something like +5V, fall through below } // a quoted string, will return DSN_STRING if( *cur == stringDelimiter ) { // Non-specctraMode, understands and deciphers escaped \, \r, \n, and \". // Strips off leading and trailing double quotes if( !specctraMode ) { // copy the token, character by character so we can remove doubled up quotes. curText.clear(); ++cur; // skip over the leading delimiter, which is always " in non-specctraMode head = cur; while( head<limit ) { // ESCAPE SEQUENCES: if( *head =='\\' ) { char tbuf[8]; char c; int i; if( ++head >= limit ) break; // throw exception at L_unterminated switch( *head++ ) { case '"': case '\\': c = head[-1]; break; case 'a': c = '\x07'; break; case 'b': c = '\x08'; break; case 'f': c = '\x0c'; break; case 'n': c = '\n'; break; case 'r': c = '\r'; break; case 't': c = '\x09'; break; case 'v': c = '\x0b'; break; case 'x': // 1 or 2 byte hex escape sequence for( i=0; i<2; ++i ) { if( !isxdigit( head[i] ) ) break; tbuf[i] = head[i]; } tbuf[i] = '\0'; if( i > 0 ) c = (char) strtoul( tbuf, NULL, 16 ); else c = 'x'; // a goofed hex escape sequence, interpret as 'x' head += i; break; default: // 1-3 byte octal escape sequence --head; for( i=0; i<3; ++i ) { if( head[i] < '0' || head[i] > '7' ) break; tbuf[i] = head[i]; } tbuf[i] = '\0'; if( i > 0 ) c = (char) strtoul( tbuf, NULL, 8 ); else c = '\\'; // a goofed octal escape sequence, interpret as '\' head += i; break; } curText += c; } else if( *head == '"' ) // end of the non-specctraMode DSN_STRING { curTok = DSN_STRING; ++head; // omit this trailing double quote goto exit; } else curText += *head++; } // while // L_unterminated: wxString errtxt(_("Un-terminated delimited string") ); THROW_PARSE_ERROR( errtxt, CurSource(), CurLine(), CurLineNumber(), CurOffset() ); } else // specctraMode DSN_STRING { ++cur; // skip over the leading delimiter: ",', or $ head = cur; while( head<limit && !isStringTerminator( *head ) ) ++head; if( head >= limit ) { wxString errtxt(_("Un-terminated delimited string") ); THROW_PARSE_ERROR( errtxt, CurSource(), CurLine(), CurLineNumber(), CurOffset() ); } curText.clear(); curText.append( cur, head ); ++head; // skip over the trailing delimiter curTok = DSN_STRING; goto exit; } } // Maybe it is a token we will find in the token table. // If not, then call it a DSN_SYMBOL. { head = cur+1; while( head<limit && !isSpace( *head ) && *head!=')' && *head!='(' ) ++head; curText.clear(); curText.append( cur, head ); int found = findToken( curText ); if( found != -1 ) curTok = found; else if( 0 == curText.compare( "string_quote" ) ) curTok = DSN_STRING_QUOTE; else // unrecogized token, call it a symbol curTok = DSN_SYMBOL; } } exit: // single point of exit, no returns elsewhere please. curOffset = cur - start; next = head; // printf("tok:\"%s\"\n", curText.c_str() ); return curTok; }
InlineBotQuery ParseInlineBotQuery(not_null<const Ui::InputField*> field) { auto result = InlineBotQuery(); const auto &text = field->getTextWithTags().text; const auto textLength = text.size(); auto inlineUsernameStart = 1; auto inlineUsernameLength = 0; if (textLength > 2 && text[0] == '@' && text[1].isLetter()) { inlineUsernameLength = 1; for (auto i = inlineUsernameStart + 1; i != textLength; ++i) { const auto ch = text[i]; if (ch.isLetterOrNumber() || ch.unicode() == '_') { ++inlineUsernameLength; continue; } else if (!ch.isSpace()) { inlineUsernameLength = 0; } break; } auto inlineUsernameEnd = inlineUsernameStart + inlineUsernameLength; auto inlineUsernameEqualsText = (inlineUsernameEnd == textLength); auto validInlineUsername = false; if (inlineUsernameEqualsText) { validInlineUsername = text.endsWith(qstr("bot")); } else if (inlineUsernameEnd < textLength && inlineUsernameLength) { validInlineUsername = text[inlineUsernameEnd].isSpace(); } if (validInlineUsername) { auto username = text.midRef(inlineUsernameStart, inlineUsernameLength); if (username != result.username) { result.username = username.toString(); if (const auto peer = App::peerByName(result.username)) { if (const auto user = peer->asUser()) { result.bot = peer->asUser(); } else { result.bot = nullptr; } result.lookingUpBot = false; } else { result.bot = nullptr; result.lookingUpBot = true; } } if (result.lookingUpBot) { result.query = QString(); return result; } else if (result.bot && (!result.bot->botInfo || result.bot->botInfo->inlinePlaceholder.isEmpty())) { result.bot = nullptr; } else { result.query = inlineUsernameEqualsText ? QString() : text.mid(inlineUsernameEnd + 1); return result; } } else { inlineUsernameLength = 0; } } if (inlineUsernameLength < 3) { result.bot = nullptr; result.username = QString(); } result.query = QString(); return result; }
void dots::Put3(STROEM * file,wint_t ch,flags & flgs) // called from PutN, Put2 and GetPut { /* Put3 generally causes a newline (ch=='\n') to be written. Exception: inside htmltags. */ if(flgs.inhtmltag) { flgs.in_abbreviation = false; flgs.person_name = not_a_name; flgs.number_final_dot = false; // This code writes the complete HTML-tag if(Option.suppressHTML) { ch = ' '; } else { if(ch) { if(isSpace(ch)) { last = ' '; //return; // 20100107 } else { if(isFlatSpace(last))// last == ' ' || last == 0xA0 || last == 0x3000) { pRegularizationFnc(file,' ',flgs); } pRegularizationFnc(file,ch,flgs); } } } } else { if(isFlatSpace(ch))//20100106 // ch == 0xA0) // 20071112 ch = ' '; else if((unsigned int)ch < ' ') // replace tabs by spaces and all other non-white space by an asterisk { if(ch != '\n') ch = '*'; } if(ch == ' ') { if(last != '\n') // Spaces at the beginning of a line are ignored. Only spaces after words are recorded in 'last'. { last = ' '; } return; } if(ch == '\n') { if(trailingDotFollowingNumber) { if(Option.tokenize) pRegularizationFnc(file, Option.tokenSplit,flgs); // insert blank before dot if number followed by dot is at the end of the line pRegularizationFnc(file,'.',flgs); trailingDotFollowingNumber = false; flgs.in_abbreviation = false; } } else if(isFlatSpace(last))// last == ' ' || last == 0xA0) { wint_t lastToWrite = ' '; if(!isLower(ch)) // Might be an indication that a new sentence starts here. // Check preceding token for trailing dot that might be a // sentence delimiter after all. { if(trailingDotFollowingNumber) // ... in 1999. Next month ... ch=='N', last is ' ' // ^ Not written from here {// Regard dot as sentence delimiter after all if(Option.tokenize) pRegularizationFnc(file, Option.tokenSplit,flgs); // Insert blank before dot if number followed by dot is followed by capitalised word. pRegularizationFnc(file,'.',flgs); trailingDotFollowingNumber = false; lastToWrite = '\n'; // Number seems to be the last word of the previous sentence. Fake history. // ... in 1999. // Next month ... flgs.in_abbreviation = false; } else if(flgs.in_abbreviation /*&& flgs.newSegment*/) { switch(flgs.person_name) { case initial: flgs.person_name = not_a_name; break; case not_a_name: default: // Skema 1. Affald fra husholdninger --> Skema 1. | Affald fra husholdninger 20040420 if(!flgs.expectCapitalizedWord) lastToWrite = '\n'; // Abbreviation seems to be the last word of the previous sentence break; } } } else if(trailingDotFollowingNumber) { // Now we suppose that the dot trailing the number is part of that number. pRegularizationFnc(file,'.',flgs); trailingDotFollowingNumber = false; } if((lastToWrite != ' ' && lastToWrite != 0xA0) || flgs.writtentoline) pRegularizationFnc(file,lastToWrite,flgs); flgs.writtentoline = (lastToWrite == ' ' || lastToWrite == 0xA0); if(Option.emptyline && flgs.in_abbreviation && !flgs.writtentoline) { // Make sure to send \n next time ensureEmptyLine = true; } flgs.in_abbreviation = false; } if(flgs.number_final_dot == 1 && !(flgs.person_name == initial)) // This can only be the case if ch == separating character { trailingDotFollowingNumber = true; } else { if(ch != '\n' && ensureEmptyLine) { pRegularizationFnc(file,'\n',flgs); } ensureEmptyLine = false; pRegularizationFnc(file,ch,flgs); flgs.writtentoline = ch != '\n'; if(Option.emptyline && flgs.in_abbreviation && !flgs.writtentoline) { // Make sure to send \n next time ensureEmptyLine = true; } } } last = ch; // flgs.newSegment = false; }
Token Lexer::nextToken() { if (isDone()) { return *finish_token; } int tokenType = END_OF_FILE; char c = nextChar(); string currentValue=""; string jsonValue =""; while (isDone() != true && isSpace(c)==true) { c = nextChar(); } if ('"' == c) { tokenType = VALUE; if (isDone() != true) { c = nextChar(); while (index < length && c != '"') { currentValue += c; if (c == '\\' && index < length) { c = nextChar(); currentValue += c; } c = nextChar(); } jsonValue = currentValue; } else { throw KevoreeException("Lexer Unterminated string nextToken"); } }else if ('{' == c) { tokenType = LEFT_BRACE; } else if ('}' == c) { tokenType = RIGHT_BRACE; } else if ('[' == c) { tokenType = LEFT_BRACKET; } else if (']' == c) { tokenType = RIGHT_BRACKET; } else if (':' == c) { tokenType = COLON; } else if (',' == c) { tokenType = COMMA; } else if (! isDone()) { while (isValueLetter(tolower(c)) == true) { currentValue += c; if (isValueLetter(tolower(peekChar())) != true) { break; } else { c = nextChar(); } } string v = currentValue; std::transform(v.begin(), v.end(), v.begin(), ::tolower); if (v.compare("true") == 0) { jsonValue = "true"; } else if (v.compare("false") == 0) { jsonValue = "false"; } else { jsonValue = v; } tokenType = VALUE; } else { tokenType = END_OF_FILE; } return Token(tokenType, jsonValue); }
void ConfigManager::loadFromStream(SeekableReadStream &stream) { String domainName; String comment; Domain domain; int lineno = 0; _appDomain.clear(); _gameDomains.clear(); _miscDomains.clear(); _transientDomain.clear(); _domainSaveOrder.clear(); #ifdef ENABLE_KEYMAPPER _keymapperDomain.clear(); #endif // TODO: Detect if a domain occurs multiple times (or likewise, if // a key occurs multiple times inside one domain). while (!stream.eos() && !stream.err()) { lineno++; // Read a line String line = stream.readLine(); if (line.size() == 0) { // Do nothing } else if (line[0] == '#') { // Accumulate comments here. Once we encounter either the start // of a new domain, or a key-value-pair, we associate the value // of the 'comment' variable with that entity. comment += line; comment += "\n"; } else if (line[0] == '[') { // It's a new domain which begins here. // Determine where the previously accumulated domain goes, if we accumulated anything. addDomain(domainName, domain); domain.clear(); const char *p = line.c_str() + 1; // Get the domain name, and check whether it's valid (that // is, verify that it only consists of alphanumerics, // dashes and underscores). while (*p && (isAlnum(*p) || *p == '-' || *p == '_')) p++; if (*p == '\0') error("Config file buggy: missing ] in line %d", lineno); else if (*p != ']') error("Config file buggy: Invalid character '%c' occurred in section name in line %d", *p, lineno); domainName = String(line.c_str() + 1, p); domain.setDomainComment(comment); comment.clear(); } else { // This line should be a line with a 'key=value' pair, or an empty one. // Skip leading whitespaces const char *t = line.c_str(); while (isSpace(*t)) t++; // Skip empty lines / lines with only whitespace if (*t == 0) continue; // If no domain has been set, this config file is invalid! if (domainName.empty()) { error("Config file buggy: Key/value pair found outside a domain in line %d", lineno); } // Split string at '=' into 'key' and 'value'. First, find the "=" delimeter. const char *p = strchr(t, '='); if (!p) error("Config file buggy: Junk found in line line %d: '%s'", lineno, t); // Extract the key/value pair String key(t, p); String value(p + 1); // Trim of spaces key.trim(); value.trim(); // Finally, store the key/value pair in the active domain domain[key] = value; // Store comment domain.setKVComment(key, comment); comment.clear(); } } addDomain(domainName, domain); // Add the last domain found }
/* The lexer is in charge of reading the file. * Some of sub-lexer (like eatComment) also read file. * lexing is finished when the lexer return Tok_EOF */ static objcKeyword lex (lexingState * st) { int retType; /* handling data input here */ while (st->cp == NULL || st->cp[0] == '\0') { st->cp = fileReadLine (); if (st->cp == NULL) return Tok_EOF; return Tok_EOL; } if (isAlpha (*st->cp)) { readIdentifier (st); retType = lookupKeyword (vStringValue (st->name), Lang_ObjectiveC); if (retType == -1) /* If it's not a keyword */ { return ObjcIDENTIFIER; } else { return retType; } } else if (*st->cp == '@') { readIdentifierObjcDirective (st); retType = lookupKeyword (vStringValue (st->name), Lang_ObjectiveC); if (retType == -1) /* If it's not a keyword */ { return Tok_any; } else { return retType; } } else if (isSpace (*st->cp)) { eatWhiteSpace (st); return lex (st); } else switch (*st->cp) { case '(': st->cp++; return Tok_PARL; case '\\': st->cp++; return Tok_Backslash; case '#': st->cp++; return Tok_Sharp; case '/': if (st->cp[1] == '*') /* ergl, a comment */ { eatComment (st); return lex (st); } else if (st->cp[1] == '/') { st->cp = NULL; return lex (st); } else { st->cp++; return Tok_any; } break; case ')': st->cp++; return Tok_PARR; case '{': st->cp++; return Tok_CurlL; case '}': st->cp++; return Tok_CurlR; case '[': st->cp++; return Tok_SQUAREL; case ']': st->cp++; return Tok_SQUARER; case ',': st->cp++; return Tok_COMA; case ';': st->cp++; return Tok_semi; case ':': st->cp++; return Tok_dpoint; case '"': eatString (st); return Tok_any; case '+': st->cp++; return Tok_PLUS; case '-': st->cp++; return Tok_MINUS; default: st->cp++; break; } /* default return if nothing is recognized, * shouldn't happen, but at least, it will * be handled without destroying the parsing. */ return Tok_any; }
Token* Tokenizer::getToken() { int startRow = row; int startCol = col; QChar c = getChar(); // get and store the next character from the string // catch the end of the input string if (atEnd) return new Token(Token::EndOfInput, "END", row, col, row, col); int cType = translator->look2type(c); // since we need to know it often we store it // catch spaces if (isSpace(c)) { QString look; do { look += (isTab(c) ? " " : " "); c = getChar(); } while (isSpace(c) && !atEnd); ungetChar(); return new Token(Token::WhiteSpace, look, startRow, startCol, row, col); } // catch EndOfLine's if (isBreak(c)) { return new Token(Token::EndOfLine, "\\n", startRow, startCol, startRow+1, 1); } // catch comments if (cType == Token::Comment) { QString look; do { look += c; c = getChar(); } while (!isBreak(c) && !atEnd); ungetChar(); return new Token(Token::Comment, look, startRow, startCol, row, col); } // catch strings if (cType == Token::StringDelimiter) { QString look = QString(c); do { c = getChar(); look += c; } while (!(translator->look2type(c) == Token::StringDelimiter && look.right(2) != "\\\"") && !isBreak(c) && !atEnd); return new Token(Token::String, look, startRow, startCol, row, col); } // catch variables if (cType == Token::VariablePrefix) { QString look; do { look += c; c = getChar(); } while (isWordChar(c) || c.category() == QChar::Number_DecimalDigit || c == '_'); ungetChar(); return new Token(Token::Variable, look, startRow, startCol, row, col); } // catch words (known commands or function calls) if (isWordChar(c)) { // first char has to be a letter QString look; do { look += c; c = getChar(); } while (isWordChar(c) || c.isDigit() || c == '_'); // next chars ungetChar(); int type = translator->look2type(look); if (type == Token::Unknown) type = Token::FunctionCall; return new Token(type, look, startRow, startCol, row, col); } // catch numbers if (c.isDigit() || cType == Token::DecimalSeparator) { bool hasDot = false; int localType = cType; QString look; do { if (localType == Token::DecimalSeparator) hasDot = true; look += c; c = getChar(); localType = translator->look2type(c); } while (c.isDigit() || (localType == Token::DecimalSeparator && !hasDot)); ungetChar(); // if all we got is a dot then this is not a number, so return an Error token here if (translator->look2type(look) == Token::DecimalSeparator) return new Token(Token::Error, look, startRow, startCol, row, col); return new Token(Token::Number, look, startRow, startCol, row, col); } // catch previously uncatched 'double charactered tokens' (tokens that ar not in letters, like: == != >= <=) { QString look = QString(c).append(getChar()); int type = translator->look2type(look); if (type != Token::Unknown) return new Token(type, look, startRow, startCol, row, col); ungetChar(); } // catch known tokens of a single character (as last...) if (cType != Token::Unknown) return new Token(cType, static_cast<QString>(c), startRow, startCol, row, col); // this does not neglect calls to functions with a name of length one (checked it) return new Token(Token::Error, static_cast<QString>(c), startRow, startCol, row, col); }
char *OpenDDLParser::parseDataList( char *in, char *end, Value::ValueType type, Value **data, size_t &numValues, Reference **refs, size_t &numRefs ) { *data = ddl_nullptr; numValues = numRefs = 0; if( ddl_nullptr == in || in == end ) { return in; } in = lookForNextToken( in, end ); if( *in == '{' ) { ++in; Value *current( ddl_nullptr ), *prev( ddl_nullptr ); while( '}' != *in ) { current = ddl_nullptr; in = lookForNextToken( in, end ); if ( Value::ddl_ref == type ) { std::vector<Name*> names; in = parseReference( in, end, names ); if ( !names.empty() ) { Reference *ref = new Reference( names.size(), &names[ 0 ] ); *refs = ref; numRefs = names.size(); } } else if ( Value::ddl_none == type ) { if (isInteger( in, end )) { in = parseIntegerLiteral( in, end, ¤t ); } else if (isFloat( in, end )) { in = parseFloatingLiteral( in, end, ¤t ); } else if (isStringLiteral( *in )) { in = parseStringLiteral( in, end, ¤t ); } else if (isHexLiteral( in, end )) { in = parseHexaLiteral( in, end, ¤t ); } } else { switch(type){ case Value::ddl_int8: case Value::ddl_int16: case Value::ddl_int32: case Value::ddl_int64: case Value::ddl_unsigned_int8: case Value::ddl_unsigned_int16: case Value::ddl_unsigned_int32: case Value::ddl_unsigned_int64: in = parseIntegerLiteral( in, end, ¤t, type); break; case Value::ddl_half: case Value::ddl_float: case Value::ddl_double: in = parseFloatingLiteral( in, end, ¤t, type); break; case Value::ddl_string: in = parseStringLiteral( in, end, ¤t ); break; default: break; } } if( ddl_nullptr != current ) { if( ddl_nullptr == *data ) { *data = current; prev = current; } else { prev->setNext( current ); prev = current; } ++numValues; } in = getNextSeparator( in, end ); if( ',' != *in && Grammar::CloseBracketToken[ 0 ] != *in && !isSpace( *in ) ) { break; } } ++in; } return in; }
bool PropertiesParser::parse(char ch) { bool ret = false; switch (state) { case state_0: if (ch == '#') state = state_comment; else if (isKeyChar(ch)) { key = ch; keypart = ch; state = state_key; } else if (!isSpace(ch) && ch != '\n' && ch != '\r') throw std::runtime_error("format error in properties"); break; case state_key: if (ch == '.') { event.onKeyPart(keypart); keypart.clear(); key += ch; } else if (isKeyChar(ch)) { keypart += ch; key += ch; } else if (isSpace(ch)) { ret = event.onKeyPart(keypart) || event.onKey(key); state = state_key_sp; } else if (ch == '=') { ret = event.onKeyPart(keypart) || event.onKey(key); state = state_value; } else throw std::runtime_error("parse error in properties while reading key " + key); break; case state_key_sp: if (ch == '=') { state = state_value; } else if (!isSpace(ch)) throw std::runtime_error("parse error while reading key " + key); break; case state_value: if (ch == '\n') { ret = event.onValue(value); value.clear(); state = state_0; } else if (ch == '\\') state = state_value_esc; else if (!value.empty() || !isSpace(ch)) value += ch; break; case state_value_esc: value += ch; state = state_value; break; case state_comment: if (ch == '\n') state = state_0; break; } return ret; }
std::string ObjLoader::LoadObj(shapes_t& shapes, const char* filename, const char* mtl_basepath) { shapes.reset(); std::stringstream err; std::istringstream ifs(FileUtils::getInstance()->getStringFromFile(filename)); std::map<vertex_index, ssize_t> vertexCache; //std::ifstream ifs(filename); if (!ifs) { err << "Cannot open file [" << filename << "]" << std::endl; return err.str(); } std::vector<float> v; std::vector<float> vn; std::vector<float> vt; std::vector<std::vector<vertex_index> > faceGroup; std::string name; // material std::map<std::string, material_t> material_map; material_t material; int maxchars = 8192; // Alloc enough size. std::vector<char> buf(maxchars); // Alloc enough size. while (ifs.peek() != -1) { ifs.getline(&buf[0], maxchars); std::string linebuf(&buf[0]); // Trim newline '\r\n' or '\r' if (linebuf.size() > 0) { if (linebuf[linebuf.size()-1] == '\n') linebuf.erase(linebuf.size()-1); } if (linebuf.size() > 0) { if (linebuf[linebuf.size()-1] == '\n') linebuf.erase(linebuf.size()-1); } // Skip if empty line. if (linebuf.empty()) { continue; } // Skip leading space. const char* token = linebuf.c_str(); token += strspn(token, " \t"); assert(token); if (token[0] == '\0') continue; // empty line if (token[0] == '#') continue; // comment line // vertex if (token[0] == 'v' && isSpace((token[1]))) { token += 2; float x, y, z; parseFloat3(x, y, z, token); v.push_back(x); v.push_back(y); v.push_back(z); continue; } // normal if (token[0] == 'v' && token[1] == 'n' && isSpace((token[2]))) { token += 3; float x, y, z; parseFloat3(x, y, z, token); vn.push_back(x); vn.push_back(y); vn.push_back(z); continue; } // texcoord if (token[0] == 'v' && token[1] == 't' && isSpace((token[2]))) { token += 3; float x, y; parseFloat2(x, y, token); vt.push_back(x); vt.push_back(y); continue; } // face if (token[0] == 'f' && isSpace((token[1]))) { token += 2; token += strspn(token, " \t"); std::vector<vertex_index> face; while (!isNewLine(token[0])) { // fix warning, cast to int, i think int is enough vertex_index vi = parseTriple(token, (int)v.size() / 3, (int)vn.size() / 3, (int)vt.size() / 2); face.push_back(vi); auto n = strspn(token, " \t\r"); token += n; } faceGroup.push_back(face); continue; } // use mtl if ((0 == strncmp(token, "usemtl", 6)) && isSpace((token[6]))) { char namebuf[4096]; token += 7; sscanf(token, "%s", namebuf); if (material_map.find(namebuf) != material_map.end()) { material = material_map[namebuf]; } else { // { error!! material not found } InitMaterial(material); } continue; } // load mtl if ((0 == strncmp(token, "mtllib", 6)) && isSpace((token[6]))) { char namebuf[4096]; token += 7; sscanf(token, "%s", namebuf); std::string err_mtl = LoadMtl(material_map, namebuf, mtl_basepath); if (!err_mtl.empty()) { faceGroup.clear(); // for safety //return err_mtl; } continue; } // group name if (token[0] == 'g' && isSpace((token[1]))) { // flush previous face group. shape_t shape; exportFaceGroupToShape(vertexCache, shapes, v, vn, vt, faceGroup, material, name); faceGroup.clear(); std::vector<std::string> names; while (!isNewLine(token[0])) { std::string str = parseString(token); names.push_back(str); token += strspn(token, " \t\r"); // skip tag } assert(names.size() > 0); // names[0] must be 'g', so skipt 0th element. if (names.size() > 1) { name = names[1]; } else { name = ""; } continue; } // object name if (token[0] == 'o' && isSpace((token[1]))) { // flush previous face group. shape_t shape; exportFaceGroupToShape(vertexCache, shapes, v, vn, vt, faceGroup, material, name); faceGroup.clear(); // @todo { multiple object name? } char namebuf[4096]; token += 2; sscanf(token, "%s", namebuf); name = std::string(namebuf); continue; } // Ignore unknown command. } shape_t shape; exportFaceGroupToShape(vertexCache, shapes, v, vn, vt, faceGroup, material, name); faceGroup.clear(); // for safety return err.str(); }
bool MaxEnt::parseLine( FILE *fin, FILE *fout, bool withClassLabel ) { vector<string> feature; int c; int cnt = 0; enum State{ SPACE, WORD, NONE }; int state = NONE; VarStr word; bool first = true; while( (c=fgetc(fin)) != EOF ) { if ( c == '\n' ) break; cnt ++; if ( state == NONE ) { if ( isSpace(c) ) state = SPACE; else { word.add(c); state = WORD; } continue; } if ( isSpace(c) ) { if ( state == WORD ) { if ( !first ) feature.push_back(word.c_str()); else { if ( !withClassLabel ) feature.push_back(word.c_str()); first = false; } state = SPACE; word.clear(); } } else { if ( state == SPACE ) { word.add(c); state = WORD; } else word.add(c); } } if ( c == EOF && cnt == 0 ) return false; if ( word.length() > 0 ) { if ( !first || (first&&!withClassLabel) ) feature.push_back(word.c_str()); } if ( feature.size() > 0 ) { vector<pair<string,double> > labelProbs; int pcls = predict(feature,labelProbs); fprintf(fout,"%s",labelProbs[pcls].first.c_str()); for( int i = 0; i < labelProbs.size(); i ++ ) fprintf(fout, "\t%s\t%lf", labelProbs[i].first.c_str(), labelProbs[i].second); fprintf(fout,"\n"); } return true; }
std::string LoadMtl ( std::map<std::string, int>& material_map, std::vector<material_t>& materials, std::istream& inStream) { material_map.clear(); std::stringstream err; material_t material; int maxchars = 8192; // Alloc enough size. std::vector<char> buf(maxchars); // Alloc enough size. while (inStream.peek() != -1) { inStream.getline(&buf[0], maxchars); std::string linebuf(&buf[0]); // Trim newline '\r\n' or '\n' if (linebuf.size() > 0) { if (linebuf[linebuf.size()-1] == '\n') linebuf.erase(linebuf.size()-1); } if (linebuf.size() > 0) { if (linebuf[linebuf.size()-1] == '\r') linebuf.erase(linebuf.size()-1); } // Skip if empty line. if (linebuf.empty()) { continue; } // Skip leading space. const char* token = linebuf.c_str(); token += strspn(token, " \t"); assert(token); if (token[0] == '\0') continue; // empty line if (token[0] == '#') continue; // comment line // new mtl if ((0 == strncmp(token, "newmtl", 6)) && isSpace((token[6]))) { // flush previous material. if (!material.name.empty()) { material_map.insert(std::pair<std::string, int>(material.name, materials.size())); materials.push_back(material); } // initial temporary material InitMaterial(material); // set new mtl name char namebuf[4096]; token += 7; sscanf(token, "%s", namebuf); material.name = namebuf; continue; } // ambient if (token[0] == 'K' && token[1] == 'a' && isSpace((token[2]))) { token += 2; float r, g, b; parseFloat3(r, g, b, token); material.ambient[0] = r; material.ambient[1] = g; material.ambient[2] = b; continue; } // diffuse if (token[0] == 'K' && token[1] == 'd' && isSpace((token[2]))) { token += 2; float r, g, b; parseFloat3(r, g, b, token); material.diffuse[0] = r; material.diffuse[1] = g; material.diffuse[2] = b; continue; } // specular if (token[0] == 'K' && token[1] == 's' && isSpace((token[2]))) { token += 2; float r, g, b; parseFloat3(r, g, b, token); material.specular[0] = r; material.specular[1] = g; material.specular[2] = b; continue; } // transmittance if (token[0] == 'K' && token[1] == 't' && isSpace((token[2]))) { token += 2; float r, g, b; parseFloat3(r, g, b, token); material.transmittance[0] = r; material.transmittance[1] = g; material.transmittance[2] = b; continue; } // ior(index of refraction) if (token[0] == 'N' && token[1] == 'i' && isSpace((token[2]))) { token += 2; material.ior = parseFloat(token); continue; } // emission if(token[0] == 'K' && token[1] == 'e' && isSpace(token[2])) { token += 2; float r, g, b; parseFloat3(r, g, b, token); material.emission[0] = r; material.emission[1] = g; material.emission[2] = b; continue; } // shininess if(token[0] == 'N' && token[1] == 's' && isSpace(token[2])) { token += 2; material.shininess = parseFloat(token); continue; } // illum model if (0 == strncmp(token, "illum", 5) && isSpace(token[5])) { token += 6; material.illum = parseInt(token); continue; } // dissolve if ((token[0] == 'd' && isSpace(token[1]))) { token += 1; material.dissolve = parseFloat(token); continue; } if (token[0] == 'T' && token[1] == 'r' && isSpace(token[2])) { token += 2; material.dissolve = parseFloat(token); continue; } // ambient texture if ((0 == strncmp(token, "map_Ka", 6)) && isSpace(token[6])) { token += 7; material.ambient_texname = token; continue; } // diffuse texture if ((0 == strncmp(token, "map_Kd", 6)) && isSpace(token[6])) { token += 7; material.diffuse_texname = token; continue; } // specular texture if ((0 == strncmp(token, "map_Ks", 6)) && isSpace(token[6])) { token += 7; material.specular_texname = token; continue; } // normal texture if ((0 == strncmp(token, "map_Ns", 6)) && isSpace(token[6])) { token += 7; material.normal_texname = token; continue; } // unknown parameter const char* _space = strchr(token, ' '); if(!_space) { _space = strchr(token, '\t'); } if(_space) { int len = _space - token; std::string key(token, len); std::string value = _space + 1; material.unknown_parameter.insert(std::pair<std::string, std::string>(key, value)); } } // flush last material. material_map.insert(std::pair<std::string, int>(material.name, materials.size())); materials.push_back(material); return err.str(); }
bool MEModel::loadTrainingExamples( const char *trainFile ) { FILE *fin = fopen(trainFile,"r"); if ( fin == NULL ) { fprintf( stderr, "open train file %s failed!\n", trainFile ); return false; } int lineNum = 0; int fieldNum = 0; int ch; bool endOfFile = false; int curFieldCnt = 0; while( !endOfFile ) { ch = fgetc(fin); curFieldCnt = 0; while( true ) { while( !isLineEnd(ch) && isSpace(ch) ) ch = fgetc(fin); if( isLineEnd(ch) ) break; while( !isLineEnd(ch) && !isSpace(ch) ) ch = fgetc(fin); curFieldCnt ++; } if ( curFieldCnt > 0 ) { lineNum ++; fieldNum += curFieldCnt; } if( ch == EOF ) endOfFile = true; } rewind(fin); M = lineNum; VarStr word; endOfFile = false; xspace = new SpElem[fieldNum]; x = new SpElem*[M]; y = new int[M]; lineNum = 0; int xIndex = 0; while( !endOfFile ) { ch = fgetc(fin); curFieldCnt = 0; int curY = 0, curX = 0; char *ystr = NULL; while( true ) { while( !isLineEnd(ch) && isSpace(ch) ) ch = fgetc(fin); if( isLineEnd(ch) ) break; word.clear(); while( !isLineEnd(ch) && !isSpace(ch) ) { word.add(ch); ch = fgetc(fin); } curFieldCnt ++; if( curFieldCnt == 1 ) { ystr = new char[word.length()+1]; strcpy(ystr,word.c_str()); } else { curX = feaIdMap.add(word.c_str()); xspace[xIndex].index = curX; xspace[xIndex].value = 1; if ( curFieldCnt == 2 ) x[lineNum] = &xspace[xIndex]; xIndex ++; } } if ( curFieldCnt > 1 ) { xspace[xIndex].index = -1; xspace[xIndex].value = 0; xIndex ++; curY = classNameIdMap.add(ystr); y[lineNum] = curY; lineNum ++; } if( ystr != NULL ) delete []ystr; if ( ch == EOF ) endOfFile = true; } fclose(fin); M = lineNum; classNumber = (int)classNameIdMap.size(); needDestroyTrain = true; return true; }
void XSingleSource::skipSpace() { for (; isSpace() ; ++m_iterator); }
inline std::string &rtrim(std::string &s) { return rtrim(s, isSpace()); }