コード例 #1
0
ファイル: txtstuff.cpp プロジェクト: QFangMobile/rtfreader
bool textSource::segment(int level
                        ,int sstatus
                        ,bool PrevIsField  // True if previous sibling block contains a \field
                        ,charprops CharProps
                        )
    {

    wint_t ch;
    curr_pos = Ftell(sourceFile);// After parsing a html-tag, seeking to curr_pos brings you back to the position where the parsed sequence started.                   

    if(Option.keepEOLsequence)
        {
        copyEOLsequence();
        // SourceFile is rewinded
        }

    do
        {
        ch = Getc(sourceFile);
        end_offset = Ftell(sourceFile);
        if(curr_pos >= tagendpos)
            {
            // We are not inside an HTML-tag.
            if(flgs.inhtmltag)
                {
                flgs.firstafterhtmltag = true;
                flgs.inhtmltag = false;
                }
            // Check whether a well-formed HTML tag is ahead. Returns sourceFile
            // in same file position.
            flgs.htmltagcoming = isHTMLtagComing(ch);
//            assert(new_pos == Ftell(sourceFile));
            assert(end_offset == Ftell(sourceFile));
            }
        else if(flgs.htmltagcoming)
            {
            // We are leaving an HTML-tag and entering a new one.
            flgs.inhtmltag = true;
            flgs.htmltagcoming = false;
            }
        /* Scan in advance, checking whether the line to come is a heading and 
        therefore must be preceded with a newline (WritePar will then be set to 
        true.)
        */

        if(  ch ==  '\n'
          || ch == '\r'
          || ch == WEOF
          || ch == 26
          )
            {
            flgs.in_fileName = false;
            heading = isHeading(firsttext,ch,WriteParAfterHeadingOrField);
            if(!skipSegmentation(firsttext,ch))
                {
                doTheSegmentation(CharProps,true,false); // Bart 20040120. true because: Suppose that end of line is end of segment
                if(!WriteParAfterHeadingOrField && heading)
                    {// A normal line after a heading has WritePar==false and heading==true
                    WriteParAfterHeadingOrField = true;
                    heading = false;
                    }
                }
            if(firsttext.EOL)
                firsttext.b.LS = 1;
            }
        else
            {
            updateFlags(ch,flgs);
            int EOL = firsttext.EOL;
            bool sentenceEnd = checkSentenceStartDueToBullet(ch);
            if(  sentenceEnd 
              || flgs.htmltagcoming
              || flgs.inhtmltag
              || (end_offset - begin_offset > MAXSEGMENTLENGTH && isSpace(ch)) // check for buffer overflow
              )
                {
                doTheSegmentation(CharProps,false,false); 
                firsttext.b.SD = 1;
                firsttext.b.LS = 0;
                }
            if(isSpace(ch))
                {
                if(EOL)
                    firsttext.b.LS = 1;
                }
            else
                {
                firsttext.b.LS = 0;
                firsttext.EOL = 0; // resets SD, CR and LF
                }
            }
        curr_pos = end_offset;
        }
    while(ch != WEOF && ch != 26);
    outputtext->PutHandlingLine('\n',flgs); // 20100106 Flush last line
    return false;
    }
コード例 #2
0
int main(void)
{
  int ret = 0;

  INIT_MACHINE();

  reg_t numCount    = 1;
  reg_t upCount     = 2;
  reg_t lowCount    = 3;
  reg_t specialCount  = 4;
  reg_t otherCount  = 5;
  reg_t spaceCount  = 6;

  MOVIM8(numCount, 0);
  MOVIM8(spaceCount, 0);
  MOVIM8(upCount, 0);
  MOVIM8(lowCount, 0);
  MOVIM8(specialCount, 0);
  MOVIM8(otherCount, 0);

  reg_t numString = 7;
  reg_t upString = 9;
  reg_t lowString = 11;
  reg_t spaceString = 13;
  reg_t specialString = 15;
  reg_t otherString = 17;
  reg_t newlineString = 19;

  MOVIM16(numString, ADDR(0));
  MOVIM16(spaceString, ADDR(16));
  MOVIM16(upString, ADDR(32));
  MOVIM16(lowString, ADDR(48));
  MOVIM16(specialString, ADDR(64));
  MOVIM16(otherString, ADDR(80));
  MOVIM16(newlineString, ADDR(96));


  reg_t i = 21;

  //                                  123456789012
  insertString(numString,     "numbers  = ", i);
  insertString(spaceString,   "spaces   = ", i);
  insertString(upString,      "uppers   = ", i);
  insertString(lowString,     "lowers   = ", i);
  insertString(specialString, "special  = ", i);
  insertString(otherString,   "other    = ", i);

  //endl
  MOVIM8(REG16(newlineString), '\n');
  MOVIM32(REG16(newlineString) + 1, 0);

  ret = getChar(i);
  while (ret != (-1))
  {
    if (isUpLetter(i, i+1))
    {
      INC8(upCount);
    }
    else if (isLowLetter(i, i+1))
    {
      INC8(lowCount);
    }
    else if (isNumber(i, i+1))
    {
      INC8(numCount);
    }
    else if (isSpace(i, i+1))
    {
#ifdef PATCHED
      INC8(spaceCount);
#else
      INC32(spaceCount);
#endif
    }
    else if (isSpecial(i, i+1))
    {
      INC8(specialCount);
    }
    else
    {
      INC8(otherCount);
    }
    printChar(i);
    ret = getChar(i);
  }

  printString(numString, i, i+4);
  printReg8(numCount, i);
  printString(newlineString, i, i+4);

  printString(upString, i, i+4);
  printReg8(upCount, i);
  printString(newlineString, i, i+4);

  printString(lowString, i, i+4);
  printReg8(lowCount, i);
  printString(newlineString, i, i+4);

  printString(spaceString, i, i+4);
  printReg8(spaceCount, i);
  printString(newlineString, i, i+4);

  printString(specialString, i, i+4);
  printReg8(specialCount, i);
  printString(newlineString, i, i+4);

  printString(otherString, i, i+4);
  printReg8(otherCount, i);
  printString(newlineString, i, i+4);

  return (0);
}
コード例 #3
0
std::string LoadObj(
  std::vector<shape_t>& shapes,
  std::vector<material_t>& materials,   // [output]
  std::istream& inStream,
  MaterialReader& readMatFn)
{
  std::stringstream err;

  std::vector<float> v;
  std::vector<float> vn;
  std::vector<float> vt;
  std::vector<std::vector<vertex_index> > faceGroup;
  std::string name;

  // material
  std::map<std::string, int> material_map;
  std::map<vertex_index, unsigned int> vertexCache;
  int  material = -1;

  shape_t shape;

  int maxchars = 8192;  // Alloc enough size.
  std::vector<char> buf(maxchars);  // Alloc enough size.
  while (inStream.peek() != -1) {
    inStream.getline(&buf[0], maxchars);

    std::string linebuf(&buf[0]);

    // Trim newline '\r\n' or '\n'
    if (linebuf.size() > 0) {
      if (linebuf[linebuf.size()-1] == '\n') linebuf.erase(linebuf.size()-1);
    }
    if (linebuf.size() > 0) {
      if (linebuf[linebuf.size()-1] == '\r') linebuf.erase(linebuf.size()-1);
    }

    // Skip if empty line.
    if (linebuf.empty()) {
      continue;
    }

    // Skip leading space.
    const char* token = linebuf.c_str();
    token += strspn(token, " \t");

    assert(token);
    if (token[0] == '\0') continue; // empty line

    if (token[0] == '#') continue;  // comment line

    // vertex
    if (token[0] == 'v' && isSpace((token[1]))) {
      token += 2;
      float x, y, z;
      parseFloat3(x, y, z, token);
      v.push_back(x);
      v.push_back(y);
      v.push_back(z);
      continue;
    }

    // normal
    if (token[0] == 'v' && token[1] == 'n' && isSpace((token[2]))) {
      token += 3;
      float x, y, z;
      parseFloat3(x, y, z, token);
      vn.push_back(x);
      vn.push_back(y);
      vn.push_back(z);
      continue;
    }

    // texcoord
    if (token[0] == 'v' && token[1] == 't' && isSpace((token[2]))) {
      token += 3;
      float x, y;
      parseFloat2(x, y, token);
      vt.push_back(x);
      vt.push_back(y);
      continue;
    }

    // face
    if (token[0] == 'f' && isSpace((token[1]))) {
      token += 2;
      token += strspn(token, " \t");

      std::vector<vertex_index> face;
      while (!isNewLine(token[0])) {
        vertex_index vi = parseTriple(token, v.size() / 3, vn.size() / 3, vt.size() / 2);
        face.push_back(vi);
        int n = strspn(token, " \t\r");
        token += n;
      }

      faceGroup.push_back(face);

      continue;
    }

    // use mtl
    if ((0 == strncmp(token, "usemtl", 6)) && isSpace((token[6]))) {

      char namebuf[4096];
      token += 7;
      sscanf(token, "%s", namebuf);

      faceGroup.clear();

      if (material_map.find(namebuf) != material_map.end()) {
        material = material_map[namebuf];
      } else {
        // { error!! material not found }
        material = -1;
      }

      continue;

    }

    // load mtl
    if ((0 == strncmp(token, "mtllib", 6)) && isSpace((token[6]))) {
      char namebuf[4096];
      token += 7;
      sscanf(token, "%s", namebuf);

      std::string err_mtl = readMatFn(namebuf, materials, material_map);
      if (!err_mtl.empty()) {
        faceGroup.clear();  // for safety
        return err_mtl;
      }

      continue;
    }

    // group name
    if (token[0] == 'g' && isSpace((token[1]))) {

      // flush previous face group.
      bool ret = exportFaceGroupToShape(shape, vertexCache, v, vn, vt, faceGroup, material, name, true);
      if (ret) {
        shapes.push_back(shape);
      }

      shape = shape_t();

      //material = -1;
      faceGroup.clear();

      std::vector<std::string> names;
      while (!isNewLine(token[0])) {
        std::string str = parseString(token);
        names.push_back(str);
        token += strspn(token, " \t\r"); // skip tag
      }

      assert(names.size() > 0);

      // names[0] must be 'g', so skipt 0th element.
      if (names.size() > 1) {
        name = names[1];
      } else {
        name = "";
      }

      continue;
    }

    // object name
    if (token[0] == 'o' && isSpace((token[1]))) {

      // flush previous face group.
      bool ret = exportFaceGroupToShape(shape, vertexCache, v, vn, vt, faceGroup, material, name, true);
      if (ret) {
        shapes.push_back(shape);
      }

      //material = -1;
      faceGroup.clear();
      shape = shape_t();

      // @todo { multiple object name? }
      char namebuf[4096];
      token += 2;
      sscanf(token, "%s", namebuf);
      name = std::string(namebuf);


      continue;
    }

    // Ignore unknown command.
  }

  bool ret = exportFaceGroupToShape(shape, vertexCache, v, vn, vt, faceGroup, material, name, true);
  if (ret) {
    shapes.push_back(shape);
  }
  faceGroup.clear();  // for safety

  return err.str();
}
コード例 #4
0
ファイル: MapperGui.cpp プロジェクト: hoodwolf/Infraelly
void MapperGui::mouseClicked(gcn::MouseEvent& mouseEvent){
    if( mouseEvent.isConsumed() ){ return; };

    //click from the map image ( worldImagePlaceHolder )
    if( mouseEvent.getSource() == worldImagePlaceHolder ){
        //consume mouse event
        mouseEvent.consume();
        //click from the tilesetImage
        if( world != NULL ){
            if( world->empty() ){ return; };

            int layerNum = layerToggleBox->getWorkingLayer();
            Layer& workingLayer = world->getLayer(layerNum);

            // tileset select
            int tileWidth = workingLayer.getTileWidth();
            int tileHeight = workingLayer.getTileHeight();

            // get the x-y co-ords reletive to the tileset region
            int col = mouseEvent.getX() - worldImagePlaceHolder->getX();
            int row = mouseEvent.getY() - worldImagePlaceHolder->getY();

            // if its a multiple of tilewidth/height (ie in the barrier between)
            if( col % tileWidth == 0){ ++col; };
            if( row % tileHeight == 0){ ++row; };

            // turn co-ords into row/col
            col = col/tileWidth;
            row = row/tileHeight;


            // calculate the amount of tiles off the screen
            int trueCol = worldImagePlaceHolder->getX();
            int trueRow = worldImagePlaceHolder->getY();

            //  if its a multiple of tilewidth/height (ie in the barrier between)
            if( trueCol % tileWidth == 0){ ++col; };
            if( trueRow % tileHeight == 0){ ++row; };
            // turn co-ords into row/col
            trueCol = trueCol/tileWidth;
            trueRow = trueRow/tileHeight;
            //add the on screen offset to the offsecreen tiles
            trueCol += col-1;
            trueRow += row-1;

            //check if within range
            if( (trueCol < workingLayer.getWidth()) &&
                (trueRow < workingLayer.getHeight()) ){
                //if left click
                if( mouseEvent.getButton() == gcn::MouseEvent::LEFT ){
                    //set the tile wher mouse is on map, to selected tile on ts
                    //if the attrib ts is on
                    if( tilesetBox->getSelectedTileset() == tilesetBox->getAttribTileset() ){
                        //only replace the attibute
                        workingLayer.at(trueCol, trueRow).setAttribute( tilesetBox->getSelectedTile().getAttribute() );
                    } else {
                        //replace old tile
                        workingLayer.at(trueCol, trueRow) = tilesetBox->getSelectedTile();
                    }
                } else if( mouseEvent.getButton() == gcn::MouseEvent::MIDDLE ){
                    //fill layer, to selected tile on ts
                    //if the attrib ts is on
                    if( tilesetBox->getSelectedTileset() == tilesetBox->getAttribTileset() ){
                        //only replace the attibute
                        for(int i = 0; i < workingLayer.getHeight(); ++i){
                            for(int j = 0; j < workingLayer.getWidth(); ++j){
                                workingLayer.at(j, i).setAttribute( tilesetBox->getSelectedTile().getAttribute() );
                            }
                        }
                    } else {
                        //replace old tile
                        for(int i = 0; i < workingLayer.getHeight(); ++i){
                            for(int j = 0; j < workingLayer.getWidth(); ++j){
                                workingLayer.at(j, i) = tilesetBox->getSelectedTile();
                            }
                        }
                    }
                } else if ( mouseEvent.getButton() == gcn::MouseEvent::RIGHT ){
                    //if the attrib ts is on
                    if( tilesetBox->getSelectedTileset() == tilesetBox->getAttribTileset() ){
                        //clear attrib
                        workingLayer.at(trueCol, trueRow).setAttribute( MapTile::REGULAR );
                    } else {
                        //clear the tileset (delete tile)
                        workingLayer.at(trueCol, trueRow).setSource( NULL, 0, 0 );
                    }
                }
                mapTouched = 1;
            }
        }//end if event from drop down
    } else




    //click from the tilset panel
    //load tileset button
    if( mouseEvent.getSource() == tilesetBox->getLoadTilesetButton() ){
        mouseEvent.consume();
        loadTilesetBox->setVisible( true );
        loadTilesetBox->requestFocus();
    } else

    //make tileset button
    if( mouseEvent.getSource() == tilesetBox->getMakeTilesetButton() ){
        mouseEvent.consume();
        makeTilesetBox->setVisible( true );
        makeTilesetBox->requestFocus();
    } else

    //close tileset button
    if( mouseEvent.getSource() == tilesetBox->getCloseTilesetButton() ){
        mouseEvent.consume();
        tilesetDetailsBox->setInfoSource( NULL );
    } else

    //display tileset info
    if( mouseEvent.getSource() == tilesetBox->getTilesetDetailsButton() ){
        mouseEvent.consume();
        tilesetDetailsBox->setInfoSource( tilesetBox->getSelectedTileset() );
        tilesetDetailsBox->setVisible( true );
        tilesetDetailsBox->requestFocus();
    } else




    // from file tool bar
    //new doc
    if( mouseEvent.getSource() == fileToolBar->getNewDocButton() ){
        mouseEvent.consume();
        loadMapBox->setVisible( true );
        loadMapBox->requestFocus();
    } else

    //save document
    if( mouseEvent.getSource() == fileToolBar->getSaveDocButton() ){
        mouseEvent.consume();
        if( world == NULL ){
            alert("Theres no map to save (try the new map button =P)");
            return;
        }
        if( world->size() == 0 ){
            alert("Theres no map to save (try the new map button =P)");
            return;
        }
        std::string a =  world->getFilename() ;
        if( world->getFilename() == "NULL" || world->getFilename().empty() ||
            isSpace(world->getFilename())  || world->getFilename() == "" ){
            filenameBox->setFilename("data/maps/");
            filenameBox->setVisible( true );
            action = SAVEAS;
        } else {
            if( !world->saveMap(world->getFilename()) ){
                alert( ("Cant save file: " + world->getFilename()) );
                return;
            } else {
                alert("Saved");
            }
        }
    } else

    // save as
    if( mouseEvent.getSource() == fileToolBar->getSaveAsButton() ){
        mouseEvent.consume();
        if( world == NULL ){
            alert("Theres no map to save (try the new map button =P)");
            return;
        }
        if( world->size() == 0 ){
            alert("Theres no map to save (try the new map button =P)");
            return;
        }
        action = SAVEAS;
        filenameBox->setFilename("data/maps/");
        filenameBox->setVisible( true );
    } else

    //open new doc (close current 1st)
    if( mouseEvent.getSource() == fileToolBar->getOpenDocButton() ){
        mouseEvent.consume();
        action = OPEN;
        filenameBox->setFilename("data/maps/");
        filenameBox->setVisible( true );
    } else









    //close (no save) and reOpen
    if( mouseEvent.getSource() == fileToolBar->getReopenDocButton() ){
        mouseEvent.consume();
        if( world != NULL ){
            std::string filename = correctFilepath(world->getFilename());
            //delete map
            notify("Releasing old world from memory");
            world->unload();
            //clear the tileset array
            notify("Releasing tilesets");
            tilesetBox->freeTilesets();
            //reopen
            notify("Reopening file");
            if( !world->loadMap(filename) ){
                alert( ("Cant open file: " + filename) );
            }
            unNotify();
            mapDetailsBox->setInfoSource( world );
            //  free old worldImage
            if( worldImage != NULL ){
                SDL_FreeSurface(worldImage);
                worldImage = NULL;
            };
            mapTouched = 1;
            action = NOTHING;
        }
    } else

    //close
    if( mouseEvent.getSource() == fileToolBar->getCloseDocButton() ){
        mouseEvent.consume();
        //delete map
        notify("Releasing old world from memory");
        if( world != NULL ){
            //delete worlds
            world->unload();
            //clear the tileset array
            notify("Releasing tilesets");
            tilesetBox->freeTilesets();
            unNotify();
            mapDetailsBox->setInfoSource( world );
            //  free old worldImage
            if( worldImage != NULL ){
                SDL_FreeSurface(worldImage);
                worldImage = NULL;
            }
            mapTouched = 1;
            action = NOTHING;
        }
    } else

    //get details
    if( mouseEvent.getSource() == fileToolBar->getDetailsButton() ){
        mouseEvent.consume();
        mapDetailsBox->setVisible( true );
    }
}
コード例 #5
0
ファイル: lexer.cpp プロジェクト: mvila/liu
 Token *Lexer::scanNewline() {
     startToken();
     do consume(); while(isNewline() || isSpace());
     return finishToken(Token::Newline);
 }
コード例 #6
0
ファイル: php.c プロジェクト: blackb1rd/ctags
static int skipWhitespaces (int c)
{
	while (isSpace (c))
		c = getcFromInputFile ();
	return c;
}
コード例 #7
0
ファイル: kilescriptdocument.cpp プロジェクト: KDE/kile
bool KileScriptDocument::isSpace(int line, int column)
{
    return isSpace(KTextEditor::Cursor(line,column));
}
コード例 #8
0
ファイル: Trim.cpp プロジェクト: Patrick-Louden/2.2
 std::string trimCopy(const std::string& input) {
   return trimCopyIf(input, isSpace());
 }
コード例 #9
0
ファイル: conf.c プロジェクト: andrewoesten/neo-myth-menu
int config_loadFromBuffer(const char* buf,const int size)
{
	const char* pa = buf;
	int ind = 0;
	ConfigEntry obj;

	config_init();

	while(*pa)
	{
		{
			//reset pointer
			ind = 0;

			obj.variable[0] = obj.value[0] = '\0';
			obj.variableLength = obj.valueLength = 0;

			//skip head WS
			while(*pa){ if(isSpace(*pa))pa++;else break;}

			//load variable
			while(*pa)
			{
				if(*pa == '=')
					break;
				else if(*pa == ' ')
					break;
				else
				{
					if(ind > CONF_ENTRY_BUFFER_Va_SIZE-1)
						break;

					{
						if(! (*pa) )
							break;

						if(*pa == '\r')
						{
							++pa;

							if(*pa == '\n')
								++pa;

							break;
						}
						else if(*pa == '\n')
						{
							++pa;
							break;
						}
					}

					obj.variable[ind++] = *pa;
				}

				++pa;
			}

			obj.variable[ind] = '\0';
			obj.variableLength = ind;
			ind = 0;

			if( (*pa == '='))
			{
				++pa;

				//skip WS after assignment --if exists
				while(*pa){ if(isSpace(*pa))pa++;else break;}

				//load value
				while(*pa)
				{
					if(*pa == '\r') 
					{
						++pa;

						if(*pa == '\n')
						{
							++pa;
							break;
						}
					}
					else if(*pa == '\n')
					{
						++pa;
						break;
					}
					else
					{
						if(ind > CONF_ENTRY_BUFFER_Pa_SIZE-1)
							break;

						{
							if(! (*pa) )
								break;

							if(*pa == '\r')
							{
								++pa;

								if(*pa == '\n')
									++pa;

								break;
							}
							else if(*pa == '\n')
							{
								++pa;
								break;
							}
						}

						obj.value[ind++] = *pa;
					}

					++pa;
				}

				obj.value[ind] = '\0';
				obj.valueLength = ind;

				if(obj.variableLength)
				{
					//printf("Adding [%s] with data [%s]\n",obj.variable,obj.value);
					internal_config_push(obj.variable,obj.value,obj.variableLength,obj.valueLength);//config_push(obj.variable,obj.value);
				}
			}
		}
	}

	return 0;
}
コード例 #10
0
ファイル: Trim.cpp プロジェクト: Patrick-Louden/2.2
 void trimRight(std::string& str) {
   trimRightIf(str, isSpace());
 }
コード例 #11
0
ファイル: Trim.cpp プロジェクト: Patrick-Louden/2.2
 void trim(std::string& str) {
   trimIf(str, isSpace());
 }
コード例 #12
0
ファイル: Trim.cpp プロジェクト: Patrick-Louden/2.2
 void trimLeft(std::string& str) {
   trimLeftIf(str, isSpace());
 }
コード例 #13
0
ファイル: tokenize.c プロジェクト: Moteesh/reactos
/*
** Return the length of the token that begins at z[0].  Return
** -1 if the token is (or might be) incomplete.  Store the token
** type in *tokenType before returning.
*/
int sqliteGetToken(const WCHAR *z, int *tokenType, int *skip){
  int i;

  *skip = 0;
  switch( *z ){
    case ' ': case '\t': case '\n': case '\f':
      for(i=1; isSpace(z[i]); i++){}
      *tokenType = TK_SPACE;
      return i;
    case '-':
      if( z[1]==0 ) return -1;
      *tokenType = TK_MINUS;
      return 1;
    case '(':
      *tokenType = TK_LP;
      return 1;
    case ')':
      *tokenType = TK_RP;
      return 1;
    case '*':
      *tokenType = TK_STAR;
      return 1;
    case '=':
      *tokenType = TK_EQ;
      return 1;
    case '<':
      if( z[1]=='=' ){
        *tokenType = TK_LE;
        return 2;
      }else if( z[1]=='>' ){
        *tokenType = TK_NE;
        return 2;
      }else{
        *tokenType = TK_LT;
        return 1;
      }
    case '>':
      if( z[1]=='=' ){
        *tokenType = TK_GE;
        return 2;
      }else{
        *tokenType = TK_GT;
        return 1;
      }
    case '!':
      if( z[1]!='=' ){
        *tokenType = TK_ILLEGAL;
        return 2;
      }else{
        *tokenType = TK_NE;
        return 2;
      }
    case '?':
      *tokenType = TK_WILDCARD;
      return 1;
    case ',':
      *tokenType = TK_COMMA;
      return 1;
    case '`': case '\'': {
      int delim = z[0];
      for(i=1; z[i]; i++){
        if( z[i]==delim )
          break;
      }
      if( z[i] ) i++;
      if( delim == '`' )
        *tokenType = TK_ID;
      else
        *tokenType = TK_STRING;
      return i;
    }
    case '.':
      if( !isDigit(z[1]) ){
        *tokenType = TK_DOT;
        return 1;
      }
      /* Fall through */
    case '0': case '1': case '2': case '3': case '4':
    case '5': case '6': case '7': case '8': case '9':
      *tokenType = TK_INTEGER;
      for(i=1; isDigit(z[i]); i++){}
      return i;
    case '[':
      for(i=1; z[i] && z[i-1]!=']'; i++){}
      *tokenType = TK_ID;
      return i;
    default:
      if( !isIdChar[*z] ){
        break;
      }
      for(i=1; isIdChar[z[i]]; i++){}
      *tokenType = sqliteKeywordCode(z, i);
      if( *tokenType == TK_ID && z[i] == '`' ) *skip = 1;
      return i;
  }
  *tokenType = TK_ILLEGAL;
  return 1;
}
コード例 #14
0
void TextDataSourceDialog::loadFile(const QString &fileName)
{
    mFileName = fileName;
    QFile inputFile(mFileName);

    if (!inputFile.open(QIODevice::ReadOnly | QIODevice::Text))
        return;  //TODO: Add normal treatment

    ui->groupBox->setTitle(fileName + " contents");

    QTextStream in(&inputFile);
    int nLines = 0;

    QMap<QChar, int> symbolsCount;
    QStringList fileContents;
    while (!in.atEnd() && nLines < 10) {
        QString line = in.readLine();
        fileContents << line;
        ui->textBrowser->append("<font color=\"blue\"><b><i>" + QString::number(++nLines)
                                + ":</i></b></font>&nbsp;&nbsp;&nbsp;" + line);
        for (auto ch : line) {
            if (symbolsCount.find(ch) == symbolsCount.end()) {
                symbolsCount[ch] = 0;
            }
            symbolsCount[ch]++;
        }
    }
    if (!in.atEnd()) {
        ui->textBrowser->append("<font color=\"blue\"><b><i> ...</i></b></font> " );
    }
    //TODO: the stuff below is to move scrollBar to the top (doesn't seem to work), need to be fixed
    ui->textBrowser->verticalScrollBar()->setValue(ui->textBrowser->verticalScrollBar()->minimum());

    //determine separator
    auto isValidSeparator = [] (QChar ch) {
      return !ch.isLetterOrNumber() && !ch.isSpace() && (ch != sDecimalMark)
              && (ch != '-') && (ch != '+');
    };
    int separatorCounter = 0;
    for (const auto& sym : symbolsCount.keys()) {
        if (isValidSeparator(sym)) {
            if (symbolsCount[sym] > separatorCounter) {
                separatorCounter = symbolsCount[sym];
                mFieldSeparator = sym.toLatin1();
            }
        }
    }
    ui->fieldSeparatorLineEdit->setText(QString(mFieldSeparator));

    auto countPercentageOfLetters = [separator = this->mFieldSeparator] (const QString& str) {
        int totalNumber = 0;
        int numberLetters = 0;
        for (auto i = str.begin(); i != str.end(); ++i) {
            if (*i != separator && ! i->isSpace()) {
                totalNumber++;
                if (i->isLetter()) {
                    numberLetters++;
                }
            }
        }
        return totalNumber ? (double)numberLetters / totalNumber : 0;
    };


    if (fileContents.size() > 0 && countPercentageOfLetters(fileContents[0]) > 0.5) {
        ui->namesLineComboBox->setCurrentValue(0);
        if (fileContents.size() > 1 && countPercentageOfLetters(fileContents[1]) > 0.5) {
            ui->descriptionLineComboBox->setCurrentValue(1);
            if (fileContents.size() > 2 && countPercentageOfLetters(fileContents[2]) > 0.5) {
                ui->unitsLineComboBox->setCurrentValue(2);
            }
        }
    }


}
コード例 #15
0
ファイル: ocaml.c プロジェクト: b4n/fishman-ctags
/* The lexer is in charge of reading the file.
 * Some of sub-lexer (like eatComment) also read file.
 * lexing is finished when the lexer return Tok_EOF */
static ocamlKeyword lex (lexingState * st)
{
	int retType;
	/* handling data input here */
	while (st->cp == NULL || st->cp[0] == '\0')
	{
		st->cp = fileReadLine ();
		if (st->cp == NULL)
			return Tok_EOF;
	}

	if (isAlpha (*st->cp))
	{
		readIdentifier (st);
		retType = lookupKeyword (vStringValue (st->name), Lang_Ocaml);

		if (retType == -1)	/* If it's not a keyword */
		{
			return OcaIDENTIFIER;
		}
		else
		{
			return retType;
		}
	}
	else if (isNum (*st->cp))
		return eatNumber (st);
	else if (isSpace (*st->cp))
	{
		eatWhiteSpace (st);
		return lex (st);
	}
	/* OCaml permit the definition of our own operators
	 * so here we check all the consecuting chars which
	 * are operators to discard them. */
	else if (isOperator[*st->cp])
		return eatOperator (st);
	else
		switch (*st->cp)
		{
		case '(':
			if (st->cp[1] == '*')	/* ergl, a comment */
			{
				eatComment (st);
				return lex (st);
			}
			else
			{
				st->cp++;
				return Tok_PARL;
			}

		case ')':
			st->cp++;
			return Tok_PARR;
		case '[':
			st->cp++;
			return Tok_BRL;
		case ']':
			st->cp++;
			return Tok_BRR;
		case '{':
			st->cp++;
			return Tok_CurlL;
		case '}':
			st->cp++;
			return Tok_CurlR;
		case '\'':
			st->cp++;
			return Tok_Prime;
		case ',':
			st->cp++;
			return Tok_comma;
		case '=':
			st->cp++;
			return Tok_EQ;
		case ';':
			st->cp++;
			return Tok_semi;
		case '"':
			eatString (st);
			return Tok_Val;
		case '_':
			st->cp++;
			return Tok_Val;
		case '#':
			st->cp++;
			return Tok_Sharp;
		case '\\':
			st->cp++;
			return Tok_Backslash;

		default:
			st->cp++;
			break;
		}

	/* default return if nothing is recognized,
	 * shouldn't happen, but at least, it will
	 * be handled without destroying the parsing. */
	return Tok_Val;
}
コード例 #16
0
int DSNLEXER::NextTok() throw( IO_ERROR )
{
    const char*   cur  = next;
    const char*   head = cur;

    prevTok = curTok;

    if( curTok != DSN_EOF )
    {
        if( cur >= limit )
        {
L_read:
            // blank lines are returned as "\n" and will have a len of 1.
            // EOF will have a len of 0 and so is detectable.
            int len = readLine();
            if( len == 0 )
            {
                cur = start;        // after readLine(), since start can change, set cur offset to start
                curTok = DSN_EOF;
                goto exit;
            }

            cur = start;    // after readLine() since start can change.

            // skip leading whitespace
            while( cur<limit && isSpace(*cur) )
                ++cur;

            // If the first non-blank character is #, this line is a comment.
            // Comments cannot follow any other token on the same line.
            if( cur<limit && *cur=='#' )
            {
                if( commentsAreTokens )
                {
                    // save the entire line, including new line as the current token.
                    // the '#' character may not be at offset zero.
                    curText = start;      // entire line is the token
                    cur     = start;      // ensure a good curOffset below
                    curTok  = DSN_COMMENT;
                    head    = limit;        // do a readLine() on next call in here.
                    goto exit;
                }
                else
                    goto L_read;
            }
        }
        else
        {
            // skip leading whitespace
            while( cur<limit && isSpace(*cur) )
                ++cur;
        }

        if( cur >= limit )
            goto L_read;

        // switching the string_quote character
        if( prevTok == DSN_STRING_QUOTE )
        {
            static const wxString errtxt( _("String delimiter must be a single character of ', \", or $"));

            char cc = *cur;
            switch( cc )
            {
            case '\'':
            case '$':
            case '"':
                break;
            default:
                THROW_PARSE_ERROR( errtxt, CurSource(), CurLine(), CurLineNumber(), CurOffset() );
            }

            curText = cc;

            head = cur+1;

            if( head<limit && *head!=')' && *head!='(' && !isSpace(*head) )
            {
                THROW_PARSE_ERROR( errtxt, CurSource(), CurLine(), CurLineNumber(), CurOffset() );
            }

            curTok = DSN_QUOTE_DEF;
            goto exit;
        }

        if( *cur == '(' )
        {
            curText = *cur;
            curTok = DSN_LEFT;
            head = cur+1;
            goto exit;
        }

        if( *cur == ')' )
        {
            curText = *cur;
            curTok = DSN_RIGHT;
            head = cur+1;
            goto exit;
        }

        /*  get the dash out of a <pin_reference> which is embedded for example
            like:  U2-14 or "U2"-"14"
            This is detectable by a non-space immediately preceeding the dash.
        */
        if( *cur == '-' && cur>start && !isSpace( cur[-1] ) )
        {
            curText = '-';
            curTok = DSN_DASH;
            head = cur+1;
            goto exit;
        }

        // handle DSN_NUMBER
        if( strchr( "+-.0123456789", *cur ) )
        {
            head = cur+1;
            while( head<limit && strchr( ".0123456789", *head )  )
                ++head;

            if( (head<limit && isSpace(*head)) || *head==')' || *head=='(' || head==limit )
            {
                curText.clear();
                curText.append( cur, head );
                curTok = DSN_NUMBER;
                goto exit;
            }

            // else it was something like +5V, fall through below
        }

        // a quoted string, will return DSN_STRING
        if( *cur == stringDelimiter )
        {
            // Non-specctraMode, understands and deciphers escaped \, \r, \n, and \".
            // Strips off leading and trailing double quotes
            if( !specctraMode )
            {
                // copy the token, character by character so we can remove doubled up quotes.
                curText.clear();

                ++cur;  // skip over the leading delimiter, which is always " in non-specctraMode

                head = cur;

                while( head<limit )
                {
                    // ESCAPE SEQUENCES:
                    if( *head =='\\' )
                    {
                        char    tbuf[8];
                        char    c;
                        int     i;

                        if( ++head >= limit )
                            break;  // throw exception at L_unterminated

                        switch( *head++ )
                        {
                        case '"':
                        case '\\':  c = head[-1];   break;
                        case 'a':   c = '\x07';     break;
                        case 'b':   c = '\x08';     break;
                        case 'f':   c = '\x0c';     break;
                        case 'n':   c = '\n';       break;
                        case 'r':   c = '\r';       break;
                        case 't':   c = '\x09';     break;
                        case 'v':   c = '\x0b';     break;

                        case 'x':   // 1 or 2 byte hex escape sequence
                            for( i=0; i<2; ++i )
                            {
                                if( !isxdigit( head[i] ) )
                                    break;
                                tbuf[i] = head[i];
                            }
                            tbuf[i] = '\0';
                            if( i > 0 )
                                c = (char) strtoul( tbuf, NULL, 16 );
                            else
                                c = 'x';   // a goofed hex escape sequence, interpret as 'x'
                            head += i;
                            break;

                        default:    // 1-3 byte octal escape sequence
                            --head;
                            for( i=0; i<3; ++i )
                            {
                                if( head[i] < '0' || head[i] > '7' )
                                    break;
                                tbuf[i] = head[i];
                            }
                            tbuf[i] = '\0';
                            if( i > 0 )
                                c = (char) strtoul( tbuf, NULL, 8 );
                            else
                                c = '\\';   // a goofed octal escape sequence, interpret as '\'
                            head += i;
                            break;
                        }

                        curText += c;
                    }

                    else if( *head == '"' )     // end of the non-specctraMode DSN_STRING
                    {
                        curTok = DSN_STRING;
                        ++head;                 // omit this trailing double quote
                        goto exit;
                    }

                    else
                        curText += *head++;

                }   // while

                // L_unterminated:
                wxString errtxt(_("Un-terminated delimited string") );
                THROW_PARSE_ERROR( errtxt, CurSource(), CurLine(), CurLineNumber(), CurOffset() );
            }

            else    // specctraMode DSN_STRING
            {
                ++cur;  // skip over the leading delimiter: ",', or $

                head = cur;

                while( head<limit  &&  !isStringTerminator( *head ) )
                    ++head;

                if( head >= limit )
                {
                    wxString errtxt(_("Un-terminated delimited string") );
                    THROW_PARSE_ERROR( errtxt, CurSource(), CurLine(), CurLineNumber(), CurOffset() );
                }

                curText.clear();
                curText.append( cur, head );

                ++head;     // skip over the trailing delimiter

                curTok  = DSN_STRING;
                goto exit;
            }
        }

        // Maybe it is a token we will find in the token table.
        // If not, then call it a DSN_SYMBOL.
        {
            head = cur+1;
            while( head<limit && !isSpace( *head ) && *head!=')' && *head!='(' )
                ++head;

            curText.clear();
            curText.append( cur, head );

            int found = findToken( curText );

            if( found != -1 )
                curTok = found;

            else if( 0 == curText.compare( "string_quote" ) )
                curTok = DSN_STRING_QUOTE;

            else                    // unrecogized token, call it a symbol
                curTok = DSN_SYMBOL;
        }
    }

exit:   // single point of exit, no returns elsewhere please.

    curOffset = cur - start;

    next = head;

    // printf("tok:\"%s\"\n", curText.c_str() );
    return curTok;
}
コード例 #17
0
ファイル: message_field.cpp プロジェクト: cuiwm/tdesktop
InlineBotQuery ParseInlineBotQuery(not_null<const Ui::InputField*> field) {
	auto result = InlineBotQuery();

	const auto &text = field->getTextWithTags().text;
	const auto textLength = text.size();

	auto inlineUsernameStart = 1;
	auto inlineUsernameLength = 0;
	if (textLength > 2 && text[0] == '@' && text[1].isLetter()) {
		inlineUsernameLength = 1;
		for (auto i = inlineUsernameStart + 1; i != textLength; ++i) {
			const auto ch = text[i];
			if (ch.isLetterOrNumber() || ch.unicode() == '_') {
				++inlineUsernameLength;
				continue;
			} else if (!ch.isSpace()) {
				inlineUsernameLength = 0;
			}
			break;
		}
		auto inlineUsernameEnd = inlineUsernameStart + inlineUsernameLength;
		auto inlineUsernameEqualsText = (inlineUsernameEnd == textLength);
		auto validInlineUsername = false;
		if (inlineUsernameEqualsText) {
			validInlineUsername = text.endsWith(qstr("bot"));
		} else if (inlineUsernameEnd < textLength && inlineUsernameLength) {
			validInlineUsername = text[inlineUsernameEnd].isSpace();
		}
		if (validInlineUsername) {
			auto username = text.midRef(inlineUsernameStart, inlineUsernameLength);
			if (username != result.username) {
				result.username = username.toString();
				if (const auto peer = App::peerByName(result.username)) {
					if (const auto user = peer->asUser()) {
						result.bot = peer->asUser();
					} else {
						result.bot = nullptr;
					}
					result.lookingUpBot = false;
				} else {
					result.bot = nullptr;
					result.lookingUpBot = true;
				}
			}
			if (result.lookingUpBot) {
				result.query = QString();
				return result;
			} else if (result.bot && (!result.bot->botInfo
				|| result.bot->botInfo->inlinePlaceholder.isEmpty())) {
				result.bot = nullptr;
			} else {
				result.query = inlineUsernameEqualsText
					? QString()
					: text.mid(inlineUsernameEnd + 1);
				return result;
			}
		} else {
			inlineUsernameLength = 0;
		}
	}
	if (inlineUsernameLength < 3) {
		result.bot = nullptr;
		result.username = QString();
	}
	result.query = QString();
	return result;
}
コード例 #18
0
ファイル: dots.cpp プロジェクト: kuhumcst/rtfreader
void dots::Put3(STROEM * file,wint_t ch,flags & flgs) // called from PutN, Put2 and GetPut
    {
    /* Put3 generally causes a newline (ch=='\n') to be written.
    Exception: inside htmltags.
    */

    if(flgs.inhtmltag)
        {
        flgs.in_abbreviation = false;
        flgs.person_name = not_a_name;
        flgs.number_final_dot = false;
        // This code writes the complete HTML-tag
        if(Option.suppressHTML)
            {
            ch = ' ';
            }
        else
            {
            if(ch)
                {
                if(isSpace(ch))
                    {
                    last = ' ';
                    //return; // 20100107
                    }
                else 
                    {
                    if(isFlatSpace(last))// last == ' ' ||  last == 0xA0 || last == 0x3000)
                        {
                        pRegularizationFnc(file,' ',flgs);
                        }
                    pRegularizationFnc(file,ch,flgs);
                    }
                }
            }
        }
    else
        {
        if(isFlatSpace(ch))//20100106 // ch == 0xA0) // 20071112
            ch = ' ';
        else if((unsigned int)ch < ' ') // replace tabs by spaces and all other non-white space by an asterisk
            {
            if(ch != '\n')
                ch = '*';
            }

        if(ch == ' ')
            {
            if(last != '\n') // Spaces at the beginning of a line are ignored. Only spaces after words are recorded in 'last'.
                {
                last = ' ';
                }
            return;
            }

        if(ch == '\n')
            {
            if(trailingDotFollowingNumber)
                {
                if(Option.tokenize)
                    pRegularizationFnc(file, Option.tokenSplit,flgs); // insert blank before dot if number followed by dot is at the end of the line
                pRegularizationFnc(file,'.',flgs);
                trailingDotFollowingNumber = false;
                flgs.in_abbreviation = false;
                }
            }
        else if(isFlatSpace(last))// last == ' ' || last == 0xA0)
            {
            wint_t lastToWrite = ' ';
            if(!isLower(ch)) // Might be an indication that a new sentence starts here. 
                // Check preceding token for trailing dot that might be a
                // sentence delimiter after all.
                {
                if(trailingDotFollowingNumber) // ... in 1999. Next month ...   ch=='N', last is ' '
                    //            ^ Not written from here
                    {// Regard dot as sentence delimiter after all
                    if(Option.tokenize)
                        pRegularizationFnc(file, Option.tokenSplit,flgs); // Insert blank before dot if number followed by dot is followed by capitalised word.
                    pRegularizationFnc(file,'.',flgs);
                    trailingDotFollowingNumber = false;
                    lastToWrite = '\n'; // Number seems to be the last word of the previous sentence. Fake history.
                    // ... in 1999.
                    // Next month ...
                    flgs.in_abbreviation = false;
                    }
                else if(flgs.in_abbreviation /*&& flgs.newSegment*/)
                    {
                    switch(flgs.person_name)
                        {
                        case initial: 
                            flgs.person_name = not_a_name;
                            break;
                        case not_a_name:
                        default: // Skema 1. Affald fra husholdninger --> Skema 1. | Affald fra husholdninger       20040420
                            if(!flgs.expectCapitalizedWord)
                                lastToWrite = '\n'; // Abbreviation seems to be the last word of the previous sentence
                            break;
                        }
                    }
                }
            else if(trailingDotFollowingNumber)
                { // Now we suppose that the dot trailing the number is part of that number.
                pRegularizationFnc(file,'.',flgs);
                trailingDotFollowingNumber = false;
                }
            if((lastToWrite != ' ' && lastToWrite != 0xA0) || flgs.writtentoline)
                pRegularizationFnc(file,lastToWrite,flgs);
            flgs.writtentoline = (lastToWrite == ' ' || lastToWrite == 0xA0);
            if(Option.emptyline && flgs.in_abbreviation && !flgs.writtentoline)
                { // Make sure to send \n next time
                ensureEmptyLine = true;
                }
            flgs.in_abbreviation = false;
            }

        if(flgs.number_final_dot == 1 && !(flgs.person_name == initial)) // This can only be the case if ch == separating character
            {
            trailingDotFollowingNumber = true;
            }
        else
            {
            if(ch != '\n' && ensureEmptyLine)
                {
                pRegularizationFnc(file,'\n',flgs);
                }
            ensureEmptyLine = false;
            pRegularizationFnc(file,ch,flgs);
            flgs.writtentoline = ch != '\n';
            if(Option.emptyline && flgs.in_abbreviation && !flgs.writtentoline)
                { // Make sure to send \n next time
                ensureEmptyLine = true;
                }
            }
        }

    last = ch;
    //    flgs.newSegment = false;
    }
コード例 #19
0
ファイル: Lexer.cpp プロジェクト: kevoree/kevoree-cpp
Token Lexer::nextToken() {
	if (isDone()) {
		return *finish_token;
	}
	int  tokenType = END_OF_FILE;
	char c = nextChar();
	string currentValue="";
	string jsonValue ="";
	while (isDone() != true && isSpace(c)==true) {
		c = nextChar();
	}
	if ('"' == c) {
		tokenType = VALUE;
		if (isDone() != true) {
			c = nextChar();
			while (index < length && c != '"') {
				currentValue += c;
				if (c == '\\' && index < length) {
					c = nextChar();
					currentValue += c;
				}
				c = nextChar();
			}
			jsonValue = currentValue;
		} else {
			throw  KevoreeException("Lexer Unterminated string nextToken");
		}
	}else if ('{' == c) {
		tokenType = LEFT_BRACE;
	} else if ('}' == c) {
		tokenType = RIGHT_BRACE;
	} else if ('[' == c) {
		tokenType = LEFT_BRACKET;
	} else if (']' == c) {
		tokenType = RIGHT_BRACKET;
	} else if (':' == c) {
		tokenType = COLON;
	} else if (',' == c) {
		tokenType = COMMA;
	} else if (! isDone()) {

		while (isValueLetter(tolower(c)) == true)
		{
			currentValue += c;
			if (isValueLetter(tolower(peekChar())) != true) {
				break;
			} else {
				c = nextChar();
			}
		}
		string v = currentValue;
		std::transform(v.begin(), v.end(), v.begin(), ::tolower);
		if (v.compare("true") == 0) {
			jsonValue = "true";
		} else if (v.compare("false") == 0) {
			jsonValue = "false";
		} else {
			jsonValue = v;
		}
		tokenType = VALUE;
	} else {
		tokenType = END_OF_FILE;
	}
	return Token(tokenType, jsonValue);
}
コード例 #20
0
void ConfigManager::loadFromStream(SeekableReadStream &stream) {
	String domainName;
	String comment;
	Domain domain;
	int lineno = 0;

	_appDomain.clear();
	_gameDomains.clear();
	_miscDomains.clear();
	_transientDomain.clear();
	_domainSaveOrder.clear();

#ifdef ENABLE_KEYMAPPER
	_keymapperDomain.clear();
#endif

	// TODO: Detect if a domain occurs multiple times (or likewise, if
	// a key occurs multiple times inside one domain).

	while (!stream.eos() && !stream.err()) {
		lineno++;

		// Read a line
		String line = stream.readLine();

		if (line.size() == 0) {
			// Do nothing
		} else if (line[0] == '#') {
			// Accumulate comments here. Once we encounter either the start
			// of a new domain, or a key-value-pair, we associate the value
			// of the 'comment' variable with that entity.
			comment += line;
			comment += "\n";
		} else if (line[0] == '[') {
			// It's a new domain which begins here.
			// Determine where the previously accumulated domain goes, if we accumulated anything.
			addDomain(domainName, domain);
			domain.clear();
			const char *p = line.c_str() + 1;
			// Get the domain name, and check whether it's valid (that
			// is, verify that it only consists of alphanumerics,
			// dashes and underscores).
			while (*p && (isAlnum(*p) || *p == '-' || *p == '_'))
				p++;

			if (*p == '\0')
				error("Config file buggy: missing ] in line %d", lineno);
			else if (*p != ']')
				error("Config file buggy: Invalid character '%c' occurred in section name in line %d", *p, lineno);

			domainName = String(line.c_str() + 1, p);

			domain.setDomainComment(comment);
			comment.clear();

		} else {
			// This line should be a line with a 'key=value' pair, or an empty one.

			// Skip leading whitespaces
			const char *t = line.c_str();
			while (isSpace(*t))
				t++;

			// Skip empty lines / lines with only whitespace
			if (*t == 0)
				continue;

			// If no domain has been set, this config file is invalid!
			if (domainName.empty()) {
				error("Config file buggy: Key/value pair found outside a domain in line %d", lineno);
			}

			// Split string at '=' into 'key' and 'value'. First, find the "=" delimeter.
			const char *p = strchr(t, '=');
			if (!p)
				error("Config file buggy: Junk found in line line %d: '%s'", lineno, t);

			// Extract the key/value pair
			String key(t, p);
			String value(p + 1);

			// Trim of spaces
			key.trim();
			value.trim();

			// Finally, store the key/value pair in the active domain
			domain[key] = value;

			// Store comment
			domain.setKVComment(key, comment);
			comment.clear();
		}
	}

	addDomain(domainName, domain); // Add the last domain found
}
コード例 #21
0
ファイル: objc.c プロジェクト: simlrh/ctags
/* The lexer is in charge of reading the file.
 * Some of sub-lexer (like eatComment) also read file.
 * lexing is finished when the lexer return Tok_EOF */
static objcKeyword lex (lexingState * st)
{
	int retType;

	/* handling data input here */
	while (st->cp == NULL || st->cp[0] == '\0')
	{
		st->cp = fileReadLine ();
		if (st->cp == NULL)
			return Tok_EOF;

		return Tok_EOL;
	}

	if (isAlpha (*st->cp))
	{
		readIdentifier (st);
		retType = lookupKeyword (vStringValue (st->name), Lang_ObjectiveC);

		if (retType == -1)	/* If it's not a keyword */
		{
			return ObjcIDENTIFIER;
		}
		else
		{
			return retType;
		}
	}
	else if (*st->cp == '@')
	{
		readIdentifierObjcDirective (st);
		retType = lookupKeyword (vStringValue (st->name), Lang_ObjectiveC);

		if (retType == -1)	/* If it's not a keyword */
		{
			return Tok_any;
		}
		else
		{
			return retType;
		}
	}
	else if (isSpace (*st->cp))
	{
		eatWhiteSpace (st);
		return lex (st);
	}
	else
		switch (*st->cp)
		{
		case '(':
			st->cp++;
			return Tok_PARL;

		case '\\':
			st->cp++;
			return Tok_Backslash;

		case '#':
			st->cp++;
			return Tok_Sharp;

		case '/':
			if (st->cp[1] == '*')	/* ergl, a comment */
			{
				eatComment (st);
				return lex (st);
			}
			else if (st->cp[1] == '/')
			{
				st->cp = NULL;
				return lex (st);
			}
			else
			{
				st->cp++;
				return Tok_any;
			}
			break;

		case ')':
			st->cp++;
			return Tok_PARR;
		case '{':
			st->cp++;
			return Tok_CurlL;
		case '}':
			st->cp++;
			return Tok_CurlR;
		case '[':
			st->cp++;
			return Tok_SQUAREL;
		case ']':
			st->cp++;
			return Tok_SQUARER;
		case ',':
			st->cp++;
			return Tok_COMA;
		case ';':
			st->cp++;
			return Tok_semi;
		case ':':
			st->cp++;
			return Tok_dpoint;
		case '"':
			eatString (st);
			return Tok_any;
		case '+':
			st->cp++;
			return Tok_PLUS;
		case '-':
			st->cp++;
			return Tok_MINUS;

		default:
			st->cp++;
			break;
		}

	/* default return if nothing is recognized,
	 * shouldn't happen, but at least, it will
	 * be handled without destroying the parsing. */
	return Tok_any;
}
コード例 #22
0
ファイル: tokenizer.cpp プロジェクト: garvitdelhi/kturtle
Token* Tokenizer::getToken()
{
	int startRow = row;
	int startCol = col;

	QChar c = getChar();  // get and store the next character from the string

	// catch the end of the input string
	if (atEnd)
		return new Token(Token::EndOfInput, "END", row, col, row, col);

	int cType = translator->look2type(c);  // since we need to know it often we store it

	// catch spaces
	if (isSpace(c)) {
		QString look;
		do {
			look += (isTab(c) ? "  " : " ");
			c = getChar();
		} while (isSpace(c) && !atEnd);
		ungetChar();
		return new Token(Token::WhiteSpace, look, startRow, startCol, row, col);
	}

	// catch EndOfLine's
	if (isBreak(c)) {
		return new Token(Token::EndOfLine, "\\n", startRow, startCol, startRow+1, 1);
	}

	// catch comments
	if (cType == Token::Comment) {
		QString look;
		do {
			look += c;
			c = getChar();
		} while (!isBreak(c) && !atEnd);
		ungetChar();
		return new Token(Token::Comment, look, startRow, startCol, row, col);
	}

	// catch strings
	if (cType == Token::StringDelimiter) {
		QString look = QString(c);
		do {
			c = getChar();
			look += c;
		} while (!(translator->look2type(c) == Token::StringDelimiter && look.right(2) != "\\\"") &&
		         !isBreak(c) && !atEnd);
		return new Token(Token::String, look, startRow, startCol, row, col);
	}

	// catch variables
	if (cType == Token::VariablePrefix) {
		QString look;
		do {
			look += c;
			c = getChar();
		} while (isWordChar(c) || c.category() == QChar::Number_DecimalDigit || c == '_');
		ungetChar();
		return new Token(Token::Variable, look, startRow, startCol, row, col);
	}

	// catch words (known commands or function calls)
	if (isWordChar(c)) {  // first char has to be a letter
		QString look;
		do {
			look += c;
			c = getChar();
		} while (isWordChar(c) || c.isDigit() || c == '_');  // next chars
		ungetChar();
		int type = translator->look2type(look);
		if (type == Token::Unknown)
			type = Token::FunctionCall;
		return new Token(type, look, startRow, startCol, row, col);
	}

	// catch numbers
	if (c.isDigit() || cType == Token::DecimalSeparator) {
		bool hasDot = false;

		int localType = cType;
		QString look;
		do {
			if (localType == Token::DecimalSeparator) hasDot = true;
			look += c;
			c = getChar();
			localType = translator->look2type(c);
		} while (c.isDigit() || (localType == Token::DecimalSeparator && !hasDot));
		ungetChar();
		
		// if all we got is a dot then this is not a number, so return an Error token here
		if (translator->look2type(look) == Token::DecimalSeparator)
			return new Token(Token::Error, look, startRow, startCol, row, col);
		
		return new Token(Token::Number, look, startRow, startCol, row, col);
	}

	// catch previously uncatched 'double charactered tokens' (tokens that ar not in letters, like: == != >= <=)
	{
		QString look = QString(c).append(getChar());
		int type = translator->look2type(look);
		if (type != Token::Unknown)
			return new Token(type, look, startRow, startCol, row, col);
		ungetChar();
	}

	// catch known tokens of a single character (as last...)
	if (cType != Token::Unknown)
		return new Token(cType, static_cast<QString>(c), startRow, startCol, row, col);

	// this does not neglect calls to functions with a name of length one (checked it)
	return new Token(Token::Error, static_cast<QString>(c), startRow, startCol, row, col);
}
コード例 #23
0
ファイル: OpenDDLParser.cpp プロジェクト: 1vanK/Urho3D
char *OpenDDLParser::parseDataList( char *in, char *end, Value::ValueType type, Value **data, 
                                    size_t &numValues, Reference **refs, size_t &numRefs ) {
    *data = ddl_nullptr;
    numValues = numRefs = 0;
    if( ddl_nullptr == in || in == end ) {
        return in;
    }

    in = lookForNextToken( in, end );
    if( *in == '{' ) {
        ++in;
        Value *current( ddl_nullptr ), *prev( ddl_nullptr );
        while( '}' != *in ) {
            current = ddl_nullptr;
            in = lookForNextToken( in, end );
            if ( Value::ddl_ref == type ) {
                std::vector<Name*> names;
                in = parseReference( in, end, names );
                if ( !names.empty() ) {
                    Reference *ref = new Reference( names.size(), &names[ 0 ] );
                    *refs = ref;
                    numRefs = names.size();
                }
            } else  if ( Value::ddl_none == type ) {
                if (isInteger( in, end )) {
                    in = parseIntegerLiteral( in, end, &current );
                } else if (isFloat( in, end )) {
                    in = parseFloatingLiteral( in, end, &current );
                } else if (isStringLiteral( *in )) {
                    in = parseStringLiteral( in, end, &current );
                } else if (isHexLiteral( in, end )) {
                    in = parseHexaLiteral( in, end, &current );
                }
            } else {
                switch(type){
                    case Value::ddl_int8:
                    case Value::ddl_int16:
                    case Value::ddl_int32:
                    case Value::ddl_int64:
                    case Value::ddl_unsigned_int8:
                    case Value::ddl_unsigned_int16:
                    case Value::ddl_unsigned_int32:
                    case Value::ddl_unsigned_int64:
                        in = parseIntegerLiteral( in, end, &current, type);
                        break;
                    case Value::ddl_half:
                    case Value::ddl_float:
                    case Value::ddl_double:
                        in = parseFloatingLiteral( in, end, &current, type);
                        break;
                    case Value::ddl_string:
                        in = parseStringLiteral( in, end, &current );
                        break;
                    default:
                        break;
                }
            }

            if( ddl_nullptr != current ) {
                if( ddl_nullptr == *data ) {
                    *data = current;
                    prev = current;
                } else {
                    prev->setNext( current );
                    prev = current;
                }
                ++numValues;
            }

            in = getNextSeparator( in, end );
            if( ',' != *in && Grammar::CloseBracketToken[ 0 ] != *in && !isSpace( *in ) ) {
                break;
            }
        }
        ++in;
    }

    return in;
}
コード例 #24
0
ファイル: properties.cpp プロジェクト: gema-arta/zim-vendor
  bool PropertiesParser::parse(char ch)
  {
    bool ret = false;
    switch (state)
    {
      case state_0:
        if (ch == '#')
          state = state_comment;
        else if (isKeyChar(ch))
        {
          key = ch;
          keypart = ch;
          state = state_key;
        }
        else if (!isSpace(ch) && ch != '\n' && ch != '\r')
          throw std::runtime_error("format error in properties");
        break;

      case state_key:
        if (ch == '.')
        {
          event.onKeyPart(keypart);
          keypart.clear();
          key += ch;
        }
        else if (isKeyChar(ch))
        {
          keypart += ch;
          key += ch;
        }
        else if (isSpace(ch))
        {
          ret = event.onKeyPart(keypart)
             || event.onKey(key);
          state = state_key_sp;
        }
        else if (ch == '=')
        {
          ret = event.onKeyPart(keypart)
             || event.onKey(key);
          state = state_value;
        }
        else
          throw std::runtime_error("parse error in properties while reading key " + key);
        break;

      case state_key_sp:
        if (ch == '=')
        {
          state = state_value;
        }
        else if (!isSpace(ch))
          throw std::runtime_error("parse error while reading key " + key);
        break;

      case state_value:
        if (ch == '\n')
        {
          ret = event.onValue(value);
          value.clear();
          state = state_0;
        }
        else if (ch == '\\')
          state = state_value_esc;
        else if (!value.empty() || !isSpace(ch))
          value += ch;
        break;

      case state_value_esc:
        value += ch;
        state = state_value;
        break;

      case state_comment:
        if (ch == '\n')
          state = state_0;
        break;
    }
    return ret;
  }
コード例 #25
0
std::string ObjLoader::LoadObj(shapes_t& shapes, const char* filename, const char* mtl_basepath)
{
    shapes.reset();
    
    std::stringstream err;
    std::istringstream ifs(FileUtils::getInstance()->getStringFromFile(filename));
    std::map<vertex_index, ssize_t> vertexCache;
    //std::ifstream ifs(filename);
    
    if (!ifs)
    {
        err << "Cannot open file [" << filename << "]" << std::endl;
        return err.str();
    }
    
    std::vector<float> v;
    std::vector<float> vn;
    std::vector<float> vt;
    std::vector<std::vector<vertex_index> > faceGroup;
    std::string name;
    
    // material
    std::map<std::string, material_t> material_map;
    material_t material;
    
    int maxchars = 8192;  // Alloc enough size.
    std::vector<char> buf(maxchars);  // Alloc enough size.
    while (ifs.peek() != -1)
    {
        ifs.getline(&buf[0], maxchars);
        
        std::string linebuf(&buf[0]);
        
        // Trim newline '\r\n' or '\r'
        if (linebuf.size() > 0)
        {
            if (linebuf[linebuf.size()-1] == '\n') linebuf.erase(linebuf.size()-1);
        }
        if (linebuf.size() > 0)
        {
            if (linebuf[linebuf.size()-1] == '\n') linebuf.erase(linebuf.size()-1);
        }
        
        // Skip if empty line.
        if (linebuf.empty())
        {
            continue;
        }
        
        // Skip leading space.
        const char* token = linebuf.c_str();
        token += strspn(token, " \t");
        
        assert(token);
        if (token[0] == '\0') continue; // empty line
        
        if (token[0] == '#') continue;  // comment line
        
        // vertex
        if (token[0] == 'v' && isSpace((token[1])))
        {
            token += 2;
            float x, y, z;
            parseFloat3(x, y, z, token);
            v.push_back(x);
            v.push_back(y);
            v.push_back(z);
            continue;
        }
        
        // normal
        if (token[0] == 'v' && token[1] == 'n' && isSpace((token[2])))
        {
            token += 3;
            float x, y, z;
            parseFloat3(x, y, z, token);
            vn.push_back(x);
            vn.push_back(y);
            vn.push_back(z);
            continue;
        }
        
        // texcoord
        if (token[0] == 'v' && token[1] == 't' && isSpace((token[2])))
        {
            token += 3;
            float x, y;
            parseFloat2(x, y, token);
            vt.push_back(x);
            vt.push_back(y);
            continue;
        }
        
        // face
        if (token[0] == 'f' && isSpace((token[1])))
        {
            token += 2;
            token += strspn(token, " \t");
            
            std::vector<vertex_index> face;
            while (!isNewLine(token[0])) {
                // fix warning, cast to int, i think int is enough
                vertex_index vi = parseTriple(token, (int)v.size() / 3, (int)vn.size() / 3, (int)vt.size() / 2);
                face.push_back(vi);
                auto n = strspn(token, " \t\r");
                token += n;
            }
            
            faceGroup.push_back(face);
            
            continue;
        }
        
        // use mtl
        if ((0 == strncmp(token, "usemtl", 6)) && isSpace((token[6])))
        {
            
            char namebuf[4096];
            token += 7;
            sscanf(token, "%s", namebuf);
            
            if (material_map.find(namebuf) != material_map.end())
            {
                material = material_map[namebuf];
            }
            else
            {
                // { error!! material not found }
                InitMaterial(material);
            }
            continue;
            
        }
        
        // load mtl
        if ((0 == strncmp(token, "mtllib", 6)) && isSpace((token[6])))
        {
            char namebuf[4096];
            token += 7;
            sscanf(token, "%s", namebuf);
            
            std::string err_mtl = LoadMtl(material_map, namebuf, mtl_basepath);
            if (!err_mtl.empty())
            {
                faceGroup.clear();  // for safety
                //return err_mtl;
            }
            continue;
        }
        
        // group name
        if (token[0] == 'g' && isSpace((token[1])))
        {
            // flush previous face group.
            shape_t shape;
            exportFaceGroupToShape(vertexCache, shapes, v, vn, vt, faceGroup, material, name);
            
            faceGroup.clear();
            
            std::vector<std::string> names;
            while (!isNewLine(token[0]))
            {
                std::string str = parseString(token);
                names.push_back(str);
                token += strspn(token, " \t\r"); // skip tag
            }
            
            assert(names.size() > 0);
            
            // names[0] must be 'g', so skipt 0th element.
            if (names.size() > 1)
            {
                name = names[1];
            }
            else
            {
                name = "";
            }
            
            continue;
        }
        
        // object name
        if (token[0] == 'o' && isSpace((token[1])))
        {
            // flush previous face group.
            shape_t shape;
            exportFaceGroupToShape(vertexCache, shapes, v, vn, vt, faceGroup, material, name);
            
            faceGroup.clear();
            
            // @todo { multiple object name? }
            char namebuf[4096];
            token += 2;
            sscanf(token, "%s", namebuf);
            name = std::string(namebuf);
            
            continue;
        }
        
        // Ignore unknown command.
    }
    
    shape_t shape;
    exportFaceGroupToShape(vertexCache, shapes, v, vn, vt, faceGroup, material, name);
    faceGroup.clear();  // for safety
    
    return err.str();
}
bool MaxEnt::parseLine( FILE *fin, FILE *fout, bool withClassLabel )
{
	vector<string> feature;
	int c;
	int cnt = 0;
	enum State{ SPACE, WORD, NONE };
	int state = NONE;
	VarStr word;
	bool first = true;
	while( (c=fgetc(fin)) != EOF )
	{
		if ( c == '\n' )
			break;
		cnt ++;
		if ( state == NONE )
		{
			if ( isSpace(c) )
				state = SPACE;
			else
			{
				word.add(c);
				state = WORD;
			}
			continue;
		}
		if ( isSpace(c) )
		{
			if ( state == WORD )
			{
				if ( !first )
					feature.push_back(word.c_str());
				else
				{
					if ( !withClassLabel )
						feature.push_back(word.c_str());
					first = false;
				}
				state = SPACE;
				word.clear();
			}
		}
		else
		{
			if ( state == SPACE )
			{
				word.add(c);
				state = WORD;
			}
			else
				word.add(c);
		}
	}
	if ( c == EOF && cnt == 0 )
		return false;
	if ( word.length() > 0 )
	{
		if ( !first || (first&&!withClassLabel) )
			feature.push_back(word.c_str());
	}

	if ( feature.size() > 0 )
	{
		vector<pair<string,double> > labelProbs;
		int pcls = predict(feature,labelProbs);
		fprintf(fout,"%s",labelProbs[pcls].first.c_str());
		for( int i = 0; i < labelProbs.size(); i ++ )
			fprintf(fout, "\t%s\t%lf", labelProbs[i].first.c_str(), labelProbs[i].second);
		fprintf(fout,"\n");
	}
	return true;
}
コード例 #27
0
std::string LoadMtl (
  std::map<std::string, int>& material_map,
  std::vector<material_t>& materials,
  std::istream& inStream)
{
  material_map.clear();
  std::stringstream err;

  material_t material;

  int maxchars = 8192;  // Alloc enough size.
  std::vector<char> buf(maxchars);  // Alloc enough size.
  while (inStream.peek() != -1) {
    inStream.getline(&buf[0], maxchars);

    std::string linebuf(&buf[0]);

    // Trim newline '\r\n' or '\n'
    if (linebuf.size() > 0) {
      if (linebuf[linebuf.size()-1] == '\n') linebuf.erase(linebuf.size()-1);
    }
    if (linebuf.size() > 0) {
      if (linebuf[linebuf.size()-1] == '\r') linebuf.erase(linebuf.size()-1);
    }

    // Skip if empty line.
    if (linebuf.empty()) {
      continue;
    }

    // Skip leading space.
    const char* token = linebuf.c_str();
    token += strspn(token, " \t");

    assert(token);
    if (token[0] == '\0') continue; // empty line

    if (token[0] == '#') continue;  // comment line

    // new mtl
    if ((0 == strncmp(token, "newmtl", 6)) && isSpace((token[6]))) {
      // flush previous material.
      if (!material.name.empty())
      {
          material_map.insert(std::pair<std::string, int>(material.name, materials.size()));
          materials.push_back(material);
      }

      // initial temporary material
      InitMaterial(material);

      // set new mtl name
      char namebuf[4096];
      token += 7;
      sscanf(token, "%s", namebuf);
      material.name = namebuf;
      continue;
    }

    // ambient
    if (token[0] == 'K' && token[1] == 'a' && isSpace((token[2]))) {
      token += 2;
      float r, g, b;
      parseFloat3(r, g, b, token);
      material.ambient[0] = r;
      material.ambient[1] = g;
      material.ambient[2] = b;
      continue;
    }

    // diffuse
    if (token[0] == 'K' && token[1] == 'd' && isSpace((token[2]))) {
      token += 2;
      float r, g, b;
      parseFloat3(r, g, b, token);
      material.diffuse[0] = r;
      material.diffuse[1] = g;
      material.diffuse[2] = b;
      continue;
    }

    // specular
    if (token[0] == 'K' && token[1] == 's' && isSpace((token[2]))) {
      token += 2;
      float r, g, b;
      parseFloat3(r, g, b, token);
      material.specular[0] = r;
      material.specular[1] = g;
      material.specular[2] = b;
      continue;
    }

    // transmittance
    if (token[0] == 'K' && token[1] == 't' && isSpace((token[2]))) {
      token += 2;
      float r, g, b;
      parseFloat3(r, g, b, token);
      material.transmittance[0] = r;
      material.transmittance[1] = g;
      material.transmittance[2] = b;
      continue;
    }

    // ior(index of refraction)
    if (token[0] == 'N' && token[1] == 'i' && isSpace((token[2]))) {
      token += 2;
      material.ior = parseFloat(token);
      continue;
    }

    // emission
    if(token[0] == 'K' && token[1] == 'e' && isSpace(token[2])) {
      token += 2;
      float r, g, b;
      parseFloat3(r, g, b, token);
      material.emission[0] = r;
      material.emission[1] = g;
      material.emission[2] = b;
      continue;
    }

    // shininess
    if(token[0] == 'N' && token[1] == 's' && isSpace(token[2])) {
      token += 2;
      material.shininess = parseFloat(token);
      continue;
    }

    // illum model
    if (0 == strncmp(token, "illum", 5) && isSpace(token[5])) {
      token += 6;
      material.illum = parseInt(token);
      continue;
    }

    // dissolve
    if ((token[0] == 'd' && isSpace(token[1]))) {
      token += 1;
      material.dissolve = parseFloat(token);
      continue;
    }
    if (token[0] == 'T' && token[1] == 'r' && isSpace(token[2])) {
      token += 2;
      material.dissolve = parseFloat(token);
      continue;
    }

    // ambient texture
    if ((0 == strncmp(token, "map_Ka", 6)) && isSpace(token[6])) {
      token += 7;
      material.ambient_texname = token;
      continue;
    }

    // diffuse texture
    if ((0 == strncmp(token, "map_Kd", 6)) && isSpace(token[6])) {
      token += 7;
      material.diffuse_texname = token;
      continue;
    }

    // specular texture
    if ((0 == strncmp(token, "map_Ks", 6)) && isSpace(token[6])) {
      token += 7;
      material.specular_texname = token;
      continue;
    }

    // normal texture
    if ((0 == strncmp(token, "map_Ns", 6)) && isSpace(token[6])) {
      token += 7;
      material.normal_texname = token;
      continue;
    }

    // unknown parameter
    const char* _space = strchr(token, ' ');
    if(!_space) {
      _space = strchr(token, '\t');
    }
    if(_space) {
      int len = _space - token;
      std::string key(token, len);
      std::string value = _space + 1;
      material.unknown_parameter.insert(std::pair<std::string, std::string>(key, value));
    }
  }
  // flush last material.
  material_map.insert(std::pair<std::string, int>(material.name, materials.size()));
  materials.push_back(material);

  return err.str();
}
bool MEModel::loadTrainingExamples( const char *trainFile )
{
	FILE *fin = fopen(trainFile,"r");
	if ( fin == NULL )
	{
		fprintf( stderr, "open train file %s failed!\n", trainFile );
		return false;
	}
	int lineNum = 0;
	int fieldNum = 0;
	int ch;
	bool endOfFile = false;
	int curFieldCnt = 0;
	while( !endOfFile )
	{
		ch = fgetc(fin);
		curFieldCnt = 0;
		while( true )
		{
			while( !isLineEnd(ch) && isSpace(ch) )
				ch = fgetc(fin);
			if( isLineEnd(ch) )
				break;
			while( !isLineEnd(ch) && !isSpace(ch) )
				ch = fgetc(fin);
			curFieldCnt ++;
		}
		if ( curFieldCnt > 0 )
		{
			lineNum ++;
			fieldNum += curFieldCnt;
		}
		if( ch == EOF )
			endOfFile = true;
	}
	rewind(fin);
	M = lineNum;
	VarStr word;
	endOfFile = false;
	xspace = new SpElem[fieldNum];
	x = new SpElem*[M];
	y = new int[M];
	lineNum = 0;
	int xIndex = 0;
	while( !endOfFile )
	{
		ch = fgetc(fin);
		curFieldCnt = 0;
		int curY = 0, curX = 0;
		char *ystr = NULL;
		while( true )
		{
			while( !isLineEnd(ch) && isSpace(ch) )
				ch = fgetc(fin);
			if( isLineEnd(ch) )
				break;
			word.clear();
			while( !isLineEnd(ch) && !isSpace(ch) )
			{
				word.add(ch);
				ch = fgetc(fin);
			}
			curFieldCnt ++;
			if( curFieldCnt == 1 )
			{
				ystr = new char[word.length()+1];
				strcpy(ystr,word.c_str());
			}
			else
			{
				curX = feaIdMap.add(word.c_str());
				xspace[xIndex].index = curX;
				xspace[xIndex].value = 1;
				if ( curFieldCnt == 2 )
					x[lineNum] = &xspace[xIndex];
				xIndex ++;
			}
		}
		if ( curFieldCnt > 1 )
		{
			xspace[xIndex].index = -1; xspace[xIndex].value = 0;
			xIndex ++;
			curY = classNameIdMap.add(ystr);
			y[lineNum] = curY;
			lineNum ++;
		}
		if( ystr != NULL )
			delete []ystr;
		if ( ch == EOF )
			endOfFile = true;
	}
	
	fclose(fin);
	M = lineNum;
	classNumber = (int)classNameIdMap.size();

	needDestroyTrain = true;
	return true;
}
コード例 #29
0
		void XSingleSource::skipSpace()
		{
			for (; isSpace() ; ++m_iterator);
		}
コード例 #30
0
ファイル: stringutils.hpp プロジェクト: nevermatch/davix
inline std::string &rtrim(std::string &s) {
    return rtrim(s, isSpace());
}