/** * ParamList = "(" [ ( Param [{ "," Param }] [ "," "..." ] ) | "..." | "void" ] ")" */ void parserParamList (parserCtx* ctx, ast* Node, bool inDecl) { debugEnter("ParamList"); tokenMatchPunct(ctx, punctLParen); if (!tokenIsPunct(ctx, punctRParen)) { tokenLocation voidloc = ctx->location; bool end = false; /*Either an empty prototype declaration, or the beginning of a void* parameter*/ if (tokenTryMatchKeyword(ctx, keywordVoid)) { if (!tokenIsPunct(ctx, punctRParen)) { ast* basic = astCreateLiteralIdent(voidloc, strdup("void")); ast* expr = parserDeclExpr(ctx, inDecl, symParam); ast* param = astCreateParam(voidloc, basic, expr); param->symbol = basic->symbol = symFind(ctx->scope, "void"); astAddChild(Node, param); if (!tokenTryMatchPunct(ctx, punctComma)) end = true; } else end = true; } if (!end) do { if (tokenIsPunct(ctx, punctEllipsis)) { astAddChild(Node, astCreate(astEllipsis, ctx->location)); tokenMatch(ctx); break; } else astAddChild(Node, parserParam(ctx, inDecl)); } while (tokenTryMatchPunct(ctx, punctComma)); } tokenMatchPunct(ctx, punctRParen); debugLeave(); }
void TriMeshLoader::loadOBJ(const char* objfile,TriMesh* pmesh){ fstream ifs; char line[LINE_SIZE]; //cout << "READIING" << endl; char *tok; ifs.open(objfile); //cout << "REED" << endl; while(!ifs.eof()){ //cout << "HEY LISTEN" << endl; ifs.getline(line,LINE_SIZE); //cout << "LISTEN HEY" << endl; //cout << line << endl; tok=strtok(line,TOK_SEPS); //cout << "BOOP" << endl; TokenPair *ptok=tokenMatch(tok); //cout << "POOB" << endl; if(ptok){ switch(ptok->id){ case T_VERT: processVertex(tok,pmesh); break; case T_FACE: processFace(tok,pmesh); break; default: /*processSkip(tok);*/ break; } } } //cout << "FINISHED READING" << endl; ifs.close(); }
void run_lexer(ifstream & file) { char ch; while (file.get(ch)) { Token * token = tokenMatch(ch, file); cout << token->toString() + " "; delete token; } }
/*************************************************************** * Function: Parser::parseRawData() * Purpose : Parse an XML CDATA region * Initial : Maxime Chevalier-Boisvert on October 20, 2008 **************************************************************** Revisions and bug fixes: */ RawData* Parser::parseRawData(const std::string& xmlString, size_t& charIndex, const PosVector& positions) { // If the CDATA region does not begin appropriately if (!tokenMatch(xmlString, charIndex, "<![CDATA[")) { // Throw an exception throw ParseError("Invalid CDATA region opening", positions[charIndex]); } // Move past the CDATA opening charIndex += 9; // Declare a string to store the region contents std::string contents; // For each character for (;; ++charIndex) { // If we are past the length of the input stream if (charIndex > xmlString.length()) { // Throw an exception throw ParseError("Unexpected end of stream inside CDATA region", positions[charIndex]); } // Update the current character char thisChar = xmlString[charIndex]; // If this is the closing of the CDATA region if (tokenMatch(xmlString, charIndex, "]]>")) { // Move past the CDATA closing charIndex += 2; // Break out of this loop break; } // Add the character to the region contents contents.push_back(thisChar); } // Create and return a new raw data node return new RawData(contents); }
/*************************************************************** * Function: Parser::parseNode() * Purpose : Parse an XML data node * Initial : Maxime Chevalier-Boisvert on October 20, 2008 **************************************************************** Revisions and bug fixes: */ Node* Parser::parseNode(const std::string& xmlString, size_t& charIndex, const PosVector& positions) { // If this is the beginning of a CDATA region if (tokenMatch(xmlString, charIndex, "<![CDATA[")) { // Parse the CDATA region return parseRawData(xmlString, charIndex, positions); } // If this is the beginning of an XML element else if (tokenMatch(xmlString, charIndex, "<")) { // Parse the XML element recursively return parseElement(xmlString, charIndex, positions); } // Otherwise, if this is a text region else { // Parse the text return parseText(xmlString, charIndex, positions); } }
/** * Storage = [ "auto" | "static" | "extern" | "typedef" ] */ static ast* parserStorage (parserCtx* ctx, symTag* tag) { debugEnter("Storage"); ast* Node = 0; markerTag marker = tokenIsKeyword(ctx, keywordAuto) ? markerAuto : tokenIsKeyword(ctx, keywordStatic) ? markerStatic : tokenIsKeyword(ctx, keywordExtern) ? markerExtern : markerUndefined; if (tokenTryMatchKeyword(ctx, keywordTypedef)) *tag = symTypedef; else if (marker) { Node = astCreateMarker(ctx->location, marker); tokenMatch(ctx); } debugLeave(); return Node; }
Token * tokenMatch(char & ch, ifstream & file) { while ((ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t') && !file.eof()) { file.get(ch); } switch (ch) { case ':': return new Token(COLON) ; case ';': return new Token(SEMICOLON); case '.': return new Token(DOT); case ',': return new Token(COMMA); case '!': return new Token(BANG); case '(': return new Token(LPAREN); case ')': return new Token(RPAREN); case '[': return new Token(LBRACKET); case ']': return new Token(RBRACKET); case '{': return new Token(LBRACE); case '}': return new Token(RBRACE); case '<': return new Token(LESSTHAN); case '+': return new Token(PLUS); case '-': return new Token(MINUS); case '*': return new Token(TIMES); case '/': file.get(ch); if (ch == '/') { while (file.peek() != '\n') { file.get(ch); } file.get(ch); //One extra time to skip the newline return tokenMatch(ch, file); } else if (ch == '*') { while (file.get(ch)) { if (ch == '*') { file.get(ch); if (ch == '/') { file.get(ch); return tokenMatch(ch, file); } } } } return new Token(DIV); case '=': file.get(ch); if (ch == '=') return new Token(EQUALS); file.seekg (-1, file.cur); //Move file pointer back one step so the loop won't miss a char return new Token(EQSIGN); case '&': file.get(ch); if (ch == '&') return new Token(AND); file.seekg (-1, file.cur); //Move file pointer back one step so the loop won't miss a char return new Token(BAD); case '|': file.get(ch); if (ch == '|') return new Token(OR); file.seekg (-1, file.cur); //Move file pointer back one step so the loop won't miss a char return new Token(BAD); default: return findID(ch, file); } }
/*************************************************************** * Function: Parser::streamline() * Purpose : Streamline an XML code string * Initial : Maxime Chevalier-Boisvert on October 20, 2008 **************************************************************** Revisions and bug fixes: */ std::string Parser::streamline(const std::string& rawInput, PosVector& positions) { // Declare a string for the streamlined output std::string output; // reserve space in the string output.reserve(rawInput.length()); // Declare characters for stream processing unsigned char lastChar = '\0'; unsigned char thisChar = '\0'; unsigned char nextChar = '\0'; // Variable to tell if we are in a CDATA region bool inCDATA = false; // Variable to tell if we are in a comment bool inComment = false; // Variable to indicate the current line number size_t lineNumber = 1; // Variable to indicate the start of the current line size_t lineStart = 0; // For each character of the input for (size_t charIndex = 0; charIndex < rawInput.length(); ++charIndex) { // Read this character and the next thisChar = rawInput[charIndex]; nextChar = rawInput[charIndex + 1]; // Construct the text position of the current character TextPos charPos(lineNumber, charIndex - lineStart + 1); // If this character is a newline if (thisChar == '\n') { // Increment the line number ++lineNumber; // Store the line start index lineStart = charIndex + 1; } // If this is the start of a comment if (tokenMatch(rawInput, charIndex, "<!--") && !inCDATA && !inComment) { // We are now in a comment inComment = true; // Move past the comment opening charIndex += 3; continue; } // If this is the end of a comment if (tokenMatch(rawInput, charIndex, "-->") && inComment) { // We are no longer in a comment inComment = false; // Move past the comment closing charIndex += 2; continue; } // If this is the start of a CDATA region if (tokenMatch(rawInput, charIndex, "<![CDATA[") && !inCDATA && !inComment) { // We are now in a CDATA region inCDATA = true; } // If this is the end of a CDATA region if (tokenMatch(rawInput, charIndex, "]]>") && inCDATA) { // We are no longer in a CDATA region inCDATA = false; } // If we are inside a comment if (inComment) { // Skip this character continue; } // If we are not in a CDATA region if (!inCDATA) { // If this character is whitespace if (isspace(thisChar)) { // Consider it a space thisChar = ' '; } // If the next character is whitespace if (isspace(nextChar)) { // Consider it a space nextChar = ' '; } // If this character is whitespace if (thisChar == ' ') { // If the next char is whitespace, // or the last was an opening delimiter or a space, // or the next is a closing delimiter if (nextChar == ' ' || lastChar == '<' || lastChar == '>' || lastChar == ' ' || nextChar == '<' || nextChar == '>' || nextChar == '\0') { // Ignore this space continue; } } } // Add this character to the stream output += thisChar; // Store the last character added lastChar = thisChar; // Store the text position of this character positions.push_back(charPos); } // Add a final text position for the string terminator positions.push_back(TextPos(lineNumber, rawInput.length() - lineStart + 1)); // Ensure that there is a position for each character in the output assert (output.length() + 1 == positions.size()); // Return the streamlined XML stream return output; }
/*************************************************************** * Function: Parser::parseDeclaration() * Purpose : Parse an XML declaration node * Initial : Maxime Chevalier-Boisvert on October 20, 2008 **************************************************************** Revisions and bug fixes: */ Declaration* Parser::parseDeclaration(const std::string& xmlString, size_t& charIndex, const PosVector& positions) { // If this is not an XML declaration node if (!tokenMatch(xmlString, charIndex, "<?xml")) { // Return nothing without attempting to parse return NULL; } // Move past the declaration opening charIndex += 5; // Declare variables for the current and next characters unsigned char thisChar; unsigned char nextChar; // Declare a map of attributes std::map<std::string, std::string> attributes; // For each character for (;; ++charIndex) { // If we are past the length of the input stream if (charIndex > xmlString.length()) { // Throw an exception throw ParseError("Unexpected end of stream inside opening tag", positions[charIndex]); } // Update the current and next characters thisChar = xmlString[charIndex]; nextChar = xmlString[charIndex + 1]; // If this is a space if (isspace(thisChar)) { // Skip the space continue; } // If this is the end of the opening tag if (thisChar == '?' && nextChar == '>') { // Move past the declaration closing charIndex += 2; // End this loop break; } // If this character is alphanumeric if (isalnum(thisChar)) { // Parse this attribute std::pair<std::string, std::string> attribute = parseAttribute(xmlString, charIndex, positions); // If another attribute with this name was already parsed if (attributes.find(attribute.first) != attributes.end()) { // Throw an exception throw ParseError("Duplicate attribute name: " + attribute.first, positions[charIndex]); } // Add this attribute to the map attributes[attribute.first] = attribute.second; // Move to the next character continue; } // If this character was not recognized, throw an exception throw ParseError("Invalid character inside opening tag", positions[charIndex]); } // Create and return a new XML declaration node with the parsed information return new Declaration(attributes); }