void readTokenMatrix(Dataset& data, const char* filename) { std::vector<string> lines; loadFile(lines, filename); assert(lines.size() > 1); std::vector<int> counts(2); tokenizeLineIntoIntVector(lines[1], counts); assert(counts.size() == 2); //cout << "Found: " << counts[0] << " and " << counts[1] << endl; data.numDocs = counts[0]; data.numTokens = counts[1]; vec trainClasses(data.numDocs); mat features(data.numDocs, data.numTokens); rowvec row(data.numTokens); for(uint i = 3; i < data.numDocs + 3; i++) { std::vector<double> tokens(getNumTokens(lines[i])); tokenizeLineIntoDoubleVector(lines[i], tokens); trainClasses(i-3) = tokens[0]; //cout << "class[" << tokens[0] << "]"; row.zeros(); int cumsum = 0; for(uint j = 1; j < tokens.size() - 1; j+=2) { cumsum += tokens[j]; //cout << "cumsum[" << cumsum << "]token[" << tokens[j+1] << "]" << flush ; row[cumsum] = tokens[j+1]; } Matrix::setMatrixRowToVector(features, i-3 ,row); } data.classifications = trainClasses; data.features = features; }
void Parser::matchNot(int t) { if ( LA(1)==t ) { // Throws inverted-sense exception throw MismatchedTokenException(getTokenNames(), getNumTokens(), LT(1), t, true, getFilename()); } else { // mark token as consumed -- fetch next token deferred until LA/LT consume(); } }
/**Make sure current lookahead symbol matches token type <tt>t</tt>. * Throw an exception upon mismatch, which is catch by either the * error handler or by the syntactic predicate. */ void Parser::match(int t) { if ( DEBUG_PARSER ) { traceIndent(); ANTLR_USE_NAMESPACE(std)cout << "enter match(" << t << ") with LA(1)=" << LA(1) << ANTLR_USE_NAMESPACE(std)endl; } if ( LA(1)!=t ) { if ( DEBUG_PARSER ) { traceIndent(); ANTLR_USE_NAMESPACE(std)cout << "token mismatch: " << LA(1) << "!=" << t << ANTLR_USE_NAMESPACE(std)endl; } throw MismatchedTokenException(getTokenNames(), getNumTokens(), LT(1), t, false, getFilename()); } else { // mark token as consumed -- fetch next token deferred until LA/LT consume(); } }
mat readDoubleMatrix(char * filename) { std::vector<string> lines; loadFile(lines, filename); uint numRows = lines.size(); assert(numRows > 0); uint numCols = getNumTokens(lines[0]); std::vector<double> rowV(numCols); mat res(numRows, rowV.size()); uint ridx = 0; while(ridx != numRows) { tokenizeLineIntoDoubleVector(lines[ridx], rowV); assignVectorToMatrix(rowV, res, ridx); ridx++; } return res; }
/**Make sure current lookahead symbol matches the given set * Throw an exception upon mismatch, which is catch by either the * error handler or by the syntactic predicate. */ void Parser::match(const BitSet& b) { if ( DEBUG_PARSER ) { traceIndent(); ANTLR_USE_NAMESPACE(std)cout << "enter match(" << "bitset" /*b.toString()*/ << ") with LA(1)=" << LA(1) << ANTLR_USE_NAMESPACE(std)endl; } if ( !b.member(LA(1)) ) { if ( DEBUG_PARSER ) { traceIndent(); ANTLR_USE_NAMESPACE(std)cout << "token mismatch: " << LA(1) << " not member of " << "bitset" /*b.toString()*/ << ANTLR_USE_NAMESPACE(std)endl; } throw MismatchedTokenException(getTokenNames(), getNumTokens(), LT(1), b, false, getFilename()); } else { // mark token as consumed -- fetch next token deferred until LA/LT consume(); } }
const char* getTokenName( int type ) const { if( type > getNumTokens() ) return 0; return FjolnirCodegen::tokenNames[type]; }
void TreeParser::matchNot(RefAST t, int ttype) { if ( !t || t == ASTNULL || t->getType() == ttype ) throw MismatchedTokenException( getTokenNames(), getNumTokens(), t, ttype, true ); }
/** Make sure current lookahead symbol matches the given set * Throw an exception upon mismatch, which is caught by either the * error handler or by the syntactic predicate. */ void TreeParser::match(RefAST t, const BitSet& b) { if ( !t || t==ASTNULL || !b.member(t->getType()) ) throw MismatchedTokenException( getTokenNames(), getNumTokens(), t, b, false ); }
virtual void match(RefAST t, int ttype) { if (!t || t == ASTNULL || t->getType() != ttype ) throw MismatchedTokenException( getTokenNames(), getNumTokens(), t, ttype, false ); }