void readTokenMatrix(Dataset& data, const char* filename) {
std::vector<string> lines;
loadFile(lines, filename);
assert(lines.size() > 1);
std::vector<int> counts(2);
tokenizeLineIntoIntVector(lines[1], counts);
assert(counts.size() == 2);
//cout << "Found: " << counts[0] << " and " << counts[1] << endl;
data.numDocs = counts[0];
data.numTokens = counts[1];
vec trainClasses(data.numDocs);
mat features(data.numDocs, data.numTokens);
rowvec row(data.numTokens);
for(uint i = 3; i < data.numDocs + 3; i++) {
std::vector<double> tokens(getNumTokens(lines[i]));
tokenizeLineIntoDoubleVector(lines[i], tokens);
trainClasses(i-3) = tokens[0];
//cout << "class[" << tokens[0] << "]";
row.zeros();
int cumsum = 0;
for(uint j = 1; j < tokens.size() - 1; j+=2) {
cumsum += tokens[j];
//cout << "cumsum[" << cumsum << "]token[" << tokens[j+1] << "]" << flush ;
row[cumsum] = tokens[j+1];
}
Matrix::setMatrixRowToVector(features, i-3 ,row);
}
data.classifications = trainClasses;
data.features = features;
}
void Parser::matchNot(int t)
{
if ( LA(1)==t ) {
// Throws inverted-sense exception
throw MismatchedTokenException(getTokenNames(), getNumTokens(), LT(1), t, true, getFilename());
} else {
// mark token as consumed -- fetch next token deferred until LA/LT
consume();
}
}
/**Make sure current lookahead symbol matches token type <tt>t</tt>.
* Throw an exception upon mismatch, which is catch by either the
* error handler or by the syntactic predicate.
*/
void Parser::match(int t)
{
if ( DEBUG_PARSER )
{
traceIndent();
ANTLR_USE_NAMESPACE(std)cout << "enter match(" << t << ") with LA(1)=" << LA(1) << ANTLR_USE_NAMESPACE(std)endl;
}
if ( LA(1)!=t ) {
if ( DEBUG_PARSER )
{
traceIndent();
ANTLR_USE_NAMESPACE(std)cout << "token mismatch: " << LA(1) << "!=" << t << ANTLR_USE_NAMESPACE(std)endl;
}
throw MismatchedTokenException(getTokenNames(), getNumTokens(), LT(1), t, false, getFilename());
} else {
// mark token as consumed -- fetch next token deferred until LA/LT
consume();
}
}
mat readDoubleMatrix(char * filename) {
std::vector<string> lines;
loadFile(lines, filename);
uint numRows = lines.size();
assert(numRows > 0);
uint numCols = getNumTokens(lines[0]);
std::vector<double> rowV(numCols);
mat res(numRows, rowV.size());
uint ridx = 0;
while(ridx != numRows) {
tokenizeLineIntoDoubleVector(lines[ridx], rowV);
assignVectorToMatrix(rowV, res, ridx);
ridx++;
}
return res;
}
/**Make sure current lookahead symbol matches the given set
* Throw an exception upon mismatch, which is catch by either the
* error handler or by the syntactic predicate.
*/
void Parser::match(const BitSet& b)
{
if ( DEBUG_PARSER )
{
traceIndent();
ANTLR_USE_NAMESPACE(std)cout << "enter match(" << "bitset" /*b.toString()*/
<< ") with LA(1)=" << LA(1) << ANTLR_USE_NAMESPACE(std)endl;
}
if ( !b.member(LA(1)) ) {
if ( DEBUG_PARSER )
{
traceIndent();
ANTLR_USE_NAMESPACE(std)cout << "token mismatch: " << LA(1) << " not member of "
<< "bitset" /*b.toString()*/ << ANTLR_USE_NAMESPACE(std)endl;
}
throw MismatchedTokenException(getTokenNames(), getNumTokens(), LT(1), b, false, getFilename());
} else {
// mark token as consumed -- fetch next token deferred until LA/LT
consume();
}
}
const char* getTokenName( int type ) const
{
if( type > getNumTokens() ) return 0;
return FjolnirCodegen::tokenNames[type];
}
void TreeParser::matchNot(RefAST t, int ttype)
{
if ( !t || t == ASTNULL || t->getType() == ttype )
throw MismatchedTokenException( getTokenNames(), getNumTokens(),
t, ttype, true );
}
/** Make sure current lookahead symbol matches the given set
* Throw an exception upon mismatch, which is caught by either the
* error handler or by the syntactic predicate.
*/
void TreeParser::match(RefAST t, const BitSet& b)
{
if ( !t || t==ASTNULL || !b.member(t->getType()) )
throw MismatchedTokenException( getTokenNames(), getNumTokens(),
t, b, false );
}
virtual void match(RefAST t, int ttype)
{
if (!t || t == ASTNULL || t->getType() != ttype )
throw MismatchedTokenException( getTokenNames(), getNumTokens(),
t, ttype, false );
}
