void EvalCommand :: Evaluate( CSVRow & row ) {
bool skipelse = false;
for ( unsigned int i = 0; i < mFieldExprs.size() ; i++ ) {
if ( mIsIf[i] ) {
if ( i < mFieldExprs.size() - 1 && mIsIf[i+2] ) {
CSVTHROW( "Cannot have consecutive -if options" );
}
if ( i >= mFieldExprs.size() - 2 ) {
CSVTHROW( "Need two -e options after -if" );
}
string r = mFieldExprs[i].mExpr.Evaluate();
if ( ALib::Expression::ToBool( r ) ) {
skipelse = true;
} else {
i++;
}
continue;
}
string r = mFieldExprs[i].mExpr.Evaluate();
if ( mFieldExprs[i].mField < 0 || mFieldExprs[i].mField >= (int) row.size() ) {
row.push_back( r );
}
else {
row[ mFieldExprs[i].mField ] = r;
}
if ( skipelse ) {
i++;
skipelse = false;
}
}
}
int JoinCommand :: Execute( ALib::CommandLine & cmd ) {
Clear();
mKeep = cmd.HasFlag( FLAG_KEEP );
mOuterJoin = cmd.HasFlag( FLAG_OUTERJ );
mInvert = cmd.HasFlag( FLAG_INVERT );
mIgnoreCase = cmd.HasFlag( FLAG_ICASE );
if ( mOuterJoin && mInvert ) {
CSVTHROW( "Cannot have both " << FLAG_OUTERJ
<< " and " << FLAG_INVERT << " flags" );
}
string js = cmd.GetValue( FLAG_COLS );
BuildJoinSpecs( js );
IOManager io( cmd );
unsigned int scount = io.InStreamCount();
if ( scount < 2 ) {
CSVTHROW( "Need at least two input streams" );
}
BuildRowMap( io.CreateStreamParser( scount - 1 ) );
CSVRow row;
for ( unsigned int i = 0; i < scount - 1; i++ ) {
std::unique_ptr <ALib::CSVStreamParser> p( io.CreateStreamParser( i ) );
while( p->ParseNext( row ) ) {
WriteJoinRows( io, row );
}
}
return 0;
}
void FixedCommand :: BuildFields( const ALib::CommandLine & cmd ) {
mFields.clear();
ALib::CommaList cl( cmd.GetValue( FLAG_COLS ) ); // chop into pairs
if ( cl.Size() == 0 ) {
CSVTHROW( "Need fields specified with " << FLAG_COLS << " flag" );
}
for ( unsigned int i = 0; i < cl.Size(); i++ ) {
vector <string> tmp;
if ( ALib::Split( cl.At(i), ':', tmp ) != 2 ) {
CSVTHROW( "Invalid field specification: " << cl.At(i) );
}
if ( ! ALib::IsInteger( tmp[0] ) || ! ALib::IsInteger( tmp[1] ) ) {
CSVTHROW( "Invalid field specification: " << cl.At(i) );
}
unsigned int f1 = ALib::ToInteger( tmp[0] );
unsigned int f2 = ALib::ToInteger( tmp[1] );
if ( f1 == 0 || f2 == 0 ) {
CSVTHROW( "Invalid field specification: " << cl.At(i) );
}
mFields.push_back( std::make_pair( f1, f2 ) );
}
}
static unsigned int GetField( const ALib::CommandLine & cmd,
const string & option ) {
string rc = cmd.GetValue( option );
if ( ! ALib::IsInteger( rc ) ) {
CSVTHROW( "Value for " << option << " must be integer" );
}
int n = ALib::ToInteger( rc );
if ( n <= 0 ) {
CSVTHROW( "Value for " << option << " must be greater than zero" );
}
return (unsigned int) n - 1;
}
void EvalCommand :: GetExpressions( ALib::CommandLine & cmd ) {
int i = 2; // skip exe name and command name
while( i < cmd.Argc() ) {
if ( cmd.Argv( i ) == FLAG_EXPR || cmd.Argv( i ) == FLAG_IF ) {
mIsIf.push_back( cmd.Argv( i ) == FLAG_IF );
if ( i + 1 >= cmd.Argc() ) {
CSVTHROW( "Missing expression" );
}
i++;
string expr = cmd.Argv( i );
ALib::Expression ex;
string emsg = ex.Compile( expr );
if ( emsg != "" ) {
CSVTHROW( emsg + " in " + expr );
}
mFieldExprs.push_back( FieldEx( -1, ex ) );
}
else if ( cmd.Argv( i ) == FLAG_REMOVE ) {
mIsIf.push_back( false );
if ( mDiscardInput ) {
CSVTHROW( "Cannot specify both " << FLAG_REMOVE
<< " and " << FLAG_DISCARD );
}
if ( i + 1 >= cmd.Argc() ) {
CSVTHROW( "Missimg field/expression" );
}
i++;
string::size_type pos = cmd.Argv(i).find_first_of( "," );
if ( pos == string::npos ) {
CSVTHROW( "Invalid field/index pair: " << cmd.Argv(i) );
}
string field = cmd.Argv(i).substr( 0, pos );
string expr = cmd.Argv(i).substr( pos + 1 );
if ( ! ALib::IsInteger( field ) ) {
CSVTHROW( "Invalid field (need integer): " << field );
}
int n = ALib::ToInteger( field );
if ( n <= 0 ) {
CSVTHROW( "Invalid field (must be greater than zero): " << field );
}
ALib::Expression ex;
string emsg = ex.Compile( expr );
if ( emsg != "" ) {
CSVTHROW( emsg + " in " + expr );
}
mFieldExprs.push_back( FieldEx( n - 1, ex ) );
}
i++;
}
if ( mFieldExprs.size() == 0 ) {
CSVTHROW( "Need at least one of -e or -r options" );
}
}
int SummaryCommand :: Execute( ALib::CommandLine & cmd ) {
ProcessFlags( cmd );
IOManager io( cmd );
CSVRow row;
SizeMap sizemap;
while( io.ReadCSV( row ) ) {
if ( mType == Size ) {
RecordSizes( row, sizemap );
}
else {
mRows.push_back( row );
}
}
if ( mType == Size ) {
PrintSizes( io, sizemap );
}
else {
if ( mRows.size() == 0 ) {
CSVTHROW( "No input" );
}
Summarise( io );
}
return 0;
}
void DSVReadCommand :: ParseDSV( const string & line, CSVRow & rv ) {
CSVRow row;
unsigned int pos = 0, len = line.size();
string val;
while( pos < len ) {
char c = line[ pos++ ];
if ( c == Delim() ) {
while( mCollapseSep && ALib::Peek( line, pos ) == Delim() ) {
pos++;
}
row.push_back( Unquote( val ) );
val = "";
}
else if ( c == '\\' ) {
if ( pos == len ) {
CSVTHROW( "Escape at end of line" );
}
else {
val += line[ pos++ ];
}
}
else {
val += c;
}
}
row.push_back( Unquote( val ) );
BuildCSVRow( row, rv );
}
static char MustGet( const string & s, unsigned int i ) {
char c = ALib::Peek( s, i );
if ( c == 0 ) {
CSVTHROW( "Invalid value for " << FLAG_EDIT << ": " << s );
}
return c;
}
void DateReadCommand :: ProcessFlags( ALib::CommandLine & cmd ) {
ALib::CommaList cl( cmd.GetValue( FLAG_COLS, "" ) );
CommaListToIndex( cl, mFields );
string mask = cmd.GetValue( FLAG_MASK, "" );
string cys = cmd.GetValue( FLAG_CDATE, ALib::Str( BASE_YEAR ) );
NotBoth( cmd, FLAG_BDLIST, FLAG_BDEXCL );
if ( cmd.HasFlag( FLAG_BDLIST ) ) {
mWriteAction = WriteBad;
}
else if ( cmd.HasFlag( FLAG_BDEXCL ) ) {
mWriteAction = WriteGood;
}
else {
mWriteAction = WriteAll;
}
if ( ! ALib::IsInteger( cys )) {
CSVTHROW( "Invalid year value " << cys );
}
int cy = ALib::ToInteger( cys );
string mnames = cmd.GetValue( FLAG_MNAMES, MONTH_NAMES );
delete mReader;
mReader = new MaskedDateReader( mask, mnames, cy );
}
static string Format( const string & fmt, const ALib::Date & date ) {
if ( fmt == "d" ) {
return ALib::Str( date.Day() );
}
else if ( fmt == "dd" ) {
return ALib::ZeroPad( date.Day(), 2 );
}
else if ( fmt == "m" ) {
return ALib::Str( date.Month() );
}
else if ( fmt == "mm" ) {
return ALib::ZeroPad( date.Month(), 2 );
}
else if ( fmt == "mmm" ) {
return date.ShortMonthName();
}
else if ( fmt == "M" ) {
return date.MonthName();
}
else if ( fmt == "y" || fmt == "yyyy" ) {
return ALib::Str( date.Year() );
}
else if ( fmt == "W" ) {
return date.DayName();
}
else if ( fmt == "w" ) {
return date.ShortDayName();
}
else {
CSVTHROW( "Invalid date format substring: " << fmt );
}
}
void CallCommand :: ProcessFlags( const ALib::CommandLine & cmd ) {
string bs = cmd.GetValue( FLAG_BSIZE, ALib::Str( DEF_OUTBUF_SIZE ) );
if ( ! ALib::IsInteger( bs )) {
CSVTHROW( "Value for buffer size must be integer" );
}
mOutBufSize = ALib::ToInteger( bs ) * 1024;
if ( mOutBufSize < DEF_OUTBUF_SIZE ) {
CSVTHROW( "Output buffer size too small" );
}
mDLL = cmd.GetValue( FLAG_DLL );
mFuncName = cmd.GetValue( FLAG_FUNC );
ALib::CommaList cl( cmd.GetValue( FLAG_COLS ) );
CommaListToIndex( cl, mFields );
}
string SQLUpdateCommand :: MakeSetClause( const CSVRow & row ) const {
string sc;
for ( unsigned int i = 0; i < DataCols().size(); i++ ) {
unsigned int fi = DataCols().at( i ).mField;
if ( fi >= row.size() ) {
CSVTHROW( "Required field " << fi + 1 << " missing from input" );
}
if ( sc != "" ) {
sc += ", ";
}
string field = EmptyToNull( row[fi] );
sc += DataCols().at(i).mColName
+ " = "
+ (( DoSQLQuote( i ) && ! NoNullQuote( field ) )
? ALib::SQuote( ALib::SQLQuote( field ) )
: field );
}
return "SET " + sc;
}
string SQLCommand :: MakeWhereClause( const CSVRow & row ) const {
string wc;
for ( unsigned int i = 0; i < WhereCols().size(); i++ ) {
unsigned int wi = WhereCols().at( i ).mField;
if ( wi >= row.size() ) {
CSVTHROW( "Required field " << wi + 1 << " missing in input" );
}
if ( wc != "" ) {
wc += " AND ";
}
string field = EmptyToNull( row[wi] );
wc += WhereCols().at(i).mColName
+ (field == "NULL" ? " IS " : " = " )
+ ( (DoSQLQuote( i ) && ! NoNullQuote( field ))
? ALib::SQuote( ALib::SQLQuote( field ) )
: field );
}
return "WHERE " + wc;
}
void TemplateCommand :: FileOut( const CSVRow & row ) {
string fname = ReplaceColumns( mFileTemplate, row );
std::ofstream ifs( fname );
if ( ! ifs.is_open() ) {
CSVTHROW( "Cannot open file " << fname << " for output" );
}
ifs << ReplaceColumns( mTemplate, row );
}
PivotCommand::ColRow PivotCommand:: MakeColRow( const CSVRow & row ) const {
if ( mCol >= row.size() || mRow >= row.size() ) {
CSVTHROW( "Invalid row/column index" );
}
return PivotCommand::ColRow( row[mCol], row[mRow] );
}
void SummaryCommand :: ProcessFlags( const ALib::CommandLine & cmd ) {
int nf = CountNonGeneric( cmd );
if ( nf == 0 ) {
CSVTHROW( "Need a summary flag" );
}
else if ( nf != 1 ) {
CSVTHROW( "Only one summary flag allowed" );
}
if ( cmd.HasFlag( FLAG_AVG ) ) {
mType = Average;
GetFields( cmd, FLAG_AVG );
}
else if ( cmd.HasFlag( FLAG_MIN ) ) {
mType = Min;
GetFields( cmd, FLAG_MIN );
}
else if ( cmd.HasFlag( FLAG_MAX ) ) {
mType = Max;
GetFields( cmd, FLAG_MAX );
}
else if ( cmd.HasFlag( FLAG_FREQ ) ) {
mType = Frequency;
GetFields( cmd, FLAG_FREQ );
}
else if ( cmd.HasFlag( FLAG_MEDIAN ) ) {
mType = Median;
GetFields( cmd, FLAG_MEDIAN);
}
else if ( cmd.HasFlag( FLAG_MODE) ) {
mType = Mode;
GetFields( cmd, FLAG_MODE );
}
else if ( cmd.HasFlag( FLAG_SUM ) ) {
mType = Sum;
GetFields( cmd, FLAG_SUM );
}
else if ( cmd.HasFlag( FLAG_SIZE ) ) {
mType = Size;
}
else {
CSVTHROW( "Should never happen in SummaryCommand::ProcessFlags" );
}
}
void EditCommand :: AddSubCmd( const string & ev ) {
if ( ALib::IsEmpty( ev ) ) {
CSVTHROW( "Empty value for " << FLAG_EDIT );
}
if ( ! isalpha( ev[0] ) ) { // for now
CSVTHROW( "Edit sub command missing from " << ev );
}
char cmd;
vector <string> tmp;
ParseSub( ev, cmd, tmp );
// for ( unsigned int i = 0; i < tmp.size(); i++ ) {
// std::cout << "[" << tmp[i] << "]" << std::endl;
// }
mSubCmds.push_back ( EditSubCmd( cmd, tmp[0], tmp[1], tmp[2] ) );
}
void PivotCommand :: ProcessFlags( const ALib::CommandLine & cmd ) {
mRow = GetField( cmd, FLAG_ROW );
mCol = GetField( cmd, FLAG_COL );
mFact = GetField( cmd, FLAG_FACT );
if ( mRow == mCol ) {
CSVTHROW( "Row and column options cannot have the same value" );
}
mAction = GetAction( cmd );
}
void DateFormatCommand :: ProcessFlags( ALib::CommandLine & cmd ) {
ALib::CommaList cl( cmd.GetValue( FLAG_COLS, "" ) );
CommaListToIndex( cl, mFields );
string fmt = cmd.GetValue( FLAG_FMT, "" );
if ( ALib::IsEmpty( fmt ) ) {
CSVTHROW( "Empty date format" );
}
BuildFormat( fmt );
}
void TrimCommand :: GetWidths( const std::string & ws ) {
ALib::CommaList wl( ws );
for ( unsigned int i = 0; i < wl.Size(); i++ ) {
if ( ! ALib::IsInteger( wl.At(i) ) ) {
CSVTHROW( "Invalid width " << wl.At(i) );
}
int n = ALib::ToInteger( wl.At(i) );
mWidths.push_back( n );
}
}
void TemplateCommand :: HandleSpecialChars( const string & tplate,
char c, unsigned int & pos,
const CSVRow & row,
string & out ) {
char t = tplate[ pos++ ];
if ( c == '\\' ) {
switch( t ) {
case '\n':
case '\r': CSVTHROW( "Invalid escape at end of line" );
case 'n': out += '\n'; break;
case 't': out += '\t'; break;
default: out += t;
}
}
else { // its a brace
string ns;
while( t != POUTRO) {
if ( t == '\n' || t == '\r' ) {
CSVTHROW( "Missing closing brace" );
}
ns += t;
t = tplate[ pos++ ];
}
if( ns.size() && ns[0] == EVALCHR ) { // it's an expression
out += Eval( row, ns );
return;
}
if ( ! ALib::IsInteger( ns ) ) {
CSVTHROW( "Invalid placeholder: " << "{" << ns << "}" );
}
int n = ALib::ToInteger( ns ) - 1; // to zero based
if ( n < 0 ) {
CSVTHROW( "Invalid placeholder: " << "{" << ns << "}" );
}
if ( (unsigned int) n < row.size() ) {
out += row[ n ];
}
}
}
void TemplateCommand :: ReadTemplate( const ALib::CommandLine & cmd ) {
string fname = cmd.GetValue( FLAG_TFILE );
if ( ALib::IsEmpty( fname ) ) {
CSVTHROW( "Need template file name specified with "
<< FLAG_TFILE << " flag" );
}
std::ifstream ifs( fname.c_str() );
if ( ! ifs.is_open() ) {
CSVTHROW( "Cannot open file " << fname << " for input" );
}
mTemplate = "";
string line;
while( std::getline( ifs, line ) ) {
mTemplate += line + "\n";
}
ifs.close();
}
static void ParseSub( const string & s, char & cmd, vector <string> & fields ) {
unsigned int i = 0;
cmd = MustGet( s, i++ );
if ( cmd != SUB_CMD ) {
CSVTHROW( "Invalid value for " << FLAG_EDIT << ": " << s );
}
char sep = MustGet( s, i++ );
if ( sep == '\\' ) {
CSVTHROW( "Invalid value for " << FLAG_EDIT << ": " << s );
}
string f = ReadField( s, i, sep );
fields.push_back( f );
f = ReadField( s, i, sep );
fields.push_back( f );
f = s.substr( i );
fields.push_back( f );
}
void JoinCommand :: BuildJoinSpecs( const string & js ) {
if ( js == "" ) {
CSVTHROW( "No join specified with " << FLAG_COLS << " flag" );
}
ALib::CommaList cl( js );
for ( unsigned int i = 0; i < cl.Size(); i++ ) {
vector <string> cols;
if ( ALib::Split( cl.At(i), ':', cols ) != 2 ) {
CSVTHROW( "Invalid join specification: " << cl.At(i) );
}
int c1 = ALib::ToInteger( cols[0], "Invalid column: " + cols[0] );
int c2 = ALib::ToInteger( cols[1], "Invalid column: " + cols[1] );
if ( c1 < 1 || c2 < 1 ) {
CSVTHROW( "Invalid join specfication: " << cl.At(i) );
}
mJoinSpecs.push_back( std::make_pair( c1 - 1, c2 - 1 ) );
}
}
void MaskedDateReader :: SetMonths( const string & months ) {
if ( months == "" ) {
mMonthNames = ALib::CommaList( MONTH_NAMES );
}
else {
mMonthNames = ALib::CommaList( months );
}
if ( mMonthNames.Size() != 12 ) {
CSVTHROW( "Invalid month list: " << months );
}
}
static void SumRow( vector <double> & sums, const CSVRow & row,
const FieldList & fl ) {
for ( unsigned int i = 0; i < fl.size(); i++ ) {
unsigned int fi = fl.at(i);
if ( fi >= row.size() ) {
CSVTHROW( "Invalid field index" );
}
sums.at(i) += ALib::ToReal( row.at(fi) );
}
}
int CallCommand :: Execute( ALib::CommandLine & cmd ) {
GetSkipOptions( cmd );
ProcessFlags( cmd );
std::vector <char> outbuf( mOutBufSize );
HMODULE dll = LoadLibrary( mDLL.c_str() );
if ( dll == NULL ) {
CSVTHROW( "LoadLibrary call on " << mDLL << " failed" );
}
mFunc = (FuncType) GetProcAddress( dll, mFuncName.c_str() );
if ( mFunc == NULL ) {
CSVTHROW( "Cannot load function " << mFuncName << "from DLL " << mDLL );
}
IOManager io( cmd );
CSVRow row;
while( io.ReadCSV( row ) ) {
if ( ! Skip( io, row ) ) {
continue;
}
if ( Pass( io, row ) ) {
io.WriteRow( row );
continue;
}
int rv = CallOnFields( row, &outbuf[0] );
if ( rv == 0 ) {
io.WriteRow( row );
}
else if ( rv > 0 ) {
CSVTHROW( mFuncName << " returned error code " << rv );
}
else {
// do nothing - negative values just mean skip output
}
}
return 0;
}
void PivotCommand :: AddFact( const ColRow & cr, const string & fact ) {
if ( (mAction == Action::Average || mAction == Action::Sum) ) {
if ( ! ALib::IsNumber( fact ) ) {
CSVTHROW( "Non-numeric fact: " << fact);
}
double fd = ALib::ToReal( fact );
mColRowValMap[cr].Update( fd );
}
else {
mColRowValMap[cr].Update(1);
}
}
int TruncPadBase :: Execute( ALib::CommandLine & cmd ) {
GetSkipOptions( cmd );
string ps = cmd.GetValue( FLAG_PAD );
ALib::CommaList padding( ps );
unsigned int ncols = padding.Size();
bool ncolspec = false; // use explicit size or not
if ( ncols == 0 || cmd.HasFlag( FLAG_NUM ) ) {
if ( ! cmd.HasFlag( FLAG_NUM ) ) {
CSVTHROW( "Need -n flag to specify field count" );
}
ncolspec = true;
string nv = cmd.GetValue( FLAG_NUM );
if ( ALib::ToInteger( nv, "-n flag needs integer value" ) < 0 ) {
CSVTHROW( FLAG_NUM << " needs value greater or equal to zero" );
}
ncols = ALib::ToInteger( nv );
}
IOManager io( cmd );
CSVRow row;
while( io.ReadCSV( row ) ) {
if ( Skip( io, row ) ) {
continue;
}
if ( ! Pass( io, row ) ) {
unsigned int nc = ncolspec ? ncols : row.size() + padding.Size();
ProcessRow( row, nc, padding );
}
io.WriteRow( row );
}
return 0;
}
void RemoveNewlineCommand :: ProcessFlags( const ALib::CommandLine & cmd ) {
string scols = cmd.GetValue( FLAG_COLS, "" );
ALib::CommaList cl( cmd.GetValue( FLAG_COLS, "" ) );
CommaListToIndex( cl, mFields);
if ( cmd.HasFlag( FLAG_EXCLNL ) && cmd.HasFlag( FLAG_STR ) ) {
CSVTHROW( "Flags " << FLAG_EXCLNL << " and " << FLAG_STR
<< " are mutually exclusive" );
}
mSep = cmd.GetValue( FLAG_STR, "" );
ExpandSep();
mExcludeAfter = cmd.HasFlag( FLAG_EXCLNL );
}