int JoinCommand :: Execute( ALib::CommandLine & cmd ) {
Clear();
mKeep = cmd.HasFlag( FLAG_KEEP );
mOuterJoin = cmd.HasFlag( FLAG_OUTERJ );
mInvert = cmd.HasFlag( FLAG_INVERT );
mIgnoreCase = cmd.HasFlag( FLAG_ICASE );
if ( mOuterJoin && mInvert ) {
CSVTHROW( "Cannot have both " << FLAG_OUTERJ
<< " and " << FLAG_INVERT << " flags" );
}
string js = cmd.GetValue( FLAG_COLS );
BuildJoinSpecs( js );
IOManager io( cmd );
unsigned int scount = io.InStreamCount();
if ( scount < 2 ) {
CSVTHROW( "Need at least two input streams" );
}
BuildRowMap( io.CreateStreamParser( scount - 1 ) );
CSVRow row;
for ( unsigned int i = 0; i < scount - 1; i++ ) {
std::unique_ptr <ALib::CSVStreamParser> p( io.CreateStreamParser( i ) );
while( p->ParseNext( row ) ) {
WriteJoinRows( io, row );
}
}
return 0;
}
int TemplateCommand :: Execute( ALib::CommandLine & cmd ) {
GetSkipOptions( cmd );
ReadTemplate( cmd );
if ( cmd.HasFlag( FLAG_FNAMES ) ) {
mFileTemplate = cmd.GetValue( FLAG_FNAMES );
}
IOManager io( cmd );
CSVRow row;
while( io.ReadCSV( row ) ) {
if ( Skip( row ) ) {
continue;
}
if ( mFileTemplate.empty() ) {
io.Out() << ReplaceColumns( mTemplate, row );
}
else {
FileOut( row );
}
}
return 0;
}
int EditCommand :: Execute( ALib::CommandLine & cmd ) {
GetSkipOptions( cmd );
for ( int i = 2; i < cmd.Argc(); i++ ) {
if ( cmd.Argv( i ) == FLAG_EDIT ) {
AddSubCmd( cmd.Argv( i + 1 ) );
i++;
}
}
ALib::CommaList cl( cmd.GetValue( FLAG_COLS, "" ) );
CommaListToIndex( cl, mCols );
IOManager io( cmd );
CSVRow row;
while( io.ReadCSV( row ) ) {
if ( Skip( io, row ) ) {
continue;
}
if ( ! Pass( io, row ) ) {
EditRow( row );
}
io.WriteRow( row );
}
return 0;
}
void FileSplitCommand :: ProcessFlags( ALib::CommandLine & cmd ) {
mDir = cmd.GetValue( FLAG_FSDIR, DEF_DIR );
mFilePrefix = cmd.GetValue( FLAG_FSPRE, DEF_PREF );
mFileExt= cmd.GetValue( FLAG_FSEXT, DEF_EXT );
ALib::CommaList cl( cmd.GetValue( FLAG_COLS ) );
CommaListToIndex( cl, mColIndex );
mUseFieldNames = cmd.HasFlag( FLAG_USEFLD );
}
void DateFormatCommand :: ProcessFlags( ALib::CommandLine & cmd ) {
ALib::CommaList cl( cmd.GetValue( FLAG_COLS, "" ) );
CommaListToIndex( cl, mFields );
string fmt = cmd.GetValue( FLAG_FMT, "" );
if ( ALib::IsEmpty( fmt ) ) {
CSVTHROW( "Empty date format" );
}
BuildFormat( fmt );
}
void AsciiTableCommand :: ProcessFlags( ALib::CommandLine & cmd ) {
mUseLineSep = cmd.HasFlag( FLAG_SEP );
mHeadings = ALib::CommaList( cmd.GetValue( FLAG_HEADER, "" ) );
ALib::CommaList ra = ALib::CommaList( cmd.GetValue( FLAG_RALIGN, "" ) );
CommaListToIndex( ra, mRightAlign );
if ( mHeadings.Size() && mHeadings.At(0) != FILE_HEADER) {
CSVRow r;
for ( unsigned int i = 0; i < mHeadings.Size() ; i++ ) {
r.push_back( mHeadings.At( i ) );
}
AddRow( r );
}
}
int EvalCommand :: Execute( ALib::CommandLine & cmd ) {
GetSkipOptions( cmd );
IOManager io( cmd );
CSVRow row;
mDiscardInput = cmd.HasFlag( FLAG_DISCARD );
GetExpressions( cmd );
while( io.ReadCSV( row ) ) {
if ( Skip( io, row ) ) {
continue;
}
if ( ! Pass( io, row ) ) {
SetParams( row, io );
if ( mDiscardInput ) {
row.clear();
}
Evaluate( row );
}
io.WriteRow( row );
}
return 0;
}
int DSVReadCommand :: Execute( ALib::CommandLine & cmd ) {
ReadFlags( cmd );
mIsCSV = cmd.HasFlag( FLAG_CSV );
mCollapseSep = cmd.HasFlag( FLAG_CMULTI );
IOManager io( cmd );
CSVRow row;
string line;
while( io.ReadLine( line ) ) {
ParseDSV( line, row );
io.WriteRow( row );
}
return 0;
}
void FixedCommand :: BuildFields( const ALib::CommandLine & cmd ) {
mFields.clear();
ALib::CommaList cl( cmd.GetValue( FLAG_COLS ) ); // chop into pairs
if ( cl.Size() == 0 ) {
CSVTHROW( "Need fields specified with " << FLAG_COLS << " flag" );
}
for ( unsigned int i = 0; i < cl.Size(); i++ ) {
vector <string> tmp;
if ( ALib::Split( cl.At(i), ':', tmp ) != 2 ) {
CSVTHROW( "Invalid field specification: " << cl.At(i) );
}
if ( ! ALib::IsInteger( tmp[0] ) || ! ALib::IsInteger( tmp[1] ) ) {
CSVTHROW( "Invalid field specification: " << cl.At(i) );
}
unsigned int f1 = ALib::ToInteger( tmp[0] );
unsigned int f2 = ALib::ToInteger( tmp[1] );
if ( f1 == 0 || f2 == 0 ) {
CSVTHROW( "Invalid field specification: " << cl.At(i) );
}
mFields.push_back( std::make_pair( f1, f2 ) );
}
}
void CallCommand :: ProcessFlags( const ALib::CommandLine & cmd ) {
string bs = cmd.GetValue( FLAG_BSIZE, ALib::Str( DEF_OUTBUF_SIZE ) );
if ( ! ALib::IsInteger( bs )) {
CSVTHROW( "Value for buffer size must be integer" );
}
mOutBufSize = ALib::ToInteger( bs ) * 1024;
if ( mOutBufSize < DEF_OUTBUF_SIZE ) {
CSVTHROW( "Output buffer size too small" );
}
mDLL = cmd.GetValue( FLAG_DLL );
mFuncName = cmd.GetValue( FLAG_FUNC );
ALib::CommaList cl( cmd.GetValue( FLAG_COLS ) );
CommaListToIndex( cl, mFields );
}
void SQLCommand :: BuildCols( const ALib::CommandLine & cmd,
const string & flag,
SQLColSpec::Vec & cols ) {
bool havenames = false;
cols.clear();
if ( cmd.FlagCount( flag ) > 1 ) {
CSVTHROW( "Need fields specified by single " << flag << " flag" );
}
ALib::CommaList cl( cmd.GetValue( flag ) );
for ( unsigned int i = 0; i < cl.Size(); i++ ) {
vector <string> tmp;
if ( ALib::Split( cl.At(i), ':', tmp ) > 2 ) {
CSVTHROW( "Invalid column specification: " << cl.At(i) );
}
if ( tmp.size() == 0 ) {
CSVTHROW( "Empty column specification" );
}
if ( ! ALib::IsInteger( tmp[0] ) ) {
CSVTHROW( "Field index must be integer in " << cl.At(i) );
}
int icol = ALib::ToInteger( tmp[0] );
if ( icol <= 0 ) {
CSVTHROW( "Field index must be greater than xero in " << cl.At(i) );
}
if ( tmp.size() == 1 && havenames ) {
CSVTHROW( "Must specify all column names" );
}
havenames = havenames || tmp.size() == 2;
string colname = tmp.size() == 2 ? tmp[1] : string("");
cols.push_back( SQLColSpec( icol - 1, colname ) );
}
}
int TrimCommand :: Execute( ALib::CommandLine & cmd ) {
GetSkipOptions( cmd );
if ( cmd.HasFlag( FLAG_TRLEAD ) || cmd.HasFlag( FLAG_TRTRAIL ) ) {
mTrimLead = cmd.HasFlag( FLAG_TRLEAD );
mTrimTrail = cmd.HasFlag( FLAG_TRTRAIL );
}
else {
mTrimLead = mTrimTrail = true;
}
if ( cmd.HasFlag( FLAG_WIDTH ) ) {
GetWidths( cmd.GetValue( FLAG_WIDTH ) );
}
ALib::CommaList cl( cmd.GetValue( FLAG_COLS, "" ) );
CommaListToIndex( cl, mFields );
IOManager io( cmd );
CSVRow row;
while( io.ReadCSV( row ) ) {
if ( Skip( row ) ) {
continue;
}
if ( ! Pass( row ) ) {
Trim( row );
}
io.WriteRow( row );
}
return 0;
}
void SQLCommand :: GetCommonValues( ALib::CommandLine & cmd ) {
GetSkipOptions( cmd );
mTable = cmd.GetValue( FLAG_TABLE );
if ( mTable == "" ) {
CSVTHROW( "Need table name specified by " << FLAG_TABLE << " flag" );
}
if ( ! cmd.HasFlag( FLAG_SQLSEP ) ) {
mSep ="\n;\n";
}
else {
mSep = ALib::UnEscape( cmd.GetValue( FLAG_SQLSEP ) );
}
string noq = cmd.GetValue( FLAG_NOQUOTE, "" );
if ( ! ALib::IsEmpty( noq ) ) {
CommaListToIndex( ALib::CommaList( noq ), mNoQuote );
}
mQuoteNulls = cmd.HasFlag( FLAG_QNULLS );
mEmptyNulls = cmd.HasFlag( FLAG_ENULLS );
}
static unsigned int GetField( const ALib::CommandLine & cmd,
const string & option ) {
string rc = cmd.GetValue( option );
if ( ! ALib::IsInteger( rc ) ) {
CSVTHROW( "Value for " << option << " must be integer" );
}
int n = ALib::ToInteger( rc );
if ( n <= 0 ) {
CSVTHROW( "Value for " << option << " must be greater than zero" );
}
return (unsigned int) n - 1;
}
void DateReadCommand :: ProcessFlags( ALib::CommandLine & cmd ) {
ALib::CommaList cl( cmd.GetValue( FLAG_COLS, "" ) );
CommaListToIndex( cl, mFields );
string mask = cmd.GetValue( FLAG_MASK, "" );
string cys = cmd.GetValue( FLAG_CDATE, ALib::Str( BASE_YEAR ) );
NotBoth( cmd, FLAG_BDLIST, FLAG_BDEXCL );
if ( cmd.HasFlag( FLAG_BDLIST ) ) {
mWriteAction = WriteBad;
}
else if ( cmd.HasFlag( FLAG_BDEXCL ) ) {
mWriteAction = WriteGood;
}
else {
mWriteAction = WriteAll;
}
if ( ! ALib::IsInteger( cys )) {
CSVTHROW( "Invalid year value " << cys );
}
int cy = ALib::ToInteger( cys );
string mnames = cmd.GetValue( FLAG_MNAMES, MONTH_NAMES );
delete mReader;
mReader = new MaskedDateReader( mask, mnames, cy );
}
int TruncPadBase :: Execute( ALib::CommandLine & cmd ) {
GetSkipOptions( cmd );
string ps = cmd.GetValue( FLAG_PAD );
ALib::CommaList padding( ps );
unsigned int ncols = padding.Size();
bool ncolspec = false; // use explicit size or not
if ( ncols == 0 || cmd.HasFlag( FLAG_NUM ) ) {
if ( ! cmd.HasFlag( FLAG_NUM ) ) {
CSVTHROW( "Need -n flag to specify field count" );
}
ncolspec = true;
string nv = cmd.GetValue( FLAG_NUM );
if ( ALib::ToInteger( nv, "-n flag needs integer value" ) < 0 ) {
CSVTHROW( FLAG_NUM << " needs value greater or equal to zero" );
}
ncols = ALib::ToInteger( nv );
}
IOManager io( cmd );
CSVRow row;
while( io.ReadCSV( row ) ) {
if ( Skip( io, row ) ) {
continue;
}
if ( ! Pass( io, row ) ) {
unsigned int nc = ncolspec ? ncols : row.size() + padding.Size();
ProcessRow( row, nc, padding );
}
io.WriteRow( row );
}
return 0;
}
int FileInfoCommand :: Execute( ALib::CommandLine & cmd ) {
GetSkipOptions( cmd );
mTwoCols = cmd.HasFlag( FLAG_TWOC );
mBasename = cmd.HasFlag( FLAG_BASEN );
IOManager io( cmd );
CSVRow row;
while( io.ReadCSV( row ) ) {
if ( Skip( io, row ) ) {
continue;
}
if ( Pass( io, row ) ) {
io.WriteRow( row );
continue;
}
string fname = mBasename
? ALib::Filename( io.CurrentFileName() ).Base()
: io.CurrentFileName();
CSVRow outrow;
if ( mTwoCols ) {
outrow.push_back( fname );
outrow.push_back( ALib::Str( io.CurrentLine() ));
}
else {
outrow.push_back( fname + " ("
+ ALib::Str( io.CurrentLine() ) + ")"
);
}
ALib::operator+=( outrow, row );
io.WriteRow( outrow );
}
return 0;
}
void ShuffleCommand :: ProcessFlags( const ALib::CommandLine & cmd ) {
string sn = cmd.GetValue( FLAG_NUM, ALib::Str(INT_MAX -1) );
if ( ! ALib::IsInteger( sn )) {
CSVTHROW( "Value for " << FLAG_NUM << " must be integeer" );
}
if ( ALib::ToInteger( sn ) < 0 ) {
CSVTHROW( "Value for " << FLAG_NUM << " must be zero or greater" );
}
mCount = ALib::ToInteger( sn );
string sr = cmd.GetValue( FLAG_RSEED, "-1" );
if ( ! ALib::IsInteger( sr ) ) {
CSVTHROW( "Value for " << FLAG_RSEED << " must be integer" );
}
int n = ALib::ToInteger( sr );
mSeed = n < 0 ? std::time(0) : n;
if ( cmd.HasFlag( FLAG_COLS ) ) {
ALib::CommaList cl( cmd.GetValue( FLAG_COLS ) );
CommaListToIndex( cl, mFields );
}
}
void ExcludeCommand :: ProcessFlags( const ALib::CommandLine & cmd ) {
NotBoth( cmd, FLAG_REVCOLS, FLAG_COLS, ReqOp::Required );
string es = cmd.GetValue( FLAG_IF, "" );
if ( es != "" ) {
string emsg = mExpr.Compile( es );
if ( emsg != "" ) {
CSVTHROW( emsg + " " + es );
}
}
mReverse = cmd.HasFlag( FLAG_REVCOLS );
string sn = cmd.GetValue( FLAG_COLS, "" );
if ( sn == "" ) {
sn = cmd.GetValue( FLAG_REVCOLS, "" );
}
CommaListToIndex( ALib::CommaList( sn ), mFields );
if ( mFields.size() == 0 ) {
CSVTHROW( "Field list specified by " << FLAG_COLS << " or "
<< FLAG_REVCOLS << " cannot be empty" );
}
}
void ValidateCommand :: GetOutputMode( const ALib::CommandLine & cl ) {
string om = cl.GetValue( FLAG_OMODE, "report" );
if ( om == "report" ) {
mOutMode = Reports;
}
else if ( om == "pass" ) {
mOutMode = Passes;
}
else if ( om == "fail" ) {
mOutMode = Fails;
}
else {
CSVTHROW( "Invalid value for " << FLAG_OMODE << ": " << om );
}
}
void DSVBase :: ReadFlags( const ALib::CommandLine & cl ) {
string delim = cl.GetValue( FLAG_SEP, ALib::Str( DEF_DELIM ) );
if ( delim.size() == 0 ) {
CSVTHROW( "Separator specified by " << FLAG_SEP
<< " cannot be empty" );
}
else if ( delim.size() == 1 ) {
mDelim = delim[0];
}
else if ( delim.size() == 2 && delim[0] == '\\' ) {
if ( delim[1] == 't' ) {
mDelim = '\t';
}
else {
CSVTHROW( "Bad escaped separator specified by " << FLAG_SEP );
}
}
else {
CSVTHROW( "Bad separator specified by " << FLAG_SEP );
}
string fields = cl.GetValue( FLAG_COLS, "" );
CommaListToIndex( ALib::CommaList( fields ), mFields );
}
void SummaryCommand :: ProcessFlags( const ALib::CommandLine & cmd ) {
int nf = CountNonGeneric( cmd );
if ( nf == 0 ) {
CSVTHROW( "Need a summary flag" );
}
else if ( nf != 1 ) {
CSVTHROW( "Only one summary flag allowed" );
}
if ( cmd.HasFlag( FLAG_AVG ) ) {
mType = Average;
GetFields( cmd, FLAG_AVG );
}
else if ( cmd.HasFlag( FLAG_MIN ) ) {
mType = Min;
GetFields( cmd, FLAG_MIN );
}
else if ( cmd.HasFlag( FLAG_MAX ) ) {
mType = Max;
GetFields( cmd, FLAG_MAX );
}
else if ( cmd.HasFlag( FLAG_FREQ ) ) {
mType = Frequency;
GetFields( cmd, FLAG_FREQ );
}
else if ( cmd.HasFlag( FLAG_MEDIAN ) ) {
mType = Median;
GetFields( cmd, FLAG_MEDIAN);
}
else if ( cmd.HasFlag( FLAG_MODE) ) {
mType = Mode;
GetFields( cmd, FLAG_MODE );
}
else if ( cmd.HasFlag( FLAG_SUM ) ) {
mType = Sum;
GetFields( cmd, FLAG_SUM );
}
else if ( cmd.HasFlag( FLAG_SIZE ) ) {
mType = Size;
}
else {
CSVTHROW( "Should never happen in SummaryCommand::ProcessFlags" );
}
}
PivotCommand::Action GetAction( const ALib::CommandLine & cmd ) {
string as = cmd.GetValue( FLAG_ACTION );
if ( as == OP_SUM ) {
return PivotCommand::Action::Sum;
}
else if ( as == OP_COUNT ) {
return PivotCommand::Action::Count;
}
else if ( as == OP_AVG ) {
return PivotCommand::Action::Average;
}
else {
CSVTHROW( "Invalid value for " << FLAG_ACTION << ". Need one of sum, count, avg" );
}
}
int ReadFixedCommand :: Execute( ALib::CommandLine & cmd ) {
mTrim = ! cmd.HasFlag( FLAG_KEEP );
BuildFields( cmd );
IOManager io( cmd );
string line;
CSVRow row;
while( io.ReadLine( line ) ) {
MakeRow( line, row );
io.WriteRow( row );
}
return 0;
}
int ValidateCommand :: Execute( ALib::CommandLine & cmd ) {
GetSkipOptions( cmd );
GetOutputMode( cmd );
ReadValidationFile( cmd );
IOManager io( cmd );
CSVRow row;
// we optionally return an error code to the shell if validation failed
int errtotal = 0;
bool errcode = cmd.HasFlag( FLAG_ERRCODE );
while( io.ReadCSV( row ) ) {
if ( Skip( io, row ) ) {
continue;
}
int errcount = 0;
for ( unsigned int i = 0; i < mRules.size(); i++ ) {
ValidationRule::Results res = mRules[i]->Apply( row );
if ( res.size() && mOutMode == Reports ) {
Report( io, res, errcount );
errcount++;
errtotal += errcount;
continue;
}
if ( res.size() ) {
errcount++;
if ( mOutMode == Fails ) {
io.WriteRow( row );
break;
}
}
}
if ( mOutMode == Passes && errcount == 0 ) {
io.WriteRow( row );
}
errtotal += errcount;
}
// exit code of 2 indicates program detected invalid data
return errtotal && errcode ? 2 : 0;
}
void RemoveNewlineCommand :: ProcessFlags( const ALib::CommandLine & cmd ) {
string scols = cmd.GetValue( FLAG_COLS, "" );
ALib::CommaList cl( cmd.GetValue( FLAG_COLS, "" ) );
CommaListToIndex( cl, mFields);
if ( cmd.HasFlag( FLAG_EXCLNL ) && cmd.HasFlag( FLAG_STR ) ) {
CSVTHROW( "Flags " << FLAG_EXCLNL << " and " << FLAG_STR
<< " are mutually exclusive" );
}
mSep = cmd.GetValue( FLAG_STR, "" );
ExpandSep();
mExcludeAfter = cmd.HasFlag( FLAG_EXCLNL );
}
void TemplateCommand :: ReadTemplate( const ALib::CommandLine & cmd ) {
string fname = cmd.GetValue( FLAG_TFILE );
if ( ALib::IsEmpty( fname ) ) {
CSVTHROW( "Need template file name specified with "
<< FLAG_TFILE << " flag" );
}
std::ifstream ifs( fname.c_str() );
if ( ! ifs.is_open() ) {
CSVTHROW( "Cannot open file " << fname << " for input" );
}
mTemplate = "";
string line;
while( std::getline( ifs, line ) ) {
mTemplate += line + "\n";
}
ifs.close();
}
int WriteFixedCommand :: Execute( ALib::CommandLine & cmd ) {
GetSkipOptions( cmd );
BuildFields( cmd );
IOManager io( cmd );
if ( cmd.HasFlag( FLAG_RULER ) ) {
io.Out() << Ruler() << "\n";
}
CSVRow row;
while( io.ReadCSV( row ) ) {
if ( Skip( row ) ) {
continue;
}
string line = MakeFixedOutput( row );
io.Out() << line << "\n";
}
return 0;
}
void ValidateCommand :: ReadValidationFile( const ALib::CommandLine & cmd ) {
Clear();
string fname = cmd.GetValue( FLAG_VFILE );
if ( fname == "" ) {
CSVTHROW( "Need validation file specified by "
<< FLAG_VFILE << " flag" );
}
std::ifstream ifs( fname.c_str() );
if ( ! ifs.is_open() ) {
CSVTHROW( "Cannot open validation file " << fname << " for input" );
}
string line;
while( std::getline( ifs, line ) ) {
if ( ALib::IsEmpty( line ) || line[0] == '#' ) { // a comment
continue;
}
unsigned int pos = 0;
string name = ReadName( line, pos );
FieldList flist = ReadFields( line, pos );
ValidationRule::Params params = ReadParams( line, pos );
ValidationRule * rule = RuleFactory::CreateRule( name, flist, params );
if ( rule == 0 ) {
CSVTHROW( "Unknown rule: " << name );
}
//rule->DumpOn( std::cout );
mRules.push_back( rule );
}
}
void SummaryCommand :: GetFields( const ALib::CommandLine & cmd,
const std::string & flag ) {
ALib::CommaList cl( cmd.GetValue( flag ) );
CommaListToIndex( cl, mFields );
}