Beispiel #1
0
void EvalCommand ::	Evaluate( CSVRow & row ) {

	bool skipelse = false;

	for ( unsigned int i = 0; i < mFieldExprs.size() ; i++ ) {
		if ( mIsIf[i] ) {
			if ( i < mFieldExprs.size() - 1  && mIsIf[i+2] ) {
				CSVTHROW( "Cannot have consecutive -if options" );
			}
			if ( i >= mFieldExprs.size() - 2 ) {
				CSVTHROW( "Need two -e options after -if" );
			}
			string r = mFieldExprs[i].mExpr.Evaluate();
			if ( ALib::Expression::ToBool( r ) ) {
				skipelse = true;
			} else {
				i++;
			}
			continue;
		}

		string r = mFieldExprs[i].mExpr.Evaluate();
		if ( mFieldExprs[i].mField < 0 || mFieldExprs[i].mField >= (int) row.size() ) {
			row.push_back( r );
		}
		else {
			row[ mFieldExprs[i].mField ] = r;
		}

		if ( skipelse ) {
			i++;
			skipelse = false;
		}
	}
}
Beispiel #2
0
int JoinCommand :: Execute( ALib::CommandLine & cmd ) {

	Clear();

    mKeep = cmd.HasFlag( FLAG_KEEP );
	mOuterJoin = cmd.HasFlag( FLAG_OUTERJ );
	mInvert = cmd.HasFlag( FLAG_INVERT );
	mIgnoreCase = cmd.HasFlag( FLAG_ICASE );
	if ( mOuterJoin && mInvert ) {
		CSVTHROW( "Cannot have both " << FLAG_OUTERJ
					<< " and " << FLAG_INVERT << " flags" );
	}
	string js = cmd.GetValue( FLAG_COLS );
	BuildJoinSpecs( js );

	IOManager io( cmd );
	unsigned int scount = io.InStreamCount();
	if (  scount < 2 ) {
		CSVTHROW( "Need at least two input streams" );
	}

	BuildRowMap( io.CreateStreamParser( scount - 1 ) );

	CSVRow row;
	for ( unsigned int i = 0; i < scount - 1; i++ ) {
		std::unique_ptr <ALib::CSVStreamParser> p( io.CreateStreamParser( i ) );
		while( p->ParseNext( row ) ) {
			WriteJoinRows( io, row );
		}
	}
	return 0;
}
Beispiel #3
0
void FixedCommand :: BuildFields( const ALib::CommandLine & cmd ) {

	mFields.clear();

	ALib::CommaList cl( cmd.GetValue( FLAG_COLS ) );  // chop into pairs

	if ( cl.Size() == 0 )  {
		CSVTHROW( "Need fields specified with " << FLAG_COLS << " flag" );
	}

	for ( unsigned int i = 0; i < cl.Size(); i++ ) {

		vector <string> tmp;
		if ( ALib::Split( cl.At(i), ':', tmp ) != 2 ) {
			CSVTHROW( "Invalid field specification: " << cl.At(i) );
		}

		if ( ! ALib::IsInteger( tmp[0] ) || ! ALib::IsInteger( tmp[1] ) ) {
 			CSVTHROW( "Invalid field specification: " << cl.At(i) );
		}

		unsigned int f1 = ALib::ToInteger( tmp[0] );
		unsigned int f2 = ALib::ToInteger( tmp[1] );
		if ( f1 == 0 || f2 == 0 ) {
			CSVTHROW( "Invalid field specification: " << cl.At(i) );
		}

		mFields.push_back( std::make_pair( f1, f2 ) );
	}
}
Beispiel #4
0
static unsigned int GetField( const ALib::CommandLine & cmd,
                                const string & option ) {
    string rc = cmd.GetValue( option );
    if ( ! ALib::IsInteger( rc ) ) {
        CSVTHROW( "Value for " << option << " must be integer" );
    }
    int n = ALib::ToInteger( rc );
    if ( n <= 0 ) {
        CSVTHROW( "Value for " << option << " must be greater than zero" );
    }
    return (unsigned int) n - 1;
}
Beispiel #5
0
void EvalCommand ::	GetExpressions( ALib::CommandLine & cmd ) {
	int i = 2;	// skip exe name and command name

	while( i < cmd.Argc() ) {
		if ( cmd.Argv( i ) == FLAG_EXPR || cmd.Argv( i ) == FLAG_IF ) {
			mIsIf.push_back( cmd.Argv( i ) == FLAG_IF );
			if ( i + 1 >= cmd.Argc() ) {
				CSVTHROW( "Missing expression" );
			}
			i++;
			string expr = cmd.Argv( i );
			ALib::Expression ex;
			string emsg = ex.Compile( expr );
			if ( emsg != "" ) {
				CSVTHROW( emsg + " in " + expr );
			}
			mFieldExprs.push_back( FieldEx( -1, ex ) );
		}
		else if ( cmd.Argv( i ) == FLAG_REMOVE ) {
			mIsIf.push_back( false );
			if ( mDiscardInput ) {
				CSVTHROW( "Cannot specify both " << FLAG_REMOVE
							<< " and " << FLAG_DISCARD );
			}
			if ( i + 1 >= cmd.Argc() ) {
				CSVTHROW( "Missimg field/expression" );
			}
			i++;
			string::size_type pos = cmd.Argv(i).find_first_of( "," );
			if ( pos == string::npos ) {
				CSVTHROW( "Invalid field/index pair: " << cmd.Argv(i) );
			}
			string field = cmd.Argv(i).substr( 0, pos );
			string expr = cmd.Argv(i).substr( pos + 1 );

			if ( ! ALib::IsInteger( field ) ) {
				CSVTHROW( "Invalid field (need integer): " << field );
			}
			int n = ALib::ToInteger( field );
			if ( n <= 0 ) {
				CSVTHROW( "Invalid field (must be greater than zero): " << field );
			}
			ALib::Expression ex;
			string emsg = ex.Compile( expr );
			if ( emsg != "" ) {
				CSVTHROW( emsg + " in " + expr );
			}

			mFieldExprs.push_back( FieldEx( n - 1, ex ) );
		}
		i++;
	}
	if ( mFieldExprs.size() == 0 ) {
		CSVTHROW( "Need at least one of -e or -r options" );
	}
}
Beispiel #6
0
int SummaryCommand :: Execute( ALib::CommandLine & cmd ) {

	ProcessFlags( cmd );
	IOManager io( cmd );

	CSVRow row;

	SizeMap sizemap;
	while( io.ReadCSV( row ) ) {
		if ( mType == Size ) {
			RecordSizes( row, sizemap );
		}
		else {
			mRows.push_back( row );
		}
	}


	if ( mType == Size ) {
		PrintSizes( io, sizemap );
	}
	else {
		if ( mRows.size() == 0 ) {
			CSVTHROW( "No input" );
		}
		Summarise( io );
	}
	return 0;
}
Beispiel #7
0
void DSVReadCommand :: ParseDSV( const string & line, CSVRow & rv ) {

    CSVRow row;

    unsigned int pos = 0, len = line.size();
    string val;

    while( pos < len ) {
        char c = line[ pos++ ];
        if ( c == Delim() ) {
            while( mCollapseSep && ALib::Peek( line, pos ) == Delim() ) {
                pos++;
            }
            row.push_back( Unquote( val ) );
            val = "";
        }
        else if ( c == '\\' ) {
            if ( pos == len ) {
                CSVTHROW( "Escape at end of line" );
            }
            else {
                val += line[ pos++ ];
            }
        }
        else {
            val += c;
        }
    }

    row.push_back( Unquote( val )  );

    BuildCSVRow( row, rv );

}
Beispiel #8
0
static char MustGet( const string & s, unsigned int  i ) {
	char c = ALib::Peek( s, i );
	if ( c == 0 ) {
		CSVTHROW( "Invalid value for " << FLAG_EDIT << ": " << s );
	}
	return c;
}
Beispiel #9
0
void DateReadCommand :: ProcessFlags( ALib::CommandLine & cmd ) {

    ALib::CommaList cl( cmd.GetValue( FLAG_COLS, "" ) );
    CommaListToIndex( cl, mFields );
    string mask = cmd.GetValue( FLAG_MASK, "" );
    string cys = cmd.GetValue( FLAG_CDATE, ALib::Str( BASE_YEAR ) );

    NotBoth( cmd, FLAG_BDLIST, FLAG_BDEXCL );

    if ( cmd.HasFlag( FLAG_BDLIST )  ) {
        mWriteAction = WriteBad;
    }
    else if ( cmd.HasFlag( FLAG_BDEXCL ) ) {
        mWriteAction = WriteGood;
    }
    else {
        mWriteAction = WriteAll;
    }

    if  ( ! ALib::IsInteger( cys )) {
        CSVTHROW( "Invalid year value " << cys );

    }
    int cy = ALib::ToInteger( cys );
    string mnames = cmd.GetValue( FLAG_MNAMES, MONTH_NAMES );
    delete mReader;
    mReader = new MaskedDateReader( mask, mnames, cy );
}
Beispiel #10
0
static string Format( const string & fmt, const ALib::Date & date ) {

    if ( fmt == "d" ) {
        return ALib::Str( date.Day() );
    }
    else if ( fmt == "dd" ) {
        return ALib::ZeroPad( date.Day(), 2 );
    }
    else if ( fmt == "m" ) {
        return ALib::Str( date.Month() );
    }
    else if ( fmt == "mm" ) {
        return ALib::ZeroPad( date.Month(), 2 );
    }
    else if ( fmt == "mmm" ) {
        return date.ShortMonthName();
    }
    else if ( fmt == "M" ) {
        return date.MonthName();
    }
    else if ( fmt == "y" || fmt == "yyyy" ) {
        return ALib::Str( date.Year() );
    }
    else if ( fmt == "W" ) {
        return date.DayName();
    }
    else if ( fmt == "w" ) {
        return date.ShortDayName();
    }
    else {
        CSVTHROW( "Invalid date format substring: "  << fmt );
    }
}
Beispiel #11
0
void CallCommand :: ProcessFlags( const ALib::CommandLine & cmd ) {

	string bs = cmd.GetValue( FLAG_BSIZE, ALib::Str( DEF_OUTBUF_SIZE ) );
	if ( ! ALib::IsInteger( bs )) {
		CSVTHROW( "Value for buffer size must be integer" );
	}
	mOutBufSize = ALib::ToInteger( bs ) * 1024;
	if ( mOutBufSize < DEF_OUTBUF_SIZE ) {
		CSVTHROW( "Output buffer size too small" );
	}
	mDLL = cmd.GetValue( FLAG_DLL );
	mFuncName = cmd.GetValue( FLAG_FUNC );
	ALib::CommaList cl( cmd.GetValue( FLAG_COLS ) );
	CommaListToIndex( cl, mFields );

}
Beispiel #12
0
string SQLUpdateCommand :: MakeSetClause( const CSVRow & row ) const {

	string sc;

	for ( unsigned int i = 0; i < DataCols().size(); i++ ) {

		unsigned int fi = DataCols().at( i ).mField;
		if ( fi >= row.size() ) {
			CSVTHROW( "Required field " << fi + 1 << " missing from input" );
		}

		if ( sc != "" ) {
			sc += ", ";
		}

		string field = EmptyToNull( row[fi] );

		sc +=  DataCols().at(i).mColName
				+ " = "
				+ (( DoSQLQuote( i ) && ! NoNullQuote( field ) )
					? ALib::SQuote( ALib::SQLQuote( field ) )
					: field );

	}

	return "SET " + sc;
}
Beispiel #13
0
string SQLCommand :: MakeWhereClause( const CSVRow & row  ) const {

	string wc;

	for ( unsigned int i = 0; i < WhereCols().size(); i++ ) {

		unsigned int wi = WhereCols().at( i ).mField;
		if ( wi >= row.size()  ) {
			CSVTHROW( "Required field " << wi + 1 << " missing in input" );
		}

		if ( wc != "" ) {
			wc += " AND ";
		}

		string field = EmptyToNull( row[wi] );

		wc +=  WhereCols().at(i).mColName
				+ (field == "NULL"  ? " IS " : " = " )
				+ ( (DoSQLQuote( i ) && ! NoNullQuote( field ))
					? ALib::SQuote( ALib::SQLQuote( field ) )
					: field );
	}

	return "WHERE " + wc;
}
Beispiel #14
0
void TemplateCommand :: FileOut( const CSVRow & row  ) {
	string fname = ReplaceColumns( mFileTemplate, row );
	std::ofstream ifs( fname );
	if ( ! ifs.is_open() ) {
		CSVTHROW( "Cannot open file " << fname << " for output" );
	}
	ifs << ReplaceColumns( mTemplate, row );
}
Beispiel #15
0
PivotCommand::ColRow PivotCommand:: MakeColRow( const CSVRow & row ) const {

    if ( mCol >= row.size() || mRow >= row.size() ) {
        CSVTHROW( "Invalid row/column index" );
    }

    return PivotCommand::ColRow( row[mCol], row[mRow] );
}
Beispiel #16
0
void SummaryCommand :: ProcessFlags( const ALib::CommandLine & cmd ) {

	int nf = CountNonGeneric( cmd );
	if ( nf == 0 ) {
		CSVTHROW( "Need a summary flag" );
	}
	else if ( nf != 1 ) {
		CSVTHROW( "Only one summary flag allowed" );
	}

	if ( cmd.HasFlag( FLAG_AVG ) ) {
		mType = Average;
		GetFields( cmd, FLAG_AVG );
	}
	else if ( cmd.HasFlag( FLAG_MIN ) ) {
		mType = Min;
		GetFields( cmd, FLAG_MIN );
	}
	else if ( cmd.HasFlag( FLAG_MAX ) ) {
		mType = Max;
		GetFields( cmd, FLAG_MAX );
	}
	else if ( cmd.HasFlag( FLAG_FREQ ) ) {
		mType = Frequency;
		GetFields( cmd, FLAG_FREQ );
	}
	else if ( cmd.HasFlag( FLAG_MEDIAN ) ) {
		mType = Median;
		GetFields( cmd, FLAG_MEDIAN);
	}
	else if ( cmd.HasFlag( FLAG_MODE) ) {
		mType = Mode;
		GetFields( cmd, FLAG_MODE );
	}
	else if ( cmd.HasFlag( FLAG_SUM ) ) {
		mType = Sum;
		GetFields( cmd, FLAG_SUM );
	}
	else if ( cmd.HasFlag( FLAG_SIZE ) ) {
		mType = Size;
	}
	else {
		CSVTHROW( "Should never happen in SummaryCommand::ProcessFlags" );
	}
}
Beispiel #17
0
void EditCommand :: AddSubCmd( const string & ev ) {
	if ( ALib::IsEmpty( ev ) ) {
		CSVTHROW( "Empty value for " << FLAG_EDIT );
	}

	if ( ! isalpha( ev[0] ) ) {			// for now
		CSVTHROW( "Edit sub command missing from " << ev );
	}

	char cmd;
	vector <string> tmp;
	ParseSub( ev, cmd, tmp );
//	for ( unsigned int i = 0; i < tmp.size(); i++ ) {
//		std::cout << "[" << tmp[i] << "]" << std::endl;
//	}

	mSubCmds.push_back ( EditSubCmd( cmd, tmp[0], tmp[1], tmp[2] ) );
}
Beispiel #18
0
void PivotCommand :: ProcessFlags( const ALib::CommandLine & cmd ) {
    mRow = GetField( cmd, FLAG_ROW );
    mCol = GetField( cmd, FLAG_COL );
    mFact = GetField( cmd, FLAG_FACT );
    if ( mRow == mCol ) {
        CSVTHROW( "Row and column options cannot have the same value" );
    }
    mAction = GetAction( cmd );
}
Beispiel #19
0
void DateFormatCommand :: ProcessFlags( ALib::CommandLine & cmd ) {
    ALib::CommaList cl( cmd.GetValue( FLAG_COLS, "" ) );
    CommaListToIndex( cl, mFields );
    string fmt = cmd.GetValue( FLAG_FMT, "" );
    if ( ALib::IsEmpty( fmt ) ) {
        CSVTHROW( "Empty date format" );
    }
    BuildFormat( fmt );
}
Beispiel #20
0
void TrimCommand ::	GetWidths( const std::string & ws ) {
    ALib::CommaList wl( ws );
    for ( unsigned int i = 0; i < wl.Size(); i++ ) {
        if ( ! ALib::IsInteger( wl.At(i) ) ) {
            CSVTHROW( "Invalid width " << wl.At(i) );
        }
        int n = ALib::ToInteger( wl.At(i) );
        mWidths.push_back( n );
    }
}
Beispiel #21
0
void TemplateCommand :: HandleSpecialChars( const string & tplate,
											char c, unsigned int & pos,
											const CSVRow & row,
											string & out ) {
	char t = tplate[ pos++ ];
	if ( c == '\\' ) {
		switch( t ) {
			case '\n':
			case '\r':	CSVTHROW( "Invalid escape at end of line" );
			case 'n':	out += '\n'; break;
			case 't':	out += '\t'; break;
			default:	out += t;
		}
	}
	else {		// its a brace
		string ns;
		while( t != POUTRO) {
			if ( t == '\n' || t == '\r' ) {
				CSVTHROW( "Missing closing brace" );
			}
			ns += t;
			t = tplate[ pos++ ];
		}

		if( ns.size() && ns[0] == EVALCHR ) {    // it's an expression
			out += Eval( row, ns );
			return;
		}

		if ( ! ALib::IsInteger( ns ) ) {
			CSVTHROW( "Invalid placeholder: " << "{" << ns << "}" );
		}

		int n =  ALib::ToInteger( ns ) - 1;		// to zero based
		if ( n < 0 ) {
			CSVTHROW( "Invalid placeholder: " << "{" << ns << "}" );
		}

		if ( (unsigned int) n < row.size() ) {
			out += row[ n ];
		}
	}
}
Beispiel #22
0
void TemplateCommand :: ReadTemplate( const ALib::CommandLine & cmd ) {
	string fname = cmd.GetValue( FLAG_TFILE );
	if ( ALib::IsEmpty( fname ) ) {
		CSVTHROW( "Need template file name specified with "
						<< FLAG_TFILE << " flag" );
	}
	std::ifstream ifs( fname.c_str() );
	if ( ! ifs.is_open() ) {
		CSVTHROW( "Cannot open file " << fname << "  for input" );
	}

	mTemplate = "";
	string line;
	while( std::getline( ifs, line ) ) {
		mTemplate += line + "\n";
	}

	ifs.close();
}
Beispiel #23
0
static void ParseSub( const string & s, char & cmd, vector <string> & fields ) {

	unsigned int i = 0;
	cmd = MustGet( s, i++ );
	if ( cmd != SUB_CMD ) {
		CSVTHROW( "Invalid value for " << FLAG_EDIT << ": " << s );
	}
	char sep = MustGet( s, i++ );
	if (  sep == '\\' ) {
		CSVTHROW( "Invalid value for " << FLAG_EDIT << ": " << s );
	}

	string f = ReadField( s, i, sep );
	fields.push_back( f );
	f = ReadField( s, i, sep );
	fields.push_back( f );
	f = s.substr( i );
	fields.push_back( f );
}
Beispiel #24
0
void JoinCommand :: BuildJoinSpecs( const string & js ) {

	if ( js == "" ) {
		CSVTHROW( "No join specified with " << FLAG_COLS << " flag" );
	}

	ALib::CommaList cl( js );
	for ( unsigned int i = 0; i < cl.Size(); i++ ) {
		vector <string> cols;
		if ( ALib::Split( cl.At(i), ':', cols ) != 2 ) {
			CSVTHROW( "Invalid join specification: " << cl.At(i) );
		}
		int c1 = ALib::ToInteger( cols[0], "Invalid column: " + cols[0] );
		int c2 = ALib::ToInteger( cols[1], "Invalid column: " + cols[1] );
		if ( c1 < 1 || c2 < 1 ) {
			CSVTHROW( "Invalid join specfication: " << cl.At(i) );
		}
		mJoinSpecs.push_back( std::make_pair( c1 - 1, c2 - 1 ) );
	}
}
Beispiel #25
0
void MaskedDateReader :: SetMonths( const string & months ) {
    if ( months == "" ) {
        mMonthNames = ALib::CommaList( MONTH_NAMES );
    }
    else {
        mMonthNames = ALib::CommaList( months );
    }
    if ( mMonthNames.Size() != 12 ) {
        CSVTHROW( "Invalid month list: " << months );
    }
}
Beispiel #26
0
static void SumRow( vector <double> & sums, const CSVRow & row,
						const FieldList & fl ) {

	for ( unsigned int i = 0; i < fl.size(); i++ ) {
		unsigned int fi = fl.at(i);
		if ( fi >= row.size() ) {
			CSVTHROW( "Invalid field index" );
		}
		sums.at(i) += ALib::ToReal( row.at(fi) );
	}
}
Beispiel #27
0
int CallCommand :: Execute( ALib::CommandLine & cmd ) {

	GetSkipOptions( cmd );
	ProcessFlags( cmd );
	std::vector <char> outbuf( mOutBufSize );

	HMODULE dll = LoadLibrary( mDLL.c_str() );
	if ( dll == NULL ) {
		CSVTHROW( "LoadLibrary call on " << mDLL << " failed" );
	}
	mFunc = (FuncType) GetProcAddress( dll, mFuncName.c_str() );
	if ( mFunc == NULL ) {
		CSVTHROW( "Cannot load function " << mFuncName << "from DLL " << mDLL );
	}

	IOManager io( cmd );
	CSVRow row;

	while( io.ReadCSV( row ) ) {

		if ( ! Skip( io, row ) ) {
			continue;
		}
		if ( Pass( io, row ) ) {
			io.WriteRow( row );
			continue;
		}

		int rv = CallOnFields( row, &outbuf[0] );
		if ( rv == 0 ) {
			io.WriteRow( row );
		}
		else if ( rv > 0 ) {
			CSVTHROW( mFuncName << " returned error code " << rv );
		}
		else {
			// do nothing - negative values just mean skip output
		}
	}
	return 0;
}
Beispiel #28
0
void PivotCommand :: AddFact( const ColRow & cr, const string & fact )  {
    if ( (mAction == Action::Average || mAction == Action::Sum) ) {
        if ( ! ALib::IsNumber( fact ) ) {
            CSVTHROW( "Non-numeric fact: " << fact);
        }
        double fd = ALib::ToReal( fact );
        mColRowValMap[cr].Update( fd );
    }
    else {
        mColRowValMap[cr].Update(1);
    }
}
Beispiel #29
0
int TruncPadBase :: Execute( ALib::CommandLine & cmd ) {

	GetSkipOptions( cmd );
	string ps = cmd.GetValue( FLAG_PAD );
	ALib::CommaList padding( ps );
	unsigned int ncols = padding.Size();

	bool ncolspec = false;	// use explicit size or not

	if ( ncols == 0 || cmd.HasFlag( FLAG_NUM ) ) {
		if ( ! cmd.HasFlag( FLAG_NUM ) ) {
			CSVTHROW( "Need -n flag to specify field count" );
		}
		ncolspec = true;
		string nv = cmd.GetValue( FLAG_NUM );
		if ( ALib::ToInteger( nv, "-n flag needs integer value" ) < 0 ) {
			CSVTHROW( FLAG_NUM << " needs value greater or equal to zero" );
		}
		ncols = ALib::ToInteger( nv );
	}

	IOManager io( cmd );
	CSVRow row;

	while( io.ReadCSV( row ) ) {

		if ( Skip( io, row ) ) {
			continue;
		}

		if ( ! Pass( io, row ) ) {
			unsigned int nc = ncolspec ? ncols : row.size() + padding.Size();
			ProcessRow( row, nc, padding );
		}

		io.WriteRow( row );
	}

	return 0;
}
Beispiel #30
0
void RemoveNewlineCommand :: ProcessFlags( const ALib::CommandLine & cmd ) {

	string scols = cmd.GetValue( FLAG_COLS, "" );
	ALib::CommaList cl( cmd.GetValue( FLAG_COLS, "" ) );
	CommaListToIndex( cl, mFields);
	if ( cmd.HasFlag( FLAG_EXCLNL ) && cmd.HasFlag( FLAG_STR ) ) {
		CSVTHROW( "Flags " << FLAG_EXCLNL << " and " << FLAG_STR
					<< " are mutually exclusive" );
 	}
	mSep = cmd.GetValue( FLAG_STR, "" );
	ExpandSep();
	mExcludeAfter = cmd.HasFlag( FLAG_EXCLNL );
}