int main( int ac, char** av )
{
try
{
comma::signal_flag is_shutdown(comma::signal_flag::hard);
comma::command_line_options options( ac, av, usage );
if( options.exists( "--bash-completion" )) bash_completion( ac, av );
options.assert_mutually_exclusive( "--by-lower,--by-upper,--nearest,--realtime" );
if( options.exists( "--by-upper" )) { method = how::by_upper; }
if( options.exists( "--nearest" )) { method = how::nearest; }
if( options.exists( "--realtime" )) { method = how::realtime; }
timestamp_only = options.exists( "--timestamp-only,--time-only" );
select_only = options.exists( "--do-not-append,--select" );
if( select_only && timestamp_only ) { std::cerr << "csv-time-join: --timestamp-only specified with --select, ignoring --timestamp-only" << std::endl; }
bool discard_bounding = options.exists( "--discard-bounding" );
boost::optional< unsigned int > buffer_size = options.optional< unsigned int >( "--buffer" );
if( options.exists( "--bound" ) ) { bound = boost::posix_time::microseconds( static_cast<unsigned int>(options.value< double >( "--bound" ) * 1000000 )); }
stdin_csv = comma::csv::options( options, "t" );
std::vector< std::string > unnamed = options.unnamed(
"--by-lower,--by-upper,--nearest,--realtime,--select,--do-not-append,--timestamp-only,--time-only,--discard-bounding",
"--binary,-b,--delimiter,-d,--fields,-f,--bound,--buffer,--verbose,-v" );
std::string properties;
bool stdin_first = true;
switch( unnamed.size() )
{
case 0:
std::cerr << "csv-time-join: please specify bounding source" << std::endl;
return 1;
case 1:
properties = unnamed[0];
break;
case 2:
if( unnamed[0] == "-" ) { properties = unnamed[1]; }
else if( unnamed[1] == "-" ) { properties = unnamed[0]; stdin_first = false; }
else { std::cerr << "csv-time-join: expected either '- <bounding>' or '<bounding> -'; got : " << comma::join( unnamed, ' ' ) << std::endl; return 1; }
break;
default:
std::cerr << "csv-time-join: expected either '- <bounding>' or '<bounding> -'; got : " << comma::join( unnamed, ' ' ) << std::endl;
return 1;
}
comma::name_value::parser parser( "filename" );
bounding_csv = parser.get< comma::csv::options >( properties );
if( bounding_csv.fields.empty() ) { bounding_csv.fields = "t"; }
comma::csv::input_stream< Point > stdin_stream( std::cin, stdin_csv );
#ifdef WIN32
if( stdin_csv.binary() ) { _setmode( _fileno( stdout ), _O_BINARY ); }
#endif // #ifdef WIN32
comma::io::istream bounding_istream( comma::split( properties, ';' )[0]
, bounding_csv.binary() ? comma::io::mode::binary : comma::io::mode::ascii );
comma::csv::input_stream< Point > bounding_stream( *bounding_istream, bounding_csv );
#ifndef WIN32
comma::io::select select;
comma::io::select bounding_stream_select;
select.read().add( 0 );
select.read().add( bounding_istream.fd() );
bounding_stream_select.read().add( bounding_istream.fd() );
#endif // #ifndef WIN32
const Point* p = NULL;
if( method == how::realtime )
{
#ifndef WIN32
bool end_of_input = false;
bool end_of_bounds = false;
boost::optional<timestring_t> joined_line;
while (!is_shutdown && !end_of_input)
{
if ( !bounding_stream.ready() && !stdin_stream.ready() )
{
select.wait(boost::posix_time::milliseconds(1));
}
if ( !is_shutdown && !end_of_input && ( stdin_stream.ready() || ( select.check() && select.read().ready( comma::io::stdin_fd ) ) ) )
{
p = stdin_stream.read();
if( p )
{
timestring_t input_line = std::make_pair( get_time( *p ), stdin_stream.last() );
if( joined_line ) { output( input_line, *joined_line, stdin_first ); }
}
else
{
comma::verbose << "end of input stream" << std::endl;
end_of_input = true;
}
}
if ( !is_shutdown && !end_of_bounds &&
( bounding_stream.ready() || ( select.check() && select.read().ready( bounding_istream.fd() ))))
{
p = bounding_stream.read();
if( p )
{
joined_line = std::make_pair( get_time( *p ), bounding_stream.last() );
}
else
{
comma::verbose << "end of bounding stream" << std::endl;
end_of_bounds = true;
}
}
}
if (is_shutdown) { comma::verbose << "got a signal" << std::endl; return 0; }
#else
COMMA_THROW(comma::exception, "--realtime mode not supported in WIN32");
#endif
}
else
{
std::deque<timestring_t> bounding_queue;
bool next = true;
bool bounding_data_available;
bool upper_bound_added = false;
// add a fake entry for an lower bound to allow stdin before first bound to match
bounding_queue.push_back( std::make_pair( boost::posix_time::neg_infin, "" ));
while( ( stdin_stream.ready() || ( std::cin.good() && !std::cin.eof() ) ) )
{
if( !std::cin.good() ) { select.read().remove( 0 ); }
if( !bounding_istream->good() ) { select.read().remove( bounding_istream.fd() ); }
bounding_data_available = bounding_stream.ready() || ( bounding_istream->good() && !bounding_istream->eof() );
#ifdef WIN32
bool bounding_stream_ready = true;
bool stdin_stream_ready = true;
#else // #ifdef WIN32
//check so we do not block
bool bounding_stream_ready = bounding_stream.ready();
bool stdin_stream_ready = stdin_stream.ready();
if( next )
{
if( !bounding_stream_ready || !stdin_stream_ready )
{
if( !bounding_stream_ready && !stdin_stream_ready )
{
select.wait( boost::posix_time::milliseconds(10) );
}
else
{
select.check();
}
if( select.read().ready( bounding_istream.fd() )) { bounding_stream_ready = true; }
if( select.read().ready(0) ) { stdin_stream_ready=true; }
}
}
else
{
if( !bounding_stream_ready )
{
bounding_stream_select.wait( boost::posix_time::milliseconds(10) );
if( bounding_stream_select.read().ready( bounding_istream.fd() )) { bounding_stream_ready=true; }
}
}
#endif //#ifdef WIN32
//keep storing available bounding data
if( bounding_stream_ready )
{
if( !buffer_size || bounding_queue.size() < *buffer_size || discard_bounding )
{
const Point* q = bounding_stream.read();
if( q )
{
bounding_queue.push_back( std::make_pair( get_time( *q ), bounding_stream.last() ));
}
else
{
bounding_data_available=false;
}
}
if( buffer_size && bounding_queue.size() > *buffer_size && discard_bounding )
{
bounding_queue.pop_front();
}
}
if( !upper_bound_added && bounding_istream->eof() )
{
// add a fake entry for an upper bound to allow stdin data above last bound to match
bounding_queue.push_back( std::make_pair( boost::posix_time::pos_infin, "" ));
upper_bound_added = true;
}
//if we are done with the last bounded point get next
if( next )
{
if(!stdin_stream_ready) { continue; }
p = stdin_stream.read();
if( !p ) { break; }
}
boost::posix_time::ptime t = get_time(*p);
//get bound
while(bounding_queue.size()>=2)
{
if( t < bounding_queue[1].first ) { break; }
bounding_queue.pop_front();
}
if(bounding_queue.size()<2)
{
//bound not found
//do we have more data?
if(!bounding_data_available) { break; }
next=false;
continue;
}
//bound available
if( method == how::by_lower && t < bounding_queue.front().first )
{
next = true;
continue;
}
bool is_first = ( method == how::by_lower )
|| ( method == how::nearest && ( t - bounding_queue[0].first ) < ( bounding_queue[1].first - t ));
const timestring_t& chosen_bound = is_first ? bounding_queue[0] : bounding_queue[1];;
timestring_t input_line = std::make_pair( t, stdin_stream.last() );
output( input_line, chosen_bound, stdin_first );
next=true;
}
}
return 0;
}
catch( std::exception& ex ) { std::cerr << "csv-time-join: " << ex.what() << std::endl; }
catch( ... ) { std::cerr << "csv-time-join: unknown exception" << std::endl; }
}
int main( int argc, char** argv )
{
comma::command_line_options options( argc, argv );
if( options.exists( "--help" ) || options.exists( "-h" ) || argc == 1 ) { usage(); }
offset=options.value("--error-margin",0.5);
bounds=comma::csv::ascii<bounds_t>().get(options.value("--bounds",std::string("0,0,0,0,0,0")));
bool output_all = options.exists( "--output-all");
std::vector<std::string> unnamed=options.unnamed("--output-all,--verbose,-v","-.*");
std::string operation=unnamed[0];
comma::csv::options csv(options);
csv.full_xpath=true;
bool flag_exists=false;
if( operation == "stream" )
{
std::vector<std::string> fields=comma::split(csv.fields,csv.delimiter);
if( csv.fields.empty() ) { csv.fields = "t,coordinates"; }
flag_exists = csv.has_field( "flag" );
std::string bounded_string("bounded/");
for(unsigned int i=0; i<fields.size(); i++)
{
if(fields[i].substr(0,bounded_string.size())!=bounded_string)
{
fields[i]=bounded_string+fields[i];
}
}
csv.fields=comma::join( fields, csv.delimiter );
}
else if( operation == "shape" )
{
}
else
{
std::cerr << "points-grep: expected operation, got: \"" << operation << "\"" << std::endl;
return 1;
}
flag_exists = csv.has_field( "bounded/flag" );
comma::csv::input_stream<joined_point> istream(std::cin,csv);
comma::csv::output_stream<joined_point> ostream(std::cout,csv);
comma::signal_flag is_shutdown;
joined_point pq;
if(operation=="stream")
{
if(unnamed.size()<2){ usage(); }
comma::name_value::parser parser( "filename" );
comma::csv::options bounding_csv = parser.get< comma::csv::options >( unnamed[1] ); // get stream options
comma::io::istream bounding_is( comma::split(unnamed[1],';')[0], bounding_csv.binary() ? comma::io::mode::binary : comma::io::mode::ascii ); // get stream name
//bounding stream
std::deque<bounding_point> bounding_queue;
comma::csv::input_stream<bounding_point> bounding_istream(*bounding_is, bounding_csv);
comma::io::select istream_select;
comma::io::select bounding_istream_select;
istream_select.read().add(0);
istream_select.read().add(bounding_is.fd());
bounding_istream_select.read().add(bounding_is.fd());
bool next=true;
while(!is_shutdown && ( istream.ready() || ( std::cin.good() && !std::cin.eof() ) ))
{
bool bounding_data_available = bounding_istream.ready() || ( bounding_is->good() && !bounding_is->eof());
//check so we do not block
bool bounding_istream_ready=bounding_istream.ready();
bool istream_ready=istream.ready();
if(next)
{
//only check istream if we need a new point
if(!bounding_istream_ready || !istream_ready)
{
if(!bounding_istream_ready && !istream_ready)
{
istream_select.wait(boost::posix_time::milliseconds(10));
}
else
{
istream_select.check();
}
if(istream_select.read().ready(bounding_is.fd()))
{
bounding_istream_ready=true;
}
if(istream_select.read().ready(0))
{
istream_ready=true;
}
}
}
else
{
if(!bounding_istream_ready)
{
bounding_istream_select.wait(boost::posix_time::milliseconds(10));
if(bounding_istream_select.read().ready(bounding_is.fd()))
{
bounding_istream_ready=true;
}
}
}
//keep storing available bounding data
if(bounding_istream_ready)
{
const bounding_point* q = bounding_istream.read();
if( q )
{
bounding_queue.push_back(*q);
}
else
{
bounding_data_available=false;
}
}
//if we are done with the last bounded point get next
if(next)
{
if(!istream_ready) { continue; }
const joined_point* pq_ptr = istream.read();
if( !pq_ptr ) { break; }
pq=*pq_ptr;
}
//get bound
while(bounding_queue.size()>=2)
{
if( pq.bounded.timestamp < bounding_queue[1].t ) { break; }
bounding_queue.pop_front();
}
if(bounding_queue.size()<2)
{
//bound not found
//do we have more data?
if(!bounding_data_available) { break; }
next=false;
continue;
}
//bound available
next=true; //get new point on next iteration
//discard late points
if(pq.bounded.timestamp < bounding_queue[0].t)
{
continue;
}
//match
bool is_first=( pq.bounded.timestamp - bounding_queue[0].t < bounding_queue[1].t - pq.bounded.timestamp );
pq.bounding = is_first ? bounding_queue[0] : bounding_queue[1]; // assign bounding point
//filter out object points
filter_point(pq);
if(!pq.bounded.flag && !output_all)
{
continue;
}
if(flag_exists)
{
ostream.write(pq);
ostream.flush();
continue;
}
//append flag
if(ostream.is_binary())
{
ostream.write(pq,istream.binary().last());
std::cout.write( reinterpret_cast< const char* >( &pq.bounded.flag ), sizeof( comma::uint32 ) );
}
else
{
std::string line=comma::join( istream.ascii().last(), csv.delimiter );
line+=","+boost::lexical_cast<std::string>(pq.bounded.flag);
ostream.write(pq,line);
}
ostream.flush();
}
}
else if(operation=="shape")
{
while(!is_shutdown && ( istream.ready() || ( std::cin.good() && !std::cin.eof() ) ))
{
const joined_point* pq_ptr = istream.read();
if( !pq_ptr ) { break; }
pq=*pq_ptr;
//filter out object points
filter_point(pq);
if(!pq.bounded.flag && !output_all)
{
continue;
}
if(flag_exists)
{
ostream.write(pq);
ostream.flush();
continue;
}
//append flag
if(ostream.is_binary())
{
ostream.write(pq,istream.binary().last());
std::cout.write( reinterpret_cast< const char* >( &pq.bounded.flag ), sizeof( comma::uint32 ) );
}
else
{
std::string line=comma::join( istream.ascii().last(), csv.delimiter );
line+=","+boost::lexical_cast<std::string>(pq.bounded.flag);
ostream.write(pq,line);
}
ostream.flush();
}
}
return(0);
}