int main( int ac, char* av[] )
{
try
{
comma::command_line_options options( ac, av, usage );
if( options.exists( "--bash-completion" ) ) bash_completion( ac, av );
options.assert_mutually_exclusive( "--from,--flush,--output-fields", "--to,--dimensions,--dim,--input-fields" );
comma::csv::options csv( options );
auto const type = options.value< std::string >( "--type" );
ros_execute( av, options );
std::cerr << comma::verbose.app_name() << ": unknown --type: " << type << std::endl;
}
catch( std::exception& ex ) { std::cerr << comma::verbose.app_name() << ": " << ex.what() << std::endl; }
catch( ... ) { std::cerr << comma::verbose.app_name() << ": unknown exception" << std::endl; }
return 1;
}
int main( int ac, char** av )
{
try
{
comma::command_line_options options( ac, av, usage );
if( options.exists( "--bash-completion" ) ) bash_completion( ac, av );
if( options.exists( "--output-fields" ))
{
if( options.exists( "--size" )) { std::cout << extended_output_fields << std::endl; }
else { std::cout << standard_output_fields << std::endl; }
return 0;
}
// Functionally equivalent to boost::optional< std::size_t > record_size
// but eliminates the gcc "maybe-uninitialized" warning
boost::optional< std::size_t > record_size = boost::make_optional< std::size_t >( false, 0 );
if( options.exists( "--size" )) { record_size = options.value< std::size_t >( "--size" ); }
boost::posix_time::time_duration update_interval = boost::posix_time::microseconds( options.value< double >( "--update,-u", default_update_interval ) * 1000000 );
double window = options.value< double >( "--window,-w", default_window );
double bucket_width = options.value< double >( "--resolution,-r", default_window_resolution );
boost::posix_time::time_duration bucket_duration = boost::posix_time::microseconds( bucket_width * 1000000 );
char delimiter = options.value( "--delimiter,-d", default_delimiter );
comma::io::select select;
select.read().add( comma::io::stdin_fd );
comma::io::istream is( "-", comma::io::mode::binary );
unsigned long long total_bytes = 0;
unsigned int bucket_bytes = 0;
boost::circular_buffer< unsigned int > window_buckets( std::ceil( window / bucket_width ));
boost::posix_time::ptime start_time = boost::posix_time::microsec_clock::universal_time();
boost::posix_time::ptime next_update = start_time + update_interval;
boost::posix_time::ptime next_bucket = start_time + bucket_duration;
comma::signal_flag is_shutdown;
bool end_of_stream = false;
boost::array< char, 65536 > buffer;
std::ios_base::sync_with_stdio( false ); // unsync to make rdbuf()->in_avail() working
std::cin.tie( NULL ); // std::cin is tied to std::cout by default
while( !is_shutdown && !end_of_stream )
{
select.wait( wait_interval );
while( is->rdbuf()->in_avail() || ( select.check() && select.read().ready( is.fd() ) && is->good() ) )
{
std::size_t available = is->rdbuf()->in_avail(); // std::size_t available = is.available_on_file_descriptor();
if( available == 0 ) { end_of_stream = true; break; }
std::size_t size = std::min( available, buffer.size() );
is->read( &buffer[0], size );
bucket_bytes += size;
total_bytes += size;
std::cout.write( &buffer[0], size );
std::cout.flush();
}
boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time();
if( now >= next_bucket )
{
window_buckets.push_back( bucket_bytes );
bucket_bytes = 0;
next_bucket += bucket_duration;
}
if( now >= next_update )
{
double elapsed_time = double( ( now - start_time ).total_milliseconds() ) / 1000.0f;
double bandwidth = (double)total_bytes / elapsed_time;
double window_bandwidth = (double)std::accumulate( window_buckets.begin()
, window_buckets.end()
, 0.0f )
/ window_buckets.size() / bucket_width;
std::cerr << boost::posix_time::to_iso_string( now )
<< std::fixed
<< delimiter << total_bytes
<< delimiter << bandwidth
<< delimiter << window_bandwidth;
std::cerr.unsetf( std::ios_base::floatfield );
if( record_size )
{
std::cerr << delimiter << bandwidth / *record_size
<< delimiter << window_bandwidth / *record_size;
}
std::cerr << std::endl;
next_update += update_interval;
}
}
return 0;
}
catch( std::exception& ex ) { std::cerr << "io-bandwidth: " << ex.what() << std::endl; }
catch( ... ) { std::cerr << "io-bandwidth: unknown exception" << std::endl; }
return 1;
}
int main( int ac, char** av )
{
try
{
comma::signal_flag is_shutdown(comma::signal_flag::hard);
comma::command_line_options options( ac, av, usage );
if( options.exists( "--bash-completion" )) bash_completion( ac, av );
options.assert_mutually_exclusive( "--by-lower,--by-upper,--nearest,--realtime" );
if( options.exists( "--by-upper" )) { method = how::by_upper; }
if( options.exists( "--nearest" )) { method = how::nearest; }
if( options.exists( "--realtime" )) { method = how::realtime; }
timestamp_only = options.exists( "--timestamp-only,--time-only" );
select_only = options.exists( "--do-not-append,--select" );
if( select_only && timestamp_only ) { std::cerr << "csv-time-join: --timestamp-only specified with --select, ignoring --timestamp-only" << std::endl; }
bool discard_bounding = options.exists( "--discard-bounding" );
boost::optional< unsigned int > buffer_size = options.optional< unsigned int >( "--buffer" );
if( options.exists( "--bound" ) ) { bound = boost::posix_time::microseconds( static_cast<unsigned int>(options.value< double >( "--bound" ) * 1000000 )); }
stdin_csv = comma::csv::options( options, "t" );
std::vector< std::string > unnamed = options.unnamed(
"--by-lower,--by-upper,--nearest,--realtime,--select,--do-not-append,--timestamp-only,--time-only,--discard-bounding",
"--binary,-b,--delimiter,-d,--fields,-f,--bound,--buffer,--verbose,-v" );
std::string properties;
bool stdin_first = true;
switch( unnamed.size() )
{
case 0:
std::cerr << "csv-time-join: please specify bounding source" << std::endl;
return 1;
case 1:
properties = unnamed[0];
break;
case 2:
if( unnamed[0] == "-" ) { properties = unnamed[1]; }
else if( unnamed[1] == "-" ) { properties = unnamed[0]; stdin_first = false; }
else { std::cerr << "csv-time-join: expected either '- <bounding>' or '<bounding> -'; got : " << comma::join( unnamed, ' ' ) << std::endl; return 1; }
break;
default:
std::cerr << "csv-time-join: expected either '- <bounding>' or '<bounding> -'; got : " << comma::join( unnamed, ' ' ) << std::endl;
return 1;
}
comma::name_value::parser parser( "filename" );
bounding_csv = parser.get< comma::csv::options >( properties );
if( bounding_csv.fields.empty() ) { bounding_csv.fields = "t"; }
comma::csv::input_stream< Point > stdin_stream( std::cin, stdin_csv );
#ifdef WIN32
if( stdin_csv.binary() ) { _setmode( _fileno( stdout ), _O_BINARY ); }
#endif // #ifdef WIN32
comma::io::istream bounding_istream( comma::split( properties, ';' )[0]
, bounding_csv.binary() ? comma::io::mode::binary : comma::io::mode::ascii );
comma::csv::input_stream< Point > bounding_stream( *bounding_istream, bounding_csv );
#ifndef WIN32
comma::io::select select;
comma::io::select bounding_stream_select;
select.read().add( 0 );
select.read().add( bounding_istream.fd() );
bounding_stream_select.read().add( bounding_istream.fd() );
#endif // #ifndef WIN32
const Point* p = NULL;
if( method == how::realtime )
{
#ifndef WIN32
bool end_of_input = false;
bool end_of_bounds = false;
boost::optional<timestring_t> joined_line;
while (!is_shutdown && !end_of_input)
{
if ( !bounding_stream.ready() && !stdin_stream.ready() )
{
select.wait(boost::posix_time::milliseconds(1));
}
if ( !is_shutdown && !end_of_input && ( stdin_stream.ready() || ( select.check() && select.read().ready( comma::io::stdin_fd ) ) ) )
{
p = stdin_stream.read();
if( p )
{
timestring_t input_line = std::make_pair( get_time( *p ), stdin_stream.last() );
if( joined_line ) { output( input_line, *joined_line, stdin_first ); }
}
else
{
comma::verbose << "end of input stream" << std::endl;
end_of_input = true;
}
}
if ( !is_shutdown && !end_of_bounds &&
( bounding_stream.ready() || ( select.check() && select.read().ready( bounding_istream.fd() ))))
{
p = bounding_stream.read();
if( p )
{
joined_line = std::make_pair( get_time( *p ), bounding_stream.last() );
}
else
{
comma::verbose << "end of bounding stream" << std::endl;
end_of_bounds = true;
}
}
}
if (is_shutdown) { comma::verbose << "got a signal" << std::endl; return 0; }
#else
COMMA_THROW(comma::exception, "--realtime mode not supported in WIN32");
#endif
}
else
{
std::deque<timestring_t> bounding_queue;
bool next = true;
bool bounding_data_available;
bool upper_bound_added = false;
// add a fake entry for an lower bound to allow stdin before first bound to match
bounding_queue.push_back( std::make_pair( boost::posix_time::neg_infin, "" ));
while( ( stdin_stream.ready() || ( std::cin.good() && !std::cin.eof() ) ) )
{
if( !std::cin.good() ) { select.read().remove( 0 ); }
if( !bounding_istream->good() ) { select.read().remove( bounding_istream.fd() ); }
bounding_data_available = bounding_stream.ready() || ( bounding_istream->good() && !bounding_istream->eof() );
#ifdef WIN32
bool bounding_stream_ready = true;
bool stdin_stream_ready = true;
#else // #ifdef WIN32
//check so we do not block
bool bounding_stream_ready = bounding_stream.ready();
bool stdin_stream_ready = stdin_stream.ready();
if( next )
{
if( !bounding_stream_ready || !stdin_stream_ready )
{
if( !bounding_stream_ready && !stdin_stream_ready )
{
select.wait( boost::posix_time::milliseconds(10) );
}
else
{
select.check();
}
if( select.read().ready( bounding_istream.fd() )) { bounding_stream_ready = true; }
if( select.read().ready(0) ) { stdin_stream_ready=true; }
}
}
else
{
if( !bounding_stream_ready )
{
bounding_stream_select.wait( boost::posix_time::milliseconds(10) );
if( bounding_stream_select.read().ready( bounding_istream.fd() )) { bounding_stream_ready=true; }
}
}
#endif //#ifdef WIN32
//keep storing available bounding data
if( bounding_stream_ready )
{
if( !buffer_size || bounding_queue.size() < *buffer_size || discard_bounding )
{
const Point* q = bounding_stream.read();
if( q )
{
bounding_queue.push_back( std::make_pair( get_time( *q ), bounding_stream.last() ));
}
else
{
bounding_data_available=false;
}
}
if( buffer_size && bounding_queue.size() > *buffer_size && discard_bounding )
{
bounding_queue.pop_front();
}
}
if( !upper_bound_added && bounding_istream->eof() )
{
// add a fake entry for an upper bound to allow stdin data above last bound to match
bounding_queue.push_back( std::make_pair( boost::posix_time::pos_infin, "" ));
upper_bound_added = true;
}
//if we are done with the last bounded point get next
if( next )
{
if(!stdin_stream_ready) { continue; }
p = stdin_stream.read();
if( !p ) { break; }
}
boost::posix_time::ptime t = get_time(*p);
//get bound
while(bounding_queue.size()>=2)
{
if( t < bounding_queue[1].first ) { break; }
bounding_queue.pop_front();
}
if(bounding_queue.size()<2)
{
//bound not found
//do we have more data?
if(!bounding_data_available) { break; }
next=false;
continue;
}
//bound available
if( method == how::by_lower && t < bounding_queue.front().first )
{
next = true;
continue;
}
bool is_first = ( method == how::by_lower )
|| ( method == how::nearest && ( t - bounding_queue[0].first ) < ( bounding_queue[1].first - t ));
const timestring_t& chosen_bound = is_first ? bounding_queue[0] : bounding_queue[1];;
timestring_t input_line = std::make_pair( t, stdin_stream.last() );
output( input_line, chosen_bound, stdin_first );
next=true;
}
}
return 0;
}
catch( std::exception& ex ) { std::cerr << "csv-time-join: " << ex.what() << std::endl; }
catch( ... ) { std::cerr << "csv-time-join: unknown exception" << std::endl; }
}
