static void calculate_distance_next()
{
comma::csv::input_stream< Eigen::Vector3d > istream( std::cin, csv );
boost::optional< Eigen::Vector3d > last;
while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
{
const Eigen::Vector3d* p = istream.read();
if( !p ) { break; }
if( last )
{
double distance = ( *p - *last ).norm();
if( csv.binary() ) { std::cout.write( reinterpret_cast< const char* >( &distance ), sizeof( double ) ); }
else { std::cout << csv.delimiter << distance << std::endl; }
}
if( csv.binary() ) { std::cout.write( istream.binary().last(), istream.binary().binary().format().size() ); }
else { std::cout << comma::join( istream.ascii().last(), csv.delimiter ); }
last = *p;
}
if( last )
{
double fake_final_distance = 0;
if( csv.binary() ) { std::cout.write( reinterpret_cast< const char* >( &fake_final_distance ), sizeof( double ) ); }
else { std::cout << csv.delimiter << fake_final_distance << std::endl; }
}
}
static void thin( double resolution )
{
comma::csv::input_stream< Eigen::Vector3d > istream( std::cin, csv );
boost::optional< Eigen::Vector3d > last;
double distance = 0;
while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
{
const Eigen::Vector3d* p = istream.read();
if( !p ) { break; }
distance += last ? ( *p - *last ).norm() : 0;
if( !last || distance >= resolution )
{
distance = 0;
if( csv.binary() )
{
std::cout.write( istream.binary().last(), istream.binary().binary().format().size() );
}
else
{
std::cout << comma::join( istream.ascii().last(), csv.delimiter ) << std::endl;
}
}
last = *p;
}
}
int main( int ac, char** av )
{
try
{
comma::command_line_options options( ac, av );
if( options.exists( "--help,-h,--long-help" ) ) { usage(); }
verbose = options.exists( "--verbose,-v" );
stdin_csv = comma::csv::options( options );
if( stdin_csv.binary() )
{
std::pair< entry_t, comma::csv::options > p = make_input_( stdin_csv );
input = p.first;
stdin_csv = p.second;
istream.reset( new comma::csv::input_stream< entry_t >( std::cin, stdin_csv, input ) );
if( verbose ) { std::cerr << "csv-calc: input fields: " << stdin_csv.fields << std::endl; } // todo: remove
}
std::vector< std::string > unnamed = options.unnamed( "--verbose,-v", "--binary,-b,--delimiter,-d,--fields,-f" );
if( unnamed.empty() ) { std::cerr << "csv-calc: please specify operations" << std::endl; return 1; }
std::vector< std::string > operation_strings = comma::split( unnamed[0], ',' );
return 0;
}
catch( std::exception& ex )
{
std::cerr << "csv-calc: " << ex.what() << std::endl;
}
catch( ... )
{
std::cerr << "csv-calc: unknown exception" << std::endl;
}
return 1;
}
static void output_bounding( std::ostream& os, const timestring_t& bounding, bool stdin_first )
{
if( !select_only )
{
if( stdin_csv.binary() )
{
if( timestamp_only )
{
static const unsigned int time_size = comma::csv::format::traits< boost::posix_time::ptime, comma::csv::format::time >::size;
static char timestamp[ time_size ];
comma::csv::format::traits< boost::posix_time::ptime, comma::csv::format::time >::to_bin( bounding.first, timestamp );
os.write( (char*)×tamp, time_size );
}
else
{
os.write( &bounding.second[0], bounding.second.size() );
}
}
else
{
if( stdin_first ) { os << stdin_csv.delimiter; }
os << ( timestamp_only ? boost::posix_time::to_iso_string( bounding.first ) : bounding.second );
if( !stdin_first ) { os << stdin_csv.delimiter; }
}
}
}
intervals( const comma::command_line_options& options ) : options( options )
, csv( options )
, ocsv( options )
, ascii_csv( options )
, empty( traits< bound_type >::cast( options.optional< std::string >( "--empty" ) ) )
, intervals_only( options.exists( "--intervals-only" ) )
, use_limits( options.exists( "--limits,-l" ) )
{
if( csv.fields.empty() ) { csv.fields = comma::join( comma::csv::names< interval_t< From, To > >(), ',' ); }
if( ocsv.fields.empty() || intervals_only )
{
ocsv.fields = comma::join( comma::csv::names< interval_t< From, To > >(), ',' );
if( ocsv.binary() && ( intervals_only || append ) ) { ocsv.format( comma::csv::format::value< interval_t< From, To > >() ); }
}
ascii_csv.fields = ocsv.fields;
ascii_csv.quote = boost::none;
if( verbose ) { std::cerr << app_name << ": empty: "; empty ? std::cerr << *empty : std::cerr << "<none>"; std::cerr << std::endl; }
options.assert_mutually_exclusive( "overlap-count-min,overlap-count-max", "overlap-count" );
if( options.exists( "--overlap-count" ) )
{
min_overlap_count = max_overlap_count = options.value< unsigned int >( "--overlap-count" );
}
else
{
min_overlap_count = options.value( "--overlap-count-min", 0 );
max_overlap_count = options.value( "--overlap-count-max", std::numeric_limits< unsigned int >::max() );
}
}
template < typename It > static void output_( It it, It end )
{
for( ; it != end; ++it )
{
for( std::size_t i = 0; i < it->second.size() ; ++i )
{
std::cout.write( &( it->second[i][0] ), stdin_csv.binary() ? stdin_csv.format().size() : it->second[i].length() );
if( stdin_csv.flush ) { std::cout.flush(); }
}
}
}
void output_points(const Eigen::Vector3d& p1, const Eigen::Vector3d& p2)
{
if( csv.binary() )
{
std::cout.write( reinterpret_cast< const char* >( &p1 ), sizeof( double ) * 3 );
std::cout.write( reinterpret_cast< const char* >( &p2 ), sizeof( double ) * 3 );
std::cout.flush();
}
else
{
std::cout << ascii.put( p1 ) << csv.delimiter << ascii.put( p2 ) << std::endl;
}
}
void run()
{
comma::csv::input_stream< interval_t< From, To > > istream( std::cin, csv );
comma::csv::ascii< interval_t< std::string > > ascii( csv.fields );
if( !first_line.empty() )
{
interval_t< From, To > interval = comma::csv::ascii< interval_t< From, To > >( csv.fields ).get( first_line );
bound_t< bound_type > from( LOWER );
bound_t< bound_type > to( UPPER );
std::string payload;
interval_t< std::string > first;
ascii.get( first, first_line );
if( !first.from.value.empty() && ( !empty || interval.from.value != *empty ) ) { from.value = interval.from.value; }
if( !first.to.value.empty() && ( !empty || interval.to.value != *empty ) ) { to.value = interval.to.value; }
payload = first_line;
if( !intervals_only ) { ascii.put( interval_t< std::string >(), payload ); } // blank out interval from payload
if( verbose ) { std::cerr << app_name << ": from: " << from << " to: " << to << " payload: " << payload << std::endl; }
add( from, to, payload );
}
while( istream.ready() || std::cin.good() )
{
const interval_t< From, To >* interval = istream.read();
if( !interval ) { break; }
bound_t< bound_type > from( LOWER );
bound_t< bound_type > to( UPPER );
std::string payload;
if( csv.binary() )
{
if( !empty || interval->from.value != *empty ) { from.value = interval->from.value; }
if( !empty || interval->to.value != *empty ) { to.value = interval->to.value; }
std::vector< char > buf( istream.binary().last(), istream.binary().last() + istream.binary().size() );
if( !intervals_only ) { istream.binary().binary().put( interval_t< From, To >(), &buf[0] ); } // blank out interval from payload
payload = std::string( buf.begin(), buf.end() );
}
else
{
interval_t< std::string > last;
ascii.get( last, istream.ascii().last() );
if( !last.from.value.empty() && ( !empty || interval->from.value != *empty ) ) { from.value = interval->from.value; }
if( !last.to.value.empty() && ( !empty || interval->to.value != *empty ) ) { to.value = interval->to.value; }
std::vector< std::string > buf( istream.ascii().last() );
if( !intervals_only ) { ascii.put( interval_t< std::string >(), buf ); } // blank out interval from payload
payload = comma::join( buf, csv.delimiter );
}
if( verbose ) { std::cerr << app_name << ": from: " << from << " to: " << to << " payload: " << ( csv.binary() ? "<binary>" : payload ) << std::endl; }
add( from, to, payload );
}
write();
}
static block_t* read_block_impl_( ::tbb::flow_control* flow = NULL )
{
static boost::array< block_t, 3 > blocks;
static boost::optional< block_t::pair_t > last;
static comma::uint32 block_id = 0;
block_t::pairs_t* points = new block_t::pairs_t;
while( true ) // quick and dirty, only if --discard
{
static comma::csv::input_stream< input_t > istream( std::cin, csv );
points->clear();
while( true )
{
if( last )
{
block_id = last->first.block;
points->push_back( *last );
last.reset();
}
if( is_shutdown || std::cout.bad() || std::cin.bad() || std::cin.eof() )
{
if( bursty_reader ) { bursty_reader->stop(); } // quick and dirty, it sucks...
if( flow ) { flow->stop(); }
break;
}
const input_t* p = istream.read();
if( !p ) { break; }
std::string line;
if( csv.binary() )
{
line.resize( csv.format().size() );
::memcpy( &line[0], istream.binary().last(), csv.format().size() );
}
else
{
line = comma::join( istream.ascii().last(), csv.delimiter );
}
last = std::make_pair( *p, line );
if( p->block != block_id ) { break; }
}
for( unsigned int i = 0; i < blocks.size(); ++i )
{
if( !blocks[i].empty ) { continue; }
blocks[i].clear();
blocks[i].id = block_id;
blocks[i].points.reset( points );
blocks[i].empty = false;
return &blocks[i];
}
}
}
void output_( const iterator_& c1, const iterator_& b, const iterator_& c3 )
{
const iterator_& a = reverse_ ? c3 : c1;
const iterator_& c = reverse_ ? c1 : c3;
if( csv_.binary() )
{
std::cout.write( &a->second[0], csv_.format().size() );
std::cout.write( &b->second[0], csv_.format().size() );
std::cout.write( &c->second[0], csv_.format().size() );
}
else
{
std::cout << a->second << csv_.delimiter << b->second << csv_.delimiter << c->second << std::endl;
}
}
stereo::stereo ( const snark::imaging::camera_parser& left, const snark::imaging::camera_parser& right, unsigned int width, unsigned int height, const comma::csv::options& csv, bool input_rectified ):
m_rotation( right.rotation() * left.rotation().transpose() ),
m_translation( right.translation() - left.translation() ),
m_rectify( left.camera(), left.distortion(), right.camera(), right.distortion(), width, height, m_rotation, m_translation, input_rectified ),
m_input_rectified( input_rectified ),
m_frame_counter( 0 )
{
if( csv.binary() )
{
m_binary.reset( new comma::csv::binary< colored_point >( csv ) );
m_output.resize( csv.format().size() );
}
else
{
m_ascii.reset( new comma::csv::ascii< colored_point >( csv ) );
}
}
Reader::Reader( QGLView& viewer, comma::csv::options& options, std::size_t size, coloured* c, unsigned int pointSize, const std::string& label, const QVector3D& offset )
: size( size )
, pointSize( pointSize )
, options( options )
, m_viewer( viewer )
, m_num_points( 0 )
, m_colored( c )
, m_shutdown( false )
, m_isStdIn( options.filename == "-" )
, m_show( true )
, m_istream( options.filename, options.binary() ? comma::io::mode::binary : comma::io::mode::ascii, comma::io::mode::non_blocking )
, updated_( false )
, id_( 0 )
, m_label( label )
, m_offset( offset )
{
std::vector< std::string > v = comma::split( options.fields, ',' ); // quick and dirty
}
intervals( const comma::command_line_options& options ) : options( options )
, csv( options )
, ocsv( options )
, ascii_csv( options )
, empty( traits< bound_type >::cast( options.optional< std::string >( "--empty" ) ) )
, intervals_only( options.exists( "--intervals-only" ) )
, use_limits( options.exists( "--limits,-l" ) )
{
if( csv.fields.empty() ) { csv.fields = comma::join( comma::csv::names< interval_t< From, To > >(), ',' ); }
if( ocsv.fields.empty() || intervals_only )
{
ocsv.fields = comma::join( comma::csv::names< interval_t< From, To > >(), ',' );
if( ocsv.binary() && intervals_only ) { ocsv.format( comma::csv::format::value< interval_t< From, To > >() ); }
}
ascii_csv.fields = ocsv.fields;
ascii_csv.quote = boost::none;
if( verbose ) { std::cerr << app_name << ": empty: "; empty ? std::cerr << *empty : std::cerr << "<none>"; std::cerr << std::endl; }
}
static void calculate_distance_for_pairs()
{
comma::csv::input_stream< point_pair_t > istream( std::cin, csv );
while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
{
const point_pair_t* p = istream.read();
if( !p ) { break; }
double norm = ( p->first - p->second ).norm();
if( csv.binary() )
{
std::cout.write( istream.binary().last(), istream.binary().binary().format().size() );
std::cout.write( reinterpret_cast< const char* >( &norm ), sizeof( double ) );
}
else
{
std::cout << comma::join( istream.ascii().last(), csv.delimiter ) << csv.delimiter << norm << std::endl;
}
}
}
frame::frame( const comma::csv::options& options, bool discardOutOfOrder, boost::optional< boost::posix_time::time_duration > maxGap, bool outputframe, bool to, bool interpolate, bool rotation_present )
: outputframe( outputframe )
, m_to( to )
, m_interpolate( interpolate )
, m_rotation( ::Eigen::Matrix3d::Identity() )
, m_istream( new comma::io::istream( options.filename, options.binary() ? comma::io::mode::binary : comma::io::mode::ascii, comma::io::mode::blocking ) )
, m_discardOutOfOrder( discardOutOfOrder )
, m_maxGap( maxGap )
, rotation_present_( rotation_present )
{
m_is.reset( new comma::csv::input_stream< position_type >( *( *m_istream )(), options ) );
const position_type* p = m_is->read(); // todo: maybe not very good, move to value()
if( p == NULL ) { COMMA_THROW( comma::exception, "failed to read from " << options.filename ); }
m_pair.first = *p;
set_position( p->value );
p = m_is->read(); // todo: maybe not very good, move to value()
if( p == NULL ) { COMMA_THROW( comma::exception, "failed to read from " << options.filename ); }
m_pair.second = *p;
}
stereo::stereo ( const camera_parser& left, const camera_parser& right,
const cv::Mat& left_x, const cv::Mat& left_y, const cv::Mat& right_x, const cv::Mat& right_y,
const comma::csv::options& csv, bool input_rectified ):
m_rotation( Eigen::Matrix3d::Identity() ),
m_translation( right.translation() - left.translation() ),
m_rectify( left.camera(), right.camera(), m_translation, left_x, left_y, right_x, right_y, input_rectified ),
m_input_rectified( input_rectified ),
m_frame_counter( 0 )
{
if( csv.binary() )
{
m_binary.reset( new comma::csv::binary< colored_point >( csv ) );
m_output.resize( csv.format().size() );
}
else
{
m_ascii.reset( new comma::csv::ascii< colored_point >( csv ) );
}
}
static void output( const timestring_t& input, const timestring_t& bounding, bool stdin_first )
{
if( bounding.first.is_infinity() ) { return; }
if( bound && ( input.first - bounding.first > bound || bounding.first - input.first > bound )) { return; }
if( stdin_first )
{
output_input( std::cout, input );
output_bounding( std::cout, bounding, stdin_first );
}
else
{
output_bounding( std::cout, bounding, stdin_first );
output_input( std::cout, input );
}
if( !stdin_csv.binary() ) { std::cout << '\n'; }
std::cout.flush();
}
void write()
{
static comma::csv::output_stream< interval_t< From, To > > ostream( std::cout, ocsv );
static comma::csv::ascii< from_t< std::string > > from_ascii( ascii_csv );
static comma::csv::ascii< to_t< std::string > > to_ascii( ascii_csv );
for( typename map_t::iterator it = map.begin(); it != map.end(); ++it )
{
bound_t< bound_type > from = it->first.lower();
bound_t< bound_type > to = it->first.upper();
interval_t< From, To > interval;
bool from_has_value = true;
bool to_has_value = true;
if( from.value ) { interval.from.value = *from.value; }
else if( use_limits ) { interval.from.value = limits< From >::lowest(); }
else if( empty ) { interval.from.value = static_cast< From >( *empty ); }
else { from_has_value = false; }
if( to.value ) { interval.to.value = *to.value; }
else if( use_limits ) { interval.to.value = limits< To >::max(); }
else if( empty ) { interval.to.value = static_cast< To >( *empty ); }
else { to_has_value = false; }
const set_t& s = it->second;
if( csv.binary() )
{
if( intervals_only ) { ostream.write( interval ); ostream.flush(); continue; }
for( typename set_t::const_iterator v = s.begin(); v != s.end(); ++v ) { ostream.write( interval, *v ); }
ostream.flush();
}
else
{
for( typename set_t::const_iterator v = s.begin(); v != s.end(); ++v )
{
std::string payload( intervals_only ? "" : *v );
ostream.ascii().ascii().put( interval, payload );
if( !from_has_value ) { from_ascii.put( from_t< std::string >(), payload ); }
if( !to_has_value ) { to_ascii.put( to_t< std::string >(), payload); }
std::cout << payload << std::endl;
if( intervals_only ) { break; }
}
}
}
}
static void write_block_( block_t* block )
{
if( !block ) { return; } // quick and dirty for now, only if --discard
for( std::size_t i = 0; i < block->points->size(); ++i )
{
const block_t::pair_t& p = block->points->operator[]( i );
if( ( p.first.foreground != foreground_t ) && !output_all ) { continue; }
comma::uint32 id = p.first.id;
comma::uint32 foreground = p.first.foreground;
std::cout.write( &p.second[0], p.second.size() );
if( csv.binary() )
{
std::cout.write( reinterpret_cast< const char* >( &id ), sizeof( comma::uint32 ) );
std::cout.write( reinterpret_cast< const char* >( &foreground ), sizeof( comma::uint32 ) );
}
else
{
std::cout << csv.delimiter << id << csv.delimiter << foreground << std::endl;
}
}
std::cout.flush();
block->clear();
}
static void calculate_distance( bool cumulative )
{
comma::csv::input_stream< Eigen::Vector3d > istream( std::cin, csv );
boost::optional< Eigen::Vector3d > last;
double distance = 0;
while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
{
const Eigen::Vector3d* p = istream.read();
if( !p ) { break; }
double norm = last ? ( *p - *last ).norm() : 0;
distance = cumulative ? distance + norm : norm;
last = *p;
if( csv.binary() )
{
std::cout.write( istream.binary().last(), istream.binary().binary().format().size() );
std::cout.write( reinterpret_cast< const char* >( &distance ), sizeof( double ) );
}
else
{
std::cout << comma::join( istream.ascii().last(), csv.delimiter ) << csv.delimiter << distance << std::endl;
}
}
}
int main( int ac, char** av )
{
try
{
comma::command_line_options options( ac, av );
if( options.exists( "--help,-h" ) ) { usage(); }
verbose = options.exists( "--verbose,-v" );
bool is_or = options.exists( "--or" );
csv = comma::csv::options( options );
fields = comma::split( csv.fields, ',' );
if( fields.size() == 1 && fields[0].empty() ) { fields.clear(); }
std::vector< std::string > unnamed = options.unnamed( "--first-matching,--or,--sorted,--input-sorted,--not-matching,--output-all,--all,--strict,--verbose,-v", "-.*" );
//for( unsigned int i = 0; i < unnamed.size(); constraints_map.insert( std::make_pair( comma::split( unnamed[i], ';' )[0], unnamed[i] ) ), ++i );
bool strict = options.exists( "--strict" );
bool first_matching = options.exists( "--first-matching" );
bool not_matching = options.exists( "--not-matching" );
bool all = options.exists( "--output-all,--all" );
for( unsigned int i = 0; i < unnamed.size(); ++i )
{
std::string field = comma::split( unnamed[i], ';' )[0];
bool found = false;
for( unsigned int j = 0; j < fields.size() && !found; found = field == fields[j], ++j );
if( !found )
{
if( strict ) { std::cerr << "csv-select: on constraint: \"" << unnamed[i] << "\" field \"" << field << "\" not found in fields: " << csv.fields << std::endl; return 1; }
std::cerr << "csv-select: warning: on constraint: \"" << unnamed[i] << "\" field \"" << field << "\" not found in fields: " << csv.fields << std::endl;
continue;
}
constraints_map.insert( std::make_pair( field, unnamed[i] ) );
}
comma::signal_flag is_shutdown;
if( csv.binary() )
{
#ifdef WIN32
_setmode( _fileno( stdout ), _O_BINARY );
#endif
init_input( csv.format(), options );
comma::csv::binary_input_stream< input_t > istream( std::cin, csv, input );
while( !is_shutdown && ( istream.ready() || ( std::cin.good() && !std::cin.eof() ) ) )
{
const input_t* p = istream.read();
if( !p || p->done( is_or ) ) { break; }
char match = ( p->is_a_match( is_or ) == !not_matching ) ? 1 : 0;
if( match || all )
{
std::cout.write( istream.last(), csv.format().size() );
if( all ) { std::cout.write( &match, 1 ); }
if( csv.flush ) { std::cout.flush(); }
if( first_matching ) { break; }
}
}
}
else
{
std::string line;
while( !is_shutdown && std::cin.good() && !std::cin.eof() )
{
std::getline( std::cin, line );
line = comma::strip( line, '\r' ); // windows, sigh...
if( !line.empty() ) { break; }
}
if( line.empty() ) { return 0; }
comma::csv::format format = options.exists( "--format" )
? comma::csv::format( options.value< std::string >( "--format" ) )
: guess_format( line );
init_input( format, options );
comma::csv::ascii_input_stream< input_t > istream( std::cin, csv, input );
// todo: quick and dirty: no time to debug why the commented section does not work (but that's the right way)
std::istringstream iss( line );
comma::csv::ascii_input_stream< input_t > isstream( iss, csv, input );
const input_t* p = isstream.read();
if( !p || p->done( is_or ) ) { return 0; }
bool match = p->is_a_match( is_or ) == !not_matching;
if( match || all )
{
std::cout << line;
if( all ) { std::cout << csv.delimiter << match; }
std::cout << std::endl;
if( first_matching ) { return 0; }
}
while( !is_shutdown && ( istream.ready() || ( std::cin.good() && !std::cin.eof() ) ) )
{
const input_t* p = istream.read();
if( !p || p->done( is_or ) ) { break; }
bool match = p->is_a_match( is_or ) == !not_matching;
if( match || all )
{
std::cout << comma::join( istream.last(), csv.delimiter );
if( all ) { std::cout << csv.delimiter << match; }
std::cout << std::endl;
if( first_matching ) { return 0; }
}
}
}
return 0;
}
catch( std::exception& ex ) { std::cerr << "csv-select: caught: " << ex.what() << std::endl; }
catch( ... ) { std::cerr << "csv-select: unknown exception" << std::endl; }
return 1;
}
CameraReader::CameraReader( comma::csv::options& options )
: options( options )
, m_shutdown( false )
, m_istream( options.filename, options.binary() ? comma::io::mode::binary : comma::io::mode::ascii )
{
}
void write()
{
static comma::csv::output_stream< interval_t< From, To > > ostream( std::cout, ocsv );
static comma::csv::ascii< from_t< std::string > > from_ascii( ascii_csv );
static comma::csv::ascii< to_t< std::string > > to_ascii( ascii_csv );
for( typename map_t::iterator it = map.begin(); it != map.end(); ++it )
{
bound_t< bound_type > from = it->first.lower();
bound_t< bound_type > to = it->first.upper();
interval_t< From, To > interval;
bool from_has_value = true;
bool to_has_value = true;
if( from.value ) { interval.from.value = *from.value; }
else if( use_limits ) { interval.from.value = limits< From >::lowest(); }
else if( empty ) { interval.from.value = static_cast< From >( *empty ); }
else { from_has_value = false; }
if( to.value ) { interval.to.value = *to.value; }
else if( use_limits ) { interval.to.value = limits< To >::max(); }
else if( empty ) { interval.to.value = static_cast< To >( *empty ); }
else { to_has_value = false; }
const set_t& s = it->second;
if( s.size() < min_overlap_count || s.size() > max_overlap_count ) { continue; }
if( append )
{
if( csv.binary() )
{
for( typename set_t::const_iterator v = s.begin(); v != s.end(); ++v )
{
std::cout.write( &( *v )[0], v->size() );
ostream.write( interval );
}
ostream.flush(); // todo: use csv.flush flag
}
else
{
//std::ostringstream oss;
//comma::csv::output_stream< interval_t< From, To > > osstream( oss ); // todo! quick and dirty, watch performance!
if( !from_has_value || !to_has_value ) { std::cerr << "csv-interval: support for empty from/to values for --append: todo" << std::endl; exit( 1 ); }
for( typename set_t::const_iterator v = s.begin(); v != s.end(); ++v )
{
std::cout << *v << csv.delimiter;
ostream.write( interval );
}
}
}
else
{
if( csv.binary() )
{
if( intervals_only ) { ostream.write( interval ); ostream.flush(); continue; }
for( typename set_t::const_iterator v = s.begin(); v != s.end(); ++v ) { ostream.write( interval, *v ); }
ostream.flush();
}
else
{
for( typename set_t::const_iterator v = s.begin(); v != s.end(); ++v )
{
std::string payload( intervals_only ? "" : *v );
ostream.ascii().ascii().put( interval, payload );
if( !from_has_value ) { from_ascii.put( from_t< std::string >(), payload ); }
if( !to_has_value ) { to_ascii.put( to_t< std::string >(), payload); }
std::cout << payload << std::endl;
if( intervals_only ) { break; }
}
}
}
}
}
int main( int ac, char** av )
{
try
{
comma::command_line_options options( ac, av );
verbose = options.exists( "--verbose,-v" );
if( options.exists( "--help,-h" ) ) { usage( verbose ); }
csv = comma::csv::options( options );
csv.full_xpath = true;
ascii = comma::csv::ascii< Eigen::Vector3d >( "x,y,z", csv.delimiter );
const std::vector< std::string >& operations = options.unnamed( "--verbose,-v,--trace,--no-antialiasing,--next,--unit", "-.*" );
if( operations.size() != 1 ) { std::cerr << "points-calc: expected one operation, got " << operations.size() << ": " << comma::join( operations, ' ' ) << std::endl; return 1; }
const std::string& operation = operations[0];
if (vector_calc::has_operation(operation))
{
vector_calc::process(operation, options, csv);
return 0;
}
if( operation == "plane-intersection" )
{
plane_intersection::process(options, csv);
return 0;
}
if( operation == "distance" )
{
if( csv.has_field( "first" ) || csv.has_field( "second" )
|| csv.has_field( "first/x" ) || csv.has_field( "second/x" )
|| csv.has_field( "first/y" ) || csv.has_field( "second/y" )
|| csv.has_field( "first/z" ) || csv.has_field( "second/z" ) )
{
calculate_distance_for_pairs();
return 0;
}
if ( options.exists( "--next" ) ) { calculate_distance_next(); }
else { calculate_distance( false ); }
return 0;
}
if( operation == "cumulative-distance" )
{
calculate_distance( true );
return 0;
}
if( operation == "nearest" )
{
if( options.exists( "--point,--to" ) )
{
Eigen::Vector3d point = comma::csv::ascii< Eigen::Vector3d >().get( options.value< std::string >( "--point,--to" ) );
comma::csv::input_stream< Eigen::Vector3d > istream( std::cin, csv );
std::string record;
double min_distance = std::numeric_limits< double >::max();
while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
{
const Eigen::Vector3d* p = istream.read();
if( !p ) { break; }
double d = ( *p - point ).norm();
if( d >= min_distance ) { continue; }
min_distance = d;
record = csv.binary() ? std::string( istream.binary().last(), csv.format().size() ) : comma::join( istream.ascii().last(), csv.delimiter );
}
if( !record.empty() ) { std::cout << record; }
if( csv.binary() ) { std::cout.write( reinterpret_cast< const char* >( &min_distance ), sizeof( double ) ); }
else { std::cout << csv.delimiter << min_distance << std::endl; }
return 0;
}
else
{
return 0;
}
}
if( operation == "thin" )
{
if( !options.exists( "--resolution" ) ) { std::cerr << "points-calc: --resolution is not specified " << std::endl; return 1; }
double resolution = options.value( "--resolution" , 0.0 );
thin( resolution );
return 0;
}
if( operation == "discretise" || operation == "discretize" )
{
if( !options.exists( "--step" ) ) { std::cerr << "points-calc: --step is not specified " << std::endl; return 1; }
double step = options.value( "--step" , 0.0 );
if( step <= 0 ) { std::cerr << "points-calc: expected positive step, got " << step << std::endl; return 1; }
// the last discretised point can be very close to the end of the interval, in which case the last two points can be identical in the output since ascii.put uses 12 digits by default
// setting --tolerance=1e-12 will not allow the last discretised point to be too close to the end of the interval and therefore the output will have two distinct points at the end
double tolerance = options.value( "--tolerance" , 0.0 );
if( tolerance < 0 ) { std::cerr << "points-calc: expected non-negative tolerance, got " << tolerance << std::endl; return 1; }
discretise( step, tolerance );
return 0;
}
if( operation == "local-max" || operation == "local-min" ) // todo: if( operation == "local-calc" ? )
{
double sign = operation == "local-max" ? 1 : -1;
if( csv.fields.empty() ) { csv.fields = "x,y,z,scalar"; }
csv.full_xpath = false;
bool has_id = csv.has_field( "id" );
comma::csv::input_stream< local_operation::point > istream( std::cin, csv );
std::deque< local_operation::record > records;
double radius = options.value< double >( "--radius" );
bool trace = options.exists( "--trace" );
Eigen::Vector3d resolution( radius, radius, radius );
snark::math::closed_interval< double, 3 > extents;
comma::uint32 id = 0;
if( verbose ) { std::cerr << "points-calc: reading input points..." << std::endl; }
while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
{
const local_operation::point* p = istream.read();
if( !p ) { break; }
std::string line;
if( csv.binary() ) // quick and dirty
{
line.resize( csv.format().size() );
::memcpy( &line[0], istream.binary().last(), csv.format().size() );
}
else
{
line = comma::join( istream.ascii().last(), csv.delimiter );
}
local_operation::point q = *p;
if( !has_id ) { q.id = id++; }
records.push_back( local_operation::record( q, line ) );
records.back().reference_record = &records.back();
extents.set_hull( p->coordinates );
}
if( verbose ) { std::cerr << "points-calc: loading " << records.size() << " points into grid..." << std::endl; }
typedef std::vector< local_operation::record* > voxel_t; // todo: is vector a good container? use deque
typedef snark::voxel_map< voxel_t, 3 > grid_t;
grid_t grid( extents.min(), resolution );
for( std::size_t i = 0; i < records.size(); ++i ) { ( grid.touch_at( records[i].point.coordinates ) )->second.push_back( &records[i] ); }
if( verbose ) { std::cerr << "points-calc: searching for local extrema..." << std::endl; }
for( grid_t::iterator it = grid.begin(); it != grid.end(); ++it )
{
grid_t::index_type i;
for( i[0] = it->first[0] - 1; i[0] < it->first[0] + 2; ++i[0] )
{
for( i[1] = it->first[1] - 1; i[1] < it->first[1] + 2; ++i[1] )
{
for( i[2] = it->first[2] - 1; i[2] < it->first[2] + 2; ++i[2] )
{
grid_t::iterator git = grid.find( i );
if( git == grid.end() ) { continue; }
for( voxel_t::iterator vit = it->second.begin(); vit != it->second.end(); ++vit )
{
for( std::size_t k = 0; k < git->second.size() && ( *vit )->is_extremum; ++k )
{
local_operation::evaluate_local_extremum( *vit, git->second[k], radius, sign );
}
}
}
}
}
}
#ifdef WIN32
_setmode( _fileno( stdout ), _O_BINARY );
#endif
if( verbose ) { std::cerr << "points-calc: filling extrema grid..." << std::endl; }
grid_t extrema( extents.min(), resolution );
for( std::size_t i = 0; i < records.size(); ++i )
{
if( records[i].is_extremum )
{
( extrema.touch_at( records[i].point.coordinates ) )->second.push_back( &records[i] );
}
else
{
records[i].extremum_id = local_operation::record::invalid_id; // quick and dirty for now
// if( records[i].extremum_id == local_operation::record::invalid_id ) { continue; }
// while( records[i].reference_record->point.id != records[i].reference_record->reference_record->point.id )
// {
// records[i].reference_record = records[i].reference_record->reference_record;
// }
// records[i].extremum_id = records[i].reference_record->point.id;
}
}
if( verbose ) { std::cerr << "points-calc: calculating distances to " << extrema.size() << " local extrema..." << std::endl; }
for( grid_t::iterator it = grid.begin(); it != grid.end(); ++it )
{
grid_t::index_type i;
for( i[0] = it->first[0] - 1; i[0] < it->first[0] + 2; ++i[0] )
{
for( i[1] = it->first[1] - 1; i[1] < it->first[1] + 2; ++i[1] )
{
for( i[2] = it->first[2] - 1; i[2] < it->first[2] + 2; ++i[2] )
{
grid_t::iterator git = extrema.find( i );
if( git == extrema.end() ) { continue; }
for( std::size_t n = 0; n < it->second.size(); ++n )
{
for( std::size_t k = 0; k < git->second.size(); ++k )
{
local_operation::update_nearest_extremum( it->second[n], git->second[k], radius );
}
}
}
}
}
}
if( trace )
{
if( verbose ) { std::cerr << "points-calc: tracing extrema..." << std::endl; }
for( std::size_t i = 0; i < records.size(); ++i )
{
if( records[i].extremum_id == local_operation::record::invalid_id ) { continue; }
while( records[i].reference_record->point.id != records[i].reference_record->reference_record->point.id )
{
records[i].reference_record = records[i].reference_record->reference_record;
}
}
}
if( verbose ) { std::cerr << "points-calc: outputting..." << std::endl; }
std::string endl = csv.binary() ? "" : "\n";
std::string delimiter = csv.binary() ? "" : std::string( 1, csv.delimiter );
comma::csv::options output_csv;
if( csv.binary() ) { output_csv.format( "ui,d" ); }
comma::csv::output_stream< local_operation::output > ostream( std::cout, output_csv );
for( std::size_t i = 0; i < records.size(); ++i )
{
std::cout.write( &records[i].line[0], records[i].line.size() );
std::cout.write( &delimiter[0], delimiter.size() );
ostream.write( records[i].output( false ) ); // quick and dirty
}
if( verbose ) { std::cerr << "points-calc: done!" << std::endl; }
return 0;
}
if( operation == "nearest-max" || operation == "nearest-min" || operation == "nearest-any" )
{
double sign = operation == "nearest-max" ? 1 : -1;
bool any = operation == "nearest-any";
if( csv.fields.empty() ) { csv.fields = "x,y,z,scalar"; }
csv.full_xpath = false;
bool has_id = csv.has_field( "id" );
comma::csv::input_stream< local_operation::point > istream( std::cin, csv );
std::deque< local_operation::record > records;
double radius = options.value< double >( "--radius" );
Eigen::Vector3d resolution( radius, radius, radius );
snark::math::closed_interval< double, 3 > extents;
comma::uint32 id = 0;
if( verbose ) { std::cerr << "points-calc: reading input points..." << std::endl; }
while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
{
const local_operation::point* p = istream.read();
if( !p ) { break; }
std::string line;
if( csv.binary() ) // quick and dirty
{
line.resize( csv.format().size() );
::memcpy( &line[0], istream.binary().last(), csv.format().size() );
}
else
{
line = comma::join( istream.ascii().last(), csv.delimiter );
}
local_operation::point q = *p;
if( !has_id ) { q.id = id++; }
records.push_back( local_operation::record( q, line ) );
records.back().reference_record = &records.back();
extents.set_hull( p->coordinates );
}
if( verbose ) { std::cerr << "points-calc: loading " << records.size() << " points into grid..." << std::endl; }
typedef std::vector< local_operation::record* > voxel_t; // todo: is vector a good container? use deque
typedef snark::voxel_map< voxel_t, 3 > grid_t;
grid_t grid( extents.min(), resolution );
for( std::size_t i = 0; i < records.size(); ++i ) { ( grid.touch_at( records[i].point.coordinates ) )->second.push_back( &records[i] ); }
if( verbose ) { std::cerr << "points-calc: searching for " << operation << "..." << std::endl; }
for( grid_t::iterator it = grid.begin(); it != grid.end(); ++it )
{
grid_t::index_type i;
for( i[0] = it->first[0] - 1; i[0] < it->first[0] + 2; ++i[0] )
{
for( i[1] = it->first[1] - 1; i[1] < it->first[1] + 2; ++i[1] )
{
for( i[2] = it->first[2] - 1; i[2] < it->first[2] + 2; ++i[2] )
{
grid_t::iterator git = grid.find( i );
if( git == grid.end() ) { continue; }
for( std::size_t n = 0; n < it->second.size(); ++n )
{
for( std::size_t k = 0; k < git->second.size(); ++k )
{
local_operation::update_nearest( it->second[n], git->second[k], radius, sign, any );
}
}
}
}
}
}
#ifdef WIN32
_setmode( _fileno( stdout ), _O_BINARY );
#endif
if( verbose ) { std::cerr << "points-calc: outputting..." << std::endl; }
std::string endl = csv.binary() ? "" : "\n";
std::string delimiter = csv.binary() ? "" : std::string( 1, csv.delimiter );
comma::csv::options output_csv;
if( csv.binary() ) { output_csv.format( "ui,d" ); }
comma::csv::output_stream< local_operation::output > ostream( std::cout, output_csv );
for( std::size_t i = 0; i < records.size(); ++i )
{
std::cout.write( &records[i].line[0], records[i].line.size() );
std::cout.write( &delimiter[0], delimiter.size() );
ostream.write( records[i].output() );
}
if( verbose ) { std::cerr << "points-calc: done!" << std::endl; }
return 0;
}
if( operation == "find-outliers" )
{
unsigned int size = options.value< unsigned int >( "--min-number-of-points-per-voxel,--size" );
double r = options.value< double >( "--resolution" );
Eigen::Vector3d resolution( r, r, r );
bool no_antialiasing = options.exists( "--no-antialiasing" );
comma::csv::input_stream< Eigen::Vector3d > istream( std::cin, csv );
std::deque< remove_outliers::record > records;
snark::math::closed_interval< double, 3 > extents;
if( verbose ) { std::cerr << "points-calc: reading input points..." << std::endl; }
while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
{
const Eigen::Vector3d* p = istream.read();
if( !p ) { break; }
std::string line;
if( csv.binary() ) // quick and dirty
{
line.resize( csv.format().size() );
::memcpy( &line[0], istream.binary().last(), csv.format().size() );
}
else
{
line = comma::join( istream.ascii().last(), csv.delimiter );
}
records.push_back( remove_outliers::record( *p, line ) );
extents.set_hull( *p );
}
if( verbose ) { std::cerr << "points-calc: loading " << records.size() << " points into grid..." << std::endl; }
typedef std::vector< remove_outliers::record* > voxel_t; // todo: is vector a good container? use deque
typedef snark::voxel_map< voxel_t, 3 > grid_t;
grid_t grid( extents.min(), resolution );
for( std::size_t i = 0; i < records.size(); ++i ) { ( grid.touch_at( records[i].point ) )->second.push_back( &records[i] ); }
if( verbose ) { std::cerr << "points-calc: removing outliers..." << std::endl; }
for( grid_t::iterator it = grid.begin(); it != grid.end(); ++it )
{
bool rejected = true;
if( no_antialiasing )
{
rejected = it->second.size() < size;
}
else
{
grid_t::index_type i;
for( i[0] = it->first[0] - 1; i[0] < it->first[0] + 2 && rejected; ++i[0] )
{
for( i[1] = it->first[1] - 1; i[1] < it->first[1] + 2 && rejected; ++i[1] )
{
for( i[2] = it->first[2] - 1; i[2] < it->first[2] + 2 && rejected; ++i[2] )
{
grid_t::iterator git = grid.find( i );
rejected = git == grid.end() || git->second.size() < size;
}
}
}
}
if( rejected ) { for( std::size_t i = 0; i < it->second.size(); ++i ) { it->second[i]->rejected = true; } }
}
#ifdef WIN32
_setmode( _fileno( stdout ), _O_BINARY );
#endif
if( verbose ) { std::cerr << "points-calc: outputting..." << std::endl; }
std::string endl = csv.binary() ? "" : "\n";
std::string delimiter = csv.binary() ? "" : std::string( 1, csv.delimiter );
for( std::size_t i = 0; i < records.size(); ++i )
{
char valid = records[i].rejected ? 0 : 1;
std::cout.write( &records[i].line[0], records[i].line.size() );
std::cout.write( &delimiter[0], delimiter.size() );
std::cout.write( &valid, 1 );
std::cout.write( &endl[0], endl.size() );
}
if( verbose ) { std::cerr << "points-calc: done!" << std::endl; }
return 0;
}
std::cerr << "points-calc: please specify an operation" << std::endl;
return 1;
}
catch( std::exception& ex )
{
std::cerr << "points-calc: " << ex.what() << std::endl;
}
catch( ... )
{
std::cerr << "points-calc: unknown exception" << std::endl;
}
return 1;
}
static void output_input( std::ostream& os, const timestring_t& input )
{
if( stdin_csv.binary() ) { os.write( &input.second[0], stdin_csv.format().size() ); }
else { os << input.second; }
}
int sort( const comma::command_line_options& options )
{
input_t::map sorted_map;
input_t default_input;
std::vector< std::string > v = comma::split( stdin_csv.fields, ',' );
std::vector< std::string > order = options.exists("--order") ? comma::split( options.value< std::string >( "--order" ), ',' ) : v;
std::vector< std::string > w (v.size());
bool unique = options.exists("--unique,-u");
std::string first_line;
comma::csv::format f;
if( stdin_csv.binary() ) { f = stdin_csv.format(); }
else if( options.exists( "--format" ) ) { f = comma::csv::format( options.value< std::string >( "--format" ) ); }
else
{
while( std::cin.good() && first_line.empty() ) { std::getline( std::cin, first_line ); }
if( first_line.empty() ) { return 0; }
f = comma::csv::impl::unstructured::guess_format( first_line, stdin_csv.delimiter );
if( verbose ) { std::cerr << "csv-sort: guessed format: " << f.string() << std::endl; }
}
for( std::size_t i = 0; i < order.size(); ++i ) // quick and dirty, wasteful, but who cares
{
if (order[i].empty()) continue;
for( std::size_t k = 0; k < v.size(); ++k )
{
if( v[k].empty() || v[k] != order[i] )
{
if ( k + 1 == v.size())
{
std::cerr << "csv-sort: order field name \"" << order[i] << "\" not found in input fields \"" << stdin_csv.fields << "\"" << std::endl;
return 1;
}
continue;
}
std::string type = default_input.keys.append( f.offset( k ).type );
w[k] = "keys/" + type;
ordering_t o;
if ( type[0] == 's' ) { o.type = ordering_t::str_type; o.index = default_input.keys.strings.size() - 1; }
else if ( type[0] == 'l' ) { o.type = ordering_t::long_type; o.index = default_input.keys.longs.size() - 1; }
else if ( type[0] == 'd' ) { o.type = ordering_t::double_type; o.index = default_input.keys.doubles.size() - 1; }
else if ( type[0] == 't' ) { o.type = ordering_t::time_type; o.index = default_input.keys.time.size() - 1; }
else { std::cerr << "csv-sort: cannot sort on field " << v[k] << " of type \"" << type << "\"" << std::endl; return 1; }
ordering.push_back(o);
break;
}
}
stdin_csv.fields = comma::join( w, ',' );
if ( verbose ) { std::cerr << "csv-sort: fields: " << stdin_csv.fields << std::endl; }
comma::csv::input_stream< input_t > stdin_stream( std::cin, stdin_csv, default_input );
#ifdef WIN32
if( stdin_stream.is_binary() ) { _setmode( _fileno( stdout ), _O_BINARY ); }
#endif
if (!first_line.empty())
{
input_t::map::mapped_type& d = sorted_map[ comma::csv::ascii< input_t >(stdin_csv,default_input).get(first_line) ];
d.push_back( first_line + "\n" );
}
while( stdin_stream.ready() || ( std::cin.good() && !std::cin.eof() ) )
{
const input_t* p = stdin_stream.read();
if( !p ) { break; }
if( stdin_stream.is_binary() )
{
input_t::map::mapped_type& d = sorted_map[ *p ];
if (unique && !d.empty()) continue;
d.push_back( std::string() );
d.back().resize( stdin_csv.format().size() );
::memcpy( &d.back()[0], stdin_stream.binary().last(), stdin_csv.format().size() );
}
else
{
input_t::map::mapped_type& d = sorted_map[ *p ];
if (unique && !d.empty()) continue;
d.push_back( comma::join( stdin_stream.ascii().last(), stdin_csv.delimiter ) + "\n" );
}
}
if( options.exists( "--reverse,-r" ) ) { output_( sorted_map.rbegin(), sorted_map.rend() ); }
else { output_( sorted_map.begin(), sorted_map.end() ); }
return 0;
}
int main( int ac, char** av )
{
try
{
comma::signal_flag is_shutdown(comma::signal_flag::hard);
comma::command_line_options options( ac, av, usage );
if( options.exists( "--bash-completion" )) bash_completion( ac, av );
options.assert_mutually_exclusive( "--by-lower,--by-upper,--nearest,--realtime" );
if( options.exists( "--by-upper" )) { method = how::by_upper; }
if( options.exists( "--nearest" )) { method = how::nearest; }
if( options.exists( "--realtime" )) { method = how::realtime; }
timestamp_only = options.exists( "--timestamp-only,--time-only" );
select_only = options.exists( "--do-not-append,--select" );
if( select_only && timestamp_only ) { std::cerr << "csv-time-join: --timestamp-only specified with --select, ignoring --timestamp-only" << std::endl; }
bool discard_bounding = options.exists( "--discard-bounding" );
boost::optional< unsigned int > buffer_size = options.optional< unsigned int >( "--buffer" );
if( options.exists( "--bound" ) ) { bound = boost::posix_time::microseconds( static_cast<unsigned int>(options.value< double >( "--bound" ) * 1000000 )); }
stdin_csv = comma::csv::options( options, "t" );
std::vector< std::string > unnamed = options.unnamed(
"--by-lower,--by-upper,--nearest,--realtime,--select,--do-not-append,--timestamp-only,--time-only,--discard-bounding",
"--binary,-b,--delimiter,-d,--fields,-f,--bound,--buffer,--verbose,-v" );
std::string properties;
bool stdin_first = true;
switch( unnamed.size() )
{
case 0:
std::cerr << "csv-time-join: please specify bounding source" << std::endl;
return 1;
case 1:
properties = unnamed[0];
break;
case 2:
if( unnamed[0] == "-" ) { properties = unnamed[1]; }
else if( unnamed[1] == "-" ) { properties = unnamed[0]; stdin_first = false; }
else { std::cerr << "csv-time-join: expected either '- <bounding>' or '<bounding> -'; got : " << comma::join( unnamed, ' ' ) << std::endl; return 1; }
break;
default:
std::cerr << "csv-time-join: expected either '- <bounding>' or '<bounding> -'; got : " << comma::join( unnamed, ' ' ) << std::endl;
return 1;
}
comma::name_value::parser parser( "filename" );
bounding_csv = parser.get< comma::csv::options >( properties );
if( bounding_csv.fields.empty() ) { bounding_csv.fields = "t"; }
comma::csv::input_stream< Point > stdin_stream( std::cin, stdin_csv );
#ifdef WIN32
if( stdin_csv.binary() ) { _setmode( _fileno( stdout ), _O_BINARY ); }
#endif // #ifdef WIN32
comma::io::istream bounding_istream( comma::split( properties, ';' )[0]
, bounding_csv.binary() ? comma::io::mode::binary : comma::io::mode::ascii );
comma::csv::input_stream< Point > bounding_stream( *bounding_istream, bounding_csv );
#ifndef WIN32
comma::io::select select;
comma::io::select bounding_stream_select;
select.read().add( 0 );
select.read().add( bounding_istream.fd() );
bounding_stream_select.read().add( bounding_istream.fd() );
#endif // #ifndef WIN32
const Point* p = NULL;
if( method == how::realtime )
{
#ifndef WIN32
bool end_of_input = false;
bool end_of_bounds = false;
boost::optional<timestring_t> joined_line;
while (!is_shutdown && !end_of_input)
{
if ( !bounding_stream.ready() && !stdin_stream.ready() )
{
select.wait(boost::posix_time::milliseconds(1));
}
if ( !is_shutdown && !end_of_input && ( stdin_stream.ready() || ( select.check() && select.read().ready( comma::io::stdin_fd ) ) ) )
{
p = stdin_stream.read();
if( p )
{
timestring_t input_line = std::make_pair( get_time( *p ), stdin_stream.last() );
if( joined_line ) { output( input_line, *joined_line, stdin_first ); }
}
else
{
comma::verbose << "end of input stream" << std::endl;
end_of_input = true;
}
}
if ( !is_shutdown && !end_of_bounds &&
( bounding_stream.ready() || ( select.check() && select.read().ready( bounding_istream.fd() ))))
{
p = bounding_stream.read();
if( p )
{
joined_line = std::make_pair( get_time( *p ), bounding_stream.last() );
}
else
{
comma::verbose << "end of bounding stream" << std::endl;
end_of_bounds = true;
}
}
}
if (is_shutdown) { comma::verbose << "got a signal" << std::endl; return 0; }
#else
COMMA_THROW(comma::exception, "--realtime mode not supported in WIN32");
#endif
}
else
{
std::deque<timestring_t> bounding_queue;
bool next = true;
bool bounding_data_available;
bool upper_bound_added = false;
// add a fake entry for an lower bound to allow stdin before first bound to match
bounding_queue.push_back( std::make_pair( boost::posix_time::neg_infin, "" ));
while( ( stdin_stream.ready() || ( std::cin.good() && !std::cin.eof() ) ) )
{
if( !std::cin.good() ) { select.read().remove( 0 ); }
if( !bounding_istream->good() ) { select.read().remove( bounding_istream.fd() ); }
bounding_data_available = bounding_stream.ready() || ( bounding_istream->good() && !bounding_istream->eof() );
#ifdef WIN32
bool bounding_stream_ready = true;
bool stdin_stream_ready = true;
#else // #ifdef WIN32
//check so we do not block
bool bounding_stream_ready = bounding_stream.ready();
bool stdin_stream_ready = stdin_stream.ready();
if( next )
{
if( !bounding_stream_ready || !stdin_stream_ready )
{
if( !bounding_stream_ready && !stdin_stream_ready )
{
select.wait( boost::posix_time::milliseconds(10) );
}
else
{
select.check();
}
if( select.read().ready( bounding_istream.fd() )) { bounding_stream_ready = true; }
if( select.read().ready(0) ) { stdin_stream_ready=true; }
}
}
else
{
if( !bounding_stream_ready )
{
bounding_stream_select.wait( boost::posix_time::milliseconds(10) );
if( bounding_stream_select.read().ready( bounding_istream.fd() )) { bounding_stream_ready=true; }
}
}
#endif //#ifdef WIN32
//keep storing available bounding data
if( bounding_stream_ready )
{
if( !buffer_size || bounding_queue.size() < *buffer_size || discard_bounding )
{
const Point* q = bounding_stream.read();
if( q )
{
bounding_queue.push_back( std::make_pair( get_time( *q ), bounding_stream.last() ));
}
else
{
bounding_data_available=false;
}
}
if( buffer_size && bounding_queue.size() > *buffer_size && discard_bounding )
{
bounding_queue.pop_front();
}
}
if( !upper_bound_added && bounding_istream->eof() )
{
// add a fake entry for an upper bound to allow stdin data above last bound to match
bounding_queue.push_back( std::make_pair( boost::posix_time::pos_infin, "" ));
upper_bound_added = true;
}
//if we are done with the last bounded point get next
if( next )
{
if(!stdin_stream_ready) { continue; }
p = stdin_stream.read();
if( !p ) { break; }
}
boost::posix_time::ptime t = get_time(*p);
//get bound
while(bounding_queue.size()>=2)
{
if( t < bounding_queue[1].first ) { break; }
bounding_queue.pop_front();
}
if(bounding_queue.size()<2)
{
//bound not found
//do we have more data?
if(!bounding_data_available) { break; }
next=false;
continue;
}
//bound available
if( method == how::by_lower && t < bounding_queue.front().first )
{
next = true;
continue;
}
bool is_first = ( method == how::by_lower )
|| ( method == how::nearest && ( t - bounding_queue[0].first ) < ( bounding_queue[1].first - t ));
const timestring_t& chosen_bound = is_first ? bounding_queue[0] : bounding_queue[1];;
timestring_t input_line = std::make_pair( t, stdin_stream.last() );
output( input_line, chosen_bound, stdin_first );
next=true;
}
}
return 0;
}
catch( std::exception& ex ) { std::cerr << "csv-time-join: " << ex.what() << std::endl; }
catch( ... ) { std::cerr << "csv-time-join: unknown exception" << std::endl; }
}
