Ejemplo n.º 1
0
static void calculate_distance_next()
{
    comma::csv::input_stream< Eigen::Vector3d > istream( std::cin, csv );
    boost::optional< Eigen::Vector3d > last;
    while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
    {
        const Eigen::Vector3d* p = istream.read();
        if( !p ) { break; }
        if( last )
        {
            double distance = ( *p - *last ).norm();
            if( csv.binary() ) { std::cout.write( reinterpret_cast< const char* >( &distance ), sizeof( double ) ); }
            else { std::cout << csv.delimiter << distance << std::endl; }
        }
        if( csv.binary() ) { std::cout.write( istream.binary().last(), istream.binary().binary().format().size() ); }
        else { std::cout << comma::join( istream.ascii().last(), csv.delimiter ); }
        last = *p;
    }
    if( last )
    {
        double fake_final_distance = 0;
        if( csv.binary() ) { std::cout.write( reinterpret_cast< const char* >( &fake_final_distance ), sizeof( double ) ); }
        else { std::cout << csv.delimiter << fake_final_distance << std::endl; }
    }
}
Ejemplo n.º 2
0
static void thin( double resolution )
{
    comma::csv::input_stream< Eigen::Vector3d > istream( std::cin, csv );
    boost::optional< Eigen::Vector3d > last;
    double distance = 0;
    while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
    {
        const Eigen::Vector3d* p = istream.read();
        if( !p ) { break; }
        distance += last ? ( *p - *last ).norm() : 0;
        if( !last || distance >= resolution )
        {
            distance = 0;
            if( csv.binary() )
            {
                std::cout.write( istream.binary().last(), istream.binary().binary().format().size() );
            }
            else
            {
                std::cout << comma::join( istream.ascii().last(), csv.delimiter ) << std::endl;
            }
        }
        last = *p;
    }
}
Ejemplo n.º 3
0
int main( int ac, char** av )
{
    try
    {
        comma::command_line_options options( ac, av );
        if( options.exists( "--help,-h,--long-help" ) ) { usage(); }
        verbose = options.exists( "--verbose,-v" );
        stdin_csv = comma::csv::options( options );
        if( stdin_csv.binary() )
        {
            std::pair< entry_t, comma::csv::options > p = make_input_( stdin_csv );
            input = p.first;
            stdin_csv = p.second;
            istream.reset( new comma::csv::input_stream< entry_t >( std::cin, stdin_csv, input ) );
            if( verbose ) { std::cerr << "csv-calc: input fields: " << stdin_csv.fields << std::endl; } // todo: remove
        }
        std::vector< std::string > unnamed = options.unnamed( "--verbose,-v", "--binary,-b,--delimiter,-d,--fields,-f" );
        if( unnamed.empty() ) { std::cerr << "csv-calc: please specify operations" << std::endl; return 1; }
        std::vector< std::string > operation_strings = comma::split( unnamed[0], ',' );
        
        
        
        return 0;
    }
    catch( std::exception& ex )
    {
        std::cerr << "csv-calc: " << ex.what() << std::endl;
    }
    catch( ... )
    {
        std::cerr << "csv-calc: unknown exception" << std::endl;
    }
    return 1;
}
Ejemplo n.º 4
0
static void output_bounding( std::ostream& os, const timestring_t& bounding, bool stdin_first )
{
    if( !select_only )
    {
        if( stdin_csv.binary() )
        {
            if( timestamp_only )
            {
                static const unsigned int time_size = comma::csv::format::traits< boost::posix_time::ptime, comma::csv::format::time >::size;
                static char timestamp[ time_size ];
                comma::csv::format::traits< boost::posix_time::ptime, comma::csv::format::time >::to_bin( bounding.first, timestamp );
                os.write( (char*)&timestamp, time_size );
            }
            else
            {
                os.write( &bounding.second[0], bounding.second.size() );
            }
        }
        else
        {
            if( stdin_first ) { os << stdin_csv.delimiter; }
            os << ( timestamp_only ? boost::posix_time::to_iso_string( bounding.first ) : bounding.second );
            if( !stdin_first ) { os << stdin_csv.delimiter; }
        }
    }
}
Ejemplo n.º 5
0
 intervals( const comma::command_line_options& options ) : options( options )
                                                         , csv( options )
                                                         , ocsv( options )
                                                         , ascii_csv( options )
                                                         , empty( traits< bound_type >::cast( options.optional< std::string >( "--empty" ) ) )
                                                         , intervals_only( options.exists( "--intervals-only" ) )
                                                         , use_limits( options.exists( "--limits,-l" ) )
 {
     if( csv.fields.empty() ) { csv.fields = comma::join( comma::csv::names< interval_t< From, To > >(), ',' ); }
     if( ocsv.fields.empty() || intervals_only )
     {
         ocsv.fields = comma::join( comma::csv::names< interval_t< From, To > >(), ',' );
         if( ocsv.binary() && ( intervals_only || append ) ) { ocsv.format( comma::csv::format::value< interval_t< From, To > >() ); }
     }
     ascii_csv.fields = ocsv.fields;
     ascii_csv.quote = boost::none;
     if( verbose ) { std::cerr << app_name << ": empty: "; empty ? std::cerr << *empty : std::cerr << "<none>"; std::cerr << std::endl; }
     options.assert_mutually_exclusive( "overlap-count-min,overlap-count-max", "overlap-count" );
     if( options.exists( "--overlap-count" ) )
     {
         min_overlap_count = max_overlap_count = options.value< unsigned int >( "--overlap-count" );
     }
     else
     {
         min_overlap_count = options.value( "--overlap-count-min", 0 );
         max_overlap_count = options.value( "--overlap-count-max", std::numeric_limits< unsigned int >::max() );
     }
 }
Ejemplo n.º 6
0
template < typename It > static void output_( It it, It end )
{
    for( ; it != end; ++it )
    {
        for( std::size_t i = 0; i < it->second.size() ; ++i )
        {
            std::cout.write( &( it->second[i][0] ), stdin_csv.binary() ? stdin_csv.format().size() : it->second[i].length() );
            if( stdin_csv.flush ) { std::cout.flush(); }
        }
    }
}
Ejemplo n.º 7
0
void output_points(const Eigen::Vector3d& p1, const Eigen::Vector3d& p2)
{
    if( csv.binary() )
    {
        std::cout.write( reinterpret_cast< const char* >( &p1 ), sizeof( double ) * 3 );
        std::cout.write( reinterpret_cast< const char* >( &p2 ), sizeof( double ) * 3 );
        std::cout.flush();
    }
    else 
    {
        std::cout << ascii.put( p1 ) << csv.delimiter << ascii.put( p2 ) << std::endl; 
    }
}
Ejemplo n.º 8
0
    void run()
    {
        comma::csv::input_stream< interval_t< From, To > > istream( std::cin, csv );
        comma::csv::ascii< interval_t< std::string > > ascii( csv.fields );

        if( !first_line.empty() )
        {
            interval_t< From, To > interval = comma::csv::ascii< interval_t< From, To > >( csv.fields ).get( first_line );
            bound_t< bound_type > from( LOWER );
            bound_t< bound_type > to( UPPER );
            std::string payload;
            interval_t< std::string > first;
            ascii.get( first, first_line );
            if( !first.from.value.empty() && ( !empty || interval.from.value != *empty ) ) { from.value = interval.from.value; }
            if( !first.to.value.empty() && ( !empty || interval.to.value != *empty  ) ) { to.value = interval.to.value; }
            payload = first_line;
            if( !intervals_only ) { ascii.put( interval_t< std::string >(), payload ); } // blank out interval from payload
            if( verbose ) { std::cerr << app_name << ": from: " << from << " to: " << to << " payload: " << payload << std::endl; }
            add( from, to, payload );
        }

        while( istream.ready() || std::cin.good()  )
        {
            const interval_t< From, To >* interval = istream.read();
            if( !interval ) { break; }
            bound_t< bound_type > from( LOWER );
            bound_t< bound_type > to( UPPER );
            std::string payload;
            if( csv.binary() )
            {
                if( !empty || interval->from.value != *empty ) { from.value = interval->from.value; }
                if( !empty || interval->to.value != *empty ) { to.value = interval->to.value; }
                std::vector< char > buf( istream.binary().last(), istream.binary().last() + istream.binary().size() );
                if( !intervals_only ) { istream.binary().binary().put( interval_t< From, To >(), &buf[0] ); } // blank out interval from payload
                payload = std::string( buf.begin(), buf.end() );
            }
            else
            {
                interval_t< std::string > last;
                ascii.get( last, istream.ascii().last() );
                if( !last.from.value.empty() && ( !empty || interval->from.value != *empty ) ) { from.value = interval->from.value; }
                if( !last.to.value.empty() && ( !empty || interval->to.value != *empty  ) ) { to.value = interval->to.value; }
                std::vector< std::string > buf( istream.ascii().last() );
                if( !intervals_only ) { ascii.put( interval_t< std::string >(), buf ); } // blank out interval from payload
                payload = comma::join( buf, csv.delimiter );
            }
            if( verbose ) { std::cerr << app_name << ": from: " << from << " to: " << to << " payload: " << ( csv.binary() ? "<binary>" : payload ) << std::endl; }
            add( from, to, payload );
        }
        write();
    }
Ejemplo n.º 9
0
static block_t* read_block_impl_( ::tbb::flow_control* flow = NULL )
{
    static boost::array< block_t, 3 > blocks;
    static boost::optional< block_t::pair_t > last;
    static comma::uint32 block_id = 0;
    block_t::pairs_t* points = new block_t::pairs_t;
    while( true ) // quick and dirty, only if --discard
    {
        static comma::csv::input_stream< input_t > istream( std::cin, csv );
        points->clear();
        while( true )
        {
            if( last )
            {
                block_id = last->first.block;
                points->push_back( *last );
                last.reset();
            }
            if( is_shutdown || std::cout.bad() || std::cin.bad() || std::cin.eof() )
            {
                if( bursty_reader ) { bursty_reader->stop(); } // quick and dirty, it sucks...
                if( flow ) { flow->stop(); }
                break;
            }
            const input_t* p = istream.read();
            if( !p ) { break; }
            std::string line;
            if( csv.binary() )
            {
                line.resize( csv.format().size() );
                ::memcpy( &line[0], istream.binary().last(), csv.format().size() );
            }
            else
            {
                line = comma::join( istream.ascii().last(), csv.delimiter );
            }
            last = std::make_pair( *p, line );
            if( p->block != block_id ) { break; }
        }
        for( unsigned int i = 0; i < blocks.size(); ++i )
        {
            if( !blocks[i].empty ) { continue; }
            blocks[i].clear();
            blocks[i].id = block_id;
            blocks[i].points.reset( points );
            blocks[i].empty = false;
            return &blocks[i];
        }
    }
}
Ejemplo n.º 10
0
 void output_( const iterator_& c1, const iterator_& b, const iterator_& c3 )
 {
     const iterator_& a = reverse_ ? c3 : c1;
     const iterator_& c = reverse_ ? c1 : c3;
     if( csv_.binary() )
     {
         std::cout.write( &a->second[0], csv_.format().size() );
         std::cout.write( &b->second[0], csv_.format().size() );
         std::cout.write( &c->second[0], csv_.format().size() );
     }
     else
     {
         std::cout << a->second << csv_.delimiter << b->second << csv_.delimiter << c->second << std::endl;
     }
 }
Ejemplo n.º 11
0
stereo::stereo ( const snark::imaging::camera_parser& left, const snark::imaging::camera_parser& right, unsigned int width, unsigned int height, const comma::csv::options& csv, bool input_rectified ):
    m_rotation( right.rotation() * left.rotation().transpose() ),
    m_translation( right.translation() - left.translation() ),
    m_rectify( left.camera(), left.distortion(), right.camera(), right.distortion(), width, height, m_rotation, m_translation, input_rectified ),
    m_input_rectified( input_rectified ),
    m_frame_counter( 0 )
{
    if( csv.binary() )
    {
        m_binary.reset( new comma::csv::binary< colored_point >( csv ) );
        m_output.resize( csv.format().size() );
    }
    else
    {
        m_ascii.reset( new comma::csv::ascii< colored_point >( csv ) );
    }
}
Ejemplo n.º 12
0
Reader::Reader( QGLView& viewer, comma::csv::options& options, std::size_t size, coloured* c, unsigned int pointSize, const std::string& label, const QVector3D& offset )
    : size( size )
    , pointSize( pointSize )
    , options( options )
    , m_viewer( viewer )
    , m_num_points( 0 )
    , m_colored( c )
    , m_shutdown( false )
    , m_isStdIn( options.filename == "-" )
    , m_show( true )
    , m_istream( options.filename, options.binary() ? comma::io::mode::binary : comma::io::mode::ascii, comma::io::mode::non_blocking )
    , updated_( false )
    , id_( 0 )
    , m_label( label )
    , m_offset( offset )
{
    std::vector< std::string > v = comma::split( options.fields, ',' ); // quick and dirty
}
Ejemplo n.º 13
0
 intervals( const comma::command_line_options& options ) : options( options )
                                                         , csv( options )
                                                         , ocsv( options )
                                                         , ascii_csv( options )
                                                         , empty( traits< bound_type >::cast( options.optional< std::string >( "--empty" ) ) )
                                                         , intervals_only( options.exists( "--intervals-only" ) )
                                                         , use_limits( options.exists( "--limits,-l" ) )
 {
     if( csv.fields.empty() ) { csv.fields = comma::join( comma::csv::names< interval_t< From, To > >(), ',' ); }
     if( ocsv.fields.empty() || intervals_only )
     {
         ocsv.fields = comma::join( comma::csv::names< interval_t< From, To > >(), ',' );
         if( ocsv.binary() && intervals_only ) { ocsv.format( comma::csv::format::value< interval_t< From, To > >() ); }
     }
     ascii_csv.fields = ocsv.fields;
     ascii_csv.quote = boost::none;
     if( verbose ) { std::cerr << app_name << ": empty: "; empty ? std::cerr << *empty : std::cerr << "<none>"; std::cerr << std::endl; }
 }
Ejemplo n.º 14
0
static void calculate_distance_for_pairs()
{
    comma::csv::input_stream< point_pair_t > istream( std::cin, csv );
    while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
    {
        const point_pair_t* p = istream.read();
        if( !p ) { break; }
        double norm = ( p->first - p->second ).norm();
        if( csv.binary() )
        {
            std::cout.write( istream.binary().last(), istream.binary().binary().format().size() );
            std::cout.write( reinterpret_cast< const char* >( &norm ), sizeof( double ) );
        }
        else
        {
            std::cout << comma::join( istream.ascii().last(), csv.delimiter ) << csv.delimiter << norm << std::endl;
        }
    }
}
Ejemplo n.º 15
0
frame::frame( const comma::csv::options& options, bool discardOutOfOrder, boost::optional< boost::posix_time::time_duration > maxGap, bool outputframe, bool to, bool interpolate, bool rotation_present )
    : outputframe( outputframe )
    , m_to( to )
    , m_interpolate( interpolate )
    , m_rotation( ::Eigen::Matrix3d::Identity() )
    , m_istream( new comma::io::istream( options.filename, options.binary() ? comma::io::mode::binary : comma::io::mode::ascii, comma::io::mode::blocking ) )
    , m_discardOutOfOrder( discardOutOfOrder )
    , m_maxGap( maxGap )
    , rotation_present_( rotation_present )
{
    m_is.reset( new comma::csv::input_stream< position_type >( *( *m_istream )(), options ) );
    const position_type* p = m_is->read(); // todo: maybe not very good, move to value()
    if( p == NULL ) { COMMA_THROW( comma::exception, "failed to read from " << options.filename ); }
    m_pair.first = *p;
    set_position( p->value );
    p = m_is->read(); // todo: maybe not very good, move to value()
    if( p == NULL ) { COMMA_THROW( comma::exception, "failed to read from " << options.filename ); }
    m_pair.second = *p;
}
Ejemplo n.º 16
0
stereo::stereo ( const camera_parser& left, const camera_parser& right,
                 const cv::Mat& left_x, const cv::Mat& left_y, const cv::Mat& right_x, const cv::Mat& right_y,
                 const comma::csv::options& csv, bool input_rectified ):
    m_rotation( Eigen::Matrix3d::Identity() ),
    m_translation( right.translation() - left.translation() ),
    m_rectify( left.camera(), right.camera(), m_translation, left_x, left_y, right_x, right_y, input_rectified ),
    m_input_rectified( input_rectified ),
    m_frame_counter( 0 )
{
    if( csv.binary() )
    {
        m_binary.reset( new comma::csv::binary< colored_point >( csv ) );
        m_output.resize( csv.format().size() );
    }
    else
    {
        m_ascii.reset( new comma::csv::ascii< colored_point >( csv ) );
    }
}
Ejemplo n.º 17
0
static void output( const timestring_t& input, const timestring_t& bounding, bool stdin_first )
{
    if( bounding.first.is_infinity() ) { return; }

    if( bound && ( input.first - bounding.first > bound || bounding.first - input.first > bound )) { return; }

    if( stdin_first )
    {
        output_input( std::cout, input );
        output_bounding( std::cout, bounding, stdin_first );
    }
    else
    {
        output_bounding( std::cout, bounding, stdin_first );
        output_input( std::cout, input );
    }

    if( !stdin_csv.binary() ) { std::cout << '\n'; }
    std::cout.flush();
}
Ejemplo n.º 18
0
 void write()
 {
     static comma::csv::output_stream< interval_t< From, To > > ostream( std::cout, ocsv );
     static comma::csv::ascii< from_t< std::string > > from_ascii( ascii_csv );
     static comma::csv::ascii< to_t< std::string > > to_ascii( ascii_csv );
     for( typename map_t::iterator it = map.begin(); it != map.end(); ++it )
     {
         bound_t< bound_type > from = it->first.lower();
         bound_t< bound_type > to = it->first.upper();
         interval_t< From, To > interval;
         bool from_has_value = true;
         bool to_has_value = true;
         if( from.value ) { interval.from.value = *from.value; }
         else if( use_limits ) { interval.from.value = limits< From >::lowest(); }
         else if( empty ) { interval.from.value = static_cast< From >( *empty ); }
         else { from_has_value = false; }
         if( to.value ) { interval.to.value = *to.value; }
         else if( use_limits ) { interval.to.value = limits< To >::max(); }
         else if( empty ) { interval.to.value = static_cast< To >( *empty ); }
         else { to_has_value = false; }
         const set_t& s = it->second;
         if( csv.binary() )
         {
             if( intervals_only ) { ostream.write( interval ); ostream.flush(); continue; }
             for( typename set_t::const_iterator v = s.begin(); v != s.end(); ++v ) { ostream.write( interval, *v ); }
             ostream.flush();
         }
         else
         {
             for( typename set_t::const_iterator v = s.begin(); v != s.end(); ++v )
             {
                 std::string payload( intervals_only ? "" : *v );
                 ostream.ascii().ascii().put( interval, payload );
                 if( !from_has_value ) { from_ascii.put( from_t< std::string >(), payload ); }
                 if( !to_has_value ) { to_ascii.put( to_t< std::string >(), payload); }
                 std::cout << payload << std::endl;
                 if( intervals_only ) { break; }
             }
         }
     }
 }
Ejemplo n.º 19
0
static void write_block_( block_t* block )
{
    if( !block ) { return; } // quick and dirty for now, only if --discard
    for( std::size_t i = 0; i < block->points->size(); ++i )
    {
        const block_t::pair_t& p = block->points->operator[]( i );
        if( ( p.first.foreground != foreground_t ) && !output_all ) { continue; }
        comma::uint32 id = p.first.id;
        comma::uint32 foreground = p.first.foreground;
        std::cout.write( &p.second[0], p.second.size() );
        if( csv.binary() )
        {
            std::cout.write( reinterpret_cast< const char* >( &id ), sizeof( comma::uint32 ) );
            std::cout.write( reinterpret_cast< const char* >( &foreground ), sizeof( comma::uint32 ) );
        }
        else
        {
            std::cout << csv.delimiter << id << csv.delimiter << foreground << std::endl;
        }
    }
    std::cout.flush();
    block->clear();
}
Ejemplo n.º 20
0
static void calculate_distance( bool cumulative )
{
    comma::csv::input_stream< Eigen::Vector3d > istream( std::cin, csv );
    boost::optional< Eigen::Vector3d > last;
    double distance = 0;
    while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
    {
        const Eigen::Vector3d* p = istream.read();
        if( !p ) { break; }
        double norm = last ? ( *p - *last ).norm() : 0;
        distance = cumulative ? distance + norm : norm;
        last = *p;
        if( csv.binary() )
        {
            std::cout.write( istream.binary().last(), istream.binary().binary().format().size() );
            std::cout.write( reinterpret_cast< const char* >( &distance ), sizeof( double ) );
        }
        else
        {
            std::cout << comma::join( istream.ascii().last(), csv.delimiter ) << csv.delimiter << distance << std::endl;
        }
    }
}
Ejemplo n.º 21
0
int main( int ac, char** av )
{
    try
    {
        comma::command_line_options options( ac, av );
        if( options.exists( "--help,-h" ) ) { usage(); }
        verbose = options.exists( "--verbose,-v" );
        bool is_or = options.exists( "--or" );
        csv = comma::csv::options( options );
        fields = comma::split( csv.fields, ',' );
        if( fields.size() == 1 && fields[0].empty() ) { fields.clear(); }
        std::vector< std::string > unnamed = options.unnamed( "--first-matching,--or,--sorted,--input-sorted,--not-matching,--output-all,--all,--strict,--verbose,-v", "-.*" );
        //for( unsigned int i = 0; i < unnamed.size(); constraints_map.insert( std::make_pair( comma::split( unnamed[i], ';' )[0], unnamed[i] ) ), ++i );
        bool strict = options.exists( "--strict" );
        bool first_matching = options.exists( "--first-matching" );
        bool not_matching = options.exists( "--not-matching" );
        bool all = options.exists( "--output-all,--all" );
        for( unsigned int i = 0; i < unnamed.size(); ++i )
        {
            std::string field = comma::split( unnamed[i], ';' )[0];
            bool found = false;
            for( unsigned int j = 0; j < fields.size() && !found; found = field == fields[j], ++j );
            if( !found )
            {
                if( strict ) { std::cerr << "csv-select: on constraint: \"" << unnamed[i] << "\" field \"" << field << "\" not found in fields: " << csv.fields << std::endl; return 1; }
                std::cerr << "csv-select: warning: on constraint: \"" << unnamed[i] << "\" field \"" << field << "\" not found in fields: " << csv.fields << std::endl;
                continue;
            }
            constraints_map.insert( std::make_pair( field, unnamed[i] ) );
        }
        comma::signal_flag is_shutdown;
        if( csv.binary() )
        {
            #ifdef WIN32
            _setmode( _fileno( stdout ), _O_BINARY );
            #endif
            init_input( csv.format(), options );
            comma::csv::binary_input_stream< input_t > istream( std::cin, csv, input );
            while( !is_shutdown && ( istream.ready() || ( std::cin.good() && !std::cin.eof() ) ) )
            {
                const input_t* p = istream.read();
                if( !p || p->done( is_or ) ) { break; }
                char match = ( p->is_a_match( is_or ) == !not_matching ) ? 1 : 0;
                if( match || all )
                {
                    std::cout.write( istream.last(), csv.format().size() );
                    if( all ) { std::cout.write( &match, 1 ); }
                    if( csv.flush ) { std::cout.flush(); }
                    if( first_matching ) { break; }
                }
            }
        }
        else
        {
            std::string line;
            while( !is_shutdown && std::cin.good() && !std::cin.eof() )
            {
                std::getline( std::cin, line );
                line = comma::strip( line, '\r' ); // windows, sigh...
                if( !line.empty() ) { break; }
            }
            if( line.empty() ) { return 0; }
            comma::csv::format format = options.exists( "--format" )
                                      ? comma::csv::format( options.value< std::string >( "--format" ) )
                                      : guess_format( line );
            init_input( format, options );
            comma::csv::ascii_input_stream< input_t > istream( std::cin, csv, input );
            // todo: quick and dirty: no time to debug why the commented section does not work (but that's the right way)
            std::istringstream iss( line );
            comma::csv::ascii_input_stream< input_t > isstream( iss, csv, input );
            const input_t* p = isstream.read();
            if( !p || p->done( is_or ) ) { return 0; }
            bool match = p->is_a_match( is_or ) == !not_matching;
            if( match || all )
            {
                std::cout << line;
                if( all ) { std::cout << csv.delimiter << match; }
                std::cout << std::endl;
                if( first_matching ) { return 0; }
            }
            while( !is_shutdown && ( istream.ready() || ( std::cin.good() && !std::cin.eof() ) ) )
            {
                const input_t* p = istream.read();
                if( !p || p->done( is_or ) ) { break; }
                bool match = p->is_a_match( is_or ) == !not_matching;
                if( match || all )
                {
                    std::cout << comma::join( istream.last(), csv.delimiter );
                    if( all ) { std::cout << csv.delimiter << match; }
                    std::cout << std::endl;
                    if( first_matching ) { return 0; }
                }
            }
        }
        return 0;
    }
    catch( std::exception& ex ) { std::cerr << "csv-select: caught: " << ex.what() << std::endl; }
    catch( ... ) { std::cerr << "csv-select: unknown exception" << std::endl; }
    return 1;
}
Ejemplo n.º 22
0
CameraReader::CameraReader( comma::csv::options& options )
    : options( options )
    , m_shutdown( false )
    , m_istream( options.filename, options.binary() ? comma::io::mode::binary : comma::io::mode::ascii )
{
}
Ejemplo n.º 23
0
 void write()
 {
     static comma::csv::output_stream< interval_t< From, To > > ostream( std::cout, ocsv );
     static comma::csv::ascii< from_t< std::string > > from_ascii( ascii_csv );
     static comma::csv::ascii< to_t< std::string > > to_ascii( ascii_csv );
     for( typename map_t::iterator it = map.begin(); it != map.end(); ++it )
     {
         bound_t< bound_type > from = it->first.lower();
         bound_t< bound_type > to = it->first.upper();
         interval_t< From, To > interval;
         bool from_has_value = true;
         bool to_has_value = true;
         if( from.value ) { interval.from.value = *from.value; }
         else if( use_limits ) { interval.from.value = limits< From >::lowest(); }
         else if( empty ) { interval.from.value = static_cast< From >( *empty ); }
         else { from_has_value = false; }
         if( to.value ) { interval.to.value = *to.value; }
         else if( use_limits ) { interval.to.value = limits< To >::max(); }
         else if( empty ) { interval.to.value = static_cast< To >( *empty ); }
         else { to_has_value = false; }
         const set_t& s = it->second;
         if( s.size() < min_overlap_count || s.size() > max_overlap_count ) { continue; }
         if( append )
         {
             if( csv.binary() )
             {
                 for( typename set_t::const_iterator v = s.begin(); v != s.end(); ++v )
                 { 
                     std::cout.write( &( *v )[0], v->size() );
                     ostream.write( interval );
                 }
                 ostream.flush(); // todo: use csv.flush flag
             }
             else
             {
                 //std::ostringstream oss;
                 //comma::csv::output_stream< interval_t< From, To > > osstream( oss ); // todo! quick and dirty, watch performance!
                 if( !from_has_value || !to_has_value ) { std::cerr << "csv-interval: support for empty from/to values for --append: todo" << std::endl; exit( 1 ); }
                 for( typename set_t::const_iterator v = s.begin(); v != s.end(); ++v )
                 {
                     std::cout << *v << csv.delimiter;
                     ostream.write( interval );
                 }
             }
         }
         else
         {
             if( csv.binary() )
             {
                 if( intervals_only ) { ostream.write( interval ); ostream.flush(); continue; }
                 for( typename set_t::const_iterator v = s.begin(); v != s.end(); ++v ) { ostream.write( interval, *v ); }
                 ostream.flush();
             }
             else
             {
                 for( typename set_t::const_iterator v = s.begin(); v != s.end(); ++v )
                 {
                     std::string payload( intervals_only ? "" : *v );
                     ostream.ascii().ascii().put( interval, payload );
                     if( !from_has_value ) { from_ascii.put( from_t< std::string >(), payload ); }
                     if( !to_has_value ) { to_ascii.put( to_t< std::string >(), payload); }
                     std::cout << payload << std::endl;
                     if( intervals_only ) { break; }
                 }
             }
         }
     }
 }
Ejemplo n.º 24
0
int main( int ac, char** av )
{
    try
    {
        comma::command_line_options options( ac, av );
        verbose = options.exists( "--verbose,-v" );
        if( options.exists( "--help,-h" ) ) { usage( verbose ); }
        csv = comma::csv::options( options );
        csv.full_xpath = true;
        ascii = comma::csv::ascii< Eigen::Vector3d >( "x,y,z", csv.delimiter );
        const std::vector< std::string >& operations = options.unnamed( "--verbose,-v,--trace,--no-antialiasing,--next,--unit", "-.*" );
        if( operations.size() != 1 ) { std::cerr << "points-calc: expected one operation, got " << operations.size() << ": " << comma::join( operations, ' ' ) << std::endl; return 1; }
        const std::string& operation = operations[0];
        if (vector_calc::has_operation(operation))
        {
            vector_calc::process(operation, options, csv);
            return 0;
        }
        if( operation == "plane-intersection" )
        {
            plane_intersection::process(options, csv);
            return 0;
        }
        if( operation == "distance" )
        {
            if(    csv.has_field( "first" )   || csv.has_field( "second" )
                || csv.has_field( "first/x" ) || csv.has_field( "second/x" )
                || csv.has_field( "first/y" ) || csv.has_field( "second/y" )
                || csv.has_field( "first/z" ) || csv.has_field( "second/z" ) )
            {
                calculate_distance_for_pairs();
                return 0;
            }
            if ( options.exists( "--next" ) ) { calculate_distance_next(); }
            else { calculate_distance( false ); }
            return 0;
        }
        if( operation == "cumulative-distance" )
        {
            calculate_distance( true );
            return 0;
        }
        if( operation == "nearest" )
        {
            if( options.exists( "--point,--to" ) )
            {
                Eigen::Vector3d point = comma::csv::ascii< Eigen::Vector3d >().get( options.value< std::string >( "--point,--to" ) );
                comma::csv::input_stream< Eigen::Vector3d > istream( std::cin, csv );
                std::string record;
                double min_distance = std::numeric_limits< double >::max();
                while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
                {
                    const Eigen::Vector3d* p = istream.read();
                    if( !p ) { break; }
                    double d = ( *p - point ).norm();
                    if( d >= min_distance ) { continue; }
                    min_distance = d;
                    record = csv.binary() ? std::string( istream.binary().last(), csv.format().size() ) : comma::join( istream.ascii().last(), csv.delimiter );
                }
                if( !record.empty() ) { std::cout << record; }
                if( csv.binary() ) { std::cout.write( reinterpret_cast< const char* >( &min_distance ), sizeof( double ) ); }
                else { std::cout << csv.delimiter << min_distance << std::endl; }
                return 0;
            }
            else
            {
                
                return 0;
            }
        }
        if( operation == "thin" )
        {
            if( !options.exists( "--resolution" ) ) { std::cerr << "points-calc: --resolution is not specified " << std::endl; return 1; }
            double resolution = options.value( "--resolution" , 0.0 );
            thin( resolution );
            return 0;
        }
        if( operation == "discretise" || operation == "discretize" )
        {
            if( !options.exists( "--step" ) ) { std::cerr << "points-calc: --step is not specified " << std::endl; return 1; }
            double step = options.value( "--step" , 0.0 );
            if( step <= 0 ) { std::cerr << "points-calc: expected positive step, got " << step << std::endl; return 1; }
            // the last discretised point can be very close to the end of the interval, in which case the last two points can be identical in the output since ascii.put uses 12 digits by default
            // setting --tolerance=1e-12 will not allow the last discretised point to be too close to the end of the interval and therefore the output will have two distinct points at the end
            double tolerance = options.value( "--tolerance" , 0.0 ); 
            if( tolerance < 0 ) { std::cerr << "points-calc: expected non-negative tolerance, got " << tolerance << std::endl; return 1; }
            discretise( step, tolerance );
            return 0;
        }
        if( operation == "local-max" || operation == "local-min" ) // todo: if( operation == "local-calc" ? )
        {
            double sign = operation == "local-max" ? 1 : -1;
            if( csv.fields.empty() ) { csv.fields = "x,y,z,scalar"; }
            csv.full_xpath = false;
            bool has_id = csv.has_field( "id" );
            comma::csv::input_stream< local_operation::point > istream( std::cin, csv );
            std::deque< local_operation::record > records;
            double radius = options.value< double >( "--radius" );
            bool trace = options.exists( "--trace" );
            Eigen::Vector3d resolution( radius, radius, radius );
            snark::math::closed_interval< double, 3 > extents;
            comma::uint32 id = 0;
            if( verbose ) { std::cerr << "points-calc: reading input points..." << std::endl; }
            while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
            {
                const local_operation::point* p = istream.read();
                if( !p ) { break; }
                std::string line;
                if( csv.binary() ) // quick and dirty
                {
                    line.resize( csv.format().size() );
                    ::memcpy( &line[0], istream.binary().last(), csv.format().size() );
                }
                else
                {
                    line = comma::join( istream.ascii().last(), csv.delimiter );
                }
                local_operation::point q = *p;
                if( !has_id ) { q.id = id++; }
                records.push_back( local_operation::record( q, line ) );
                records.back().reference_record = &records.back();
                extents.set_hull( p->coordinates );
            }
            if( verbose ) { std::cerr << "points-calc: loading " << records.size() << " points into grid..." << std::endl; }
            typedef std::vector< local_operation::record* > voxel_t; // todo: is vector a good container? use deque
            typedef snark::voxel_map< voxel_t, 3 > grid_t;
            grid_t grid( extents.min(), resolution );
            for( std::size_t i = 0; i < records.size(); ++i ) { ( grid.touch_at( records[i].point.coordinates ) )->second.push_back( &records[i] ); }
            if( verbose ) { std::cerr << "points-calc: searching for local extrema..." << std::endl; }
            for( grid_t::iterator it = grid.begin(); it != grid.end(); ++it )
            {
                grid_t::index_type i;
                for( i[0] = it->first[0] - 1; i[0] < it->first[0] + 2; ++i[0] )
                {
                    for( i[1] = it->first[1] - 1; i[1] < it->first[1] + 2; ++i[1] )
                    {
                        for( i[2] = it->first[2] - 1; i[2] < it->first[2] + 2; ++i[2] )
                        {
                            grid_t::iterator git = grid.find( i );
                            if( git == grid.end() ) { continue; }
                            for( voxel_t::iterator vit = it->second.begin(); vit != it->second.end(); ++vit )
                            {
                                for( std::size_t k = 0; k < git->second.size() && ( *vit )->is_extremum; ++k )
                                {
                                    local_operation::evaluate_local_extremum( *vit, git->second[k], radius, sign );
                                }
                            }
                        }
                    }
                }
            }
            #ifdef WIN32
            _setmode( _fileno( stdout ), _O_BINARY );
            #endif
            if( verbose ) { std::cerr << "points-calc: filling extrema grid..." << std::endl; }
            grid_t extrema( extents.min(), resolution );
            for( std::size_t i = 0; i < records.size(); ++i )
            {
                if( records[i].is_extremum )
                { 
                    ( extrema.touch_at( records[i].point.coordinates ) )->second.push_back( &records[i] );
                }
                else
                { 
                    records[i].extremum_id = local_operation::record::invalid_id; // quick and dirty for now
//                     if( records[i].extremum_id == local_operation::record::invalid_id ) { continue; }
//                     while( records[i].reference_record->point.id != records[i].reference_record->reference_record->point.id )
//                     {
//                         records[i].reference_record = records[i].reference_record->reference_record;
//                     }
//                     records[i].extremum_id = records[i].reference_record->point.id;
                }
            }
            if( verbose ) { std::cerr << "points-calc: calculating distances to " << extrema.size() << " local extrema..." << std::endl; }
            for( grid_t::iterator it = grid.begin(); it != grid.end(); ++it )
            {
                grid_t::index_type i;
                for( i[0] = it->first[0] - 1; i[0] < it->first[0] + 2; ++i[0] )
                {
                    for( i[1] = it->first[1] - 1; i[1] < it->first[1] + 2; ++i[1] )
                    {
                        for( i[2] = it->first[2] - 1; i[2] < it->first[2] + 2; ++i[2] )
                        {
                            grid_t::iterator git = extrema.find( i );
                            if( git == extrema.end() ) { continue; }
                            for( std::size_t n = 0; n < it->second.size(); ++n )
                            {                            
                                for( std::size_t k = 0; k < git->second.size(); ++k )
                                {
                                    local_operation::update_nearest_extremum( it->second[n], git->second[k], radius );
                                }
                            }
                        }
                    }
                }
            }
            if( trace )
            {
                if( verbose ) { std::cerr << "points-calc: tracing extrema..." << std::endl; }
                for( std::size_t i = 0; i < records.size(); ++i )
                {
                    if( records[i].extremum_id == local_operation::record::invalid_id ) { continue; }
                    while( records[i].reference_record->point.id != records[i].reference_record->reference_record->point.id )
                    {
                        records[i].reference_record = records[i].reference_record->reference_record;
                    }
                }
            }
            if( verbose ) { std::cerr << "points-calc: outputting..." << std::endl; }
            std::string endl = csv.binary() ? "" : "\n";
            std::string delimiter = csv.binary() ? "" : std::string( 1, csv.delimiter );
            comma::csv::options output_csv;
            if( csv.binary() ) { output_csv.format( "ui,d" ); }
            comma::csv::output_stream< local_operation::output > ostream( std::cout, output_csv );
            for( std::size_t i = 0; i < records.size(); ++i )
            {
                std::cout.write( &records[i].line[0], records[i].line.size() );
                std::cout.write( &delimiter[0], delimiter.size() );
                ostream.write( records[i].output( false ) ); // quick and dirty
            }
            if( verbose ) { std::cerr << "points-calc: done!" << std::endl; }
            return 0;
        }
        if( operation == "nearest-max" || operation == "nearest-min" || operation == "nearest-any" )
        {
            double sign = operation == "nearest-max" ? 1 : -1;
            bool any = operation == "nearest-any";
            if( csv.fields.empty() ) { csv.fields = "x,y,z,scalar"; }
            csv.full_xpath = false;
            bool has_id = csv.has_field( "id" );
            comma::csv::input_stream< local_operation::point > istream( std::cin, csv );
            std::deque< local_operation::record > records;
            double radius = options.value< double >( "--radius" );
            Eigen::Vector3d resolution( radius, radius, radius );
            snark::math::closed_interval< double, 3 > extents;
            comma::uint32 id = 0;
            if( verbose ) { std::cerr << "points-calc: reading input points..." << std::endl; }
            while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
            {
                const local_operation::point* p = istream.read();
                if( !p ) { break; }
                std::string line;
                if( csv.binary() ) // quick and dirty
                {
                    line.resize( csv.format().size() );
                    ::memcpy( &line[0], istream.binary().last(), csv.format().size() );
                }
                else
                {
                    line = comma::join( istream.ascii().last(), csv.delimiter );
                }
                local_operation::point q = *p;
                if( !has_id ) { q.id = id++; }
                records.push_back( local_operation::record( q, line ) );
                records.back().reference_record = &records.back();
                extents.set_hull( p->coordinates );
            }
            if( verbose ) { std::cerr << "points-calc: loading " << records.size() << " points into grid..." << std::endl; }
            typedef std::vector< local_operation::record* > voxel_t; // todo: is vector a good container? use deque
            typedef snark::voxel_map< voxel_t, 3 > grid_t;
            grid_t grid( extents.min(), resolution );
            for( std::size_t i = 0; i < records.size(); ++i ) { ( grid.touch_at( records[i].point.coordinates ) )->second.push_back( &records[i] ); }
            if( verbose ) { std::cerr << "points-calc: searching for " << operation << "..." << std::endl; }
            for( grid_t::iterator it = grid.begin(); it != grid.end(); ++it )
            {
                grid_t::index_type i;
                for( i[0] = it->first[0] - 1; i[0] < it->first[0] + 2; ++i[0] )
                {
                    for( i[1] = it->first[1] - 1; i[1] < it->first[1] + 2; ++i[1] )
                    {
                        for( i[2] = it->first[2] - 1; i[2] < it->first[2] + 2; ++i[2] )
                        {
                            grid_t::iterator git = grid.find( i );
                            if( git == grid.end() ) { continue; }
                            for( std::size_t n = 0; n < it->second.size(); ++n )
                            {                            
                                for( std::size_t k = 0; k < git->second.size(); ++k )
                                {
                                    local_operation::update_nearest( it->second[n], git->second[k], radius, sign, any );
                                }
                            }
                        }
                    }
                }
            }
            #ifdef WIN32
            _setmode( _fileno( stdout ), _O_BINARY );
            #endif
            if( verbose ) { std::cerr << "points-calc: outputting..." << std::endl; }
            std::string endl = csv.binary() ? "" : "\n";
            std::string delimiter = csv.binary() ? "" : std::string( 1, csv.delimiter );
            comma::csv::options output_csv;
            if( csv.binary() ) { output_csv.format( "ui,d" ); }
            comma::csv::output_stream< local_operation::output > ostream( std::cout, output_csv );
            for( std::size_t i = 0; i < records.size(); ++i )
            {
                std::cout.write( &records[i].line[0], records[i].line.size() );
                std::cout.write( &delimiter[0], delimiter.size() );
                ostream.write( records[i].output() );
            }
            if( verbose ) { std::cerr << "points-calc: done!" << std::endl; }
            return 0;
        }
        if( operation == "find-outliers" )
        {
            unsigned int size = options.value< unsigned int >( "--min-number-of-points-per-voxel,--size" );
            double r = options.value< double >( "--resolution" );
            Eigen::Vector3d resolution( r, r, r );
            bool no_antialiasing = options.exists( "--no-antialiasing" );            
            comma::csv::input_stream< Eigen::Vector3d > istream( std::cin, csv );
            std::deque< remove_outliers::record > records;
            snark::math::closed_interval< double, 3 > extents;
            if( verbose ) { std::cerr << "points-calc: reading input points..." << std::endl; }
            while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
            {
                const Eigen::Vector3d* p = istream.read();
                if( !p ) { break; }
                std::string line;
                if( csv.binary() ) // quick and dirty
                {
                    line.resize( csv.format().size() );
                    ::memcpy( &line[0], istream.binary().last(), csv.format().size() );
                }
                else
                {
                    line = comma::join( istream.ascii().last(), csv.delimiter );
                }
                records.push_back( remove_outliers::record( *p, line ) );
                extents.set_hull( *p );
            }
            if( verbose ) { std::cerr << "points-calc: loading " << records.size() << " points into grid..." << std::endl; }
            typedef std::vector< remove_outliers::record* > voxel_t; // todo: is vector a good container? use deque
            typedef snark::voxel_map< voxel_t, 3 > grid_t;
            grid_t grid( extents.min(), resolution );
            for( std::size_t i = 0; i < records.size(); ++i ) { ( grid.touch_at( records[i].point ) )->second.push_back( &records[i] ); }
            if( verbose ) { std::cerr << "points-calc: removing outliers..." << std::endl; }
            for( grid_t::iterator it = grid.begin(); it != grid.end(); ++it )
            {
                bool rejected = true;
                if( no_antialiasing )
                {
                    rejected = it->second.size() < size;
                }
                else
                {
                    grid_t::index_type i;
                    for( i[0] = it->first[0] - 1; i[0] < it->first[0] + 2 && rejected; ++i[0] )
                    {
                        for( i[1] = it->first[1] - 1; i[1] < it->first[1] + 2 && rejected; ++i[1] )
                        {
                            for( i[2] = it->first[2] - 1; i[2] < it->first[2] + 2 && rejected; ++i[2] )
                            {
                                grid_t::iterator git = grid.find( i );
                                rejected = git == grid.end() || git->second.size() < size;
                            }
                        }
                    }
                }
                if( rejected ) { for( std::size_t i = 0; i < it->second.size(); ++i ) { it->second[i]->rejected = true; } }
            }
            #ifdef WIN32
            _setmode( _fileno( stdout ), _O_BINARY );
            #endif
            if( verbose ) { std::cerr << "points-calc: outputting..." << std::endl; }
            std::string endl = csv.binary() ? "" : "\n";
            std::string delimiter = csv.binary() ? "" : std::string( 1, csv.delimiter );
            for( std::size_t i = 0; i < records.size(); ++i )
            {
                char valid = records[i].rejected ? 0 : 1;
                std::cout.write( &records[i].line[0], records[i].line.size() );
                std::cout.write( &delimiter[0], delimiter.size() );
                std::cout.write( &valid, 1 );
                std::cout.write( &endl[0], endl.size() );
            }
            if( verbose ) { std::cerr << "points-calc: done!" << std::endl; }
            return 0;
        }
        std::cerr << "points-calc: please specify an operation" << std::endl;
        return 1;
    }
    catch( std::exception& ex )
    {
        std::cerr << "points-calc: " << ex.what() << std::endl;
    }
    catch( ... )
    {
        std::cerr << "points-calc: unknown exception" << std::endl;
    }
    return 1;
}
Ejemplo n.º 25
0
static void output_input( std::ostream& os, const timestring_t& input )
{
    if( stdin_csv.binary() ) { os.write( &input.second[0], stdin_csv.format().size() ); }
    else { os << input.second; }
}
Ejemplo n.º 26
0
int sort( const comma::command_line_options& options )
{
    input_t::map sorted_map;
    input_t default_input;
    std::vector< std::string > v = comma::split( stdin_csv.fields, ',' );
    std::vector< std::string > order = options.exists("--order") ? comma::split( options.value< std::string >( "--order" ), ',' ) : v;
    std::vector< std::string > w (v.size());
    bool unique = options.exists("--unique,-u");
    
    std::string first_line;
    comma::csv::format f;
    if( stdin_csv.binary() ) { f = stdin_csv.format(); }
    else if( options.exists( "--format" ) ) { f = comma::csv::format( options.value< std::string >( "--format" ) ); }
    else
    {
        while( std::cin.good() && first_line.empty() ) { std::getline( std::cin, first_line ); }
        if( first_line.empty() ) { return 0; }
        f = comma::csv::impl::unstructured::guess_format( first_line, stdin_csv.delimiter );
        if( verbose ) { std::cerr << "csv-sort: guessed format: " << f.string() << std::endl; }
    }
    for( std::size_t i = 0; i < order.size(); ++i ) // quick and dirty, wasteful, but who cares
    {
        if (order[i].empty()) continue;
        for( std::size_t k = 0; k < v.size(); ++k )
        {
            if( v[k].empty() || v[k] != order[i] ) 
            { 
                if ( k + 1 == v.size()) 
                { 
                    std::cerr << "csv-sort: order field name \"" << order[i] << "\" not found in input fields \"" << stdin_csv.fields << "\"" << std::endl;
                    return 1;
                }
                continue; 
            }
            std::string type = default_input.keys.append( f.offset( k ).type );
            w[k] = "keys/" + type;
            
            ordering_t o;
            if ( type[0] == 's' ) { o.type = ordering_t::str_type; o.index = default_input.keys.strings.size() - 1; }
            else if ( type[0] == 'l' ) { o.type = ordering_t::long_type; o.index = default_input.keys.longs.size() - 1; }
            else if ( type[0] == 'd' ) { o.type = ordering_t::double_type; o.index = default_input.keys.doubles.size() - 1; }
            else if ( type[0] == 't' ) { o.type = ordering_t::time_type; o.index = default_input.keys.time.size() - 1; }
            else { std::cerr << "csv-sort: cannot sort on field " << v[k] << " of type \"" << type << "\"" << std::endl; return 1; }
            
            ordering.push_back(o);
            
            break;
        }
    }
    stdin_csv.fields = comma::join( w, ',' );
    if ( verbose ) { std::cerr << "csv-sort: fields: " << stdin_csv.fields << std::endl; }
    comma::csv::input_stream< input_t > stdin_stream( std::cin, stdin_csv, default_input );
    #ifdef WIN32
    if( stdin_stream.is_binary() ) { _setmode( _fileno( stdout ), _O_BINARY ); }
    #endif
    
    if (!first_line.empty()) 
    { 
        input_t::map::mapped_type& d = sorted_map[ comma::csv::ascii< input_t >(stdin_csv,default_input).get(first_line) ];
        d.push_back( first_line + "\n" );
    }
    
    while( stdin_stream.ready() || ( std::cin.good() && !std::cin.eof() ) )
    {
        const input_t* p = stdin_stream.read();
        if( !p ) { break; }
        if( stdin_stream.is_binary() )
        {
            input_t::map::mapped_type& d = sorted_map[ *p ];
            if (unique && !d.empty()) continue;
            d.push_back( std::string() );
            d.back().resize( stdin_csv.format().size() );
            ::memcpy( &d.back()[0], stdin_stream.binary().last(), stdin_csv.format().size() );
        }
        else
        {
            input_t::map::mapped_type& d = sorted_map[ *p ];
            if (unique && !d.empty()) continue;
            d.push_back( comma::join( stdin_stream.ascii().last(), stdin_csv.delimiter ) + "\n" );
        }
    }
    
    if( options.exists( "--reverse,-r" ) ) { output_( sorted_map.rbegin(), sorted_map.rend() ); }
    else { output_( sorted_map.begin(), sorted_map.end() ); }
    
    return 0;
}
Ejemplo n.º 27
0
int main( int ac, char** av )
{
    try
    {
        comma::signal_flag is_shutdown(comma::signal_flag::hard);
        comma::command_line_options options( ac, av, usage );

        if( options.exists( "--bash-completion" )) bash_completion( ac, av );
        options.assert_mutually_exclusive( "--by-lower,--by-upper,--nearest,--realtime" );
        if( options.exists( "--by-upper" )) { method = how::by_upper; }
        if( options.exists( "--nearest" )) { method = how::nearest; }
        if( options.exists( "--realtime" )) { method = how::realtime; }
        timestamp_only = options.exists( "--timestamp-only,--time-only" );
        select_only = options.exists( "--do-not-append,--select" );
        if( select_only && timestamp_only ) { std::cerr << "csv-time-join: --timestamp-only specified with --select, ignoring --timestamp-only" << std::endl; }
        bool discard_bounding = options.exists( "--discard-bounding" );
        boost::optional< unsigned int > buffer_size = options.optional< unsigned int >( "--buffer" );
        if( options.exists( "--bound" ) ) { bound = boost::posix_time::microseconds( static_cast<unsigned int>(options.value< double >( "--bound" ) * 1000000 )); }
        stdin_csv = comma::csv::options( options, "t" );

        std::vector< std::string > unnamed = options.unnamed(
            "--by-lower,--by-upper,--nearest,--realtime,--select,--do-not-append,--timestamp-only,--time-only,--discard-bounding",
            "--binary,-b,--delimiter,-d,--fields,-f,--bound,--buffer,--verbose,-v" );
        std::string properties;
        bool stdin_first = true;
        switch( unnamed.size() )
        {
            case 0:
                std::cerr << "csv-time-join: please specify bounding source" << std::endl;
                return 1;
            case 1:
                properties = unnamed[0];
                break;
            case 2:
                if( unnamed[0] == "-" ) { properties = unnamed[1]; }
                else if( unnamed[1] == "-" ) { properties = unnamed[0]; stdin_first = false; }
                else { std::cerr << "csv-time-join: expected either '- <bounding>' or '<bounding> -'; got : " << comma::join( unnamed, ' ' ) << std::endl; return 1; }
                break;
            default:
                std::cerr << "csv-time-join: expected either '- <bounding>' or '<bounding> -'; got : " << comma::join( unnamed, ' ' ) << std::endl;
                return 1;
        }
        comma::name_value::parser parser( "filename" );
        bounding_csv = parser.get< comma::csv::options >( properties );
        if( bounding_csv.fields.empty() ) { bounding_csv.fields = "t"; }

        comma::csv::input_stream< Point > stdin_stream( std::cin, stdin_csv );
        #ifdef WIN32
        if( stdin_csv.binary() ) { _setmode( _fileno( stdout ), _O_BINARY ); }
        #endif // #ifdef WIN32

        comma::io::istream bounding_istream( comma::split( properties, ';' )[0]
                                           , bounding_csv.binary() ? comma::io::mode::binary : comma::io::mode::ascii );
        comma::csv::input_stream< Point > bounding_stream( *bounding_istream, bounding_csv );

        #ifndef WIN32
        comma::io::select select;
        comma::io::select bounding_stream_select;
        select.read().add( 0 );
        select.read().add( bounding_istream.fd() );
        bounding_stream_select.read().add( bounding_istream.fd() );
        #endif // #ifndef WIN32

        const Point* p = NULL;

        if( method == how::realtime )
        {
            #ifndef WIN32
            bool end_of_input = false;
            bool end_of_bounds = false;
            
            boost::optional<timestring_t> joined_line;
            
            while (!is_shutdown && !end_of_input)
            {
                if ( !bounding_stream.ready() && !stdin_stream.ready() )
                {
                    select.wait(boost::posix_time::milliseconds(1));
                }
                
                if ( !is_shutdown && !end_of_input && ( stdin_stream.ready() || ( select.check() && select.read().ready( comma::io::stdin_fd ) ) ) )
                {
                    p = stdin_stream.read();
                    if( p )
                    {
                        timestring_t input_line = std::make_pair( get_time( *p ), stdin_stream.last() );
                        if( joined_line ) { output( input_line, *joined_line, stdin_first ); }
                    }
                    else
                    {
                        comma::verbose << "end of input stream" << std::endl;
                        end_of_input = true;
                    }
                }
                
                if ( !is_shutdown && !end_of_bounds &&
                   ( bounding_stream.ready() || ( select.check() && select.read().ready( bounding_istream.fd() ))))
                {
                    p = bounding_stream.read();
                    if( p )
                    {
                        joined_line = std::make_pair( get_time( *p ), bounding_stream.last() );
                    }
                    else
                    {
                        comma::verbose << "end of bounding stream" << std::endl;
                        end_of_bounds = true;
                    }
                }
            }
            if (is_shutdown) { comma::verbose << "got a signal" << std::endl; return 0; }
            #else
            COMMA_THROW(comma::exception, "--realtime mode not supported in WIN32");
            #endif
        }
        else
        {
            std::deque<timestring_t> bounding_queue;
            bool next = true;
            bool bounding_data_available;
            bool upper_bound_added = false;

            // add a fake entry for an lower bound to allow stdin before first bound to match
            bounding_queue.push_back( std::make_pair( boost::posix_time::neg_infin, "" ));

            while( ( stdin_stream.ready() || ( std::cin.good() && !std::cin.eof() ) ) )
            {
                if( !std::cin.good() ) { select.read().remove( 0 ); }
                if( !bounding_istream->good() ) { select.read().remove( bounding_istream.fd() ); }
                bounding_data_available = bounding_stream.ready() || ( bounding_istream->good() && !bounding_istream->eof() );
                #ifdef WIN32
                bool bounding_stream_ready = true;
                bool stdin_stream_ready = true;
                #else // #ifdef WIN32
                //check so we do not block
                bool bounding_stream_ready = bounding_stream.ready();
                bool stdin_stream_ready = stdin_stream.ready();

                if( next )
                {
                    if( !bounding_stream_ready || !stdin_stream_ready )
                    {
                        if( !bounding_stream_ready && !stdin_stream_ready )
                        {
                            select.wait( boost::posix_time::milliseconds(10) );
                        }
                        else
                        {
                            select.check();
                        }
                        if( select.read().ready( bounding_istream.fd() )) { bounding_stream_ready = true; }
                        if( select.read().ready(0) ) { stdin_stream_ready=true; }
                    }
                }
                else
                {
                    if( !bounding_stream_ready )
                    {
                        bounding_stream_select.wait( boost::posix_time::milliseconds(10) );
                        if( bounding_stream_select.read().ready( bounding_istream.fd() )) { bounding_stream_ready=true; }
                    }
                }
                #endif //#ifdef WIN32

                //keep storing available bounding data
                if( bounding_stream_ready )
                {
                    if( !buffer_size || bounding_queue.size() < *buffer_size || discard_bounding )
                    {
                        const Point* q = bounding_stream.read();
                        if( q )
                        {
                            bounding_queue.push_back( std::make_pair( get_time( *q ), bounding_stream.last() ));
                        }
                        else
                        {
                            bounding_data_available=false;
                        }
                    }
                    if( buffer_size && bounding_queue.size() > *buffer_size && discard_bounding )
                    {
                        bounding_queue.pop_front();
                    }
                }
                if( !upper_bound_added && bounding_istream->eof() )
                {
                    // add a fake entry for an upper bound to allow stdin data above last bound to match
                    bounding_queue.push_back( std::make_pair( boost::posix_time::pos_infin, "" ));
                    upper_bound_added = true;
                }

                //if we are done with the last bounded point get next
                if( next )
                {
                    if(!stdin_stream_ready) { continue; }
                    p = stdin_stream.read();
                    if( !p ) { break; }
                }

                boost::posix_time::ptime t = get_time(*p);
              
                //get bound
                while(bounding_queue.size()>=2)
                {
                    if( t < bounding_queue[1].first ) { break; }
                    bounding_queue.pop_front();
                }

                if(bounding_queue.size()<2)
                {
                    //bound not found
                    //do we have more data?
                    if(!bounding_data_available) { break; }
                    next=false;
                    continue;
                }

                //bound available

                if( method == how::by_lower && t < bounding_queue.front().first )
                {
                    next = true;
                    continue;
                }

                bool is_first = ( method == how::by_lower )
                    || ( method == how::nearest && ( t - bounding_queue[0].first ) < ( bounding_queue[1].first - t ));

                const timestring_t& chosen_bound = is_first ? bounding_queue[0] : bounding_queue[1];;
                timestring_t input_line = std::make_pair( t, stdin_stream.last() );

                output( input_line, chosen_bound, stdin_first );
                next=true;
            }
        }
        return 0;     
    }
    catch( std::exception& ex ) { std::cerr << "csv-time-join: " << ex.what() << std::endl; }
    catch( ... ) { std::cerr << "csv-time-join: unknown exception" << std::endl; }
}