Esempio n. 1
0
void MasterSession::on_connect( )
{
    if ( Variable::mode == "p" )
    { 
        Variable::istream.open( Variable::local_path , std::ios::in | std::ios::binary );
        Variable::istream.seekg( 0 , std::ios::end );
        size_t size = Variable::istream.tellg( );
        Logger::log( "Put File Size: %" , size );
        uptr<MessageRequestPut> msg = make_uptr( MessageRequestPut );
        msg->set_path( Variable::remote_path );
        msg->set_request_id( MRT::UUID::create( ) );
        msg->set_size( size );
        this->send_message( move_ptr( msg ) );

        Variable::istream.close( );
    }
    else if( Variable::mode == "g" )
    {
        uptr<MessageRequestGet> msg = make_uptr( MessageRequestGet );
        msg->set_path( Variable::remote_path );
        msg->set_request_id( MRT::UUID::create( ) );
        this->send_message( move_ptr( msg ) );
    }
    else
    {
        this->close( );
    }
}
    bool MessageHub::AddAllHandlers()
    {
        AddHandler( std::move( make_uptr( Protocal::MessageGreetingHandler         ) ) );
        AddHandler( std::move( make_uptr( Protocal::MessageRegistHandler           ) ) );
        AddHandler( std::move( make_uptr( Protocal::MessageTaskDeliverHandler      ) ) );
        AddHandler( std::move( make_uptr( Protocal::MessageTaskDeliverReplyHandler ) ) );

        AddHandler(std::move(make_uptr(Protocal::MessageTaskAbortHandler)));

        return true;
    }
    static int HTTPMessageListExecutorHandler( HTTPMessageListExecutor msg )
    {
        // UserDefineHandler Begin
        // Your Codes here!

        uptr<HTTPMessageResult> result = make_uptr( HTTPMessageResult );

        result->result( 0 );

        auto list = ExecutorManager::instance()->list();
        MRT::json json;

        for ( auto exe : list )
        {
            MRT::json json_exe;
            json_exe["id"] = exe->id();
            json_exe["status"] = exe->status();
            json_exe["mem"] = exe->memory_size();
            json_exe["disk"] = exe->disk_size();
            json_exe["ability"] = exe->ability();
            json_exe["task_id"] = exe->current_task() == nullptr ? "" : exe->current_task()->id();
            json.push_back( json_exe );
        }

        result->object( json.dump() );
        
        msg.owner()->send_message( move_ptr(result) );

        return 0;

        // UserDefineHandler End 
    }
    static int HTTPMessageTaskStartHandler( HTTPMessageTaskStart msg )
    {
        // UserDefineHandler Begin
        // Your Codes here!

        uptr<HTTPMessageResult> result = make_uptr( HTTPMessageResult );

        TaskDescripter* task = new TaskDescripter();
        task->aligner( msg.aligner() );
        task->args( msg.args() );
        task->fastq( msg.fastq() );
        task->id( msg.id() );
        task->executor( msg.executor() );
        task->reference( msg.reference( ) );
        task->sorter( msg.sorter( ) );

        auto err = TaskManager::instance()->launch( task );
       
        result->result( static_cast< int >( err.code() ) );
        result->message( err.message() );

        msg.owner()->send_message( move_ptr(result) );

        return 0;

        // UserDefineHandler End 
    }
void Listener::uv_read_callback( uv_stream_t * stream , 
                                 ssize_t nread , 
                                 const uv_buf_t * buf )
{
    Session* session = scast<Session*>( stream->data );

    if ( session == nullptr )
    {
        delete buf->base;
        LOG_DEBUG( "Session is nullptr!" );
        return;
    }

    if ( nread < 0 )
    {
        delete buf->base;
        LOG_DEBUG_UV( nread );
        uv_close( ( uv_handle_t* ) &session->uv_tcp_ , 
                  Listener::uv_close_callback );
        return;
    }

    uptr<Buffer> pbuf = make_uptr( Buffer , buf->base , nread );
    session->on_read( move_ptr( pbuf ) );

    delete buf->base;
    //delete buf;
}
void ClusterSession::send_message( uptr<Message> message )
{
    uptr<MRT::Buffer> header  = make_uptr( MRT::Buffer , "YH" );
    uptr<MRT::Buffer> length  = make_uptr( MRT::Buffer , 4 );
    uptr<MRT::Buffer> body    = message->to_bytes( );

    int* pshort = ( int* ) ( length->data( ) );
    *pshort = scast<int>( body->size( ) );
    //++pshort;
    //*pshort = scast<int>( body->size( ) );

    this->send( move_ptr( header ) );
    this->send( move_ptr( length ) );
    this->send( move_ptr( body ) );

    LOG_DEBUG( "send message %lld to %lld" , this->id( ) , message->command( ) );
}
Esempio n. 7
0
bool BlockDistributer::get_file( sptr<FileMeta> file ,
                                 sptr<ClientSession> client ,
                                 uptr<MessageRequestGet> msg )
{
    if ( file == nullptr || client == nullptr || msg == nullptr )
         return false;

    auto blocks = file->blocks( );

    //sptr<RequestTokenCollection> token = 
    //    RequestTokenTable::instance( )->create( sptr<ClusterSession>( client ) , 
    //                                            msg->request_id( ) ,
    //                                            blocks.size( ) );

    for ( auto & block : blocks )
    {
        auto nodes = block->nodes( );

        size_t min_block_node_id = MAX_SIZE_T;
        sptr<NodeSession> node_session = nullptr;
        sptr<NodeMeta> node_meta = nullptr;

        for ( auto & node : nodes )
        {
            auto n = NodeManager::instance( )->find_node( [node] ( sptr<NodeSession> ns )
            {
                return ns->id( ) == node->node_id( );
            } );

            if ( n != nullptr )
            {
                if ( n->block_num( ) <= min_block_node_id )
                {
                    min_block_node_id = n->id( );
                    node_session = n;
                    node_meta = node;
                }
            }
        }

        if ( node_meta == nullptr )
        {
            return false;
        }

        uptr<MessageRequestGetToken> req = make_uptr( MessageRequestGetToken );
        req->set_index( node_meta->index( ) );
        req->set_request_id( msg->request_id( ) );
        req->set_block_id( block->id( ) );
        req->set_client_id( client->id( ) );
        node_session->send_message( move_ptr( req ) );
    }

    client->token_num( blocks.size( ) );

    return true;
}
Esempio n. 8
0
void MasterSession::on_connect( )
{
    uptr<MessageRegister> msg = make_uptr( MessageRegister );
    msg->set_id( MRT::UUID::create( ) );
    msg->set_block_num( BlockTable::instance( )->block_num( ) );
    msg->set_data_addr ( "" );
    msg->set_data_port ( 0 );
    msg->set_disk_space( 1000 );
    this->send_message( move_ptr( msg ) );
}
Esempio n. 9
0
bool BlockDistributer::put_file( size_t file_size ,
                                 sptr<ClientSession> client ,
                                 uptr<MessageRequestPut> msg )
{
    LOG_DEBUG( "file size num %lld " , file_size );

    auto size_left = file_size;

    size_t block_num        = ( size_t ) ceil( ( double ) file_size / ( double ) ( BLOCK_SIZE ) );
    size_t max_block_num    = 0;
    size_t offset           = 0;

    NodeManager::instance( )->sort( [ ] ( sptr<NodeSession> n1 , sptr<NodeSession> n2 )
    {
        return n1->block_num( ) > n2->block_num( );
    } );

    if ( NodeManager::instance( )->count( ) == 0 )
    {
        LOG_DEBUG( "node count is 0 exit" , block_num );
        client->close( );
        return false;
    }

    for ( size_t i = 0; i < block_num; i++ )
    {
        if ( size_left == 0 ) break;

        auto c_size = size_left > BLOCK_SIZE ? BLOCK_SIZE : size_left;

        auto n = NodeManager::instance( )->get_node( i % NodeManager::instance( )->count( ) );

        if ( n == nullptr )
        {
            LOG_DEBUG( "No node found exit..." , n->id( ) );
            return false;
        }


        uptr<MessageRequestPutToken> req = make_uptr( MessageRequestPutToken );
        req->set_request_id( msg->request_id( ) );
        req->set_block_id( i );
        req->set_size( c_size );
        req->set_client_id( client->id( ) );
        req->set_file_name( msg->path( ) );
        req->set_offset( offset );
        n->send_message( move_ptr( req ) );

        size_left -= c_size;
        offset += c_size;
    }

    client->token_num( block_num );
    return true;
}
Esempio n. 10
0
int main(int argc,char** argv)
{     
    while(true)
    {
        MRT::SyncWorker::create( 1 , [ ] ( MRT::SyncWorker* worker )
        { 
            ExecutorManager::instance( )->run( );
            TaskManager::instance( )->run( );

            return false;
        } , nullptr, nullptr );

        MRT::Maraton::instance( )->regist(
            make_uptr( ExecutorListener , "0.0.0.0" ) );
        MRT::Maraton::instance( )->regist(
            make_uptr( RestAPIListener , "0.0.0.0" ) );
        MRT::Maraton::instance( )->loop( );
    }
    return 0;
}
Esempio n. 11
0
int main( int argc , char * argv[] )
{     
    if ( !CMDParameter::parse( argc , argv ) )
    {
        return 0;
    }

    // Server mode
    if ( Variable::mode == 1 )
    {
        LOG_SYS( "system start in master mode" );

        MRT::Maraton::instance( )->regist( make_uptr( NodeListener , 
                                           "0.0.0.0" , 
                                           100)); 

        MRT::Maraton::instance( )->regist( make_uptr( ClientListener , 
                                           "0.0.0.0"  , 
                                           101));

        MRT::Maraton::instance( )->regist( make_uptr( MasterHTTPListener ,
                                           "0.0.0.0"  ,
                                           80 ) );

        MRT::Maraton::instance( )->run( );

        LOG_SYS( "system shutdown" );
    }
    // Node mode
    else if ( Variable::mode == 0 )
    {
        LOG_SYS( "system start in node mode" );
        LOG_SYS( "load index file" );
        
        BlockTable::instance( )->load_from_file( );
        
        LOG_SYS( "connecting %:%" , Variable::server_ip.c_str( ) , Variable::port );

        MRT::Maraton::instance( )->regist( make_uptr( ClientListener ,
                                           "0.0.0.0"  ,
                                           101 ) );

        MRT::Maraton::instance( )->regist( make_uptr( NodeHTTPListener ,
                                           "0.0.0.0"  ,
                                           80 ) );

        MRT::Maraton::instance( )->regist( make_uptr( MasterConnector , 
                                           Variable::server_ip , 
                                           100));
        while ( true )
        {
            
            MRT::Maraton::instance( )->run( );
            LOG_SYS( "disconnected to server , reconnecting" );
        }
    }
    
    return 0;
}
Esempio n. 12
0
void MasterSession::alive( )
{
    worker_ = MRT::SyncWorker::create( 1000 , [this] ( MRT::SyncWorker* worker )
    {
        uptr<MessageAlive> alive = make_uptr( MessageAlive );
        alive->set_block_num( BlockTable::instance( )->block_num( ) );
        alive->set_disk_space( 0 );
        alive->set_session_id( this->session_id( ) );
        alive->set_token_num( TokenPool::instance( )->token_num( ) );
        alive->set_local_time( Timer::tick( ) );
        this->send_message( move_ptr( alive ) );
        return false;
    } , nullptr , nullptr );
}
    static int HTTPMessageCommandHandler( HTTPMessageCommand msg )
    {
        // UserDefineHandler Begin
        // Your Codes here!

        uptr<HTTPMessageResult> result = make_uptr( HTTPMessageResult );

        result->result( 0 );

        std::string ret_data = "";
        for ( std::string exe_id : msg.executors() )
        {
            auto exe = ExecutorManager::instance()->find( exe_id );

            if ( exe == nullptr )
            {
                ret_data += exe_id + " not found\r\n";
                result->result( 1 );
                break;
            }

            if ( exe->status() != Executor::ExecutorStatus::kStandby )
            {
                ret_data += exe_id + " not kIdle\r\n";
                result->result( 1 );
                break;
            }

            //MessageCommand cmd_msg;
            //cmd_msg.command_line( msg.cmd() );
            //cmd_msg.uri_list( msg.uris() );
            //cmd_msg.run_as( msg.runAsUser() );

            //exe->session()->send( &cmd_msg );
            //ret_data += "Executor " + exe_id + " send";
        }

        result->message( ret_data );
        //msg.owner()->send( MOVE( result ) );

        return 0;

        // UserDefineHandler End 
    }
void DuplicateConnector::OnSessionClose( Session * session )
{
    auto s = DuplicateSessionPool::Instance()->Pop( scast<DuplicateSession*>( session ) );

    if ( session->LastError().Code() != 0 &&
         session->LastError().Code() != -4095 )
    {
        Logger::Error( "duplicate error <%>%" , session->LastError().Code() ,
                       session->LastError().Message() );
    }

    if ( session->LastError().Code() == -4078 )
    {
        Logger::Log( "retring duplicate % from %" ,
                     this->message_->path() ,
                     this->message_->address() );
        auto connector = make_uptr( DuplicateConnector , move_ptr( this->message_ ) );
        MRT::Maraton::Instance()->Regist( move_ptr( connector ) );
    }
     
    SAFE_DELETE( session );
}
Esempio n. 15
0
void MasterSession::sync_all_block( )
{ 
    auto block_num = BlockTable::instance( )->block_num( );

    for ( size_t i = 0; i < block_num; i++ )
    {
        auto block = BlockTable::instance( )->find_block( i );

        if ( false == block->is_used ) continue;

        uptr<MessageSyncBlock> msg = make_uptr( MessageSyncBlock );

        msg->set_id( block->block_id );
        msg->set_index( block->index );
        msg->set_path( block->path );
        msg->set_size( block->size );
        msg->set_status( 0 );

        this->send_message( move_ptr( msg ) );
    }

    LOG_DEBUG( "Sync % files in total" , block_num );

}
Esempio n. 16
0
void StructCompactor::ItemVec::make_uptv( String &os, const String &pr, int par_level, const String &sp, bool save ) {
    String i = char( 'i' + par_level );
    os << sp << "for( ST " << i << " = 0; " << i << " < " << pr << ".size_; ++" << i << " ) {\n";
    make_uptr( os, pr + ".data_[ " + i + " ]", par_level + 1, sp + "    ", save );
    os << sp << "}\n";
}
Esempio n. 17
0
void ClusterSession::try_pop_message( )
{
    while ( true )
    {
        switch ( this->state_ )
        {
            case MessageParseState::Header:
                {
                    auto tmp = this->buffer_.pop( 2 );
                    if ( tmp == nullptr ) return;


                    auto kf = ( *tmp )[0];
                    auto ks = ( *tmp )[1];

                    if ( kf == 'Y' && ks == 'H' )
                    {
                        this->state_ = MessageParseState::Length;
                    }
                    else
                    {
                        this->close( );
                        return;
                    }
                }
                break;
            case MessageParseState::Length:
                {
                    auto tmp = this->buffer_.pop( 4 );

                    if ( tmp == nullptr ) return;

                    short* pshort           = ( short* ) ( tmp->data( ) );
                    this->oringal_size_     = *pshort;
                    ++pshort;
                    this->compressed_size_  = *pshort;

                    if ( this->compressed_size_ < MAX_CIRCLE_BUFFER_SIZE &&
                         this->oringal_size_ < MAX_CIRCLE_BUFFER_SIZE )
                    {
                        this->state_ = MessageParseState::Body;
                    }
                    else
                    {
                        this->state_ = MessageParseState::Header;
                        this->close( );
                        return;
                    }
                }
                break;
            case MessageParseState::Body:
                {
                    auto tmp = this->buffer_.pop( this->compressed_size_ );

                    if ( tmp == nullptr ) return;

                    try
                    {
                        auto msg = make_uptr( Message , std::string( tmp->data( ) , tmp->size( ) ) );
                        msg->owner( this );
                        this->on_message( move_ptr( msg ) );
                    }
                    catch ( ... )
                    {

                    }

                    this->state_ = MessageParseState::Header;
                }
                break;
            default:
                break;
        }
    }
}