void MasterSession::on_connect( ) { if ( Variable::mode == "p" ) { Variable::istream.open( Variable::local_path , std::ios::in | std::ios::binary ); Variable::istream.seekg( 0 , std::ios::end ); size_t size = Variable::istream.tellg( ); Logger::log( "Put File Size: %" , size ); uptr<MessageRequestPut> msg = make_uptr( MessageRequestPut ); msg->set_path( Variable::remote_path ); msg->set_request_id( MRT::UUID::create( ) ); msg->set_size( size ); this->send_message( move_ptr( msg ) ); Variable::istream.close( ); } else if( Variable::mode == "g" ) { uptr<MessageRequestGet> msg = make_uptr( MessageRequestGet ); msg->set_path( Variable::remote_path ); msg->set_request_id( MRT::UUID::create( ) ); this->send_message( move_ptr( msg ) ); } else { this->close( ); } }
bool MessageHub::AddAllHandlers() { AddHandler( std::move( make_uptr( Protocal::MessageGreetingHandler ) ) ); AddHandler( std::move( make_uptr( Protocal::MessageRegistHandler ) ) ); AddHandler( std::move( make_uptr( Protocal::MessageTaskDeliverHandler ) ) ); AddHandler( std::move( make_uptr( Protocal::MessageTaskDeliverReplyHandler ) ) ); AddHandler(std::move(make_uptr(Protocal::MessageTaskAbortHandler))); return true; }
static int HTTPMessageListExecutorHandler( HTTPMessageListExecutor msg ) { // UserDefineHandler Begin // Your Codes here! uptr<HTTPMessageResult> result = make_uptr( HTTPMessageResult ); result->result( 0 ); auto list = ExecutorManager::instance()->list(); MRT::json json; for ( auto exe : list ) { MRT::json json_exe; json_exe["id"] = exe->id(); json_exe["status"] = exe->status(); json_exe["mem"] = exe->memory_size(); json_exe["disk"] = exe->disk_size(); json_exe["ability"] = exe->ability(); json_exe["task_id"] = exe->current_task() == nullptr ? "" : exe->current_task()->id(); json.push_back( json_exe ); } result->object( json.dump() ); msg.owner()->send_message( move_ptr(result) ); return 0; // UserDefineHandler End }
static int HTTPMessageTaskStartHandler( HTTPMessageTaskStart msg ) { // UserDefineHandler Begin // Your Codes here! uptr<HTTPMessageResult> result = make_uptr( HTTPMessageResult ); TaskDescripter* task = new TaskDescripter(); task->aligner( msg.aligner() ); task->args( msg.args() ); task->fastq( msg.fastq() ); task->id( msg.id() ); task->executor( msg.executor() ); task->reference( msg.reference( ) ); task->sorter( msg.sorter( ) ); auto err = TaskManager::instance()->launch( task ); result->result( static_cast< int >( err.code() ) ); result->message( err.message() ); msg.owner()->send_message( move_ptr(result) ); return 0; // UserDefineHandler End }
void Listener::uv_read_callback( uv_stream_t * stream , ssize_t nread , const uv_buf_t * buf ) { Session* session = scast<Session*>( stream->data ); if ( session == nullptr ) { delete buf->base; LOG_DEBUG( "Session is nullptr!" ); return; } if ( nread < 0 ) { delete buf->base; LOG_DEBUG_UV( nread ); uv_close( ( uv_handle_t* ) &session->uv_tcp_ , Listener::uv_close_callback ); return; } uptr<Buffer> pbuf = make_uptr( Buffer , buf->base , nread ); session->on_read( move_ptr( pbuf ) ); delete buf->base; //delete buf; }
void ClusterSession::send_message( uptr<Message> message ) { uptr<MRT::Buffer> header = make_uptr( MRT::Buffer , "YH" ); uptr<MRT::Buffer> length = make_uptr( MRT::Buffer , 4 ); uptr<MRT::Buffer> body = message->to_bytes( ); int* pshort = ( int* ) ( length->data( ) ); *pshort = scast<int>( body->size( ) ); //++pshort; //*pshort = scast<int>( body->size( ) ); this->send( move_ptr( header ) ); this->send( move_ptr( length ) ); this->send( move_ptr( body ) ); LOG_DEBUG( "send message %lld to %lld" , this->id( ) , message->command( ) ); }
bool BlockDistributer::get_file( sptr<FileMeta> file , sptr<ClientSession> client , uptr<MessageRequestGet> msg ) { if ( file == nullptr || client == nullptr || msg == nullptr ) return false; auto blocks = file->blocks( ); //sptr<RequestTokenCollection> token = // RequestTokenTable::instance( )->create( sptr<ClusterSession>( client ) , // msg->request_id( ) , // blocks.size( ) ); for ( auto & block : blocks ) { auto nodes = block->nodes( ); size_t min_block_node_id = MAX_SIZE_T; sptr<NodeSession> node_session = nullptr; sptr<NodeMeta> node_meta = nullptr; for ( auto & node : nodes ) { auto n = NodeManager::instance( )->find_node( [node] ( sptr<NodeSession> ns ) { return ns->id( ) == node->node_id( ); } ); if ( n != nullptr ) { if ( n->block_num( ) <= min_block_node_id ) { min_block_node_id = n->id( ); node_session = n; node_meta = node; } } } if ( node_meta == nullptr ) { return false; } uptr<MessageRequestGetToken> req = make_uptr( MessageRequestGetToken ); req->set_index( node_meta->index( ) ); req->set_request_id( msg->request_id( ) ); req->set_block_id( block->id( ) ); req->set_client_id( client->id( ) ); node_session->send_message( move_ptr( req ) ); } client->token_num( blocks.size( ) ); return true; }
void MasterSession::on_connect( ) { uptr<MessageRegister> msg = make_uptr( MessageRegister ); msg->set_id( MRT::UUID::create( ) ); msg->set_block_num( BlockTable::instance( )->block_num( ) ); msg->set_data_addr ( "" ); msg->set_data_port ( 0 ); msg->set_disk_space( 1000 ); this->send_message( move_ptr( msg ) ); }
bool BlockDistributer::put_file( size_t file_size , sptr<ClientSession> client , uptr<MessageRequestPut> msg ) { LOG_DEBUG( "file size num %lld " , file_size ); auto size_left = file_size; size_t block_num = ( size_t ) ceil( ( double ) file_size / ( double ) ( BLOCK_SIZE ) ); size_t max_block_num = 0; size_t offset = 0; NodeManager::instance( )->sort( [ ] ( sptr<NodeSession> n1 , sptr<NodeSession> n2 ) { return n1->block_num( ) > n2->block_num( ); } ); if ( NodeManager::instance( )->count( ) == 0 ) { LOG_DEBUG( "node count is 0 exit" , block_num ); client->close( ); return false; } for ( size_t i = 0; i < block_num; i++ ) { if ( size_left == 0 ) break; auto c_size = size_left > BLOCK_SIZE ? BLOCK_SIZE : size_left; auto n = NodeManager::instance( )->get_node( i % NodeManager::instance( )->count( ) ); if ( n == nullptr ) { LOG_DEBUG( "No node found exit..." , n->id( ) ); return false; } uptr<MessageRequestPutToken> req = make_uptr( MessageRequestPutToken ); req->set_request_id( msg->request_id( ) ); req->set_block_id( i ); req->set_size( c_size ); req->set_client_id( client->id( ) ); req->set_file_name( msg->path( ) ); req->set_offset( offset ); n->send_message( move_ptr( req ) ); size_left -= c_size; offset += c_size; } client->token_num( block_num ); return true; }
int main(int argc,char** argv) { while(true) { MRT::SyncWorker::create( 1 , [ ] ( MRT::SyncWorker* worker ) { ExecutorManager::instance( )->run( ); TaskManager::instance( )->run( ); return false; } , nullptr, nullptr ); MRT::Maraton::instance( )->regist( make_uptr( ExecutorListener , "0.0.0.0" ) ); MRT::Maraton::instance( )->regist( make_uptr( RestAPIListener , "0.0.0.0" ) ); MRT::Maraton::instance( )->loop( ); } return 0; }
int main( int argc , char * argv[] ) { if ( !CMDParameter::parse( argc , argv ) ) { return 0; } // Server mode if ( Variable::mode == 1 ) { LOG_SYS( "system start in master mode" ); MRT::Maraton::instance( )->regist( make_uptr( NodeListener , "0.0.0.0" , 100)); MRT::Maraton::instance( )->regist( make_uptr( ClientListener , "0.0.0.0" , 101)); MRT::Maraton::instance( )->regist( make_uptr( MasterHTTPListener , "0.0.0.0" , 80 ) ); MRT::Maraton::instance( )->run( ); LOG_SYS( "system shutdown" ); } // Node mode else if ( Variable::mode == 0 ) { LOG_SYS( "system start in node mode" ); LOG_SYS( "load index file" ); BlockTable::instance( )->load_from_file( ); LOG_SYS( "connecting %:%" , Variable::server_ip.c_str( ) , Variable::port ); MRT::Maraton::instance( )->regist( make_uptr( ClientListener , "0.0.0.0" , 101 ) ); MRT::Maraton::instance( )->regist( make_uptr( NodeHTTPListener , "0.0.0.0" , 80 ) ); MRT::Maraton::instance( )->regist( make_uptr( MasterConnector , Variable::server_ip , 100)); while ( true ) { MRT::Maraton::instance( )->run( ); LOG_SYS( "disconnected to server , reconnecting" ); } } return 0; }
void MasterSession::alive( ) { worker_ = MRT::SyncWorker::create( 1000 , [this] ( MRT::SyncWorker* worker ) { uptr<MessageAlive> alive = make_uptr( MessageAlive ); alive->set_block_num( BlockTable::instance( )->block_num( ) ); alive->set_disk_space( 0 ); alive->set_session_id( this->session_id( ) ); alive->set_token_num( TokenPool::instance( )->token_num( ) ); alive->set_local_time( Timer::tick( ) ); this->send_message( move_ptr( alive ) ); return false; } , nullptr , nullptr ); }
static int HTTPMessageCommandHandler( HTTPMessageCommand msg ) { // UserDefineHandler Begin // Your Codes here! uptr<HTTPMessageResult> result = make_uptr( HTTPMessageResult ); result->result( 0 ); std::string ret_data = ""; for ( std::string exe_id : msg.executors() ) { auto exe = ExecutorManager::instance()->find( exe_id ); if ( exe == nullptr ) { ret_data += exe_id + " not found\r\n"; result->result( 1 ); break; } if ( exe->status() != Executor::ExecutorStatus::kStandby ) { ret_data += exe_id + " not kIdle\r\n"; result->result( 1 ); break; } //MessageCommand cmd_msg; //cmd_msg.command_line( msg.cmd() ); //cmd_msg.uri_list( msg.uris() ); //cmd_msg.run_as( msg.runAsUser() ); //exe->session()->send( &cmd_msg ); //ret_data += "Executor " + exe_id + " send"; } result->message( ret_data ); //msg.owner()->send( MOVE( result ) ); return 0; // UserDefineHandler End }
void DuplicateConnector::OnSessionClose( Session * session ) { auto s = DuplicateSessionPool::Instance()->Pop( scast<DuplicateSession*>( session ) ); if ( session->LastError().Code() != 0 && session->LastError().Code() != -4095 ) { Logger::Error( "duplicate error <%>%" , session->LastError().Code() , session->LastError().Message() ); } if ( session->LastError().Code() == -4078 ) { Logger::Log( "retring duplicate % from %" , this->message_->path() , this->message_->address() ); auto connector = make_uptr( DuplicateConnector , move_ptr( this->message_ ) ); MRT::Maraton::Instance()->Regist( move_ptr( connector ) ); } SAFE_DELETE( session ); }
void MasterSession::sync_all_block( ) { auto block_num = BlockTable::instance( )->block_num( ); for ( size_t i = 0; i < block_num; i++ ) { auto block = BlockTable::instance( )->find_block( i ); if ( false == block->is_used ) continue; uptr<MessageSyncBlock> msg = make_uptr( MessageSyncBlock ); msg->set_id( block->block_id ); msg->set_index( block->index ); msg->set_path( block->path ); msg->set_size( block->size ); msg->set_status( 0 ); this->send_message( move_ptr( msg ) ); } LOG_DEBUG( "Sync % files in total" , block_num ); }
void StructCompactor::ItemVec::make_uptv( String &os, const String &pr, int par_level, const String &sp, bool save ) { String i = char( 'i' + par_level ); os << sp << "for( ST " << i << " = 0; " << i << " < " << pr << ".size_; ++" << i << " ) {\n"; make_uptr( os, pr + ".data_[ " + i + " ]", par_level + 1, sp + " ", save ); os << sp << "}\n"; }
void ClusterSession::try_pop_message( ) { while ( true ) { switch ( this->state_ ) { case MessageParseState::Header: { auto tmp = this->buffer_.pop( 2 ); if ( tmp == nullptr ) return; auto kf = ( *tmp )[0]; auto ks = ( *tmp )[1]; if ( kf == 'Y' && ks == 'H' ) { this->state_ = MessageParseState::Length; } else { this->close( ); return; } } break; case MessageParseState::Length: { auto tmp = this->buffer_.pop( 4 ); if ( tmp == nullptr ) return; short* pshort = ( short* ) ( tmp->data( ) ); this->oringal_size_ = *pshort; ++pshort; this->compressed_size_ = *pshort; if ( this->compressed_size_ < MAX_CIRCLE_BUFFER_SIZE && this->oringal_size_ < MAX_CIRCLE_BUFFER_SIZE ) { this->state_ = MessageParseState::Body; } else { this->state_ = MessageParseState::Header; this->close( ); return; } } break; case MessageParseState::Body: { auto tmp = this->buffer_.pop( this->compressed_size_ ); if ( tmp == nullptr ) return; try { auto msg = make_uptr( Message , std::string( tmp->data( ) , tmp->size( ) ) ); msg->owner( this ); this->on_message( move_ptr( msg ) ); } catch ( ... ) { } this->state_ = MessageParseState::Header; } break; default: break; } } }