int ObClientManager::send_request(const ObServer& server, const int32_t pcode, const int32_t version, const int64_t timeout, ObDataBuffer& in_buffer, ObDataBuffer& out_buffer, int64_t& session_id) const { int rc = OB_SUCCESS; ObPacket* response = NULL; rc = do_send_request(server, pcode, version, timeout, in_buffer, response); // deserialize response packet to out_buffer if (OB_SUCCESS == rc && NULL != response) { session_id = response->get_session_id() ; // TODO // copy response's inner_buffer to out_buffer. int64_t data_length = response->get_data_length(); ObDataBuffer* response_buffer = response->get_buffer(); if (out_buffer.get_remain() < data_length) { TBSYS_LOG(ERROR, "insufficient memory in out_buffer, remain:%ld, length=%ld", out_buffer.get_remain(), data_length); rc = OB_ERROR; } else { memcpy(out_buffer.get_data() + out_buffer.get_position(), response_buffer->get_data() + response_buffer->get_position(), data_length); out_buffer.get_position() += data_length; } } return rc; }
static int process(easy_request_t *r) { int ret = EASY_OK; if (NULL == r) { TBSYS_LOG(WARN, "request is NULL, r = %p", r); ret = EASY_BREAK; } else if (NULL == r->ipacket) { TBSYS_LOG(WARN, "request is NULL, r->ipacket = %p", r->ipacket); ret = EASY_BREAK; } else { ObFileServer* server = (ObFileServer*)r->ms->c->handler->user_data; ObPacket* packet = (ObPacket*)r->ipacket; packet->set_request(r); r->ms->c->pool->ref++; easy_atomic_inc(&r->ms->pool->ref); easy_pool_set_lock(r->ms->pool); if (OB_REQUIRE_HEARTBEAT == packet->get_packet_code()) { server->handle_request(packet); } else { server->handlePacket(packet); } ret = EASY_AGAIN; } return ret; }
int ObTbnetCallback::shadow_process(easy_request_t* r) { int ret = EASY_OK; if (NULL == r || NULL == r->ipacket) { TBSYS_LOG(ERROR, "request is empty, r = %p, r->ipacket = %p", r, r->ipacket); ret = EASY_BREAK; } else { ObShadowServer *server = (ObShadowServer*)r->ms->c->handler->user_data; ObPacket *req = (ObPacket*) r->ipacket; req->set_request(r); ret = server->handlePacket(req); if (OB_SUCCESS == ret) { r->ms->c->pool->ref++; easy_atomic_inc(&r->ms->pool->ref); easy_pool_set_lock(r->ms->pool); ret = EASY_AGAIN; } else { ret = EASY_OK; TBSYS_LOG(WARN, "can not push packet(src is %s, pcode is %u) to packet queue", inet_ntoa_r(r->ms->c->addr), req->get_packet_code()); } } return ret; }
tbnet::IPacketHandler::HPRetCode MockNodeServer::handlePacket(tbnet::Connection* connection, tbnet::Packet *packet) { tbnet::IPacketHandler::HPRetCode rc = tbnet::IPacketHandler::FREE_CHANNEL; if (!packet->isRegularPacket()) { TBSYS_LOG(WARN, "control packet, packet code: %d", ((tbnet::ControlPacket*)packet)->getCommand()); } else { ObPacket* req = (ObPacket*) packet; req->set_connection(connection); int packet_code = req->get_packet_code(); bool ps = false; switch(packet_code) { case OB_MMS_HEART_BEAT: case OB_MMS_STOP_SLAVE: case OB_MMS_SLAVE_DOWN: case OB_MMS_TRANSFER_2_MASTER: ps = node_.handlePacket(req); break; default: ps = MockServer::handlePacket(connection, packet); break; } if (!ps) { TBSYS_LOG(WARN, "overflow packet dropped, packet code: %d", req->getPCode()); rc = tbnet::IPacketHandler::KEEP_CHANNEL; } } return rc; }
int MockNodeServer::do_request(ObPacket* base_packet) { int err = OB_SUCCESS; ObPacket* req = static_cast<ObPacket*>(base_packet); int packet_code = req->get_packet_code(); err = req->deserialize(); if(OB_SUCCESS == err) { switch(packet_code) { case OB_MMS_HEART_BEAT: case OB_MMS_STOP_SLAVE: case OB_MMS_SLAVE_DOWN: case OB_MMS_TRANSFER_2_MASTER: err = node_.handlePacket(req); break; default: err = OB_ERROR; TBSYS_LOG(ERROR, "wrong packet code:packet[%d]", packet_code); break; } } if(err != OB_SUCCESS) { TBSYS_LOG(WARN, "check handle failed:ret[%d]", err); } return err; }
int ObMsSqlRpcEvent::deserialize_packet(ObPacket & packet, ObNewScanner & result) { ObDataBuffer * data_buff = NULL; int ret = packet.deserialize(); if (ret != OB_SUCCESS) { TBSYS_LOG(WARN, "deserialize the packet failed:ret[%d]", ret); } else { data_buff = packet.get_buffer(); if (NULL == data_buff) { ret = OB_INNER_STAT_ERROR; TBSYS_LOG(WARN, "check packet data buff failed:buff[%p]", data_buff); } if (packet.get_packet_code() == OB_SESSION_END) { /// when session end, set session id to 0 set_session_end(); } else { set_session_id(packet.get_session_id()); } } ObResultCode code; if (OB_SUCCESS == ret) { ret = code.deserialize(data_buff->get_data(), data_buff->get_capacity(), data_buff->get_position()); if (OB_SUCCESS != ret) { TBSYS_LOG(ERROR, "deserialize result failed:pos[%ld], ret[%d]", data_buff->get_position(), ret); } else { ObCommonSqlRpcEvent::set_result_code(code.result_code_); } } /// result.clear(); if ((OB_SUCCESS == ret) && (OB_SUCCESS == code.result_code_)) { ret = result.deserialize(data_buff->get_data(), data_buff->get_capacity(), data_buff->get_position()); if (ret != OB_SUCCESS) { TBSYS_LOG(WARN, "deserialize scanner failed:pos[%ld], ret[%d]", data_buff->get_position(), ret); } } return ret; }
int ObClientManager::do_post_request(const ObServer& server, const int32_t pcode, const int32_t version, const int64_t session_id, const int64_t timeout, const ObDataBuffer& in_buffer, tbnet::IPacketHandler* handler, void* args) const { int rc = OB_SUCCESS; ObPacket* packet = new (std::nothrow) ObPacket(); if (NULL == packet) { rc = OB_ALLOCATE_MEMORY_FAILED; } else if (OB_SUCCESS != error_) { packet->free(); rc = error_; TBSYS_LOG(ERROR, "prev_error=%d", error_); } else { packet->set_packet_code(pcode); packet->setChannelId(0); packet->set_source_timeout(timeout); packet->set_session_id(session_id); packet->set_api_version(version); packet->set_data(in_buffer); if (timeout > max_request_timeout_) { max_request_timeout_ = timeout; connmgr_->setDefaultQueueTimeout(0, static_cast<int32_t>(max_request_timeout_ / 1000)); } rc = packet->serialize(); if (OB_SUCCESS != rc) { TBSYS_LOG(WARN, "packet serialize error"); packet->free(); packet = NULL; } else { rc = do_post_packet(server, packet, handler, args); } } return rc; }
int ObProxyCallback::process(easy_request_t *r) { int ret = EASY_OK; if (NULL == r) { TBSYS_LOG(WARN, "request is NULL, r = %p", r); ret = EASY_BREAK; } else if (NULL == r->ipacket) { TBSYS_LOG(WARN, "request is NULL, r->ipacket = %p", r->ipacket); ret = EASY_BREAK; } else { ObProxyServer* server = (ObProxyServer*)r->ms->c->handler->user_data; ObPacket* packet = (ObPacket*)r->ipacket; packet->set_request(r); //handle_request will send response if (OB_REQUIRE_HEARTBEAT == packet->get_packet_code()) { server->handle_request(packet); ret = EASY_OK; } else { ret = server->handlePacket(packet); if (OB_SUCCESS == ret) { r->ms->c->pool->ref++; easy_atomic_inc(&r->ms->pool->ref); easy_pool_set_lock(r->ms->pool); ret = EASY_AGAIN; } else { ret = EASY_OK; TBSYS_LOG(WARN, "can not push packet(src is %s, pcode is %u) to packet queue", inet_ntoa_r(r->ms->c->addr), packet->get_packet_code()); } } } return ret; }
int ObUpdateCallback::process(easy_request_t* r) { int ret = EASY_OK; if (NULL == r) { TBSYS_LOG(ERROR, "request is empty, r = %p", r); ret = EASY_BREAK; } else if (NULL == r->ipacket) { TBSYS_LOG(ERROR, "request is empty, r->ipacket = %p", r->ipacket); ret = EASY_BREAK; } else { ObUpdateServer *server = reinterpret_cast<ObUpdateServer*>(r->ms->c->handler->user_data); ObPacket *req = reinterpret_cast<ObPacket*>(r->ipacket); req->set_request(r); r->ms->c->pool->ref ++; easy_atomic_inc(&r->ms->pool->ref); easy_pool_set_lock(r->ms->pool); ret = server->handlePacket(req); if (OB_SUCCESS == ret) { // enqueue success ret = EASY_AGAIN; } else if (OB_ENQUEUE_FAILED == ret) { TBSYS_LOG(WARN, "can not push packet(src is %s, pcode is %u) to packet queue", inet_ntoa_r(r->ms->c->addr), req->get_packet_code()); r->ms->c->pool->ref --; easy_atomic_dec(&r->ms->pool->ref); ret = EASY_OK; } else /* OB_ERROR */ { ret = EASY_AGAIN; } } return ret; }
ObPacket* ObPacketQueue::pop() { if (head_ == NULL) { return NULL; } ObPacket* packet = head_; ObPacket* ret_packet = NULL; head_ = (ObPacket*)head_->_next; if (head_ == NULL) { tail_ = NULL; } size_ --; int err = ring_buffer_.pop_task(packet); if (err == OB_SUCCESS) { ThreadSpecificBuffer::Buffer* tb = thread_buffer_->get_buffer(); if (tb == NULL) { TBSYS_LOG(ERROR, "get packet thread buffer failed, return NULL"); } else { int64_t total_size = sizeof(ObPacket) + packet->get_packet_buffer()->get_capacity(); char* buf = tb->current(); memcpy(buf, packet, total_size); ret_packet = (ObPacket*)buf; buf += sizeof(ObPacket); ret_packet->set_packet_buffer(buf, packet->get_packet_buffer()->get_capacity()); } } else { TBSYS_LOG(ERROR, "pop task from ring buffer failed, task: %p, err: %d", packet, err); } return ret_packet; }
int ObBaseServer::send_response(const int32_t pcode, const int32_t version, const ObDataBuffer& buffer, tbnet::Connection* connection, const int32_t channel_id) { int rc = OB_SUCCESS; if (connection == NULL) { rc = OB_ERROR; TBSYS_LOG(WARN, "connection is NULL"); } ObPacket* packet = new(std::nothrow) ObPacket(); if (packet == NULL) { rc = OB_ALLOCATE_MEMORY_FAILED; TBSYS_LOG(ERROR, "create packet failed"); } else { packet->set_packet_code(pcode); packet->setChannelId(channel_id); packet->set_api_version(version); packet->set_data(buffer); } if (rc == OB_SUCCESS) { rc = packet->serialize(); if (rc != OB_SUCCESS) TBSYS_LOG(WARN, "packet serialize error, error: %d", rc); } if (rc == OB_SUCCESS) { if (!connection->postPacket(packet)) { uint64_t peer_id = connection->getPeerId(); TBSYS_LOG(WARN, "send packet to [%s] failed", tbsys::CNetUtil::addrToString(peer_id).c_str()); rc = OB_ERROR; } } if (rc != OB_SUCCESS) { if (NULL != packet) { packet->free(); packet = NULL; } } return rc; }
bool WaitObjectManager::wakeup_wait_object(const int64_t id, tbnet::Packet* response) { bool ret = true; tbsys::CThreadGuard guard(&mutex_); WaitObject* wait_object = NULL; if (wait_objects_map_.get(id, wait_object) != hash::HASH_EXIST) { TBSYS_LOG(INFO, "wait object not found, id: [%ld]", id); } else { if (response != NULL && response->isRegularPacket()) { ObPacket* packet = dynamic_cast<ObPacket*>(response); if (packet != NULL) { int64_t total_size = sizeof(ObPacket) + packet->get_packet_buffer()->get_position(); char* res_buf = wait_object->response_buffer_; memcpy(res_buf, packet, total_size); wait_object->response_ = (ObPacket*)res_buf; res_buf += sizeof(ObPacket); wait_object->response_->set_packet_buffer(res_buf, packet->get_packet_buffer()->get_position()); } else { TBSYS_LOG(WARN, "receive packet is not ObPacket, pcode: %d", response->getPCode()); ret = false; } } // always add the done count wait_object->wakeup(); } return ret; }
int MockMergeServer::do_request(ObPacket * base_packet) { ObPacket * packet = base_packet; int32_t packet_code = packet->get_packet_code(); int ret = OB_SUCCESS; ret = packet->deserialize(); int64_t receive_time = packet->get_receive_ts(); int32_t version = packet->get_api_version(); int32_t channelId = packet->getChannelId(); tbnet::Connection *connection = packet->get_connection(); ObDataBuffer *in_buff = packet->get_buffer(); ThreadSpecificBuffer::Buffer *thread_buff = response_packet_buffer_.get_buffer(); //貌似刨去了网络时间? int64_t timeout_us = packet->get_source_timeout() - (tbsys::CTimeUtil::getTime() - packet->get_receive_ts()); if (NULL != thread_buff) { thread_buff->reset(); ObDataBuffer out_buff(thread_buff->current(), thread_buff->remain()); if (OB_SUCCESS == ret) { switch(packet_code) { case OB_SCAN_REQUEST: ret = ms_scan(receive_time, version, channelId, connection, *in_buff, out_buff, timeout_us); if (ret != OB_SUCCESS) { TBSYS_LOG(WARN, "ms_scan failed"); } break; } } } return ret; }
int ObClientManager::do_send_request( const ObServer& server, const int32_t pcode, const int32_t version, const int64_t timeout, ObDataBuffer& in_buffer, ObPacket* &response) const { int rc = OB_SUCCESS; ObPacket* packet = new (std::nothrow) ObPacket(); if (NULL == packet) { rc = OB_ALLOCATE_MEMORY_FAILED; } else { packet->set_packet_code(pcode); packet->setChannelId(0); packet->set_api_version(version); packet->set_data(in_buffer); packet->set_source_timeout(timeout); } if (OB_SUCCESS == rc) { rc = packet->serialize(); if (OB_SUCCESS != rc) TBSYS_LOG(WARN, "packet serialize error"); } // serialize failed if (OB_SUCCESS != rc && NULL != packet) { packet->free(); } if (OB_SUCCESS == rc) { rc = do_send_packet(server, packet, timeout, response); } return rc; }
int handle_async_request(ObMsSqlRpcEvent & result) { int rc = OB_SUCCESS; ObPacket* packet = new (std::nothrow) ObPacket(); if (NULL == packet) { rc = OB_ALLOCATE_MEMORY_FAILED; } else { ObDataBuffer in_buffer; int64_t data_len = 2 * 1024 * 1024; char *data = (char *)ob_malloc(data_len); const int32_t pcode = OB_SCAN_RESPONSE; const int32_t version = 1; const int64_t session_id = 0; const int64_t timeout = 1000000; in_buffer.set_data(data, data_len); // fill scanner to buffer handle_scan_table(in_buffer); packet->set_packet_code(pcode); packet->setChannelId(0); packet->set_source_timeout(timeout); packet->set_session_id(session_id); packet->set_api_version(version); packet->set_data(in_buffer); rc = packet->serialize(); result.set_req_type(ObMergerRpcEvent::SCAN_RPC); result.set_result_code(OB_SUCCESS); if (rc != OB_SUCCESS) { TBSYS_LOG(WARN, "packet serialize error, error: %d", rc); } handler.push(&result, packet); //result.handlePacket(packet, NULL); } return OB_SUCCESS; }
int ObTbnetCallback::encode(easy_request_t *r, void *data) { int ret = EASY_OK; easy_buf_t* b = NULL; char* buff = NULL; ObPacket* packet = reinterpret_cast<ObPacket*>(data); int64_t header_size = 0; uint16_t ob_packet_header_size = 0; #if !defined(_OB_VERSION) || _OB_VERSION<=300 /* 0.3 ObPacket 头格式: 16 bytes */ /* 2 bytes 2 bytes 8 bytes 4 bytes *---------------------------------------------------------------------- *| ob_packet_header_size_== 0 | api_version_ | session_id_ | timeout_ | *---------------------------------------------------------------------- */ ob_packet_header_size = 0; header_size = sizeof(uint16_t)/* ob_packet_header_size_ */ + sizeof(int16_t) /* api_version_ */+ sizeof(int64_t) /* session_id_ */+ sizeof(int32_t)/* timeout_ */; #elif _OB_VERSION>300 /* 0.4 ObPacket 头格式: 24 bytes */ /* 2 bytes 2 bytes 8 bytes 4 bytes 8 bytes 8 bytes *-------------------------------------------------------------------------------------- *| ob_packet_header_size_ | api_version_ | session_id_ | timeout_ |trace_id_|req_sign_| *-------------------------------------------------------------------------------------- */ ob_packet_header_size = (uint16_t)(sizeof(ob_packet_header_size) + sizeof(int16_t)/* api_version_ */ + sizeof(int64_t) /* session_id_ */ + sizeof(int32_t)/* timeout_ */ + sizeof(uint64_t)/* trace_id_ */ + sizeof(uint64_t) /* req_sign_ */); header_size = ob_packet_header_size; #endif packet->set_ob_packet_header_size(ob_packet_header_size); int64_t size = packet->get_inner_buffer()->get_position() + OB_TBNET_HEADER_LENGTH + header_size; b = reinterpret_cast<easy_buf_t *>(easy_pool_alloc(r->ms->pool, static_cast<uint32_t>(sizeof(easy_buf_t) + size))); if (NULL == b) { TBSYS_LOG(WARN, "alloc mem for send buffer failed buf=%p size is %lu", b, sizeof(easy_buf_t) + size); ret = EASY_ERROR; } else { //skip sizeof(easy_buf_t) bytes buff = reinterpret_cast<char *>(b + 1); //b->pos = buff; //b->end = buff + size; //set packet length init_easy_buf(b, buff, r, size); packet->set_packet_len(static_cast<int>( packet->get_inner_buffer()->get_position() + header_size)); if (false == packet->encode(buff, size)) { ret = EASY_ERROR; TBSYS_LOG(WARN, "encode failed packet is %p, buff is %p, size is %ld", packet, buff, size); } if (EASY_OK == ret) { easy_buf_set_data(r->ms->pool, b, buff, static_cast<uint32_t>(size)); easy_request_addbuf(r, b); //add rpc bytes out statistics OB_STAT_INC(COMMON, RPC_BYTES_OUT, size); TBSYS_LOG(DEBUG, "encode packet success packet code is %d, c is %s",packet->get_packet_code(), easy_connection_str(r->ms->c)); } } return ret; }
int ObClientManager::get_next(const ObServer& server, const int64_t session_id, const int64_t timeout, ObDataBuffer& in_buffer, ObDataBuffer& out_buffer) const { int rc = OB_SUCCESS; ObPacket* response = NULL; //rc = send_request(server, pcode, version, timeout, in_buffer, response); ObPacket* packet = new (std::nothrow) ObPacket(); if (NULL == packet) { rc = OB_ALLOCATE_MEMORY_FAILED; } else { packet->set_packet_code(OB_SESSION_NEXT_REQUEST); packet->setChannelId(0); packet->set_api_version(0); packet->set_data(in_buffer); packet->set_source_timeout(timeout); packet->set_session_id(session_id); //TODO } if (OB_SUCCESS == rc) { rc = packet->serialize(); if (OB_SUCCESS != rc) TBSYS_LOG(WARN, "packet serialize error, code=%d", packet->get_packet_code()); } // serialize failed if (OB_SUCCESS != rc && NULL != packet) { packet->free(); } if (OB_SUCCESS == rc) { rc = do_send_packet(server, packet, timeout, response); } // deserialize response packet to out_buffer if (OB_SUCCESS == rc && NULL != response) { // copy response's inner_buffer to out_buffer. int64_t data_length = response->get_data_length(); ObDataBuffer* response_buffer = response->get_buffer(); if (out_buffer.get_remain() < data_length) { TBSYS_LOG(ERROR, "insufficient memory in out_buffer, remain:%ld, length=%ld", out_buffer.get_remain(), data_length); rc = OB_ERROR; } else { memcpy(out_buffer.get_data() + out_buffer.get_position(), response_buffer->get_data() + response_buffer->get_position(), data_length); out_buffer.get_position() += data_length; } } return rc; }
int ObMsSqlRpcEvent::handle_packet(ObPacket * packet, void * args) { int ret = OB_SUCCESS; if (ObMergeServerMain::get_instance()->get_merge_server().is_stoped()) { TBSYS_LOG(WARN, "server stoped, cannot handle anything."); ret = OB_ERROR; } else { /// stat event process time this->end(); /// parse the packet for get result code and result scanner ret = parse_packet(packet, args); if (ret != OB_SUCCESS) { TBSYS_LOG(WARN, "parse packet from server[%s] failed, request_id[%lu], event_id[%lu], rpc_event[%p]", server_.to_cstring(), client_request_id_, get_event_id(), this); /// set result code, maybe timeout packet, connection errors. if (NULL != args && 1 == reinterpret_cast<easy_connection_t*>(args)->conn_has_error) { ObCommonSqlRpcEvent::set_result_code(OB_CONN_ERROR); } else { ObCommonSqlRpcEvent::set_result_code(ret); } } ObPacket* obpacket = packet; if (NULL != obpacket) { TBSYS_LOG(DEBUG, "get packet from server[%s] success, event_id[%lu], time_used[%ld]," "result code=%d, packet code=%d, session_id=%ld", server_.to_cstring(), get_event_id(), get_time_used(), get_result_code(), obpacket->get_packet_code(), obpacket->get_session_id()); } //lock_.lock(); if (client_request_ == NULL) { TBSYS_LOG(WARN, "rpc event(%p) unknown status, destroy myself, event_id[%lu] " "request_id[%lu], req_type[%d]", this, this->get_event_id(), this->get_client_id(), this->get_req_type()); this->~ObMsSqlRpcEvent(); ob_tc_free(this); } else { /// no matter parse succ or failed push to finish queue /// not check the event valid only push to the finish queue if (OB_SUCCESS != (ret = client_request_->signal(*this))) { TBSYS_LOG(WARN, "failed to push event to finish queue, destroy myself, req_type[%s], request_id[%lu], ret=%d", (this->get_req_type() == SCAN_RPC) ? "SCAN":"GET", this->get_client_id(), ret); OB_ASSERT(magic_ == 0x1234abcd); this->~ObMsSqlRpcEvent(); ob_tc_free(this); } else { //OB_ASSERT(magic_ == 0x1234abcd); //lock_.unlock(); } } } return ret; }
int ObPacketQueue::pop_packets(tbnet::Packet** packet_arr, const int64_t ary_size, int64_t& ret_size) { int err = OB_SUCCESS; ThreadSpecificBuffer::Buffer* tb = NULL; ret_size = 0; if (NULL == packet_arr || ary_size <= 0) { TBSYS_LOG(WARN, "invalid param, packet_arr=%p, arr_size=%ld", packet_arr, ary_size); err = OB_ERROR; } else { tb = thread_buffer_->get_buffer(); if (tb == NULL) { TBSYS_LOG(ERROR, "get packet thread buffer failed, return NULL"); err = OB_ERROR; } else { tb->reset(); } } while (OB_SUCCESS == err && ret_size < ary_size) { if (head_ == NULL) { break; } else { ObPacket* packet = head_; int64_t total_size = sizeof(ObPacket) + packet->get_packet_buffer()->get_capacity(); if (tb->remain() < total_size) { if (0 == ret_size) { TBSYS_LOG(ERROR, "the thread buffer is too small, packet_size=%ld, buffer_size=%ld", total_size, tb->remain()); // discard packet ring_buffer_.pop_task(head_); head_ = (ObPacket*) head_->_next; if (head_ == NULL) { tail_ = NULL; } size_ --; err = OB_ERROR; } break; } else { char* buf = tb->current(); memcpy(buf, packet, total_size); packet_arr[ret_size] = (ObPacket*) buf; buf += sizeof(ObPacket); ((ObPacket*) packet_arr[ret_size])->set_packet_buffer(buf, packet->get_packet_buffer()->get_capacity()); // pop from queue head_ = (ObPacket*) head_->_next; if (head_ == NULL) { tail_ = NULL; } size_ --; // pop from ring buffer err = ring_buffer_.pop_task(packet); if (OB_SUCCESS != err) { TBSYS_LOG(ERROR, "failed to pop task, err=%d", err); } tb->advance(total_size); ++ret_size; } } } return err; }