int main(int argc, char *argv[])
{
void * handle;
if (argc < 2)
{
fprintf(stderr, "Invalid arguments\n");
exit(1);
}
if (msd_load_so(&handle, syms, argv[1]) < 0)
{
fprintf(stderr, "load so file failed\n");
exit(1);
}
if (dll.handle_init)
{
dll.handle_init("handle_init", 0);
}
dll.handle_task("handle_task");
if (dll.handle_fini)
{
dll.handle_fini("handle_fnit", 1);
}
msd_unload_so(&handle);
exit(0);
}
/**
* @brief 重启已停止work进程
* @param
* @return
*/
int monitor_work_proc(int argc, char* argv[], pid_t *wpids, int pid_count, bool /* use_barrier */)
{
int result;
pid_t old_pid, new_pid;
int sec = 10;
while (!g_stop) {
struct timeval tv = {sec, 0};
int sleep_ok = select(0, NULL, NULL, NULL, &tv);
for (int i = 0; i < pid_count && sleep_ok == 0; ++i) {
if(wpids[i] != 0) {
result = kill(wpids[i], 0);
if (0 == result || errno != ESRCH) {
continue;
}
}
if (!g_stop) {
old_pid = wpids[i];
/// WORK进程异常退出时，需要清理过滤器中这个进程的资源
if (g_bench_conf.use_barrier) {
AFUTEX_LOCK_ERR_CHK(g_rcv_rq_lock);
g_prcv_brq->clean_4pid(old_pid);
AFUTEX_UNLOCK_ERR_CHK(g_rcv_rq_lock);
}
new_pid = fork();
if(new_pid < 0) {
wpids[i] = 0;
ERROR_LOG("Work[%u] exited, and fork failed: %s", new_pid, strerror(errno));
} else if(new_pid > 0) {///parent process
wpids[i] = new_pid;
ERROR_LOG("Work[%u] exited, fork[%u] to replace.", old_pid, new_pid);
} else {
run_work_proc(argc, argv, g_bench_conf.use_barrier);
}
}
}
if (dll.handle_timer) {
dll.handle_timer(&sec);
}
}
if (dll.handle_fini) {
dll.handle_fini(PROC_MAIN);
}
///wait for all child processes end
for (int i = 0; i < pid_count; ) {
if (wpids[i] != 0) {
result = kill(wpids[i], 0);
if (result != 0 && errno == ESRCH) {
/// 等待下一个进程
++i;
} else {
/// 继续等待 50 毫秒
usleep(50000);
continue;
}
}
}
return 0;
}
/**
* @brief 启动work进程
* @param
* @return
*/
int run_work_proc(int argc, char *argv[], bool /* use_barrier */)
{
setproctitle("%s:[WORK]", g_bench_conf.prog_name);
if (dll.handle_init != NULL && dll.handle_init(argc, argv, PROC_WORK) != 0) {
print_prompt(false, "Worker process handle_init failed.");
return -1;
}
/// 由于环形队列中有头 shm_block_t，所以需要给头预留空间
int prcv_data_len = g_bench_conf.max_pkg_len + sizeof(shm_block_t);
char *prcv_data = (char *)malloc(prcv_data_len);
if (prcv_data == NULL) {
print_prompt(false, "Malloc work proc pop buffer fail, size: %d B", prcv_data_len);
return -1;
}
shm_block_t *mb = (shm_block_t *)prcv_data;
int sndlen = 0;
int ret = 0;
char *puser_data = NULL;
if (g_bench_conf.use_barrier == true) {
if (g_prcv_brq == NULL) {
print_prompt(false, "Use barrier mode, but i_barrier_ring_queue is null.");
goto work_proc_end;
}
while (! g_stop) {
if (dll.handle_schedule) {
dll.handle_schedule();
}
if (!g_prcv_brq->is_able_pop()) {
/// 等待数据
struct timespec tv = {0, 200000};
nanosleep(&tv, NULL);
continue;
}
AFUTEX_LOCK_ERR_CHK(g_rcv_rq_lock);
ret = g_prcv_brq->pop(prcv_data, prcv_data_len);
AFUTEX_UNLOCK_ERR_CHK(g_rcv_rq_lock);
if (ret < 0) {
ERROR_LOG("Barrier pop error: %d", g_prcv_brq->get_last_errno());
sleep(1); /// 防止出错以后打印日志速度太快
continue;
} else if (ret == 0) {
/// 没有取出任何数据
continue;
} else {
if(ret != mb->length || (u_int)mb->length <= sizeof(shm_block_t) ||
mb->length > prcv_data_len) {
/// 数据包包长有误，关闭这个连接
ERROR_LOG("Pop data len(%d) err, pkg len(%d), prcv_data_len(%d), shm_block_t(%zd), close conn: %d.",
ret, mb->length, prcv_data_len, sizeof(shm_block_t), mb->blk_id);
work_proc_close_conn(mb);
continue;
}
}
TRACE_LOG("Pop barrier ring queue len: %d", mb->length);
sndlen = 0;
ret = dll.handle_process(mb->data, mb->length - sizeof(shm_block_t), &puser_data, &sndlen, &mb->skinfo);
if (ret != 0) {
work_proc_close_conn(mb);
TRACE_LOG("Handle_process fail, close conn: %d", mb->blk_id);
} else if(sndlen + sizeof(shm_block_t) > (u_int)prcv_data_len) {
work_proc_close_conn(mb);
ERROR_LOG("Handle_process sndlen(%d) > buf len, close conn: %d", sndlen, mb->blk_id);
} else {
if (sndlen > 0) {
mb->length = sndlen + sizeof(shm_block_t);
mb->type = DAT_BLOCK;
memcpy(mb->data, puser_data, sndlen);
AFUTEX_LOCK_ERR_CHK(g_snd_rq_lock);
ret = g_psnd_rq->push_data(prcv_data, mb->length, true);
AFUTEX_UNLOCK_ERR_CHK(g_snd_rq_lock);
if (ret != mb->length) {
ERROR_LOG("Push data(len: %d) error (ret: %d, data: %d, empty: %d)", mb->length,
g_psnd_rq->get_last_errno(), g_psnd_rq->get_data_len(), g_psnd_rq->get_empty_buffer_len());
} else {
TRACE_LOG("ring queue push data len: %d", mb->length);
g_p_net_io_notifier->popup();
}
}
}
}
} else {
if (g_prcv_rq == NULL) {
print_prompt(false, "Use nonbarrier mode, but i_ring_queue object is null.");
goto work_proc_end;
}
while (!g_stop) {
if (dll.handle_schedule != NULL) {
dll.handle_schedule();
}
if (g_prcv_rq->get_data_len() <= 0) {
struct timespec tv = {0, 200000};
nanosleep (&tv, NULL);
continue;
}
AFUTEX_LOCK_ERR_CHK(g_rcv_rq_lock);
ret = g_prcv_rq->pop_data(prcv_data, prcv_data_len, 0);
AFUTEX_UNLOCK_ERR_CHK(g_rcv_rq_lock);
if (ret < 0) {
ERROR_LOG("Common ring queue pop fail(ret: %d): %s", ret, g_prcv_rq->get_last_errstr());
sleep(1); /// 防止日志打印过快
continue;
} else if (ret == 0) {
/// 数据还没准备好
continue;
} else {
if(ret != mb->length || (u_int)mb->length <= sizeof(shm_block_t) || mb->length > prcv_data_len) {
/// 数据包包长有误，关闭这个连接
ERROR_LOG("Pop data len(%d) err, pkg len(%d), prcv_data_len(%d), shm_block_t(%zd), close conn: %d.",
ret, mb->length, prcv_data_len, sizeof(shm_block_t), mb->blk_id);
work_proc_close_conn(mb);
continue;
}
}
TRACE_LOG("Common ring queue pop len: %d", mb->length);
sndlen = 0;
ret = dll.handle_process(mb->data, mb->length - sizeof(shm_block_t), &puser_data, &sndlen, &mb->skinfo);
if (ret != 0) {
work_proc_close_conn(mb);
TRACE_LOG("Handle process fail, close conn: %d", mb->blk_id);
} else if(sndlen + sizeof(shm_block_t) > (u_int)prcv_data_len) {
work_proc_close_conn(mb);
ERROR_LOG("Handle_process sndlen(%zd) > buf len %d, close conn: %d",
sndlen + sizeof(shm_block_t), prcv_data_len, mb->blk_id);
} else {
if (sndlen > 0) {
mb->length = sndlen + sizeof(shm_block_t);
mb->type = DAT_BLOCK;
memcpy(mb->data, puser_data, sndlen);
AFUTEX_LOCK_ERR_CHK(g_snd_rq_lock);
ret = g_psnd_rq->push_data(prcv_data, mb->length, true);
AFUTEX_UNLOCK_ERR_CHK(g_snd_rq_lock);
if (ret != mb->length) {
ERROR_LOG("Push data(len: %d) error (ret: %d, data: %d, empty: %d)", mb->length,
g_psnd_rq->get_last_errno(), g_psnd_rq->get_data_len(), g_psnd_rq->get_empty_buffer_len());
} else {
TRACE_LOG("Common ring queue push len: %d", mb->length);
g_p_net_io_notifier->popup();
}
}
}
}
}
work_proc_end:
free(prcv_data);
if (g_p_send_buff != NULL) {
free(g_p_send_buff);
g_p_send_buff = NULL;
g_send_buff_len = 0;
}
if (dll.handle_fini != NULL) {
dll.handle_fini(PROC_WORK);
}
return 0;
}
/**
* @brief 创建conn进程
* @param
* @return 0:success, -1:fail
*/
int conn_proc(int argc, char **argv)
{
//建立TCP连接
if (g_bench_conf.bind_list == NULL)
{
NB_BOOT_LOG(-1, "Bind info is null.");
return -1;
}
//暂时仅支持一个端口
struct bind_config *bc = g_bench_conf.bind_list;
if (bc->type != SOCK_STREAM)
{
NB_BOOT_LOG(-1, "Warning: cannot support UDP now.");
return -1;
}
c_newbench_net_io_event nb_event;
if (nb_event.init() != 0)
{
NB_BOOT_LOG(-1, "Init c_newbench_net_io_event object failed.");
return -1;
}
if (pnb_io_server->init(bc->bind_ip,
bc->bind_port,
&nb_event,
g_p_net_io_notifier,
1,
1) != 0)
{
NB_BOOT_LOG(-1, "Bind [%s:%u]: %s", bc->bind_ip,
bc->bind_port, pnb_io_server->get_last_errstr());
return -1;
}
NB_BOOT_LOG(0, "Listen on %s:%u", bc->bind_ip, bc->bind_port);
int psnd_data_len = g_bench_conf.max_pkg_len + sizeof(shm_block_t);
char *psnd_data = (char *)malloc(psnd_data_len);
if (psnd_data == NULL)
{
NB_BOOT_LOG(-1, "Malloc package send buffer fail, size: %d B", g_bench_conf.max_pkg_len);
return -1;
}
if (dll.handle_init && dll.handle_init(argc, argv, PROC_CONN) != 0)
{
NB_BOOT_LOG(-1, "Conn handle init fail.");
return -1;
}
setproctitle("%s:[CONN]", g_bench_conf.prog_name);
shm_block_t* mb = (shm_block_t*)psnd_data;
while(!g_stop)
{
// 每次阻塞1毫秒, 因后面要发送数据
// TODO:时间应该是可配的
if (pnb_io_server->do_io(4,
NET_IO_SERVER_CMD_ACCEPT
| NET_IO_SERVER_CMD_READ
| NET_IO_SERVER_CMD_WRITE) != 0)
{
ERROR_LOG("do_io fail: %s", pnb_io_server->get_last_errstr());
}
// 发送发送环形队列中的数据包, 发干净为止
while (!g_stop)
{
// 取出数据包
int ret = g_psnd_rq->pop_data(psnd_data, psnd_data_len, 0);
if (ret == 0)
{
break;
}
else if (ret < 0)
{
// 发生错误
ERROR_LOG("Pop data error, ret: %d, err: %s", ret, g_psnd_rq->get_last_errstr());
break;
}
else
{
if (ret != mb->length || mb->length > psnd_data_len)
{
// 出错, 打印日志以后退出发送
ERROR_LOG("pop data len %d != pkg len %d or > psnd_data_len", mb->length, psnd_data_len);
break;
}
}
if (mb->type == FIN_BLOCK)
{
pnb_io_server->close_connection(mb->blk_id, true);
continue;
}
TRACE_LOG("common ring queue pop data length: %d", mb->length);
// 发送数据
int bytes_sent = 0;
int data_len = mb->length - sizeof(shm_block_t);
if (data_len)
{
if (mb->type == BROADCAST_BLOCK)
{
ret = pnb_io_server->broadcast(
mb->blk_id,
(char*)mb->data + bytes_sent,
data_len - bytes_sent,
1);
if (ret)
{
// 广播报文为全部成功
ERROR_LOG("broadcast not success at all, %d failed",
ret);
}
}
else
{
ret = pnb_io_server->send_data_atomic(
mb->blk_id,
(char*)mb->data + bytes_sent,
data_len - bytes_sent);
if (ret != data_len - bytes_sent)
{
// 发送数据失败时丢弃这个包
ERROR_LOG("Send to conn(%d) fail, len: %d, discard this pkg.", mb->blk_id, mb->length);
}
else
{
if (mb->skinfo.ptr_lookout)
{
// 传回storage
reinterpret_cast<union net_io_storage *>(mb->skinfo.ptr_lookout)->u64
= mb->skinfo.storage.u64;
}
}
}
}
}
}
free(psnd_data);
if (dll.handle_fini)
{
dll.handle_fini(PROC_CONN);
}
return 0;
}