int cfio_init(int x_proc_num, int y_proc_num, int ratio)
{
int rc, i;
int size;
int root = 0;
int error, ret;
int server_proc_num;
int best_server_amount;
MPI_Group group, client_group, server_group;
int *ranks;
//set_debug_mask(DEBUG_CFIO | DEBUG_SERVER);// | DEBUG_MSG | DEBUG_SERVER);
rc = MPI_Initialized(&i);
if( !i )
{
error("MPI should be initialized before the cfio\n");
return -1;
}
MPI_Comm_size(MPI_COMM_WORLD, &size);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
//if(rank == 100)
//{
// set_debug_mask(DEBUG_MAP);
//}
client_num = x_proc_num * y_proc_num;
server_proc_num = size - client_num;
if(server_proc_num < 0)
{
server_proc_num = 0;
}
best_server_amount = (int)((double)client_num / ratio);
if(best_server_amount <= 0)
{
best_server_amount = 1;
}
MPI_Comm_group(MPI_COMM_WORLD, &group);
//ranks = malloc(client_num * sizeof(int));
//for(i = 0; i < client_num; i ++)
//{
// ranks[i] = i;
//}
//MPI_Group_incl(group, client_num, ranks, &client_group);
//MPI_Comm_create(MPI_COMM_WORLD, client_group, &client_comm);
//free(ranks);
ranks = malloc(server_proc_num * sizeof(int));
for(i = 0; i < server_proc_num; i ++)
{
ranks[i] = i + client_num;
}
MPI_Group_incl(group, server_proc_num, ranks, &server_group);
MPI_Comm_create(MPI_COMM_WORLD, server_group, &server_comm);
free(ranks);
//times_start();
if((ret = cfio_map_init(
x_proc_num, y_proc_num, server_proc_num,
best_server_amount, MPI_COMM_WORLD, server_comm)) < 0)
{
error("Map Init Fail.");
return ret;
}
if(cfio_map_proc_type(rank) == CFIO_MAP_TYPE_SERVER)
{
if((ret = cfio_server_init()) < 0)
{
error("");
return ret;
}
if((ret = cfio_server_start()) < 0)
{
error("");
return ret;
}
}else if(cfio_map_proc_type(rank) == CFIO_MAP_TYPE_CLIENT)
{
if((ret = cfio_send_init(CLIENT_BUF_SIZE)) < 0)
{
error("");
return ret;
}
if((ret = cfio_id_init(CFIO_ID_INIT_CLIENT)) < 0)
{
error("");
return ret;
}
}
debug(DEBUG_CFIO, "success return.");
return CFIO_ERROR_NONE;
}
int cfio_server_init()
{
init_time = times_cur();
int i;
int ret = 0;
int x_proc_num, y_proc_num;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if((ret = cfio_recv_init()) < 0)
{
error("");
return ret;
}
if((ret = cfio_id_init(CFIO_ID_INIT_SERVER)) < 0)
{
error("");
return ret;
}
if((ret = cfio_io_init(rank)) < 0)
{
error("");
return ret;
}
times_init();
client_num = cfio_map_get_client_num_of_server(rank);
recv_bufs = malloc(sizeof(char *) * client_num);
if (!recv_bufs) {
printf("line %d malloc error \n", __LINE__);
return CFIO_ERROR_MALLOC;
}
SERVER_REGION_SIZE = RECV_BUF_SIZE / client_num;
for (i = 0; i < client_num; ++i) {
recv_bufs[i] = malloc(sizeof(char) * SERVER_REGION_SIZE);
if (!recv_bufs[i]) {
printf("line %d malloc error \n", __LINE__);
return CFIO_ERROR_MALLOC;
}
}
buf_addrs = malloc(sizeof(cfio_buf_addr_t *) * client_num);
if (!buf_addrs) {
printf("line %d malloc error \n", __LINE__);
return CFIO_ERROR_MALLOC;
}
for (i = 0; i < client_num; ++i) {
buf_addrs[i] = malloc(sizeof(cfio_buf_addr_t));
if (!buf_addrs[i]) {
printf("line %d malloc error \n", __LINE__);
return CFIO_ERROR_MALLOC;
}
cfio_rdma_addr_init(buf_addrs[i], NULL);
}
begin_time = times_cur();
cfio_rdma_server_init(client_num, SERVER_REGION_SIZE, recv_bufs,
sizeof(cfio_buf_addr_t), (char **)buf_addrs);
end_time = times_cur();
// printf("server %d rdma init %f. \n", rank, end_time - begin_time);
return CFIO_ERROR_NONE;
}