/* Create connections between clients and servers */
void armci_init_connections()
{
armci_mpi2_debug(0, "armci_init_connections\n");
_armci_buf_init(); /* CHECK: Is this correct ? */
MPI_Check(MPI_Barrier(ARMCI_COMM_WORLD));
/* Abhinav Vishnu */
armci_create_server_MPIprocess();
armci_mpi2_debug(0, "armci_init_connections completed\n");
}
/*\ Create Sockets for clients and servers
\*/
void armci_init_connections()
{
int i,n,p,master = armci_clus_info[armci_clus_me].master;
_armci_buf_init();
/* sockets for communication with data server */
SRV_sock = (int*) malloc(sizeof(int)*armci_nclus);
if(!SRV_sock)armci_die("ARMCI cannot allocate SRV_sock",armci_nclus);
/* array that will be used to exchange port info */
AR_port = (int*) calloc(armci_nproc * armci_nclus, sizeof(int));
if(!AR_port)armci_die("ARMCI cannot allocate AR_port",armci_nproc*armci_nclus);
/* create active sockets list select */
if (!(_armci_active_socks = malloc(sizeof(active_socks_t))))
armci_die("dataserv.c, malloc _armci_active_socks failed",0);
for(i=0,n=MAX_BUFS+MAX_SMALL_BUFS;i<n;i++)_armci_active_socks->socks[i]=-1;
/* create sockets for communication with each user process */
if(master==armci_me){
CLN_sock = (int*) malloc(sizeof(int)*armci_nproc);
if(!CLN_sock)armci_die("ARMCI cannot allocate CLN_sock",armci_nproc);
armci_determine_sock_buf_size();
for(p=0; p< armci_nproc; p++){
int off_port = armci_clus_me*armci_nproc;
# ifdef SERVER_THREAD
if(p >=armci_clus_first && p <= armci_clus_last) CLN_sock[p]=-1;
else
# endif
armci_CreateSocketAndBind(CLN_sock + p, AR_port + p +off_port);
}
#ifdef SERVER_THREAD
/* skip sockets associated with processes on the current node */
if(armci_clus_first>0)
armci_ListenSockAll(CLN_sock, armci_clus_first);
if(armci_clus_last< armci_nproc-1)
armci_ListenSockAll(CLN_sock + armci_clus_last+1,
armci_nproc-armci_clus_last-1);
#else
armci_ListenSockAll(CLN_sock, armci_nproc);
#endif
}
}
/*\ initialization of LAPI related data structures
\*/
void armci_init_lapi()
{
int rc, p;
int lapi_max_uhdr_sz;
lapi_cmpl_t *pcntr;
lapi_remote_cxt_t util_cxt; /* For call to obtain rCxt */
#ifndef TCGMSG
rc = LAPI_Init(&lapi_handle, &lapi_info);
if(rc) ERROR("lapi_init failed",rc);
#endif
/* set the max limit for AM header data length */
rc = LAPI_Qenv(lapi_handle,MAX_UHDR_SZ, &lapi_max_uhdr_sz);
if(rc) ERROR("armci_init_lapi: LAPI_Qenv failed", rc);
/* fprintf(stderr,"max header size = %d\n",lapi_max_uhdr_sz);*/
/* how much data can fit into AM header ? */
lapi_max_uhdr_data_sz = lapi_max_uhdr_sz - sizeof(request_header_t);
/* allocate memory for completion state array */
cmpl_arr = (lapi_cmpl_t*)malloc(armci_nproc*sizeof(lapi_cmpl_t));
if(cmpl_arr==NULL) ERROR("armci_init_lapi:malloc for cmpl_arr failed",0);
/* allocate memory for ack and get counters, 1 if not thread safe */
#ifdef THREAD_SAFE
ack_cntr = calloc(armci_user_threads.max, sizeof(lapi_cmpl_t));
get_cntr = calloc(armci_user_threads.max, sizeof(lapi_cmpl_t));
#else
ack_cntr = calloc(1, sizeof(lapi_cmpl_t));
get_cntr = calloc(1, sizeof(lapi_cmpl_t));
#endif
if (!(ack_cntr && get_cntr))
ERROR("armci_init_lapi:calloc for ack or get counters failed",0);
/* initialize completion state array */
for(p = 0; p< armci_nproc; p++){
rc = LAPI_Setcntr(lapi_handle, &cmpl_arr[p].cntr, 0);
if(rc) ERROR("armci_init_lapi: LAPI_Setcntr failed (arr)",rc);
cmpl_arr[p].oper = -1;
cmpl_arr[p].val = 0;
}
/* initialize ack/buf/hdr counters */
#ifdef THREAD_SAFE
# define N armci_user_threads.max
#else
# define N 1
#endif
for (p = 0; p < N; p++) {
rc = LAPI_Setcntr(lapi_handle, &(ack_cntr[p].cntr), 0);
if(rc) ERROR("armci_init_lapi: LAPI_Setcntr failed (ack)",rc);
ack_cntr[p].val = 0;
rc = LAPI_Setcntr(lapi_handle, &(get_cntr[p].cntr), 0);
if(rc) ERROR("armci_init_lapi: LAPI_Setcntr failed (get)",rc);
get_cntr[p].val = 0;
}
rc = LAPI_Setcntr(lapi_handle, &hdr_cntr.cntr, 0);
if(rc) ERROR("armci_init_lapi: LAPI_Setcntr failed (hdr)",rc);
hdr_cntr.val = 0;
rc = LAPI_Setcntr(lapi_handle, &buf_cntr.cntr, 0);
if(rc) ERROR("armci_init_lapi: LAPI_Setcntr failed (buf)",rc);
buf_cntr.val = 0;
#if 0
pcntr = (lapi_cmpl_t*)MessageSndBuffer;
rc = LAPI_Setcntr(lapi_handle, &pcntr->cntr, 0);
if(rc) ERROR("armci_init_lapi: LAPI_Setcntr failed (bufcntr)",rc);
pcntr->val = 0;
#endif
#ifdef LAPI_RDMA
/* allocate rCxt */
lapi_remote_cxt = (lapi_user_cxt_t*)malloc(armci_nproc *
sizeof(lapi_user_cxt_t));
if(lapi_remote_cxt==NULL) ERROR("armci_init_lapi: rCxt malloc failed",0);
/* obtain remote context "rCxt" for RDMA Operation of all procs */
for(p = 0; p< armci_nproc; p++){
if(p==armci_me) continue;
util_cxt.Util_type = LAPI_REMOTE_RCXT;
util_cxt.operation = LAPI_RDMA_ACQUIRE;
util_cxt.dest = p;
CHECK(LAPI_Util(lapi_handle, (lapi_util_t *) &util_cxt));
lapi_remote_cxt[p] = util_cxt.usr_rcxt;
}
#endif
#if !defined(LAPI2)
/* for high performance, disable LAPI internal error checking */
LAPI_Senv(lapi_handle, ERROR_CHK, 0);
#endif
/* make sure that interrupt mode is on */
LAPI_Senv(lapi_handle, INTERRUPT_SET, 1);
/* initialize buffer managment module */
_armci_buf_init();
#ifdef LAPI_RDMA
CHECK((LAPI_Gfence(lapi_handle)));
#endif
#if ARMCI_ENABLE_GPC_CALLS
gpc_req = (gpc_buf_t *)malloc(sizeof(gpc_buf_t)*MAX_GPC_REQ);
if(gpc_req==NULL)armci_die("malloc for gpc failed",sizeof(gpc_buf_t));
gpc_init();
#endif
}
/* Create connections between clients and servers */
void armci_init_connections()
{
armci_mpi2_debug(0, "armci_init_connections\n");
_armci_buf_init(); /* CHECK: Is this correct ? */
}