void dwf_dslash_5_plus_start(Vector *out, Vector *in, Float mass, int dag, Dwf *dwf_lib_arg) { int x; int s; // getPlusData(comm_buf, f_in, 24*vol_4d_cb, 4); // f_in = (IFloat *) in; // f_in = f_in + (local_ls-1)*ls_stride; // getMinusData(comm_buf, f_in, 24*vol_4d_cb, 4); // Initializations //------------------------------------------------------------------ local_ls = GJP.SnodeSites(); s_nodes = GJP.Snodes(); s_node_coor = GJP.SnodeCoor(); vol_4d_cb = dwf_lib_arg->vol_4d / 2; ls_stride = 24 * vol_4d_cb; if(s_nodes<2) return; if(!rbuf_up){ rbuf_up = (Float *)malloc(24*vol_4d_cb*sizeof(IFloat)); msgmem_up[1] = QMP_declare_msgmem(rbuf_up,24*vol_4d_cb*sizeof(IFloat)); msghandle_up[1] = QMP_declare_receive_relative(msgmem_up[1],4,+1,0); // fprintf(stderr,"rbuf_up=%p\n",rbuf_up); // fprintf(stderr,"msgmem_up[1]=%p\n",msgmem_up[1]); // fprintf(stderr,"msghandle_up[1]=%p\n",msghandle_up[1]); } if(!rbuf_down){ rbuf_down = (Float *)malloc(24*vol_4d_cb*sizeof(IFloat)); msgmem_down[1] = QMP_declare_msgmem(rbuf_down,24*vol_4d_cb*sizeof(IFloat)); msghandle_down[1] = QMP_declare_receive_relative(msgmem_down[1],4,-1,0); } if (in_p != in){ if (in_p!=NULL){ QMP_free_msghandle(msghandle_up[0]); QMP_free_msgmem(msgmem_up[0]); QMP_free_msghandle(msghandle_down[0]); QMP_free_msgmem(msgmem_down[0]); } in_p = in; IFloat *f_in = (IFloat *)in; msgmem_up[0] = QMP_declare_msgmem(f_in,24*vol_4d_cb*sizeof(IFloat)); msghandle_up[0] = QMP_declare_send_relative(msgmem_up[0],4,-1,0); f_in = f_in + (local_ls-1)*ls_stride; msgmem_down[0] = QMP_declare_msgmem(f_in,24*vol_4d_cb*sizeof(IFloat)); msghandle_down[0] = QMP_declare_send_relative(msgmem_down[0],4,1,0); } #ifndef USE_GETPLUS QMP_start(msghandle_up[0]); QMP_start(msghandle_up[1]); QMP_start(msghandle_down[0]); QMP_start(msghandle_down[1]); #endif }
CPS_START_NAMESPACE #ifndef USE_QMP #define QMP #endif void GlobalDataShift::Shift(int direction, int n_disp){ if (n_disp==0) return; SCUDir s_dir,r_dir; int a_disp; void *send_p,*recv_p,*temp_p; #ifndef USE_QMP if (n_disp>0){ a_disp = n_disp; s_dir = gjp_scu_dir[i*2]; r_dir = gjp_scu_dir[i*2+1]; } else { a_disp = -n_disp; s_dir = gjp_scu_dir[i*2+1]; r_dir = gjp_scu_dir[i*2]; } #else // int direction = i; int sflag; if (n_disp > 0) sflag = +1; else sflag = -1; #endif send_p = addr; recv_p = temp_buf; #ifndef USE_QMP SCUDirArgIR Send(send_p,s_dir,SCU_SEND,data_len); SCUDirArgIR Recv(recv_p,r_dir,SCU_REC,data_len); #else QMP_msgmem_t msgmem[2]; QMP_msghandle_t msghandle[2]; QMP_status_t status; QMP_msghandle_t multiple; #endif // sys_cacheflush(0); for(int i = 0;i<a_disp-1;i++){ #ifndef USE_QMP Send.StartTrans();Recv.StartTrans(); Send.TransComplete();Recv.TransComplete(); #else msgmem[0] = QMP_declare_msgmem((void *)send_p, data_len); msgmem[1] = QMP_declare_msgmem((void *)recv_p, data_len); msghandle[0] = QMP_declare_send_relative(msgmem[0], direction, sflag, 0); msghandle[1] = QMP_declare_receive_relative(msgmem[1], direction, -sflag, 0); multiple = QMP_declare_multiple(msghandle, 2); QMP_start(multiple); status = QMP_wait(multiple); if (status != QMP_SUCCESS) QMP_error("Error in GlobalDataShift::Shift:%s\n", QMP_error_string(status)); QMP_free_msghandle(multiple); QMP_free_msgmem(msgmem[0]); QMP_free_msgmem(msgmem[1]); #endif temp_p = send_p; send_p = recv_p; recv_p = temp_p; #ifndef USE_QMP Send.Addr(send_p); Recv.Addr(recv_p); #endif } #ifndef USE_QMP Send.StartTrans();Recv.StartTrans(); Send.TransComplete();Recv.TransComplete(); #else msgmem[0] = QMP_declare_msgmem((void *)send_p, data_len); msgmem[1] = QMP_declare_msgmem((void *)recv_p, data_len); msghandle[0] = QMP_declare_send_relative(msgmem[0], direction, sflag, 0); msghandle[1] = QMP_declare_receive_relative(msgmem[1], direction, -sflag, 0); multiple = QMP_declare_multiple(msghandle, 2); QMP_start(multiple); status = QMP_wait(multiple); if (status != QMP_SUCCESS) QMP_error("Error in GlobalDataShift::Shift:%s\n", QMP_error_string(status)); QMP_free_msghandle(multiple); QMP_free_msgmem(msgmem[0]); QMP_free_msgmem(msgmem[1]); #endif if (recv_p != addr) memcpy(addr,recv_p,data_len); }
void PT::mat_cb_norm(int n, IFloat **mout, IFloat **min, const int *dir, int parity, IFloat * gauge) { //List of the different directions int wire[MAX_DIR]; int i; // printf("PT::mat_cb_norm\n"); QMP_msgmem_t *msg_mem_p = (QMP_msgmem_t *)Alloc("","vec_cb_norm", "msg_mem_p", 2*non_local_dirs*sizeof(QMP_msgmem_t)); QMP_msghandle_t* msg_handle_p = (QMP_msghandle_t *)Alloc("","vec_cb_norm", "msg_handle_p", 2*non_local_dirs*sizeof(QMP_msghandle_t)); QMP_msghandle_t multiple; static int call_num = 0; int vlen = VECT_LEN; int vlen2 = VECT_LEN; call_num++; //Name our function char *fname="pt_mat_cb()"; // VRB.Func("",fname); //Set the transfer directions //If wire[i] is even, then we have communication in the negative direction //If wire[i] is odd, then we have communication in the positive direction for(i=0;i<n;i++) wire[i]=dir[i]; #ifdef PROFILE Float dtime = - dclock(); #endif int non_local_dir=0; //#pragma omp parallel default(shared) { //If wire[i] is odd, then we have parallel transport in the //positive direction. In this case, multiplication by the link matrix is //done before the field is transferred over to the adjacent node // //If we have transfer in the negative T direction (wire[i] = 6), then //we have to copy the appropriate fields to a send buffer //#pragma omp for for(i=0;i<n;i++) { if(!local[wire[i]/2]) { if(wire[i]%2) { if(conjugated) pt_cmm_cpp(non_local_chi_cb[wire[i]],(long)uc_nl_cb_pre[parity][wire[i]/2],(long)min[i],(long)snd_buf_cb[wire[i]/2],(long)gauge); else pt_cmm_dag_cpp(non_local_chi_cb[wire[i]],(long)uc_nl_cb_pre[parity][wire[i]/2],(long)min[i],(long)snd_buf_cb[wire[i]/2],(long)gauge); } else if((wire[i] == 6)) { for(int j = 0; j < non_local_chi_cb[6];j++) memcpy(snd_buf_t_cb + j*GAUGE_LEN,min[i] + 3 * *(Toffset[parity]+j)*3,GAUGE_LEN*sizeof(IFloat)); } } } //#pragma omp barrier //#pragma omp master { for(i=0;i<n;i++) if(!local[wire[i]/2]) { //Calculate the starting address for the data to be sent IFloat *addr = min[i] + GAUGE_LEN * offset_cb[wire[i]]; msg_mem_p[2*non_local_dir] = QMP_declare_msgmem((void *)rcv_buf[wire[i]], 3*non_local_chi_cb[wire[i]]*VECT_LEN*sizeof(IFloat)); //Initialize the msg_mem for sends if(wire[i]%2) msg_mem_p[2*non_local_dir+1] = QMP_declare_msgmem((void *)snd_buf_cb[wire[i]/2], 3*non_local_chi_cb[wire[i]]*VECT_LEN*sizeof(IFloat)); else if(wire[i] == 6) msg_mem_p[2*non_local_dir+1] = QMP_declare_msgmem((void *)snd_buf_t_cb, 3*non_local_chi_cb[wire[i]]*VECT_LEN*sizeof(IFloat)); else msg_mem_p[2*non_local_dir+1] = QMP_declare_strided_msgmem((void *)addr, (size_t)(3*blklen_cb[wire[i]]), numblk_cb[wire[i]], (ptrdiff_t)(3*stride_cb[wire[i]]+3*blklen_cb[wire[i]])); msg_handle_p[2*non_local_dir] = QMP_declare_receive_relative(msg_mem_p[2*non_local_dir], wire[i]/2, 1-2*(wire[i]%2), 0); msg_handle_p[2*non_local_dir+1] = QMP_declare_send_relative(msg_mem_p[2*non_local_dir+1], wire[i]/2, 2*(wire[i]%2)-1, 0); non_local_dir++; } if(non_local_dir) { multiple = QMP_declare_multiple(msg_handle_p, 2*non_local_dir); QMP_start(multiple); } } //#pragma omp master { //Do local calculations //#pragma omp for for(i=0;i<n;i++) { if((wire[i]%2 && conjugated) || ((wire[i]%2 == 0) && (conjugated == 0))) pt_cmm_cpp(local_chi_cb[wire[i]],(long)uc_l_cb[parity][wire[i]],(long)min[i],(long)mout[i],(long)gauge); else pt_cmm_dag_cpp(local_chi_cb[wire[i]],(long)uc_l_cb[parity][wire[i]],(long)min[i],(long)mout[i],(long)gauge); } //#pragma omp barrier //#pragma omp master { if(non_local_dir) { QMP_status_t qmp_complete_status = QMP_wait(multiple); if (qmp_complete_status != QMP_SUCCESS) QMP_error("Send failed in vec_cb_norm: %s\n", QMP_error_string(qmp_complete_status)); QMP_free_msghandle(multiple); for(int i = 0; i < 2*non_local_dir; i++) QMP_free_msgmem(msg_mem_p[i]); Free(msg_handle_p); Free(msg_mem_p); } } //#pragma omp master { //If wire[i] is even, then we have transport in the negative direction //In this case, the vector field is multiplied by the SU(3) link matrix //after all communication is complete IFloat *fp0,*fp1; //#pragma omp for for(i=0;i<n;i++) { if(!local[wire[i]/2]) { if(!(wire[i]%2)) { if(conjugated) pt_cmm_dag_cpp(non_local_chi_cb[wire[i]],(long)uc_nl_cb[parity][wire[i]],(long)rcv_buf[wire[i]],(long)mout[i],(long)gauge); else pt_cmm_cpp(non_local_chi_cb[wire[i]],(long)uc_nl_cb[parity][wire[i]],(long)rcv_buf[wire[i]],(long)mout[i],(long)gauge); } //Otherwise we have parallel transport in the positive direction. //In this case, the received data has already been pre-multiplied //All we need to do is to put the transported field in the correct place else { //int destination, source; //Place the data in the receive buffer into the result vector for(int s=0;s<non_local_chi_cb[wire[i]];s++) { //source = uc_nl_cb[parity][wire[i]][s].src; fp0 = (IFloat *)((long)rcv_buf[wire[i]]+3*uc_nl_cb[parity][wire[i]][s].src); //destination = uc_nl_cb[parity][wire[i]][s].dest; fp1 = (IFloat *)(mout[i]+3*uc_nl_cb[parity][wire[i]][s].dest); memcpy(fp1,fp0,GAUGE_LEN*sizeof(IFloat)); } } } } } //#pragma omp parallel #ifdef PROFILE dtime +=dclock(); print_flops("",fname,99*vol*n,dtime); #endif // ParTrans::PTflops +=99*n*vol; }
void PT::mat(int n, matrix **mout, matrix **min, const int *dir){ int wire[MAX_DIR]; int i; QMP_msgmem_t msg_mem_p[2*MAX_DIR]; QMP_msghandle_t msg_handle_p[2*MAX_DIR]; QMP_msghandle_t multiple; static double setup=0.,qmp=0.,localt=0.,nonlocal=0.; static int call_num = 0; call_num++; // char *fname="pt_mat()"; // VRB.Func("",fname); // if (call_num%100==1) printf("PT:mat()\n"); for(i=0;i<n;i++) wire[i] = dir[i]; #ifdef PROFILE Float dtime2 = - dclock(); #endif double dtime = -dclock(); int non_local_dir=0; for(i=0;i<n;i++) if (!local[wire[i]/2]) { //Calculate the address for transfer in a particular direction Float * addr = ((Float *)min[i]+GAUGE_LEN*offset[wire[i]]); msg_mem_p[2*non_local_dir] = QMP_declare_msgmem((void *)rcv_buf[wire[i]], 3*non_local_chi[wire[i]]*VECT_LEN*sizeof(IFloat)); msg_mem_p[2*non_local_dir+1] = QMP_declare_strided_msgmem((void *)addr, (size_t)(3*blklen[wire[i]]), numblk[wire[i]], (ptrdiff_t)(3*stride[wire[i]]+3*blklen[wire[i]])); msg_handle_p[2*non_local_dir] = QMP_declare_receive_relative(msg_mem_p[2*non_local_dir], wire[i]/2, 1-2*(wire[i]%2), 0); msg_handle_p[2*non_local_dir+1] = QMP_declare_send_relative(msg_mem_p[2*non_local_dir+1], wire[i]/2, 2*(wire[i]%2)-1, 0); non_local_dir++; } if (call_num==1 && !QMP_get_node_number()) printf("non_local_dir=%d\n",non_local_dir); if(non_local_dir) { multiple = QMP_declare_multiple(msg_handle_p, 2*non_local_dir); QMP_start(multiple); } dtime += dclock(); setup +=dtime; dtime = -dclock(); int if_print = 0; // if ( (call_num%10000==1) && (!QMP_get_node_number()) ) if_print=1; #define USE_TEST2 #ifdef USE_TEST2 //assume nt > n! static char *cname="mat()"; #pragma omp parallel default(shared) { int iam,nt,ipoints,istart,offset; iam = omp_get_thread_num(); nt = omp_get_num_threads(); int nt_dir = nt/n; int n_t = iam/nt_dir; int i_t = iam%nt_dir; if (n_t >= n ){ n_t = n-1; i_t = iam - (n-1)*nt_dir; nt_dir = nt -(n-1)*nt_dir; } int w_t = wire[n_t]; ipoints = (local_chi[w_t]/2)/nt_dir; offset = ipoints*i_t; if (i_t == (nt_dir-1)) ipoints = (local_chi[w_t]/2)-offset; if ( if_print ) printf("thread %d of %d nt_dir n_t i_t ipoints offset= %d %d %d %d %d\n",iam,nt,nt_dir,n_t,i_t,ipoints,offset); //Interleaving of local computation of matrix multiplication partrans_cmm_agg((uc_l[w_t]+offset*2),min[n_t],mout[n_t],ipoints); if ( if_print ) printf("thread %d of %d done\n",iam,nt); } #else { //Interleaving of local computation of matrix multiplication #pragma omp parallel for default(shared) for(i=0;i<n;i++){ partrans_cmm_agg(uc_l[wire[i]],min[i],mout[i],local_chi[wire[i]]/2); } } #endif dtime += dclock(); localt +=dtime; dtime = -dclock(); //#pragma omp barrier //#pragma omp master { if(non_local_dir) { QMP_status_t qmp_complete_status = QMP_wait(multiple); if (qmp_complete_status != QMP_SUCCESS) QMP_error("Send failed in vec_cb_norm: %s\n", QMP_error_string(qmp_complete_status)); QMP_free_msghandle(multiple); for(int i = 0; i < 2*non_local_dir; i++) QMP_free_msgmem(msg_mem_p[i]); // Free(msg_handle_p); // Free(msg_mem_p); } } //#pragma omp master { dtime += dclock(); qmp +=dtime; dtime = -dclock(); //Do non-local computations #ifdef USE_TEST2 //assume nt > n! #pragma omp parallel default(shared) { int iam,nt,ipoints,istart,offset; iam = omp_get_thread_num(); nt = omp_get_num_threads(); int nt_dir = nt/n; int n_t = iam/nt_dir; int i_t = iam%nt_dir; if (n_t >= n ){ n_t = n-1; i_t = iam - (n-1)*nt_dir; nt_dir = nt -(n-1)*nt_dir; } int w_t = wire[n_t]; ipoints = (non_local_chi[w_t]/2)/nt_dir; offset = ipoints*i_t; if (i_t == (nt_dir-1)) ipoints = (non_local_chi[w_t]/2)-offset; if ( if_print ) printf("thread %d of %d nt_dir n_t i_t ipoints offset= %d %d %d %d %d\n",iam,nt,nt_dir,n_t,i_t,ipoints,offset); //Non-local computation if (ipoints>0) partrans_cmm_agg((uc_nl[w_t]+offset*2),(matrix *)rcv_buf[w_t],mout[n_t],ipoints); if ( if_print ) printf("thread %d of %d done\n",iam,nt); } #else { #pragma omp parallel for for(i=0;i<n;i++) if (!local[wire[i]/2]) { #ifdef USE_OMP if (call_num%10000==1 && !QMP_get_node_number() ) printf("thread %d of %d i=%d\n",omp_get_thread_num(),omp_get_num_threads(),i); #endif partrans_cmm_agg(uc_nl[wire[i]],(matrix *)rcv_buf[wire[i]],mout[i],non_local_chi[wire[i]]/2); } }//#pragma omp parallel #endif dtime += dclock(); nonlocal +=dtime; if (call_num%100==0){ static char *cname="mat()"; if (!QMP_get_node_number() ) { print_flops("mat():local*100",0,localt); print_flops("mat():nonlocal*100",0,nonlocal); print_flops("mat():qmp*100",0,qmp); print_flops("mat():setup*100",0,setup); } localt=nonlocal=qmp=setup=0.; } #ifdef PROFILE dtime2 +=dclock(); print_flops("",fname,198*vol*n,dtime2); #endif // ParTrans::PTflops +=198*n*vol; }
/*! Computes sum[x] = vect2[x] vect[x + hop dir]^dagger where the sum is over n_vect vectors and the hop is in a forward direction. */ void PT::vvpd(IFloat **vect2, IFloat ***vect, int n_vect, const int *dir, int n_dir, int hop, IFloat **sum, int overwrite){ char *fname = "pt_vvpd()"; #if 1 // ERR.NotImplemented(cname,fname); QMP_error("%s""%s Not implemented\n"); #else // VRB.Func("",fname); int i, s, v; Float f = 2.0; int wire[MAX_DIR]; for(i=0;i<n_dir;i++) wire[i] = dir[i]; // from (x,y,z,t) to (t,x,y,z) QMP_msgmem_t *msg_mem_p = (QMP_msgmem_t *)Alloc("","vvpd", "msg_mem_p", 2*non_local_dirs*sizeof(QMP_msgmem_t)); QMP_msgmem_t *msg_mem_p2 = (QMP_msgmem_t *)Alloc("","vvpd", "msg_mem_p", 2*non_local_dirs*sizeof(QMP_msgmem_t)); QMP_msghandle_t* msg_handle_p = (QMP_msghandle_t *)Alloc("","vvpd", "msg_handle_p", 2*non_local_dirs*sizeof(QMP_msghandle_t)); QMP_msghandle_t* msg_handle_p2 = (QMP_msghandle_t *)Alloc("","vvpd", "msg_handle_p", 2*non_local_dirs*sizeof(QMP_msghandle_t)); QMP_msghandle_t multiple; //Setup communciation int comms=0; for(i=0;i<n_dir;i++) if( !local[wire[i]/2]) { if ( size[wire[i]/2] <hop) fprintf(stderr, "%s:size(%d) in direction %d is smaller than the hop(%d)\n", fname,size[wire[i]],wire[i],hop); comms++; } for(v=0; v<n_vect; v++){ if (v%2==0) { comms=0; for(i=0;i<n_dir;i++) if( !local[wire[i]/2]){ msg_mem_p[2*comms] = QMP_declare_msgmem((void *)rcv_buf[wire[i]], hop*non_local_chi[wire[i]]*VECT_LEN*sizeof(IFloat)); msg_handle_p[2*comms] = QMP_declare_receive_relative(msg_mem_p[2*comms], wire[i]/2, 1-2*(wire[i]%2), 0); msg_mem_p[2*comms+1] = QMP_declare_strided_msgmem((void *)(vect[v][i]+VECT_LEN*set_offset(wire[i], hop)), (size_t)(hop*blklen[wire[i]]), numblk[wire[i]], (ptrdiff_t)(stride[wire[i]] + blklen[wire[i]])); msg_handle_p[2*comms+1] = QMP_declare_send_relative(msg_mem_p[2*comms+1], wire[i]/2, 2*(wire[i]%2)-1, 0); comms++; } // Start communication if(comms) { multiple = QMP_declare_multiple(msg_handle_p, 2*comms); } if (comms) { QMP_start(multiple); QMP_status_t qmp_complete_status = QMP_wait(multiple); if (qmp_complete_status != QMP_SUCCESS) QMP_error("Send failed in vvpd: %s\n", QMP_error_string(qmp_complete_status)); QMP_free_msghandle(multiple); for(int i = 0; i < 2*comms; i++) QMP_free_msgmem(msg_mem_p[i]); } } else { comms=0; for(i=0;i<n_dir;i++) if( !local[wire[i]/2]){ msg_mem_p2[2*comms] = QMP_declare_msgmem((void *)rcv_buf2[wire[i]], hop*non_local_chi[wire[i]]*VECT_LEN*sizeof(IFloat)); msg_handle_p2[2*comms] = QMP_declare_receive_relative(msg_mem_p2[2*comms], wire[i]/2, 1-2*(wire[i]%2), 0); msg_mem_p2[2*comms+1] = QMP_declare_strided_msgmem((void *)(vect[v][i]+VECT_LEN*set_offset(wire[i], hop)), (size_t)(hop*blklen[wire[i]]), numblk[wire[i]], (ptrdiff_t)(stride[wire[i]] + blklen[wire[i]])); msg_handle_p2[2*comms+1] = QMP_declare_send_relative(msg_mem_p2[2*comms+1], wire[i]/2, 2*(wire[i]%2)-1, 0); comms++; } // Start communication if(comms) { multiple = QMP_declare_multiple(msg_handle_p2, 2*comms); } if (comms) { QMP_start(multiple); QMP_status_t qmp_complete_status = QMP_wait(multiple); if (qmp_complete_status != QMP_SUCCESS) QMP_error("Send failed in vvpd: %s\n", QMP_error_string(qmp_complete_status)); QMP_free_msghandle(multiple); for(int i = 0; i < 2*comms; i++) QMP_free_msgmem(msg_mem_p2[i]); } } // Perform non-local calculation for previous v if (v>0) if (v==1 && overwrite==1) { for(i=0; i<n_dir; i++) if(non_local_chi[wire[i]]>0) cross_over_lin(sum[i], &f, vect2[v-1],rcv_buf[wire[i]], hop*non_local_chi[wire[i]], src_nl[hop-1][wire[i]], dest_nl[hop-1][wire[i]]); } else if (v%2==1) { for(i=0; i<n_dir; i++) if(non_local_chi[wire[i]]>0) cross_lin(sum[i], &f, vect2[v-1],rcv_buf[wire[i]], hop*non_local_chi[wire[i]], src_nl[hop-1][wire[i]], dest_nl[hop-1][wire[i]]); } else { for(i=0; i<n_dir; i++) if(non_local_chi[wire[i]]>0) cross_lin(sum[i], &f,vect2[v-1],rcv_buf2[wire[i]], hop*non_local_chi[wire[i]], src_nl[hop-1][wire[i]], dest_nl[hop-1][wire[i]]); } // Perform local calculation for current v if (v==0 && overwrite==1) { for(i=0; i<n_dir; i++) if((vol-hop*non_local_chi[wire[i]])>0) cross_over_look(sum[i], &f, vect2[v], vect[v][i], vol-hop*non_local_chi[wire[i]], src_l[hop-1][wire[i]], dest_l[hop-1][wire[i]]); } else { for(i=0; i<n_dir; i++) if((vol-hop*non_local_chi[wire[i]])>0) cross_look(sum[i], &f, vect2[v], vect[v][i], vol-hop*non_local_chi[wire[i]], src_l[hop-1][wire[i]], dest_l[hop-1][wire[i]]); } } if (v==1 && overwrite==1) { for(i=0; i<n_dir; i++) if(non_local_chi[wire[i]]>0) cross_over_lin(sum[i], &f, vect2[v-1],rcv_buf[wire[i]], hop*non_local_chi[wire[i]], src_nl[hop-1][wire[i]], dest_nl[hop-1][wire[i]]); } else if (v%2==1) { for(i=0; i<n_dir; i++) if(non_local_chi[wire[i]]>0) cross_lin(sum[i], &f, vect2[v-1],rcv_buf[wire[i]], hop*non_local_chi[wire[i]], src_nl[hop-1][wire[i]], dest_nl[hop-1][wire[i]]); } else { for(i=0; i<n_dir; i++) if(non_local_chi[wire[i]]>0) cross_lin(sum[i], &f,vect2[v-1],rcv_buf2[wire[i]], hop*non_local_chi[wire[i]], src_nl[hop-1][wire[i]], dest_nl[hop-1][wire[i]]); } #endif // ParTrans::PTflops += 90*n_vect*n_dir*vol; }
//! u[x] = v[x+dir] for n_dir forward or backward directions dir. void PT::shift_field(IFloat **v, const int *dir, int n_dir, int hop, IFloat **u){ int i, length; int wire[n_dir]; for (i=0; i<n_dir;i++) wire[i] = dir[i]; #ifdef USE_QMP QMP_msgmem_t msg_mem_p[20]; QMP_msghandle_t msg_handle_p[20]; QMP_msghandle_t multiple; #else SCUDirArgMulti SCUmulti; SCUDirArgIR *SCUarg_p[2*n_dir]; #endif int comms=0; for (i=0; i<n_dir; i++) if (!local[wire[i]/2]){ #ifndef USE_QMP SCUarg_p[2*comms] = SCUarg_mat[hop-1][2*wire[i]]; SCUarg_p[2*comms+1] = SCUarg_mat[hop-1][2*wire[i]+1]; SCUarg_p[2*comms+1]->Addr((void *)(v[i]+GAUGE_LEN*set_offset(wire[i], hop))); #else msg_mem_p[2*comms] = QMP_declare_msgmem((void *)rcv_buf[wire[i]], 3*hop*non_local_chi[wire[i]]*VECT_LEN*sizeof(IFloat)); msg_mem_p[2*comms+1] = QMP_declare_strided_msgmem((void *)(v[i]+GAUGE_LEN*set_offset(wire[i], hop)), (size_t)(3*hop*blklen[wire[i]]), numblk[wire[i]], (ptrdiff_t)(3*stride[wire[i]]+3*blklen[wire[i]])); msg_handle_p[2*comms] = QMP_declare_receive_relative(msg_mem_p[2*comms], wire[i]/2, 1-2*(wire[i]%2), 0); msg_handle_p[2*comms+1] = QMP_declare_send_relative(msg_mem_p[2*comms+1], wire[i]/2, 2*(wire[i]%2)-1, 0); #endif comms++; } #ifndef USE_QMP if (comms) SCUmulti.Init(SCUarg_p,2*comms); if (comms) SCUmulti.SlowStartTrans(); #else if(comms) { multiple = QMP_declare_multiple(msg_handle_p, 2*comms); QMP_start(multiple); } #endif // SCUmulti.TransComplete(); for (i=0; i<n_dir; i++) { length = vol-hop*non_local_chi[wire[i]]; copy_matrix(u[i],v[i],&length,dest_l[hop-1][wire[i]], src_l[hop-1][wire[i]]); } #ifndef USE_QMP if (comms) SCUmulti.TransComplete(); #else if(comms) { QMP_status_t qmp_complete_status = QMP_wait(multiple); if (qmp_complete_status != QMP_SUCCESS) QMP_error("Send failed in shift_field: %s\n", QMP_error_string(qmp_complete_status)); QMP_free_msghandle(multiple); for(int i = 0; i < 2*comms; i++) QMP_free_msgmem(msg_mem_p[i]); } #endif for (i=0; i<n_dir; i++) { length = hop*non_local_chi[wire[i]]; copy_matrix(u[i],(IFloat*)rcv_buf[wire[i]],&length, dest_nl[hop-1][wire[i]],src_nl[hop-1][wire[i]]); } }
void create_msgs(int **smem, int **rmem, QMP_msgmem_t *sendmem, QMP_msgmem_t *recvmem, QMP_msghandle_t *sendh, QMP_msghandle_t *recvh, int ndims, int nc, int size, struct perf_argv *pargv) { int i, j, n; for (i = 0; i < nc; i++) { if(strided_array_send) { void *base[NAMAX]; size_t bsize[NAMAX]; int nblocks[NAMAX]; ptrdiff_t stride[NAMAX]; int tsize, skip; int na, k, bs, nb, nbt, ab, as, st; bs = strided_array_send; nbt = size/bs; na = sqrt(nbt); if(na<2) na = nbt; if(na>NAMAX) na = NAMAX; nb = nbt/na; st = 2*bs; skip = 3*bs; ab = bs*nb; as = st*nb+skip; tsize = 0; for(k=0; k<na; k++) { bsize[k] = bs*sizeof(int); stride[k] = st*sizeof(int); nblocks[k] = nb; if(k==na-1) nblocks[k] = nbt - nb*(na-1); tsize += skip + st * nblocks[k]; } smem[i] = (int *)malloc(ndims*tsize*sizeof(int)); for(n=0; n<ndims; n++) { for (j = 0; j < tsize; j++) { smem[i][n*tsize+j] = 0; } for (j = 0; j < size; j++) { int ai, ak; ak = j/ab; if(ak>=na) ak = na-1; ai = j-(ab*ak); k = (as*ak) + st*(ai/bs) + (ai%bs); smem[i][n*tsize+k] = i+j+1; } for(k=0; k<na; k++) { base[k] = (void *)&(smem[i][n*tsize+as*k]); } sendmem[n*nc+i] = QMP_declare_strided_array_msgmem(base, bsize, nblocks, stride, na); if(!sendmem[n*nc+i]) { QMP_printf("error in declare strided msgmem\n"); QMP_abort(1); } } } else if(strided_send) { int tsize, bsize, stride, nblocks; bsize = strided_send; stride = 2*bsize; nblocks = size/bsize; tsize = stride * nblocks; smem[i] = (int *)malloc(ndims*tsize*sizeof (int)); for(n=0; n<ndims; n++) { for (j = 0; j < tsize; j++) { smem[i][n*tsize+j] = 0; } for (j = 0; j < size; j++) { int k = stride*(j/bsize) + (j%bsize); smem[i][n*tsize+k] = i+j+1; } sendmem[n*nc+i] = QMP_declare_strided_msgmem(smem[i]+(n*tsize), bsize*sizeof(int), nblocks, stride*sizeof(int)); if(!sendmem[n*nc+i]) { QMP_printf("error in declare strided msgmem\n"); QMP_abort(1); } } } else { smem[i] = (int *)malloc(ndims*size*sizeof(int)); for(n=0; n<ndims; n++) { for (j = 0; j < size; j++) { smem[i][n*size+j] = i+j+1; } sendmem[n*nc+i] = QMP_declare_msgmem(smem[i]+(n*size), size*sizeof(int)); if(!sendmem[n*nc+i]) { QMP_printf("error in declare msgmem\n"); QMP_abort(1); } } } if(strided_recv) { int tsize, bsize, stride, nblocks; bsize = strided_recv; stride = 2*bsize; nblocks = size/bsize; tsize = stride * nblocks; rmem[i] = (int *)malloc(ndims*tsize*sizeof (int)); for(n=0; n<ndims; n++) { for (j = 0; j < tsize; j++) { rmem[i][n*tsize+j] = 0; } recvmem[n*nc+i] = QMP_declare_strided_msgmem(rmem[i]+(n*tsize), bsize*sizeof(int), nblocks, stride*sizeof(int)); if(!recvmem[n*nc+i]) { QMP_printf("error in declare strided msgmem\n"); QMP_abort(1); } } } else { rmem[i] = (int *)malloc(ndims*size*sizeof (int)); for(n=0; n<ndims; n++) { for (j = 0; j < size; j++) { rmem[i][n*size+j] = 0; } recvmem[n*nc+i] = QMP_declare_msgmem (rmem[i]+(n*size), size*sizeof(int)); if(!recvmem[n*nc+i]) { QMP_printf("error in declare msgmem\n"); QMP_abort(1); } } } if(ndims>0) { // always use QMP_msghandle_t *tsend, *trecv; tsend = (QMP_msghandle_t *)malloc(ndims*sizeof(QMP_msghandle_t)); trecv = (QMP_msghandle_t *)malloc(ndims*sizeof(QMP_msghandle_t)); for(n=0; n<ndims; n++) { trecv[n] = QMP_declare_receive_relative (recvmem[n*nc+i], n, 1, 0); if (!trecv[n]) { QMP_printf ("Recv Handle Error: %s\n", QMP_get_error_string(0)); exit (1); } tsend[n] = QMP_declare_send_relative (sendmem[n*nc+i], n, -1, 0); if (!tsend[n]) { QMP_printf ("Send Handle Error: %s\n", QMP_get_error_string(0)); exit (1); } } if(pargv->option & TEST_PINGPONG) { if(pargv->sender) { sendh[i] = QMP_declare_send_recv_pairs(tsend, ndims); recvh[i] = QMP_declare_send_recv_pairs(trecv, ndims); } else { recvh[i] = QMP_declare_send_recv_pairs(trecv, ndims); sendh[i] = QMP_declare_send_recv_pairs(tsend, ndims); } } else { recvh[i] = QMP_declare_multiple(trecv, ndims); sendh[i] = QMP_declare_multiple(tsend, ndims); } if (!recvh[i]) { QMP_printf ("Recv Handle Error: %s\n", QMP_get_error_string(0)); exit (1); } if (!sendh[i]) { QMP_printf ("Send Handle Error: %s\n", QMP_get_error_string(0)); exit (1); } free(tsend); free(trecv); } else { recvh[i] = QMP_declare_receive_relative (recvmem[i], 0, 1, 0); if (!recvh[i]) { QMP_printf ("Recv Handle Error: %s\n", QMP_get_error_string(0)); exit (1); } sendh[i] = QMP_declare_send_relative (sendmem[i], 0, -1, 0); if (!sendh[i]) { QMP_printf ("Send Handle Error: %s\n", QMP_get_error_string(0)); exit (1); } } } }
void wfm_comm(){ char *fname="wfm_comm()"; const int group = 16; void *addr[group]; size_t blksize[group]; int numblk[group]; ptrdiff_t stride[group]; int index; const int MAX_MSGHANDLE=20; if (wfm_max_numchunk/group+1 >MAX_MSGHANDLE) ERR.General("",fname,"wfm_max_numchunk(%d)/group+1 >MAX_MSGHANDLE",wfm_max_numchunk); static QMP_msgmem_t send_mem[8][MAX_MSGHANDLE]; static QMP_msgmem_t recv_mem[8][MAX_MSGHANDLE]; static QMP_msghandle_t send_h[8][MAX_MSGHANDLE]; static QMP_msghandle_t recv_h[8][MAX_MSGHANDLE]; static int pir=0; static int wfm_blocks[8]; if (wilson_initted || !initted){ VRB.Flow("",fname,"wilson_initted=%d initted=%d\n",wilson_initted,initted); for(int dir=0;dir<8;dir++) wfm_blocks[dir]=1; for(int ig=0; ig<1; ig++){ // VRB.Flow("",fname,"ig=%d",ig); for(int dir=0;dir<8;dir++){ int sign=1; if(dir>3) sign = -1; // VRB.Flow("",fname,"dir=%d",dir); int n_site=0; addr[n_site] = wfm_s_start[dir]; blksize[n_site] = wfm_blklen[dir]; numblk[n_site] = wfm_numblk[dir]; stride[n_site] = wfm_stride[dir]; n_site++; if (initted){ QMP_free_msghandle(send_h[dir][ig]); QMP_free_msghandle(recv_h[dir][ig]); QMP_free_msgmem(send_mem[dir][ig]); QMP_free_msgmem(recv_mem[dir][ig]); } if(n_site>0){ send_mem[dir][ig] = QMP_declare_strided_array_msgmem(addr,blksize,numblk,stride,n_site); send_h[dir][ig] = QMP_declare_send_relative(send_mem[dir][ig],dir%4,sign,0); wfm_blocks[dir]=ig+1; } int r_site=0; addr[r_site] = wfm_r_start[dir]; blksize[r_site] = wfm_blklen[dir]; numblk[r_site] = wfm_numblk[dir]; stride[r_site] = wfm_stride[dir]; r_site++; if (n_site!=r_site) ERR.General("",fname,"n_site(%d)!=r_site(%d)\n",n_site,r_site); // VRB.Flow("",fname,"n_site=%d r_site=%d",n_site,r_site); if(r_site>0){ recv_mem[dir][ig] = QMP_declare_strided_array_msgmem(addr,blksize,numblk,stride,r_site); recv_h[dir][ig] = QMP_declare_receive_relative(recv_mem[dir][ig],dir%4,-sign,0); } } // dir } // ig for(int dir=0;dir<8;dir++) VRB.Flow("",fname,"wfm_blocks[%d]=%d",dir,wfm_blocks[dir]); pir = CoorT()%2; // pir = 0; initted=1; wilson_initted=0; } #if 0 for(int dir=0;dir<8;dir++) for ( index=0; index <wfm_numchunk[dir];index++){ Float *tmp_p = wfm_send_ad[dir+8*index]; if ( (*tmp_p)*(*tmp_p) >0.0001) printf("Node %d: wfm_send_ad[%d][%d]=%e\n",UniqueID(),dir,index,*tmp_p); } #endif int dir_g=4; for(int ig=0; ig<1; ig++){ for(index=0;index<8;index++){ QMP_start(send_h[index][ig]); QMP_start(recv_h[index][ig]); } for(index=0;index<8;index++){ QMP_wait(send_h[index][ig]); QMP_wait(recv_h[index][ig]); } } #if 0 for(int dir=0;dir<8;dir++) for ( index=0; index <wfm_numchunk[dir];index++){ Float *tmp_p = wfm_recv_ad[dir+8*index]; if ( (*tmp_p)*(*tmp_p) >1e-10) printf("Node %d: wfm_recv_ad[%d][%d]=%e\n",UniqueID(),dir,index,*tmp_p); } #endif }