void calc_node_neighbors(int node)
{
int dir;
map_node_array(node,node_pos);
for(dir=0;dir<3;dir++) {
int buf;
MPI_Cart_shift(comm_cart, dir, -1, &buf, node_neighbors + 2*dir);
MPI_Cart_shift(comm_cart, dir, 1, &buf, node_neighbors + 2*dir + 1);
/* left boundary ? */
if (node_pos[dir] == 0) {
boundary[2*dir] = 1;
}
else {
boundary[2*dir] = 0;
}
/* right boundary ? */
if (node_pos[dir] == node_grid[dir]-1) {
boundary[2*dir+1] = -1;
}
else {
boundary[2*dir+1] = 0;
}
}
GRID_TRACE(printf("%d: node_grid %d %d %d, pos %d %d %d, node_neighbors ", this_node, node_grid[0], node_grid[1], node_grid[2], node_pos[0], node_pos[1], node_pos[2]));
}
/** Calculate a velocity profile for the LB fluid. */
void lb_calc_velprof(double *result, int *params) {
int index, dir[3], grid[3];
int newroot=0, subrank, involved=0;
double rho, j[3], *velprof;
MPI_Comm slice_comm = NULL;
MPI_Status status;
if (this_node != 0) params = malloc(4*sizeof(int));
MPI_Bcast(params, 4, MPI_INT, 0, MPI_COMM_WORLD);
int vcomp = params[0];
int pdir = params[1];
int x1 = params[2];
int x2 = params[3];
dir[pdir] = 0;
dir[(pdir+1)%3] = x1;
dir[(pdir+2)%3] = x2;
//fprintf(stderr,"%d: (%d,%d,%d)\n",this_node,dir[0],dir[1],dir[2]);
newroot = map_lattice_to_node(&lblattice, dir, grid);
map_node_array(this_node, node_pos);
//fprintf(stderr,"%d: newroot=%d (%d,%d,%d)\n",this_node,newroot,grid[0],grid[1],grid[2]);
if ( (grid[(pdir+1)%3] == node_pos[(pdir+1)%3])
&& (grid[(pdir+2)%3] == node_pos[(pdir+2)%3]) ) {
involved = 1;
}
MPI_Comm_split(MPI_COMM_WORLD, involved, this_node, &slice_comm);
MPI_Comm_rank(slice_comm, &subrank);
if (this_node == newroot)
result = realloc(result,box_l[pdir]/lblattice.agrid*sizeof(double));
//fprintf(stderr,"%d (%d,%d): result=%p vcomp=%d pdir=%d x1=%d x2=%d involved=%d\n",this_node,subrank,newroot,result,vcomp,pdir,x1,x2,involved);
if (involved) {
velprof = malloc(lblattice.grid[pdir]*sizeof(double));
//dir[(pdir+1)%3] += 1;
//dir[(pdir+2)%3] += 1;
for (dir[pdir]=1;dir[pdir]<=lblattice.grid[pdir];dir[pdir]++) {
index = get_linear_index(dir[0],dir[1],dir[2],lblattice.halo_grid);
lb_calc_local_fields(index, &rho, j, NULL);
//fprintf(stderr,"%p %d %.12e %.12e %d\n",lbfluid[0],index,rho,j[0],vcomp);
if (rho < ROUND_ERROR_PREC) {
velprof[dir[pdir]-1] = 0.0;
} else {
//velprof[dir[pdir]-1] = local_j / (SQR(lbpar.agrid)*lbpar.tau);
velprof[dir[pdir]-1] = j[vcomp]/rho * lblattice.agrid/lbpar.tau;
//fprintf(stderr,"%f %f %f\n",velprof[dir[pdir]-1],local_j,local_rho);
}
//if (dir[pdir]==lblattice.grid[pdir]) {
// int i;
// fprintf(stderr,"(%d,%d,%d)\n",dir[0],dir[1],dir[2]);
// fprintf(stderr,"%d\n",lbfluid[index].boundary);
// for (i=0;i<lbmodel.n_veloc;i++) fprintf(stderr,"local_n[%p][%d]=%.12e\n",lbfluid[index].n,i,lbfluid[index].n[i]+lbmodel.coeff[i][0]*lbpar.rho);
// fprintf(stderr,"local_rho=%e\n",local_rho);
// fprintf(stderr,"local_j=%e\n",local_j);
//}
}
MPI_Gather(velprof, lblattice.grid[pdir], MPI_DOUBLE, result, lblattice.grid[pdir], MPI_DOUBLE, newroot, slice_comm);
free(velprof);
}
MPI_Comm_free(&slice_comm);
if (newroot != 0) {
if (this_node == newroot) {
MPI_Send(result, lblattice.grid[pdir]*node_grid[pdir], MPI_DOUBLE, 0, REQ_VELPROF, MPI_COMM_WORLD);
free(result);
}
if (this_node == 0) {
//fprintf(stderr,"%d: I'm just here!\n",this_node);
MPI_Recv(result, lblattice.grid[pdir]*node_grid[pdir], MPI_DOUBLE, newroot, REQ_VELPROF, MPI_COMM_WORLD, &status);
//fprintf(stderr,"%d: And now I'm here!\n",this_node);
}
}
if (this_node !=0) free(params);
}
/** Calculate a density profile of the fluid. */
void lb_calc_densprof(double *result, int *params) {
int index, dir[3], grid[3];
int newroot=0, subrank, involved=0;
double *profile;
MPI_Comm slice_comm = NULL;
MPI_Status status;
if (this_node !=0) params = malloc(3*sizeof(int));
MPI_Bcast(params, 3, MPI_INT, 0, MPI_COMM_WORLD);
int pdir = params[0];
int x1 = params[1];
int x2 = params[2];
dir[pdir] = 0;
dir[(pdir+1)%3] = x1;
dir[(pdir+2)%3] = x2;
newroot = map_lattice_to_node(&lblattice, dir, grid);
map_node_array(this_node, node_pos);
if ( (grid[(pdir+1)%3] == node_pos[(pdir+1)%3])
&& (grid[(pdir+2)%3] == node_pos[(pdir+2)%3]) ) {
involved = 1;
}
MPI_Comm_split(MPI_COMM_WORLD, involved, this_node, &slice_comm);
MPI_Comm_rank(slice_comm, &subrank);
if (this_node == newroot)
result = realloc(result,box_l[pdir]/lblattice.agrid*sizeof(double));
if (involved) {
profile = malloc(lblattice.grid[pdir]*sizeof(double));
//dir[(pdir+1)%3] += 1;
//dir[(pdir+2)%3] += 1;
for (dir[pdir]=1;dir[pdir]<=lblattice.grid[pdir];dir[pdir]++) {
index = get_linear_index(dir[0],dir[1],dir[2],lblattice.halo_grid);
lb_calc_local_rho(index,&profile[dir[pdir]-1]);
//profile[dir[pdir]-1] = *lbfluid[index].rho;
//if (dir[pdir]==lblattice.grid[pdir]) {
// int i;
// fprintf(stderr,"(%d,%d,%d)\n",dir[0],dir[1],dir[2]);
// fprintf(stderr,"%d\n",lbfluid[index].boundary);
// for (i=0;i<lbmodel.n_veloc;i++) fprintf(stderr,"local_n[%p][%d]=%.12e\n",lbfluid[index].n,i,lbfluid[index].n[i]+lbmodel.coeff[i][0]*lbpar.rho);
// fprintf(stderr,"local_rho=%e\n",*lbfluid[index].rho);
//}
}
MPI_Gather(profile, lblattice.grid[pdir], MPI_DOUBLE, result, lblattice.grid[pdir], MPI_DOUBLE, 0, slice_comm);
free(profile);
}
MPI_Comm_free(&slice_comm);
if (newroot != 0) {
if (this_node == newroot) {
MPI_Send(result, lblattice.grid[pdir]*node_grid[pdir], MPI_DOUBLE, 0, REQ_DENSPROF, MPI_COMM_WORLD);
free(result);
}
if (this_node == 0) {
MPI_Recv(result, lblattice.grid[pdir]*node_grid[pdir], MPI_DOUBLE, newroot, REQ_DENSPROF, MPI_COMM_WORLD, &status);
}
}
if (this_node != 0) free(params);
}
int dfft_init(double **data,
int *local_mesh_dim, int *local_mesh_margin,
int* global_mesh_dim, double *global_mesh_off,
int *ks_pnum)
{
int i,j;
/* helpers */
int mult[3];
int n_grid[4][3]; /* The four node grids. */
int my_pos[4][3]; /* The position of this_node in the node grids. */
int *n_id[4]; /* linear node identity lists for the node grids. */
int *n_pos[4]; /* positions of nodes in the node grids. */
/* FFTW WISDOM stuff. */
char wisdom_file_name[255];
FILE *wisdom_file;
int wisdom_status;
FFT_TRACE(fprintf(stderr,"%d: dipolar dfft_init():\n",this_node));
dfft.max_comm_size=0; dfft.max_mesh_size=0;
for(i=0;i<4;i++) {
n_id[i] = (int *) malloc(1*n_nodes*sizeof(int));
n_pos[i] = (int *) malloc(3*n_nodes*sizeof(int));
}
/* === node grids === */
/* real space node grid (n_grid[0]) */
for(i=0;i<3;i++) {
n_grid[0][i] = node_grid[i];
my_pos[0][i] = node_pos[i];
}
for(i=0;i<n_nodes;i++) {
map_node_array(i,&(n_pos[0][3*i+0]));
n_id[0][get_linear_index( n_pos[0][3*i+0],n_pos[0][3*i+1],n_pos[0][3*i+2], n_grid[0])] = i;
}
/* FFT node grids (n_grid[1 - 3]) */
calc_2d_grid(n_nodes,n_grid[1]);
/* resort n_grid[1] dimensions if necessary */
dfft.plan[1].row_dir = map_3don2d_grid(n_grid[0], n_grid[1], mult);
dfft.plan[0].n_permute = 0;
for(i=1;i<4;i++) dfft.plan[i].n_permute = (dfft.plan[1].row_dir+i)%3;
for(i=0;i<3;i++) {
n_grid[2][i] = n_grid[1][(i+1)%3];
n_grid[3][i] = n_grid[1][(i+2)%3];
}
dfft.plan[2].row_dir = (dfft.plan[1].row_dir-1)%3;
dfft.plan[3].row_dir = (dfft.plan[1].row_dir-2)%3;
/* === communication groups === */
/* copy local mesh off real space charge assignment grid */
for(i=0;i<3;i++) dfft.plan[0].new_mesh[i] = local_mesh_dim[i];
for(i=1; i<4;i++) {
dfft.plan[i].g_size=fft_find_comm_groups(n_grid[i-1], n_grid[i], n_id[i-1], n_id[i],
dfft.plan[i].group, n_pos[i], my_pos[i]);
if(dfft.plan[i].g_size==-1) {
/* try permutation */
j = n_grid[i][(dfft.plan[i].row_dir+1)%3];
n_grid[i][(dfft.plan[i].row_dir+1)%3] = n_grid[i][(dfft.plan[i].row_dir+2)%3];
n_grid[i][(dfft.plan[i].row_dir+2)%3] = j;
dfft.plan[i].g_size=fft_find_comm_groups(n_grid[i-1], n_grid[i], n_id[i-1], n_id[i],
dfft.plan[i].group, n_pos[i], my_pos[i]);
if(dfft.plan[i].g_size==-1) {
fprintf(stderr,"%d: dipolar INTERNAL ERROR: fft_find_comm_groups error\n", this_node);
errexit();
}
}
dfft.plan[i].send_block = (int *)realloc(dfft.plan[i].send_block, 6*dfft.plan[i].g_size*sizeof(int));
dfft.plan[i].send_size = (int *)realloc(dfft.plan[i].send_size, 1*dfft.plan[i].g_size*sizeof(int));
dfft.plan[i].recv_block = (int *)realloc(dfft.plan[i].recv_block, 6*dfft.plan[i].g_size*sizeof(int));
dfft.plan[i].recv_size = (int *)realloc(dfft.plan[i].recv_size, 1*dfft.plan[i].g_size*sizeof(int));
dfft.plan[i].new_size = fft_calc_local_mesh(my_pos[i], n_grid[i], global_mesh_dim,
global_mesh_off, dfft.plan[i].new_mesh,
dfft.plan[i].start);
permute_ifield(dfft.plan[i].new_mesh,3,-(dfft.plan[i].n_permute));
permute_ifield(dfft.plan[i].start,3,-(dfft.plan[i].n_permute));
dfft.plan[i].n_ffts = dfft.plan[i].new_mesh[0]*dfft.plan[i].new_mesh[1];
/* === send/recv block specifications === */
for(j=0; j<dfft.plan[i].g_size; j++) {
int k, node;
/* send block: this_node to comm-group-node i (identity: node) */
node = dfft.plan[i].group[j];
dfft.plan[i].send_size[j]
= fft_calc_send_block(my_pos[i-1], n_grid[i-1], &(n_pos[i][3*node]), n_grid[i],
global_mesh_dim, global_mesh_off, &(dfft.plan[i].send_block[6*j]));
permute_ifield(&(dfft.plan[i].send_block[6*j]),3,-(dfft.plan[i-1].n_permute));
permute_ifield(&(dfft.plan[i].send_block[6*j+3]),3,-(dfft.plan[i-1].n_permute));
if(dfft.plan[i].send_size[j] > dfft.max_comm_size)
dfft.max_comm_size = dfft.plan[i].send_size[j];
/* First plan send blocks have to be adjusted, since the CA grid
may have an additional margin outside the actual domain of the
node */
if(i==1) {
for(k=0;k<3;k++)
dfft.plan[1].send_block[6*j+k ] += local_mesh_margin[2*k];
}
/* recv block: this_node from comm-group-node i (identity: node) */
dfft.plan[i].recv_size[j]
= fft_calc_send_block(my_pos[i], n_grid[i], &(n_pos[i-1][3*node]), n_grid[i-1],
global_mesh_dim, global_mesh_off,&(dfft.plan[i].recv_block[6*j]));
permute_ifield(&(dfft.plan[i].recv_block[6*j]),3,-(dfft.plan[i].n_permute));
permute_ifield(&(dfft.plan[i].recv_block[6*j+3]),3,-(dfft.plan[i].n_permute));
if(dfft.plan[i].recv_size[j] > dfft.max_comm_size)
dfft.max_comm_size = dfft.plan[i].recv_size[j];
}
for(j=0;j<3;j++) dfft.plan[i].old_mesh[j] = dfft.plan[i-1].new_mesh[j];
if(i==1)
dfft.plan[i].element = 1;
else {
dfft.plan[i].element = 2;
for(j=0; j<dfft.plan[i].g_size; j++) {
dfft.plan[i].send_size[j] *= 2;
dfft.plan[i].recv_size[j] *= 2;
}
}
/* DEBUG */
for(j=0;j<n_nodes;j++) {
/* MPI_Barrier(comm_cart); */
if(j==this_node) FFT_TRACE(fft_print_fft_plan(dfft.plan[i]));
}
}
/* Factor 2 for complex fields */
dfft.max_comm_size *= 2;
dfft.max_mesh_size = (local_mesh_dim[0]*local_mesh_dim[1]*local_mesh_dim[2]);
for(i=1;i<4;i++)
if(2*dfft.plan[i].new_size > dfft.max_mesh_size) dfft.max_mesh_size = 2*dfft.plan[i].new_size;
FFT_TRACE(fprintf(stderr,"%d: dfft.max_comm_size = %d, dfft.max_mesh_size = %d\n",
this_node,dfft.max_comm_size,dfft.max_mesh_size));
/* === pack function === */
for(i=1;i<4;i++) {
dfft.plan[i].pack_function = fft_pack_block_permute2;
FFT_TRACE(fprintf(stderr,"%d: forw plan[%d] permute 2 \n",this_node,i));
}
(*ks_pnum)=6;
if(dfft.plan[1].row_dir==2) {
dfft.plan[1].pack_function = fft_pack_block;
FFT_TRACE(fprintf(stderr,"%d: forw plan[%d] permute 0 \n",this_node,1));
(*ks_pnum)=4;
}
else if(dfft.plan[1].row_dir==1) {
dfft.plan[1].pack_function = fft_pack_block_permute1;
FFT_TRACE(fprintf(stderr,"%d: forw plan[%d] permute 1 \n",this_node,1));
(*ks_pnum)=5;
}
/* Factor 2 for complex numbers */
dfft.send_buf = (double *)realloc(dfft.send_buf, dfft.max_comm_size*sizeof(double));
dfft.recv_buf = (double *)realloc(dfft.recv_buf, dfft.max_comm_size*sizeof(double));
(*data) = (double *)realloc((*data), dfft.max_mesh_size*sizeof(double));
dfft.data_buf = (double *)realloc(dfft.data_buf, dfft.max_mesh_size*sizeof(double));
if(!(*data) || !dfft.data_buf || !dfft.recv_buf || !dfft.send_buf) {
fprintf(stderr,"%d: Could not allocate FFT data arays\n",this_node);
errexit();
}
fftw_complex *c_data = (fftw_complex *) (*data);
/* === FFT Routines (Using FFTW / RFFTW package)=== */
for(i=1;i<4;i++) {
dfft.plan[i].dir = FFTW_FORWARD;
/* FFT plan creation.
Attention: destroys contents of c_data/data and c_data_buf/data_buf. */
wisdom_status = FFTW_FAILURE;
sprintf(wisdom_file_name,"dfftw3_1d_wisdom_forw_n%d.file",
dfft.plan[i].new_mesh[2]);
if( (wisdom_file=fopen(wisdom_file_name,"r"))!=NULL ) {
wisdom_status = fftw_import_wisdom_from_file(wisdom_file);
fclose(wisdom_file);
}
if(dfft.init_tag==1) fftw_destroy_plan(dfft.plan[i].our_fftw_plan);
//printf("dfft.plan[%d].n_ffts=%d\n",i,dfft.plan[i].n_ffts);
dfft.plan[i].our_fftw_plan =
fftw_plan_many_dft(1,&dfft.plan[i].new_mesh[2],dfft.plan[i].n_ffts,
c_data,NULL,1,dfft.plan[i].new_mesh[2],
c_data,NULL,1,dfft.plan[i].new_mesh[2],
dfft.plan[i].dir,FFTW_PATIENT);
if( wisdom_status == FFTW_FAILURE &&
(wisdom_file=fopen(wisdom_file_name,"w"))!=NULL ) {
fftw_export_wisdom_to_file(wisdom_file);
fclose(wisdom_file);
}
dfft.plan[i].fft_function = fftw_execute;
}
/* === The BACK Direction === */
/* this is needed because slightly different functions are used */
for(i=1;i<4;i++) {
dfft.back[i].dir = FFTW_BACKWARD;
wisdom_status = FFTW_FAILURE;
sprintf(wisdom_file_name,"dfftw3_1d_wisdom_back_n%d.file",
dfft.plan[i].new_mesh[2]);
if( (wisdom_file=fopen(wisdom_file_name,"r"))!=NULL ) {
wisdom_status = fftw_import_wisdom_from_file(wisdom_file);
fclose(wisdom_file);
}
if(dfft.init_tag==1) fftw_destroy_plan(dfft.back[i].our_fftw_plan);
dfft.back[i].our_fftw_plan =
fftw_plan_many_dft(1,&dfft.plan[i].new_mesh[2],dfft.plan[i].n_ffts,
c_data,NULL,1,dfft.plan[i].new_mesh[2],
c_data,NULL,1,dfft.plan[i].new_mesh[2],
dfft.back[i].dir,FFTW_PATIENT);
if( wisdom_status == FFTW_FAILURE &&
(wisdom_file=fopen(wisdom_file_name,"w"))!=NULL ) {
fftw_export_wisdom_to_file(wisdom_file);
fclose(wisdom_file);
}
dfft.back[i].fft_function = fftw_execute;
dfft.back[i].pack_function = fft_pack_block_permute1;
FFT_TRACE(fprintf(stderr,"%d: back plan[%d] permute 1 \n",this_node,i));
}
if(dfft.plan[1].row_dir==2) {
dfft.back[1].pack_function = fft_pack_block;
FFT_TRACE(fprintf(stderr,"%d: back plan[%d] permute 0 \n",this_node,1));
}
else if(dfft.plan[1].row_dir==1) {
dfft.back[1].pack_function = fft_pack_block_permute2;
FFT_TRACE(fprintf(stderr,"%d: back plan[%d] permute 2 \n",this_node,1));
}
dfft.init_tag=1;
/* free(data); */
for(i=0;i<4;i++) { free(n_id[i]); free(n_pos[i]); }
return dfft.max_mesh_size;
}
/** Initialize boundary conditions for all constraints in the system. */
void lb_init_boundaries() {
int n, x, y, z;
//char *errtxt;
double pos[3], dist, dist_tmp=0.0, dist_vec[3];
if (lattice_switch & LATTICE_LB_GPU) {
#if defined (LB_GPU) && defined (LB_BOUNDARIES_GPU)
int number_of_boundnodes = 0;
int *host_boundary_node_list= (int*)Utils::malloc(sizeof(int));
int *host_boundary_index_list= (int*)Utils::malloc(sizeof(int));
size_t size_of_index;
int boundary_number = -1; // the number the boundary will actually belong to.
#ifdef EK_BOUNDARIES
ekfloat *host_wallcharge_species_density = NULL;
float node_wallcharge = 0.0f;
int wallcharge_species = -1, charged_boundaries = 0;
int node_charged = 0;
for(n = 0; n < int(n_lb_boundaries); n++)
lb_boundaries[n].net_charge = 0.0;
if (ek_initialized)
{
host_wallcharge_species_density = (ekfloat*) Utils::malloc(ek_parameters.number_of_nodes * sizeof(ekfloat));
for(n = 0; n < int(n_lb_boundaries); n++) {
if(lb_boundaries[n].charge_density != 0.0) {
charged_boundaries = 1;
break;
}
}
if (pdb_charge_lattice) {
charged_boundaries = 1;
}
for(n = 0; n < int(ek_parameters.number_of_species); n++)
if(ek_parameters.valency[n] != 0.0) {
wallcharge_species = n;
break;
}
if(wallcharge_species == -1 && charged_boundaries) {
runtimeErrorMsg() <<"no charged species available to create wall charge\n";
}
}
#endif
for(z=0; z<int(lbpar_gpu.dim_z); z++) {
for(y=0; y<int(lbpar_gpu.dim_y); y++) {
for (x=0; x<int(lbpar_gpu.dim_x); x++) {
pos[0] = (x+0.5)*lbpar_gpu.agrid;
pos[1] = (y+0.5)*lbpar_gpu.agrid;
pos[2] = (z+0.5)*lbpar_gpu.agrid;
dist = 1e99;
#ifdef EK_BOUNDARIES
if (ek_initialized)
{
host_wallcharge_species_density[ek_parameters.dim_y*ek_parameters.dim_x*z + ek_parameters.dim_x*y + x] = 0.0f;
node_charged = 0;
node_wallcharge = 0.0f;
}
#endif
for (n=0; n < n_lb_boundaries; n++) {
switch (lb_boundaries[n].type) {
case LB_BOUNDARY_WAL:
calculate_wall_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.wal, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_SPH:
calculate_sphere_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.sph, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_CYL:
calculate_cylinder_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.cyl, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_RHOMBOID:
calculate_rhomboid_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.rhomboid, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_POR:
calculate_pore_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.pore, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_STOMATOCYTE:
calculate_stomatocyte_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.stomatocyte, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_HOLLOW_CONE:
calculate_hollow_cone_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.hollow_cone, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_SPHEROCYLINDER:
calculate_spherocylinder_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.spherocyl, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_VOXEL: // voxel data do not need dist
//calculate_voxel_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.voxel, &dist_tmp, dist_vec);
dist_tmp=1e99;
break;
default:
runtimeErrorMsg() <<"lbboundary type "<< lb_boundaries[n].type << " not implemented in lb_init_boundaries()\n";
}
if (dist > dist_tmp || n == 0) {
dist = dist_tmp;
boundary_number = n;
}
#ifdef EK_BOUNDARIES
if (ek_initialized)
{
if(dist_tmp <= 0 && lb_boundaries[n].charge_density != 0.0f) {
node_charged = 1;
node_wallcharge += lb_boundaries[n].charge_density * ek_parameters.agrid*ek_parameters.agrid*ek_parameters.agrid;
lb_boundaries[n].net_charge += lb_boundaries[n].charge_density * ek_parameters.agrid*ek_parameters.agrid*ek_parameters.agrid;
}
}
#endif
}
#ifdef EK_BOUNDARIES
if(pdb_boundary_lattice &&
pdb_boundary_lattice[ek_parameters.dim_y*ek_parameters.dim_x*z + ek_parameters.dim_x*y + x]) {
dist = -1;
boundary_number = n_lb_boundaries; // Makes sure that boundary_number is not used by a constraint
}
#endif
if (dist <= 0 && boundary_number >= 0 && (n_lb_boundaries > 0 || pdb_boundary_lattice)) {
size_of_index = (number_of_boundnodes+1)*sizeof(int);
host_boundary_node_list = (int *) Utils::realloc(host_boundary_node_list, size_of_index);
host_boundary_index_list = (int *) Utils::realloc(host_boundary_index_list, size_of_index);
host_boundary_node_list[number_of_boundnodes] = x + lbpar_gpu.dim_x*y + lbpar_gpu.dim_x*lbpar_gpu.dim_y*z;
host_boundary_index_list[number_of_boundnodes] = boundary_number + 1;
number_of_boundnodes++;
//printf("boundindex %i: \n", number_of_boundnodes);
}
#ifdef EK_BOUNDARIES
if (ek_initialized)
{
ek_parameters.number_of_boundary_nodes = number_of_boundnodes;
if(wallcharge_species != -1) {
if(pdb_charge_lattice &&
pdb_charge_lattice[ek_parameters.dim_y*ek_parameters.dim_x*z + ek_parameters.dim_x*y + x] != 0.0f) {
node_charged = 1;
node_wallcharge += pdb_charge_lattice[ek_parameters.dim_y*ek_parameters.dim_x*z + ek_parameters.dim_x*y + x];
}
if(node_charged)
host_wallcharge_species_density[ek_parameters.dim_y*ek_parameters.dim_x*z + ek_parameters.dim_x*y + x] = node_wallcharge / ek_parameters.valency[wallcharge_species];
else if(dist <= 0)
host_wallcharge_species_density[ek_parameters.dim_y*ek_parameters.dim_x*z + ek_parameters.dim_x*y + x] = 0.0f;
else
host_wallcharge_species_density[ek_parameters.dim_y*ek_parameters.dim_x*z + ek_parameters.dim_x*y + x] = ek_parameters.density[wallcharge_species] * ek_parameters.agrid*ek_parameters.agrid*ek_parameters.agrid;
}
}
#endif
}
}
}
/**call of cuda fkt*/
float* boundary_velocity = (float *) Utils::malloc(3*(n_lb_boundaries+1)*sizeof(float));
for (n=0; n<n_lb_boundaries; n++) {
boundary_velocity[3*n+0]=lb_boundaries[n].velocity[0];
boundary_velocity[3*n+1]=lb_boundaries[n].velocity[1];
boundary_velocity[3*n+2]=lb_boundaries[n].velocity[2];
}
boundary_velocity[3*n_lb_boundaries+0] = 0.0f;
boundary_velocity[3*n_lb_boundaries+1] = 0.0f;
boundary_velocity[3*n_lb_boundaries+2] = 0.0f;
if (n_lb_boundaries || pdb_boundary_lattice)
lb_init_boundaries_GPU(n_lb_boundaries, number_of_boundnodes, host_boundary_node_list, host_boundary_index_list, boundary_velocity);
free(boundary_velocity);
free(host_boundary_node_list);
free(host_boundary_index_list);
#ifdef EK_BOUNDARIES
if (ek_initialized)
{
ek_init_species_density_wallcharge(host_wallcharge_species_density, wallcharge_species);
free(host_wallcharge_species_density);
}
#endif
#endif /* defined (LB_GPU) && defined (LB_BOUNDARIES_GPU) */
}
else {
#if defined (LB) && defined (LB_BOUNDARIES)
int node_domain_position[3], offset[3];
int the_boundary=-1;
map_node_array(this_node, node_domain_position);
offset[0] = node_domain_position[0]*lblattice.grid[0];
offset[1] = node_domain_position[1]*lblattice.grid[1];
offset[2] = node_domain_position[2]*lblattice.grid[2];
for (n=0;n<lblattice.halo_grid_volume;n++) {
lbfields[n].boundary = 0;
}
if (lblattice.halo_grid_volume==0)
return;
for (z=0; z<lblattice.grid[2]+2; z++) {
for (y=0; y<lblattice.grid[1]+2; y++) {
for (x=0; x<lblattice.grid[0]+2; x++) {
pos[0] = (offset[0]+(x-0.5))*lblattice.agrid[0];
pos[1] = (offset[1]+(y-0.5))*lblattice.agrid[1];
pos[2] = (offset[2]+(z-0.5))*lblattice.agrid[2];
dist = 1e99;
for (n=0;n<n_lb_boundaries;n++) {
switch (lb_boundaries[n].type) {
case LB_BOUNDARY_WAL:
calculate_wall_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.wal, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_SPH:
calculate_sphere_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.sph, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_CYL:
calculate_cylinder_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.cyl, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_RHOMBOID:
calculate_rhomboid_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.rhomboid, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_POR:
calculate_pore_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.pore, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_STOMATOCYTE:
calculate_stomatocyte_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.stomatocyte, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_HOLLOW_CONE:
calculate_hollow_cone_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.hollow_cone, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_VOXEL: // voxel data do not need dist
dist_tmp=1e99;
//calculate_voxel_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.voxel, &dist_tmp, dist_vec);
break;
default:
runtimeErrorMsg() <<"lbboundary type " << lb_boundaries[n].type << " not implemented in lb_init_boundaries()\n";
}
if (dist_tmp<dist || n == 0) {
dist = dist_tmp;
the_boundary = n;
}
}
if (dist <= 0 && the_boundary >= 0 && n_lb_boundaries > 0) {
lbfields[get_linear_index(x,y,z,lblattice.halo_grid)].boundary = the_boundary+1;
//printf("boundindex %i: \n", get_linear_index(x,y,z,lblattice.halo_grid));
}
else {
lbfields[get_linear_index(x,y,z,lblattice.halo_grid)].boundary = 0;
}
}
}
}
//printf("init voxels\n\n");
// SET VOXEL BOUNDARIES DIRECTLY
int xxx,yyy,zzz=0;
char line[80];
for (n=0;n<n_lb_boundaries;n++) {
switch (lb_boundaries[n].type) {
case LB_BOUNDARY_VOXEL:
//lbfields[get_linear_index(lb_boundaries[n].c.voxel.pos[0],lb_boundaries[n].c.voxel.pos[1],lb_boundaries[n].c.voxel.pos[2],lblattice.halo_grid)].boundary = n+1;
FILE *fp;
//fp=fopen("/home/mgusenbauer/Daten/Copy/DUK/GentlePump/Optimierer/NSvsLBM/geometry_files/bottleneck_fine_voxel_data_d20_converted_noMirror.csv", "r");
//fp=fopen("/home/mgusenbauer/Daten/Copy/DUK/GentlePump/Optimierer/NSvsLBM/geometry_files/bottleneck_fine_voxel_data_d80_converted_noMirror.csv", "r");
//fp=fopen("/home/mgusenbauer/Daten/Copy/DUK/GentlePump/Optimierer/NSvsLBM/geometry_files/bottleneck_fine_voxel_data_d80_converted.csv", "r");
fp=fopen(lb_boundaries[n].c.voxel.filename, "r");
while(fgets(line, 80, fp) != NULL)
{
/* get a line, up to 80 chars from fp, done if NULL */
sscanf (line, "%d %d %d", &xxx,&yyy,&zzz);
//printf("%d %d %d\n", xxx,yyy,zzz);
//lbfields[get_linear_index(xxx,yyy+30,zzz,lblattice.halo_grid)].boundary = n+1;
lbfields[get_linear_index(xxx,yyy,zzz,lblattice.halo_grid)].boundary = n+1;
}
fclose(fp);
break;
default:
break;
}
}
// CHECK FOR BOUNDARY NEIGHBOURS AND SET FLUID NORMAL VECTOR
//int neighbours = {0,0,0,0,0,0};
//int x=0,y=0,z=0;
//double nn[]={0.0,0.0,0.0,0.0,0.0,0.0};
//for (n=0;n<n_lb_boundaries;n++) {
//switch (lb_boundaries[n].type) {
//case LB_BOUNDARY_VOXEL:
//x=lb_boundaries[n].c.voxel.pos[0];
//y=lb_boundaries[n].c.voxel.pos[1];
//z=lb_boundaries[n].c.voxel.pos[2];
//if(((x-1) >= 0) && (lbfields[get_linear_index(x-1,y,z,lblattice.halo_grid)].boundary == 0)) nn[0] = -1.0;//neighbours[0] = -1;
//if(((x+1) <= lblattice.grid[0]) && (lbfields[get_linear_index(x+1,y,z,lblattice.halo_grid)].boundary == 0)) nn[1] = 1.0;//neighbours[1] = 1;
////printf("%.0lf %.0lf ",nn[0],nn[1]);
//lb_boundaries[n].c.voxel.n[0] = nn[0]+nn[1];
////nn=0.0;
//if(((y-1) >= 0) && (lbfields[get_linear_index(x,y-1,z,lblattice.halo_grid)].boundary == 0)) nn[2] = -1.0;//neighbours[2] = -1;
//if(((y+1) <= lblattice.grid[1]) && (lbfields[get_linear_index(x,y+1,z,lblattice.halo_grid)].boundary == 0)) nn[3] = 1.0;//neighbours[3] = 1;
////printf("%.0lf %.0lf ",nn[2],nn[3]);
//lb_boundaries[n].c.voxel.n[1] = nn[2]+nn[3];
////nn=0.0;
//if(((z-1) >= 0) && (lbfields[get_linear_index(x,y,z-1,lblattice.halo_grid)].boundary == 0)) nn[4] = -1.0;//neighbours[4] = -1;
//if(((z+1) <= lblattice.grid[2]) && (lbfields[get_linear_index(x,y,z+1,lblattice.halo_grid)].boundary == 0)) nn[5] = 1.0;//neighbours[5]= 1;
////printf("%.0lf %.0lf ",nn[4],nn[5]);
//lb_boundaries[n].c.voxel.n[2] = nn[4]+nn[5];
//nn[0]=0.0,nn[1]=0.0,nn[2]=0.0,nn[3]=0.0,nn[4]=0.0,nn[5]=0.0;
////printf("t %d pos: %.0lf %.0lf %.0lf, fluid normal %.0lf %.0lf %.0lf\n",n, x,y,z,lb_boundaries[n].c.voxel.normal[0],lb_boundaries[n].c.voxel.normal[1],lb_boundaries[n].c.voxel.normal[2]);
////printf("boundaries: %d %d %d %d %d %d\n",lbfields[get_linear_index(x-1,y,z,lblattice.halo_grid)].boundary,lbfields[get_linear_index(x+1,y,z,lblattice.halo_grid)].boundary,lbfields[get_linear_index(x,y-1,z,lblattice.halo_grid)].boundary,lbfields[get_linear_index(x,y+1,z,lblattice.halo_grid)].boundary,lbfields[get_linear_index(x,y,z-1,lblattice.halo_grid)].boundary,lbfields[get_linear_index(x,y,z+1,lblattice.halo_grid)].boundary);
//break;
//default:
//break;
//}
//}
//// DO THE SAME FOR THE CONSTRAINTS: CONSTRAINTS MUST BE SET AND THE SAME AS LB_BOUNDARY !!!
//for(n=0;n<n_constraints;n++) {
//switch(constraints[n].type) {
//case CONSTRAINT_VOXEL:
//x=constraints[n].c.voxel.pos[0];
//y=constraints[n].c.voxel.pos[1];
//z=constraints[n].c.voxel.pos[2];
//if(((x-1) >= 0) && (lbfields[get_linear_index(x-1,y,z,lblattice.halo_grid)].boundary == 0)) nn[0] = -1.0;//neighbours[0] = -1;
//if(((x+1) <= lblattice.grid[0]) && (lbfields[get_linear_index(x+1,y,z,lblattice.halo_grid)].boundary == 0)) nn[1] = 1.0;//neighbours[1] = 1;
////printf("%.0lf %.0lf ",nn[0],nn[1]);
//constraints[n].c.voxel.n[0] = nn[0]+nn[1];
////nn=0.0;
//if(((y-1) >= 0) && (lbfields[get_linear_index(x,y-1,z,lblattice.halo_grid)].boundary == 0)) nn[2] = -1.0;//neighbours[2] = -1;
//if(((y+1) <= lblattice.grid[1]) && (lbfields[get_linear_index(x,y+1,z,lblattice.halo_grid)].boundary == 0)) nn[3] = 1.0;//neighbours[3] = 1;
////printf("%.0lf %.0lf ",nn[2],nn[3]);
//constraints[n].c.voxel.n[1] = nn[2]+nn[3];
////nn=0.0;
//if(((z-1) >= 0) && (lbfields[get_linear_index(x,y,z-1,lblattice.halo_grid)].boundary == 0)) nn[4] = -1.0;//neighbours[4] = -1;
//if(((z+1) <= lblattice.grid[2]) && (lbfields[get_linear_index(x,y,z+1,lblattice.halo_grid)].boundary == 0)) nn[5] = 1.0;//neighbours[5]= 1;
////printf("%.0lf %.0lf ",nn[4],nn[5]);
//constraints[n].c.voxel.n[2] = nn[4]+nn[5];
//nn[0]=0.0,nn[1]=0.0,nn[2]=0.0,nn[3]=0.0,nn[4]=0.0,nn[5]=0.0;
//break;
//default:
//break;
//}
//}
//#ifdef VOXEL_BOUNDARIES
/*
for (z=0; z<lblattice.grid[2]+2; z++) {
for (y=0; y<lblattice.grid[1]+2; y++) {
for (x=0; x<lblattice.grid[0]+2; x++) {
lbfields[get_linear_index(x,y,z,lblattice.halo_grid)].boundary = 1;
}
}
}
static const char filename[] = "/home/mgusenbauer/Daten/Copy/DUK/GentlePump/Optimierer/voxels/stl/data_final.csv";
FILE *file = fopen ( filename, "r" );
int coords[3];
printf("start new\n");
if ( file != NULL ){
char line [ 128 ]; // or other suitable maximum line size
while ( fgets ( line, sizeof line, file ) != NULL ) {// read a line
//fputs ( line, stdout ); // write the line
//coords = line.Split(' ').Select(n => Convert.ToInt32(n)).ToArray();
//printf("readline: %s\n",line);
int i;
sscanf(line, "%d %d %d", &coords[0],&coords[1],&coords[2]);
//printf("%d %d %d\n", coords[0],coords[1],coords[2]);
lbfields[get_linear_index(coords[0]+5,coords[1]+5,coords[2]+5,lblattice.halo_grid)].boundary = 0;
}
fclose ( file );
}
printf("end new\n");
*/
#endif
}
}
/** Initialize boundary conditions for all constraints in the system. */
void lb_init_boundaries() {
int n, x, y, z;
char *errtxt;
double pos[3], dist, dist_tmp=0.0, dist_vec[3];
if (lattice_switch & LATTICE_LB_GPU) {
#if defined (LB_GPU) && defined (LB_BOUNDARIES_GPU)
int number_of_boundnodes = 0;
int *host_boundary_node_list= (int*)malloc(sizeof(int));
int *host_boundary_index_list= (int*)malloc(sizeof(int));
size_t size_of_index;
int boundary_number = -1; // the number the boundary will actually belong to.
for(z=0; z<lbpar_gpu.dim_z; z++) {
for(y=0; y<lbpar_gpu.dim_y; y++) {
for (x=0; x<lbpar_gpu.dim_x; x++) {
pos[0] = (x+0.5)*lbpar_gpu.agrid;
pos[1] = (y+0.5)*lbpar_gpu.agrid;
pos[2] = (z+0.5)*lbpar_gpu.agrid;
dist = 1e99;
for (n=0;n<n_lb_boundaries;n++) {
switch (lb_boundaries[n].type) {
case LB_BOUNDARY_WAL:
calculate_wall_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.wal, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_SPH:
calculate_sphere_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.sph, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_CYL:
calculate_cylinder_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.cyl, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_RHOMBOID:
calculate_rhomboid_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.rhomboid, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_POR:
calculate_pore_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.pore, &dist_tmp, dist_vec);
break;
default:
errtxt = runtime_error(128);
ERROR_SPRINTF(errtxt, "{109 lbboundary type %d not implemented in lb_init_boundaries()\n", lb_boundaries[n].type);
}
if (dist > dist_tmp || n == 0) {
dist = dist_tmp;
boundary_number = n;
}
}
if (dist <= 0 && boundary_number >= 0 && n_lb_boundaries > 0) {
size_of_index = (number_of_boundnodes+1)*sizeof(int);
host_boundary_node_list = realloc(host_boundary_node_list, size_of_index);
host_boundary_index_list = realloc(host_boundary_index_list, size_of_index);
host_boundary_node_list[number_of_boundnodes] = x + lbpar_gpu.dim_x*y + lbpar_gpu.dim_x*lbpar_gpu.dim_y*z;
host_boundary_index_list[number_of_boundnodes] = boundary_number + 1;
number_of_boundnodes++;
// printf("boundindex %i: \n", number_of_boundnodes);
}
}
}
}
/**call of cuda fkt*/
float* boundary_velocity = malloc(3*n_lb_boundaries*sizeof(float));
for (n=0; n<n_lb_boundaries; n++) {
boundary_velocity[3*n+0]=lb_boundaries[n].velocity[0];
boundary_velocity[3*n+1]=lb_boundaries[n].velocity[1];
boundary_velocity[3*n+2]=lb_boundaries[n].velocity[2];
}
if (n_lb_boundaries)
lb_init_boundaries_GPU(n_lb_boundaries, number_of_boundnodes, host_boundary_node_list, host_boundary_index_list, boundary_velocity);
free(boundary_velocity);
free(host_boundary_node_list);
free(host_boundary_index_list);
#endif
} else {
#if defined (LB) && defined (LB_BOUNDARIES)
int node_domain_position[3], offset[3];
int the_boundary=-1;
map_node_array(this_node, node_domain_position);
offset[0] = node_domain_position[0]*lblattice.grid[0];
offset[1] = node_domain_position[1]*lblattice.grid[1];
offset[2] = node_domain_position[2]*lblattice.grid[2];
for (n=0;n<lblattice.halo_grid_volume;n++) {
lbfields[n].boundary = 0;
}
if (lblattice.halo_grid_volume==0)
return;
for (z=0; z<lblattice.grid[2]+2; z++) {
for (y=0; y<lblattice.grid[1]+2; y++) {
for (x=0; x<lblattice.grid[0]+2; x++) {
pos[0] = (offset[0]+(x-0.5))*lblattice.agrid;
pos[1] = (offset[1]+(y-0.5))*lblattice.agrid;
pos[2] = (offset[2]+(z-0.5))*lblattice.agrid;
dist = 1e99;
for (n=0;n<n_lb_boundaries;n++) {
switch (lb_boundaries[n].type) {
case LB_BOUNDARY_WAL:
calculate_wall_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.wal, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_SPH:
calculate_sphere_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.sph, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_CYL:
calculate_cylinder_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.cyl, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_RHOMBOID:
calculate_rhomboid_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.rhomboid, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_POR:
calculate_pore_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.pore, &dist_tmp, dist_vec);
break;
default:
errtxt = runtime_error(128);
ERROR_SPRINTF(errtxt, "{109 lbboundary type %d not implemented in lb_init_boundaries()\n", lb_boundaries[n].type);
}
if (dist_tmp<dist || n == 0) {
dist = dist_tmp;
the_boundary = n;
}
}
if (dist <= 0 && the_boundary >= 0 && n_lb_boundaries > 0) {
lbfields[get_linear_index(x,y,z,lblattice.halo_grid)].boundary = the_boundary+1;
//printf("boundindex %i: \n", get_linear_index(x,y,z,lblattice.halo_grid));
}
else {
lbfields[get_linear_index(x,y,z,lblattice.halo_grid)].boundary = 0;
}
}
}
}
#endif
}
}
/** Initialize boundary conditions for all constraints in the system. */
void lb_init_boundaries() {
int n, x, y, z, node_domain_position[3], offset[3];
char *errtxt;
double pos[3], dist, dist_tmp=0.0, dist_vec[3];
int the_boundary=-1;
map_node_array(this_node, node_domain_position);
offset[0] = node_domain_position[0]*lblattice.grid[0];
offset[1] = node_domain_position[1]*lblattice.grid[1];
offset[2] = node_domain_position[2]*lblattice.grid[2];
for (n=0;n<lblattice.halo_grid_volume;n++) {
lbfields[n].boundary = 0;
}
if (lblattice.halo_grid_volume==0)
return;
for (z=0; z<lblattice.grid[2]+2; z++) {
for (y=0; y<lblattice.grid[1]+2; y++) {
for (x=0; x<lblattice.grid[0]+2; x++) {
pos[0] = (offset[0]+(x-1))*lblattice.agrid;
pos[1] = (offset[1]+(y-1))*lblattice.agrid;
pos[2] = (offset[2]+(z-1))*lblattice.agrid;
dist = 1e99;
for (n=0;n<n_lb_boundaries;n++) {
switch (lb_boundaries[n].type) {
case LB_BOUNDARY_WAL:
calculate_wall_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.wal, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_SPH:
calculate_sphere_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.sph, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_CYL:
calculate_cylinder_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.cyl, &dist_tmp, dist_vec);
break;
case LB_BOUNDARY_POR:
calculate_pore_dist((Particle*) NULL, pos, (Particle*) NULL, &lb_boundaries[n].c.pore, &dist_tmp, dist_vec);
break;
default:
errtxt = runtime_error(128);
ERROR_SPRINTF(errtxt, "{109 lbboundary type %d not implemented in lb_init_boundaries()\n", lb_boundaries[n].type);
}
// if (abs(dist) > abs(dist_tmp) || n == 0) {
if (dist_tmp<dist) { //If you try to create a wall of finite thickness ...|xxx|..., it makes every node a wall node! We still leave it like that, since it allows for corners without problems. We will add a box type to allow for walls of finite thickness. (Georg Rempfer, Stefan Kesselheim, 05.10.2011)
dist = dist_tmp;
the_boundary = n;
}
}
if (dist <= 0 && n_lb_boundaries > 0) {
lbfields[get_linear_index(x,y,z,lblattice.halo_grid)].boundary = the_boundary+1;
} else {
lbfields[get_linear_index(x,y,z,lblattice.halo_grid)].boundary=0;
}
}
}
}
}
int calc_node_neighbors(int node)
{
int dir, neighbor_count;
#ifdef LEES_EDWARDS
node_neighbors = (int *)realloc(node_neighbors, 6 * sizeof(int));
node_neighbor_lr = (int *)realloc(node_neighbor_lr, 6 * sizeof(int));
node_neighbor_wrap = (int *)realloc(node_neighbor_wrap, 6 * sizeof(int));
#endif
map_node_array(node,node_pos);
for(dir=0;dir<3;dir++) {
int buf;
/* Writes to node_neighbors[] the integer rank of that neighbor
* ... the 'buf' stores own rank, which is discarded. */
if( node_pos[dir] % 2 == 0 ){
MPI_Cart_shift(comm_cart, dir, -1, &buf, &(node_neighbors[2*dir]));
MPI_Cart_shift(comm_cart, dir, 1, &buf, &(node_neighbors[2*dir + 1]));
#ifdef LEES_EDWARDS
node_neighbor_lr[2*dir] = 0;
node_neighbor_lr[2*dir+1] = 1;
#endif
}else{
MPI_Cart_shift(comm_cart, dir, 1, &buf, &(node_neighbors[2*dir]));
MPI_Cart_shift(comm_cart, dir, -1, &buf, &(node_neighbors[2*dir + 1]));
#ifdef LEES_EDWARDS
node_neighbor_lr[2*dir] = 1;
node_neighbor_lr[2*dir+1] = 0;
#endif
}
/* left boundary ? */
if (node_pos[dir] == 0) {
boundary[2*dir] = 1;
}
else {
boundary[2*dir] = 0;
}
/* right boundary ? */
if (node_pos[dir] == node_grid[dir]-1) {
boundary[2*dir+1] = -1;
}
else {
boundary[2*dir+1] = 0;
}
}
neighbor_count = 6;
#ifdef LEES_EDWARDS
if( boundary[2] == 1 || boundary[3] == -1 ){
int x_index, abs_coords[3];
abs_coords[2] = node_pos[2]; /* z constant */
if( ( boundary[2] == 1 && boundary[3] != -1) || ( boundary[3] == -1 && boundary[2] != 1) ){
node_neighbors = (int *)realloc(node_neighbors, (neighbor_count + node_grid[0] - 1)*sizeof(int));
node_neighbor_lr = (int *)realloc(node_neighbor_lr, (neighbor_count + node_grid[0] - 1)*sizeof(int));
node_neighbor_wrap = (int *)realloc(node_neighbor_wrap, (neighbor_count + node_grid[0] - 1)*sizeof(int));
}else if( boundary[3] == -1 && boundary[2] == 1){
node_neighbors = (int *)realloc(node_neighbors, (neighbor_count + 2 * node_grid[0] - 2)*sizeof(int));
node_neighbor_lr = (int *)realloc(node_neighbor_lr, (neighbor_count + 2 * node_grid[0] - 2)*sizeof(int));
node_neighbor_wrap = (int *)realloc(node_neighbor_wrap, (neighbor_count + 2 * node_grid[0] - 2)*sizeof(int));
}
for( x_index = 0; x_index < node_grid[0]; x_index++){
if( x_index != node_pos[0] ){
abs_coords[0] = x_index;
if( x_index > node_pos[0] ){
if( boundary[2] == 1 ){
abs_coords[1] = node_grid[1] - 1; /* wraps to upper y */
node_neighbors[neighbor_count] = map_array_node(abs_coords);
node_neighbor_lr[neighbor_count] = 0;
neighbor_count++;
}
if( boundary[3] == -1 ){
abs_coords[1] = 0; /* wraps to lower y */
node_neighbors[neighbor_count] = map_array_node(abs_coords);
node_neighbor_lr[neighbor_count] = 1;
neighbor_count++;
}
}else{
if( boundary[3] == -1 ){
abs_coords[1] = 0; /* wraps to lower y */
node_neighbors[neighbor_count] = map_array_node(abs_coords);
node_neighbor_lr[neighbor_count] = 1;
neighbor_count++;
}
if( boundary[2] == 1 ){
abs_coords[1] = node_grid[1] - 1; /* wraps to upper y */
node_neighbors[neighbor_count] = map_array_node(abs_coords);
node_neighbor_lr[neighbor_count] = 0;
neighbor_count++;
}
}
}
}
}
#else
GRID_TRACE(printf("%d: node_grid %d %d %d, pos %d %d %d, node_neighbors ", this_node, node_grid[0], node_grid[1], node_grid[2], node_pos[0], node_pos[1], node_pos[2]));
#endif
#ifdef LEES_EDWARDS
my_neighbor_count = neighbor_count;/* set the global neighbor count */
for(neighbor_count = 0; neighbor_count < my_neighbor_count; neighbor_count++ ){
if( neighbor_count < 6 ) dir = neighbor_count / 2;
else dir = 1;
if( boundary[2*dir] == 1 && node_neighbor_lr[neighbor_count] == 0 ){
node_neighbor_wrap[neighbor_count] = 1;
}
else if(boundary[2*dir + 1] == -1 && node_neighbor_lr[neighbor_count] == 1 ){
node_neighbor_wrap[neighbor_count] = -1;
}else{
node_neighbor_wrap[neighbor_count] = 0;
}
}
GRID_TRACE(
for( dir=0; dir < my_neighbor_count; dir++){ fprintf(stderr, "%d: neighbour %d --> %d lr: %i wrap: %i\n", this_node, dir,node_neighbors[dir],node_neighbor_lr[dir],node_neighbor_wrap[dir]);}
);
void printAllComms(FILE *ofile, GhostCommunicator *comm, int backwards){
int i,j,cnt,ccnt, num;
int neighbor_index;
char ***comGrid;
int l1, l2;
l1 = dd.cell_grid[0]+3;
l2 = dd.cell_grid[1]+2;
comGrid = new char**[l1];
for( i = 0; i < l1;i++){
comGrid[i] = new char*[l2];
for( j = 0; j < l2;j++){
comGrid[i][j]=new char[8];
sprintf(&comGrid[i][j][0],".......");
comGrid[i][j][3]='\0';
comGrid[i][j][7]='\0';
}
}
/* prepare communicator... need 2 comms for node-node, 1 comm for intranode exchange */
num = my_neighbor_count * 2;
for(i = 0; i < my_neighbor_count; i++) {
if( node_neighbors[i] == this_node ) num--;
}
fprintf(ofile,"printing comms for node %i dimensions %i-%i-%i\n",this_node,dd.cell_grid[0],dd.cell_grid[1],dd.cell_grid[2]);
fprintf(ofile,"list of partners and sizes:\n");
for(ccnt = 0; ccnt < num; ccnt++){
if( backwards ) cnt = num - 1 - ccnt;
else cnt = ccnt;
fprintf(ofile," %i %i\n",comm->comm[cnt].node,comm->comm[cnt].n_part_lists);
}
fprintf(ofile,"\n");
/*loop over comms */
for(ccnt = 0; ccnt < num; ccnt++) {
int neighbor_coords[3];
if( backwards == LE_COMM_BACKWARDS ) cnt = num - 1 - ccnt;
else cnt = ccnt;
neighbor_index = comm->comm[cnt].node;
map_node_array(neighbor_index, neighbor_coords);
if( ( comm->comm[cnt].type == GHOST_SEND
|| comm->comm[cnt].type == GHOST_RECV )
&& comm->comm[cnt].node == this_node ){
fprintf(ofile,"serious comms error\n");
fprintf(stderr,"serious comms error\n");
exit(8);
}
/* clear the representation of this comm */
for( i = 0; i < l1;i++){
for( j = 0; j < l2;j++){
sprintf(&comGrid[i][j][0],".......");
comGrid[i][j][3]='\0';
comGrid[i][j][7]='\0';
}
}
if( comm->comm[cnt].type == GHOST_LOCL ){
for( i = 0; i < comm->comm[cnt].n_part_lists/2; i++){
if( comm->comm[cnt].part_lists[i]->myIndex[2] > 1
&& comm->comm[cnt].part_lists[i]->myIndex[2] < 10){
//fprintf(stderr,"%i: local send %i %i\n",this_node,comm->comm[cnt].part_lists[i]->myIndex[0],comm->comm[cnt].part_lists[i]->myIndex[1]);
sprintf( comGrid[comm->comm[cnt].part_lists[i]->myIndex[0]]
[comm->comm[cnt].part_lists[i]->myIndex[1]],
"%is%i",this_node,comm->comm[cnt].part_lists[i]->myIndex[2]);
}
}
for( i = comm->comm[cnt].n_part_lists/2; i < comm->comm[cnt].n_part_lists; i++){
if( comm->comm[cnt].part_lists[i]->myIndex[2] > 1
&& comm->comm[cnt].part_lists[i]->myIndex[2] < 10){
//fprintf(stderr,"%i: local rec %i %i\n",this_node,comm->comm[cnt].part_lists[i]->myIndex[0],comm->comm[cnt].part_lists[i]->myIndex[1]);
sprintf(&comGrid[comm->comm[cnt].part_lists[i]->myIndex[0]]
[comm->comm[cnt].part_lists[i]->myIndex[1]][4],
"%ir%i",this_node,comm->comm[cnt].part_lists[i]->myIndex[2]);
}
}
}else if (comm->comm[cnt].type == GHOST_SEND){
for( i = 0; i < comm->comm[cnt].n_part_lists; i++){
if( comm->comm[cnt].part_lists[i]->myIndex[2] > 1
&& comm->comm[cnt].part_lists[i]->myIndex[2] < 10){
//fprintf(stderr,"%i: ghost send %i %i\n",this_node,comm->comm[cnt].part_lists[i]->myIndex[0],comm->comm[cnt].part_lists[i]->myIndex[1]);
sprintf( comGrid[comm->comm[cnt].part_lists[i]->myIndex[0]]
[comm->comm[cnt].part_lists[i]->myIndex[1]],
"%iS%i",neighbor_index,comm->comm[cnt].part_lists[i]->myIndex[2]);
}
}
}else if (comm->comm[cnt].type == GHOST_RECV){
for( i = 0; i < comm->comm[cnt].n_part_lists; i++){
if( comm->comm[cnt].part_lists[i]->myIndex[2] > 1
&& comm->comm[cnt].part_lists[i]->myIndex[2] < 10){
//fprintf(stderr,"%i: ghost rec %i %i\n",this_node,comm->comm[cnt].part_lists[i]->myIndex[0],comm->comm[cnt].part_lists[i]->myIndex[1]);
sprintf(&comGrid[comm->comm[cnt].part_lists[i]->myIndex[0]]
[comm->comm[cnt].part_lists[i]->myIndex[1]][4],
"%iR%i",neighbor_index,comm->comm[cnt].part_lists[i]->myIndex[2]);
}
}
}
/* print a diagram of this comm, only if it is in a particular z-slice */
if( comm->comm[cnt].type == GHOST_LOCL
|| comm->comm[cnt].type == GHOST_SEND
|| comm->comm[cnt].type == GHOST_RECV ){
fprintf(ofile,"COMM TYPE: %i partner %i, size %i \n",
comm->comm[cnt].type,comm->comm[cnt].node,comm->comm[cnt].n_part_lists);
for( j = dd.cell_grid[1]+1; j >=0; j--){
for( i = 0; i < dd.cell_grid[0]+3;i++){
fprintf(ofile,"|%s%s",comGrid[i][j],&comGrid[i][j][4]);
}
fprintf(ofile,"|\n");
}
fprintf(ofile,"\n*****************\n");
if( comm->comm[cnt].type == GHOST_LOCL ){
fprintf(ofile,"COMM TYPE: %i\n",comm->comm[cnt].type);
for( j = dd.cell_grid[1]+1; j >=0; j--){
fprintf(ofile,".\n");
}
fprintf(ofile,"\n*****************\n");
}
}
}
for( i = 0; i < dd.cell_grid[0]+3;i++){
for( j = 0; j < dd.cell_grid[1]+2;j++){
delete [] comGrid[i][j];
}
delete [] comGrid[i];
}
delete [] comGrid;
}
