static
void hpccg_alloc_and_fill( const int np ,
const int my_p ,
const int gbox[][2] ,
const int ghost ,
struct cgsolve_data * const data )
{
int (*pbox)[3][2] = NULL ;
int * map_local_ord = NULL;
data->nRow = 0 ;
data->A_pc = NULL ;
data->A_ia = NULL ;
data->A_a = NULL ;
data->np = np ;
data->ip = my_p ;
data->recv_pc = NULL ;
data->send_pc = NULL ;
data->send_id = NULL ;
box_partition_rcb( np, my_p,
(const int (*)[2]) gbox, ghost,
& pbox ,
& map_local_ord ,
& data->recv_pc ,
& data->send_pc ,
& data->send_id );
{
const int (* const my_box)[2] = (const int (*)[2]) pbox[my_p] ;
const int bx = my_box[0][0] ;
const int by = my_box[1][0] ;
const int bz = my_box[2][0] ;
const int nx = my_box[0][1] - bx ;
const int ny = my_box[1][1] - by ;
const int nz = my_box[2][1] - bz ;
const int n = nx * ny * nz ;
const int nnz = 27 * n ; /* Upper bound */
int * const pc = (int *) malloc( sizeof(int) * ( n + 1 ) );
int * const ia = (int *) malloc( sizeof(int) * nnz );
MATRIX_SCALAR * const a = (MATRIX_SCALAR *) malloc( sizeof(MATRIX_SCALAR) * nnz );
int irow = 0 ;
int ipc = 0 ;
int ix , iy , iz ;
int sx , sy , sz ;
for ( iz = 0 ; iz < nz ; ++iz ) {
for ( iy = 0 ; iy < ny ; ++iy ) {
for ( ix = 0 ; ix < nx ; ++ix , ++irow ) {
if ( irow != box_map_local( my_box, ghost, map_local_ord,ix,iy,iz) ) {
fprintf(stderr,"P%d: irow[%d] != box_map_local(%d,%d,%d) = %d\n",
my_p,irow,ix,iy,iz,
box_map_local( my_box, ghost, map_local_ord, ix, iy, iz) );
}
pc[ irow ] = ipc ; /* Beginning of row coefficients */
/* Diagonal term first */
ia[ ipc ] = irow ;
a[ ipc ] = 27.0f ;
++ipc ;
/* Off-diagonal terms to follow */
for ( sz = -1 ; sz <= 1 ; ++sz ) {
for ( sy = -1 ; sy <= 1 ; ++sy ) {
for ( sx = -1 ; sx <= 1 ; ++sx ) {
const int dx = ix + sx ;
const int dy = iy + sy ;
const int dz = iz + sz ;
const int global_x = dx + bx ;
const int global_y = dy + by ;
const int global_z = dz + bz ;
if ( gbox[0][0] <= global_x && global_x < gbox[0][1] &&
gbox[1][0] <= global_y && global_y < gbox[1][1] &&
gbox[2][0] <= global_z && global_z < gbox[2][1] &&
! ( sz == 0 && sy == 0 && sx == 0 ) ) {
/* 'icol' is mapped for communication */
const int icol =
box_map_local(my_box,ghost,map_local_ord,dx,dy,dz);
if ( icol < 0 ) {
fprintf(stderr,"P%d : bad column at local (%d,%d,%d) global(%d,%d,%d)\n",
my_p, dx,dy,dz,global_x,global_y,global_z);
fflush(stderr);
abort();
}
ia[ ipc ] = icol ;
a[ ipc ] = -1.0f ;
++ipc ;
}
}
}
}
}
}
}
pc[irow] = ipc ;
data->nRow = irow ;
data->A_pc = pc ;
data->A_ia = ia ;
data->A_a = a ;
}
free( map_local_ord );
free( pbox );
}
static void test_maps( const int root_box[][2] , const int np )
{
const int ghost = 1 ;
const int nx_global = root_box[0][1] - root_box[0][0] ;
const int ny_global = root_box[1][1] - root_box[1][0] ;
int ieq , i , j ;
int (*pbox)[3][2] ;
int **local_values ;
int **map_local_id ;
int **map_recv_pc ;
int **map_send_pc ;
int **map_send_id ;
pbox = (int (*)[3][2]) malloc( sizeof(int) * np * 3 * 2 );
box_partition( 0 , np , 2 , root_box , pbox );
local_values = (int **) malloc( sizeof(int*) * np );
map_local_id = (int **) malloc( sizeof(int*) * np );
map_recv_pc = (int **) malloc( sizeof(int*) * np );
map_send_pc = (int **) malloc( sizeof(int*) * np );
map_send_id = (int **) malloc( sizeof(int*) * np );
/* Set each local value to the global equation number */
for ( ieq = i = 0 ; i < np ; ++i ) {
const int (*mybox)[2] = (const int (*)[2]) pbox[i] ;
const int nx = mybox[0][1] - mybox[0][0] ;
const int ny = mybox[1][1] - mybox[1][0] ;
const int nz = mybox[2][1] - mybox[2][0] ;
int ix , iy , iz ;
/* Generate the partition maps for this rank */
box_partition_maps( np , i , (const int (*)[3][2]) pbox , ghost ,
& map_local_id[i] , & map_recv_pc[i] ,
& map_send_pc[i] , & map_send_id[i] );
local_values[i] = (int *) malloc( sizeof(int) * map_recv_pc[i][np] );
for ( iz = -ghost ; iz < nz + ghost ; ++iz ) {
for ( iy = -ghost ; iy < ny + ghost ; ++iy ) {
for ( ix = -ghost ; ix < nx + ghost ; ++ix ) {
const int ieq = box_map_local(mybox,ghost,map_local_id[i],ix,iy,iz);
if ( 0 <= ieq ) {
const int ix_global = ix + mybox[0][0] ;
const int iy_global = iy + mybox[1][0] ;
const int iz_global = iz + mybox[2][0] ;
if ( root_box[0][0] <= ix_global && ix_global < root_box[0][1] &&
root_box[1][0] <= iy_global && iy_global < root_box[1][1] &&
root_box[2][0] <= iz_global && iz_global < root_box[2][1] ) {
local_values[i][ ieq ] = ix_global +
iy_global * nx_global +
iz_global * nx_global * ny_global ;
}
else {
local_values[i][ ieq ] = -1 ;
}
}
}
}
}
}
/* Pair-wise compare the local values */
/* i == receiving processor rank */
/* ip == sending processor rank */
/* j == receiving processor data entry for message from 'ip' */
/* jp == sending processor data entry for message to 'i' */
for ( i = 0 ; i < np ; ++i ) {
for ( j = 1 ; j < np ; ++j ) {
const int ip = ( i + j ) % np ;
const int jp = ( i + np - ip ) % np ;
const int nrecv = map_recv_pc[i] [j+1] - map_recv_pc[i] [j] ;
const int nsend = map_send_pc[ip][jp+1] - map_send_pc[ip][jp] ;
int k ;
if ( nrecv != nsend ) {
fprintf(stderr,"P%d recv %d from P%d\n",i,nrecv,ip);
fprintf(stderr,"P%d send %d to P%d\n",ip,nsend,i);
abort();
}
for ( k = 0 ; k < nrecv ; ++k ) {
const int irecv = map_recv_pc[i][j] + k ;
const int isend = map_send_pc[ip][jp] + k ;
const int val_irecv = local_values[i][irecv] ;
const int val_isend = local_values[ip][ map_send_id[ip][isend] ] ;
if ( val_irecv != val_isend ) {
fprintf(stderr,"P%d recv[%d] = %d , from P%d\n",i,k,val_irecv,ip);
fprintf(stderr,"P%d send[%d] = %d , to P%d\n",ip,k,val_isend,i);
abort();
}
}
}
}
for ( i = 0 ; i < np ; ++i ) {
free( map_local_id[i] );
free( map_recv_pc[i] );
free( map_send_pc[i] );
free( map_send_id[i] );
free( local_values[i] );
}
free( map_send_id );
free( map_send_pc );
free( map_recv_pc );
free( map_local_id );
free( local_values );
free( pbox );
}
