float get_convergence_sqd (float ** current_ptr,float ** next_ptr,int rank) {
int i,j,my_start,my_end,my_num_rows;
float sum;
my_start = get_start(rank);
my_end = get_end(rank);
my_num_rows = get_num_rows(rank);
sum = 0.0;
for (j=my_start;j<=my_end;j++) {
for (i=0;i<(int)floor(WIDTH/H);i++) {
sum += pow(next_ptr[global_to_local(rank,j)][i]-current_ptr[global_to_local(rank,j)][i],2);
}
}
return sum;
}
void Ioss::Map::reverse_map_data(void *data, const Ioss::Field &field, size_t count) const
{
assert(!map.empty());
if (!is_sequential(map)) {
if (field.get_type() == Ioss::Field::INTEGER) {
int* connect = static_cast<int*>(data);
for (size_t i=0; i < count; i++) {
int global_id = connect[i];
connect[i] = global_to_local(global_id, true);
}
} else {
int64_t* connect = static_cast<int64_t*>(data);
for (size_t i=0; i < count; i++) {
int64_t global_id = connect[i];
connect[i] = global_to_local(global_id, true);
}
}
}
}
void enforce_bc_par (float ** domain_ptr,int rank,int i,int j) {
/* enforce bc's first */
if(i == ((int)floor(WIDTH/H/2)-1) && j == 0) {
/* This is the heat source location */
domain_ptr[j][i] = T_SRC0;
} else if (i <= 0 || j <= 0 || i >= ((int)floor(WIDTH/H)-1) || j >= ((int)floor(HEIGHT/H)-1)) {
/* All edges and beyond are set to 0.0 */
domain_ptr[global_to_local(rank,j)][i] = 0.0;
}
}
float get_val_par (float * above_ptr,float ** domain_ptr,float * below_ptr,int rank,int i,int j) {
float ret_val;
int p;
MPI_Comm_size(MPI_COMM_WORLD,&p);
/* enforce bc's first */
if(i == ((int)floor(WIDTH/H/2)-1) && j == 0) {
/* This is the heat source location */
ret_val = T_SRC0;
} else if (i <= 0 || j <= 0 || i >= ((int)floor(WIDTH/H)-1) || j >= ((int)floor(HEIGHT/H)-1)) {
/* All edges and beyond are set to 0.0 */
ret_val = 0.0;
} else {
/* Else, return value for matrix supplied or ghost rows */
if (j < get_start(rank)) {
if (rank == ROOT) {
/* not interested in above ghost row */
ret_val = 0.0;
} else {
ret_val = above_ptr[i];
/*printf("%d: Used ghost (%d,%d) row from above = %f\n",rank,i,j,above_ptr[i]);
fflush(stdout);*/
}
} else if (j > get_end(rank)) {
if (rank == (p-1)) {
/* not interested in below ghost row */
ret_val = 0.0;
} else {
ret_val = below_ptr[i];
/*printf("%d: Used ghost (%d,%d) row from below = %f\n",rank,i,j,below_ptr[i]);
fflush(stdout);*/
}
} else {
/* else, return the value in the domain asked for */
ret_val = domain_ptr[global_to_local(rank,j)][i];
/*printf("%d: Used real (%d,%d) row from self = %f\n",rank,i,global_to_local(rank,j),domain_ptr[global_to_local(rank,j)][i]);
fflush(stdout);*/
}
}
return ret_val;
}
void create_wand_monst()
{
short r1,r2,r3,i = 0,num_tries = 0;
location p_loc;
r1 = get_ran(1,0,3);
if (overall_mode == MODE_OUTDOORS)
if (!univ.out.outdoors[univ.party.i_w_c.x][univ.party.i_w_c.y].wandering[r1].isNull()) {
r2 = get_ran(1,0,3);
while ((point_onscreen(univ.out.outdoors[univ.party.i_w_c.x][univ.party.i_w_c.y].wandering_locs[r2],
global_to_local(univ.party.p_loc))) && (num_tries++ < 100))
r2 = get_ran(1,0,3);
if (!is_blocked(univ.out.outdoors[univ.party.i_w_c.x][univ.party.i_w_c.y].wandering_locs[r2]))
place_outd_wand_monst(univ.out.outdoors[univ.party.i_w_c.x][univ.party.i_w_c.y].wandering_locs[r2],
univ.out.outdoors[univ.party.i_w_c.x][univ.party.i_w_c.y].wandering[r1],0);
}
else // won't place wandering is more than 50 monsters
if ((!univ.town->wandering[r1].isNull()) && (univ.town.countMonsters() <= 50)
&& (univ.party.m_killed[univ.town.num] < univ.town->max_num_monst)) {
r2 = get_ran(1,0,3);
while (point_onscreen(univ.town->wandering_locs[r2],univ.town.p_loc) &&
!loc_off_act_area(univ.town->wandering_locs[r2]) && (num_tries++ < 100))
r2 = get_ran(1,0,3);
for (i = 0; i < 4; i++) {
if (univ.town->wandering[r1].monst[i] != 0) { // place a monster
p_loc = univ.town->wandering_locs[r2];
p_loc.x += get_ran(1,0,4) - 2;
p_loc.y += get_ran(1,0,4) - 2;
if (!is_blocked(p_loc))
place_monster(univ.town->wandering[r1].monst[i],p_loc);
}
p_loc = univ.town->wandering_locs[r2];
p_loc.x += get_ran(1,0,4) - 2;
p_loc.y += get_ran(1,0,4) - 2;
r3 = get_ran(1,0,3);
if ((r3 >= 2) && (!is_blocked(p_loc))) // place extra monsters?
place_monster(univ.town->wandering[r1].monst[3],p_loc);
}
}
}
int main(int argc, char ** argv)
{
/* ----------- VARIABLE DECLARATIONS ----------- */
/* Size of matrix - the matrix will be n x n */
int n;
/* Number of processes and rank of this process*/
int nprocs, rank;
/* Size and periodicity of cartesian grid */
int dim_size[2];
int periods[2];
/* Number of processes in x and y directions */
int npx, npy;
/* Offset for this process in x and y directions */
int x_offset, y_offset;
/* The standard (normal) number of rows and cols
per core - not valid for the last in a row or col */
int std_rows_in_core, std_cols_in_core;
/* Coordinates in cartesian grid, and num of rows and cols
this process has */
int coords[2];
int rows_in_core;
int cols_in_core;
/* Array to store the chunk of the matrix that we have */
double **array;
double **new_array;
/* Loop variables */
int i, j;
/* Cartesian communicator */
MPI_Comm cart_comm;
int coords2[2];
int rank2;
int global_x1, global_x2;
int global_y1, global_y2;
struct global_index gi;
struct local_index li;
MPI_Request send_request;
MPI_Request recv_request;
/* Stores the number of requests this core will need to keep track of
and creates a pointer to eventually hold the array of these requests */
int n_requests;
MPI_Request *all_requests;
int request_counter;
int tag;
/* Determines whether the final result should be printed or not */
int print = 0;
/* Holds start and end times for the transpose */
double tstart, tend, walltime;
/* ----------- PROCESS CMD LINE ARGUMENTS ----------- */
/* If there isn't one command-line argument (argc also counts the program name)
then print an error message and exit.
Only do this if we're on process 0 - otherwise nprocs error messages will be printed!*/
if (argc < 2 && rank == 0)
{
printf("You must specify the size of the matrix (n) as a command-line argument\n");
printf("Exiting\n");
MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
return EXIT_FAILURE;
}
/* As we've got here, there must be at least one command-line argument, so try and convert it to an
integer */
n = atoi(argv[1]);
/* Process other command line arguments */
/* Process command line arguments checking them and running the appropriate function */
for (i = 2; i < argc; i++) /* Skip argv[0] (program name) and argv[1] (size of matrix)*/
{
/* if strcmp returns 0 then the strings are identical */
if (strcmp(argv[i], "print") == 0)
{
print = 1;
}
}
/* If atoi returns zero it is either because it couldn't convert the string to an integer,
or the integer given was 0. In either case, we must print an error message because it is a
nonsensical size for the matrix */
if (n == 0)
{
printf("You must an integer > 0 as the size of the matrix\n");
printf("Exiting\n");
MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
return EXIT_FAILURE;
}
/* ----------- MPI Setup ----------- */
/* Initialise MPI */
MPI_Init(&argc, &argv);
/* Get the rank for this process, and the number of processes */
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
/* Work out how big the grid should be, given the number of processes */
factorise( nprocs, &npx, &npy );
/* Create the cartesian communicator */
dim_size[0] = npy;
dim_size[1] = npx;
periods[0] = 1;
periods[1] = 1;
MPI_Cart_create(MPI_COMM_WORLD, 2, dim_size, periods, 1, &cart_comm);
/* Get our co-ordinates within that communicator */
MPI_Cart_coords(cart_comm, rank, 2, coords);
rows_in_core = ceil(n / (float) npx);
cols_in_core = ceil(n / (float) npy);
std_rows_in_core = rows_in_core;
std_cols_in_core = cols_in_core;
if (coords[0] == (npy - 1))
{
/* We're at the far end of a row */
cols_in_core = n - (cols_in_core * (npy - 1));
}
if (coords[1] == (npx - 1))
{
/* We're at the bottom of a col */
rows_in_core = n - (rows_in_core * (npx - 1));
}
/* Calculate the number of individual communications that are needed to transpose the matrix.
We know that this will be the number of requests we need to store, so malloc some memory for the
array of the right size. Each process has its own requests array - so it's simply 2 * ncols * nrows
*/
n_requests = 2 * rows_in_core * cols_in_core;
all_requests = malloc(n_requests * sizeof(MPI_Request));
/* ----------- INITIALISE MATRIX CHUNKS ----------- */
/* Allocate an array to store this chunk of the matrix.
If the allocation fails, print an error */
array = alloc_2d_double(rows_in_core, cols_in_core);
new_array = alloc_2d_double(rows_in_core, cols_in_core);
if (array == NULL || new_array == NULL)
{
printf("Problem with array allocation.\nExiting\n");
return 1;
}
for (i = 0; i < rows_in_core; i++)
{
for (j = 0; j < cols_in_core; j++)
{
new_array[i][j] = NAN;
}
}
/* Calculate the offset of the top left-hand corner of our chunk of the matrix from the
top left-hand corner of the whole matrix */
x_offset = coords[0] * std_cols_in_core;
y_offset = coords[1] * std_rows_in_core;
for (i = 0; i < rows_in_core; i++)
{
/*printf("Process (%d, %d): ", coords[0], coords[1]);*/
for (j = 0; j < cols_in_core; j++)
{
array[i][j] = (float) ( (i + y_offset) * n) + ( (j + x_offset) + 1);
/*printf("%f ", array[i][j]);*/
}
/*printf("\n");*/
}
MPI_Barrier(MPI_COMM_WORLD);
/* ----------- DO TRANSPOSE ----------- */
/* Record the start time, making sure all processes have got there at the same time using
a barrier */
MPI_Barrier(cart_comm);
tstart = timer();
request_counter = 0;
for (i = 0; i < rows_in_core; i++)
{
for (j = 0; j < cols_in_core; j++)
{
/* For each item in this chunk of the array:
Send it to the opposite index
Receive from the opposite index */
/* Calculate the opposite index */
gi = local_to_global(coords[0], coords[1], j, i, std_cols_in_core, std_rows_in_core);
/* The two global indices for swapping */
global_x1 = gi.x;
global_y1 = gi.y;
global_x2 = global_y1;
global_y2 = global_x1;
/* We know which rank one of them is (as we're running as it!)
but we need to calculate the rank of the other which involves:
1. Getting the local index of the swapped global index
2. Converting the X, Y process co-ordinates to a rank */
li = global_to_local(global_x2, global_y2, std_cols_in_core, std_rows_in_core);
coords2[0] = li.procX;
coords2[1] = li.procY;
MPI_Cart_rank(cart_comm, coords2, &rank2);
tag = (global_x2 + 1) * (global_y2 + 1);
/*printf("Swapping\n");
printf("Value = %f\n", array[i][j]);
printf("(%f) From global (%d, %d) - Processor (%d, %d) with local (%d, %d)\n", array[i][j], global_x1, global_y1, coords[0], coords[1], j, i);
printf("(%f) To global (%d, %d)- Processor (%d, %d) with local (%d, %d)\n", array[i][j], global_x2, global_y2, coords2[0], coords2[1], li.arrayX, li.arrayY);
printf("(%f) Going to rank %d\n", array[i][j], rank2); */
MPI_Isend(&array[i][j], 1, MPI_DOUBLE, rank2, (global_y2 * n) + global_x2, MPI_COMM_WORLD, &send_request);
MPI_Irecv(&new_array[i][j], 1, MPI_DOUBLE, rank2, (global_x2 * n) + global_y2, MPI_COMM_WORLD, &recv_request);
all_requests[request_counter] = send_request;
request_counter++;
all_requests[request_counter] = recv_request;
request_counter++;
}
}
MPI_Waitall(request_counter, all_requests, MPI_STATUSES_IGNORE);
/* Store end time (using barrier before-hand to make the calculated times for all of the
processes almost exactly equal, meaning we can just take the time from one process
for output */
MPI_Barrier(cart_comm);
tend = timer();
/* ----------- PRINT OUTPUT AND FINALISE ENVIRONMENT ----------- */
walltime = tend - tstart;
if (print == 1)
{
for (i = 0; i < rows_in_core; i++)
{
printf("Result at Process (%d, %d): ", coords[0], coords[1]);
for (j = 0; j < cols_in_core; j++)
{
printf("%f ", new_array[i][j]);
}
printf("\n");
}
}
if (rank == 0)
{
printf("%d\t%d\t%f\n", nprocs, n, walltime);
}
/* Free the memory we grabbed for the requests array */
free(all_requests);
/* Free the memory we grabbed earlier for the data arrays */
free_2d_double(array);
free_2d_double(new_array);
/* Close down the MPI environment */
MPI_Finalize();
return EXIT_SUCCESS;
}
void Ioss::Map::build_reorder_map(int64_t start, int64_t count)
{
// This routine builds a map that relates the current node id order
// to the original node ordering in affect at the time the file was
// created. That is, the node map used to define the topology of the
// model. Now, if there are changes in node ordering at the
// application level, we build the node reorder map to map the
// current order into the original order. An added complication is
// that this is more than just a reordering... It may be that the
// application has 'ghosted' nodes that it doesnt want put out on
// the database, so the reorder map must handle a node that is not
// in the original mesh and map that to an invalid value (currently
// using -1 as invalid value...)
// Note: To further add confusion, the reorder map is 0-based
// and the reverse map and 'map' are 1-baed. This is
// just a consequence of how they are intended to be used...
//
// start is based on a 0-based array -- start of the reorderMap to build.
if (reorder.empty()) {
// See if actually need a reorder map first...
bool need_reorder_map = false;
int64_t my_end = start+count;
for (int64_t i=start; i < my_end; i++) {
int64_t global_id = map[i+1];
int64_t orig_local_id = global_to_local(global_id) - 1;
// The reordering should only be a permutation of the original
// ordering within this entity block...
assert(orig_local_id >= start && orig_local_id <= my_end);
if (i != orig_local_id) {
need_reorder_map = true;
break;
}
}
if (need_reorder_map) {
int64_t map_size = map.size()-1;
reorder.resize(map_size);
// If building a partial reorder map, assume all entries
// are a direct 1-1 and then let the partial fills overwrite
// if needed.
if (start > 0 || my_end < map_size) {
for (size_t i=0; i < reorder.size(); i++) {
reorder[i] = i;
}
}
} else {
return;
}
}
int64_t my_end = start+count;
for (int64_t i=start; i < my_end; i++) {
int64_t global_id = map[i+1];
int64_t orig_local_id = global_to_local(global_id) - 1;
// The reordering should only be a permutation of the original
// ordering within this entity block...
assert(orig_local_id >= start && orig_local_id <= my_end);
reorder[i] = orig_local_id;
}
}
