int main(int argc, char** argv) {
init_mpi(&argc, &argv);
// disable log from caffe (disable glog entirely)
FLAGS_minloglevel = 3;
// Run tool or show usage.
caffe::GlobalInit(&argc, &argv);
int iters = FLAGS_iterations;
string netdefs = FLAGS_model;
Dtype lr = Dtype(FLAGS_lr);
// at least iters in total
// actuall number of workers is np - 1
iters = (iters + np - 2) / (np - 1);
if (pid == 0)
iters *= (np - 1);
auto net = init_net(netdefs);
std::vector<Blob<Dtype>*> bottom_vec;
// sync parameters
broadcast_params(net);
if (pid == 0)
root_node(net, iters, lr);
else
worker_node(net, iters);
return finish_mpi();
}
int main(int argc, char **argv)
{
int ts = 0, var = 0;
init_mpi(argc, argv);
parse_args(argc, argv);
check_args();
calculate_per_process_offsets();
create_synthetic_simulation_data();
rank_0_print("Simulation Data Created\n");
create_pidx_var_point_and_access();
for (ts = 0; ts < time_step_count; ts++)
{
set_pidx_file(ts);
for (var = 0; var < variable_count; var++)
set_pidx_variable(var);
PIDX_close(file);
}
destroy_pidx_var_point_and_access();
destroy_synthetic_simulation_data();
shutdown_mpi();
return 0;
}
int main(int argc, char **argv)
{
const int rank = init_mpi(&argc, &argv);
/* parse args (unsafe) */
xensure(argc > 3);
int i = atoi(argv[1]);
int j = atoi(argv[2]);
size_t m = atoi(argv[3]);
unsigned char *buf = calloc((m ? m : 1) * 2, 1);
xensure(buf);
unsigned char *sendbuf = buf;
unsigned char *recvbuf = buf + (m ? m : 1);
init_data(sendbuf, m, rank);
if (rank == i) {
double tick = MPI_Wtick();
printf("tick /s = %.17g\n", tick);
double min_time = tick * TICK_FACTOR;
printf("min_time /s = %.17g\n", min_time);
bench_all(1, sendbuf, recvbuf, m, j, min_time);
} else if (rank == j) {
bench_all(0, sendbuf, recvbuf, m, i, NAN);
}
xtry(MPI_Finalize());
free(buf);
}
//Initalize
static void init(void)
{
init_mpi();
allocateMemories();
setCoefficient();
initDebug();
}
int main (int argc, char *argv[])
{
struct pe_vars v;
long * msg_buffer;
/*
* Initialize
*/
init_mpi(&v);
check_usage(argc, argv, v.npes, v.me);
print_header(v.me);
if (v.me == 0) printf("Total processes = %d\n",v.npes);
/*
* Allocate Memory
*/
msg_buffer = allocate_memory(v.me, &(v.win) );
memset(msg_buffer, 0, MAX_MSG_SZ * ITERS_LARGE * sizeof(long));
/*
* Time Put Message Rate
*/
benchmark(msg_buffer, v.me, v.pairs, v.nxtpe, v.win);
/*
* Finalize
*/
MPI_Win_unlock_all(v.win);
MPI_Win_free(&v.win);
MPI_Free_mem(msg_buffer);
MPI_Finalize();
return EXIT_SUCCESS;
}
int main(int argc, char **argv)
{
init_mpi(&argc, &argv);
if (mpi.rank == 0) {
init_gwclock();
}
// xtry(MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN));
double t = get_gwclock();
cpp_main(atoi(argv[1]));
printf0("time_all=%.17g\n", get_gwclock() - t);
MPI_Finalize();
return 0;
}
/// main
int main(int argc, char **argv)
{
init_mpi(argc, argv);
parse_args(argc, argv);
check_args();
calculate_per_process_offsets();
create_synthetic_simulation_data();
int var, p;
PIDX_point global_bounding_box, **local_offset_point, **local_box_count_point;
local_offset_point = malloc(sizeof(PIDX_point*) * variable_count);
local_box_count_point = malloc(sizeof(PIDX_point*) * variable_count);
for(var = 0; var < variable_count; var++)
{
local_offset_point[var] = malloc(sizeof(PIDX_point) * patch_count);
local_box_count_point[var] = malloc(sizeof(PIDX_point) * patch_count);
for(p = 0 ; p < patch_count ; p++)
{
PIDX_set_point_5D(local_offset_point[var][p], (int64_t)var_offset[var][p][0], (int64_t)var_offset[var][p][1], (int64_t)var_offset[var][p][2], 0, 0);
PIDX_set_point_5D(local_box_count_point[var][p], (int64_t)var_count[var][p][0], (int64_t)var_count[var][p][1], (int64_t)var_count[var][p][2], 1, 1);
}
}
PIDX_file file; // IDX file descriptor
PIDX_variable* variable; // variable descriptor
variable = malloc(sizeof(*variable) * variable_count);
memset(variable, 0, sizeof(*variable) * variable_count);
PIDX_set_point_5D(global_bounding_box, (int64_t)global_box_size[0], (int64_t)global_box_size[1], (int64_t)global_box_size[2], 1, 1);
PIDX_access access;
PIDX_create_access(&access);
#if PIDX_HAVE_MPI
PIDX_set_mpi_access(access, MPI_COMM_WORLD);
#endif
int ts;
for (ts = 0; ts < time_step_count; ts++)
{
PIDX_file_create(output_file_name, PIDX_MODE_CREATE, access, &file);
PIDX_set_dims(file, global_bounding_box);
PIDX_set_current_time_step(file, ts);
PIDX_set_variable_count(file, variable_count);
PIDX_debug_rst(file, 1);
PIDX_debug_hz(file, 1);
for (var = 0; var < variable_count; var++)
{
char variable_name[512];
sprintf(variable_name, "variable_%d", var);
PIDX_variable_create(variable_name, sizeof(double) * 8, FLOAT64, &variable[var]);
for (p = 0 ; p < patch_count ; p++)
PIDX_variable_write_data_layout(variable[var], local_offset_point[var][p], local_box_count_point[var][p], double_data[var][p], PIDX_row_major);
PIDX_append_and_write_variable(file, variable[var]/*, local_offset_point[var][p], local_box_count_point[var][p], double_data[var][p], PIDX_row_major*/);
}
PIDX_close(file);
}
PIDX_close_access(access);
destroy_synthetic_simulation_data();
for(var = 0; var < variable_count; var++)
{
free(local_offset_point[var]);
free(local_box_count_point[var]);
}
free(local_offset_point);
free(local_box_count_point);
free(variable);
variable = 0;
shutdown_mpi();
return 0;
}
int main(int argc, char** argv)
{
initialize_global_variables();
int ret = init_mpi(&argc, &argv);
if (ret != POTFIT_SUCCESS) {
shutdown_mpi();
return EXIT_FAILURE;
}
read_input_files(argc, argv);
g_mpi.init_done = 1;
ret = broadcast_params_mpi();
switch (ret) {
case POTFIT_ERROR_MPI_CLEAN_EXIT:
shutdown_mpi();
return EXIT_SUCCESS;
case POTFIT_ERROR:
shutdown_mpi();
return EXIT_FAILURE;
}
g_calc.ndim = g_pot.opt_pot.idxlen;
g_calc.ndimtot = g_pot.opt_pot.len;
// main force vector, all forces, energies, stresses, etc. will be stored here
g_calc.force = (double*)Malloc(g_calc.mdim * sizeof(double));
// starting positions for the force vector
set_force_vector_pointers();
#if defined(APOT)
#if defined(MPI)
MPI_Bcast(g_pot.opt_pot.table, g_calc.ndimtot, MPI_DOUBLE, 0, MPI_COMM_WORLD);
#endif // MPI
update_calc_table(g_pot.opt_pot.table, g_pot.calc_pot.table, 1);
#endif // APOT
if (g_mpi.myid > 0) {
start_mpi_worker(g_calc.force);
} else {
#if defined(MPI)
if (g_mpi.num_cpus > g_config.nconf) {
warning("You are using more CPUs than you have configurations!\n");
warning("While this will not do any harm, you are wasting %d CPUs.\n",
g_mpi.num_cpus - g_config.nconf);
}
#endif // MPI
time_t start_time;
time_t end_time;
time(&start_time);
if (g_param.opt && g_calc.ndim > 0) {
run_optimization();
} else if (g_calc.ndim == 0) {
printf(
"\nOptimization disabled due to 0 free parameters. Calculating "
"errors.\n");
} else {
printf("\nOptimization disabled. Calculating errors.\n\n");
}
time(&end_time);
#if defined(APOT)
double tot = calc_forces(g_pot.opt_pot.table, g_calc.force, 0);
#else
double tot = calc_forces(g_pot.calc_pot.table, g_calc.force, 0);
#endif // APOT
#if defined(UQ)
return uncertainty_quantification(tot,g_files.sloppyfile);
#endif //UQ
write_pot_table_potfit(g_files.endpot);
return 0;
{
int format = -1;
switch (g_pot.format_type) {
case POTENTIAL_FORMAT_UNKNOWN:
error(1, "Unknown potential format detected! (%s:%d)\n", __FILE__,
__LINE__);
case POTENTIAL_FORMAT_ANALYTIC:
format = 0;
break;
case POTENTIAL_FORMAT_TABULATED_EQ_DIST:
format = 3;
break;
case POTENTIAL_FORMAT_TABULATED_NON_EQ_DIST:
format = 4;
break;
}
printf("\nPotential in format %d written to file \t%s\n", format,
g_files.endpot);
}
if (g_param.writeimd == 1)
write_pot_table_imd(g_files.imdpot);
if (g_param.plot == 1)
write_plotpot_pair(&g_pot.calc_pot, g_files.plotfile);
if (g_param.write_lammps == 1)
write_pot_table_lammps();
// will not work with MPI
#if defined(PDIST) && !defined(MPI)
write_pairdist(&g_pot.opt_pot, g_files.distfile);
#endif // PDIST && !MPI
// write the error files for forces, energies, stresses, ...
write_errors(g_calc.force, tot);
/* calculate total runtime */
if (g_param.opt && g_mpi.myid == 0 && g_calc.ndim > 0) {
printf("\nRuntime: %d hours, %d minutes and %d seconds.\n",
(int)difftime(end_time, start_time) / 3600,
((int)difftime(end_time, start_time) % 3600) / 60,
(int)difftime(end_time, start_time) % 60);
printf("%d force calculations, each took %f seconds\n", g_calc.fcalls,
(double)difftime(end_time, start_time) / g_calc.fcalls);
}
#if defined(MPI)
calc_forces(NULL, NULL, 1); /* go wake up other threads */
#endif // MPI
} /* myid == 0 */
// do some cleanups before exiting
#if defined(MPI)
// kill MPI
shutdown_mpi();
#endif // MPI
free_allocated_memory();
return 0;
}
/* Initialize all subsystems:
* MPI System: initialize and setup cluster system
* If rank = 0 (head):
* Initialize DISPLAY and render environment
* Else:
* Await instructions from head!
*/
void start_sim(int plants, int herbivores, int predators,
int argc, char **argv){
num_plants = plants;
num_herbivores = herbivores;
num_predators = predators;
init_mpi(argc, argv);
if(rank == 0){
// print the initial starting values for organisms
printf("Head node started... plants=%d, herbs=%d, preds=%d.\n",
num_plants, num_herbivores, num_predators);
printf("Simulating a total of %d organisms.\n",
num_plants + num_herbivores + num_predators);
// send startup data:
// #plants, #herbivores, #predators, scrnWidth, scrnHeight
int init_data[NUMBER_OF_ORGANISMS];
init_data[PLANTS] = num_plants; // PLANTS = 0
init_data[HERBIVORES] = num_herbivores; // HERBIVORES = 1
init_data[PREDATORS] = num_predators; // PREDATORS = 2
MPISendStatus(init_data, NUMBER_OF_ORGANISMS);
// note: display idle_func handles all simulation polling events
// all buffer receiving activity handled from this point forward
// in DISPLAY subsystem (display.c)
init_display(argc, argv, num_plants, num_herbivores, num_predators);
terminate();
}
// if node > 5, this is no good. DO nothing!
else if(rank > 5){
// loop aimlessly doing nothing until the head
// terminates all operations
while(MPIReceiveContinue()){
continue;
}
printf("Unused node %d is done.\n", rank);
MPIDone();
}
/*******************************************************/
/*******************************************************/
/*** WORKER NODES CODE: HERE IS WHAT WORKER NODES DO ***/
// run the individual MPI processor and await instructions:
/* Node 1: handle positioning all PLANTS
* Node 2: handle positioning all herbivores
* Node 3: handle positioning all predators
* Node 4: handle collisions between predators and herbivores
* Node 5: handle collisions between herbivores and plants
*/
// NODE 4: herbivore-plant collisions
else if(rank == COLL_PLANTS_HERBIVORES){ // rank 4
int max_plants = num_plants;
int max_herbivores = num_herbivores;
// create initial position buffers (defaults to 0)
int plant_positions[num_plants*2];
int herbivore_positions[num_herbivores*2];
// create initial death buffer (defaults to (char)0 = alive)
char plant_deaths[num_plants];
// create initial feed buffer (defaults to 0);
int herbivore_feed[num_herbivores];
// collision processing loop:
do{
// send feed and death data
MPISendDeathReports(plant_deaths, num_plants, PLANTS+1);
MPISendFeedReports(herbivore_feed, num_herbivores, HERBIVORES+1);
// receive position data for both (predators and herbivores)
MPIRecvCollisionPos_PLANTS_HERBIVORES(
(int*)&plant_positions, max_plants*2,
(int*)&herbivore_positions, max_herbivores*2);
// clear out the arrays
memset(plant_deaths, 0, num_plants*sizeof(char));
memset(herbivore_feed, 0, num_herbivores*sizeof(int));
// processes collisions and apply feed and death data
int i; // index variable
int j; // index variable
for(i=0; i<num_plants; i++){
for(j=0; j<num_herbivores; j++){
int plantX = plant_positions[i*2];
int plantY = plant_positions[i*2+1];
int herbX = herbivore_positions[j*2];
int herbY = herbivore_positions[j*2+1];
// check for collision in alive herbivores
if( plant_deaths[i] == 0 &&
plantX <= herbX+2 && plantX >= herbX-2 &&
plantY <= herbY+2 && plantY >= herbY-2){
plant_deaths[i] = 1;
//num_deaths++;
herbivore_feed[j]++;
}
}
}
}
while(MPIReceiveContinue());
printf("Plant-Herbivore collision node (%d) done.\n", rank);
MPIDone();
}
// NODE 5: predator-herbivore collisions
else if(rank == COLL_HERBIVORES_PREDATORS){ // rank 5
int max_herbivores = num_herbivores;
int max_predators = num_predators;
// create initial position buffers (defaults to 0)
int herbivore_positions[num_herbivores*2];
int predator_positions[num_predators*2];
// create initial death buffer (defaults to (char)0 = alive)
char herbivore_deaths[num_herbivores];
// create initial feed buffer (defaults to 0);
int predator_feed[num_predators];
int num_deaths = 0;
// collision processing loop:
do{
// send feed and death data
//void MPISendDeathReports(char buffer[], int count, int destination)
// PLANTS+1 = the actual nodes (offset from 0, nodes start at 1)
MPISendDeathReports(herbivore_deaths, num_herbivores, HERBIVORES+1);
MPISendFeedReports(predator_feed, num_predators, PREDATORS+1);
// update number of herbivores since last death cycle
//num_herbivores = num_herbivores - num_deaths;
//num_deaths = 0;
// receive position data for both (predators and herbivores)
MPIRecvCollisionPos_HERBIVORES_PREDATORS(
(int*)&herbivore_positions, max_herbivores*2,
(int*)&predator_positions, max_predators*2);
// clear out the arrays
memset(herbivore_deaths, 0, num_herbivores*sizeof(char));
memset(predator_feed, 0, num_predators*sizeof(int));
// processes collisions and apply feed and death data
int i; // index variable
int j; // index variable
for(i=0; i<num_herbivores; i++){
for(j=0; j<num_predators; j++){
int herbX = herbivore_positions[i*2];
int herbY = herbivore_positions[i*2+1];
int predX = predator_positions[j*2];
int predY = predator_positions[j*2+1];
// check for collision in alive herbivores
if( herbivore_deaths[i] == 0 &&
herbX <= predX+1 && herbX >= predX-1 &&
herbY <= predY+1 && herbY >= predY-1){
herbivore_deaths[i] = 1;
//num_deaths++;
predator_feed[j]++;
}
}
}
//do float while int true if for 5 double "times"
}
while(MPIReceiveContinue());
printf("Herbivore-Predator collision node (%d) done.\n", rank);
MPIDone();
}
// NODE 1-3 (moving herbivores, plants, predators)
else {
// establish organism type:
// 0 = plant
// 1 = herbivore
// 2 = predator
organism_type = rank - 1;
// collect status from head node
// this will tell how many organisms this node will have
// have to deal with.
MPIRecvStatus();
// statistics data
int num_eaten = 0;
int num_starved = 0;
int num_reproductions = 0;
// create position and movement arrays for each organism
// each organism is an index in all four arrays
int positions[num_organisms*2]; // absolute positions
float posF[num_organisms*2]; // relative GL positions
int x_velocity[num_organisms]; // x velocities
int y_velocity[num_organisms]; // y velocities
// set values for minumum and maximum X, Y positions
int x_min = 15;
int y_min = 15;
int x_max = WINDOW_WIDTH - 30;
int y_max = WINDOW_HEIGHT - 30;
// create random x and y positions, and random x and y
// velocities for each organism
unsigned int randSeed = (unsigned int)time(NULL);
srand(randSeed);
// insert randomly generated positions/velocities into arrays
int i;
for(i=0; i<num_organisms; i++){
// random works as follows:
// rand() % x : generates a pseudonumber from 0 to (x-1)
// then add offsets
positions[2*i] = (rand() % x_max + 15);
positions[2*i+1] = (rand() % y_max + 15);
if(organism_type == PLANTS){
x_velocity[i] = 0;
y_velocity[i] = 0;
}
else{
x_velocity[i] = (rand() % 10 + 1);
y_velocity[i] = (rand() % 10 + 1);
int dir = (rand() % 2);
if(dir == 0)
x_velocity[i] *= -1;
dir = (rand() % 2);
if(dir == 0)
y_velocity[i] *= -1;
}
}
// feed buffer
int total_feeds[num_organisms];
memset(total_feeds, 0, num_organisms*sizeof(int));
// loop until all ogranisms are dead
while(num_organisms > 0){
// update all positions, and check for possible reversal
// of velocity (if out of bounds!)
for(i=0; i<num_organisms; i++){
// update x-position
positions[2*i] += x_velocity[i];
// if x position is out of bounds, reverse velocity
if(positions[2*i] < x_min || positions[2*i] > x_max){
x_velocity[i] *= -1;
positions[2*i] += x_velocity[i];
}
// update y-position
positions[2*i+1] += y_velocity[i];
// if y position is out of bounds, reverse velocity
if(positions[2*i+1] < y_min || positions[2*i+1] > y_max){
y_velocity[i] *= -1;
positions[2*i+1] += y_velocity[i];
}
}
// death and feed buffers
char deaths[num_organisms];
int feeds[num_organisms];
// send collision detecting nodes all data
if(organism_type == PLANTS){
// get death data
MPIRecvDeathReports((char*)&deaths, num_organisms,
COLL_PLANTS_HERBIVORES);
// update deaths
for(i=0; i<num_organisms; i++){
if(deaths[i] == (char)1){
// move last organism to this position
// and decrease number of organisms
positions[2*i] = positions[2*num_organisms-2];
positions[2*i+1] = positions[2*num_organisms-1];
deaths[i] = deaths[num_organisms-1];
total_feeds[i] = total_feeds[num_organisms-1];
num_organisms--;
i--;
num_eaten++;
//printf("killed a plant! Now we have %d\n", num_organisms);
}
}
// regrow a new plant if under limit
if(num_organisms < num_plants && num_organisms != 0){
// generate 1 new plant for every 100 alive
int added = 0;
i=0;
while((num_organisms+added) < num_plants && i < num_organisms){
positions[2*num_organisms-2] = (rand() % x_max + 15);
positions[2*num_organisms-1] = (rand() % y_max + 15);
total_feeds[num_organisms-1] = 0;
added++;
i+=30;
}
num_organisms += added;
num_reproductions++;
}
// send to node COLL_PLANTS_HERBIVORES
MPISendCollisionPos_PLANTS_HERBIVORES(
positions, num_organisms*2);
}
else if(organism_type == HERBIVORES){
// receive feed reports from COLL_PLANTS_HERBIVORES
MPIRecvFeedReports((int*)&feeds, num_organisms,
COLL_PLANTS_HERBIVORES);
// receive death reports from COLL_HERBIVORES_PREDATORS
MPIRecvDeathReports((char*)&deaths, num_organisms,
COLL_HERBIVORES_PREDATORS);
// check for reproductions
for(i=0; i<num_organisms; i++){
total_feeds[i]--;
//printf("After: %d\n", total_feeds[i]);
total_feeds[i] += 10*feeds[i];
// organism starves if it hasn't fed
if(total_feeds[i] < -100){
// move last organism to this position
// and decrease number of organisms
positions[2*i] = positions[2*num_organisms-2];
positions[2*i+1] = positions[2*num_organisms-1];
feeds[i] = feeds[num_organisms-1];
total_feeds[i] = total_feeds[num_organisms-1];
num_organisms--;
i--;
num_starved++;
}
// organism reproduces
if(total_feeds[i] >= 10 && num_organisms < num_herbivores){
positions[2*num_organisms-2] = (rand() % x_max + 15);
positions[2*num_organisms-1] = (rand() % y_max + 15);
x_velocity[num_organisms] = (rand() % 10 + 1);
y_velocity[num_organisms] = (rand() % 10 + 1);
int dir = (rand() % 2);
if(dir == 0)
x_velocity[num_organisms] *= -1;
dir = (rand() % 2);
if(dir == 0)
y_velocity[num_organisms] *= -1;
total_feeds[num_organisms] = 0;
// create a new herbivore at a random position
num_organisms++;
num_reproductions++;
}
}
// check for deaths
for(i=0; i<num_organisms; i++){
if(deaths[i] == (char)1){
// move last organism to this position
// and decrease number of organisms
positions[2*i] = positions[2*num_organisms-2];
positions[2*i+1] = positions[2*num_organisms-1];
deaths[i] = deaths[num_organisms-1];
total_feeds[i] = total_feeds[num_organisms-1];
num_organisms--;
i--;
num_eaten++;
}
}
// send to node COLL_PLANTS_HERBIVORES
MPISendCollisionPos_PLANTS_HERBIVORES(
positions, num_organisms*2);
// and COLL_HERBIVORES_PREDATORS
MPISendCollisionPos_HERBIVORES_PREDATORS(
positions, num_organisms*2);
}
else if(organism_type == PREDATORS){
// receive feed reports from COLL_HERBIVORES_PREDATORS
MPIRecvFeedReports((int*)&feeds, num_organisms,
COLL_HERBIVORES_PREDATORS);
// check feed data:
for(i=0; i<num_organisms; i++){
//printf("Before: %d\n", total_feeds[i]);
total_feeds[i]--;
//printf("After: %d\n", total_feeds[i]);
total_feeds[i] += 20*feeds[i];
// organism starves if it hasn't fed
if(total_feeds[i] < -1000){
// move last organism to this position
// and decrease number of organisms
positions[2*i] = positions[2*num_organisms-2];
positions[2*i+1] = positions[2*num_organisms-1];
feeds[i] = feeds[num_organisms-1];
total_feeds[i] = total_feeds[num_organisms-1];
num_organisms--;
i--;
num_starved++;
}
// organism reproduces
if(total_feeds[i] >= 10 && num_organisms < num_predators){
positions[2*num_organisms] = (rand() % x_max + 15);
positions[2*num_organisms+1] = (rand() % y_max + 15);
x_velocity[num_organisms] = (rand() % 10 + 1);
y_velocity[num_organisms] = x_velocity[num_organisms];
total_feeds[num_organisms] = 0;
// create a new predator at a random position
num_organisms++;
num_reproductions++;
}
}
// send to node COLL_HERBIVORES_PREDATORS
MPISendCollisionPos_HERBIVORES_PREDATORS(
positions, num_organisms*2);
}
// ERROR CHECK:
// make sure number of organisms is not negative
if(num_organisms < 0)
num_organisms = 0;
// update all positions in the OpenGL float format
// to display in the head node
for(i=0; i<num_organisms; i++){
// scale the x position to relative (-1, 1) scale
// for OpenGL to render
float xpos = (float)positions[2*i] / WINDOW_WIDTH;
xpos = xpos * 2 - 1.0;
posF[2*i] = xpos;
// scale the y position to relative (-1, 1) scale
// for OpenGL to render
float ypos = (float)positions[2*i+1] / WINDOW_HEIGHT;
ypos = ypos * 2 - 1.0;
posF[2*i+1] = ypos;
}
// send new positions (posF is the relative float positions
// for OpenGL to display)
MPISendPosReport(posF, num_organisms*2);
// receive acknowledgement / report
if(!MPIReceiveContinue()){
// if ack is not received (Head node called for stop
// of operation), break the loop
break;
}
}
// once loop is broken, finish node
// loop is broken if:
// 1) Head node terminates parallel operation
// or 2) This node's organism goes extinct
printf("Organism location node (%d) is done.\n", rank);
printf("(%d) ### STATISTICS:\n", rank);
printf("(%d) #### Got eaten: %d\n", rank, num_eaten);
printf("(%d) #### Starved to death: %d\n", rank, num_starved);
printf("(%d) #### Times reproduced: %d\n", rank, num_reproductions);
MPIDone();
}
/****************** WORKER NODES END *******************/
/*******************************************************/
/*******************************************************/
}
//----------------------------------------------------------------
int main(int argc, char **argv)
{
init_mpi(argc, argv);
printf("start %d %d\n",rank,process_count);
int i;
if(rank==0) {
parse_args(argc, argv);
// var_count = read_list_in_file(var_file, &netcdf_var_names);
// rank_0_print("Number of variables = %d\n", var_count);
// time_step_count = read_list_in_file(netcdf_file_list, &netcdf_file_names);
// rank_0_print("Number of timesteps = %d\n", time_step_count);
check_args();
}
// The command line arguments are shared by all processes
#if PIDX_HAVE_MPI
MPI_Bcast(global_box_size, 3, MPI_LONG_LONG, 0, MPI_COMM_WORLD);
MPI_Bcast(local_box_size, 3, MPI_LONG_LONG, 0, MPI_COMM_WORLD);
// MPI_Bcast(&time_step_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&var_file, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
MPI_Bcast(&netcdf_file_list, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
// MPI_Bcast(&var_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&output_file_name, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
#endif
//TODO: Only the rank 0 should read the input files and broadcast the data
var_count = read_list_in_file(var_file, &netcdf_var_names);
rank_0_print("Number of variables = %d\n", var_count);
time_step_count = read_list_in_file(netcdf_file_list, &netcdf_file_names);
rank_0_print("Number of timesteps = %d\n", time_step_count);
calculate_per_process_offsets();
create_pidx_var_names();
PIDX_access pidx_access;
create_pidx_access(&pidx_access);
PIDX_time_step_caching_ON();
determine_var_types();
create_pidx_vars();
int t = 0,plist_id;
for (t = 0; t < time_step_count; ++t)
{
rank_0_print("Processing time step %d (file %s)\n", t, netcdf_file_names[t]);
PIDX_file pidx_file;
int ret = PIDX_file_create(output_file_name, PIDX_MODE_CREATE, pidx_access, &pidx_file);
if (ret != PIDX_success)
terminate_with_error_msg("ERROR: Failed to create PIDX file\n");
set_pidx_params(pidx_file);
PIDX_set_current_time_step(pidx_file, t);
int file_id = ncmpi_open(MPI_COMM_WORLD,netcdf_file_names[t], NC_NOWRITE, MPI_INFO_NULL, &plist_id);
if (file_id !=0)
terminate_with_error_msg("ERROR: Failed to open file %s\n", netcdf_file_names[t]);
int v = 0;
for(v = 0; v < var_count; ++v)
{
rank_0_print("Processing variable %s\n", netcdf_var_names[v]);
var_data = malloc(var_types[v].atomic_type * var_types[v].num_values * local_box_size[0] * local_box_size[1] * local_box_size[2]);
read_var_from_netcdf(plist_id, netcdf_var_names[v], var_types[v]);
write_var_to_idx(pidx_file, pidx_var_names[v], pidx_vars[v]);
if (PIDX_flush(pidx_file) != PIDX_success)
terminate_with_error_msg("ERROR: Failed to flush variable %s, time step %d\n", pidx_var_names[v], t);
free(var_data);
}
ncmpi_close(plist_id);
PIDX_close(pidx_file);
}
PIDX_time_step_caching_OFF();
PIDX_close_access(pidx_access);
free_memory();
shutdown_mpi();
return 0;
}
int main(int argc, char **argv)
{
double start_time, end_time;
start_time = get_time();
int ret = 0;
init_mpi(argc, argv);
parse_args(argc, argv);
rank_0_print("Merge Program\n");
#if 0
comm = MPI_COMM_WORLD;
#endif
ret = IDX_file_open(output_file_name);
if (ret != 0) terminate_with_error_msg("PIDX_file_create");
maxh = strlen(bitSequence);
fprintf(stderr, "Partition size :: and count %d %d %d :: %d %d %d\n", idx_count[0], idx_count[1], idx_count[2], idx_size[0], idx_size[1], idx_size[2]);
fprintf(stderr, "bitstring %s maxh = %d\n", bitSequence, maxh);
// shared_block_level is the level upto which the idx blocks are shared
int shared_block_level = (int)log2(idx_count[0] * idx_count[1] * idx_count[2]) + bits_per_block + 1;
if (shared_block_level >= maxh)
shared_block_level = maxh;
int shared_block_count = pow(2, shared_block_level - 1) / samples_per_block;
fprintf(stderr, "Shared block level = %d Shared block count = %d\n", shared_block_level, shared_block_count);
int level = 0;
int ts = 0;
// Iteration through all the timesteps
for (ts = start_time_step; ts <= end_time_step; ts++)
{
// Iteration through all the shared blocks
//for (level = 0; level < shared_block_level; level = level + 1)
{
int hz_index = (int)pow(2, level - 1);
int file_no = hz_index / (blocks_per_file * samples_per_block);
int file_count;
char existing_file_name[PIDX_FILE_PATH_LENGTH];
char new_file_name[PIDX_FILE_PATH_LENGTH];
int ic = 0;
if (level <= bits_per_block + log2(blocks_per_file) + 1)
file_count = 1;
else
file_count = (int)pow(2, level - (bits_per_block + log2(blocks_per_file) + 1));
// file_no is the index of the file that needs to be opened to read from all the partitions
// they contain the shared blocks
fprintf(stderr, "Opening file %d\n", file_no);
#if 1
// iterate throuh all the files that contains the shared blocks
// most likely this will be only the first file of all the partitions
// so fc = 1
int fc = 0;
for (fc = file_no; fc < file_no + file_count; fc++)
{
// malloc for the header for the outpur blocks, i.e. the merged blocks
uint32_t* write_binheader;
int write_binheader_count;
write_binheader_count = 10 + 10 * blocks_per_file * variable_count;
int write_binheader_length = write_binheader_count * sizeof (*write_binheader);
write_binheader = malloc(write_binheader_length);
memset(write_binheader, 0, write_binheader_length);
//iterate through all the variables/fields
int var = 0;
off_t var_offset = 0;
for (var = 0; var < 1; var++)
{
unsigned char *write_data_buffer = malloc(samples_per_block * shared_block_count * bpv[var]/8);
memset(write_data_buffer, 0, samples_per_block * shared_block_count * bpv[var]/8);
//fprintf(stderr, "Write bufer size = %d [%d x %d x %d]\n", samples_per_block * shared_block_count * bpv[var]/8, (int)pow(2, bits_per_block), shared_block_count, bpv[var]/8);
// shared block data
// doube pointer (number o fpartitions x number of shared blocks)
unsigned char **read_data_buffer = malloc(idx_count[0] * idx_count[1] * idx_count[2] * sizeof(*read_data_buffer));
memset(read_data_buffer, 0, idx_count[0] * idx_count[1] * idx_count[2] * sizeof(*read_data_buffer));
// shared block header info
uint32_t** read_binheader = malloc(idx_count[0] * idx_count[1] * idx_count[2] * sizeof(*read_binheader));
memset(read_binheader, 0, idx_count[0] * idx_count[1] * idx_count[2] * sizeof(*read_binheader));
file_initialize_time_step(ts, output_file_name, output_file_template);
generate_file_name(blocks_per_file, output_file_template, fc, new_file_name, PATH_MAX);
//fprintf(stderr, "Merged blocks to be written in %s\n", new_file_name);
// iterate through all the parttions
for (ic = 0; ic < idx_count[0] * idx_count[1] * idx_count[2]; ic++)
{
char file_name_skeleton[PIDX_FILE_PATH_LENGTH];
strncpy(file_name_skeleton, output_file_name, strlen(output_file_name) - 4);
file_name_skeleton[strlen(output_file_name) - 4] = '\0';
if (idx_count[0] != 1 || idx_count[1] != 1 || idx_count[2] != 1)
sprintf(partition_file_name, "%s_%d.idx", file_name_skeleton, ic);
else
strcpy(partition_file_name, output_file_name);
file_initialize_time_step(ts, partition_file_name, partition_file_template);
generate_file_name(blocks_per_file, partition_file_template, fc, existing_file_name, PATH_MAX);
int read_binheader_count;
read_binheader_count = 10 + 10 * blocks_per_file * variable_count;
read_binheader[ic] = (uint32_t*) malloc(sizeof (*read_binheader[ic])*(read_binheader_count));
memset(read_binheader[ic], 0, sizeof (*(read_binheader[ic]))*(read_binheader_count));
fprintf(stderr, "[%d] Partition File name %s\n", ic, existing_file_name);
// file exists
if ( access( partition_file_name, F_OK ) != -1 )
{
// contins data from the shared blocks
read_data_buffer[ic] = malloc(samples_per_block * shared_block_count * bpv[var]/8);
memset(read_data_buffer[ic], 0, samples_per_block * shared_block_count * bpv[var]/8);
int fd;
fd = open(existing_file_name, O_RDONLY | O_BINARY);
if (fd < 0)
{
fprintf(stderr, "[File : %s] [Line : %d] open\n", __FILE__, __LINE__);
continue;
return 0;
}
// reads the header infor from binary file of the partitions
ret = read(fd, read_binheader[ic], (sizeof (*(read_binheader[ic])) * read_binheader_count));
if (ret < 0)
{
fprintf(stderr, "[File : %s] [Line : %d] read\n", __FILE__, __LINE__);
return 0;
}
//assert(ret == (sizeof (*(read_binheader[ic])) * read_binheader_count));
// copy the header from the partition file to the merged output file
// do it only for first partition (this gets all info other than block offset nd count)
if (ic == 0)
memcpy(write_binheader, read_binheader[ic], 10 * sizeof (*write_binheader));
int bpf = 0;
size_t data_size = 0;
off_t data_offset = 0;
for (bpf = 0; bpf < shared_block_count; bpf++)
{
data_offset = ntohl(read_binheader[ic][(bpf + var * blocks_per_file)*10 + 12]);
data_size = ntohl(read_binheader[ic][(bpf + var * blocks_per_file)*10 + 14]);
fprintf(stderr, "[%s] [Partition %d Block %d Variable %d] --> Offset %d Count %d\n", partition_file_name, ic, bpf, var, (int)data_offset, (int)data_size);
if (data_offset != 0 && data_size != 0)
{
pread(fd, read_data_buffer[ic] + (bpf * samples_per_block * (bpv[var] / 8)), data_size, data_offset);
write_binheader[((bpf + var * blocks_per_file)*10 + 12)] = htonl(write_binheader_length + (bpf * data_size) + var * shared_block_count);
write_binheader[((bpf + var * blocks_per_file)*10 + 14)] = htonl(data_size);
// Merge happening while the shared block is being read
// Hardcoded stupid merge
// checks if value is not zero then copies to the write block
int m = 0;
for (m = 0; m < data_size / (bpv[var] / 8) ; m++)
{
double temp;
memcpy(&temp, read_data_buffer[ic] + (bpf * samples_per_block + m) * sizeof(double), sizeof(double));
if (temp != 0)
memcpy(write_data_buffer + ((bpf * samples_per_block) + m) * sizeof(double), &temp, sizeof(double));
}
}
}
close(fd);
}
else
continue;
}
//Merge after all the reads
for (ic = 0; ic < idx_count[0] * idx_count[1] * idx_count[2]; ic++)
{
//input is read_data_buffer**
//output is write_data_buffer*
}
if ( access( new_file_name, F_OK ) != -1 )
{
// file exists
int fd;
fd = open(new_file_name, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR);
{
}
close(fd);
}
else
{
// file doesn't exist
/*
int r;
for (r = 0; r < (shared_block_count * samples_per_block * bpv[var]/8) / sizeof(double); r++)
{
double dval;
memcpy(&dval, write_data_buffer + r * sizeof(double), sizeof(double));
fprintf(stderr, "value at %d = %f\n", r, dval);
}
*/
int fd;
fd = open(new_file_name, O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR);
pwrite(fd, write_binheader, sizeof (*write_binheader)*(write_binheader_count), 0);
pwrite(fd, write_data_buffer, shared_block_count * samples_per_block * bpv[var]/8, sizeof (*write_binheader)*(write_binheader_count));
close(fd);
}
}
}
#endif
}
}
shutdown_mpi();
end_time = get_time();
fprintf(stderr, "Total time taken = %f %f\n", end_time, start_time);
return 0;
}
int main(int argc, char **argv)
{
init_mpi(argc, argv);
parse_args(argc, argv);
check_args();
calculate_per_process_offsets();
create_synthetic_simulation_data();
rank_0_print("Simulation Data Created\n");
int ret;
int var;
int ts;
PIDX_file file; // IDX file descriptor
PIDX_variable* variable; // variable descriptor
variable = malloc(sizeof(*variable) * variable_count);
memset(variable, 0, sizeof(*variable) * variable_count);
PIDX_point global_size, local_offset, local_size;
PIDX_set_point_5D(global_size, global_box_size[0], global_box_size[1], global_box_size[2], 1, 1);
PIDX_set_point_5D(local_offset, local_box_offset[0], local_box_offset[1], local_box_offset[2], 0, 0);
PIDX_set_point_5D(local_size, local_box_size[0], local_box_size[1], local_box_size[2], 1, 1);
// Creating access
PIDX_access access;
PIDX_create_access(&access);
#if PIDX_HAVE_MPI
PIDX_set_mpi_access(access, MPI_COMM_WORLD);
#endif
for (ts = 0; ts < time_step_count; ts++)
{
// PIDX mandatory calls
ret = PIDX_file_create(output_file_name, PIDX_MODE_CREATE, access, global_size, &file);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_file_create");
ret = PIDX_set_current_time_step(file, ts);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_current_time_step");
ret = PIDX_set_variable_count(file, variable_count);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_variable_count");
ret = PIDX_set_resolution(file, 0, reduced_resolution);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_resolution");
char var_name[512];
for (var = 0; var < variable_count; var++)
{
sprintf(var_name, "variable_%d", var);
ret = PIDX_variable_create(var_name, sizeof(unsigned long long) * 8, FLOAT64, &variable[var]);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_create");
ret = PIDX_variable_write_data_layout(variable[var], local_offset, local_size, data[var], PIDX_row_major);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_data_layout");
ret = PIDX_append_and_write_variable(file, variable[var]);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_append_and_write_variable");
}
ret = PIDX_close(file);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_close");
}
ret = PIDX_close_access(access);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_close_access");
free(variable);
variable = 0;
destroy_synthetic_simulation_data();
shutdown_mpi();
return 0;
}
int main(int argc, char **argv)
{
init_mpi(argc, argv,rank,grid_size,dims,coords,periods);
if(grid_size==0) grid_size =1;/*For debugging and with only 1 processor*/
if( argc < 2 ) {
printf("need a problem size\n");
return 1;
}
double pi, h, l_umax;
int n, m, nn, i, j, cols, l_diag_rows;
n = atoi(argv[1]);
m = n-1;
nn = 4*n;
struct Array *len = newArray(grid_size);
struct Array *displ = newArray(grid_size);
splitVector(m,grid_size,)
cols = m/grid_size; /*number of coulums of b to each processor*/
l_diag_rows =m/grid_size;
struct Array *diag = newArray(m,1);
struct Array *l_diag = newArray(l_diag_rows,1);
struct Array *b = newArray(m,cols);
struct Array *bt = newArray(m,cols);
struct Array *z = newArray(nn,1);
createDatatype(b,&grid_size);
h = 1./(double)n;
pi = 4.*atan(1.);
/*struct timeval start, end;
gettimeofday(&start,NULL);
*/
for(i=0;i<diag->rows;i++){
diag->data[i] = 2.*(1.-cos((i+1)*pi/(double)n));
}
/*
#pragma omp for schedule(static)
for(i=0+rank*l_diag->rows;i<((rank+1)*l_diag->rows);i++){
*(l_diag->data+i) = 2.*(1.-cos((i+1)*pi/(double)n));
}
MPI_Allgather(l_diag->data,l_diag->rows,MPI_DOUBLE,diag->data,diag->size,MPI_DOUBLE,cart_comm);
*/
#pragma omp parallel for schedule(static)
for(i=0;i<m;i++){
b->data[i] = h*h;
}
#pragma omp parallel for schedule(static)
for(i=0;i<b->cols;i++){
fst_(b->data+b->rows*i, &b->rows, z->data,&nn);
}
/*Transpose b by sending/receiving rows of the columns in each local b; eg (l_b_0 = {1,2,a,b})+(l_b_1 = {3,4,c,d}) -> b_all = {1,a,3,c,2,b,4,d}*/
MPI_Alltoall(b->data,grid_size,transpose_select_t,bt->data,1,transpose_insert_t,cart_comm);
#pragma omp parallel for schedule(static)
for(i=0;i<m;i++){
fstinv_(bt->data+i*bt->rows,&bt->rows,z->data,&nn);
}
#pragma omp parallel for schedule(static) private(i)
for(j=0;j<m;j++){
for(i=0;i<cols;i++){
*(bt->data + j*bt->rows + i)=*(bt->data + j*bt->rows + i)/(*(diag->data+i)+*(diag->data+j));
}
}
#pragma omp parallel for schedule(static)
for(i=0;i<cols;i++){
fst_(bt->data+i*b->rows, &bt->rows, z->data,&nn);
}
MPI_Alltoall(bt->data,grid_size,transpose_select_t,b->data,1,transpose_insert_t,cart_comm);
#pragma omp parallel for schedule(static)
for(i=0;i<m;i++){
fstinv_(b->data+i*b->rows,&b->rows,z->data,&nn);
}
l_umax = 0.0;
#pragma omp parallel for schedule(static) reduction(max:l_umax)
for(i=0;i<b->size;i++){
if(l_umax<*(b->data +i)) l_umax=*(b->data+i);
}
if(rank==0){
double umax = 0.0;
struct Array *umaxArray = newArray(grid_size,1);
MPI_Gather(&l_umax,1,MPI_DOUBLE,umaxArray->data,grid_size,MPI_DOUBLE,0,cart_comm);
#pragma omp parallel for schedule(static) reduction(max:umax)
for(i=0;i<grid_size;i++){
if(umax<*(umaxArray->data +i)) umax=*(umaxArray->data+i);
}
printf (" umax = %e \n",umax);
/*gettimeofday(&end,NULL);
print_time(start,end);
*/
}
freeArray(b);
freeArray(bt);
freeArray(l_diag);
freeArray(diag);
freeArray(z);
freeDatatype();
MPI_Finalize();
return 0;
}
int main(int argc, char **argv)
{
init_mpi(argc, argv);
parse_args(argc, argv);
check_args();
calculate_per_process_offsets();
create_synthetic_simulation_data();
rank_0_print("Simulation Data Created\n");
int ret, var, ts;
PIDX_file file; // IDX file descriptor
PIDX_variable* variable; // variable descriptor
variable = (PIDX_variable*)malloc(sizeof(*variable) * variable_count);
memset(variable, 0, sizeof(*variable) * variable_count);
PIDX_point global_size, local_offset, local_size;
PIDX_set_point_5D(global_size, global_box_size[0], global_box_size[1], global_box_size[2], 1, 1);
PIDX_set_point_5D(local_offset, local_box_offset[0], local_box_offset[1], local_box_offset[2], 0, 0);
PIDX_set_point_5D(local_size, local_box_size[0], local_box_size[1], local_box_size[2], 1, 1);
// Creating access
PIDX_access access;
PIDX_create_access(&access);
#if PIDX_HAVE_MPI
PIDX_set_mpi_access(access, MPI_COMM_WORLD);
#endif
for (ts = 0; ts < time_step_count; ts++)
{
// PIDX mandatory calls
ret = PIDX_file_create(output_file_name, PIDX_MODE_CREATE, access, global_size, &file);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_file_create");
ret = PIDX_set_current_time_step(file, ts);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_current_time_step");
ret = PIDX_set_variable_count(file, variable_count);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_variable_count");
PIDX_disable_agg(file);
PIDX_save_big_endian(file);
//PIDX_dump_rst_info(file, 1);
//PIDX_debug_rst(file, 1);
//PIDX_debug_hz(file, 1);
PIDX_point reg_patch_size;
PIDX_set_point_5D(reg_patch_size, 128, 128, 128, 1, 1);
PIDX_set_restructuring_box(file, reg_patch_size);
//PIDX_GLOBAL_PARTITION_IDX_IO
//PIDX_IDX_IO
ret = PIDX_set_io_mode(file, PIDX_RAW_IO);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_variable_count");
ret = PIDX_set_partition_size(file, partition_size[0], partition_size[1], partition_size[2]);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_partition_size");
PIDX_set_block_count(file, 128);
//ret = PIDX_set_aggregator_multiplier(file, aggregator_multiplier);
//if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_partition_size");
/*
int io_type = PIDX_IDX_IO;
switch (io_type)
{
case PIDX_GLOBAL_PARTITION_IDX_IO:
PIDX_set_block_count(file,blocks_per_file);
PIDX_set_block_size(file, 13);
break;
case PIDX_IDX_IO:
PIDX_set_block_count(file,blocks_per_file);
break;
case PIDX_RAW_IO:
PIDX_raw_io_pipe_length(file, 2);
PIDX_point reg_patch_size;
PIDX_set_point_5D(reg_patch_size, 128, 128, 128, 1, 1);
PIDX_set_restructuring_box(file, reg_patch_size);
break;
}
*/
//ret = PIDX_debug_disable_agg(file);
//if (ret != PIDX_success) terminate_with_error_msg("PIDX_debug_output");
//ret = PIDX_debug_disable_io(file);
//if (ret != PIDX_success) terminate_with_error_msg("PIDX_debug_output");
//ret = PIDX_debug_disable_hz(file);
//if (ret != PIDX_success) terminate_with_error_msg("PIDX_debug_output");
for (var = 0; var < variable_count; var++)
{
if (bpv[var] == 32)
{
ret = PIDX_variable_create(var_name[var], bpv[var], FLOAT32 , &variable[var]);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_create");
}
else if (bpv[var] == 192)
{
ret = PIDX_variable_create(var_name[var], bpv[var], FLOAT64_RGB , &variable[var]);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_create");
}
else if (bpv[var] == 64)
{
ret = PIDX_variable_create(var_name[var], bpv[var], FLOAT64 , &variable[var]);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_create");
}
ret = PIDX_variable_write_data_layout(variable[var], local_offset, local_size, data[var], PIDX_row_major);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_data_layout");
ret = PIDX_append_and_write_variable(file, variable[var]);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_append_and_write_variable");
//PIDX_flush(file);
}
ret = PIDX_close(file);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_close");
}
ret = PIDX_close_access(access);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_close_access");
free(variable);
variable = 0;
destroy_synthetic_simulation_data();
shutdown_mpi();
return 0;
}
