int main(int argc, char **argv) {
// Needed for code to compile correctly on the Mac
int mype=0;
int numpe=-1;
// Process command-line arguments, if any.
parseInput(argc, argv);
struct timeval tstart_setup;
cpu_timer_start(&tstart_setup);
numpe = 16;
double circ_radius = 6.0;
// Scale the circle appropriately for the mesh size.
circ_radius = circ_radius * (double) nx / 128.0;
int boundary = 1;
int parallel_in = 0;
mesh = new Mesh(nx, ny, levmx, ndim, boundary, parallel_in, do_gpu_calc);
if (DEBUG) {
//if (mype == 0) mesh->print();
char filename[10];
sprintf(filename,"out%1d",mype);
mesh->fp=fopen(filename,"w");
//mesh->print_local();
}
mesh->init(nx, ny, circ_radius, initial_order, do_gpu_calc);
size_t &ncells = mesh->ncells;
state = new State(mesh);
state->init(do_gpu_calc);
mesh->proc.resize(ncells);
mesh->calc_distribution(numpe);
state->fill_circle(circ_radius, 100.0, 5.0);
mesh->nlft = NULL;
mesh->nrht = NULL;
mesh->nbot = NULL;
mesh->ntop = NULL;
// Kahan-type enhanced precision sum implementation.
double H_sum = state->mass_sum(enhanced_precision_sum);
printf ("Mass of initialized cells equal to %14.12lg\n", H_sum);
H_sum_initial = H_sum;
double cpu_time_main_setup = cpu_timer_stop(tstart_setup);
state->parallel_timer_output(numpe,mype,"CPU: setup time time was",cpu_time_main_setup);
long long mem_used = memstats_memused();
if (mem_used > 0) {
printf("Memory used in startup %lld kB\n",mem_used);
printf("Memory peak in startup %lld kB\n",memstats_mempeak());
printf("Memory free at startup %lld kB\n",memstats_memfree());
printf("Memory available at startup %lld kB\n",memstats_memtotal());
}
printf("Iteration 0 timestep n/a Sim Time 0.0 cells %ld Mass Sum %14.12lg\n", ncells, H_sum);
mesh->cpu_calc_neigh_counter=0;
mesh->cpu_time_calc_neighbors=0.0;
mesh->cpu_rezone_counter=0;
mesh->cpu_time_rezone_all=0.0;
mesh->cpu_refine_smooth_counter=0;
// Set up grid.
#ifdef GRAPHICS_OUTPUT
mesh->write_grid(n);
#endif
#ifdef HAVE_GRAPHICS
set_mysize(ncells);
set_viewmode(view_mode);
set_window(mesh->xmin, mesh->xmax, mesh->ymin, mesh->ymax);
set_outline((int)outline);
init_display(&argc, argv, "Shallow Water", mype);
set_cell_coordinates(&mesh->x[0], &mesh->dx[0], &mesh->y[0], &mesh->dy[0]);
set_cell_data(&state->H[0]);
set_cell_proc(&mesh->proc[0]);
set_circle_radius(circle_radius);
draw_scene();
//if (verbose) sleep(5);
sleep(2);
// Set flag to show mesh results rather than domain decomposition.
view_mode = 1;
// Clear superposition of circle on grid output.
circle_radius = -1.0;
cpu_timer_start(&tstart);
set_idle_function(&do_calc);
start_main_loop();
#else
cpu_timer_start(&tstart);
for (int it = 0; it < 10000000; it++) {
do_calc();
}
#endif
return 0;
}
int main(int argc, char **argv) {
// Process command-line arguments, if any.
int mype=0;
int numpe=0;
parseInput(argc, argv);
L7_Init(&mype, &numpe, &argc, argv);
#if 1 // SKG make things sane for debugging
signal(SIGSEGV, SIG_DFL);
#endif
struct timeval tstart_setup;
cpu_timer_start(&tstart_setup);
real_t circ_radius = 6.0;
// Scale the circle appropriately for the mesh size.
circ_radius = circ_radius * (real_t) nx / 128.0;
int boundary = 1;
int parallel_in = 1;
// figure out the max number of threads that can be spawned
if (0 == mype) {
int nt = omp_get_max_threads();
printf("--- num openmp threads: %d\n", nt);
fflush(stdout);
}
mesh = new Mesh(nx, ny, levmx, ndim, boundary, parallel_in, do_gpu_calc);
if (DEBUG) {
//if (mype == 0) mesh->print();
char filename[10];
sprintf(filename,"out%1d",mype);
mesh->fp=fopen(filename,"w");
//mesh->print_local();
}
mesh->init(nx, ny, circ_radius, initial_order, do_gpu_calc);
size_t &ncells = mesh->ncells;
size_t &ncells_global = mesh->ncells_global;
int &noffset = mesh->noffset;
state = new State(mesh);
state->init(do_gpu_calc);
vector<int> &nsizes = mesh->nsizes;
vector<int> &ndispl = mesh->ndispl;
vector<spatial_t> &x = mesh->x;
vector<spatial_t> &dx = mesh->dx;
vector<spatial_t> &y = mesh->y;
vector<spatial_t> &dy = mesh->dy;
nsizes.resize(numpe);
ndispl.resize(numpe);
int ncells_int = ncells;
MPI_Allgather(&ncells_int, 1, MPI_INT, &nsizes[0], 1, MPI_INT, MPI_COMM_WORLD);
ndispl[0]=0;
for (int ip=1; ip<numpe; ip++){
ndispl[ip] = ndispl[ip-1] + nsizes[ip-1];
}
noffset = ndispl[mype];
state->resize(ncells);
state->fill_circle(circ_radius, 100.0, 7.0);
x.clear();
dx.clear();
y.clear();
dy.clear();
// Kahan-type enhanced precision sum implementation.
double H_sum = state->mass_sum(enhanced_precision_sum);
if (mype == 0) printf ("Mass of initialized cells equal to %14.12lg\n", H_sum);
H_sum_initial = H_sum;
double cpu_time_main_setup = cpu_timer_stop(tstart_setup);
mesh->parallel_timer_output("CPU: setup time time was",cpu_time_main_setup, 0);
long long mem_used = memstats_memused();
if (mem_used > 0) {
mesh->parallel_memory_output("Memory used in startup ",mem_used, 0);
mesh->parallel_memory_output("Memory peak in startup ",memstats_mempeak(), 0);
mesh->parallel_memory_output("Memory free at startup ",memstats_memfree(), 0);
mesh->parallel_memory_output("Memory available at startup ",memstats_memtotal(), 0);
}
if (mype == 0) {
printf("Iteration 0 timestep n/a Sim Time 0.0 cells %ld Mass Sum %14.12lg\n", ncells_global, H_sum);
}
for (int i = 0; i < MESH_COUNTER_SIZE; i++){
mesh->cpu_counters[i]=0;
}
for (int i = 0; i < MESH_TIMER_SIZE; i++){
mesh->cpu_timers[i]=0.0;
}
#ifdef HAVE_GRAPHICS
#ifdef HAVE_OPENGL
set_mysize(ncells_global);
//vector<state_t> H_global;
//vector<spatial_t> x_global;
//vector<spatial_t> dx_global;
//vector<spatial_t> y_global;
//vector<spatial_t> dy_global;
//vector<int> proc_global;
if (mype == 0){
H_global.resize(ncells_global);
x_global.resize(ncells_global);
dx_global.resize(ncells_global);
y_global.resize(ncells_global);
dy_global.resize(ncells_global);
proc_global.resize(ncells_global);
}
MPI_Gatherv(&x[0], nsizes[mype], MPI_SPATIAL_T, &x_global[0], &nsizes[0], &ndispl[0], MPI_SPATIAL_T, 0, MPI_COMM_WORLD);
MPI_Gatherv(&dx[0], nsizes[mype], MPI_SPATIAL_T, &dx_global[0], &nsizes[0], &ndispl[0], MPI_SPATIAL_T, 0, MPI_COMM_WORLD);
MPI_Gatherv(&y[0], nsizes[mype], MPI_SPATIAL_T, &y_global[0], &nsizes[0], &ndispl[0], MPI_SPATIAL_T, 0, MPI_COMM_WORLD);
MPI_Gatherv(&dy[0], nsizes[mype], MPI_SPATIAL_T, &dy_global[0], &nsizes[0], &ndispl[0], MPI_SPATIAL_T, 0, MPI_COMM_WORLD);
MPI_Gatherv(&state->H[0], nsizes[mype], MPI_STATE_T, &H_global[0], &nsizes[0], &ndispl[0], MPI_STATE_T, 0, MPI_COMM_WORLD);
set_cell_data(&H_global[0]);
set_cell_coordinates(&x_global[0], &dx_global[0], &y_global[0], &dy_global[0]);
if (view_mode == 0) {
mesh->proc.resize(ncells);
for (size_t ii = 0; ii<ncells; ii++){
mesh->proc[ii] = mesh->mype;
}
MPI_Gatherv(&mesh->proc[0], nsizes[mype], MPI_INT, &proc_global[0], &nsizes[0], &ndispl[0], MPI_C_REAL, 0, MPI_COMM_WORLD);
}
set_cell_proc(&proc_global[0]);
#endif
#ifdef HAVE_MPE
set_mysize(ncells);
set_cell_data(&state->H[0]);
set_cell_coordinates(&mesh->x[0], &mesh->dx[0], &mesh->y[0], &mesh->dy[0]);
set_cell_proc(&mesh->proc[0]);
#endif
set_window((float)mesh->xmin, (float)mesh->xmax, (float)mesh->ymin, (float)mesh->ymax);
set_viewmode(view_mode);
set_outline((int)outline);
init_display(&argc, argv, "Shallow Water");
set_circle_radius(circle_radius);
draw_scene();
if (verbose) sleep(5);
sleep(2);
// Set flag to show mesh results rather than domain decomposition.
view_mode = 1;
// Clear superposition of circle on grid output.
circle_radius = -1.0;
MPI_Barrier(MPI_COMM_WORLD);
cpu_timer_start(&tstart);
set_idle_function(&do_calc);
start_main_loop();
#else
MPI_Barrier(MPI_COMM_WORLD);
cpu_timer_start(&tstart);
for (int it = 0; it < 10000000; it++) {
do_calc();
}
#endif
return 0;
}
int main(int argc, char **argv) {
int ierr;
// Needed for code to compile correctly on the Mac
int mype=0;
int numpe=-1;
// Process command-line arguments, if any.
parseInput(argc, argv);
numpe = 16;
ierr = ezcl_devtype_init(CL_DEVICE_TYPE_GPU, 0);
if (ierr == EZCL_NODEVICE) {
ierr = ezcl_devtype_init(CL_DEVICE_TYPE_CPU, 0);
}
if (ierr != EZCL_SUCCESS) {
printf("No opencl device available -- aborting\n");
exit(-1);
}
real_t circ_radius = 6.0;
// Scale the circle appropriately for the mesh size.
circ_radius = circ_radius * (real_t) nx / 128.0;
int boundary = 1;
int parallel_in = 0;
mesh = new Mesh(nx, ny, levmx, ndim, boundary, parallel_in, do_gpu_calc);
if (DEBUG) {
//if (mype == 0) mesh->print();
char filename[10];
sprintf(filename,"out%1d",mype);
mesh->fp=fopen(filename,"w");
//mesh->print_local();
}
mesh->init(nx, ny, circ_radius, initial_order, do_gpu_calc);
size_t &ncells = mesh->ncells;
state = new State(mesh);
state->init(do_gpu_calc);
mesh->proc.resize(ncells);
mesh->calc_distribution(numpe);
state->fill_circle(circ_radius, 100.0, 7.0);
cl_mem &dev_celltype = mesh->dev_celltype;
cl_mem &dev_i = mesh->dev_i;
cl_mem &dev_j = mesh->dev_j;
cl_mem &dev_level = mesh->dev_level;
cl_mem &dev_H = state->dev_H;
cl_mem &dev_U = state->dev_U;
cl_mem &dev_V = state->dev_V;
state_t *H = state->H;
state_t *U = state->U;
state_t *V = state->V;
state->allocate_device_memory(ncells);
size_t one = 1;
state->dev_deltaT = ezcl_malloc(NULL, const_cast<char *>("dev_deltaT"), &one, sizeof(cl_real_t), CL_MEM_READ_WRITE, 0);
size_t mem_request = (int)((float)ncells*mesh->mem_factor);
dev_celltype = ezcl_malloc(NULL, const_cast<char *>("dev_celltype"), &mem_request, sizeof(cl_int), CL_MEM_READ_ONLY, 0);
dev_i = ezcl_malloc(NULL, const_cast<char *>("dev_i"), &mem_request, sizeof(cl_int), CL_MEM_READ_ONLY, 0);
dev_j = ezcl_malloc(NULL, const_cast<char *>("dev_j"), &mem_request, sizeof(cl_int), CL_MEM_READ_ONLY, 0);
dev_level = ezcl_malloc(NULL, const_cast<char *>("dev_level"), &mem_request, sizeof(cl_int), CL_MEM_READ_ONLY, 0);
cl_command_queue command_queue = ezcl_get_command_queue();
ezcl_enqueue_write_buffer(command_queue, dev_celltype, CL_FALSE, 0, ncells*sizeof(cl_int), (void *)&mesh->celltype[0], &start_write_event);
ezcl_enqueue_write_buffer(command_queue, dev_i, CL_FALSE, 0, ncells*sizeof(cl_int), (void *)&mesh->i[0], NULL );
ezcl_enqueue_write_buffer(command_queue, dev_j, CL_FALSE, 0, ncells*sizeof(cl_int), (void *)&mesh->j[0], NULL );
ezcl_enqueue_write_buffer(command_queue, dev_level, CL_FALSE, 0, ncells*sizeof(cl_int), (void *)&mesh->level[0], NULL );
ezcl_enqueue_write_buffer(command_queue, dev_H, CL_FALSE, 0, ncells*sizeof(cl_state_t), (void *)&H[0], NULL );
ezcl_enqueue_write_buffer(command_queue, dev_U, CL_FALSE, 0, ncells*sizeof(cl_state_t), (void *)&U[0], NULL );
ezcl_enqueue_write_buffer(command_queue, dev_V, CL_TRUE, 0, ncells*sizeof(cl_state_t), (void *)&V[0], &end_write_event );
state->gpu_time_write += ezcl_timer_calc(&start_write_event, &end_write_event);
mesh->nlft = NULL;
mesh->nrht = NULL;
mesh->nbot = NULL;
mesh->ntop = NULL;
mesh->dev_nlft = NULL;
mesh->dev_nrht = NULL;
mesh->dev_nbot = NULL;
mesh->dev_ntop = NULL;
if (ezcl_get_compute_device() == COMPUTE_DEVICE_ATI) enhanced_precision_sum = false;
// Kahan-type enhanced precision sum implementation.
double H_sum = state->mass_sum(enhanced_precision_sum);
printf ("Mass of initialized cells equal to %14.12lg\n", H_sum);
H_sum_initial = H_sum;
printf("Iteration 0 timestep n/a Sim Time 0.0 cells %ld Mass Sum %14.12lg\n", ncells, H_sum);
mesh->cpu_calc_neigh_counter=0;
mesh->cpu_time_calc_neighbors=0.0;
mesh->cpu_rezone_counter=0;
mesh->cpu_time_rezone_all=0.0;
mesh->cpu_refine_smooth_counter=0;
// Set up grid.
#ifdef GRAPHICS_OUTPUT
mesh->write_grid(n);
#endif
#ifdef HAVE_GRAPHICS
set_mysize(ncells);
set_viewmode(view_mode);
set_window((float)mesh->xmin, (float)mesh->xmax, (float)mesh->ymin, (float)mesh->ymax);
set_outline((int)outline);
init_display(&argc, argv, "Shallow Water", mype);
set_cell_coordinates(&mesh->x[0], &mesh->dx[0], &mesh->y[0], &mesh->dy[0]);
set_cell_data(&H[0]);
set_cell_proc(&mesh->proc[0]);
set_circle_radius(circle_radius);
draw_scene();
//if (verbose) sleep(5);
sleep(2);
// Set flag to show mesh results rather than domain decomposition.
view_mode = 1;
// Clear superposition of circle on grid output.
circle_radius = -1.0;
cpu_timer_start(&tstart);
set_idle_function(&do_calc);
start_main_loop();
#else
cpu_timer_start(&tstart);
for (int it = 0; it < 10000000; it++) {
do_calc();
}
#endif
return 0;
}
