static int
main_write(int argi, int argc, char **argv, json_object *main_obj)
{
int rank = 0, dumpNum = 0, dumpCount = 0;
unsigned long long problem_nbytes, dumpBytes = 0, summedBytes = 0;
char nbytes_str[32], seconds_str[32], bandwidth_str[32], seconds_str2[32];
double dumpTime = 0;
double bandwidth, summedBandwidth;
MACSIO_TIMING_GroupMask_t main_wr_grp = MACSIO_TIMING_GroupMask("main_write");
double dump_loop_start, dump_loop_end;
double min_dump_loop_start, max_dump_loop_end;
int exercise_scr = JsonGetInt(main_obj, "clargs/exercise_scr");
/* Sanity check args */
/* Generate a static problem object to dump on each dump */
json_object *problem_obj = MACSIO_DATA_GenerateTimeZeroDumpObject(main_obj,0);
problem_nbytes = (unsigned long long) json_object_object_nbytes(problem_obj, JSON_C_FALSE);
#warning MAKE JSON OBJECT KEY CASE CONSISTENT
json_object_object_add(main_obj, "problem", problem_obj);
/* Just here for debugging for the moment */
if (MACSIO_LOG_DebugLevel >= 2)
{
char outfName[256];
FILE *outf;
int json_c_print_flags = JSON_C_TO_STRING_PRETTY | JSON_C_TO_STRING_SPACED;
if (MACSIO_LOG_DebugLevel < 3)
json_c_print_flags |= JSON_C_TO_STRING_NO_EXTARR_VALS;
snprintf(outfName, sizeof(outfName), "main_obj_write_%03d.json", MACSIO_MAIN_Rank);
outf = fopen(outfName, "w");
fprintf(outf, "\"%s\"\n", json_object_to_json_string_ext(main_obj, json_c_print_flags));
fclose(outf);
}
#warning WERE NOT GENERATING OR WRITING ANY METADATA STUFF
#warning MAKE THIS LOOP MORE LIKE A MAIN SIM LOOP WITH SIMPLE COMPUTE AND COMM STEP
dump_loop_start = MT_Time();
dumpTime = 0.0;
for (dumpNum = 0; dumpNum < json_object_path_get_int(main_obj, "clargs/num_dumps"); dumpNum++)
{
double dt;
int scr_need_checkpoint_flag = 1;
MACSIO_TIMING_TimerId_t heavy_dump_tid;
#warning ADD OPTION TO UNLINK OLD FILE SETS
#ifdef HAVE_SCR
if (exercise_scr)
SCR_Need_checkpoint(&scr_need_checkpoint_flag);
#endif
const MACSIO_IFACE_Handle_t *iface = MACSIO_IFACE_GetByName(
json_object_path_get_string(main_obj, "clargs/interface"));
/* log dump start */
if (!exercise_scr || scr_need_checkpoint_flag)
{
int scr_valid = 0;
#ifdef HAVE_SCR
if (exercise_scr)
SCR_Start_checkpoint();
#endif
/* Start dump timer */
heavy_dump_tid = MT_StartTimer("heavy dump", main_wr_grp, dumpNum);
#warning REPLACE DUMPN AND DUMPT WITH A STATE TUPLE
#warning SHOULD HAVE PLUGIN RETURN FILENAMES SO MACSIO CAN STAT FOR TOTAL BYTES ON DISK
/* do the dump */
(*(iface->dumpFunc))(argi, argc, argv, main_obj, dumpNum, dumpTime);
#ifdef HAVE_MPI
mpi_errno = 0;
#endif
errno = 0;
dt = MT_StopTimer(heavy_dump_tid);
#ifdef HAVE_SCR
if (exercise_scr)
SCR_Complete_checkpoint(scr_valid);
#endif
}
/* stop timer */
dumpTime += dt;
dumpBytes += problem_nbytes;
dumpCount += 1;
/* log dump timing */
MACSIO_LOG_MSG(Info, ("Dump %02d BW: %s/%s = %s", dumpNum,
MU_PrByts(problem_nbytes, 0, nbytes_str, sizeof(nbytes_str)),
MU_PrSecs(dt, 0, seconds_str, sizeof(seconds_str)),
MU_PrBW(problem_nbytes, dt, 0, bandwidth_str, sizeof(bandwidth_str))));
}
dump_loop_end = MT_Time();
MACSIO_LOG_MSG(Info, ("Overall BW: %s/%s = %s",
MU_PrByts(dumpBytes, 0, nbytes_str, sizeof(nbytes_str)),
MU_PrSecs(dumpTime, 0, seconds_str, sizeof(seconds_str)),
MU_PrBW(dumpBytes, dumpTime, 0, bandwidth_str, sizeof(bandwidth_str))));
bandwidth = dumpBytes / dumpTime;
summedBandwidth = bandwidth;
min_dump_loop_start = dump_loop_start;
max_dump_loop_end = dump_loop_end;
#ifdef HAVE_MPI
MPI_Comm_rank(MACSIO_MAIN_Comm, &rank);
MPI_Reduce(&bandwidth, &summedBandwidth, 1, MPI_DOUBLE, MPI_SUM, 0, MACSIO_MAIN_Comm);
MPI_Reduce(&dumpBytes, &summedBytes, 1, MPI_UNSIGNED_LONG_LONG, MPI_SUM, 0, MACSIO_MAIN_Comm);
MPI_Reduce(&dump_loop_start, &min_dump_loop_start, 1, MPI_DOUBLE, MPI_MIN, 0, MACSIO_MAIN_Comm);
MPI_Reduce(&dump_loop_end, &max_dump_loop_end, 1, MPI_DOUBLE, MPI_MAX, 0, MACSIO_MAIN_Comm);
#endif
if (rank == 0)
{
MACSIO_LOG_MSG(Info, ("Summed BW: %s",
MU_PrBW(summedBandwidth, 1.0, 0, bandwidth_str, sizeof(bandwidth_str))));
MACSIO_LOG_MSG(Info, ("Total Bytes: %s; Last finisher - First starter = %s; BW = %s",
MU_PrByts(summedBytes, 0, nbytes_str, sizeof(nbytes_str)),
MU_PrSecs(max_dump_loop_end - min_dump_loop_start, 0, seconds_str, sizeof(seconds_str)),
MU_PrBW(summedBytes, max_dump_loop_end - min_dump_loop_start, 0, bandwidth_str, sizeof(bandwidth_str))));
}
}
float
jacobi(int nn)
{
int i,j,k,n;
float gosa,wgosa,s0,ss;
double s, cpu, com;
for(n=0 ; n<nn ; ++n){
gosa = 0.0;
wgosa= 0.0;
// if (id == 0)fprintf(stderr, "%d: nn:%d, imax:%d, jmax:%d, kmax:%d\n", n, nn, imax, jmax, kmax);
// if (id == 128)fprintf(stderr, "%d: nn:%d, imax:%d, jmax:%d, kmax:%d\n", n, nn, imax, jmax, kmax);
s = MPI_Wtime();
for(i=1 ; i<imax-1 ; ++i)
for(j=1 ; j<jmax-1 ; ++j)
for(k=1 ; k<kmax-1 ; ++k){
s0 = a[0][i][j][k] * p[i+1][j ][k ]
+ a[1][i][j][k] * p[i ][j+1][k ]
+ a[2][i][j][k] * p[i ][j ][k+1]
+ b[0][i][j][k] * ( p[i+1][j+1][k ] - p[i+1][j-1][k ]
- p[i-1][j+1][k ] + p[i-1][j-1][k ] )
+ b[1][i][j][k] * ( p[i ][j+1][k+1] - p[i ][j-1][k+1]
- p[i ][j+1][k-1] + p[i ][j-1][k-1] )
+ b[2][i][j][k] * ( p[i+1][j ][k+1] - p[i-1][j ][k+1]
- p[i+1][j ][k-1] + p[i-1][j ][k-1] )
+ c[0][i][j][k] * p[i-1][j ][k ]
+ c[1][i][j][k] * p[i ][j-1][k ]
+ c[2][i][j][k] * p[i ][j ][k-1]
+ wrk1[i][j][k];
ss = ( s0 * a[3][i][j][k] - p[i][j][k] ) * bnd[i][j][k];
wgosa += ss*ss;
wrk2[i][j][k] = p[i][j][k] + omega * ss;
}
for(i=1 ; i<imax-1 ; ++i)
for(j=1 ; j<jmax-1 ; ++j)
for(k=1 ; k<kmax-1 ; ++k)
p[i][j][k] = wrk2[i][j][k];
cpu = MPI_Wtime() - s;
s = MPI_Wtime();
sendp(ndx,ndy,ndz);
com = MPI_Wtime() - s;
s = MPI_Wtime();
if (id == 0) {
fprintf(stderr, "%d: time: %f cpu: %f com: %f nn:%d, imax:%d, jmax:%d, kmax:%d\n", cpu + com, cpu, com, n, nn, imax, jmax, kmax);
}
// if (n % 10 == 0 ) fprintf(stdout, "LLLL %d %f\n", n, s);
#ifdef SCR_ENABLE
int flag;
char SCR_testFileName[SCR_MAX_FILENAME];
char testFileName[SCR_MAX_FILENAME];
double gs, ge, start, end, dump, encoding;
gs = start = MPI_Wtime();
SCR_Need_checkpoint(&flag);
if(flag){
SCR_Start_checkpoint();
sprintf(testFileName, "testfile.%d", id);
strcpy(SCR_testFileName, testFileName);
SCR_Route_file(testFileName, SCR_testFileName);
if(id == 0) printf(" ***** SCR_ROUTE_FILE %s\n", SCR_testFileName);
file_dump(SCR_testFileName, p, MIMAX * MJMAX * MKMAX * sizeof(float));
end = MPI_Wtime();
dump = end - start;
start = MPI_Wtime();
SCR_Complete_checkpoint(1);
ge = end = MPI_Wtime();
encoding = end - start;
//printf(" ***** SCR_START_CHECKPOINT\n");
if (id == 0) fprintf(stderr, "time: %f (write: %f , enco: %f)\n", ge - gs, dump, encoding);
}
// SCR
#endif
MPI_Allreduce(&wgosa,
&gosa,
1,
MPI_FLOAT,
MPI_SUM,
MPI_COMM_WORLD);
} /* end n loop */
return(gosa);
}
