FORT_DLL_SPEC void FORT_CALL mpi_file_write_at_all_begin_ ( MPI_Fint *v1, MPI_Offset *v2, void*v3, MPI_Fint *v4, MPI_Fint *v5, MPI_Fint *ierr ){
#ifdef MPI_MODE_RDONLY
if (v3 == MPIR_F_MPI_BOTTOM) v3 = MPI_BOTTOM;
*ierr = MPI_File_write_at_all_begin( MPI_File_f2c(*v1), (MPI_Offset)*v2, v3, (int)*v4, (MPI_Datatype)(*v5) );
#else
*ierr = MPI_ERR_INTERN;
#endif
}
FORTRAN_API void FORT_CALL mpi_file_write_at_all_begin_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
int *count,MPI_Datatype *datatype, int *ierr )
{
MPI_File fh_c;
fh_c = MPI_File_f2c(*fh);
*ierr = MPI_File_write_at_all_begin(fh_c,*offset,buf,*count,*datatype);
}
JNIEXPORT void JNICALL Java_mpi_File_writeAtAllBegin(
JNIEnv *env, jobject jthis, jlong fh, jlong fileOffset,
jobject buf, jint count, jlong type)
{
void *ptr = (*env)->GetDirectBufferAddress(env, buf);
int rc = MPI_File_write_at_all_begin((MPI_File)fh, (MPI_Offset)fileOffset,
ptr, count, (MPI_Datatype)type);
ompi_java_exceptionCheck(env, rc);
}
void mpi_file_write_at_all_begin_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
int *count,MPI_Fint *datatype, int *ierr )
{
MPI_File fh_c;
MPI_Datatype datatype_c;
fh_c = MPI_File_f2c(*fh);
datatype_c = MPI_Type_f2c(*datatype);
*ierr = MPI_File_write_at_all_begin(fh_c,*offset,buf,*count,datatype_c);
}
void mpi_file_write_at_all_begin_f(MPI_Fint *fh, MPI_Offset *offset,
char *buf, MPI_Fint *count,
MPI_Fint *datatype, MPI_Fint *ierr)
{
MPI_File c_fh = MPI_File_f2c(*fh);
MPI_Datatype c_type = MPI_Type_f2c(*datatype);
*ierr = OMPI_FINT_2_INT(MPI_File_write_at_all_begin(c_fh,
(MPI_Offset) *offset,
OMPI_F2C_BOTTOM(buf),
OMPI_FINT_2_INT(*count),
c_type));
}
void writefield4_double_(int *fldid, double *vals, int *numNodes)
#endif
{
#ifdef MPI
double fldval[3];
int i, j ;
char fldname[100];
getfieldname_(*fldid, fldname);
if( myrank == 0) {
char* sHeader = (char*) malloc (1024 * sizeof(char));
memset((void*)sHeader, '\0', 1024);
sprintf(sHeader, "VECTORS %s double \n", fldname);
memcpy(&mfBuffer[mfBufferCur], sHeader, strlen(sHeader));
mfBufferCur += strlen(sHeader);
free(sHeader);
}
//printf("*numNodes = %d, myrank=%d\n", *numNodes, myrank);
for (i = 0; i < *numNodes; i++) {
fldval[0] = vals[3*i+0];
fldval[1] = vals[3*i+1];
fldval[2] = vals[3*i+2];
//printf("vals[3*%d]: %lf, %lf, %lf, mfBufferCur=%lld, myrank=%d\n", i,vals[3*i+0], vals[3*i+1], vals[3*i+2], mfBufferCur, myrank);
swap_double_byte( &fldval[0]);
swap_double_byte( &fldval[1]);
swap_double_byte( &fldval[2]);
memcpy( &mfBuffer[mfBufferCur], fldval, sizeof(double)*3);
mfBufferCur += sizeof(double) *3;
}
//printf("finished for loop, myrank=%d\n", myrank);
MPI_Barrier(MPI_COMM_WORLD);
//printf("double ======= 111 ");
long long my_data_offset = 0;
MPI_Status write_status;
MPI_Scan(&mfBufferCur, &my_data_offset, 1, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD);
MPI_Allreduce(&mfBufferCur, &fieldSizeSum, 1, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD);
my_data_offset += mfileCur;
my_data_offset -= mfBufferCur;
MPI_File_write_at_all_begin(mfile, my_data_offset, mfBuffer, mfBufferCur, MPI_CHAR);
MPI_File_write_at_all_end(mfile, mfBuffer, &write_status);
mfileCur += fieldSizeSum;
mfBufferCur = 0;
// add this return symbol into mpi file...
if( myrank == 0)
{
char* sHeader = (char*) malloc (1024 * sizeof(char));
memset((void*)sHeader, '\0', 1024);
sprintf(sHeader, " \n");
memcpy(&mfBuffer[mfBufferCur], sHeader, strlen(sHeader));
mfBufferCur += strlen(sHeader);
free(sHeader);
}
#endif
}
void writecells4_( int *eConnect, int *numElems, int *numCells, int *numNodes)
#endif
{
#ifdef MPI
int* conn;
int* conn_new;
int i, j;
int elemType;
int ncolumn; // indicate lines of CELLs
//ndim = 2, meshtype = 0; //TODO: solver should pass this info, fake for now
//if(myrank ==0) printf("-------------------> now in new version of write2dcells4, ndim = %d, meshtype = %d\n", ndim, meshtype);
if(meshtype == 0 && ndim == 2) { // old hex 2d case
ncolumn = 5;
elemType = 9;
}
else if(meshtype == 1 && ndim == 2) { // new tet 3d case
ncolumn = 4;
elemType = 5;
}
else if(meshtype == 0 && ndim == 3) { // old hex 3d case
ncolumn = 9;
elemType = 12;
}
else if(meshtype == 1 && ndim == 3) { // new tet 3d case
ncolumn = 5;
elemType = 10;
}
else {
printf("Meshtype %d not recognized, quit\n", meshtype);
exit(1);
}
conn = (int*) malloc(sizeof(int) * ncolumn);
assert(conn != NULL);
conn_new = (int*) malloc(sizeof(int) * ncolumn);
assert(conn_new != NULL);
// fprintf( fp, "CELLS %d %d \n", *numCells, 5*(*numCells));
//cell number would be aggregated here
//following conn number would add an offset - myeConnOffset
int totalNumCells = 0;
MPI_Allreduce( numCells, &totalNumCells, 1, MPI_INTEGER, MPI_SUM, MPI_COMM_WORLD);
int myeConnOffset = 0;
MPI_Scan( numNodes, &myeConnOffset, 1, MPI_INTEGER, MPI_SUM, MPI_COMM_WORLD);
myeConnOffset -= *numNodes;
int totalNumNodes = 0;
MPI_Allreduce(numNodes, &totalNumNodes, 1, MPI_INTEGER, MPI_SUM, MPI_COMM_WORLD);
if( myrank == 0)
{
char* sHeader = (char*) malloc (1024 * sizeof(char));
memset((void*)sHeader, '\0', 1024);
sprintf(sHeader, "CELLS %d %d \n", totalNumCells, ncolumn*(totalNumCells) );
memcpy(&mfBuffer[mfBufferCur], sHeader, strlen(sHeader));
mfBufferCur += strlen(sHeader);
free(sHeader);
}
for (i = 0; i < *numCells; i++) {
/*
conn[0] = 4;
conn[1] = eConnect[4*i+0];
conn[2] = eConnect[4*i+1];
conn[3] = eConnect[4*i+2];
conn[4] = eConnect[4*i+3];
for( j = 0; j < 5; j++) swap_int_byte( &conn[j] );
fwrite(conn, sizeof(int), 5, fp);
*/
//mpi-io part
conn_new[0] = ncolumn - 1;
int k;
for(k = 1; k < ncolumn; k++) {
conn_new[k] = eConnect[(ncolumn-1)*i+k-1] + myeConnOffset;
}
/*
conn_new[0] = 4;
conn_new[1] = eConnect[4*i+0] + myeConnOffset;
conn_new[2] = eConnect[4*i+1] + myeConnOffset;
conn_new[3] = eConnect[4*i+2] + myeConnOffset;
conn_new[4] = eConnect[4*i+3] + myeConnOffset;
*/
for( j = 0; j < ncolumn; j++) swap_int_byte( &conn_new[j] );
memcpy(&mfBuffer[mfBufferCur], conn_new, sizeof(int)*ncolumn);
mfBufferCur += sizeof(int) * ncolumn;
}
//flush to disk
long long my_data_offset = 0;
MPI_Status write_status;
MPI_Scan(&mfBufferCur, &my_data_offset, 1, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD);
MPI_Allreduce(&mfBufferCur, &fieldSizeSum, 1, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD);
my_data_offset += mfileCur;
my_data_offset -= mfBufferCur;
MPI_File_write_at_all_begin(mfile, my_data_offset, mfBuffer, mfBufferCur, MPI_CHAR);
MPI_File_write_at_all_end(mfile, mfBuffer, &write_status);
mfileCur += fieldSizeSum;
mfBufferCur = 0;
/*
fprintf( fp, "\n");
fprintf( fp, "CELL_TYPES %d \n", *numCells);
*/
//mpi-io part
if( myrank == 0)
{
char* sHeader = (char*) malloc (1024 * sizeof(char));
memset((void*)sHeader, '\0', 1024);
sprintf(sHeader, "\nCELL_TYPES %d \n", totalNumCells );
memcpy(&mfBuffer[mfBufferCur], sHeader, strlen(sHeader));
mfBufferCur += strlen(sHeader);
free(sHeader);
}
swap_int_byte(&elemType);
for( i = 0; i < *numCells; i++)
{
// fwrite(&elemType, sizeof(int), 1, fp);
//mpi-io
memcpy(&mfBuffer[mfBufferCur], &elemType, sizeof(int));
mfBufferCur += sizeof(int);
}
my_data_offset = 0;
MPI_Scan(&mfBufferCur, &my_data_offset, 1, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD);
MPI_Allreduce(&mfBufferCur, &fieldSizeSum, 1, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD);
my_data_offset += mfileCur;
my_data_offset -= mfBufferCur;
MPI_File_write_at_all_begin(mfile, my_data_offset, mfBuffer, mfBufferCur, MPI_CHAR);
MPI_File_write_at_all_end(mfile, mfBuffer, &write_status);
mfileCur += fieldSizeSum;
mfBufferCur = 0;
/*
fprintf( fp, "\n");
fprintf( fp, "POINT_DATA %d \n", *numNodes);
*/
//mpi-io
if( myrank == 0)
{
char* sHeader = (char*) malloc (1024 * sizeof(char));
memset((void*)sHeader, '\0', 1024);
sprintf(sHeader, "\nPOINT_DATA %d \n", totalNumNodes );
memcpy(&mfBuffer[mfBufferCur], sHeader, strlen(sHeader));
mfBufferCur += strlen(sHeader);
free(sHeader);
}
free(conn);
free(conn_new);
#endif
}
void writenodes4_(double *xyzCoords, int *numNodes)
#endif
{
#ifdef MPI
float coord[3];
int i, j;
/*
fprintf(fp, "POINTS %d ", *numNodes );
fprintf(fp, " float \n");
*/
//has to change *numNodes here
//numNodes would be aggregated
//xyzCoords itself doesn't change (absolute value)
int totalNumNodes = 0;
MPI_Allreduce(numNodes, &totalNumNodes, 1, MPI_INTEGER, MPI_SUM, MPI_COMM_WORLD);
if( myrank == 0)
{
char* sHeader = (char*) malloc( 1024*sizeof(char));
memset((void*)sHeader, '\0', 1024);
sprintf(sHeader, "POINTS %d ", totalNumNodes );
sprintf(sHeader+strlen(sHeader), " float \n");
memcpy(&mfBuffer[mfBufferCur], sHeader, strlen(sHeader));
mfBufferCur += strlen(sHeader);
free(sHeader);
}
for( i = 0; i < *numNodes; i++) {
coord[0] = (float)xyzCoords[3*i+0];
coord[1] = (float)xyzCoords[3*i+1];
coord[2] = (float)xyzCoords[3*i+2];
swap_float_byte( &coord[0] );
swap_float_byte( &coord[1] );
swap_float_byte( &coord[2] );
// fwrite(coord, sizeof(float), 3, fp);
memcpy(&mfBuffer[mfBufferCur], coord, sizeof(float)*3);
mfBufferCur += sizeof(float) *3;
}
// fprintf( fp, " \n");
// mfBuffer[mfBufferCur++] = '\n';
//my_data_offset is local offset for big chunk of field/cell data
//mfileCur is the glocal file offset shared with every proc
long long my_data_offset = 0;
MPI_Status write_status;
MPI_Scan(&mfBufferCur, &my_data_offset, 1, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD);
MPI_Allreduce(&mfBufferCur, &fieldSizeSum, 1, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD);
my_data_offset += mfileCur;
my_data_offset -= mfBufferCur;
MPI_File_write_at_all_begin(mfile, my_data_offset, mfBuffer, mfBufferCur, MPI_CHAR);
MPI_File_write_at_all_end(mfile, mfBuffer, &write_status);
mfileCur += fieldSizeSum;
// MPI_File_close( & mfile );
//clean up mfBuffer/mfBufferCur
mfBufferCur = 0;
//simply adding "\n", following original format
if( myrank == 0)
{
mfBuffer[mfBufferCur] = '\n';
mfBufferCur++;
}
//printf("writenodes4() done\n");
#endif
}
void writecomputetrace_(int* pcompstep, double* pdtime, double* pcpu_t)
#endif
{
COMPUTE_TRACE_FLAG = 1; // only for param print
// now the control is off to the solver, i/o kernel just provides the function
// it's up to solver to decide when to use compute trace
// if(COMPUTE_TRACE_FLAG != 1)
// return;
char tracefname[kMaxPathLen];
int formatparam = trace_ioop;
int nfile = trace_nf;
int stepnum = *pcompstep;
double dtime = *pdtime;
double cpu_t = *pcpu_t;
// only write every few steps TODO: get param(13) IOCOMM and compare with it
if(stepnum%100 != 0) return;
//printf("iostep is %d, dtime = %lf\n", *pcompstep, *pdtime);
int temp_rank;
MPI_Comm_rank(MPI_COMM_WORLD, &temp_rank);
MPI_Comm_size(MPI_COMM_WORLD, &mysize);
MPI_Barrier(MPI_COMM_WORLD);
memset((void*)tracefname, 0, kMaxPathLen);
//sprintf(tracefname, "%s/compute-trace-%d-proc-ioop-%d-nf-%d-t%.5d.dat",
// kOutputPath, mysize, formatparam, nfile, stepnum);
// note: this might be called before going into any io func, so "path" is not set yet
sprintf(tracefname, "%s/compute-trace-%d-proc-istep-%.5d-ioop-%d-nf-%d.dat",
kOutputPath, mysize, stepnum, trace_ioop, trace_nf);
//printf("my filename %s (myrank=%d) \n", tracefname, temp_rank);
// write the actual file
if (1) {
MPI_File timefile;
int rc;
rc = MPI_File_open(MPI_COMM_WORLD, tracefname,
MPI_MODE_CREATE | MPI_MODE_WRONLY , MPI_INFO_NULL, &timefile);
if(rc) {
if(temp_rank == 0) printf("Unble to open file %s, error code:%d! \n", tracefname, rc);
}
char mytime[128];
memset(mytime, 0, 128);
sprintf(mytime, "%10d %10.3lf %10.3lf\n",
temp_rank, dtime, cpu_t);
int len = strlen(mytime);
//printf("str len = %d\n", len);
long long offsets = temp_rank * len ;
MPI_Status write_data_status;
MPI_File_write_at_all_begin(timefile,
offsets,
mytime,
len,
MPI_CHAR);
MPI_File_write_at_all_end(timefile,
mytime,
&write_data_status);
MPI_File_close( & timefile );
}
//printf("writecomputetrace() finished, myrank = %d\n", temp_rank);
}
void printio_(int *fparam, int* piostep)
#endif
{
//printf("format param is %d, iostep is %d\n", (int)*fparam, *piostep);
int formatparam = *fparam;
int iostep = *piostep;
double overall_max, overall_min, overall_avg, overall_sum;
double io_time_max = 0.0;
double file_io_max = 0.0;
if( formatparam == 2 || formatparam == 3 || formatparam == 4 || formatparam == 5)
{
MPI_Comm_size(MPI_COMM_WORLD, &mysize);
MPI_Allreduce( &overall_time, &overall_min, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);
MPI_Allreduce( &overall_time, &overall_max, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce( &overall_time, &overall_sum, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
overall_avg = overall_sum / mysize;
}
else if(formatparam == 6 || formatparam == -6 || formatparam == 8 || formatparam == 18)
{
if(mySpecies == 1)
{
MPI_Allreduce( &overall_time, &overall_min, 1, MPI_DOUBLE, MPI_MIN, localcomm);
MPI_Allreduce( &overall_time, &overall_max, 1, MPI_DOUBLE, MPI_MAX, localcomm);
MPI_Allreduce( &overall_time, &overall_sum, 1, MPI_DOUBLE, MPI_SUM, localcomm);
overall_avg = overall_sum / localsize;
MPI_Allreduce( &file_io_time, &file_io_max, 1, MPI_DOUBLE, MPI_MAX, localcomm);
MPI_Allreduce( &io_time, &io_time_max, 1, MPI_DOUBLE, MPI_MAX, localcomm);
}
else if(mySpecies == 2)
{
overall_time = 0;
overall_min = 0;
overall_max = 0;
overall_avg = 0;
}
}
int temp_rank;
MPI_Comm_rank(MPI_COMM_WORLD, &temp_rank);
if(temp_rank == 0) {
printf("**************************************\n");
printf("I/O time (io_step=%d) stats: overall avg = %lf sec, min = %lf sec, max = %lf sec "
"(io_max = %lf sec, file_io_max = %lf sec, wtick=%lf sec),"
"checkpoint file path is %s, machine is %s, io_option = %d, num_groups = %d "
"(DEBUG_FLAG=%d, COMPUTE_TRACE_FLAG=%d).\n",
io_step, overall_avg, overall_min, overall_max, io_time_max, file_io_max, MPI_Wtick(),
path, mach_name, formatparam, numGroups,
DEBUG_FLAG, COMPUTE_TRACE_FLAG);
printf("**************************************\n");
}
MPI_Barrier(MPI_COMM_WORLD);
// return if IO trace flag not set, otherwise write timing trace of each i/o op
if(IOTRACE_FLAG != 1)
return;
char tracefname[128];
memset((void*)tracefname, 0, 128);
sprintf(tracefname, "iotrace-t%.5d.dat", iostep);
// write the actual file
if (1) {
MPI_File timefile;
int rc;
rc = MPI_File_open(MPI_COMM_WORLD, tracefname,
MPI_MODE_CREATE | MPI_MODE_WRONLY , MPI_INFO_NULL, &timefile);
char mytime[128];
sprintf(mytime, "\n%10d %10.3lf %10.3lf %10.3lf %10.3lf ",
temp_rank, overall_time, overall_avg, overall_min, overall_max);
long long offsets = temp_rank * 56 ;
MPI_Status write_data_status;
MPI_File_write_at_all_begin(timefile,
offsets,
mytime,
56,
MPI_CHAR);
MPI_File_write_at_all_end(timefile,
mytime,
&write_data_status);
MPI_File_close( & timefile );
}
}
