PetscErrorCode readProfileSurfaceScalarDataRecord(char *fileName, PetscScalar *arr, PetscInt numValsPerProfile, PetscInt iRec)
{
/* Random access version of readProfileSurfaceScalarData */
/* This version takes 1 additional argument: */
/* iRec: the record to read (iRec=1 is the first record) */
PetscErrorCode ierr;
off_t off, offset;
PetscViewer fd;
PetscInt fp;
PetscInt iShift;
PetscMPIInt numProcessors, myId;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&myId);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&numProcessors);CHKERRQ(ierr);
/* Shift file pointer to start of data owned by local process */
iShift = (iRec-1)*numValsPerProfile*totalNumProfiles + numValsPerProfile*numPrevProfiles;
off = PETSC_BINARY_SCALAR_SIZE*iShift;
ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,fileName,FILE_MODE_READ,&fd);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetDescriptor(fd,&fp);CHKERRQ(ierr);
ierr = PetscBinarySeek(fp,off,PETSC_BINARY_SEEK_SET,&offset);CHKERRQ(ierr);
ierr = PetscBinaryRead(fp,arr,numValsPerProfile*lNumProfiles,PETSC_SCALAR);CHKERRQ(ierr);
ierr = PetscViewerDestroy(fd);CHKERRQ(ierr);
return 0;
}
PetscErrorCode iniPeriodicTimer( const char pre[], PeriodicTimer *thetimer )
{
PetscErrorCode ierr;
PetscBool flg, flg1;
PetscInt it;
PetscViewer fd;
PetscInt fp;
char timeFile[PETSC_MAX_PATH_LEN];
/* read time data */
ierr = PetscOptionsGetReal(pre,"-cycle_period",&thetimer->cyclePeriod,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ1(PETSC_COMM_WORLD,1,"Must indicate matrix cycling time with the -%scycle_period option",pre);
ierr = PetscOptionsGetReal(pre,"-cycle_step",&thetimer->cycleStep,&flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"WARNING!: Cycling step has been specified for periodic object %s\n",pre);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," This is a legacy option retained for backward compatibility and will be removed in future releases\n");CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Use -%snum_per_period instead\n",pre);CHKERRQ(ierr);
thetimer->numPerPeriod=thetimer->cyclePeriod/thetimer->cycleStep;
/* array for holding extended time array */
PetscMalloc((thetimer->numPerPeriod+2)*sizeof(PetscScalar), &thetimer->tdp);
for (it=0; it<=thetimer->numPerPeriod+1; it++) {
thetimer->tdp[it]=(-thetimer->cycleStep/2.0) + it*thetimer->cycleStep;
}
} else {
ierr = PetscOptionsGetInt(pre,"-num_per_period",&thetimer->numPerPeriod,&flg1);CHKERRQ(ierr);
if (!flg1) SETERRQ1(PETSC_COMM_WORLD,1,"Must indicate number of fields per period with the -%snum_per_period option",pre);
/* array for holding extended time array */
PetscMalloc((thetimer->numPerPeriod+2)*sizeof(PetscScalar), &thetimer->tdp);
ierr = PetscOptionsGetString(pre,"-periodic_times_file",timeFile,PETSC_MAX_PATH_LEN-1,&flg1);CHKERRQ(ierr);
if (flg1) {
ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,timeFile,FILE_MODE_READ,&fd);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetDescriptor(fd,&fp);CHKERRQ(ierr);
ierr = PetscBinaryRead(fp,&thetimer->tdp[1],thetimer->numPerPeriod,PETSC_SCALAR);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&fd);CHKERRQ(ierr);
thetimer->tdp[0]=thetimer->tdp[thetimer->numPerPeriod]-thetimer->cyclePeriod;
thetimer->tdp[thetimer->numPerPeriod+1]=thetimer->tdp[1]+thetimer->cyclePeriod;
} else {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Assuming equally-spaced fields for periodic object %s\n",pre);CHKERRQ(ierr);
thetimer->cycleStep=thetimer->cyclePeriod/thetimer->numPerPeriod;
for (it=0; it<=thetimer->numPerPeriod+1; it++) {
thetimer->tdp[it]=(-thetimer->cycleStep/2.0) + it*thetimer->cycleStep;
}
}
}
ierr = PetscPrintf(PETSC_COMM_WORLD,"Periodic object %s specified at times:\n",pre);CHKERRQ(ierr);
for (it=0; it<=thetimer->numPerPeriod+1; it++) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"tdp=%10.5f\n", thetimer->tdp[it]);CHKERRQ(ierr);
}
return 0;
}
PetscErrorCode writeProfileSurfaceScalarData(char *fileName, PetscScalar *arr, PetscInt numValsPerProfile, PetscTruth appendToFile)
{
PetscErrorCode ierr;
PetscScalar *tmpArr;
PetscInt *displs, *rcounts, cumpro;
PetscInt ipro;
size_t m1, m2;
/* off_t off, offset; */
PetscViewer fd;
PetscInt fp;
/* PetscInt iShift; */
PetscMPIInt numProcessors, myId;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&myId);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&numProcessors);CHKERRQ(ierr);
m1 = numValsPerProfile*totalNumProfiles*sizeof(PetscScalar);
m2 = numProcessors*sizeof(PetscInt);
/* Allocate memory for temporary arrays */
ierr = PetscMalloc(m1,&tmpArr);CHKERRQ(ierr);
ierr = PetscMalloc(m2,&displs);CHKERRQ(ierr);
ierr = PetscMalloc(m2,&rcounts);CHKERRQ(ierr);
cumpro=0;
for (ipro=1; ipro<=numProcessors; ipro++) {
displs[ipro-1]=numValsPerProfile*cumpro;
rcounts[ipro-1]=numValsPerProfile*gNumProfiles[ipro-1];
cumpro = cumpro + gNumProfiles[ipro-1];
/* ierr=PetscPrintf(PETSC_COMM_WORLD,"cumpro=%d, displs=%d\n",cumpro,displs[ipro-1],rcounts[ipro-1]);CHKERRQ(ierr); */
}
MPI_Gatherv(arr,numValsPerProfile*lNumProfiles,MPI_DOUBLE,tmpArr,rcounts,displs,MPI_DOUBLE,0, PETSC_COMM_WORLD);
if (myId==0) { /* this shouldn't really be necessary, but without it, all processors seem to be writing in append mode */
if (appendToFile) {
ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,fileName,FILE_MODE_APPEND,&fd);CHKERRQ(ierr);
} else {
ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,fileName,FILE_MODE_WRITE,&fd);CHKERRQ(ierr);
}
ierr = PetscViewerBinaryGetDescriptor(fd,&fp);CHKERRQ(ierr);
ierr = PetscBinaryWrite(fp,tmpArr,numValsPerProfile*totalNumProfiles,PETSC_SCALAR,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscViewerDestroy(fd);CHKERRQ(ierr);
}
ierr = PetscFree(tmpArr);CHKERRQ(ierr);
ierr = PetscFree(displs);CHKERRQ(ierr);
ierr = PetscFree(rcounts);CHKERRQ(ierr);
return 0;
}
PetscErrorCode ISView_General_Binary(IS is,PetscViewer viewer)
{
PetscErrorCode ierr;
IS_General *isa = (IS_General*) is->data;
PetscMPIInt rank,size,mesgsize,tag = ((PetscObject)viewer)->tag, mesglen;
PetscInt n,N,len,j,tr[2];
int fdes;
MPI_Status status;
PetscInt message_count,flowcontrolcount,*values;
PetscFunctionBegin;
ierr = PetscLayoutGetLocalSize(is->map, &n);CHKERRQ(ierr);
ierr = PetscLayoutGetSize(is->map, &N);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetDescriptor(viewer,&fdes);CHKERRQ(ierr);
/* determine maximum message to arrive */
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)is),&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)is),&size);CHKERRQ(ierr);
tr[0] = IS_FILE_CLASSID;
tr[1] = N;
ierr = PetscViewerBinaryWrite(viewer,tr,2,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr);
ierr = MPI_Reduce(&n,&len,1,MPIU_INT,MPI_SUM,0,PetscObjectComm((PetscObject)is));CHKERRQ(ierr);
ierr = PetscViewerFlowControlStart(viewer,&message_count,&flowcontrolcount);CHKERRQ(ierr);
if (!rank) {
ierr = PetscBinaryWrite(fdes,isa->idx,n,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr);
ierr = PetscMalloc1(len,&values);CHKERRQ(ierr);
ierr = PetscMPIIntCast(len,&mesgsize);CHKERRQ(ierr);
/* receive and save messages */
for (j=1; j<size; j++) {
ierr = PetscViewerFlowControlStepMaster(viewer,j,&message_count,flowcontrolcount);CHKERRQ(ierr);
ierr = MPI_Recv(values,mesgsize,MPIU_INT,j,tag,PetscObjectComm((PetscObject)is),&status);CHKERRQ(ierr);
ierr = MPI_Get_count(&status,MPIU_INT,&mesglen);CHKERRQ(ierr);
ierr = PetscBinaryWrite(fdes,values,(PetscInt)mesglen,PETSC_INT,PETSC_TRUE);CHKERRQ(ierr);
}
ierr = PetscViewerFlowControlEndMaster(viewer,&message_count);CHKERRQ(ierr);
ierr = PetscFree(values);CHKERRQ(ierr);
} else {
ierr = PetscViewerFlowControlStepWorker(viewer,rank,&message_count);CHKERRQ(ierr);
ierr = PetscMPIIntCast(n,&mesgsize);CHKERRQ(ierr);
ierr = MPI_Send(isa->idx,mesgsize,MPIU_INT,0,tag,PetscObjectComm((PetscObject)is));CHKERRQ(ierr);
ierr = PetscViewerFlowControlEndWorker(viewer,&message_count);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode readProfileSurfaceScalarData(char *fileName, PetscScalar *arr, PetscInt numValsPerProfile)
{
PetscErrorCode ierr;
/* PetscScalar *tmpArr; */
/* PetscInt ip; */
/* size_t m1, m2; */
off_t off, offset;
PetscViewer fd;
PetscInt fp;
PetscInt iShift;
PetscMPIInt numProcessors, myId;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&myId);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&numProcessors);CHKERRQ(ierr);
/* m1 = numValsPerProfile*totalNumProfiles*sizeof(PetscScalar); */
/* m2 = numValsPerProfile*lNumProfiles*sizeof(PetscScalar); */
/*Read all data into temporary array */
/* ierr = PetscMalloc(m1,&tmpArr);CHKERRQ(ierr); */
/* ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,fileName,FILE_MODE_READ,&fd);CHKERRQ(ierr); */
/* ierr = PetscViewerBinaryGetDescriptor(fd,&fp);CHKERRQ(ierr); */
/* ierr = PetscBinaryRead(fp,tmpArr,numValsPerProfile*totalNumProfiles,PETSC_SCALAR);CHKERRQ(ierr); */
/* ierr = PetscViewerDestroy(fd);CHKERRQ(ierr); */
/* Shift file pointer to start of data owned by local process */
iShift = numValsPerProfile*numPrevProfiles;
/* printf("ipro=%d,iShift=%d\n",myId,iShift); */
off = PETSC_BINARY_SCALAR_SIZE*iShift;
ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,fileName,FILE_MODE_READ,&fd);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetDescriptor(fd,&fp);CHKERRQ(ierr);
ierr = PetscBinarySeek(fp,off,PETSC_BINARY_SEEK_SET,&offset);CHKERRQ(ierr);
ierr = PetscBinaryRead(fp,arr,numValsPerProfile*lNumProfiles,PETSC_SCALAR);CHKERRQ(ierr);
ierr = PetscViewerDestroy(fd);CHKERRQ(ierr);
/* ierr = PetscMalloc(m2,&arr);CHKERRQ(ierr); */
/* for (ip=1; ip<=lNumProfiles; ip++) { */
/* arr[ip-1]=tmpArr[numPrevProfiles+ip-1]; */
/* } */
/* ierr = PetscFree(tmpArr);CHKERRQ(ierr); */
return 0;
}
PetscErrorCode readProfileSurfaceIntData(char *fileName, PetscInt *arr, PetscInt numValsPerProfile)
{
PetscErrorCode ierr;
/* PetscInt *tmpArr; */
/* PetscInt ip; */
/* size_t m1, m2; */
off_t off, offset;
PetscViewer fd;
PetscInt fp;
PetscInt iShift;
/* m1 = totalNumProfiles*sizeof(PetscInt); */
/* m2 = lNumProfiles*sizeof(PetscInt); */
/* ierr = PetscMalloc(m1,&tmpArr);CHKERRQ(ierr); */
/* ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,fileName,FILE_MODE_READ,&fd);CHKERRQ(ierr); */
/* ierr = PetscViewerBinaryGetDescriptor(fd,&fp);CHKERRQ(ierr); */
/* ierr = PetscBinaryRead(fp,tmpArr,totalNumProfiles,PETSC_INT);CHKERRQ(ierr); */
/* ierr = PetscViewerDestroy(fd);CHKERRQ(ierr); */
/* Shift file pointer to start of data owned by local process */
iShift = numValsPerProfile*numPrevProfiles;
off = PETSC_BINARY_INT_SIZE*iShift;
ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,fileName,FILE_MODE_READ,&fd);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetDescriptor(fd,&fp);CHKERRQ(ierr);
ierr = PetscBinarySeek(fp,off,PETSC_BINARY_SEEK_SET,&offset);CHKERRQ(ierr);
ierr = PetscBinaryRead(fp,arr,numValsPerProfile*lNumProfiles,PETSC_INT);CHKERRQ(ierr);
ierr = PetscViewerDestroy(fd);CHKERRQ(ierr);
/* for (ip=0; ip<totalNumProfiles; ip++) { */
/* ierr = PetscPrintf(PETSC_COMM_WORLD,"ip=%d,kc=%d\n",ip,tmpArr[ip]);CHKERRQ(ierr); */
/* } */
/* ierr = PetscMalloc(m2,&arr);CHKERRQ(ierr); */
/* for (ip=1; ip<=lNumProfiles; ip++) { */
/* arr[ip-1]=tmpArr[numPrevProfiles+ip-1]; */
/* ierr = PetscPrintf(PETSC_COMM_WORLD,"ip=%d,kc=%d\n",ip,arr[ip-1]);CHKERRQ(ierr); */
/* } */
/* ierr = PetscFree(tmpArr);CHKERRQ(ierr); */
return 0;
}
PetscErrorCode writeBinaryScalarData(char *fileName, PetscScalar *arr, PetscInt N, PetscTruth appendToFile)
{
PetscErrorCode ierr;
PetscViewer fd;
PetscInt fp;
PetscMPIInt myId;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&myId);CHKERRQ(ierr);
if (myId==0) { /* this shouldn't really be necessary, but without it, all processors seem to be writing in append mode */
if (appendToFile) {
ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,fileName,FILE_MODE_APPEND,&fd);CHKERRQ(ierr);
} else {
ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,fileName,FILE_MODE_WRITE,&fd);CHKERRQ(ierr);
}
ierr = PetscViewerBinaryGetDescriptor(fd,&fp);CHKERRQ(ierr);
ierr = PetscBinaryWrite(fp,arr,N,PETSC_SCALAR,PETSC_FALSE);CHKERRQ(ierr);
ierr = PetscViewerDestroy(fd);CHKERRQ(ierr);
}
return 0;
}
int main(int argc,char **args)
{
PetscErrorCode ierr;
PetscMPIInt size;
int fd;
PetscInt i,m = 10,sz;
PetscScalar *avec,*array;
Vec vec;
PetscViewer view_out,view_in;
PetscInitialize(&argc,&args,(char *)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 1) SETERRQ(PETSC_COMM_SELF,1,"This is a uniprocessor example only!");
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
/* ---------------------------------------------------------------------- */
/* PART 1: Write some data to a file in binary format */
/* ---------------------------------------------------------------------- */
/* Allocate array and set values */
ierr = PetscMalloc(m*sizeof(PetscScalar),&array);CHKERRQ(ierr);
for (i=0; i<m; i++) {
array[i] = i*10.0;
}
/* Open viewer for binary output */
ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,"input.dat",FILE_MODE_WRITE,&view_out);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetDescriptor(view_out,&fd);CHKERRQ(ierr);
/* Write binary output */
ierr = PetscBinaryWrite(fd,&m,1,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr);
ierr = PetscBinaryWrite(fd,array,m,PETSC_SCALAR,PETSC_FALSE);CHKERRQ(ierr);
/* Destroy the output viewer and work array */
ierr = PetscViewerDestroy(&view_out);CHKERRQ(ierr);
ierr = PetscFree(array);CHKERRQ(ierr);
/* ---------------------------------------------------------------------- */
/* PART 2: Read data from file and form a vector */
/* ---------------------------------------------------------------------- */
/* Open input binary viewer */
ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,"input.dat",FILE_MODE_READ,&view_in);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetDescriptor(view_in,&fd);CHKERRQ(ierr);
/* Create vector and get pointer to data space */
ierr = VecCreate(PETSC_COMM_SELF,&vec);CHKERRQ(ierr);
ierr = VecSetSizes(vec,PETSC_DECIDE,m);CHKERRQ(ierr);
ierr = VecSetFromOptions(vec);CHKERRQ(ierr);
ierr = VecGetArray(vec,&avec);CHKERRQ(ierr);
/* Read data into vector */
ierr = PetscBinaryRead(fd,&sz,1,PETSC_INT);CHKERRQ(ierr);
if (sz <=0) SETERRQ(PETSC_COMM_SELF,1,"Error: Must have array length > 0");
ierr = PetscPrintf(PETSC_COMM_SELF,"reading data in binary from input.dat, sz =%D ...\n",sz);CHKERRQ(ierr);
ierr = PetscBinaryRead(fd,avec,sz,PETSC_SCALAR);CHKERRQ(ierr);
/* View vector */
ierr = VecRestoreArray(vec,&avec);CHKERRQ(ierr);
ierr = VecView(vec,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
/* Free data structures */
ierr = VecDestroy(&vec);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&view_in);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode VecLoad_Binary(Vec vec, PetscViewer viewer)
{
PetscMPIInt size,rank,tag;
int fd;
PetscInt i,rows = 0,n,*range,N,bs;
PetscErrorCode ierr;
PetscBool flag;
PetscScalar *avec,*avecwork;
MPI_Comm comm;
MPI_Request request;
MPI_Status status;
#if defined(PETSC_HAVE_MPIIO)
PetscBool useMPIIO;
#endif
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)viewer,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetDescriptor(viewer,&fd);CHKERRQ(ierr);
ierr = PetscViewerBinaryReadVecHeader_Private(viewer,&rows);CHKERRQ(ierr);
/* Set Vec sizes,blocksize,and type if not already set. Block size first so that local sizes will be compatible. */
ierr = PetscOptionsGetInt(((PetscObject)vec)->prefix, "-vecload_block_size", &bs, &flag);CHKERRQ(ierr);
if (flag) {
ierr = VecSetBlockSize(vec, bs);CHKERRQ(ierr);
}
if (vec->map->n < 0 && vec->map->N < 0) {
ierr = VecSetSizes(vec,PETSC_DECIDE,rows);CHKERRQ(ierr);
}
/* If sizes and type already set,check if the vector global size is correct */
ierr = VecGetSize(vec, &N);CHKERRQ(ierr);
if (N != rows) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_FILE_UNEXPECTED, "Vector in file different length (%d) then input vector (%d)", rows, N);
#if defined(PETSC_HAVE_MPIIO)
ierr = PetscViewerBinaryGetMPIIO(viewer,&useMPIIO);CHKERRQ(ierr);
if (useMPIIO) {
ierr = VecLoad_Binary_MPIIO(vec, viewer);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
#endif
ierr = VecGetLocalSize(vec,&n);CHKERRQ(ierr);
ierr = PetscObjectGetNewTag((PetscObject)viewer,&tag);CHKERRQ(ierr);
ierr = VecGetArray(vec,&avec);CHKERRQ(ierr);
if (!rank) {
ierr = PetscBinaryRead(fd,avec,n,PETSC_SCALAR);CHKERRQ(ierr);
if (size > 1) {
/* read in other chuncks and send to other processors */
/* determine maximum chunck owned by other */
range = vec->map->range;
n = 1;
for (i=1; i<size; i++) n = PetscMax(n,range[i+1] - range[i]);
ierr = PetscMalloc(n*sizeof(PetscScalar),&avecwork);CHKERRQ(ierr);
for (i=1; i<size; i++) {
n = range[i+1] - range[i];
ierr = PetscBinaryRead(fd,avecwork,n,PETSC_SCALAR);CHKERRQ(ierr);
ierr = MPI_Isend(avecwork,n,MPIU_SCALAR,i,tag,comm,&request);CHKERRQ(ierr);
ierr = MPI_Wait(&request,&status);CHKERRQ(ierr);
}
ierr = PetscFree(avecwork);CHKERRQ(ierr);
}
} else {
ierr = MPI_Recv(avec,n,MPIU_SCALAR,0,tag,comm,&status);CHKERRQ(ierr);
}
ierr = VecRestoreArray(vec,&avec);CHKERRQ(ierr);
ierr = VecAssemblyBegin(vec);CHKERRQ(ierr);
ierr = VecAssemblyEnd(vec);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode iniProfileData(PetscInt myId)
{
PetscMPIInt numProcessors;
PetscErrorCode ierr;
PetscInt ipro, ip;
PetscViewer fd;
PetscInt fp;
PetscTruth flg;
PetscInt avgprofiles, totalprofiles;
PetscInt dum;
useProfiles = PETSC_FALSE;
ierr = PetscOptionsHasName(PETSC_NULL,"-use_profiles",&useProfiles);CHKERRQ(ierr);
if (useProfiles) {
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&numProcessors);CHKERRQ(ierr);
/* Read in number of profiles per processor */
ierr = PetscMalloc(numProcessors*sizeof(PetscInt),&gNumProfiles);CHKERRQ(ierr);
/* ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,"numprofiles.bin",FILE_MODE_READ,&fd);CHKERRQ(ierr); */
/* ierr = PetscViewerBinaryGetDescriptor(fd,&fp);CHKERRQ(ierr); */
/* ierr = PetscBinaryRead(fp,gNumProfiles,numProcessors,PETSC_INT);CHKERRQ(ierr); */
/* ierr = PetscViewerDestroy(fd);CHKERRQ(ierr); */
/* ierr = PetscPrintf(PETSC_COMM_WORLD,"Done reading numprofiles.bin\n");CHKERRQ(ierr); */
/* Compute total number of profiles */
/* totalNumProfiles=0; */
/* for (ipro=1; ipro<=numProcessors; ipro++) { */
/* totalNumProfiles = totalNumProfiles + gNumProfiles[ipro-1]; */
/* } */
/* ierr=PetscPrintf(PETSC_COMM_WORLD,"totalNumProfiles = %d\n",totalNumProfiles);CHKERRQ(ierr); */
ierr = PetscMalloc(totalNumProfiles*sizeof(PetscInt),&gStartIndices);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,"gStartIndices.bin",FILE_MODE_READ,&fd);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetDescriptor(fd,&fp);CHKERRQ(ierr);
ierr = PetscBinaryRead(fp,&totalNumProfiles,1,PETSC_INT);CHKERRQ(ierr);
if (totalNumProfiles<=0) SETERRQ(1,"Invalid total number of profiles! Must be >0");
/* Read in starting and ending global indices of profiles. NOTE: these have a base 1 index. */
ierr = PetscMalloc(totalNumProfiles*sizeof(PetscInt),&gStartIndices);CHKERRQ(ierr);
ierr = PetscBinaryRead(fp,gStartIndices,totalNumProfiles,PETSC_INT);CHKERRQ(ierr);
ierr = PetscViewerDestroy(fd);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Done reading gStartIndices.bin\n");CHKERRQ(ierr);
ierr = PetscMalloc(totalNumProfiles*sizeof(PetscInt),&gEndIndices);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,"gEndIndices.bin",FILE_MODE_READ,&fd);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetDescriptor(fd,&fp);CHKERRQ(ierr);
ierr = PetscBinaryRead(fp,&dum,1,PETSC_INT);CHKERRQ(ierr);
if (dum != totalNumProfiles) SETERRQ(1,"Total number of profiles don't match!");
ierr = PetscBinaryRead(fp,gEndIndices,totalNumProfiles,PETSC_INT);CHKERRQ(ierr);
ierr = PetscViewerDestroy(fd);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Done reading gEndIndices.bin\n");CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Total number of profiles specified: %d\n",totalNumProfiles);CHKERRQ(ierr);
avgprofiles = floor(totalNumProfiles/numProcessors);
totalprofiles = 0;
for (ipro=1; ipro<=numProcessors; ipro++) {
gNumProfiles[ipro-1] = avgprofiles;
totalprofiles = totalprofiles + avgprofiles;
}
gNumProfiles[0] = gNumProfiles[0] + (totalNumProfiles - totalprofiles);
for (ipro=1; ipro<=numProcessors; ipro++) {
ierr=PetscPrintf(PETSC_COMM_WORLD,"Number of profiles on processor %d = %d\n",ipro-1,gNumProfiles[ipro-1]);CHKERRQ(ierr);
}
/* for (ip=1; ip<=totalNumProfiles; ip++) { */
/* gStartIndices[ip-1]=gStartIndices[ip-1]-1; */
/* gEndIndices[ip-1]=gEndIndices[ip-1]-1; */
/* ierr=PetscPrintf(PETSC_COMM_WORLD,"gStartIndices=%d, gEndIndices=%d\n",gStartIndices[ip-1],gEndIndices[ip-1]);CHKERRQ(ierr); */
/* } */
/* Compute total number of profiles upto (but not including) current processor */
/* NOTE: myId starts at 1 */
numPrevProfiles=0;
for (ipro=1; ipro<=myId-1; ipro++) {
numPrevProfiles = numPrevProfiles + gNumProfiles[ipro-1];
}
/* ierr=PetscPrintf(PETSC_COMM_WORLD,"myId = %d\n",myId);CHKERRQ(ierr); */
/* ierr=PetscPrintf(PETSC_COMM_WORLD,"nn = %d\n",nn);CHKERRQ(ierr); */
/* for (ipro=1; ipro<=myId; ipro++) { */
/* ierr=PetscPrintf(PETSC_COMM_WORLD,"ID=%d, gNumProfiles=%d\n",ipro,gNumProfiles[ipro-1]);CHKERRQ(ierr); */
/* } */
/* Compute starting and ending LOCAL indices of profiles on current processor. NOTE: These have a base 0 index. */
lNumProfiles = gNumProfiles[myId-1];
/* ierr=PetscPrintf(PETSC_COMM_WORLD,"lNumProfiles = %d\n",lNumProfiles);CHKERRQ(ierr); */
ierr = PetscMalloc(lNumProfiles*sizeof(PetscInt),&lStartIndices);CHKERRQ(ierr);
ierr = PetscMalloc(lNumProfiles*sizeof(PetscInt),&lEndIndices);CHKERRQ(ierr);
ierr = PetscMalloc(lNumProfiles*sizeof(PetscInt),&lProfileLength);CHKERRQ(ierr);
lSize=0; /* local size of vectors */
for (ip=1; ip<=lNumProfiles; ip++) {
lStartIndices[ip-1]=gStartIndices[numPrevProfiles+ip-1]-gStartIndices[numPrevProfiles+1-1];
lEndIndices[ip-1]=gEndIndices[numPrevProfiles+ip-1]-gStartIndices[numPrevProfiles+1-1];
lProfileLength[ip-1]=lEndIndices[ip-1]-lStartIndices[ip-1]+1;
/* ierr=PetscPrintf(PETSC_COMM_WORLD,"lStartIndices=%d, lEndIndices=%d, lProfileLength=%d\n",lStartIndices[ip-1],lEndIndices[ip-1],lProfileLength[ip-1]);CHKERRQ(ierr); */
lSize=lSize+lProfileLength[ip-1];
}
} /* useProfiles */
return 0;
}
PetscErrorCode VecView_Seq_Binary(Vec xin,PetscViewer viewer)
{
PetscErrorCode ierr;
int fdes;
PetscInt n = xin->map->n,classid=VEC_FILE_CLASSID;
FILE *file;
const PetscScalar *xv;
#if defined(PETSC_HAVE_MPIIO)
PetscBool isMPIIO;
#endif
PetscBool skipHeader;
PetscViewerFormat format;
PetscFunctionBegin;
/* Write vector header */
ierr = PetscViewerBinaryGetSkipHeader(viewer,&skipHeader);CHKERRQ(ierr);
if (!skipHeader) {
ierr = PetscViewerBinaryWrite(viewer,&classid,1,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr);
ierr = PetscViewerBinaryWrite(viewer,&n,1,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr);
}
/* Write vector contents */
#if defined(PETSC_HAVE_MPIIO)
ierr = PetscViewerBinaryGetUseMPIIO(viewer,&isMPIIO);CHKERRQ(ierr);
if (!isMPIIO) {
#endif
ierr = PetscViewerBinaryGetDescriptor(viewer,&fdes);CHKERRQ(ierr);
ierr = VecGetArrayRead(xin,&xv);CHKERRQ(ierr);
ierr = PetscBinaryWrite(fdes,(void*)xv,n,PETSC_SCALAR,PETSC_FALSE);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(xin,&xv);CHKERRQ(ierr);
ierr = PetscViewerGetFormat(viewer,&format);CHKERRQ(ierr);
if (format == PETSC_VIEWER_BINARY_MATLAB) {
MPI_Comm comm;
FILE *info;
const char *name;
ierr = PetscObjectGetName((PetscObject)xin,&name);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)viewer,&comm);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetInfoPointer(viewer,&info);CHKERRQ(ierr);
ierr = PetscFPrintf(comm,info,"#--- begin code written by PetscViewerBinary for MATLAB format ---#\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,info,"#$$ Set.%s = PetscBinaryRead(fd);\n",name);CHKERRQ(ierr);
ierr = PetscFPrintf(comm,info,"#--- end code written by PetscViewerBinary for MATLAB format ---#\n\n");CHKERRQ(ierr);
}
#if defined(PETSC_HAVE_MPIIO)
} else {
MPI_Offset off;
MPI_File mfdes;
PetscMPIInt lsize;
ierr = PetscMPIIntCast(n,&lsize);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetMPIIODescriptor(viewer,&mfdes);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetMPIIOOffset(viewer,&off);CHKERRQ(ierr);
ierr = MPI_File_set_view(mfdes,off,MPIU_SCALAR,MPIU_SCALAR,(char*)"native",MPI_INFO_NULL);CHKERRQ(ierr);
ierr = VecGetArrayRead(xin,&xv);CHKERRQ(ierr);
ierr = MPIU_File_write_all(mfdes,(void*)xv,lsize,MPIU_SCALAR,MPI_STATUS_IGNORE);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(xin,&xv);CHKERRQ(ierr);
ierr = PetscViewerBinaryAddMPIIOOffset(viewer,n*sizeof(PetscScalar));CHKERRQ(ierr);
}
#endif
ierr = PetscViewerBinaryGetInfoPointer(viewer,&file);CHKERRQ(ierr);
if (file) {
if (((PetscObject)xin)->prefix) {
ierr = PetscFPrintf(PETSC_COMM_SELF,file,"-%svecload_block_size %D\n",((PetscObject)xin)->prefix,PetscAbs(xin->map->bs));CHKERRQ(ierr);
} else {
ierr = PetscFPrintf(PETSC_COMM_SELF,file,"-vecload_block_size %D\n",PetscAbs(xin->map->bs));CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
static PetscErrorCode ISView_General_Binary(IS is,PetscViewer viewer)
{
PetscErrorCode ierr;
PetscBool skipHeader,useMPIIO;
IS_General *isa = (IS_General*) is->data;
PetscMPIInt rank,size,mesgsize,tag = ((PetscObject)viewer)->tag, mesglen;
PetscInt n,N,len,j,tr[2];
int fdes;
MPI_Status status;
PetscInt message_count,flowcontrolcount,*values;
PetscFunctionBegin;
/* ierr = ISGetLayout(is,&map);CHKERRQ(ierr); */
ierr = PetscLayoutGetLocalSize(is->map, &n);CHKERRQ(ierr);
ierr = PetscLayoutGetSize(is->map, &N);CHKERRQ(ierr);
tr[0] = IS_FILE_CLASSID;
tr[1] = N;
ierr = PetscViewerBinaryGetSkipHeader(viewer,&skipHeader);CHKERRQ(ierr);
if (!skipHeader) {
ierr = PetscViewerBinaryWrite(viewer,tr,2,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr);
}
ierr = PetscViewerBinaryGetUseMPIIO(viewer,&useMPIIO);CHKERRQ(ierr);
#if defined(PETSC_HAVE_MPIIO)
if (useMPIIO) {
MPI_File mfdes;
MPI_Offset off;
PetscMPIInt lsize;
PetscInt rstart;
const PetscInt *iarray;
ierr = PetscMPIIntCast(n,&lsize);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetMPIIODescriptor(viewer,&mfdes);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetMPIIOOffset(viewer,&off);CHKERRQ(ierr);
ierr = PetscLayoutGetRange(is->map,&rstart,NULL);CHKERRQ(ierr);
off += rstart*(MPI_Offset)sizeof(PetscInt); /* off is MPI_Offset, not PetscMPIInt */
ierr = MPI_File_set_view(mfdes,off,MPIU_INT,MPIU_INT,(char*)"native",MPI_INFO_NULL);CHKERRQ(ierr);
ierr = ISGetIndices(is,&iarray);CHKERRQ(ierr);
ierr = MPIU_File_write_all(mfdes,(void*)iarray,lsize,MPIU_INT,MPI_STATUS_IGNORE);CHKERRQ(ierr);
ierr = ISRestoreIndices(is,&iarray);CHKERRQ(ierr);
ierr = PetscViewerBinaryAddMPIIOOffset(viewer,N*(MPI_Offset)sizeof(PetscInt));CHKERRQ(ierr);
PetscFunctionReturn(0);
}
#endif
ierr = PetscViewerBinaryGetDescriptor(viewer,&fdes);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)is),&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)is),&size);CHKERRQ(ierr);
/* determine maximum message to arrive */
ierr = MPI_Reduce(&n,&len,1,MPIU_INT,MPI_MAX,0,PetscObjectComm((PetscObject)is));CHKERRQ(ierr);
ierr = PetscViewerFlowControlStart(viewer,&message_count,&flowcontrolcount);CHKERRQ(ierr);
if (!rank) {
ierr = PetscBinaryWrite(fdes,isa->idx,n,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr);
ierr = PetscMalloc1(len,&values);CHKERRQ(ierr);
ierr = PetscMPIIntCast(len,&mesgsize);CHKERRQ(ierr);
/* receive and save messages */
for (j=1; j<size; j++) {
ierr = PetscViewerFlowControlStepMaster(viewer,j,&message_count,flowcontrolcount);CHKERRQ(ierr);
ierr = MPI_Recv(values,mesgsize,MPIU_INT,j,tag,PetscObjectComm((PetscObject)is),&status);CHKERRQ(ierr);
ierr = MPI_Get_count(&status,MPIU_INT,&mesglen);CHKERRQ(ierr);
ierr = PetscBinaryWrite(fdes,values,(PetscInt)mesglen,PETSC_INT,PETSC_TRUE);CHKERRQ(ierr);
}
ierr = PetscViewerFlowControlEndMaster(viewer,&message_count);CHKERRQ(ierr);
ierr = PetscFree(values);CHKERRQ(ierr);
} else {
ierr = PetscViewerFlowControlStepWorker(viewer,rank,&message_count);CHKERRQ(ierr);
ierr = PetscMPIIntCast(n,&mesgsize);CHKERRQ(ierr);
ierr = MPI_Send(isa->idx,mesgsize,MPIU_INT,0,tag,PetscObjectComm((PetscObject)is));CHKERRQ(ierr);
ierr = PetscViewerFlowControlEndWorker(viewer,&message_count);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode waitForSignal(PetscInt waitTime)
{
PetscErrorCode ierr;
FILE *fd1;
PetscViewer fd;
PetscInt iDone=0;
PetscInt zero=0;
PetscInt one=1;
PetscInt fp;
PetscInt numsignalfiles = 2;
PetscTruth flg;
if (firstTime) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Initializing signaling module ...\n");CHKERRQ(ierr);
ierr = PetscOptionsGetString(PETSC_NULL,"-signalscript",externalSignalScriptName,PETSC_MAX_PATH_LEN-1,&useExternalSignalScript);CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL,"-signalfiles",&useSignalFiles);CHKERRQ(ierr);
if ((useExternalSignalScript) && (useSignalFiles)) {
SETERRQ(1,"Cannot specify both an external signal script and signal files!");
}
if ((!useExternalSignalScript) && (!useSignalFiles)) {
SETERRQ(1,"Must specify an external signal script OR signal files!");
}
if (useExternalSignalScript) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"External signaling script has been specified: %s\n",externalSignalScriptName);CHKERRQ(ierr);
}
if (useSignalFiles) {
ierr = PetscOptionsGetStringArray(PETSC_NULL,"-signalfiles",signalfiles,&numsignalfiles,&flg);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Signal files have been specified: %s, %s\n",signalfiles[0],signalfiles[1]);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-signalwaittime",&signalWaitTime,&flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Signal wait time of %d seconds has been specified\n",signalWaitTime);CHKERRQ(ierr);
} else {
signalWaitTime = -1;
}
}
firstTime = PETSC_FALSE;
} else {
if (useExternalSignalScript) {
ierr = PetscPOpen(PETSC_COMM_WORLD,PETSC_NULL,externalSignalScriptName,"r",&fd1);CHKERRQ(ierr);
ierr = PetscPClose(PETSC_COMM_WORLD,fd1);CHKERRQ(ierr);
} else {
if (signalWaitTime>0) waitTime = signalWaitTime; /* overwrite with runtime option */
/* overwrite external signal file */
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,signalfiles[1],FILE_MODE_WRITE,&fd);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetDescriptor(fd,&fp);CHKERRQ(ierr);
ierr = PetscBinarySynchronizedWrite(PETSC_COMM_WORLD,fp,&zero,1,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr);
ierr = PetscViewerDestroy(fd);CHKERRQ(ierr);
/* send "ready" signal */
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD,signalfiles[0],&fd);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(fd,"%d\n",one);CHKERRQ(ierr);
ierr = PetscViewerDestroy(fd);CHKERRQ(ierr);
/* wait for external signal */
while (iDone==0) {
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,signalfiles[1],FILE_MODE_READ,&fd);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetDescriptor(fd,&fp);CHKERRQ(ierr);
ierr = PetscBinarySynchronizedRead(PETSC_COMM_WORLD,fp,&iDone,1,PETSC_INT);CHKERRQ(ierr);
ierr = PetscViewerDestroy(fd);CHKERRQ(ierr);
PetscSleep(waitTime);
}
/* send "busy" signal */
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD,signalfiles[0],&fd);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(fd,"%d\n",zero);CHKERRQ(ierr);
ierr = PetscViewerDestroy(fd);CHKERRQ(ierr);
}
}
return 0;
}
