PetscErrorCode readData(MPI_Comm comm,himaInfo *hinfo)
{
PetscInt i;
FILE *fd;
char temp[50];
PetscErrorCode ierr;
PetscMPIInt rank;
PetscReal *v = hinfo->vol, *t = hinfo->St0;
PetscInt num=hinfo->n;
PetscFunctionBeginUser;
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
if (!rank) {
ierr = PetscFOpen(PETSC_COMM_SELF,DATAFILENAME,"r",&fd);CHKERRQ(ierr);
for (i=0;i<num;i++) {
double vv,tt;
if (fscanf(fd,"%s%lf%lf",temp,&vv,&tt) != 3) SETERRQ(PETSC_COMM_SELF,1,"Badly formatted input file\n");
v[i] = vv;
t[i] = tt;
}
fclose(fd);
}
ierr = MPI_Bcast(v,2*num,MPIU_REAL,0,PETSC_COMM_WORLD);CHKERRQ(ierr);
/* ierr = PetscPrintf(PETSC_COMM_SELF,"[%d] vol %g, ... %g; St0 %g, ... %g\n",rank,hinfo->vol[0],hinfo->vol[num-1],hinfo->St0 [0],hinfo->St0[num-1]); */
PetscFunctionReturn(0);
}
PetscErrorCode readData(MPI_Comm comm,himaInfo *hinfo)
{
int i;
FILE *fd;
char temp[50];
PetscErrorCode ierr;
PetscMPIInt rank;
double *v = hinfo->vol, *t = hinfo->St0;
int num=hinfo->n;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm,&rank);
CHKERRQ(ierr);
if (!rank) {
ierr = PetscFOpen(PETSC_COMM_SELF,DATAFILENAME,"r",&fd);
CHKERRQ(ierr);
for (i=0; i<num; i++) {
fscanf(fd,"%s%lf%lf",temp,v+i,t+i);
}
fclose(fd);
}
ierr = MPI_Bcast(v,2*num,MPI_DOUBLE,0,PETSC_COMM_WORLD);
CHKERRQ(ierr);
//ierr = PetscPrintf(PETSC_COMM_SELF,"[%d] vol %g, ... %g; St0 %g, ... %g\n",rank,hinfo->vol[0],hinfo->vol[num-1],hinfo->St0 [0],hinfo->St0[num-1]);
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode PetscDrawCreate_TikZ(PetscDraw draw)
{
PetscDraw_TikZ *win;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscMemcpy(draw->ops,&DvOps,sizeof(DvOps));CHKERRQ(ierr);
ierr = PetscNew(PetscDraw_TikZ,&win);CHKERRQ(ierr);
ierr = PetscLogObjectMemory(draw,sizeof(PetscDraw_TikZ));CHKERRQ(ierr);
draw->data = (void*) win;
if (draw->title) {
ierr = PetscStrallocpy(draw->title,&win->filename);CHKERRQ(ierr);
} else {
const char *fname;
ierr = PetscObjectGetName((PetscObject)draw,&fname);CHKERRQ(ierr);
ierr = PetscStrallocpy(fname,&win->filename);CHKERRQ(ierr);
}
ierr = PetscFOpen(PetscObjectComm((PetscObject)draw),win->filename,"w",&win->fd);CHKERRQ(ierr);
ierr = PetscFPrintf(PetscObjectComm((PetscObject)draw),win->fd,TikZ_BEGIN_DOCUMENT);CHKERRQ(ierr);
ierr = PetscFPrintf(PetscObjectComm((PetscObject)draw),win->fd,TikZ_BEGIN_FRAME);CHKERRQ(ierr);
win->written = PETSC_FALSE;
PetscFunctionReturn(0);
}
int readmm(char s[], Mat *pA){
FILE *file;
int *Is,*Js;
PetscScalar *Vs;
PetscInt m,n,nnz,i;
PetscErrorCode ierr;
ierr = PetscFOpen(PETSC_COMM_SELF,s,"r",&file);CHKERRQ(ierr);
char buf[100];
/* process header with comments */
do fgets(buf,PETSC_MAX_PATH_LEN-1,file);
while (buf[0] == '%');
sscanf(buf,"%d %d %d\n",&m,&n,&nnz);
//ierr = PetscPrintf (PETSC_COMM_SELF,"m = %d, n = %d, nnz = %d\n",m,n,nnz);
/* reseve memory for matrices */
ierr = PetscMalloc3(nnz,&Is, nnz,&Js, nnz,&Vs); CHKERRQ(ierr);
for (i=0; i<nnz; i++) {
ierr = fscanf(file,"%d %d %le\n",&Is[i],&Js[i],(double*)&Vs[i]);
//ierr = PetscPrintf(PETSC_COMM_WORLD,"%d,%d,%le\n",Is[i],Js[i],Vs[i]);CHKERRQ(ierr);
if (ierr == EOF) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_USER,"i=%d, reach EOF\n",i);
Is[i]--; Js[i]--; /* adjust from 1-based to 0-based */
}
fclose(file);
//ierr = PetscPrintf(PETSC_COMM_SELF,"Read file completes.\n");CHKERRQ(ierr);
/* Creat and asseble matrix */
ierr = MatCreate(PETSC_COMM_SELF,pA);CHKERRQ(ierr);
ierr = MatSetType(*pA, /*MATDENSE*/ MATSEQAIJ );CHKERRQ(ierr);
ierr = MatSetSizes(*pA,PETSC_DECIDE,PETSC_DECIDE,m,n);CHKERRQ(ierr);
ierr = MatSetFromOptions(*pA);CHKERRQ(ierr);
ierr = MatSetUp(*pA);CHKERRQ(ierr);
for (i=0; i<nnz; i++) {
ierr = MatSetValues(*pA,1,&Is[i],1,&Js[i],&Vs[i],INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(*pA,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*pA,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
//ierr = MatView(*pA,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscFree3(Is,Js,Vs);CHKERRQ(ierr);
return 0;
}
/*@C
PetscInfoAllow - Causes PetscInfo() messages to be printed to standard output.
Not Collective, each processor may call this separately, but printing is only
turned on if the lowest processor number associated with the PetscObject associated
with the call to PetscInfo() has called this routine.
Input Parameter:
+ flag - PETSC_TRUE or PETSC_FALSE
- filename - optional name of file to write output to (defaults to stdout)
Options Database Key:
. -info [optional filename] - Activates PetscInfoAllow()
Level: advanced
Concepts: debugging^detailed runtime information
Concepts: dumping detailed runtime information
.seealso: PetscInfo()
@*/
PetscErrorCode PetscInfoAllow(PetscBool flag, const char filename[])
{
char fname[PETSC_MAX_PATH_LEN], tname[5];
PetscMPIInt rank;
PetscErrorCode ierr;
PetscFunctionBegin;
if (flag && filename) {
ierr = PetscFixFilename(filename, fname);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr);
sprintf(tname, ".%d", rank);
ierr = PetscStrcat(fname, tname);CHKERRQ(ierr);
ierr = PetscFOpen(MPI_COMM_SELF, fname, "w", &PetscInfoFile);CHKERRQ(ierr);
if (!PetscInfoFile) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN, "Cannot open requested file for writing: %s",fname);
} else if (flag) PetscInfoFile = PETSC_STDOUT;
PetscLogPrintInfo = flag;
PetscLogPrintInfoNull = flag;
PetscFunctionReturn(0);
}
static PetscErrorCode DMDAVTKWriteAll_VTS(DM da,PetscViewer viewer)
{
#if defined(PETSC_USE_REAL_SINGLE)
const char precision[] = "Float32";
#elif defined(PETSC_USE_REAL_DOUBLE)
const char precision[] = "Float64";
#else
const char precision[] = "UnknownPrecision";
#endif
MPI_Comm comm = ((PetscObject)da)->comm;
PetscViewer_VTK *vtk = (PetscViewer_VTK*)viewer->data;
PetscViewerVTKObjectLink link;
FILE *fp;
PetscMPIInt rank,size,tag;
DMDALocalInfo info;
PetscInt dim,mx,my,mz,bs,boffset,maxnnodes,i,j,k,f,r;
PetscInt rloc[6],(*grloc)[6] = PETSC_NULL;
PetscScalar *array,*array2;
PetscReal gmin[3],gmax[3];
PetscErrorCode ierr;
PetscFunctionBegin;
#if defined(PETSC_USE_COMPLEX)
SETERRQ(comm,PETSC_ERR_SUP,"Complex values not supported");
#endif
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,&dim, &mx,&my,&mz, 0,0,0, &bs,0,0,0,0,0);CHKERRQ(ierr);
ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr);
ierr = DMDAGetBoundingBox(da,gmin,gmax);CHKERRQ(ierr);
ierr = PetscFOpen(comm,vtk->filename,"wb",&fp);CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp,"<?xml version=\"1.0\"?>\n");CHKERRQ(ierr);
#ifdef PETSC_WORDS_BIGENDIAN
ierr = PetscFPrintf(comm,fp,"<VTKFile type=\"StructuredGrid\" version=\"0.1\" byte_order=\"BigEndian\">\n");CHKERRQ(ierr);
#else
ierr = PetscFPrintf(comm,fp,"<VTKFile type=\"StructuredGrid\" version=\"0.1\" byte_order=\"LittleEndian\">\n");CHKERRQ(ierr);
#endif
ierr = PetscFPrintf(comm,fp," <StructuredGrid WholeExtent=\"%D %D %D %D %D %D\">\n",0,mx-1,0,my-1,0,mz-1);CHKERRQ(ierr);
if (!rank) {ierr = PetscMalloc(size*6*sizeof(PetscInt),&grloc);CHKERRQ(ierr);}
rloc[0] = info.xs;
rloc[1] = info.xm;
rloc[2] = info.ys;
rloc[3] = info.ym;
rloc[4] = info.zs;
rloc[5] = info.zm;
ierr = MPI_Gather(rloc,6,MPIU_INT,&grloc[0][0],6,MPIU_INT,0,comm);CHKERRQ(ierr);
/* Write XML header */
maxnnodes = 0; /* Used for the temporary array size on rank 0 */
boffset = 0; /* Offset into binary file */
for (r=0; r<size; r++) {
PetscInt xs=-1,xm=-1,ys=-1,ym=-1,zs=-1,zm=-1,nnodes = 0;
if (!rank) {
xs = grloc[r][0];
xm = grloc[r][1];
ys = grloc[r][2];
ym = grloc[r][3];
zs = grloc[r][4];
zm = grloc[r][5];
nnodes = xm*ym*zm;
}
maxnnodes = PetscMax(maxnnodes,nnodes);
#if 0
switch (dim) {
case 1:
ierr = PetscFPrintf(comm,fp," <Piece Extent=\"%D %D %D %D %D %D\">\n",xs,xs+xm-1,0,0,0,0);CHKERRQ(ierr);
break;
case 2:
ierr = PetscFPrintf(comm,fp," <Piece Extent=\"%D %D %D %D %D %D\">\n",xs,xs+xm,ys+ym-1,xs,xs+xm-1,0,0);CHKERRQ(ierr);
break;
case 3:
ierr = PetscFPrintf(comm,fp," <Piece Extent=\"%D %D %D %D %D %D\">\n",xs,xs+xm-1,ys,ys+ym-1,zs,zs+zm-1);CHKERRQ(ierr);
break;
default: SETERRQ1(((PetscObject)da)->comm,PETSC_ERR_SUP,"No support for dimension %D",dim);
}
#endif
ierr = PetscFPrintf(comm,fp," <Piece Extent=\"%D %D %D %D %D %D\">\n",xs,xs+xm-1,ys,ys+ym-1,zs,zs+zm-1);CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp," <Points>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp," <DataArray type=\"%s\" Name=\"Position\" NumberOfComponents=\"3\" format=\"appended\" offset=\"%D\" />\n",precision,boffset);CHKERRQ(ierr);
boffset += 3*nnodes*sizeof(PetscScalar) + sizeof(int);
ierr = PetscFPrintf(comm,fp," </Points>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp," <PointData Scalars=\"ScalarPointData\">\n");CHKERRQ(ierr);
for (link=vtk->link; link; link=link->next) {
Vec X = (Vec)link->vec;
const char *vecname = "";
if (((PetscObject)X)->name || link != vtk->link) { /* If the object is already named, use it. If it is past the first link, name it to disambiguate. */
ierr = PetscObjectGetName((PetscObject)X,&vecname);CHKERRQ(ierr);
}
for (i=0; i<bs; i++) {
char buf[256];
const char *fieldname;
ierr = DMDAGetFieldName(da,i,&fieldname);CHKERRQ(ierr);
if (!fieldname) {
ierr = PetscSNPrintf(buf,sizeof(buf),"Unnamed%D",i);CHKERRQ(ierr);
fieldname = buf;
}
ierr = PetscFPrintf(comm,fp," <DataArray type=\"%s\" Name=\"%s%s\" NumberOfComponents=\"1\" format=\"appended\" offset=\"%D\" />\n",precision,vecname,fieldname,boffset);CHKERRQ(ierr);
boffset += nnodes*sizeof(PetscScalar) + sizeof(int);
}
}
ierr = PetscFPrintf(comm,fp," </PointData>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp," </Piece>\n");CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm,fp," </StructuredGrid>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp," <AppendedData encoding=\"raw\">\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp,"_");CHKERRQ(ierr);
/* Now write the arrays. */
tag = ((PetscObject)viewer)->tag;
ierr = PetscMalloc2(maxnnodes*PetscMax(3,bs),PetscScalar,&array,maxnnodes*3,PetscScalar,&array2);CHKERRQ(ierr);
for (r=0; r<size; r++) {
MPI_Status status;
PetscInt xs=-1,xm=-1,ys=-1,ym=-1,zs=-1,zm=-1,nnodes = 0;
if (!rank) {
xs = grloc[r][0];
xm = grloc[r][1];
ys = grloc[r][2];
ym = grloc[r][3];
zs = grloc[r][4];
zm = grloc[r][5];
nnodes = xm*ym*zm;
} else if (r == rank) {
nnodes = info.xm*info.ym*info.zm;
}
{ /* Write the coordinates */
Vec Coords;
ierr = DMGetCoordinates(da,&Coords);CHKERRQ(ierr);
if (Coords) {
const PetscScalar *coords;
ierr = VecGetArrayRead(Coords,&coords);CHKERRQ(ierr);
if (!rank) {
if (r) {
PetscMPIInt nn;
ierr = MPI_Recv(array,nnodes*dim,MPIU_SCALAR,r,tag,comm,&status);CHKERRQ(ierr);
ierr = MPI_Get_count(&status,MPIU_SCALAR,&nn);CHKERRQ(ierr);
if (nn != nnodes*dim) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Array size mismatch");
} else {
ierr = PetscMemcpy(array,coords,nnodes*dim*sizeof(PetscScalar));CHKERRQ(ierr);
}
/* Transpose coordinates to VTK (C-style) ordering */
for (k=0; k<zm; k++) {
for (j=0; j<ym; j++) {
for (i=0; i<xm; i++) {
PetscInt Iloc = i+xm*(j+ym*k);
array2[Iloc*3+0] = array[Iloc*dim + 0];
array2[Iloc*3+1] = dim > 1 ? array[Iloc*dim + 1] : 0;
array2[Iloc*3+2] = dim > 2 ? array[Iloc*dim + 2] : 0;
}
}
}
} else if (r == rank) {
ierr = MPI_Send((void*)coords,nnodes*dim,MPIU_SCALAR,0,tag,comm);CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(Coords,&coords);CHKERRQ(ierr);
} else { /* Fabricate some coordinates using grid index */
for (k=0; k<zm; k++) {
for (j=0; j<ym; j++) {
for (i=0; i<xm; i++) {
PetscInt Iloc = i+xm*(j+ym*k);
array2[Iloc*3+0] = xs+i;
array2[Iloc*3+1] = ys+j;
array2[Iloc*3+2] = zs+k;
}
}
}
}
ierr = PetscViewerVTKFWrite(viewer,fp,array2,nnodes*3,PETSC_SCALAR);CHKERRQ(ierr);
}
/* Write each of the objects queued up for this file */
for (link=vtk->link; link; link=link->next) {
Vec X = (Vec)link->vec;
const PetscScalar *x;
ierr = VecGetArrayRead(X,&x);CHKERRQ(ierr);
if (!rank) {
if (r) {
PetscMPIInt nn;
ierr = MPI_Recv(array,nnodes*bs,MPIU_SCALAR,r,tag,comm,&status);CHKERRQ(ierr);
ierr = MPI_Get_count(&status,MPIU_SCALAR,&nn);CHKERRQ(ierr);
if (nn != nnodes*bs) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Array size mismatch receiving from rank %D",r);
} else {
ierr = PetscMemcpy(array,x,nnodes*bs*sizeof(PetscScalar));CHKERRQ(ierr);
}
for (f=0; f<bs; f++) {
/* Extract and transpose the f'th field */
for (k=0; k<zm; k++) {
for (j=0; j<ym; j++) {
for (i=0; i<xm; i++) {
PetscInt Iloc = i+xm*(j+ym*k);
array2[Iloc] = array[Iloc*bs + f];
}
}
}
ierr = PetscViewerVTKFWrite(viewer,fp,array2,nnodes,PETSC_SCALAR);CHKERRQ(ierr);
}
} else if (r == rank) {
ierr = MPI_Send((void*)x,nnodes*bs,MPIU_SCALAR,0,tag,comm);CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(X,&x);CHKERRQ(ierr);
}
}
ierr = PetscFree2(array,array2);CHKERRQ(ierr);
ierr = PetscFree(grloc);CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp,"\n </AppendedData>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp,"</VTKFile>\n");CHKERRQ(ierr);
ierr = PetscFClose(comm,fp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode DMPlexVTKWriteAll_ASCII(DM dm, PetscViewer viewer)
{
MPI_Comm comm;
PetscViewer_VTK *vtk = (PetscViewer_VTK*) viewer->data;
FILE *fp;
PetscViewerVTKObjectLink link;
PetscSection coordSection, globalCoordSection;
PetscLayout vLayout;
Vec coordinates;
PetscReal lengthScale;
PetscInt vMax, totVertices, totCells;
PetscBool hasPoint = PETSC_FALSE, hasCell = PETSC_FALSE, writePartition = PETSC_FALSE;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscFOpen(comm, vtk->filename, "wb", &fp);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "# vtk DataFile Version 2.0\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "Simplicial Mesh Example\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "ASCII\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "DATASET UNSTRUCTURED_GRID\n");CHKERRQ(ierr);
/* Vertices */
ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionCreateGlobalSection(coordSection, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
if (vMax >= 0) {
PetscInt pStart, pEnd, p, localSize = 0;
ierr = PetscSectionGetChart(globalCoordSection, &pStart, &pEnd);CHKERRQ(ierr);
pEnd = PetscMin(pEnd, vMax);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof;
ierr = PetscSectionGetDof(globalCoordSection, p, &dof);CHKERRQ(ierr);
if (dof > 0) ++localSize;
}
ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &vLayout);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(vLayout, localSize);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr);
} else {
ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject)dm), globalCoordSection, &vLayout);CHKERRQ(ierr);
}
ierr = PetscLayoutGetSize(vLayout, &totVertices);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "POINTS %d double\n", totVertices);CHKERRQ(ierr);
ierr = DMPlexVTKWriteSection_ASCII(dm, coordSection, globalCoordSection, coordinates, fp, 3, PETSC_DETERMINE, lengthScale);CHKERRQ(ierr);
/* Cells */
ierr = DMPlexVTKWriteCells_ASCII(dm, fp, &totCells);CHKERRQ(ierr);
/* Vertex fields */
for (link = vtk->link; link; link = link->next) {
if ((link->ft == PETSC_VTK_POINT_FIELD) || (link->ft == PETSC_VTK_POINT_VECTOR_FIELD)) hasPoint = PETSC_TRUE;
if ((link->ft == PETSC_VTK_CELL_FIELD) || (link->ft == PETSC_VTK_CELL_VECTOR_FIELD)) hasCell = PETSC_TRUE;
}
if (hasPoint) {
ierr = PetscFPrintf(comm, fp, "POINT_DATA %d\n", totVertices);CHKERRQ(ierr);
for (link = vtk->link; link; link = link->next) {
Vec X = (Vec) link->vec;
DM dmX;
PetscSection section, globalSection, newSection = NULL;
const char *name;
PetscInt enforceDof = PETSC_DETERMINE;
if ((link->ft != PETSC_VTK_POINT_FIELD) && (link->ft != PETSC_VTK_POINT_VECTOR_FIELD)) continue;
if (link->ft == PETSC_VTK_POINT_VECTOR_FIELD) enforceDof = 3;
ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr);
ierr = VecGetDM(X, &dmX);CHKERRQ(ierr);
if (dmX) {
DMLabel subpointMap, subpointMapX;
PetscInt dim, dimX, pStart, pEnd, qStart, qEnd;
ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr);
/* Here is where we check whether dmX is a submesh of dm */
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetDimension(dmX, &dimX);CHKERRQ(ierr);
ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
ierr = DMPlexGetChart(dmX, &qStart, &qEnd);CHKERRQ(ierr);
ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr);
ierr = DMPlexGetSubpointMap(dmX, &subpointMapX);CHKERRQ(ierr);
if (((dim != dimX) || ((pEnd-pStart) < (qEnd-qStart))) && subpointMap && !subpointMapX) {
const PetscInt *ind = NULL;
IS subpointIS;
PetscInt n = 0, q;
ierr = PetscSectionGetChart(section, &qStart, &qEnd);CHKERRQ(ierr);
ierr = DMPlexCreateSubpointIS(dm, &subpointIS);CHKERRQ(ierr);
if (subpointIS) {
ierr = ISGetLocalSize(subpointIS, &n);CHKERRQ(ierr);
ierr = ISGetIndices(subpointIS, &ind);CHKERRQ(ierr);
}
ierr = PetscSectionCreate(comm, &newSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(newSection, pStart, pEnd);CHKERRQ(ierr);
for (q = qStart; q < qEnd; ++q) {
PetscInt dof, off, p;
ierr = PetscSectionGetDof(section, q, &dof);CHKERRQ(ierr);
if (dof) {
ierr = PetscFindInt(q, n, ind, &p);CHKERRQ(ierr);
if (p >= pStart) {
ierr = PetscSectionSetDof(newSection, p, dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, q, &off);CHKERRQ(ierr);
ierr = PetscSectionSetOffset(newSection, p, off);CHKERRQ(ierr);
}
}
}
if (subpointIS) {
ierr = ISRestoreIndices(subpointIS, &ind);CHKERRQ(ierr);
ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
}
/* No need to setup section */
section = newSection;
}
} else {
ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) §ion);CHKERRQ(ierr);
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
}
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
if (newSection) {ierr = PetscSectionDestroy(&newSection);CHKERRQ(ierr);}
}
}
/* Cell Fields */
ierr = PetscOptionsGetBool(((PetscObject) dm)->prefix, "-dm_view_partition", &writePartition, NULL);CHKERRQ(ierr);
if (hasCell || writePartition) {
ierr = PetscFPrintf(comm, fp, "CELL_DATA %d\n", totCells);CHKERRQ(ierr);
for (link = vtk->link; link; link = link->next) {
Vec X = (Vec) link->vec;
DM dmX;
PetscSection section, globalSection;
const char *name;
PetscInt enforceDof = PETSC_DETERMINE;
if ((link->ft != PETSC_VTK_CELL_FIELD) && (link->ft != PETSC_VTK_CELL_VECTOR_FIELD)) continue;
if (link->ft == PETSC_VTK_CELL_VECTOR_FIELD) enforceDof = 3;
ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr);
ierr = VecGetDM(X, &dmX);CHKERRQ(ierr);
if (dmX) {
ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr);
} else {
PetscContainer c;
ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) &c);CHKERRQ(ierr);
if (!c) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscContainerGetPointer(c, (void**) §ion);CHKERRQ(ierr);
}
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
}
if (writePartition) {
ierr = PetscFPrintf(comm, fp, "SCALARS partition int 1\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "LOOKUP_TABLE default\n");CHKERRQ(ierr);
ierr = DMPlexVTKWritePartition_ASCII(dm, fp);CHKERRQ(ierr);
}
}
/* Cleanup */
ierr = PetscSectionDestroy(&globalCoordSection);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr);
ierr = PetscFClose(comm, fp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
PetscInt rank,size,npt;
PetscErrorCode ierr;
Vec x,y0,tempvec, *vinda,*vindb,*vindc;
PetscInt i,j,k,l,n,p,m,m2,pmax,puse,Istart,Iend,localsize,niter;
PetscScalar dx,dy,dx2,dy2;
PetscScalar *Mixnorm;
PetscInt iter,*iterind,*nind;
FILE *fidoutput;
char fname[50];
PetscViewer socketviewer;
PetscInt withMatlab;
PetscTruth Matlabflag;
PetscLogDouble v1,v2,elapsed_time;
PetscInitialize(&argc,&args,(char *)0,help);
MPI_Comm_size(PETSC_COMM_WORLD,&size);
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nPETSC: Petsc Initializes successfully! \n");
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: comm_size is %d \n", size);
ierr = PetscOptionsGetInt(PETSC_NULL,"-withMatlab",&withMatlab,&Matlabflag);CHKERRQ(ierr);
if (Matlabflag == PETSC_FALSE){withMatlab = 0;}else{withMatlab = 1;}
if(withMatlab==1){
// Rank 0 connects to socket, use default socket
PetscViewerSocketOpen(PETSC_COMM_WORLD,0,PETSC_DEFAULT,&socketviewer);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: socket opened! \n");CHKERRQ(ierr);
// Receive n from Matlab
IntReceive(socketviewer, &nind);
n = *nind;
// Receive iter from Matlab
IntReceive(socketviewer, &iterind);
iter = *iterind;
}else{
ierr = PetscOptionsGetInt(PETSC_NULL,"-ngrid",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-niter",&iter,PETSC_NULL);CHKERRQ(ierr);
}
/////////////////////////////////////////////////////////////////////////////////////
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of grid is %d \n", n);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of iteration is %d \n", iter);
Mixnorm = malloc(iter*sizeof(PetscScalar));
dx = 1.0/n;
dy = 1.0/n;
dx2 = dx/2-dx/1e6;
dy2 = dy/2-dy/1e6;
npt = 5;
pmax = 4e6;
puse = pmax;
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated buffer size (per processer) %f Mbytes \n", pmax*1.0/1e6*8*16 );
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated variable size %f Mbytes\n", 1.0*n*n/1e6*8*2);
/////////////////////////////////////////////////////////////////////////////////////
ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE ,n,&tempvec);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(tempvec,&Istart,&Iend);CHKERRQ(ierr);
localsize = Iend-Istart;
ierr = VecDestroy(tempvec);CHKERRQ(ierr);
/////////////////////////////////////////////////////////////////////////////////////
// Create initial vector
Vec x0;
PetscScalar *x0array;
x0array = malloc((localsize)*n*sizeof(PetscScalar));
k = 0;
for(i=Istart;i<Iend;i++){
for(j=0;j<n;j++){
*(x0array+k) = cos(2*M_PI*(dx/2+i*dx));
//*(x0array+k) = cos(2*M_PI*(dy/2+j*dy));
k++;
}
}
ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,n*localsize,PETSC_DECIDE,x0array,&x0);CHKERRQ(ierr);
ierr = VecDuplicate(x0,&x);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,pmax*npt,&y0);CHKERRQ(ierr);
ierr = VecNorm(x0,NORM_2,Mixnorm); CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"PETSC: initial norm= %f \n",*(Mixnorm+0)/n );
///////////////////////////////////////////////////////////////////////////
// Map Center Points
PetscInt *NzindJ,*idx,*idy,*idp;
PetscScalar *CenterX,*CenterY,*VecVal,*pty;
PetscScalar *ShiftX,*ShiftY,CX,CY, *yarray;
IS isx,isy;
VecScatter ctx;
CenterX = malloc(npt*sizeof(PetscScalar));
CenterY = malloc(npt*sizeof(PetscScalar));
ShiftX = malloc(npt*sizeof(PetscScalar));
ShiftY = malloc(npt*sizeof(PetscScalar));
VecVal = malloc(npt*sizeof(PetscScalar));
yarray = malloc(pmax*sizeof(PetscScalar));
NzindJ = malloc(pmax*npt*sizeof(PetscInt));
idx = malloc(pmax*npt*sizeof(PetscInt));
idy = malloc(pmax*npt*sizeof(PetscInt));
idp = malloc(pmax*sizeof(PetscInt));
*(ShiftX+0) = 0;
*(ShiftY+0) = 0;
*(ShiftX+1) = -dx2;
*(ShiftY+1) = -dy2;
*(ShiftX+2) = dx2;
*(ShiftY+2) = -dy2;
*(ShiftX+3) = -dx2;
*(ShiftY+3) = dy2;
*(ShiftX+4) = dy2;
*(ShiftY+4) = dx2;
//*(ShiftX+5) = 0;
//*(ShiftY+5) = -dy2;
//*(ShiftX+6) = -dx2;
//*(ShiftY+6) = 0;
//*(ShiftX+7) = dx2;
//*(ShiftY+7) = 0;
//*(ShiftX+8) = 0;
//*(ShiftY+9) = dy2;
for(i=0;i<npt*pmax;i++){ *(idy+i)=i; }
ISCreateGeneralWithArray(PETSC_COMM_SELF,npt*pmax,idy,&isy);
vinda = &x0;
vindb = &x;
sprintf(fname, "mixnorm_%d_%d",n,iter);
ierr =PetscPrintf(PETSC_COMM_WORLD,"\n iter norm time unit time\n");CHKERRQ(ierr);
ierr =PetscFOpen(PETSC_COMM_WORLD,fname,"w",&fidoutput);CHKERRQ(ierr);
for(niter=0;niter<iter;niter++){
ierr = PetscGetTime(&v1);CHKERRQ(ierr);
l = 0; p = 0;
if (n*localsize-l<=pmax){puse = n*localsize-l;}else{puse=pmax;}
for(i=Istart;i<Iend;i++){
for(j=0;j<n;j++){
CX = dx2+i*dx;
CY = dy2+j*dy;
for(k=0;k<npt;k++){
*(CenterX+k) = CX + *(ShiftX+k);
*(CenterY+k) = CY + *(ShiftY+k);
InverseStandardMap((CenterX+k),(CenterY+k));
*(NzindJ+p*npt +k) = floor(*(CenterX+k)*n)*n + floor(*(CenterY+k)*n);
}
*(idp+p) = Istart*n+ l;
if(p>=puse-1){
ierr = ISCreateGeneralWithArray(PETSC_COMM_WORLD,npt*puse,NzindJ,&isx);CHKERRQ(ierr);
for(m=0;m<npt*puse;m++){ *(idy+m)=m; }
ierr = ISCreateGeneralWithArray(PETSC_COMM_SELF,npt*puse,idy,&isy);CHKERRQ(ierr);
ierr = VecScatterCreate(*vinda,isx,y0,isy,&ctx);CHKERRQ(ierr);
ierr = VecScatterBegin(*vinda,y0,INSERT_VALUES,SCATTER_FORWARD,ctx);CHKERRQ(ierr);
ierr = VecScatterEnd(*vinda,y0,INSERT_VALUES,SCATTER_FORWARD,ctx);CHKERRQ(ierr);
ierr = VecScatterDestroy(ctx);
ierr = VecGetArray(y0,&pty);CHKERRQ(ierr);
for(m=0;m<puse;m++){
for(m2=0;m2<npt;m2++){
*(yarray+m) = *(yarray+m)+*(pty+m*npt+m2);
}
*(yarray+m) = *(yarray+m)/npt;
}
VecRestoreArray(y0,&pty);
VecSetValues(*vindb,puse,idp,yarray,INSERT_VALUES);
for(m=0;m<pmax;m++){*(yarray+m) = 0; }
p = 0;
if (n*localsize-l<=pmax){puse = n*localsize-l-1;}else{puse=pmax;}
}else{p++;}
l++;
}
}
VecAssemblyBegin(*vindb);
VecAssemblyEnd(*vindb);
vindc = vindb;
vindb = vinda;
vinda = vindc;
//ierr = VecCopy(x,x0);CHKERRQ(ierr);
ierr = VecNorm(*vinda,NORM_2,Mixnorm+niter); CHKERRQ(ierr);
*(Mixnorm+niter) = *(Mixnorm+niter)/n;
ierr = PetscGetTime(&v2);CHKERRQ(ierr);
elapsed_time = v2 - v1;
PetscPrintf(PETSC_COMM_WORLD," %d %f %f %f \n",niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 );
PetscFPrintf(PETSC_COMM_WORLD,fidoutput," %d %f %f %f\n"
,niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 );
}
PetscFClose(PETSC_COMM_WORLD,fidoutput);
///////////////////////////////////////////////////////////////////////////
if(withMatlab==1){
VecView(x0,socketviewer);
PetscScalarView(iter,Mixnorm,socketviewer);
}
free(CenterX);
free(CenterY);
free(ShiftX);
free(ShiftY);
free(x0array);
free(idx);
free(idy);
free(idp);
free(yarray);
free(NzindJ);
free(Mixnorm);
ierr = VecDestroy(x0);CHKERRQ(ierr);
ierr = VecDestroy(x);CHKERRQ(ierr);
ierr = VecDestroy(y0);CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"Done!");
//////////////////////////////////////////////////////////////////////////////////////
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
int main(int argc,char **args)
{
PetscInt rank,size,npt;
PetscScalar dx,dy,cx,cy;
PetscErrorCode ierr;
Vec x,x0,tempvec, *vinda,*vindb,*vindc;
PetscInt i,j,k,n,n2,pmax,puse,Istart,Iend,localsize,niter;
PetscScalar **x0array, **aarray,**barray;
PetscInt *cacheInt;
PetscScalar *cacheScalar;
DA myDA;
PetscScalar *Mixnorm;
PetscInt iter,*iterind,*nind;
FILE *fidoutput, *fidtimelog;
char fname[50],ftimelog[50];
PetscViewer socketviewer;
PetscInt withMatlab, doFFT, doSmoothing;
PetscTruth Matlabflag, FFTflag, Smoothingflag;
PetscInt timelogcount;
MPI_Status status;
PetscLogDouble v1,v2,elapsed_time;
timelogcount = 0;
PetscInitialize(&argc,&args,(char *)0,help);
MPI_Comm_size(PETSC_COMM_WORLD,&size);
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nPETSC: Petsc Initializes successfully! \n");
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: comm_size is %d \n", size);
ierr = PetscOptionsGetInt(PETSC_NULL,"-withMatlab",&withMatlab,&Matlabflag);CHKERRQ(ierr);
if (Matlabflag == PETSC_FALSE){withMatlab = 0;}else{withMatlab = 1;}
ierr = PetscOptionsGetInt(PETSC_NULL,"-doFFT",&doFFT,&FFTflag);CHKERRQ(ierr);
if (FFTflag == PETSC_FALSE){doFFT = 0;}else{doFFT = 1;}
ierr = PetscOptionsGetInt(PETSC_NULL,"-doSmoothing",&doSmoothing,&Smoothingflag);CHKERRQ(ierr);
if (Smoothingflag == PETSC_FALSE){doSmoothing = 0;}else{doSmoothing = 1;}
if(withMatlab==1){
// Rank 0 connects to socket, use default socket
PetscViewerSocketOpen(PETSC_COMM_WORLD,0,PETSC_DEFAULT,&socketviewer);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: socket opened! \n");CHKERRQ(ierr);
// Receive n from Matlab
IntReceive(socketviewer, &nind);
n = *nind;
// Receive iter from Matlab
IntReceive(socketviewer, &iterind);
iter = *iterind;
}else{
ierr = PetscOptionsGetInt(PETSC_NULL,"-ngrid",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-niter",&iter,PETSC_NULL);CHKERRQ(ierr);
}
/////////////////////////////////////////////////////////////////////////////////////
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of grid is %d \n", n);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of iteration is %d \n", iter);
Mixnorm = malloc(iter*sizeof(PetscScalar));
dx = 1.0/n;
dy = 1.0/n;
n2 = (PetscInt)(n*0.5);
npt = 5;
pmax = 5e5;
puse = pmax;
PetscInt logmax = 1000;
PetscScalar Timelog[logmax];
PetscLogDouble t1,t2;
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated buffer size (per processer) %f Mbytes \n", pmax*1.0/1e6*8*17 );
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated variable size %f Mbytes\n", 1.0*n*n/1e6*8*1);
/////////////////////////////////////////////////////////////////////////////////////
// ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE ,n,&tempvec);CHKERRQ(ierr);
// ierr = VecGetOwnershipRange(tempvec,&Istart,&Iend);CHKERRQ(ierr);
// localsize = Iend-Istart;
// ierr = VecDestroy(tempvec);CHKERRQ(ierr);
/////////////////////////////////////////////////////////////////////////////////////
if(doSmoothing==1){
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n\n\n\nPETSC: Now Do DACreate2d \n\n\n\n" );
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n\n\n\nPETSC: %d %d %d\n\n\n\n",n2,n,size);
DACreate2d(MPI_COMM_WORLD,DA_XYPERIODIC,DA_STENCIL_BOX,n2,n,1,size,1,2,PETSC_NULL,PETSC_NULL,&myDA);
DACreateGlobalVector(myDA,&x0);
DAGetCorners(myDA,PETSC_NULL,&Istart,PETSC_NULL,PETSC_NULL,&localsize,PETSC_NULL);
Iend = Istart+localsize;
}else{
ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE ,n,&tempvec);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(tempvec,&Istart,&Iend);CHKERRQ(ierr);
localsize = Iend-Istart;
ierr = VecDestroy(tempvec);CHKERRQ(ierr);
VecCreateMPI(PETSC_COMM_WORLD,localsize*n2,PETSC_DETERMINE ,&x0);
}
//ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n\n\n\nPETSC: So far so good\n\n\n\n");
VecGetArray2d(x0,n2,localsize,0,0,&x0array);
// Create initial vector
for(j=0;j<localsize;j++){
for(i=0;i<n2;i++){
cx = (Istart+j+0.5)*dx;
x0array[i][j] = cos(2*M_PI*cx);
}
}
VecRestoreArray2d(x0,n2,localsize,0,0,&x0array);
ierr = VecDuplicate(x0,&x);CHKERRQ(ierr);
ierr = VecNorm(x0,NORM_2,Mixnorm); CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"PETSC: initial norm= %f \n",*(Mixnorm+0)/n );
vinda = &x0;
vindb = &x;
sprintf(fname, "mixnorm_%d_%d",n,iter);
ierr =PetscPrintf(PETSC_COMM_WORLD,"\n iter norm time unit time\n");CHKERRQ(ierr);
ierr =PetscFOpen(PETSC_COMM_WORLD,fname,"w",&fidoutput);CHKERRQ(ierr);
///////////////////////////////////////////////////////////////////////////////////////////////////
// Memory allocation for the iteration scheme
// cacheInt = malloc(1*pmax*sizeof(PetscInt));
// cacheScalar = malloc(2*pmax*sizeof(PetscScalar));
cacheInt = malloc(2*pmax*sizeof(PetscInt));
cacheScalar = malloc(2*pmax*sizeof(PetscScalar));
///////////////////////////////////////////////////////////////////////////////////////////////////
// Iteration here!
for(niter=0;niter<iter;niter++){
ierr = PetscGetTime(&v1);CHKERRQ(ierr);
// BackwardAverage(vinda, vindb, cacheInt, cacheScalar, n, npt, pmax, Istart,Iend);
// BackwardAverageR(vinda, vindb, cacheInt, cacheScalar, n, npt, pmax, Istart,Iend);
BackwardAverageRL(vinda, vindb, cacheInt, cacheScalar, n, npt, pmax, Istart,Iend);
vindc = vindb;
vindb = vinda;
vinda = vindc;
// if(doSmoothing==1){Smoothing(vinda, vindb,n, myDA, Istart,Iend);}
ierr = PetscGetTime(&v2);CHKERRQ(ierr);
//vindc = vindb;
//vindb = vinda;
//vinda = vindc;
ierr = VecNorm(*vinda,NORM_2,Mixnorm+niter); CHKERRQ(ierr);
*(Mixnorm+niter) = *(Mixnorm+niter)/n;
elapsed_time = v2 - v1;
PetscPrintf(PETSC_COMM_WORLD," %d %f %f %f \n",niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 );
PetscFPrintf(PETSC_COMM_WORLD,fidoutput," %d %f %f %f\n"
,niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 );
}
////////////////////////////////////////////////////////////////////////////////////////////////////
//Change oremtation of vector
VecGetArray2d(*vinda,n2,localsize,0,0,&aarray);
VecGetArray2d(*vindb,localsize,n2,0,0,&barray);
for(j=0;j<localsize;j++){
for(i=0;i<n2;i++){
barray[j][i] = aarray[i][j];
}
}
VecRestoreArray2d(*vinda,n2,localsize,0,0,&aarray);
VecRestoreArray2d(*vindb,localsize,n2,0,0,&barray);
vindc = vindb;
vindb = vinda;
vinda = vindc;
////////////////////////////////////////////////////////////////////////////////////////////////////
// FFT part
if(doFFT==1){FFT2D(*vinda,*vindb, localsize, n, Istart,Iend, iter,doSmoothing);}
////////////////////////////////////////////////////////////////////////////////////////////////////
/*
if(rank==0){
sprintf(ftimelog, "timelog_%d_%d",n,iter);
fidtimelog = fopen(ftimelog,"w");
for(i=0;i<timelogcount;i++){ fprintf(fidtimelog,"%f ",Timelog[i]); }
fprintf(fidtimelog,"\n ");
for(j = 1;j<size;j++){
MPI_Recv(Timelog,timelogcount,MPI_DOUBLE,j,j,PETSC_COMM_WORLD,&status);
for(i=0;i<timelogcount;i++){ fprintf(fidtimelog,"%f ",Timelog[i]); }
fprintf(fidtimelog,"\n ");
}
fclose(fidtimelog);
}else{
MPI_Send(Timelog ,timelogcount,MPI_DOUBLE,0,rank,PETSC_COMM_WORLD);
}
PetscFClose(PETSC_COMM_WORLD,fidoutput);
*/
///////////////////////////////////////////////////////////////////////////
if(withMatlab==1){
VecView(*vinda,socketviewer);
PetscScalarView(iter,Mixnorm,socketviewer);
}
// free(x0array);
free(Mixnorm);
free(cacheInt);
free(cacheScalar);
ierr = VecDestroy(x0);CHKERRQ(ierr);
ierr = VecDestroy(x);CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"Done!");
//////////////////////////////////////////////////////////////////////////////////////
ierr = PetscFinalize();CHKERRQ(ierr);
int main(int argc,char **args)
{
/*PETSc Mat Object */
Mat pMat;
/* Input matrix market file and output PETSc binary file */
char inputFile[128],outputFile[128],buf[128];
/* number rows, columns, non zeros etc */
int i,j,m,n,nnz,ierr,col,row;
/*We compute no of nozeros per row for PETSc Mat object pre-allocation*/
int *nnzPtr;
/*Maximum nonzero in nay row */
int maxNNZperRow=0;
/*Row number containing max non zero elements */
int maxRowNum = 0;
/*Just no of comments that will be ignore during successive read of file */
int numComments=0;
PetscScalar zero=0;
/* This is variable of type double */
PetscScalar val;
/*File handle for read and write*/
FILE* file;
/*File handle for writing nonzero elements distribution per row */
FILE *fileRowDist;
/*PETSc Viewer is used for writing PETSc Mat object in binary format */
PetscViewer view;
/*Just record time required for conversion */
PetscLogDouble t1,t2,elapsed_time;
/* MatrixMarket struct */
MM_typecode matcode;
/*Initialise PETSc lib */
PetscInitialize(&argc,&args,(char *)0,PETSC_NULL);
/* Just record time */
//ierr = PetscGetTime(&t1); CHKERRQ(ierr);
/*Get name of matrix market file from command line options and Open file*/
ierr = PetscOptionsGetString(PETSC_NULL,"-fin",inputFile,127,PETSC_NULL); CHKERRQ(ierr);
ierr = PetscFOpen(PETSC_COMM_SELF,inputFile,"r",&file); CHKERRQ(ierr);
if (mm_read_banner(file, &matcode)) {
PetscPrintf(PETSC_COMM_SELF, "Could not read Matrix Market banner.\n");
exit(1);
}
/********************* MM_typecode query fucntions ***************************/
/* #define mm_is_matrix(typecode) ((typecode)[0]=='M') */
/* #define mm_is_sparse(typecode) ((typecode)[1]=='C') */
/* #define mm_is_coordinate(typecode)((typecode)[1]=='C') */
/* #define mm_is_dense(typecode) ((typecode)[1]=='A') */
/* #define mm_is_array(typecode) ((typecode)[1]=='A') */
/* #define mm_is_complex(typecode) ((typecode)[2]=='C') */
/* #define mm_is_real(typecode) ((typecode)[2]=='R') */
/* #define mm_is_pattern(typecode) ((typecode)[2]=='P') */
/* #define mm_is_integer(typecode) ((typecode)[2]=='I') */
/* #define mm_is_symmetric(typecode)((typecode)[3]=='S') */
/* #define mm_is_general(typecode) ((typecode)[3]=='G') */
/* #define mm_is_skew(typecode) ((typecode)[3]=='K') */
/* #define mm_is_hermitian(typecode)((typecode)[3]=='H') */
/* int mm_is_valid(MM_typecode matcode); */
/* Do not convert pattern matrices */
if (mm_is_pattern(matcode)) {
ierr = PetscPrintf(PETSC_COMM_SELF, "%s: Pattern matrix -- skipping.\n", inputFile);
exit(0);
}
/* find out size of sparse matrix .... */
/*Reads size of sparse matrix from matrix market file */
int ret_code;
if ((ret_code = mm_read_mtx_crd_size(file, &m, &n, &nnz)) !=0)
exit(1);
ierr = PetscPrintf(PETSC_COMM_SELF, "%s: ROWS = %d, COLUMNS = %d, NO OF NON-ZEROS = %d\n",inputFile,m,n,nnz);
/* Only consider square matrices */
if (m != n) {
ierr = PetscPrintf(PETSC_COMM_SELF, "%s: Nonsquare matrix -- skipping.\n", inputFile);
exit(0);
}
ierr = MatCreate(PETSC_COMM_WORLD,&pMat);CHKERRQ(ierr);
ierr = MatSetFromOptions(pMat);CHKERRQ(ierr);
//ierr = MatSetOption(pMat, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE); CHKERRQ(ierr);
if (mm_is_symmetric(matcode)) {
ierr = MatSetOption(pMat,MAT_SYMMETRIC,PETSC_TRUE); CHKERRQ(ierr);
ierr = MatSetOption(pMat,MAT_SYMMETRY_ETERNAL,PETSC_TRUE); CHKERRQ(ierr);
}
ierr = MatSetSizes(pMat,PETSC_DECIDE,PETSC_DECIDE,m,n);CHKERRQ(ierr);
ierr = MatSetUp(pMat);CHKERRQ(ierr);
//printf("\n MAX NONZERO FOR ANY ROW ARE : %d & ROW NUM IS : %d", maxNNZperRow, maxRowNum );
/* Its important to pre-allocate memory by passing max non zero for any row in the matrix */
//ierr = MatCreateSeqAIJ(PETSC_COMM_WORLD,m,n,maxNNZperRow,PETSC_NULL,&pMat);
/* OR we can also pass row distribution of nozero elements for every row */
/* ierr = MatCreateSeqAIJ(PETSC_COMM_WORLD,m,n,0,nnzPtr,&pMat);*/
/*Now Set matrix elements values form matrix market file */
for (i=0; i < m; i++){
for (j = 0; j < n; j++) {
if (i != j) continue;
ierr = MatSetValues(pMat,1,&i,1,&j,&zero,INSERT_VALUES); CHKERRQ(ierr);
}
}
for (i=0; i<nnz; i++)
{
/*Read matrix element from matrix market file*/
fscanf(file,"%d %d %le\n",&row,&col,&val);
/*In matrix market format, rows and columns starts from 1 */
row = row-1; col = col-1 ;
/* For every non zero element,insert that value at row,col position */
ierr = MatSetValues(pMat,1,&row,1,&col,&val,INSERT_VALUES); CHKERRQ(ierr);
}
fclose(file);
/*Matrix Read Complete */
ierr = PetscPrintf(PETSC_COMM_SELF,"%s MATRIX READ...DONE!\n", inputFile);
/*Now assemeble the matrix */
ierr = MatAssemblyBegin(pMat,MAT_FINAL_ASSEMBLY);
ierr = MatAssemblyEnd(pMat,MAT_FINAL_ASSEMBLY);
/* Now open output file for writing into PETSc Binary FOrmat*/
ierr = PetscOptionsGetString(PETSC_NULL,"-fout",outputFile,127,PETSC_NULL);CHKERRQ(ierr);
/*With the PETSc Viewer write output to File*/
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,outputFile,FILE_MODE_WRITE,&view);CHKERRQ(ierr);
/*Matview will dump the Mat object to binary file */
ierr = MatView(pMat,view);CHKERRQ(ierr);
//ierr = MatView(pMat,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF,"%s PETSC MATRIX STORED\n", outputFile);
/* Destroy the data structure */
ierr = PetscViewerDestroy(&view);CHKERRQ(ierr);
ierr = MatDestroy(&pMat);CHKERRQ(ierr);
/*Just for statistics*/
/*
ierr = PetscGetTime(&t2);CHKERRQ(ierr);
elapsed_time = t2 - t1;
ierr = PetscPrintf(PETSC_COMM_SELF,"ELAPSE TIME: %g\n",elapsed_time);CHKERRQ(ierr);
*/
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
int main(int argc, char *argv[])
{
PetscErrorCode ierr;
PetscBool flg;
PetscMPIInt rank, size;
int i, status;
idx_t ni,isize,*vtxdist, *xadj, *adjncy, *vwgt, *part;
idx_t wgtflag=0, numflag=0, ncon=1, ndims=3, edgecut=0;
idx_t options[5];
real_t *xyz, *tpwgts, ubvec[1];
MPI_Comm comm;
FILE *fp;
char fname[PETSC_MAX_PATH_LEN],prefix[PETSC_MAX_PATH_LEN] = "";
PetscInitialize(&argc,&argv,NULL,help);
#if defined(PETSC_USE_64BIT_INDICES)
ierr = PetscPrintf(PETSC_COMM_WORLD,"This example only works with 32 bit indices\n");
PetscFinalize();
#endif
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
MPI_Comm_size(PETSC_COMM_WORLD,&size);
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,NULL,"Parmetis test options","");CHKERRQ(ierr);
ierr = PetscOptionsString("-prefix","Path and prefix of test file","",prefix,prefix,sizeof(prefix),&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_USER,"Must specify -prefix");CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
ierr = PetscMalloc1(size+1,&vtxdist);CHKERRQ(ierr);
ierr = PetscSNPrintf(fname,sizeof(fname),"%s.%d.graph",prefix,rank);CHKERRQ(ierr);
ierr = PetscFOpen(PETSC_COMM_SELF,fname,"r",&fp);CHKERRQ(ierr);
fread(vtxdist, sizeof(idx_t), size+1, fp);
ni = vtxdist[rank+1]-vtxdist[rank];
ierr = PetscMalloc1(ni+1,&xadj);CHKERRQ(ierr);
fread(xadj, sizeof(idx_t), ni+1, fp);
ierr = PetscMalloc1(xadj[ni],&adjncy);CHKERRQ(ierr);
for (i=0; i<ni; i++) fread(&adjncy[xadj[i]], sizeof(idx_t), xadj[i+1]-xadj[i], fp);
ierr = PetscFClose(PETSC_COMM_SELF,fp);CHKERRQ(ierr);
ierr = PetscSNPrintf(fname,sizeof(fname),"%s.%d.graph.xyz",prefix,rank);CHKERRQ(ierr);
ierr = PetscFOpen(PETSC_COMM_SELF,fname,"r",&fp);CHKERRQ(ierr);
ierr = PetscMalloc3(ni*ndims,&xyz,ni,&part,size,&tpwgts);CHKERRQ(ierr);
fread(xyz, sizeof(real_t), ndims*ni, fp);
ierr = PetscFClose(PETSC_COMM_SELF,fp);CHKERRQ(ierr);
vwgt = NULL;
for (i = 0; i < size; i++) tpwgts[i] = 1. / size;
isize = size;
ubvec[0] = 1.05;
options[0] = 1;
options[1] = 2;
options[2] = 15;
options[3] = 0;
options[4] = 0;
ierr = MPI_Comm_dup(MPI_COMM_WORLD, &comm);CHKERRQ(ierr);
status = ParMETIS_V3_PartGeomKway(vtxdist, xadj, adjncy, vwgt,
NULL, &wgtflag, &numflag, &ndims, xyz, &ncon, &isize, tpwgts, ubvec,
options, &edgecut, part, &comm);CHKERRQPARMETIS(status);
ierr = MPI_Comm_free(&comm);CHKERRQ(ierr);
ierr = PetscFree(vtxdist);CHKERRQ(ierr);
ierr = PetscFree(xadj);CHKERRQ(ierr);
ierr = PetscFree(adjncy);CHKERRQ(ierr);
ierr = PetscFree3(xyz,part,tpwgts);CHKERRQ(ierr);
PetscFinalize();
return 0;
}
/*
Write all fields that have been provided to the viewer
Multi-block XML format with binary appended data.
*/
PetscErrorCode DMPlexVTKWriteAll_VTU(DM dm,PetscViewer viewer)
{
MPI_Comm comm;
PetscViewer_VTK *vtk = (PetscViewer_VTK*)viewer->data;
PetscViewerVTKObjectLink link;
FILE *fp;
PetscMPIInt rank,size,tag;
PetscErrorCode ierr;
PetscInt dim,cellHeight,cStart,cEnd,vStart,vEnd,cMax,numLabelCells,hasLabel,c,v,r,i;
PieceInfo piece,*gpiece = NULL;
void *buffer = NULL;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
#if defined(PETSC_USE_COMPLEX)
SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"Complex values not supported");
#endif
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
tag = ((PetscObject)viewer)->tag;
ierr = PetscFOpen(comm,vtk->filename,"wb",&fp);CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp,"<?xml version=\"1.0\"?>\n");CHKERRQ(ierr);
#if defined(PETSC_WORDS_BIGENDIAN)
ierr = PetscFPrintf(comm,fp,"<VTKFile type=\"UnstructuredGrid\" version=\"0.1\" byte_order=\"BigEndian\">\n");CHKERRQ(ierr);
#else
ierr = PetscFPrintf(comm,fp,"<VTKFile type=\"UnstructuredGrid\" version=\"0.1\" byte_order=\"LittleEndian\">\n");CHKERRQ(ierr);
#endif
ierr = PetscFPrintf(comm,fp," <UnstructuredGrid>\n");CHKERRQ(ierr);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
ierr = DMPlexGetStratumSize(dm, "vtk", 1, &numLabelCells);CHKERRQ(ierr);
hasLabel = numLabelCells > 0 ? PETSC_TRUE : PETSC_FALSE;
piece.nvertices = vEnd - vStart;
piece.ncells = 0;
piece.nconn = 0;
for (c = cStart; c < cEnd; ++c) {
PetscInt *closure = NULL;
PetscInt closureSize;
if (hasLabel) {
PetscInt value;
ierr = DMPlexGetLabelValue(dm, "vtk", c, &value);CHKERRQ(ierr);
if (value != 1) continue;
}
ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
for (v = 0; v < closureSize*2; v += 2) {
if ((closure[v] >= vStart) && (closure[v] < vEnd)) piece.nconn++;
}
ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
piece.ncells++;
}
if (!rank) {ierr = PetscMalloc(size*sizeof(piece),&gpiece);CHKERRQ(ierr);}
ierr = MPI_Gather((PetscInt*)&piece,sizeof(piece)/sizeof(PetscInt),MPIU_INT,(PetscInt*)gpiece,sizeof(piece)/sizeof(PetscInt),MPIU_INT,0,comm);CHKERRQ(ierr);
/*
* Write file header
*/
if (!rank) {
PetscInt boffset = 0;
for (r=0; r<size; r++) {
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <Piece NumberOfPoints=\"%D\" NumberOfCells=\"%D\">\n",gpiece[r].nvertices,gpiece[r].ncells);CHKERRQ(ierr);
/* Coordinate positions */
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <Points>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <DataArray type=\"%s\" Name=\"Position\" NumberOfComponents=\"3\" format=\"appended\" offset=\"%D\" />\n",precision,boffset);CHKERRQ(ierr);
boffset += gpiece[r].nvertices*3*sizeof(PetscScalar) + sizeof(int);
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," </Points>\n");CHKERRQ(ierr);
/* Cell connectivity */
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <Cells>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <DataArray type=\"Int32\" Name=\"connectivity\" NumberOfComponents=\"1\" format=\"appended\" offset=\"%D\" />\n",boffset);CHKERRQ(ierr);
boffset += gpiece[r].nconn*sizeof(PetscInt) + sizeof(int);
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <DataArray type=\"Int32\" Name=\"offsets\" NumberOfComponents=\"1\" format=\"appended\" offset=\"%D\" />\n",boffset);CHKERRQ(ierr);
boffset += gpiece[r].ncells*sizeof(PetscInt) + sizeof(int);
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <DataArray type=\"UInt8\" Name=\"types\" NumberOfComponents=\"1\" format=\"appended\" offset=\"%D\" />\n",boffset);CHKERRQ(ierr);
boffset += gpiece[r].ncells*sizeof(unsigned char) + sizeof(int);
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," </Cells>\n");CHKERRQ(ierr);
/*
* Cell Data headers
*/
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <CellData>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <DataArray type=\"Int32\" Name=\"Rank\" NumberOfComponents=\"1\" format=\"appended\" offset=\"%D\" />\n",boffset);CHKERRQ(ierr);
boffset += gpiece[r].ncells*sizeof(int) + sizeof(int);
/* all the vectors */
for (link=vtk->link; link; link=link->next) {
Vec X = (Vec)link->vec;
PetscInt bs,nfields,field;
const char *vecname = "";
if ((link->ft != PETSC_VTK_CELL_FIELD) && (link->ft != PETSC_VTK_CELL_VECTOR_FIELD)) continue;
if (((PetscObject)X)->name || link != vtk->link) { /* If the object is already named, use it. If it is past the first link, name it to disambiguate. */
ierr = PetscObjectGetName((PetscObject)X,&vecname);CHKERRQ(ierr);
}
ierr = PetscSectionGetDof(dm->defaultSection,cStart,&bs);CHKERRQ(ierr);
ierr = PetscSectionGetNumFields(dm->defaultSection,&nfields);CHKERRQ(ierr);
for (field=0,i=0; field<(nfields?nfields:1); field++) {
PetscInt fbs,j;
const char *fieldname = NULL;
char buf[256];
if (nfields) { /* We have user-defined fields/components */
ierr = PetscSectionGetFieldDof(dm->defaultSection,cStart,field,&fbs);CHKERRQ(ierr);
ierr = PetscSectionGetFieldName(dm->defaultSection,field,&fieldname);CHKERRQ(ierr);
} else fbs = bs; /* Say we have one field with 'bs' components */
if (!fieldname) {
ierr = PetscSNPrintf(buf,sizeof(buf),"CellField%D",field);CHKERRQ(ierr);
fieldname = buf;
}
for (j=0; j<fbs; j++) {
ierr = PetscFPrintf(comm,fp," <DataArray type=\"%s\" Name=\"%s%s.%D\" NumberOfComponents=\"1\" format=\"appended\" offset=\"%D\" />\n",precision,vecname,fieldname,j,boffset);CHKERRQ(ierr);
boffset += gpiece[r].ncells*sizeof(PetscScalar) + sizeof(int);
i++;
}
}
if (i != bs) SETERRQ2(comm,PETSC_ERR_PLIB,"Total number of field components %D != block size %D",i,bs);
}
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," </CellData>\n");CHKERRQ(ierr);
/*
* Point Data headers
*/
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <PointData>\n");CHKERRQ(ierr);
for (link=vtk->link; link; link=link->next) {
Vec X = (Vec)link->vec;
PetscInt bs,nfields,field;
const char *vecname = "";
if ((link->ft != PETSC_VTK_POINT_FIELD) && (link->ft != PETSC_VTK_POINT_VECTOR_FIELD)) continue;
if (((PetscObject)X)->name || link != vtk->link) { /* If the object is already named, use it. If it is past the first link, name it to disambiguate. */
ierr = PetscObjectGetName((PetscObject)X,&vecname);CHKERRQ(ierr);
}
ierr = PetscSectionGetDof(dm->defaultSection,vStart,&bs);CHKERRQ(ierr);
ierr = PetscSectionGetNumFields(dm->defaultSection,&nfields);CHKERRQ(ierr);
for (field=0,i=0; field<(nfields?nfields:1); field++) {
PetscInt fbs,j;
const char *fieldname = NULL;
char buf[256];
if (nfields) { /* We have user-defined fields/components */
ierr = PetscSectionGetFieldDof(dm->defaultSection,vStart,field,&fbs);CHKERRQ(ierr);
ierr = PetscSectionGetFieldName(dm->defaultSection,field,&fieldname);CHKERRQ(ierr);
} else fbs = bs; /* Say we have one field with 'bs' components */
if (!fieldname) {
ierr = PetscSNPrintf(buf,sizeof(buf),"PointField%D",field);CHKERRQ(ierr);
fieldname = buf;
}
for (j=0; j<fbs; j++) {
ierr = PetscFPrintf(comm,fp," <DataArray type=\"%s\" Name=\"%s%s.%D\" NumberOfComponents=\"1\" format=\"appended\" offset=\"%D\" />\n",precision,vecname,fieldname,j,boffset);CHKERRQ(ierr);
boffset += gpiece[r].nvertices*sizeof(PetscScalar) + sizeof(int);
}
}
}
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," </PointData>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," </Piece>\n");CHKERRQ(ierr);
}
}
ierr = PetscFPrintf(comm,fp," </UnstructuredGrid>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp," <AppendedData encoding=\"raw\">\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp,"_");CHKERRQ(ierr);
if (!rank) {
PetscInt maxsize = 0;
for (r=0; r<size; r++) {
maxsize = PetscMax(maxsize, (PetscInt) (gpiece[r].nvertices*3*sizeof(PetscScalar)));
maxsize = PetscMax(maxsize, (PetscInt) (gpiece[r].ncells*sizeof(PetscScalar)));
maxsize = PetscMax(maxsize, (PetscInt) (gpiece[r].nconn*sizeof(PetscVTKInt)));
}
ierr = PetscMalloc(maxsize,&buffer);CHKERRQ(ierr);
}
for (r=0; r<size; r++) {
if (r == rank) {
PetscInt nsend;
{ /* Position */
const PetscScalar *x;
PetscScalar *y = NULL;
Vec coords;
nsend = piece.nvertices*3;
ierr = DMGetCoordinatesLocal(dm,&coords);CHKERRQ(ierr);
ierr = VecGetArrayRead(coords,&x);CHKERRQ(ierr);
if (dim != 3) {
ierr = PetscMalloc(piece.nvertices*3*sizeof(PetscScalar),&y);CHKERRQ(ierr);
for (i=0; i<piece.nvertices; i++) {
y[i*3+0] = x[i*dim+0];
y[i*3+1] = (dim > 1) ? x[i*dim+1] : 0;
y[i*3+2] = 0;
}
}
ierr = TransferWrite(viewer,fp,r,0,y ? y : x,buffer,nsend,PETSC_SCALAR,tag);CHKERRQ(ierr);
ierr = PetscFree(y);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(coords,&x);CHKERRQ(ierr);
}
{ /* Connectivity, offsets, types */
PetscVTKInt *connectivity = NULL,*offsets;
PetscVTKType *types;
ierr = DMPlexGetVTKConnectivity(dm,&piece,&connectivity,&offsets,&types);CHKERRQ(ierr);
ierr = TransferWrite(viewer,fp,r,0,connectivity,buffer,piece.nconn,PETSC_INT32,tag);CHKERRQ(ierr);
ierr = TransferWrite(viewer,fp,r,0,offsets,buffer,piece.ncells,PETSC_INT32,tag);CHKERRQ(ierr);
ierr = TransferWrite(viewer,fp,r,0,types,buffer,piece.ncells,PETSC_UINT8,tag);CHKERRQ(ierr);
ierr = PetscFree3(connectivity,offsets,types);CHKERRQ(ierr);
}
{ /* Owners (cell data) */
PetscVTKInt *owners;
ierr = PetscMalloc(piece.ncells*sizeof(PetscVTKInt),&owners);CHKERRQ(ierr);
for (i=0; i<piece.ncells; i++) owners[i] = rank;
ierr = TransferWrite(viewer,fp,r,0,owners,buffer,piece.ncells,PETSC_INT32,tag);CHKERRQ(ierr);
ierr = PetscFree(owners);CHKERRQ(ierr);
}
/* Cell data */
for (link=vtk->link; link; link=link->next) {
Vec X = (Vec)link->vec;
const PetscScalar *x;
PetscScalar *y;
PetscInt bs;
if ((link->ft != PETSC_VTK_CELL_FIELD) && (link->ft != PETSC_VTK_CELL_VECTOR_FIELD)) continue;
ierr = PetscSectionGetDof(dm->defaultSection,cStart,&bs);CHKERRQ(ierr);
ierr = VecGetArrayRead(X,&x);CHKERRQ(ierr);
ierr = PetscMalloc(piece.ncells*sizeof(PetscScalar),&y);CHKERRQ(ierr);
for (i=0; i<bs; i++) {
PetscInt cnt;
for (c=cStart,cnt=0; c<cEnd; c++) {
const PetscScalar *xpoint;
if (hasLabel) { /* Ignore some cells */
PetscInt value;
ierr = DMPlexGetLabelValue(dm, "vtk", c, &value);CHKERRQ(ierr);
if (value != 1) continue;
}
ierr = DMPlexPointLocalRead(dm,c,x,&xpoint);CHKERRQ(ierr);
y[cnt++] = xpoint[i];
}
if (cnt != piece.ncells) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Count does not match");
ierr = TransferWrite(viewer,fp,r,0,y,buffer,piece.ncells,PETSC_SCALAR,tag);CHKERRQ(ierr);
}
ierr = PetscFree(y);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(X,&x);CHKERRQ(ierr);
}
for (link=vtk->link; link; link=link->next) {
Vec X = (Vec)link->vec;
const PetscScalar *x;
PetscScalar *y;
PetscInt bs;
if ((link->ft != PETSC_VTK_POINT_FIELD) && (link->ft != PETSC_VTK_POINT_VECTOR_FIELD)) continue;
ierr = PetscSectionGetDof(dm->defaultSection,vStart,&bs);CHKERRQ(ierr);
ierr = VecGetArrayRead(X,&x);CHKERRQ(ierr);
ierr = PetscMalloc(piece.nvertices*sizeof(PetscScalar),&y);CHKERRQ(ierr);
for (i=0; i<bs; i++) {
PetscInt cnt;
for (v=vStart,cnt=0; v<vEnd; v++) {
const PetscScalar *xpoint;
ierr = DMPlexPointLocalRead(dm,v,x,&xpoint);CHKERRQ(ierr);
y[cnt++] = xpoint[i];
}
if (cnt != piece.nvertices) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Count does not match");
ierr = TransferWrite(viewer,fp,r,0,y,buffer,piece.nvertices,PETSC_SCALAR,tag);CHKERRQ(ierr);
}
ierr = PetscFree(y);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(X,&x);CHKERRQ(ierr);
}
} else if (!rank) {
ierr = TransferWrite(viewer,fp,r,0,NULL,buffer,gpiece[r].nvertices*3,PETSC_SCALAR,tag);CHKERRQ(ierr); /* positions */
ierr = TransferWrite(viewer,fp,r,0,NULL,buffer,gpiece[r].nconn,PETSC_INT32,tag);CHKERRQ(ierr); /* connectivity */
ierr = TransferWrite(viewer,fp,r,0,NULL,buffer,gpiece[r].ncells,PETSC_INT32,tag);CHKERRQ(ierr); /* offsets */
ierr = TransferWrite(viewer,fp,r,0,NULL,buffer,gpiece[r].ncells,PETSC_UINT8,tag);CHKERRQ(ierr); /* types */
ierr = TransferWrite(viewer,fp,r,0,NULL,buffer,gpiece[r].ncells,PETSC_INT32,tag);CHKERRQ(ierr); /* owner rank (cells) */
/* all cell data */
for (link=vtk->link; link; link=link->next) {
PetscInt bs;
if ((link->ft != PETSC_VTK_CELL_FIELD) && (link->ft != PETSC_VTK_CELL_VECTOR_FIELD)) continue;
ierr = PetscSectionGetDof(dm->defaultSection,cStart,&bs);CHKERRQ(ierr);
for (i=0; i<bs; i++) {
ierr = TransferWrite(viewer,fp,r,0,NULL,buffer,gpiece[r].ncells,PETSC_SCALAR,tag);CHKERRQ(ierr);
}
}
/* all point data */
for (link=vtk->link; link; link=link->next) {
PetscInt bs;
if ((link->ft != PETSC_VTK_POINT_FIELD) && (link->ft != PETSC_VTK_POINT_VECTOR_FIELD)) continue;
ierr = PetscSectionGetDof(dm->defaultSection,vStart,&bs);CHKERRQ(ierr);
for (i=0; i<bs; i++) {
ierr = TransferWrite(viewer,fp,r,0,NULL,buffer,gpiece[r].nvertices,PETSC_SCALAR,tag);CHKERRQ(ierr);
}
}
}
}
ierr = PetscFree(gpiece);CHKERRQ(ierr);
ierr = PetscFree(buffer);CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp,"\n </AppendedData>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp,"</VTKFile>\n");CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main( int argc, char **argv )
{
Mat hmat,smat;
PetscInt n=0,row,col, nev;
PetscScalar number;
PetscViewer view;
PetscBool flag;
int ierr,i;
char path[PETSC_MAX_PATH_LEN]="", temporary[PETSC_MAX_PATH_LEN]="";
char infilename[PETSC_MAX_PATH_LEN]="", outfilename[PETSC_MAX_PATH_LEN]="";
SlepcInitialize(&argc,&argv,(char*)0,help);
ierr = PetscOptionsGetString(PETSC_NULL,"-in",infilename,PETSC_MAX_PATH_LEN-1,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetString(PETSC_NULL,"-out",outfilename,PETSC_MAX_PATH_LEN-1,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscGetWorkingDirectory(path,PETSC_MAX_PATH_LEN);
if ( strcmp(infilename, "") == 0 ||
strcmp(outfilename, "") == 0 ||
n == 0 ){
SETERRQ(PETSC_COMM_WORLD,1,"Error in parameterlist.\nExample call: ./convertToPETScBin -in matrix.hmat -out matrix.dat -n 1912");
}
// number = 2.0 + 3.0 * PETSC_i;
// PetscPrintf(PETSC_COMM_WORLD,"number: %g i%g\n", number);
// std::cout << number
PetscPrintf(PETSC_COMM_WORLD,"processing %s into %s\n", infilename, outfilename);
ierr = MatCreateSeqAIJ(PETSC_COMM_WORLD,n,n,n,PETSC_NULL,&hmat);
FILE *file;
double real, imag;
PetscScalar complex;
PetscScalar myreal;
strcat(temporary, path); strcat(temporary, "/"); strcat(temporary, infilename); strcpy(infilename, temporary);
ierr = PetscFOpen(PETSC_COMM_SELF,infilename,"r",&file);CHKERRQ(ierr);
while ( fscanf(file,"%d %d %le %le\n",&row,&col,(double*)&real, (double*)&imag) != EOF ){
row = row - 1; col = col -1; // adjustment
if ( row == col ){
complex = real /*+ imag*PETSC_i*/;
ierr = MatSetValues(hmat,1,&row,1,&col,&complex,INSERT_VALUES);CHKERRQ(ierr);
}else{
complex = real /*+ imag*PETSC_i */;
ierr = MatSetValues(hmat,1,&row,1,&col,&complex,INSERT_VALUES);CHKERRQ(ierr);
complex = real /*- imag*PETSC_i */;
ierr = MatSetValues(hmat,1,&col,1,&row,&complex,INSERT_VALUES);CHKERRQ(ierr);
}
}
fclose(file);
ierr = MatAssemblyBegin(hmat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(hmat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatIsHermitian(hmat,0.00000000000001,&flag); CHKERRQ(ierr);
if ( !flag ){
SETERRQ(PETSC_COMM_WORLD,1,"Error. Output matrix is not hermitian.");
}
// writing
strcpy(temporary,"");
strcat(temporary, path); strcat(temporary, "/"); strcat(temporary, outfilename); strcpy(outfilename, temporary);
ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,outfilename,FILE_MODE_WRITE,&view);CHKERRQ(ierr);
ierr = MatView(hmat,view);CHKERRQ(ierr);
ierr = PetscViewersDestroy(&view);
ierr = SlepcFinalize();CHKERRQ(ierr);
return 0;
}
int main(int argc,char **args)
{
/*PETSc Mat Object */
Mat pMat;
/* Input matrix market file and output PETSc binary file */
char inputFile[128],outputFile[128],buf[128];
/* number rows, columns, non zeros etc */
int i, m,n,nnz,ierr,col,row;
/*We compute no of nozeros per row for PETSc Mat object pre-allocation*/
int *nnzPtr;
/*Maximum nonzero in nay row */
int maxNNZperRow=0;
/*Row number containing max non zero elements */
int maxRowNum = 0;
/*Just no of comments that will be ignore during successive read of file */
int numComments=0;
/* This is variable of type double */
PetscScalar val;
/*File handle for read and write*/
FILE* file;
/*File handle for writing nonzero elements distribution per row */
FILE *fileRowDist;
/*PETSc Viewer is used for writing PETSc Mat object in binary format */
PetscViewer view;
/*Just record time required for conversion */
PetscLogDouble t1,t2,elapsed_time;
/*Initialise PETSc lib */
PetscInitialize(&argc,&args,(char *)0,PETSC_NULL);
/* Just record time */
ierr = PetscGetTime(&t1); CHKERRQ(ierr);
/*Get name of matrix market file from command line options and Open file*/
ierr = PetscOptionsGetString(PETSC_NULL,"-fin",inputFile,127,PETSC_NULL);
ierr = PetscFOpen(PETSC_COMM_SELF,inputFile,"r",&file);
/* Just count the comment lines in the file */
while(1)
{
fgets(buf,128,file);
/*If line starts with %, its a comment */
if(buf[0] == '%')
{
printf("\n IGNORING COMMENT LINE : IGNORING....");
numComments++;
}
else
{
/*Set Pointer to Start of File */
fseek(file, 0, SEEK_SET );
int num = numComments;
/* and just move pointer to the entry in the file which indicates row nums, col nums and non zero elements */
while(num--)
fgets(buf,128,file);
break;
}
}
/*Reads size of sparse matrix from matrix market file */
fscanf(file,"%d %d %d\n",&m,&n,&nnz);
printf ("ROWS = %d, COLUMNS = %d, NO OF NON-ZEROS = %d\n",m,n,nnz);
/*Now we will calculate non zero elelments distribution per row */
nnzPtr = (int *) calloc (sizeof(int), m);
/*This is similar to calculate histogram or frequency of elements in the array */
for (i=0; !feof(file); i++)
{
fscanf(file,"%d %d %le\n",&row,&col,&val);
row = row-1; col = col-1 ;
nnzPtr[row]++;
}
printf("\n ROW DISTRIBUTION CALCULATED....WRITING TO THE FILE..!");
fflush(stdout);
/*Write row distribution to the file ROW_STR.dat */
fileRowDist = fopen ("ROW_DISTR.dat", "w");
for (i=0; i< m; i++)
{
fprintf(fileRowDist, "%d\t %d\n", i, nnzPtr[i]);
/*Find max num of of nonzero for any row of the matrix and that row number */
if( maxNNZperRow < nnzPtr[i] )
{ /*store max nonzero for any row*/
maxNNZperRow = nnzPtr[i];
/*row that contains max non zero elements*/
maxRowNum = i;
}
}
/*Close File */
fclose(fileRowDist);
printf("\n MAX NONZERO FOR ANY ROW ARE : %d & ROW NUM IS : %d", maxNNZperRow, maxRowNum );
/* Again set the file pointer the fist data record in matrix market file*
* Note that we can directly move ponts with fseek, but as this is text file
* we are simple reading line by line
*/
fseek(file, 0, SEEK_SET );
numComments++;
while(numComments--)
fgets(buf,128,file);
/* Its important to pre-allocate memory by passing max non zero for any row in the matrix */
ierr = MatCreateSeqAIJ(PETSC_COMM_WORLD,m,n,maxNNZperRow,PETSC_NULL,&pMat);
/* OR we can also pass row distribution of nozero elements for every row */
/* ierr = MatCreateSeqAIJ(PETSC_COMM_WORLD,m,n,0,nnzPtr,&pMat);*/
/*Now Set matrix elements values form matrix market file */
for (i=0; i<nnz; i++)
{
/*Read matrix element from matrix market file*/
fscanf(file,"%d %d %le\n",&row,&col,&val);
/*In matrix market format, rows and columns starts from 1 */
row = row-1; col = col-1 ;
/* For every non zero element,insert that value at row,col position */
ierr = MatSetValues(pMat,1,&row,1,&col,&val,INSERT_VALUES);
}
fclose(file);
/*Matrix Read Complete */
ierr = PetscPrintf(PETSC_COMM_SELF,"\n MATRIX READ...DONE!");
/*Now assemeble the matrix */
ierr = MatAssemblyBegin(pMat,MAT_FINAL_ASSEMBLY);
ierr = MatAssemblyEnd(pMat,MAT_FINAL_ASSEMBLY);
/* Now open output file for writing into PETSc Binary FOrmat*/
ierr = PetscOptionsGetString(PETSC_NULL,"-fout",outputFile,127,PETSC_NULL);CHKERRQ(ierr);
/*With the PETSc Viewer write output to File*/
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,outputFile,FILE_MODE_WRITE,&view);CHKERRQ(ierr);
/*Matview will dump the Mat object to binary file */
ierr = MatView(pMat,view);CHKERRQ(ierr);
/* Destroy the data structure */
ierr = PetscViewerDestroy(&view);CHKERRQ(ierr);
ierr = MatDestroy(&pMat);CHKERRQ(ierr);
/*Just for statistics*/
ierr = PetscGetTime(&t2);CHKERRQ(ierr);
elapsed_time = t2 - t1;
ierr = PetscPrintf(PETSC_COMM_SELF,"ELAPSE TIME: %g\n",elapsed_time);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
#include <petsc/private/fortranimpl.h>
#if defined(PETSC_HAVE_FORTRAN_CAPS)
#define petscfopen_ PETSCFOPEN
#elif !defined(PETSC_HAVE_FORTRAN_UNDERSCORE)
#define petscfopen_ petscfopen
#endif
PETSC_EXTERN void PETSC_STDCALL petscfopen_(MPI_Comm *comm,CHAR fname PETSC_MIXED_LEN(len1),CHAR fmode PETSC_MIXED_LEN(len2),
FILE **file,PetscErrorCode *ierr PETSC_END_LEN(len1) PETSC_END_LEN(len2))
{
char *c1,*c2;
FIXCHAR(fname,len1,c1);
FIXCHAR(fmode,len2,c2);
*ierr = PetscFOpen(MPI_Comm_f2c(*(MPI_Fint*)&*comm),c1,c2,file);
FREECHAR(fname,c1);
FREECHAR(fmode,c2);
}
