int main( int argc, char** argv )
{
int numProcs, rank;
#if DO_MPI
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &numProcs);
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
#else
rank = 0;
#endif
double EndTime = 10;
int ADVECT_PARTICLES = 7;
int DO_FTLE = 1;
int PRINT_VTK = 1;
double STEPSIZE = 0.001;
double Samples_cubic = 10;
int dataSet = 1;
long int fnx = 3;
long int fny = 3;
long int fnz = 2;
double xmin = -118;
double ymin = -50;
double zmin = -81.6;
double xmax = xmin + 168;
double ymax = ymin + 100;
double zmax = zmin + 203.6;
double SampleXmax = xmax;
double SampleXmin = xmin;
double SampleYmax = ymax;
double SampleYmin = ymin;
double SampleZmax = zmax;
double SampleZmin = zmin;
for(int flag = argc; flag > 0; flag--){
if(!strcmp(argv[flag-1],"-nx")) fnx = atol( argv[flag] );
if(!strcmp(argv[flag-1],"-ny")) fny = atol( argv[flag] );
if(!strcmp(argv[flag-1],"-nz")) fnz = atol( argv[flag] );
if(!strcmp(argv[flag-1],"-t")) EndTime = strtod( argv[flag], NULL );
if(!strcmp(argv[flag-1],"-p")) PRINT_VTK = atoi( argv[flag] );
if(!strcmp(argv[flag-1],"-ftle")) DO_FTLE = atoi( argv[flag] );
if(!strcmp(argv[flag-1],"-a")) ADVECT_PARTICLES = atoi( argv[flag] );
if(!strcmp(argv[flag-1],"-step")) STEPSIZE = strtod( argv[flag], NULL );
if(!strcmp(argv[flag-1],"-c")) Samples_cubic = strtod (argv[flag], NULL );
if(!strcmp(argv[flag-1],"-d")) dataSet = atoi( argv[flag] );
if(!strcmp(argv[flag-1],"-u") || !strcmp(argv[flag-1],"-usage")){
usage(argv);
return 0;
}
}
if( fnx < 2 )
{
cerr << "fnx is less than 2, please use a value greater than 2." << endl;
usage(argv);
return 1;
}
if( fny < 2 )
{
cerr << "fny is less than 2, please use a value greater than 2." << endl;
usage(argv);
return 1;
}
if( fnz < 2 )
{
cerr << "fnz is less than 2, please use a value greater than 2." << endl;
usage(argv);
return 1;
}
if( ADVECT_PARTICLES < 0 || ADVECT_PARTICLES > 7 )
{
cerr << "ADVECT_PARTICLES is not valid, should be between 0 and 7 (inclusive)." << endl;
usage(argv);
return 1;
}
if(PRINT_VTK != 0 && PRINT_VTK != 1){
cerr << "PRINT_VTK is not valid, should be 0 or 1." << endl;
usage(argv);
return 1;
}
if( DO_FTLE != 0 && DO_FTLE != 1 )
{
cerr << "DO_FTLE is not valid, should be 0 or 1." << endl;
usage(argv);
return 1;
}
if( STEPSIZE <= 0 )
{
cerr << "STEPSIZE should be greater than 0." << endl;
usage(argv);
return 1;
}
if(EndTime <= 0 ){
cerr << "EndTime should be greater than 0." << endl;
usage(argv);
return 1;
}
if( Samples_cubic < 1 )
{
cerr << "Samples_cubic should be at least 1." << endl;
usage(argv);
return 1;
}
cout << "[---------- START ----------]" << endl;
/*
*Building the problem mesh
*/
cout << "\tCreating the Problem Mesh" << endl;
// Number of Points in the XYZ dimensions
// 10x10x10 cells
long int nx, ny, nz;
if( dataSet == 1 )
{
nx = 100;
ny = 100;
nz = 100;
}
else if( dataSet == 2 )
{
nx = 200;
ny = 200;
nz = 200;
}
else if( dataSet == 3 )
{
nx = 300;
ny = 300;
nz = 300;
}
else if( dataSet == 4 )
{
nx = 400;
ny = 400;
nz = 400;
}
else if( dataSet == 5 )
{
nx = 500;
ny = 500;
nz = 500;
}
else
{
nx = 500;
ny = 500;
nz = 500;
}
long int size = nx*ny*nz*3;
double* v_field = new double [ size ];
float *buff = new float [ size ];
// Step Size on Mesh
double dx = 168 / (nx-1);
double dy = 100 / (ny-1);
double dz = 203.6 / (nz-1);
double BoundBox[6] = { xmin, xmax, ymin, ymax, zmin, zmax };
#if DO_MPI
if( rank == 0 )
{
#endif
cout << "\t\tReading Data Set" << endl;
long int byteSize = size*sizeof(float);
ifstream is;
if( dataSet == 1 )
{
is.open( "/home/user/Research/DATA/1to5hund/nimrod_100", ios::in | ios::binary );
}
else if( dataSet == 2 )
{
is.open( "/home/user/Research/DATA/1to5hund/nimrod_200", ios::in | ios::binary );
}
else if( dataSet == 3 )
{
is.open( "/home/user/Research/DATA/1to5hund/nimrod_300", ios::in | ios::binary );
}
else if( dataSet == 4 )
{
is.open( "/home/user/Research/DATA/1to5hund/nimrod_400", ios::in | ios::binary );
}
else if( dataSet == 5 )
{
is.open( "/home/user/Research/DATA/1to5hund/nimrod_500", ios::in | ios::binary );
}
else
{
is.open( "/home/user/Research/DATA/Fishtank/fishtank", ios::in | ios::binary );
}
is.read( reinterpret_cast<char*>(buff), byteSize );
is.close();
#if DO_MPI
}
MPI_Bcast( buff, size, MPI_FLOAT, 0, MPI_COMM_WORLD );
if( rank == 0 )
{
#endif
cout << "\t\tBuilding Velocity Field" << endl;
#if DO_MPI
}
#endif
#if DO_VMAG
double magMin = 999999999;
double magMax = 0.0;
double magSum = 0.0;
int numberZeroMag = 0;
#endif
for( long int z = 0; z < nz; z++ ){
for( long int y = 0; y < ny; y++ ){
for( long int x = 0; x < nx; x++ )
{
long int id = 3*(x + y*nx + z*nx*ny);
v_field[ id ] = buff[id];
v_field[ id+1 ] = buff[id+1];
v_field[ id+2 ] = buff[id+2];
#if DO_VMAG
double mag = buff[id]*buff[id] + buff[id+1]*buff[id+1] + buff[id+2]*buff[id+2];
if( mag == 0.0 )
{
numberZeroMag++;
}
else
{
magSum += sqrt(mag);
if( magMin > mag )
{
magMin = mag;
}
if( magMax < mag )
{
magMax = mag;
}
}
#endif
}
}
}
#if DO_VMAG
int numberVs = nx*ny*nz - numberZeroMag;
double avg = magSum / ( (numberVs==0) ? 1 : numberVs );
if( avg == magSum ) cerr << "Error" << endl;
cerr << "Min Mag: " << magMin << endl;
cerr << "Avg Mag: " << avg << endl;
cerr << "Max Mag: " << magMax << endl;
cerr << "AVG: StepSize with Z: " << ((0.8)*((double)(xmax-xmin)/nx) + (0.2)*((double)(zmax-zmin)/nz))/avg << endl;
cerr << "AVGH: StepSize with Z: " << ((0.8)*((double)(xmax-xmin)/(2*nx)) + (0.2)*((double)(zmax-zmin)/(2*nz)))/avg << endl;
cerr << "AVG: StepSize no Z : " << (((double)(xmax-xmin)/nx))/avg << endl;
cerr << "MIN: StepSize with Z: " << ((0.8)*((double)(xmax-xmin)/nx) + (0.2)*((double)(zmax-zmin)/nz))/magMin << endl;
cerr << "MIN: StepSize no Z : " << (((double)(xmax-xmin)/nx))/magMin << endl;
cerr << "MAX: StepSize with Z: " << ((0.8)*((double)(xmax-xmin)/nx) + (0.2)*((double)(zmax-zmin)/nz))/magMax << endl;
cerr << "MAX: StepSize no Z : " << (((double)(xmax-xmin)/nx))/magMax << endl;
exit(0);
#endif
if( buff != NULL ) delete [] buff;
cout << "\tBuilding Fine_Mesh" << endl;
Mesh FineMesh( nx, ny, nz, dx, dy, dz, xmin, ymin, zmin, v_field );
/*
* Building a flow map mesh
*/
// Number of points for flow map in the X & Y & Z dims
// 2x2x2 cells
// good 15
// const long int fnx = 15;
// const long int fny = 15;
// const long int fnz = 15;
// Step Size on Mesh
double fdx = (xmax-xmin)/(fnx-1);
double fdy = (ymax-ymin)/(fny-1);
double fdz = (zmax-zmin)/(fnz-1);
cout << "\tBuilding Uber_Mesh" << endl;
DomainMesh UberMesh( fnx, fny, fnz, fdx, fdy, fdz, xmin, ymin, zmin, &FineMesh );
cout << "\t\tAllocating Uber_Mesh Variables" << endl;
// Establish the flows in each planar set
long int num_X_planar_flows = fnx * ny * nz;
long int num_Y_planar_flows = fny * nz * nx;
long int num_Z_planar_flows = fnz * nx * ny;
long int num_X_planar_cells = (fnx) * (ny-1) * (nz-1);
long int num_Y_planar_cells = (fny) * (nz-1) * (nx-1);
long int num_Z_planar_cells = (fnz) * (nx-1) * (ny-1);
// Allocate the flows for the flow map
UberMesh.flowField[0] = new Flow [ num_X_planar_flows ];
UberMesh.flowField[1] = new Flow [ num_Y_planar_flows ];
UberMesh.flowField[2] = new Flow [ num_Z_planar_flows ];
// Allocate the accepted flow bool for the flow map
UberMesh.AcceptableFlow[0] = new bool [ num_X_planar_cells ];
UberMesh.AcceptableFlow[1] = new bool [ num_Y_planar_cells ];
UberMesh.AcceptableFlow[2] = new bool [ num_Z_planar_cells ];
// Allocate the max_diff for the flow map
UberMesh.max_diff[0] = new double [ num_X_planar_cells ]();
UberMesh.max_diff[1] = new double [ num_Y_planar_cells ]();
UberMesh.max_diff[2] = new double [ num_Z_planar_cells ]();
cerr << "Init Flows" << endl;
//Initisalize flow data
UberMesh.fillInFlows();
UberMesh.setFlowsCellIDs();
cout << "\tBuilding Particle List" << endl;
ParticleContainer flowXParticles( UberMesh.flowField[0], num_X_planar_flows, STEPSIZE );
ParticleContainer flowYParticles( UberMesh.flowField[1], num_Y_planar_flows, STEPSIZE );
ParticleContainer flowZParticles( UberMesh.flowField[2], num_Z_planar_flows, STEPSIZE );
ParticleContainer* flowParticles[3] = { &flowXParticles, &flowYParticles, &flowZParticles };
long int numFlows[3] = { num_X_planar_flows, num_Y_planar_flows, num_Z_planar_flows };
cout << "\tGenerating Precomputed Flow Map" << endl;
double start,end;
GET_TIME( start );
for( int d = 0; d < 3; d++ )
{
#pragma omp parallel for
for( long int i = 0; i < numFlows[d]; i++ )
{
Flow* flow = &UberMesh.flowField[d][i];
long int cellID = flow->cellID;
if( cellID != -1 )
{
double cbb[6];
UberMesh.getCellBounds( cellID, cbb );
Particle part = flowParticles[d]->particle[i];
//cerr << "Particle Cell Advection: " << d << " " << i << endl;
UberMesh.EulerCellAdvection( cellID, 100.0, cbb, part );
flow->out.setPoint( part.x, part.y, part.z, part.t );
flow->setFID( cbb );
}
}
}
GET_TIME( end );
cout << "\t\tTime To Compute FlowMap: " << end-start << " (seconds)" << endl;
cout << "\tComputing Acceptable flows" << endl;
long int num_accepted = UberMesh.computeAllAcceptableFlows();
cout << "[----------- END -----------]" << endl;
if( ADVECT_PARTICLES != 0 )
{
cout << "Building Advection List" << endl;
int cbe = 100;
long int numParticles = 1;
Particle* advectionList;
if (ADVECT_PARTICLES == 1 )
BuildParticleContainerOne( advectionList, STEPSIZE );
else if (ADVECT_PARTICLES == 2 )
{
numParticles = 2;
BuildParticleContainerTwo( advectionList, STEPSIZE );
}
else if (ADVECT_PARTICLES == 3 )
{
#if DO_MPI
BuildParticleContainerFull( &FineMesh, advectionList, numParticles, STEPSIZE, rank, numProcs );
#else
BuildParticleContainerFull( &FineMesh, advectionList, numParticles, STEPSIZE );
#endif
}
else if (ADVECT_PARTICLES == 4 )
{
BuildParticleContainerFullX2( &FineMesh, advectionList, numParticles, STEPSIZE );
}
else if (ADVECT_PARTICLES == 5 )
{
numParticles = cbe*cbe*cbe;
double cube[6] = { -1, 1, -1, 1, -1, 1 };
BuildParticleContainerCube( advectionList, cube, cbe, STEPSIZE );
}
else if( ADVECT_PARTICLES == 6 )
{
long int uN[3] = { UberMesh.nx, UberMesh.ny, UberMesh.nz };
double uD[3] = { UberMesh.dx, UberMesh.dy, UberMesh.dz };
cerr << "Building Particles" << endl;
numParticles = num_X_planar_flows + num_Y_planar_flows + num_Z_planar_flows;
BuildParticleContainerPlanar( &FineMesh, uN, uD, advectionList, numParticles, STEPSIZE );
cerr << "Built" << endl;
}
else if( ADVECT_PARTICLES == 7 )
{
Particle** SampleParticlesI;
Particle** SampleParticlesE;
Particle** SampleParticlesL;
int numSamples = Samples_cubic;
int SampleSize = numSamples*numSamples*numSamples;
SampleParticlesI = new Particle* [ SampleSize ];
SampleParticlesE = new Particle* [ SampleSize ];
SampleParticlesL = new Particle* [ SampleSize ];
double FTLE_PD_Avg = 0.0;
double FTLE_DF_Avg = 0.0;
int cbe = 3;
int numParticlesPerCube = 27;
double samDx = (SampleXmax-SampleXmin)/(double)numSamples;
double samDy = (SampleYmax-SampleYmin)/(double)numSamples;
double samDz = (SampleZmax-SampleZmin)/(double)numSamples;
double deltaX = 0.01*samDx;
double deltaY = 0.01*samDy;
double deltaZ = 0.01*samDz;
double totalLagrange = 0.0;
double totalEuler = 0.0;
double lag_start, lag_end;
double eul_start, eul_end;
int totalL = 0;
int totalE = 0;
cerr << "looping over samples" << endl;
for( int samplesZ = 0; samplesZ < numSamples; samplesZ++ ){
for( int samplesY = 0; samplesY < numSamples; samplesY++ ){
for( int samplesX = 0; samplesX < numSamples; samplesX++ )
{
int sampleID = samplesX + samplesY*numSamples + samplesZ*numSamples*numSamples;
double xp = SampleXmin + samplesX*samDx;
double yp = SampleYmin + samplesY*samDy;
double zp = SampleZmin + samplesZ*samDz;
double cube[6] = { xp-deltaX, xp+deltaX, yp-deltaY, yp+deltaY, zp-deltaZ, zp+deltaZ };
BuildParticleContainerCube( SampleParticlesI[sampleID], cube, cbe, STEPSIZE );
BuildParticleContainerCube( SampleParticlesE[sampleID], cube, cbe, STEPSIZE );
BuildParticleContainerCube( SampleParticlesL[sampleID], cube, cbe, STEPSIZE );
ParticleContainer advectListE( SampleParticlesE[sampleID], numParticlesPerCube );
ParticleContainer advectListL( SampleParticlesL[sampleID], numParticlesPerCube );
GET_TIME( eul_start );
AdvectParticleList( &FineMesh, &advectListE, EndTime, BoundBox );
GET_TIME( eul_end );
GET_TIME( lag_start );
AdvectParticleList( &FineMesh, &UberMesh, &advectListL, EndTime, BoundBox, totalL, totalE );
GET_TIME( lag_end );
totalLagrange += lag_end-lag_start;
totalEuler += eul_end-eul_start;
Particle* Input = SampleParticlesI[ sampleID ];
Particle* OutputE = SampleParticlesE[ sampleID ];
Particle* OutputL = SampleParticlesL[ sampleID ];
double FTLE_E, FTLE_L;
FTLE_E = compute_FTLE( 1, 1, 1, cbe, cbe, cbe, Input, OutputE, EndTime );
FTLE_L = compute_FTLE( 1, 1, 1, cbe, cbe, cbe, Input, OutputL, EndTime );
double avg_ftle = ( FTLE_E + FTLE_L ) / 2.0;
FTLE_PD_Avg += (( fabs( FTLE_E - FTLE_L ) ) / ( (avg_ftle==0) ? 1 : avg_ftle ) ) * 100;
FTLE_DF_Avg += fabs( FTLE_E - FTLE_L );
/*
cerr << "FTLE_E: " << FTLE_E << endl;
cerr << "FTLE_L: " << FTLE_L << endl;
cerr << "FTLE_PD: " << (( fabs( FTLE_E - FTLE_L ) ) / ( (avg_ftle==0) ? 1 : avg_ftle ) ) * 100 << endl;
cerr << "FTLE_DF: " << fabs( FTLE_E - FTLE_L ) << endl;
*/
}
}
}
FTLE_PD_Avg /= SampleSize;
FTLE_DF_Avg /= SampleSize;
cerr << "Num Particles Advected in Total: " << SampleSize*27 << endl;
cerr << "Num Particles Used in FTLE: " << SampleSize << endl;
cerr << "Total Lagrangian Steps: " << totalL << endl;
cerr << "Total Eulerian Steps: " << totalE << endl;
cerr << "Average Lagrange PP: " << (double)totalL / (double)(SampleSize*27) << endl;
cerr << "Average Eulerian PP: " << (double)totalE / (double)(SampleSize*27) << endl;
cerr << "Percent Lagrange Steps: " << ((double)totalL / (double)( totalL + totalE ) )* 100.0 << endl;
cerr << "Sample FTLE Percent Diff: " << FTLE_PD_Avg << endl;
cerr << "Sample FTLE Difference: " << FTLE_DF_Avg << endl;
cerr << "Total Lag_time: " << totalLagrange << endl;
cerr << "Total Eul_time: " << totalEuler << endl;
cerr << "Speedup Eul / Lag: " << totalEuler / totalLagrange << endl;
}
if( ADVECT_PARTICLES != 7 )
{
ParticleContainer advectList( advectionList, numParticles );
cout << "Advecting Particle List" << endl;
double Lagrange_Full_Start;
double Lagrange_Full_End;
int totalL = 0;
int totalE = 0;
GET_TIME (Lagrange_Full_Start );
#if DO_MPI
AdvectParticleList( &FineMesh, &UberMesh, &advectList, EndTime, BoundBox, totalL, totalE, rank, numProcs );
#else
AdvectParticleList( &FineMesh, &UberMesh, &advectList, EndTime, BoundBox, totalL, totalE );
#endif
GET_TIME (Lagrange_Full_End );
stringstream mpi_cerr;
mpi_cerr << endl;
mpi_cerr << rank << ": Total Lagrangian Steps: " << totalL << endl;
mpi_cerr << rank << ": Total Eulerian Steps: " << totalE << endl;
mpi_cerr << rank << ": Average Lagrange PP: " << (double)totalL / (double)(numParticles) << endl;
mpi_cerr << rank << ": Average Eulerian PP: " << (double)totalE / (double)(numParticles) << endl;
mpi_cerr << rank << ": Percent Lagrange Steps: " << ((double)totalL / (double)( totalL + totalE ) )* 100.0 << endl;
mpi_cerr << endl;
cerr << mpi_cerr.rdbuf();
if( DO_FTLE != 0 )
{
double *FTLE1;
double *FTLE2;
double *FTLEDiff;
if( ADVECT_PARTICLES != 3 )
{
cerr << "Can only do FTLE for full point seeding currently" << endl;
}
else
{
/* We need to pull all particles into global list for everyone to have. ALL Reduce? */
cerr << "Building Advection Lists for Euler only run" << endl;
Particle* eulerOnly;
#if DO_MPI
BuildParticleContainerFull( &FineMesh, eulerOnly, numParticles, STEPSIZE, rank, numProcs );
#else
BuildParticleContainerFull( &FineMesh, eulerOnly, numParticles, STEPSIZE );
#endif
ParticleContainer advectList2( eulerOnly, numParticles );
cerr << "Advecting euler list" << endl;
double Euler_Full_Start;
double Euler_Full_End;
GET_TIME( Euler_Full_Start );
AdvectParticleList( &FineMesh, &advectList2, EndTime, BoundBox );
GET_TIME( Euler_Full_End );
cerr << "building input list for comparison " << endl;
#if DO_MPI
Particle* inputPointsL;
BuildParticleContainerFull( &FineMesh, inputPointsL, numParticles, STEPSIZE, rank, numProcs );
ParticleContainer advectList3( inputPointsL, numParticles );
#else
Particle* inputPoints;
BuildParticleContainerFull( &FineMesh, inputPoints, numParticles, STEPSIZE );
#endif
int totalNumP = nx*ny*nz;
#if DO_MPI
int start_id = rank*(totalNumP/numProcs);
int end_id = start_id + (totalNumP/numProcs);
if( rank == numProcs - 1 )
{
end_id += totalNumP - end_id;
}
MPI_Barrier(MPI_COMM_WORLD);
Particle* inputPoints = new Particle [totalNumP];
Particle* outputPoints = new Particle [totalNumP];
Particle* outputPoints2 = new Particle [totalNumP];
double *message = new double [4*totalNumP];
double *mailbox = new double [4*totalNumP];
memset(mailbox, 0.0, 4*totalNumP*sizeof(double));
serializeParticles(start_id, end_id, totalNumP, message, &advectList);
MPI_Allreduce(message, mailbox, totalNumP*4, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
serializeParticles(totalNumP, mailbox, outputPoints);
memset(mailbox, 0.0, 4*totalNumP*sizeof(double));
serializeParticles(start_id, end_id, totalNumP, message, &advectList2);
MPI_Allreduce(message, mailbox, totalNumP*4, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
serializeParticles(totalNumP, mailbox, outputPoints2);
memset(mailbox, 0.0, 4*totalNumP*sizeof(double));
serializeParticles(start_id, end_id, totalNumP, message, &advectList3);
MPI_Allreduce(message, mailbox, totalNumP*4, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
serializeParticles(totalNumP, mailbox, inputPoints);
if( message != NULL)
{
delete [] message;
}
if( mailbox != NULL )
{
delete [] mailbox;
}
if( inputPointsL != NULL )
{
delete [] inputPointsL;
}
#else
Particle* outputPoints = advectList.particle;
Particle* outputPoints2 = advectList2.particle;
#endif
FTLE1 = new double [ totalNumP ];
FTLE2 = new double [ totalNumP ];
FTLEDiff = new double [ totalNumP ];
memset( FTLE1, 0.0, totalNumP*sizeof(double));
memset( FTLE2, 0.0, totalNumP*sizeof(double));
memset( FTLEDiff, 0.0, totalNumP*sizeof(double));
cerr << "Doing FTLE calculation" << endl;
#if DO_MPI
#pragma omp parallel for schedule(static,1)
for( int id = start_id; id < end_id; id++ )
{
long int ids[3];
FineMesh.D1to3P( id, ids );
int x_id = ids[0];
int y_id = ids[1];
int z_id = ids[2];
#else
#pragma omp parallel for schedule(static,1)
for( int z_id = 0; z_id < nz; z_id++ ){
for( int y_id = 0; y_id < ny; y_id++ ){
for( int x_id = 0; x_id < nx; x_id++ )
{
unsigned int id = x_id + y_id * nx + z_id * nx * ny;
#endif
FTLE1[ id ] = compute_FTLE( x_id, y_id, z_id, nx, ny, nz, inputPoints, outputPoints, EndTime );
FTLE2[ id ] = compute_FTLE( x_id, y_id, z_id, nx, ny, nz, inputPoints, outputPoints2, EndTime );
FTLEDiff[ id ] = fabs( FTLE1[id] - FTLE2[id] );
#if DO_MPI
}
#else
}
}
}
#endif
#if DO_MPI
double *FTLE1_F, *FTLE2_F, *FTLEDiff_F;
if( rank == 0 )
{
FTLE1_F = new double [totalNumP];
FTLE2_F = new double [totalNumP];
FTLEDiff_F = new double [totalNumP];
memset( FTLE1_F, 0.0, totalNumP*sizeof(double));
memset( FTLE2_F, 0.0, totalNumP*sizeof(double));
memset( FTLEDiff_F, 0.0, totalNumP*sizeof(double));
}
MPI_Reduce( FTLE1, FTLE1_F, totalNumP, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD );
MPI_Reduce( FTLE2, FTLE2_F, totalNumP, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD );
MPI_Reduce( FTLEDiff, FTLEDiff_F, totalNumP, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD );
if( FTLE1 != NULL )
{
delete [] FTLE1;
}
if( FTLE2 != NULL )
{
delete [] FTLE2;
}
if( FTLEDiff != NULL )
{
delete [] FTLEDiff;
}
#endif
#if DO_MPI
if ( rank == 0 )
{
printVtkFTLE( nx, ny, nz, xmin, ymin, zmin, dx, dy, dz, FTLE1_F, FTLE2_F, FTLEDiff_F );
}
#else
printVtkFTLE( nx, ny, nz, xmin, ymin, zmin, dx, dy, dz, FTLE1, FTLE2, FTLEDiff );
#endif
double Euler_Full = Euler_Full_End - Euler_Full_Start;
double Lagrange_Full = Lagrange_Full_End - Lagrange_Full_Start;
stringstream mpi_cerr2;
mpi_cerr2 << rank << ": Timing Full: " << endl;
mpi_cerr2 << rank << ": Lagrangian: " << Lagrange_Full << endl;
mpi_cerr2 << rank << ": Eulerian: " << Euler_Full << endl;
mpi_cerr2 << rank << ": Lagrange to Euler Full Speedup: " << Euler_Full / Lagrange_Full << endl;
cerr << mpi_cerr2.rdbuf();
}
}
}
}
if ( PRINT_VTK != 0 && rank == 0 )
{
cout << "Printing VTK Files" << endl;
/*VTK File Printer*/
int umN[3] = {fnx, fny, fnz};
int fmN[3] = {nx, ny, nz};
double umD[3] = {fdx, fdy, fdz};
double fmD[3] = {dx, dy, dz};
double orig[3] = { xmin, ymin, zmin };
VTKFilePrinter printer( umN, fmN, umD, fmD, orig, UberMesh.AcceptableFlow, UberMesh.max_diff, "Tok" );
printer.printVtkDs();
}
/*Clean Up of allocatd variables*/
if( UberMesh.flowField[0] != NULL )
{
delete [] UberMesh.flowField[0];
}
if( UberMesh.flowField[1] != NULL )
{
delete [] UberMesh.flowField[1];
}
if( UberMesh.flowField[2] != NULL )
{
delete [] UberMesh.flowField[2];
}
if( UberMesh.AcceptableFlow[0] != NULL )
{
delete [] UberMesh.AcceptableFlow[0];
}
if( UberMesh.AcceptableFlow[1] != NULL )
{
delete [] UberMesh.AcceptableFlow[1];
}
if( UberMesh.AcceptableFlow[2] != NULL )
{
delete [] UberMesh.AcceptableFlow[2];
}
#if DO_MPI
MPI_Finalize();
#endif
}
