void CopyNcVar(
NcFile & ncIn,
NcFile & ncOut,
const std::string & strVarName,
bool fCopyAttributes,
bool fCopyData
) {
if (!ncIn.is_valid()) {
_EXCEPTIONT("Invalid input file specified");
}
if (!ncOut.is_valid()) {
_EXCEPTIONT("Invalid output file specified");
}
NcVar * var = ncIn.get_var(strVarName.c_str());
if (var == NULL) {
_EXCEPTION1("NetCDF file does not contain variable \"%s\"",
strVarName.c_str());
}
NcVar * varOut;
std::vector<NcDim *> dimOut;
dimOut.resize(var->num_dims());
std::vector<long> counts;
counts.resize(var->num_dims());
long nDataSize = 1;
for (int d = 0; d < var->num_dims(); d++) {
NcDim * dimA = var->get_dim(d);
dimOut[d] = ncOut.get_dim(dimA->name());
if (dimOut[d] == NULL) {
if (dimA->is_unlimited()) {
dimOut[d] = ncOut.add_dim(dimA->name());
} else {
dimOut[d] = ncOut.add_dim(dimA->name(), dimA->size());
}
if (dimOut[d] == NULL) {
_EXCEPTION2("Failed to add dimension \"%s\" (%i) to file",
dimA->name(), dimA->size());
}
}
if (dimOut[d]->size() != dimA->size()) {
if (dimA->is_unlimited() && !dimOut[d]->is_unlimited()) {
_EXCEPTION2("Mismatch between input file dimension \"%s\" and "
"output file dimension (UNLIMITED / %i)",
dimA->name(), dimOut[d]->size());
} else if (!dimA->is_unlimited() && dimOut[d]->is_unlimited()) {
_EXCEPTION2("Mismatch between input file dimension \"%s\" and "
"output file dimension (%i / UNLIMITED)",
dimA->name(), dimA->size());
} else if (!dimA->is_unlimited() && !dimOut[d]->is_unlimited()) {
_EXCEPTION3("Mismatch between input file dimension \"%s\" and "
"output file dimension (%i / %i)",
dimA->name(), dimA->size(), dimOut[d]->size());
}
}
counts[d] = dimA->size();
nDataSize *= counts[d];
}
// ncByte / ncChar type
if ((var->type() == ncByte) || (var->type() == ncChar)) {
DataVector<char> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
// ncShort type
if (var->type() == ncShort) {
DataVector<short> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
if (fCopyData) {
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
}
// ncInt type
if (var->type() == ncInt) {
DataVector<int> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
if (fCopyData) {
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
}
// ncFloat type
if (var->type() == ncFloat) {
DataVector<float> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
if (fCopyData) {
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
}
// ncDouble type
if (var->type() == ncDouble) {
DataVector<double> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
if (fCopyData) {
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
}
// ncInt64 type
if (var->type() == ncInt64) {
DataVector<ncint64> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
if (fCopyData) {
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
}
// Check output variable exists
if (varOut == NULL) {
_EXCEPTION1("Unable to create output variable \"%s\"",
var->name());
}
// Copy attributes
if (fCopyAttributes) {
CopyNcVarAttributes(var, varOut);
}
}
Time OutputManagerComposite::Input(
const std::string & strFileName
) {
#ifdef USE_MPI
// Set the flag indicating that output came from a restart file
m_fFromRestartFile = true;
// Determine processor rank
int nRank;
MPI_Comm_rank(MPI_COMM_WORLD, &nRank);
// The active model
const Model & model = m_grid.GetModel();
// Open binary input stream
std::ifstream ifsActiveInput;
ifsActiveInput.open(
strFileName.c_str(), std::ios::binary | std::ios::in);
if (!ifsActiveInput) {
_EXCEPTION1("Unable to open input file \"%s\"",
strFileName.c_str());
}
// Read check bits
int iCheckInput;
ifsActiveInput.read((char *)(&iCheckInput), sizeof(int));
if (iCheckInput != m_iCheck) {
_EXCEPTION1("Invalid or incompatible input file \"%s\"",
strFileName.c_str());
}
// Read current time
Time timeCurrent;
ifsActiveInput.read((char *)(&(timeCurrent)), sizeof(Time));
// Read Grid parameters from file
DataContainer & dcGridParameters = m_grid.GetDataContainerParameters();
int nGridParametersByteSize =
dcGridParameters.GetTotalByteSize();
char * pGridParameters =
(char *)(dcGridParameters.GetPointer());
ifsActiveInput.read(pGridParameters, nGridParametersByteSize);
// Initialize the Grid from specified parameters
m_grid.InitializeDataLocal();
// Load Grid data from file
DataContainer & dcGridPatchData = m_grid.GetDataContainerPatchData();
int nGridPatchDataByteSize =
dcGridPatchData.GetTotalByteSize();
char * pGridPatchData =
(char *)(dcGridPatchData.GetPointer());
ifsActiveInput.read(pGridPatchData, nGridPatchDataByteSize);
// Distribute GridPatches to processors
m_grid.DistributePatches();
// Determine space allocation for each GridPatch
m_vecGridPatchByteSize.Allocate(m_grid.GetPatchCount(), 2);
for (int i = 0; i < m_grid.GetActivePatchCount(); i++) {
const GridPatch * pPatch = m_grid.GetActivePatch(i);
int iPatchIx = pPatch->GetPatchIndex();
if (iPatchIx > m_grid.GetPatchCount()) {
_EXCEPTION2("PatchIndex (%i) out of range [0,%i)",
iPatchIx, m_grid.GetPatchCount());
}
const DataContainer & dcGeometric =
pPatch->GetDataContainerGeometric();
m_vecGridPatchByteSize[iPatchIx][0] =
dcGeometric.GetTotalByteSize();
const DataContainer & dcActiveState =
pPatch->GetDataContainerActiveState();
m_vecGridPatchByteSize[iPatchIx][1] =
dcActiveState.GetTotalByteSize();
}
MPI_Allreduce(
MPI_IN_PLACE,
&(m_vecGridPatchByteSize[0][0]),
m_vecGridPatchByteSize.GetTotalSize(),
MPI_INT,
MPI_MAX,
MPI_COMM_WORLD);
// Initialize byte location for each GridPatch
m_vecGridPatchByteLoc.Allocate(m_grid.GetPatchCount(), 2);
m_vecGridPatchByteLoc[0][0] = 0;
m_vecGridPatchByteLoc[0][1] = m_vecGridPatchByteSize[0][0];
for (int i = 1; i < m_vecGridPatchByteSize.GetRows(); i++) {
m_vecGridPatchByteLoc[i][0] =
m_vecGridPatchByteLoc[i-1][1]
+ m_vecGridPatchByteSize[i-1][1];
m_vecGridPatchByteLoc[i][1] =
m_vecGridPatchByteLoc[i][0]
+ m_vecGridPatchByteSize[i][0];
}
// Reference position
std::streampos posRefFile = ifsActiveInput.tellg();
if (posRefFile == (-1)) {
_EXCEPTIONT("ActiveInput::tellg() fail");
}
// Load in GridPatch data from file
for (int i = 0; i < m_grid.GetActivePatchCount(); i++) {
GridPatch * pPatch = m_grid.GetActivePatch(i);
int iPatchIx = pPatch->GetPatchIndex();
if (iPatchIx > m_grid.GetPatchCount()) {
_EXCEPTION2("PatchIndex (%i) out of range [0,%i)",
iPatchIx, m_grid.GetPatchCount());
}
DataContainer & dcGeometric =
pPatch->GetDataContainerGeometric();
int nGeometricDataByteSize =
dcGeometric.GetTotalByteSize();
char * pGeometricData =
(char *)(dcGeometric.GetPointer());
ifsActiveInput.seekg(
posRefFile + m_vecGridPatchByteLoc[iPatchIx][0]);
ifsActiveInput.read(
pGeometricData, m_vecGridPatchByteSize[iPatchIx][0]);
DataContainer & dcActiveState =
pPatch->GetDataContainerActiveState();
int nActiveStateDataByteSize =
dcActiveState.GetTotalByteSize();
char * pActiveStateData =
(char *)(dcActiveState.GetPointer());
ifsActiveInput.seekg(
posRefFile + m_vecGridPatchByteLoc[iPatchIx][1]);
ifsActiveInput.read(
pActiveStateData, m_vecGridPatchByteSize[iPatchIx][1]);
}
// Close the file
ifsActiveInput.close();
// Barrier
MPI_Barrier(MPI_COMM_WORLD);
#else
_EXCEPTIONT("Not implemented without USE_MPI");
#endif
return timeCurrent;
}
void MeshUtilities::FindFaceFromNode(
const Mesh & mesh,
const Node & node,
FindFaceStruct & aFindFaceStruct
) {
// Reset the FaceStruct
aFindFaceStruct.vecFaceIndices.clear();
aFindFaceStruct.vecFaceLocations.clear();
aFindFaceStruct.loc = Face::NodeLocation_Undefined;
// Loop through all faces to find overlaps
// Note: This algorithm can likely be dramatically improved
for (int l = 0; l < mesh.faces.size(); l++) {
Face::NodeLocation loc;
int ixLocation;
ContainsNode(
mesh.faces[l],
mesh.nodes,
node,
loc,
ixLocation);
if (loc == Face::NodeLocation_Exterior) {
continue;
}
#ifdef VERBOSE
printf("%i\n", l);
printf("n: %1.5e %1.5e %1.5e\n", node.x, node.y, node.z);
printf("n0: %1.5e %1.5e %1.5e\n",
mesh.nodes[mesh.faces[l][0]].x,
mesh.nodes[mesh.faces[l][0]].y,
mesh.nodes[mesh.faces[l][0]].z);
printf("n1: %1.5e %1.5e %1.5e\n",
mesh.nodes[mesh.faces[l][1]].x,
mesh.nodes[mesh.faces[l][1]].y,
mesh.nodes[mesh.faces[l][1]].z);
printf("n2: %1.5e %1.5e %1.5e\n",
mesh.nodes[mesh.faces[l][2]].x,
mesh.nodes[mesh.faces[l][2]].y,
mesh.nodes[mesh.faces[l][2]].z);
printf("n3: %1.5e %1.5e %1.5e\n",
mesh.nodes[mesh.faces[l][3]].x,
mesh.nodes[mesh.faces[l][3]].y,
mesh.nodes[mesh.faces[l][3]].z);
#endif
if (aFindFaceStruct.loc == Face::NodeLocation_Undefined) {
aFindFaceStruct.loc = loc;
}
// Node is in the interior of this face
if (loc == Face::NodeLocation_Interior) {
if (loc != aFindFaceStruct.loc) {
_EXCEPTIONT("No consensus on location of Node");
}
aFindFaceStruct.vecFaceIndices.push_back(l);
aFindFaceStruct.vecFaceLocations.push_back(ixLocation);
break;
}
// Node is on the edge of this face
if (loc == Face::NodeLocation_Edge) {
if (loc != aFindFaceStruct.loc) {
_EXCEPTIONT("No consensus on location of Node");
}
aFindFaceStruct.vecFaceIndices.push_back(l);
aFindFaceStruct.vecFaceLocations.push_back(ixLocation);
}
// Node is at the corner of this face
if (loc == Face::NodeLocation_Corner) {
if (loc != aFindFaceStruct.loc) {
_EXCEPTIONT("No consensus on location of Node");
}
aFindFaceStruct.vecFaceIndices.push_back(l);
aFindFaceStruct.vecFaceLocations.push_back(ixLocation);
}
}
// Edges can only have two adjacent Faces
if (aFindFaceStruct.loc == Face::NodeLocation_Edge) {
if (aFindFaceStruct.vecFaceIndices.size() != 2) {
printf("n: %1.5e %1.5e %1.5e\n", node.x, node.y, node.z);
_EXCEPTION2("Node found on edge with %i neighboring face(s) (%i)",
aFindFaceStruct.vecFaceIndices.size(),
(int)(aFindFaceStruct.vecFaceIndices.size()));
}
}
// Corners must have at least three adjacent Faces
if (aFindFaceStruct.loc == Face::NodeLocation_Corner) {
if (aFindFaceStruct.vecFaceIndices.size() < 3) {
printf("n: %1.5e %1.5e %1.5e\n", node.x, node.y, node.z);
_EXCEPTION1("Two Faced corner detected (%i)",
(int)(aFindFaceStruct.vecFaceIndices.size()));
}
}
}
void OutputManagerComposite::Output(
const Time & time
) {
#ifdef USE_MPI
// Check for open file
if (!IsFileOpen()) {
_EXCEPTIONT("No file available for output");
}
// Verify that only one output has been performed
if (m_ixOutputTime != 0) {
_EXCEPTIONT("Only one Composite output allowed per file");
}
// Determine processor rank
int nRank;
MPI_Comm_rank(MPI_COMM_WORLD, &nRank);
// Data types
const int ExchangeDataType_Geometric = 0;
const int ExchangeDataType_ActiveState = 1;
const int ExchangeDataType_Count = 2;
// Determine space allocation for each GridPatch
m_vecGridPatchByteSize.Allocate(m_grid.GetPatchCount(), 2);
for (int i = 0; i < m_grid.GetActivePatchCount(); i++) {
const GridPatch * pPatch = m_grid.GetActivePatch(i);
int iPatchIx = pPatch->GetPatchIndex();
if (iPatchIx > m_grid.GetPatchCount()) {
_EXCEPTION2("PatchIndex (%i) out of range [0,%i)",
iPatchIx, m_grid.GetPatchCount());
}
const DataContainer & dcGeometric =
pPatch->GetDataContainerGeometric();
m_vecGridPatchByteSize[iPatchIx][0] =
dcGeometric.GetTotalByteSize();
const DataContainer & dcActiveState =
pPatch->GetDataContainerActiveState();
m_vecGridPatchByteSize[iPatchIx][1] =
dcActiveState.GetTotalByteSize();
}
// Reduce GridPatch byte size
if (nRank != 0) {
MPI_Reduce(
&(m_vecGridPatchByteSize[0][0]),
&(m_vecGridPatchByteSize[0][0]),
m_vecGridPatchByteSize.GetTotalSize(),
MPI_INT,
MPI_MAX,
0,
MPI_COMM_WORLD);
// Reduce GridPatch byte size and write Grid information
} else if (nRank == 0) {
// Reduce
MPI_Reduce(
MPI_IN_PLACE,
&(m_vecGridPatchByteSize[0][0]),
m_vecGridPatchByteSize.GetTotalSize(),
MPI_INT,
MPI_MAX,
0,
MPI_COMM_WORLD);
// The active Model
const Model & model = m_grid.GetModel();
// Get maximum byte size for exchange
int sMaxRecvBufferByteSize = 0;
for (int i = 0; i < m_vecGridPatchByteSize.GetRows(); i++) {
if (m_vecGridPatchByteSize[i][0] > sMaxRecvBufferByteSize) {
sMaxRecvBufferByteSize = m_vecGridPatchByteSize[i][0];
}
if (m_vecGridPatchByteSize[i][1] > sMaxRecvBufferByteSize) {
sMaxRecvBufferByteSize = m_vecGridPatchByteSize[i][1];
}
}
// Allocate Recv buffer at root
if (m_vecRecvBuffer.GetRows() < sMaxRecvBufferByteSize) {
m_vecRecvBuffer.Allocate(sMaxRecvBufferByteSize);
}
// Initialize byte location for each GridPatch
m_vecGridPatchByteLoc.Allocate(m_grid.GetPatchCount(), 2);
m_vecGridPatchByteLoc[0][0] = 0;
m_vecGridPatchByteLoc[0][1] = m_vecGridPatchByteSize[0][0];
for (int i = 1; i < m_vecGridPatchByteSize.GetRows(); i++) {
m_vecGridPatchByteLoc[i][0] =
m_vecGridPatchByteLoc[i-1][1]
+ m_vecGridPatchByteSize[i-1][1];
m_vecGridPatchByteLoc[i][1] =
m_vecGridPatchByteLoc[i][0]
+ m_vecGridPatchByteSize[i][0];
}
// Write check bits
m_ofsActiveOutput.write((const char *)(&m_iCheck), sizeof(int));
// Write current time
const Time & timeCurrent = model.GetCurrentTime();
m_ofsActiveOutput.write((const char *)(&(timeCurrent)), sizeof(Time));
// Write Grid information to file
const DataContainer & dcGridParameters =
m_grid.GetDataContainerParameters();
int nGridParametersByteSize =
dcGridParameters.GetTotalByteSize();
const char * pGridParameters =
(const char *)(dcGridParameters.GetPointer());
m_ofsActiveOutput.write(
pGridParameters, nGridParametersByteSize);
const DataContainer & dcGridPatchData =
m_grid.GetDataContainerPatchData();
int nGridPatchDataByteSize =
dcGridPatchData.GetTotalByteSize();
const char * pGridPatchData =
(const char *)(dcGridPatchData.GetPointer());
m_ofsActiveOutput.write(
pGridPatchData, nGridPatchDataByteSize);
}
// Send data from GridPatches to root
if (nRank != 0) {
int nActivePatches = m_grid.GetActivePatchCount();
if (nActivePatches == 0) {
_EXCEPTIONT("No GridPatches on processor");
}
DataArray1D<MPI_Request> vecSendReqGeo(nActivePatches);
DataArray1D<MPI_Request> vecSendReqAcS(nActivePatches);
for (int i = 0; i < nActivePatches; i++) {
const GridPatch * pPatch = m_grid.GetActivePatch(i);
const DataContainer & dcGeometric =
pPatch->GetDataContainerGeometric();
int nGeometricDataByteSize =
dcGeometric.GetTotalByteSize();
const unsigned char * pGeometricData =
dcGeometric.GetPointer();
MPI_Isend(
const_cast<unsigned char *>(pGeometricData),
nGeometricDataByteSize,
MPI_BYTE,
0,
ExchangeDataType_Geometric,
MPI_COMM_WORLD,
&(vecSendReqGeo[i]));
const DataContainer & dcActiveState =
pPatch->GetDataContainerActiveState();
int nActiveStateDataByteSize =
dcActiveState.GetTotalByteSize();
const unsigned char * pActiveStateData =
dcActiveState.GetPointer();
MPI_Isend(
const_cast<unsigned char *>(pActiveStateData),
nActiveStateDataByteSize,
MPI_BYTE,
0,
ExchangeDataType_ActiveState,
MPI_COMM_WORLD,
&(vecSendReqAcS[i]));
/*
printf("Send Geo %i %i\n", pPatch->GetPatchIndex(), nGeometricDataByteSize);
printf("Send AcS %i %i\n", pPatch->GetPatchIndex(), nActiveStateDataByteSize);
*/
}
int iWaitAllMsgGeo =
MPI_Waitall(
nActivePatches,
&(vecSendReqGeo[0]),
MPI_STATUSES_IGNORE);
if (iWaitAllMsgGeo == MPI_ERR_IN_STATUS) {
_EXCEPTIONT("MPI_Waitall returned MPI_ERR_IN_STATUS");
}
int iWaitAllMsgAcS =
MPI_Waitall(
nActivePatches,
&(vecSendReqAcS[0]),
MPI_STATUSES_IGNORE);
if (iWaitAllMsgAcS == MPI_ERR_IN_STATUS) {
_EXCEPTIONT("MPI_Waitall returned MPI_ERR_IN_STATUS");
}
//std::cout << "WAIT DONE" << std::endl;
// Receive data and output to file
} else {
// Reference position
std::streampos posRefFile = m_ofsActiveOutput.tellp();
if (posRefFile == (-1)) {
_EXCEPTIONT("ActiveOutput::tellp() fail");
}
// Write my GridPatch data to file
for (int i = 0; i < m_grid.GetActivePatchCount(); i++) {
const GridPatch * pPatch = m_grid.GetActivePatch(i);
int iPatchIx = pPatch->GetPatchIndex();
if (iPatchIx > m_grid.GetPatchCount()) {
_EXCEPTION2("PatchIndex (%i) out of range [0,%i)",
iPatchIx, m_grid.GetPatchCount());
}
/*
printf("Write %i %i %i %i %i\n",
iPatchIx,
m_vecGridPatchByteLoc[iPatchIx][0],
m_vecGridPatchByteSize[iPatchIx][0],
m_vecGridPatchByteLoc[iPatchIx][1],
m_vecGridPatchByteSize[iPatchIx][1]);
*/
const DataContainer & dcGeometric =
pPatch->GetDataContainerGeometric();
int nGeometricDataByteSize =
dcGeometric.GetTotalByteSize();
const char * pGeometricData =
(const char *)(dcGeometric.GetPointer());
m_ofsActiveOutput.seekp(
posRefFile + m_vecGridPatchByteLoc[iPatchIx][0]);
m_ofsActiveOutput.write(
pGeometricData, m_vecGridPatchByteSize[iPatchIx][0]);
const DataContainer & dcActiveState =
pPatch->GetDataContainerActiveState();
int nActiveStateDataByteSize =
dcActiveState.GetTotalByteSize();
const char * pActiveStateData =
(const char *)(dcActiveState.GetPointer());
m_ofsActiveOutput.seekp(
posRefFile + m_vecGridPatchByteLoc[iPatchIx][1]);
m_ofsActiveOutput.write(
pActiveStateData, m_vecGridPatchByteSize[iPatchIx][1]);
}
// Recieve all data objects from neighbors
int nRemainingMessages =
2 * (m_grid.GetPatchCount() - m_grid.GetActivePatchCount());
for (; nRemainingMessages > 0; nRemainingMessages--) {
// Receive a consolidation message
MPI_Status status;
MPI_Recv(
&(m_vecRecvBuffer[0]),
m_vecRecvBuffer.GetRows(),
MPI_CHAR,
MPI_ANY_SOURCE,
MPI_ANY_TAG,
MPI_COMM_WORLD,
&status);
// Data type from TAG
int iDataType = status.MPI_TAG;
if ((iDataType < 0) || (iDataType >= ExchangeDataType_Count)) {
_EXCEPTION1("MPI_TAG (%i) out of range", iDataType);
}
// Check patch index
int iPatchIx = *((int*)(&(m_vecRecvBuffer[0])));
if (iPatchIx > m_vecGridPatchByteLoc.GetRows()) {
_EXCEPTION2("PatchIndex (%i) out of range [0,%i)",
iPatchIx, m_vecGridPatchByteLoc.GetRows());
}
/*
if (iDataType == 0) {
printf("Recv Geo %i %i\n",
pPatchIx[0],
m_vecGridPatchByteSize[pPatchIx[0]][iDataType]);
} else {
printf("Recv AcS %i %i\n",
pPatchIx[0],
m_vecGridPatchByteSize[pPatchIx[0]][iDataType]);
}
*/
m_ofsActiveOutput.seekp(
posRefFile + m_vecGridPatchByteLoc[iPatchIx][iDataType]);
m_ofsActiveOutput.write(
(const char *)(&(m_vecRecvBuffer[0])),
m_vecGridPatchByteSize[iPatchIx][iDataType]);
}
}
// Barrier
MPI_Barrier(MPI_COMM_WORLD);
#else
_EXCEPTIONT("Not implemented without USE_MPI");
#endif
}
int main(int argc, char** argv) {
NcError error(NcError::silent_nonfatal);
try {
// Input / Output types
enum DiscretizationType {
DiscretizationType_FV,
DiscretizationType_CGLL,
DiscretizationType_DGLL
};
// Input mesh file
std::string strInputMesh;
// Overlap mesh file
std::string strOverlapMesh;
// Input metadata file
std::string strInputMeta;
// Output metadata file
std::string strOutputMeta;
// Input data type
std::string strInputType;
// Output data type
std::string strOutputType;
// Order of polynomial in each element
int nPin;
// Order of polynomial in each output element
int nPout;
// Use bubble on interior of spectral element nodes
bool fBubble;
// Enforce monotonicity
bool fMonotoneType1;
// Enforce monotonicity
bool fMonotoneType2;
// Enforce monotonicity
bool fMonotoneType3;
// Volumetric remapping
bool fVolumetric;
// No conservation
bool fNoConservation;
// Turn off checking for conservation / consistency
bool fNoCheck;
// Output mesh file
std::string strOutputMesh;
// Variable list
std::string strVariables;
// Output map file
std::string strOutputMap;
// Input data file
std::string strInputData;
// Output data file
std::string strOutputData;
// Name of the ncol variable
std::string strNColName;
// Output as double
bool fOutputDouble;
// List of variables to preserve
std::string strPreserveVariables;
// Preserve all non-remapped variables
bool fPreserveAll;
// Fill value override
double dFillValueOverride;
// Parse the command line
BeginCommandLine()
//CommandLineStringD(strMethod, "method", "", "[se]");
CommandLineString(strInputMesh, "in_mesh", "");
CommandLineString(strOutputMesh, "out_mesh", "");
CommandLineString(strOverlapMesh, "ov_mesh", "");
CommandLineString(strInputMeta, "in_meta", "");
CommandLineString(strOutputMeta, "out_meta", "");
CommandLineStringD(strInputType, "in_type", "fv", "[fv|cgll|dgll]");
CommandLineStringD(strOutputType, "out_type", "fv", "[fv|cgll|dgll]");
CommandLineInt(nPin, "in_np", 4);
CommandLineInt(nPout, "out_np", 4);
CommandLineBool(fBubble, "bubble");
CommandLineBool(fMonotoneType1, "mono");
CommandLineBool(fMonotoneType2, "mono2");
CommandLineBool(fMonotoneType3, "mono3");
CommandLineBool(fVolumetric, "volumetric");
CommandLineBool(fNoConservation, "noconserve");
CommandLineBool(fNoCheck, "nocheck");
CommandLineString(strVariables, "var", "");
CommandLineString(strOutputMap, "out_map", "");
CommandLineString(strInputData, "in_data", "");
CommandLineString(strOutputData, "out_data", "");
CommandLineString(strNColName, "ncol_name", "ncol");
CommandLineBool(fOutputDouble, "out_double");
CommandLineString(strPreserveVariables, "preserve", "");
CommandLineBool(fPreserveAll, "preserveall");
CommandLineDouble(dFillValueOverride, "fillvalue", 0.0);
ParseCommandLine(argc, argv);
EndCommandLine(argv)
AnnounceBanner();
// Check command line parameters (mesh arguments)
if (strInputMesh == "") {
_EXCEPTIONT("No input mesh (--in_mesh) specified");
}
if (strOutputMesh == "") {
_EXCEPTIONT("No output mesh (--out_mesh) specified");
}
// Overlap mesh
if (strOverlapMesh == "") {
_EXCEPTIONT("No overlap mesh specified");
}
// Check command line parameters (data arguments)
if ((strInputData != "") && (strOutputData == "")) {
_EXCEPTIONT("--in_data specified without --out_data");
}
if ((strInputData == "") && (strOutputData != "")) {
_EXCEPTIONT("--out_data specified without --in_data");
}
// Check metadata parameters
if ((strInputMeta != "") && (strInputType == "fv")) {
_EXCEPTIONT("--in_meta cannot be used with --in_type fv");
}
if ((strOutputMeta != "") && (strOutputType == "fv")) {
_EXCEPTIONT("--out_meta cannot be used with --out_type fv");
}
// Check command line parameters (data type arguments)
STLStringHelper::ToLower(strInputType);
STLStringHelper::ToLower(strOutputType);
DiscretizationType eInputType;
DiscretizationType eOutputType;
if (strInputType == "fv") {
eInputType = DiscretizationType_FV;
} else if (strInputType == "cgll") {
eInputType = DiscretizationType_CGLL;
} else if (strInputType == "dgll") {
eInputType = DiscretizationType_DGLL;
} else {
_EXCEPTION1("Invalid \"in_type\" value (%s), expected [fv|cgll|dgll]",
strInputType.c_str());
}
if (strOutputType == "fv") {
eOutputType = DiscretizationType_FV;
} else if (strOutputType == "cgll") {
eOutputType = DiscretizationType_CGLL;
} else if (strOutputType == "dgll") {
eOutputType = DiscretizationType_DGLL;
} else {
_EXCEPTION1("Invalid \"out_type\" value (%s), expected [fv|cgll|dgll]",
strOutputType.c_str());
}
// Monotonicity flags
int nMonotoneType = 0;
if (fMonotoneType1) {
nMonotoneType = 1;
}
if (fMonotoneType2) {
if (nMonotoneType != 0) {
_EXCEPTIONT("Only one of --mono, --mono2 and --mono3 may be set");
}
nMonotoneType = 2;
}
if (fMonotoneType3) {
if (nMonotoneType != 0) {
_EXCEPTIONT("Only one of --mono, --mono2 and --mono3 may be set");
}
nMonotoneType = 3;
}
/*
// Volumetric
if (fVolumetric && (nMonotoneType != 0)) {
_EXCEPTIONT("--volumetric cannot be used in conjunction with --mono#");
}
*/
// Create Offline Map
OfflineMap mapRemap;
// Initialize dimension information from file
AnnounceStartBlock("Initializing dimensions of map");
Announce("Input mesh");
mapRemap.InitializeSourceDimensionsFromFile(strInputMesh);
Announce("Output mesh");
mapRemap.InitializeTargetDimensionsFromFile(strOutputMesh);
AnnounceEndBlock(NULL);
// Parse variable list
std::vector< std::string > vecVariableStrings;
ParseVariableList(strVariables, vecVariableStrings);
// Parse preserve variable list
std::vector< std::string > vecPreserveVariableStrings;
ParseVariableList(strPreserveVariables, vecPreserveVariableStrings);
if (fPreserveAll && (vecPreserveVariableStrings.size() != 0)) {
_EXCEPTIONT("--preserveall and --preserve cannot both be specified");
}
// Load input mesh
AnnounceStartBlock("Loading input mesh");
Mesh meshInput(strInputMesh);
meshInput.RemoveZeroEdges();
AnnounceEndBlock(NULL);
// Calculate Face areas
AnnounceStartBlock("Calculating input mesh Face areas");
double dTotalAreaInput = meshInput.CalculateFaceAreas();
Announce("Input Mesh Geometric Area: %1.15e", dTotalAreaInput);
AnnounceEndBlock(NULL);
// Input mesh areas
if (eInputType == DiscretizationType_FV) {
mapRemap.SetSourceAreas(meshInput.vecFaceArea);
}
// Load output mesh
AnnounceStartBlock("Loading output mesh");
Mesh meshOutput(strOutputMesh);
meshOutput.RemoveZeroEdges();
AnnounceEndBlock(NULL);
// Calculate Face areas
AnnounceStartBlock("Calculating output mesh Face areas");
Real dTotalAreaOutput = meshOutput.CalculateFaceAreas();
Announce("Output Mesh Geometric Area: %1.15e", dTotalAreaOutput);
AnnounceEndBlock(NULL);
// Output mesh areas
if (eOutputType == DiscretizationType_FV) {
mapRemap.SetTargetAreas(meshOutput.vecFaceArea);
}
// Load overlap mesh
AnnounceStartBlock("Loading overlap mesh");
Mesh meshOverlap(strOverlapMesh);
meshOverlap.RemoveZeroEdges();
// Verify that overlap mesh is in the correct order
int ixSourceFaceMax = (-1);
int ixTargetFaceMax = (-1);
if (meshOverlap.vecSourceFaceIx.size() !=
meshOverlap.vecTargetFaceIx.size()
) {
_EXCEPTIONT("Invalid overlap mesh:\n"
" Possible mesh file corruption?");
}
for (int i = 0; i < meshOverlap.vecSourceFaceIx.size(); i++) {
if (meshOverlap.vecSourceFaceIx[i] + 1 > ixSourceFaceMax) {
ixSourceFaceMax = meshOverlap.vecSourceFaceIx[i] + 1;
}
if (meshOverlap.vecTargetFaceIx[i] + 1 > ixTargetFaceMax) {
ixTargetFaceMax = meshOverlap.vecTargetFaceIx[i] + 1;
}
}
// Check for forward correspondence in overlap mesh
if (ixSourceFaceMax == meshInput.faces.size() //&&
//(ixTargetFaceMax == meshOutput.faces.size())
) {
Announce("Overlap mesh forward correspondence found");
// Check for reverse correspondence in overlap mesh
} else if (
ixSourceFaceMax == meshOutput.faces.size() //&&
//(ixTargetFaceMax == meshInput.faces.size())
) {
Announce("Overlap mesh reverse correspondence found (reversing)");
// Reorder overlap mesh
meshOverlap.ExchangeFirstAndSecondMesh();
// No correspondence found
} else {
_EXCEPTION2("Invalid overlap mesh:\n"
" No correspondence found with input and output meshes (%i,%i)",
ixSourceFaceMax, ixTargetFaceMax);
}
AnnounceEndBlock(NULL);
// Calculate Face areas
AnnounceStartBlock("Calculating overlap mesh Face areas");
Real dTotalAreaOverlap = meshOverlap.CalculateFaceAreas();
Announce("Overlap Mesh Area: %1.15e", dTotalAreaOverlap);
AnnounceEndBlock(NULL);
// Partial cover
if (fabs(dTotalAreaOverlap - dTotalAreaInput) > 1.0e-10) {
if (!fNoCheck) {
Announce("WARNING: Significant mismatch between overlap mesh area "
"and input mesh area.\n Automatically enabling --nocheck");
fNoCheck = true;
}
}
/*
// Recalculate input mesh area from overlap mesh
if (fabs(dTotalAreaOverlap - dTotalAreaInput) > 1.0e-10) {
AnnounceStartBlock("Overlap mesh only covers a sub-area of the sphere");
Announce("Recalculating source mesh areas");
dTotalAreaInput = meshInput.CalculateFaceAreasFromOverlap(meshOverlap);
Announce("New Input Mesh Geometric Area: %1.15e", dTotalAreaInput);
AnnounceEndBlock(NULL);
}
*/
// Finite volume input / Finite volume output
if ((eInputType == DiscretizationType_FV) &&
(eOutputType == DiscretizationType_FV)
) {
// Generate reverse node array and edge map
meshInput.ConstructReverseNodeArray();
meshInput.ConstructEdgeMap();
// Initialize coordinates for map
mapRemap.InitializeSourceCoordinatesFromMeshFV(meshInput);
mapRemap.InitializeTargetCoordinatesFromMeshFV(meshOutput);
// Construct OfflineMap
AnnounceStartBlock("Calculating offline map");
LinearRemapFVtoFV(
meshInput, meshOutput, meshOverlap, nPin, mapRemap);
// Finite volume input / Finite element output
} else if (eInputType == DiscretizationType_FV) {
DataMatrix3D<int> dataGLLNodes;
DataMatrix3D<double> dataGLLJacobian;
if (strOutputMeta != "") {
AnnounceStartBlock("Loading meta data file");
LoadMetaDataFile(strOutputMeta, dataGLLNodes, dataGLLJacobian);
AnnounceEndBlock(NULL);
} else {
AnnounceStartBlock("Generating output mesh meta data");
double dNumericalArea =
GenerateMetaData(
meshOutput,
nPout,
fBubble,
dataGLLNodes,
dataGLLJacobian);
Announce("Output Mesh Numerical Area: %1.15e", dNumericalArea);
AnnounceEndBlock(NULL);
}
// Initialize coordinates for map
mapRemap.InitializeSourceCoordinatesFromMeshFV(meshInput);
mapRemap.InitializeTargetCoordinatesFromMeshFE(
meshOutput, nPout, dataGLLNodes);
// Generate the continuous Jacobian
bool fContinuous = (eOutputType == DiscretizationType_CGLL);
if (eOutputType == DiscretizationType_CGLL) {
GenerateUniqueJacobian(
dataGLLNodes,
dataGLLJacobian,
mapRemap.GetTargetAreas());
} else {
GenerateDiscontinuousJacobian(
dataGLLJacobian,
mapRemap.GetTargetAreas());
}
// Generate reverse node array and edge map
meshInput.ConstructReverseNodeArray();
meshInput.ConstructEdgeMap();
// Generate remap weights
AnnounceStartBlock("Calculating offline map");
if (fVolumetric) {
LinearRemapFVtoGLL_Volumetric(
meshInput,
meshOutput,
meshOverlap,
dataGLLNodes,
dataGLLJacobian,
mapRemap.GetTargetAreas(),
nPin,
mapRemap,
nMonotoneType,
fContinuous,
fNoConservation);
} else {
LinearRemapFVtoGLL(
meshInput,
meshOutput,
meshOverlap,
dataGLLNodes,
dataGLLJacobian,
mapRemap.GetTargetAreas(),
nPin,
mapRemap,
nMonotoneType,
fContinuous,
fNoConservation);
}
// Finite element input / Finite volume output
} else if (
(eInputType != DiscretizationType_FV) &&
(eOutputType == DiscretizationType_FV)
) {
DataMatrix3D<int> dataGLLNodes;
DataMatrix3D<double> dataGLLJacobian;
if (strInputMeta != "") {
AnnounceStartBlock("Loading meta data file");
LoadMetaDataFile(strInputMeta, dataGLLNodes, dataGLLJacobian);
AnnounceEndBlock(NULL);
} else {
AnnounceStartBlock("Generating input mesh meta data");
double dNumericalArea =
GenerateMetaData(
meshInput,
nPin,
fBubble,
dataGLLNodes,
dataGLLJacobian);
Announce("Input Mesh Numerical Area: %1.15e", dNumericalArea);
AnnounceEndBlock(NULL);
if (fabs(dNumericalArea - dTotalAreaInput) > 1.0e-12) {
Announce("WARNING: Significant mismatch between input mesh "
"numerical area and geometric area");
}
}
if (dataGLLNodes.GetSubColumns() != meshInput.faces.size()) {
_EXCEPTIONT("Number of element does not match between metadata and "
"input mesh");
}
// Initialize coordinates for map
mapRemap.InitializeSourceCoordinatesFromMeshFE(
meshInput, nPin, dataGLLNodes);
mapRemap.InitializeTargetCoordinatesFromMeshFV(meshOutput);
// Generate the continuous Jacobian for input mesh
bool fContinuousIn = (eInputType == DiscretizationType_CGLL);
if (eInputType == DiscretizationType_CGLL) {
GenerateUniqueJacobian(
dataGLLNodes,
dataGLLJacobian,
mapRemap.GetSourceAreas());
} else {
GenerateDiscontinuousJacobian(
dataGLLJacobian,
mapRemap.GetSourceAreas());
}
// Generate offline map
AnnounceStartBlock("Calculating offline map");
if (fVolumetric) {
_EXCEPTIONT("Unimplemented: Volumetric currently unavailable for"
"GLL input mesh");
}
LinearRemapSE4(
meshInput,
meshOutput,
meshOverlap,
dataGLLNodes,
dataGLLJacobian,
nMonotoneType,
fContinuousIn,
fNoConservation,
mapRemap
);
// Finite element input / Finite element output
} else if (
(eInputType != DiscretizationType_FV) &&
(eOutputType != DiscretizationType_FV)
) {
DataMatrix3D<int> dataGLLNodesIn;
DataMatrix3D<double> dataGLLJacobianIn;
DataMatrix3D<int> dataGLLNodesOut;
DataMatrix3D<double> dataGLLJacobianOut;
// Input metadata
if (strInputMeta != "") {
AnnounceStartBlock("Loading input meta data file");
LoadMetaDataFile(
strInputMeta, dataGLLNodesIn, dataGLLJacobianIn);
AnnounceEndBlock(NULL);
} else {
AnnounceStartBlock("Generating input mesh meta data");
double dNumericalAreaIn =
GenerateMetaData(
meshInput,
nPin,
fBubble,
dataGLLNodesIn,
dataGLLJacobianIn);
Announce("Input Mesh Numerical Area: %1.15e", dNumericalAreaIn);
AnnounceEndBlock(NULL);
if (fabs(dNumericalAreaIn - dTotalAreaInput) > 1.0e-12) {
Announce("WARNING: Significant mismatch between input mesh "
"numerical area and geometric area");
}
}
// Output metadata
if (strOutputMeta != "") {
AnnounceStartBlock("Loading output meta data file");
LoadMetaDataFile(
strOutputMeta, dataGLLNodesOut, dataGLLJacobianOut);
AnnounceEndBlock(NULL);
} else {
AnnounceStartBlock("Generating output mesh meta data");
double dNumericalAreaOut =
GenerateMetaData(
meshOutput,
nPout,
fBubble,
dataGLLNodesOut,
dataGLLJacobianOut);
Announce("Output Mesh Numerical Area: %1.15e", dNumericalAreaOut);
AnnounceEndBlock(NULL);
if (fabs(dNumericalAreaOut - dTotalAreaOutput) > 1.0e-12) {
Announce("WARNING: Significant mismatch between output mesh "
"numerical area and geometric area");
}
}
// Initialize coordinates for map
mapRemap.InitializeSourceCoordinatesFromMeshFE(
meshInput, nPin, dataGLLNodesIn);
mapRemap.InitializeTargetCoordinatesFromMeshFE(
meshOutput, nPout, dataGLLNodesOut);
// Generate the continuous Jacobian for input mesh
bool fContinuousIn = (eInputType == DiscretizationType_CGLL);
if (eInputType == DiscretizationType_CGLL) {
GenerateUniqueJacobian(
dataGLLNodesIn,
dataGLLJacobianIn,
mapRemap.GetSourceAreas());
} else {
GenerateDiscontinuousJacobian(
dataGLLJacobianIn,
mapRemap.GetSourceAreas());
}
// Generate the continuous Jacobian for output mesh
bool fContinuousOut = (eOutputType == DiscretizationType_CGLL);
if (eOutputType == DiscretizationType_CGLL) {
GenerateUniqueJacobian(
dataGLLNodesOut,
dataGLLJacobianOut,
mapRemap.GetTargetAreas());
} else {
GenerateDiscontinuousJacobian(
dataGLLJacobianOut,
mapRemap.GetTargetAreas());
}
// Generate offline map
AnnounceStartBlock("Calculating offline map");
LinearRemapGLLtoGLL2(
meshInput,
meshOutput,
meshOverlap,
dataGLLNodesIn,
dataGLLJacobianIn,
dataGLLNodesOut,
dataGLLJacobianOut,
mapRemap.GetTargetAreas(),
nPin,
nPout,
nMonotoneType,
fContinuousIn,
fContinuousOut,
fNoConservation,
mapRemap
);
} else {
_EXCEPTIONT("Not implemented");
}
//#pragma warning "NOTE: VERIFICATION DISABLED"
// Verify consistency, conservation and monotonicity
if (!fNoCheck) {
AnnounceStartBlock("Verifying map");
mapRemap.IsConsistent(1.0e-8);
mapRemap.IsConservative(1.0e-8);
if (nMonotoneType != 0) {
mapRemap.IsMonotone(1.0e-12);
}
AnnounceEndBlock(NULL);
}
AnnounceEndBlock(NULL);
// Initialize element dimensions from input/output Mesh
AnnounceStartBlock("Writing output");
// Output the Offline Map
if (strOutputMap != "") {
AnnounceStartBlock("Writing offline map");
mapRemap.Write(strOutputMap);
AnnounceEndBlock(NULL);
}
// Apply Offline Map to data
if (strInputData != "") {
AnnounceStartBlock("Applying offline map to data");
mapRemap.SetFillValueOverride(static_cast<float>(dFillValueOverride));
mapRemap.Apply(
strInputData,
strOutputData,
vecVariableStrings,
strNColName,
fOutputDouble,
false);
AnnounceEndBlock(NULL);
}
AnnounceEndBlock(NULL);
// Copy variables from input file to output file
if ((strInputData != "") && (strOutputData != "")) {
if (fPreserveAll) {
AnnounceStartBlock("Preserving variables");
mapRemap.PreserveAllVariables(strInputData, strOutputData);
AnnounceEndBlock(NULL);
} else if (vecPreserveVariableStrings.size() != 0) {
AnnounceStartBlock("Preserving variables");
mapRemap.PreserveVariables(
strInputData,
strOutputData,
vecPreserveVariableStrings);
AnnounceEndBlock(NULL);
}
}
AnnounceBanner();
return (0);
} catch(Exception & e) {
Announce(e.ToString().c_str());
return (-1);
} catch(...) {
return (-2);
}
}
