Exemple #1
0
void GridPatchCSGLL::ComputeVorticityDivergence(
	int iDataIndex
) {
	// Physical constants
	const PhysicalConstants & phys = m_grid.GetModel().GetPhysicalConstants();

	// Working data
	DataArray4D<double> & dataState =
		GetDataState(iDataIndex, DataLocation_Node);

	if (dataState.GetSize(0) < 2) {
		_EXCEPTIONT(
			"Insufficient components for vorticity calculation");
	}

	// Get the alpha and beta components of vorticity
	DataArray3D<double> dataUa;
	dataUa.SetSize(
		dataState.GetSize(1),
		dataState.GetSize(2),
		dataState.GetSize(3));

	DataArray3D<double> dataUb;
	dataUb.SetSize(
		dataState.GetSize(1),
		dataState.GetSize(2),
		dataState.GetSize(3));

	dataUa.AttachToData(&(dataState[0][0][0][0]));
	dataUb.AttachToData(&(dataState[1][0][0][0]));

	// Compute the radial component of the curl of the velocity field
	ComputeCurlAndDiv(dataUa, dataUb);
}
Exemple #2
0
void Grid::ReduceInterpolate(
	const DataArray1D<double> & dAlpha,
	const DataArray1D<double> & dBeta,
	const DataArray1D<int> & iPatch,
	DataType eDataType,
	DataLocation eDataLocation,
	bool fInterpAllVariables,
	DataArray3D<double> & dInterpData,
	bool fIncludeReferenceState,
	bool fConvertToPrimitive
) const {
	// Check interpolation data array size
	if ((dAlpha.GetRows() != dBeta.GetRows()) ||
		(dAlpha.GetRows() != iPatch.GetRows())
	) {
		_EXCEPTIONT("Inconsistency in vector lengths.");
	}

	if ((eDataType == DataType_Tracers) &&
		(m_model.GetEquationSet().GetTracers() == 0)
	) {
		_EXCEPTIONT("Unable to Interpolate with no tracers.");
	}

	// Check interpolation data array size
	if ((eDataType == DataType_State) &&
		(dInterpData.GetRows() != m_model.GetEquationSet().GetComponents())
	) {
		_EXCEPTIONT("InterpData dimension mismatch (0)");
	}

	if ((eDataType == DataType_Tracers) &&
		(dInterpData.GetRows() != m_model.GetEquationSet().GetTracers())
	) {
		_EXCEPTIONT("InterpData dimension mismatch (0)");
	}

	if ((eDataType == DataType_Topography) &&
		(dInterpData.GetRows() != 1)
	) {
		_EXCEPTIONT("InterpData dimension mismatch (0)");
	}

	if ((eDataType == DataType_Vorticity) &&
		(dInterpData.GetRows() != 1)
	) {
		_EXCEPTIONT("InterpData dimension mismatch (0)");
	}

	if ((eDataType == DataType_Divergence) &&
		(dInterpData.GetRows() != 1)
	) {
		_EXCEPTIONT("InterpData dimension mismatch (0)");
	}

	if ((eDataType == DataType_Temperature) &&
		(dInterpData.GetRows() != 1)
	) {
		_EXCEPTIONT("InterpData dimension mismatch (0)");
	}

	if ((eDataLocation == DataLocation_None) &&
		(dInterpData.GetColumns() != 1)
	) {
		_EXCEPTIONT("InterpData dimension mismatch (1)");
	}

	if ((eDataLocation == DataLocation_Node) &&
		(dInterpData.GetColumns() != GetRElements())
	) {
		_EXCEPTIONT("InterpData dimension mismatch (1)");
	}

	if ((eDataLocation == DataLocation_REdge) &&
		(dInterpData.GetColumns() != GetRElements() + 1)
	) {
		_EXCEPTIONT("InterpData dimension mismatch (1)");
	}

	if (dInterpData.GetSubColumns() != dAlpha.GetRows()) {
		_EXCEPTIONT("InterpData dimension mismatch (2)");
	}

	// Zero the interpolated data
	dInterpData.Zero();

	// Interpolate state data
	for (int n = 0; n < m_vecActiveGridPatches.size(); n++) {
		m_vecActiveGridPatches[n]->InterpolateData(
			dAlpha, dBeta, iPatch,
			eDataType,
			eDataLocation,
			fInterpAllVariables,
			dInterpData,
			fIncludeReferenceState,
			fConvertToPrimitive);
	}

#ifdef USE_MPI
	// Perform an Reduce operation to combine all data
	int nRank;
	MPI_Comm_rank(MPI_COMM_WORLD, &nRank);

	if (nRank == 0) {
		MPI_Reduce(
			MPI_IN_PLACE,
			&(dInterpData[0][0][0]),
			dInterpData.GetRows()
				* dInterpData.GetColumns()
				* dInterpData.GetSubColumns(),
			MPI_DOUBLE,
			MPI_SUM,
			0,
			MPI_COMM_WORLD);

	} else {
		MPI_Reduce(
			&(dInterpData[0][0][0]),
			NULL,
			dInterpData.GetRows()
				* dInterpData.GetColumns()
				* dInterpData.GetSubColumns(),
			MPI_DOUBLE,
			MPI_SUM,
			0,
			MPI_COMM_WORLD);
	}
#endif
}
void ExchangeBuffer::Unpack(
	DataArray3D<double> & data
) {
	const size_t sAElements = data.GetSize(0);
	const size_t sBElements = data.GetSize(1);
	const size_t sRElements = data.GetSize(2);

	// Index for halo elements along boundary
	int ixBoundaryBegin;
	int ixBoundaryEnd;

	int ixABoundaryBegin;
	int ixABoundaryEnd;

	int ixBBoundaryBegin;
	int ixBBoundaryEnd;

	// Unpack data from right
	if (m_dir == Direction_Right) {
		ixBoundaryBegin = sAElements - m_sHaloElements;
		ixBoundaryEnd   = sAElements;

		// Check that sufficient data remains in send buffer
		int nTotalValues =
			  sRElements
			* (ixBoundaryEnd - ixBoundaryBegin)
			* (m_ixSecond - m_ixFirst);

		if (m_ixRecvBuffer + nTotalValues > m_dRecvBuffer.GetRows()) {
			_EXCEPTIONT("Insufficient space in RecvBuffer for operation.");
		}

		// Unpack data
		for (int i = ixBoundaryEnd-1; i >= ixBoundaryBegin; i--) {
		for (int j = m_ixFirst; j < m_ixSecond; j++) {
#pragma simd
			for (int k = 0; k < sRElements; k++) {
				data(i,j,k) =
					m_dRecvBuffer[m_ixRecvBuffer + k];
			}
			m_ixRecvBuffer += sRElements;
		}
		}

	// Unpack data from top
	} else if (m_dir == Direction_Top) {
		ixBoundaryBegin = sBElements - m_sHaloElements;
		ixBoundaryEnd   = sBElements;

		// Check that sufficient data remains in send buffer
		int nTotalValues =
			  sRElements
			* (ixBoundaryEnd - ixBoundaryBegin)
			* (m_ixSecond - m_ixFirst);

		if (m_ixRecvBuffer + nTotalValues > m_dRecvBuffer.GetRows()) {
			_EXCEPTIONT("Insufficient space in RecvBuffer for operation.");
		}

		// Unpack data
		for (int j = ixBoundaryEnd-1; j >= ixBoundaryBegin; j--) {
		for (int i = m_ixFirst; i < m_ixSecond; i++) {
#pragma simd
			for (int k = 0; k < sRElements; k++) {
				data(i,j,k) =
					m_dRecvBuffer[m_ixRecvBuffer + k];
			}
			m_ixRecvBuffer += sRElements;
		}
		}

	// Unpack data from left
	} else if (m_dir == Direction_Left) {
		ixBoundaryBegin = 0;
		ixBoundaryEnd   = m_sHaloElements;

		// Check that sufficient data remains in send buffer
		int nTotalValues =
			  sRElements
			* (ixBoundaryEnd - ixBoundaryBegin)
			* (m_ixSecond - m_ixFirst);

		if (m_ixRecvBuffer + nTotalValues > m_dRecvBuffer.GetRows()) {
			_EXCEPTIONT("Insufficient space in RecvBuffer for operation.");
		}

		// Unpack data
		for (int i = ixBoundaryBegin; i < ixBoundaryEnd; i++) {
		for (int j = m_ixFirst; j < m_ixSecond; j++) {
#pragma simd
			for (int k = 0; k < sRElements; k++) {
				data(i,j,k) =
					m_dRecvBuffer[m_ixRecvBuffer + k];
			}
			m_ixRecvBuffer += sRElements;
		}
		}

	// Unpack data from bottom
	} else if (m_dir == Direction_Bottom) {
		ixBoundaryBegin = 0;
		ixBoundaryEnd   = m_sHaloElements;

		// Check that sufficient data remains in send buffer
		int nTotalValues =
			  sRElements
			* (ixBoundaryEnd - ixBoundaryBegin)
			* (m_ixSecond - m_ixFirst);

		if (m_ixRecvBuffer + nTotalValues > m_dRecvBuffer.GetRows()) {
			_EXCEPTIONT("Insufficient space in RecvBuffer for operation.");
		}

		// Unpack data
		for (int j = ixBoundaryBegin; j < ixBoundaryEnd; j++) {
		for (int i = m_ixFirst; i < m_ixSecond; i++) {
#pragma simd
			for (int k = 0; k < sRElements; k++) {
				data(i,j,k) =
					m_dRecvBuffer[m_ixRecvBuffer + k];
			}
			m_ixRecvBuffer += sRElements;
		}
		}

	// Unpack data from top-right
	} else if (m_dir == Direction_TopRight) {
		ixABoundaryBegin = m_ixFirst + 1;
		ixABoundaryEnd   = m_ixFirst + m_sHaloElements + 1;
		ixBBoundaryBegin = m_ixSecond + 1;
		ixBBoundaryEnd   = m_ixSecond + m_sHaloElements + 1;

		// Check that sufficient data remains in send buffer
		int nTotalValues =
			  sRElements
			* (ixABoundaryEnd - ixABoundaryBegin)
			* (ixBBoundaryEnd - ixBBoundaryBegin);

		if (m_ixRecvBuffer + nTotalValues > m_dRecvBuffer.GetRows()) {
			_EXCEPTIONT("Insufficient space in RecvBuffer for operation.");
		}

		// Unpack data
		for (int j = ixBBoundaryEnd-1; j >= ixBBoundaryBegin; j--) {
		for (int i = ixABoundaryEnd-1; i >= ixABoundaryBegin; i--) {
#pragma simd
			for (int k = 0; k < sRElements; k++) {
				data(i,j,k) =
					m_dRecvBuffer[m_ixRecvBuffer + k];
			}
			m_ixRecvBuffer += sRElements;
		}
		}

	// Unpack data from top-left
	} else if (m_dir == Direction_TopLeft) {
		ixABoundaryBegin = m_ixFirst - m_sHaloElements;
		ixABoundaryEnd   = m_ixFirst;
		ixBBoundaryBegin = m_ixSecond + 1;
		ixBBoundaryEnd   = m_ixSecond + m_sHaloElements + 1;

		// Check that sufficient data remains in send buffer
		int nTotalValues =
			  sRElements
			* (ixABoundaryEnd - ixABoundaryBegin)
			* (ixBBoundaryEnd - ixBBoundaryBegin);

		if (m_ixRecvBuffer + nTotalValues > m_dRecvBuffer.GetRows()) {
			_EXCEPTIONT("Insufficient space in RecvBuffer for operation.");
		}

		// Unpack data
		for (int j = ixBBoundaryEnd-1; j >= ixBBoundaryBegin; j--) {
		for (int i = ixABoundaryBegin; i < ixABoundaryEnd; i++) {
#pragma simd
			for (int k = 0; k < sRElements; k++) {
				data(i,j,k) =
					m_dRecvBuffer[m_ixRecvBuffer + k];
			}
			m_ixRecvBuffer += sRElements;
		}
		}

	// Unpack data from bottom-left
	} else if (m_dir == Direction_BottomLeft) {
		ixABoundaryBegin = m_ixFirst - m_sHaloElements;
		ixABoundaryEnd   = m_ixFirst;
		ixBBoundaryBegin = m_ixSecond - m_sHaloElements;
		ixBBoundaryEnd   = m_ixSecond;

		// Check that sufficient data remains in send buffer
		int nTotalValues =
			  sRElements
			* (ixABoundaryEnd - ixABoundaryBegin)
			* (ixBBoundaryEnd - ixBBoundaryBegin);

		if (m_ixRecvBuffer + nTotalValues > m_dRecvBuffer.GetRows()) {
			_EXCEPTIONT("Insufficient space in RecvBuffer for operation.");
		}

		// Unpack data
		for (int j = ixBBoundaryBegin; j < ixBBoundaryEnd; j++) {
		for (int i = ixABoundaryBegin; i < ixABoundaryEnd; i++) {
#pragma simd
			for (int k = 0; k < sRElements; k++) {
				data(i,j,k) =
					m_dRecvBuffer[m_ixRecvBuffer + k];
			}
			m_ixRecvBuffer += sRElements;
		}
		}

	// Unpack data from bottom-right
	} else if (m_dir == Direction_BottomRight) {
		ixABoundaryBegin = m_ixFirst + 1;
		ixABoundaryEnd   = m_ixFirst + m_sHaloElements + 1;
		ixBBoundaryBegin = m_ixSecond - m_sHaloElements;
		ixBBoundaryEnd   = m_ixSecond;

		// Check that sufficient data remains in send buffer
		int nTotalValues =
			  sRElements
			* (ixABoundaryEnd - ixABoundaryBegin)
			* (ixBBoundaryEnd - ixBBoundaryBegin);

		if (m_ixRecvBuffer + nTotalValues > m_dRecvBuffer.GetRows()) {
			_EXCEPTIONT("Insufficient space in RecvBuffer for operation.");
		}

		// Unpack data
		for (int j = ixBBoundaryBegin; j < ixBBoundaryEnd; j++) {
		for (int i = ixABoundaryEnd-1; i >= ixABoundaryBegin; i--) {
#pragma simd
			for (int k = 0; k < sRElements; k++) {
				data(i,j,k) =
					m_dRecvBuffer[m_ixRecvBuffer + k];
			}
			m_ixRecvBuffer += sRElements;
		}
		}

	// Invalid direction
	} else {
		_EXCEPTIONT("Invalid direction");
	}
}
void ExchangeBuffer::Pack(
	const DataArray3D<double> & data
) {
	const size_t sAElements = data.GetSize(0);
	const size_t sBElements = data.GetSize(1);
	const size_t sRElements = data.GetSize(2);

	// Check matrix bounds
	if (((m_dir == Direction_Right) || (m_dir == Direction_Left)) &&
		(m_ixSecond > sBElements)
	) {
		_EXCEPTIONT("GridData / ExteriorNeighbor inconsistency.");
	}
	if (((m_dir == Direction_Top) || (m_dir == Direction_Bottom)) &&
		(m_ixSecond > sAElements)
	) {
		_EXCEPTIONT("GridData / ExteriorNeighbor inconsistency.");
	}

	// Index for halo elements along boundary
	int ixBoundaryBegin;
	int ixBoundaryEnd;

	int ixABoundaryBegin;
	int ixABoundaryEnd;
	int ixBBoundaryBegin;
	int ixBBoundaryEnd;

	// Maximum index for radial elements
	int nVarRElements;

	// Pack data to send right
	if (m_dir == Direction_Right) {
		ixBoundaryBegin = sAElements - 2 * m_sHaloElements;
		ixBoundaryEnd   = sAElements - m_sHaloElements;

		// Check that sufficient data remains in send buffer
		int nTotalValues =
			  sRElements
			* (ixBoundaryEnd - ixBoundaryBegin)
			* (m_ixSecond - m_ixFirst);

		if (m_ixSendBuffer + nTotalValues > m_dSendBuffer.GetRows()) {
			_EXCEPTIONT("Insufficient space in SendBuffer for operation.");
		}

		// Pack the SendBuffer
		if (m_fReverseDirection) {
			for (int i = ixBoundaryBegin; i < ixBoundaryEnd; i++) {
			for (int j = m_ixSecond-1; j >= m_ixFirst; j--) {
#pragma simd
				for (int k = 0; k < sRElements; k++) {
					m_dSendBuffer[m_ixSendBuffer + k] =
						data(i,j,k);
				}
				m_ixSendBuffer += sRElements;
			}
			}

		} else {
			for (int i = ixBoundaryBegin; i < ixBoundaryEnd; i++) {
			for (int j = m_ixFirst; j < m_ixSecond; j++) {
#pragma simd
				for (int k = 0; k < sRElements; k++) {
					m_dSendBuffer[m_ixSendBuffer + k] =
						data(i,j,k);	
				}
				m_ixSendBuffer += sRElements;
			}
			}
		}
	
	// Pack data to send topward
	} else if (m_dir == Direction_Top) {
		ixBoundaryBegin = sBElements - 2 * m_sHaloElements;
		ixBoundaryEnd   = sBElements - m_sHaloElements;

		// Check that sufficient data remains in send buffer
		int nTotalValues =
			  sRElements
			* (ixBoundaryEnd - ixBoundaryBegin)
			* (m_ixSecond - m_ixFirst);

		if (m_ixSendBuffer + nTotalValues > m_dSendBuffer.GetRows()) {
			_EXCEPTIONT("Insufficient space in SendBuffer for operation.");
		}

		// Pack the SendBuffer
		if (m_fReverseDirection) {
			for (int j = ixBoundaryBegin; j < ixBoundaryEnd; j++) {
			for (int i = m_ixSecond-1; i >= m_ixFirst; i--) {
#pragma simd
				for (int k = 0; k < sRElements; k++) {
					m_dSendBuffer[m_ixSendBuffer + k] =
						data(i,j,k);
				}
				m_ixSendBuffer += sRElements;
			}
			}

		} else {
			for (int j = ixBoundaryBegin; j < ixBoundaryEnd; j++) {
			for (int i = m_ixFirst; i < m_ixSecond; i++) {
#pragma simd
				for (int k = 0; k < sRElements; k++) {
					m_dSendBuffer[m_ixSendBuffer + k] =
						data(i,j,k);
				}
				m_ixSendBuffer += sRElements;
			}
			}
		}

	// Pack data to send left
	} else if (m_dir == Direction_Left) {
		ixBoundaryBegin = m_sHaloElements;
		ixBoundaryEnd   = 2 * m_sHaloElements;

		// Check that sufficient data remains in send buffer
		int nTotalValues =
			  sRElements
			* (ixBoundaryEnd - ixBoundaryBegin)
			* (m_ixSecond - m_ixFirst);

		if (m_ixSendBuffer + nTotalValues > m_dSendBuffer.GetRows()) {
			_EXCEPTIONT("Insufficient space in SendBuffer for operation.");
		}

		// Pack the SendBuffer
		if (m_fReverseDirection) {
			for (int i = ixBoundaryEnd-1; i >= ixBoundaryBegin; i--) {
			for (int j = m_ixSecond-1; j >= m_ixFirst; j--) {
#pragma simd
				for (int k = 0; k < sRElements; k++) {
					m_dSendBuffer[m_ixSendBuffer + k] =
						data(i,j,k);
				}
				m_ixSendBuffer += sRElements;
			}
			}

		} else {
			for (int i = ixBoundaryEnd-1; i >= ixBoundaryBegin; i--) {
			for (int j = m_ixFirst; j < m_ixSecond; j++) {
#pragma simd
				for (int k = 0; k < sRElements; k++) {
					m_dSendBuffer[m_ixSendBuffer + k] =
						data(i,j,k);
				}
				m_ixSendBuffer += sRElements;
			}
			}
		}

	// Pack data to send bottomward
	} else if (m_dir == Direction_Bottom) {
		ixBoundaryBegin = m_sHaloElements;
		ixBoundaryEnd   = 2 * m_sHaloElements;

		// Check that sufficient data remains in send buffer
		int nTotalValues =
			  sRElements
			* (ixBoundaryEnd - ixBoundaryBegin)
			* (m_ixSecond - m_ixFirst);

		if (m_ixSendBuffer + nTotalValues > m_dSendBuffer.GetRows()) {
			_EXCEPTIONT("Insufficient space in SendBuffer for operation.");
		}

		// Pack the SendBuffer
		if (m_fReverseDirection) {
			for (int j = ixBoundaryEnd-1; j >= ixBoundaryBegin; j--) {
			for (int i = m_ixSecond-1; i >= m_ixFirst; i--) {
#pragma simd
				for (int k = 0; k < sRElements; k++) {
					m_dSendBuffer[m_ixSendBuffer + k] =
						data(i,j,k);
				}
				m_ixSendBuffer += sRElements;
			}
			}

		} else {
			for (int j = ixBoundaryEnd-1; j >= ixBoundaryBegin; j--) {
			for (int i = m_ixFirst; i < m_ixSecond; i++) {
#pragma simd
				for (int k = 0; k < sRElements; k++) {
					m_dSendBuffer[m_ixSendBuffer + k] =
						data(i,j,k);
				}
				m_ixSendBuffer += sRElements;
			}
			}
		}

	// Pack data to send toprightward 
	} else if (m_dir == Direction_TopRight) {
		ixABoundaryBegin = m_ixFirst - m_sHaloElements + 1;
		ixABoundaryEnd   = m_ixFirst + 1;
		ixBBoundaryBegin = m_ixSecond - m_sHaloElements + 1;
		ixBBoundaryEnd   = m_ixSecond + 1;
 
		// Check that sufficient data remains in send buffer
		int nTotalValues =
			  sRElements
			* (ixABoundaryEnd - ixABoundaryBegin)
			* (ixBBoundaryEnd - ixBBoundaryBegin);

		if (m_ixSendBuffer + nTotalValues > m_dSendBuffer.GetRows()) {
			_EXCEPTIONT("Insufficient space in SendBuffer for operation.");
		}

		// Pack the SendBuffer
		if (m_fReverseDirection) {
			for (int i = ixABoundaryBegin; i < ixABoundaryEnd; i++) {
			for (int j = ixBBoundaryBegin; j < ixBBoundaryEnd; j++) {
#pragma simd
				for (int k = 0; k < sRElements; k++) {
					m_dSendBuffer[m_ixSendBuffer + k] =
						data(i,j,k);
				}
				m_ixSendBuffer += sRElements;
			}
			}

		} else {
			for (int j = ixBBoundaryBegin; j < ixBBoundaryEnd; j++) {
			for (int i = ixABoundaryBegin; i < ixABoundaryEnd; i++) {
#pragma simd
				for (int k = 0; k < sRElements; k++) {
					m_dSendBuffer[m_ixSendBuffer + k] =
						data(i,j,k);
				}
				m_ixSendBuffer += sRElements;
			}
			}
		}

	// Pack data to send topleftward 
	} else if (m_dir == Direction_TopLeft) {
		ixABoundaryBegin = m_ixFirst;
		ixABoundaryEnd   = m_ixFirst + m_sHaloElements;
		ixBBoundaryBegin = m_ixSecond - m_sHaloElements + 1;
		ixBBoundaryEnd   = m_ixSecond + 1;
 
		// Check that sufficient data remains in send buffer
		int nTotalValues =
			  sRElements
			* (ixABoundaryEnd - ixABoundaryBegin)
			* (ixBBoundaryEnd - ixBBoundaryBegin);

		if (m_ixSendBuffer + nTotalValues > m_dSendBuffer.GetRows()) {
			_EXCEPTIONT("Insufficient space in SendBuffer for operation.");
		}

		// Pack the SendBuffer
		if (m_fReverseDirection) {
			for (int i = ixABoundaryEnd-1; i >= ixABoundaryBegin; i--) {
			for (int j = ixBBoundaryBegin; j < ixBBoundaryEnd; j++) {
#pragma simd
				for (int k = 0; k < sRElements; k++) {
					m_dSendBuffer[m_ixSendBuffer + k] =
						data(i,j,k);
				}
				m_ixSendBuffer += sRElements;
			}
			}

		} else {
			for (int j = ixBBoundaryBegin; j < ixBBoundaryEnd; j++) {
			for (int i = ixABoundaryEnd-1; i >= ixABoundaryBegin; i--) {
#pragma simd
				for (int k = 0; k < sRElements; k++) {
					m_dSendBuffer[m_ixSendBuffer + k] =
						data(i,j,k);
				}
				m_ixSendBuffer += sRElements;
			}
			}
		}

	// Pack data to send bottomleftward 
	} else if (m_dir == Direction_BottomLeft) {
		ixABoundaryBegin = m_ixFirst;
		ixABoundaryEnd   = m_ixFirst + m_sHaloElements;
		ixBBoundaryBegin = m_ixSecond;
		ixBBoundaryEnd   = m_ixSecond + m_sHaloElements;
 
		// Check that sufficient data remains in send buffer
		int nTotalValues =
			  sRElements
			* (ixABoundaryEnd - ixABoundaryBegin)
			* (ixBBoundaryEnd - ixBBoundaryBegin);

		if (m_ixSendBuffer + nTotalValues > m_dSendBuffer.GetRows()) {
			_EXCEPTIONT("Insufficient space in SendBuffer for operation.");
		}

		// Pack the SendBuffer
		if (m_fReverseDirection) {
			for (int i = ixABoundaryEnd-1; i >= ixABoundaryBegin; i--) {
			for (int j = ixBBoundaryEnd-1; j >= ixBBoundaryBegin; j--) {
#pragma simd
				for (int k = 0; k < sRElements; k++) {
					m_dSendBuffer[m_ixSendBuffer + k] =
						data(i,j,k);
				}
				m_ixSendBuffer += sRElements;
			}
			}

		} else {
			for (int j = ixBBoundaryEnd-1; j >= ixBBoundaryBegin; j--) {
			for (int i = ixABoundaryEnd-1; i >= ixABoundaryBegin; i--) {
#pragma simd
				for (int k = 0; k < sRElements; k++) {
					m_dSendBuffer[m_ixSendBuffer + k] =
						data(i,j,k);
				}
				m_ixSendBuffer += sRElements;
			}
			}
		}

	// Pack data to send bottomrightward 
	} else if (m_dir == Direction_BottomRight) {
		ixABoundaryBegin = m_ixFirst - m_sHaloElements + 1;
		ixABoundaryEnd   = m_ixFirst + 1;
		ixBBoundaryBegin = m_ixSecond;
		ixBBoundaryEnd   = m_ixSecond + m_sHaloElements;
 
		// Check that sufficient data remains in send buffer
		int nTotalValues =
			  sRElements
			* (ixABoundaryEnd - ixABoundaryBegin)
			* (ixBBoundaryEnd - ixBBoundaryBegin);

		if (m_ixSendBuffer + nTotalValues > m_dSendBuffer.GetRows()) {
			_EXCEPTIONT("Insufficient space in SendBuffer for operation.");
		}

		// Pack the SendBuffer
		if (m_fReverseDirection) {
			for (int i = ixABoundaryBegin; i < ixABoundaryEnd; i++) {
			for (int j = ixBBoundaryEnd-1; j >= ixBBoundaryBegin; j--) {
#pragma simd
				for (int k = 0; k < sRElements; k++) {
					m_dSendBuffer[m_ixSendBuffer + k] =
						data(i,j,k);
				}
				m_ixSendBuffer += sRElements;
			}
			}

		} else {
			for (int j = ixBBoundaryEnd-1; j >= ixBBoundaryBegin; j--) {
			for (int i = ixABoundaryBegin; i < ixABoundaryEnd; i++) {
#pragma simd
				for (int k = 0; k < sRElements; k++) {
					m_dSendBuffer[m_ixSendBuffer + k] =
						data(i,j,k);
				}
				m_ixSendBuffer += sRElements;
			}
			}
		}

	// Invalid direction
	} else {
		_EXCEPTIONT("Invalid direction");
	}
}
int main(int argc, char ** argv) {

	MPI_Init(&argc, &argv);

	NcError error(NcError::silent_nonfatal);

try {

	// Input filename
	std::string strInputFile;

	// Output filename
	std::string strOutputFile;

	// Separate topography file
	std::string strTopographyFile;

	// List of variables to extract
	std::string strVariables;

	// Extract geopotential height
	bool fGeopotentialHeight;

	// Pressure levels to extract
	std::string strPressureLevels;

	// Height levels to extract
	std::string strHeightLevels;

	// Extract variables at the surface
	bool fExtractSurface;

	// Extract total energy
	bool fExtractTotalEnergy;

	// Parse the command line
	BeginCommandLine()
		CommandLineString(strInputFile, "in", "");
		CommandLineString(strOutputFile, "out", "");
		CommandLineString(strVariables, "var", "");
		CommandLineBool(fGeopotentialHeight, "output_z");
		CommandLineBool(fExtractTotalEnergy, "output_energy");
		CommandLineString(strPressureLevels, "p", "");
		CommandLineString(strHeightLevels, "z", "");
		CommandLineBool(fExtractSurface, "surf");

		ParseCommandLine(argc, argv);
	EndCommandLine(argv)

	AnnounceBanner();

	// Check command line arguments
	if (strInputFile == "") {
		_EXCEPTIONT("No input file specified");
	}
	if (strOutputFile == "") {
		_EXCEPTIONT("No output file specified");
	}
	if (strVariables == "") {
		_EXCEPTIONT("No variables specified");
	}

	// Parse variable string
	std::vector< std::string > vecVariableStrings;

	ParseVariableList(strVariables, vecVariableStrings);

	// Check variables
	if (vecVariableStrings.size() == 0) {
		_EXCEPTIONT("No variables specified");
	}

	// Parse pressure level string
	std::vector<double> vecPressureLevels;

	ParseLevelArray(strPressureLevels, vecPressureLevels);

	int nPressureLevels = (int)(vecPressureLevels.size());

	for (int k = 0; k < nPressureLevels; k++) {
		if (vecPressureLevels[k] <= 0.0) {
			_EXCEPTIONT("Non-positive pressure values not allowed");
		}
	}

	// Parse height level string
	std::vector<double> vecHeightLevels;

	ParseLevelArray(strHeightLevels, vecHeightLevels);

	int nHeightLevels = (int)(vecHeightLevels.size());

	// Check pressure levels
	if ((nPressureLevels == 0) &&
		(nHeightLevels == 0) &&
		(!fExtractSurface)
	) {
		_EXCEPTIONT("No pressure / height levels to process");
	}

	// Open input file
	AnnounceStartBlock("Loading input file");
	NcFile ncdf_in(strInputFile.c_str(), NcFile::ReadOnly);
	if (!ncdf_in.is_valid()) {
		_EXCEPTION1("Unable to open file \"%s\" for reading",
			strInputFile.c_str());
	}

	// Load time array
	Announce("Time");
	NcVar * varTime = ncdf_in.get_var("time");
	if (varTime == NULL) {
		_EXCEPTION1("File \"%s\" does not contain variable \"time\"",
			strInputFile.c_str());
	}
	int nTime = varTime->get_dim(0)->size();

	DataArray1D<double> dTime(nTime);
	varTime->set_cur((long)0);
	varTime->get(&(dTime[0]), nTime);

	// Load latitude array
	Announce("Latitude");
	NcVar * varLat = ncdf_in.get_var("lat");
	if (varLat == NULL) {
		_EXCEPTION1("File \"%s\" does not contain variable \"lat\"",
			strInputFile.c_str());
	}
	int nLat = varLat->get_dim(0)->size();

	DataArray1D<double> dLat(nLat);
	varLat->set_cur((long)0);
	varLat->get(&(dLat[0]), nLat);

	// Load longitude array
	Announce("Longitude");
	NcVar * varLon = ncdf_in.get_var("lon");
	if (varLon == NULL) {
		_EXCEPTION1("File \"%s\" does not contain variable \"lon\"",
			strInputFile.c_str());
	}
	int nLon = varLon->get_dim(0)->size();

	DataArray1D<double> dLon(nLon);
	varLon->set_cur((long)0);
	varLon->get(&(dLon[0]), nLon);

	// Load level array
	Announce("Level");
	NcVar * varLev = ncdf_in.get_var("lev");
	if (varLev == NULL) {
		_EXCEPTION1("File \"%s\" does not contain variable \"lev\"",
			strInputFile.c_str());
	}
	int nLev = varLev->get_dim(0)->size();

	DataArray1D<double> dLev(nLev);
	varLev->set_cur((long)0);
	varLev->get(&(dLev[0]), nLev);

	// Load level interface array
	Announce("Interface");
	NcVar * varILev = ncdf_in.get_var("ilev");
	int nILev = 0;
	DataArray1D<double> dILev;
	if (varILev == NULL) {
		Announce("Warning: Variable \"ilev\" not found");
	} else {
		nILev = varILev->get_dim(0)->size();
		if (nILev != nLev + 1) {
			_EXCEPTIONT("Variable \"ilev\" must have size lev+1");
		}
		dILev.Allocate(nILev);
		varILev->set_cur((long)0);
		varILev->get(&(dILev[0]), nILev);
	}

	// Load topography
	Announce("Topography");
	NcVar * varZs = ncdf_in.get_var("Zs");
	if (varZs == NULL) {
		_EXCEPTION1("File \"%s\" does not contain variable \"Zs\"",
			strInputFile.c_str());
	}

	DataArray2D<double> dZs(nLat, nLon);
	varZs->set_cur((long)0, (long)0);
	varZs->get(&(dZs[0][0]), nLat, nLon);

	AnnounceEndBlock("Done");

	// Open output file
	AnnounceStartBlock("Constructing output file");

	NcFile ncdf_out(strOutputFile.c_str(), NcFile::Replace);
	if (!ncdf_out.is_valid()) {
		_EXCEPTION1("Unable to open file \"%s\" for writing",
			strOutputFile.c_str());
	}

	CopyNcFileAttributes(&ncdf_in, &ncdf_out);

	// Output time array
	Announce("Time");
	NcDim * dimOutTime = ncdf_out.add_dim("time");
	NcVar * varOutTime = ncdf_out.add_var("time", ncDouble, dimOutTime);
	varOutTime->set_cur((long)0);
	varOutTime->put(&(dTime[0]), nTime);

	CopyNcVarAttributes(varTime, varOutTime);

	// Output pressure array
	NcDim * dimOutP = NULL;
	NcVar * varOutP = NULL;
	if (nPressureLevels > 0) {
		Announce("Pressure");
		dimOutP = ncdf_out.add_dim("p", nPressureLevels);
		varOutP = ncdf_out.add_var("p", ncDouble, dimOutP);
		varOutP->set_cur((long)0);
		varOutP->put(&(vecPressureLevels[0]), nPressureLevels);
	}

	// Output height array
	NcDim * dimOutZ = NULL;
	NcVar * varOutZ = NULL;
	if (nHeightLevels > 0) {
		Announce("Height");
		dimOutZ = ncdf_out.add_dim("z", nHeightLevels);
		varOutZ = ncdf_out.add_var("z", ncDouble, dimOutZ);
		varOutZ->set_cur((long)0);
		varOutZ->put(&(vecHeightLevels[0]), nHeightLevels);
	}

	// Output latitude and longitude array
	Announce("Latitude");
	NcDim * dimOutLat = ncdf_out.add_dim("lat", nLat);
	NcVar * varOutLat = ncdf_out.add_var("lat", ncDouble, dimOutLat);
	varOutLat->set_cur((long)0);
	varOutLat->put(&(dLat[0]), nLat);

	CopyNcVarAttributes(varLat, varOutLat);

	Announce("Longitude");
	NcDim * dimOutLon = ncdf_out.add_dim("lon", nLon);
	NcVar * varOutLon = ncdf_out.add_var("lon", ncDouble, dimOutLon);
	varOutLon->set_cur((long)0);
	varOutLon->put(&(dLon[0]), nLon);

	CopyNcVarAttributes(varLon, varOutLon);

	// Output topography
	Announce("Topography");
	NcVar * varOutZs = ncdf_out.add_var(
		"Zs", ncDouble, dimOutLat, dimOutLon);

	varOutZs->set_cur((long)0, (long)0);
	varOutZs->put(&(dZs[0][0]), nLat, nLon);

	AnnounceEndBlock("Done");

	// Done
	AnnounceEndBlock("Done");

	// Load all variables
	Announce("Loading variables");

	std::vector<NcVar *> vecNcVar;
	for (int v = 0; v < vecVariableStrings.size(); v++) {
		vecNcVar.push_back(ncdf_in.get_var(vecVariableStrings[v].c_str()));
		if (vecNcVar[v] == NULL) {
			_EXCEPTION1("Unable to load variable \"%s\" from file",
				vecVariableStrings[v].c_str());
		}
	}

	// Physical constants
	Announce("Initializing thermodynamic variables");

	NcAtt * attEarthRadius = ncdf_in.get_att("earth_radius");
	double dEarthRadius = attEarthRadius->as_double(0);

	NcAtt * attRd = ncdf_in.get_att("Rd");
	double dRd = attRd->as_double(0);

	NcAtt * attCp = ncdf_in.get_att("Cp");
	double dCp = attCp->as_double(0);

	double dGamma = dCp / (dCp - dRd);

	NcAtt * attP0 = ncdf_in.get_att("P0");
	double dP0 = attP0->as_double(0);

	double dPressureScaling = dP0 * std::pow(dRd / dP0, dGamma);

	NcAtt * attZtop = ncdf_in.get_att("Ztop");
	double dZtop = attZtop->as_double(0);

	// Input data
	DataArray3D<double> dataIn(nLev, nLat, nLon);
	DataArray3D<double> dataInt(nILev, nLat, nLon);

	// Output data
	DataArray2D<double> dataOut(nLat, nLon);

	// Pressure in column
	DataArray1D<double> dataColumnP(nLev);

	// Height in column
	DataArray1D<double> dataColumnZ(nLev);
	DataArray1D<double> dataColumnIZ(nILev);

	// Column weights
	DataArray1D<double> dW(nLev);
	DataArray1D<double> dIW(nILev);

	// Loop through all times, pressure levels and variables
	AnnounceStartBlock("Interpolating");

	// Add energy variable
	NcVar * varEnergy;
	if (fExtractTotalEnergy) {
		varEnergy = ncdf_out.add_var("TE", ncDouble, dimOutTime);
	}

	// Create output pressure variables
	std::vector<NcVar *> vecOutNcVarP;
	if (nPressureLevels > 0) {
		for (int v = 0; v < vecVariableStrings.size(); v++) {
			vecOutNcVarP.push_back(
				ncdf_out.add_var(
					vecVariableStrings[v].c_str(), ncDouble,
						dimOutTime, dimOutP, dimOutLat, dimOutLon));

			// Copy attributes
			CopyNcVarAttributes(vecNcVar[v], vecOutNcVarP[v]);
		}
	}

	// Create output height variables
	std::vector<NcVar *> vecOutNcVarZ;
	if (nHeightLevels > 0) {
		for (int v = 0; v < vecVariableStrings.size(); v++) {
			std::string strVarName = vecVariableStrings[v];
			if (nPressureLevels > 0) {
				strVarName += "z";
			}
			vecOutNcVarZ.push_back(
				ncdf_out.add_var(
					strVarName.c_str(), ncDouble,
						dimOutTime, dimOutZ, dimOutLat, dimOutLon));

			// Copy attributes
			CopyNcVarAttributes(vecNcVar[v], vecOutNcVarZ[v]);
		}
	}

	// Create output surface variable
	std::vector<NcVar *> vecOutNcVarS;
	if (fExtractSurface) {
		for (int v = 0; v < vecVariableStrings.size(); v++) {
			std::string strVarName = vecVariableStrings[v];
			strVarName += "S";

			vecOutNcVarS.push_back(
				ncdf_out.add_var(
					strVarName.c_str(), ncDouble,
						dimOutTime, dimOutLat, dimOutLon));

			// Copy attributes
			CopyNcVarAttributes(vecNcVar[v], vecOutNcVarS[v]);
		}
	}

	// Loop over all times
	for (int t = 0; t < nTime; t++) {

		char szAnnounce[256];
		sprintf(szAnnounce, "Time %i", t); 
		AnnounceStartBlock(szAnnounce);

		// Rho
		DataArray3D<double> dataRho(nLev, nLat, nLon);

		NcVar * varRho = ncdf_in.get_var("Rho");
		if (varRho == NULL) {
			_EXCEPTIONT("Unable to load variable \"Rho\" from file");
		}
		varRho->set_cur(t, 0, 0, 0);
		varRho->get(&(dataRho[0][0][0]), 1, nLev, nLat, nLon);

		// Pressure
		DataArray3D<double> dataP(nLev, nLat, nLon);

		if (nPressureLevels != 0) {
			NcVar * varP = ncdf_in.get_var("P");
			if (varP == NULL) {
				_EXCEPTIONT("Unable to load variable \"P\" from file");
			}
			varP->set_cur(t, 0, 0, 0);
			varP->get(&(dataP[0][0][0]), 1, nLev, nLat, nLon);
		}
/*
		// Populate pressure array
		if (nPressureLevels > 0) {

			// Calculate pointwise pressure
			for (int k = 0; k < nLev; k++) {
			for (int i = 0; i < nLat; i++) {
			for (int j = 0; j < nLon; j++) {
				dataP[k][i][j] = dPressureScaling
					* exp(log(dataRho[k][i][j] * dataP[k][i][j]) * dGamma);
			}
			}
			}
		}
*/
		// Height everywhere
		DataArray3D<double> dataZ(nLev, nLat, nLon);
		DataArray3D<double> dataIZ;
		if (nILev != 0) {
			dataIZ.Allocate(nILev, nLat, nLon);
		}

		// Populate height array
		if ((nHeightLevels > 0) || (fGeopotentialHeight)) {
			for (int k = 0; k < nLev; k++) {
			for (int i = 0; i < nLat; i++) {
			for (int j = 0; j < nLon; j++) {
				dataZ[k][i][j] = dZs[i][j] + dLev[k] * (dZtop - dZs[i][j]);
			}
			}
			}

			for (int k = 0; k < nILev; k++) {
			for (int i = 0; i < nLat; i++) {
			for (int j = 0; j < nLon; j++) {
				dataIZ[k][i][j] = dZs[i][j] + dILev[k] * (dZtop - dZs[i][j]);
			}
			}
			}
		}

		// Loop through all pressure levels and variables
		for (int v = 0; v < vecNcVar.size(); v++) {

			bool fOnInterfaces = false;

			// Load in the data array
			vecNcVar[v]->set_cur(t, 0, 0, 0);

			if (vecNcVar[v]->get_dim(1)->size() == nLev) {
				vecNcVar[v]->get(&(dataIn[0][0][0]), 1, nLev, nLat, nLon);

				Announce("%s (n)", vecVariableStrings[v].c_str());

			} else if (vecNcVar[v]->get_dim(1)->size() == nILev) {
				vecNcVar[v]->get(&(dataInt[0][0][0]), 1, nILev, nLat, nLon);
				fOnInterfaces = true;

				Announce("%s (i)", vecVariableStrings[v].c_str());
			} else {
				_EXCEPTION1("Variable \"%s\" has invalid level dimension",
					vecVariableStrings[v].c_str());
			}

			// At the physical surface
			if (fExtractSurface) {

				if (fOnInterfaces) {
					for (int i = 0; i < nLat; i++) {
					for (int j = 0; j < nLon; j++) {
						dataOut[i][j] = dataInt[0][i][j];
					}
					}

				} else {

					int kBegin = 0;
					int kEnd = 3;

					PolynomialInterp::LagrangianPolynomialCoeffs(
						3, dLev, dW, 0.0);

					// Loop thorugh all latlon indices
					for (int i = 0; i < nLat; i++) {
					for (int j = 0; j < nLon; j++) {

						// Interpolate in the vertical
						dataOut[i][j] = 0.0;
						for (int k = kBegin; k < kEnd; k++) {
							dataOut[i][j] += dW[k] * dataIn[k][i][j];
						}
					}
					}
				}

				// Write variable
				vecOutNcVarS[v]->set_cur(t, 0, 0);
				vecOutNcVarS[v]->put(&(dataOut[0][0]), 1, nLat, nLon);

			}

			// Loop through all pressure levels
			for (int p = 0; p < nPressureLevels; p++) {

				// Loop thorugh all latlon indices
				for (int i = 0; i < nLat; i++) {
				for (int j = 0; j < nLon; j++) {

					// Store column pressure
					for (int k = 0; k < nLev; k++) {
						dataColumnP[k] = dataP[k][i][j];
					}

					// Find weights
					int kBegin = 0;
					int kEnd = 0;

					// On a pressure surface
					InterpolationWeightsLinear(
						vecPressureLevels[p],
						dataColumnP,
						kBegin,
						kEnd,
						dW);

					// Interpolate in the vertical
					dataOut[i][j] = 0.0;
					for (int k = kBegin; k < kEnd; k++) {
						dataOut[i][j] += dW[k] * dataIn[k][i][j];
					}

				}
				}

				// Write variable
				vecOutNcVarP[v]->set_cur(t, p, 0, 0);
				vecOutNcVarP[v]->put(&(dataOut[0][0]), 1, 1, nLat, nLon);
			}

			// Loop through all height levels
			for (int z = 0; z < nHeightLevels; z++) {

				// Loop thorugh all latlon indices
				for (int i = 0; i < nLat; i++) {
				for (int j = 0; j < nLon; j++) {

					// Find weights
					int kBegin = 0;
					int kEnd = 0;

					// Interpolate from levels to z surfaces
					if (!fOnInterfaces) {
						for (int k = 0; k < nLev; k++) {
							dataColumnZ[k] = dataZ[k][i][j];
						}

						InterpolationWeightsLinear(
							vecHeightLevels[z],
							dataColumnZ,
							kBegin,
							kEnd,
							dW);

						dataOut[i][j] = 0.0;
						for (int k = kBegin; k < kEnd; k++) {
							dataOut[i][j] += dW[k] * dataIn[k][i][j];
						}

					// Interpolate from interfaces to z surfaces
					} else {
						for (int k = 0; k < nILev; k++) {
							dataColumnIZ[k] = dataIZ[k][i][j];
						}

						InterpolationWeightsLinear(
							vecHeightLevels[z],
							dataColumnIZ,
							kBegin,
							kEnd,
							dIW);

						dataOut[i][j] = 0.0;
						for (int k = kBegin; k < kEnd; k++) {
							dataOut[i][j] += dIW[k] * dataInt[k][i][j];
						}
					}
				}
				}

				// Write variable
				vecOutNcVarZ[v]->set_cur(t, z, 0, 0);
				vecOutNcVarZ[v]->put(&(dataOut[0][0]), 1, 1, nLat, nLon);
			}
		}

		// Output geopotential height
		if (fGeopotentialHeight) {

			Announce("Geopotential height");

			// Output variables
			NcVar * varOutZ;
			NcVar * varOutZs;

			if (nPressureLevels > 0) {
				varOutZ = ncdf_out.add_var(
					"PHIZ", ncDouble, dimOutTime, dimOutP, dimOutLat, dimOutLon);
			}
			if (fExtractSurface) {
				varOutZs = ncdf_out.add_var(
					"PHIZS", ncDouble, dimOutTime, dimOutLat, dimOutLon);
			}

			// Interpolate onto pressure levels
			for (int p = 0; p < nPressureLevels; p++) {

				// Loop thorugh all latlon indices
				for (int i = 0; i < nLat; i++) {
				for (int j = 0; j < nLon; j++) {

					int kBegin = 0;
					int kEnd = 0;

					for (int k = 0; k < nLev; k++) {
						dataColumnP[k] = dataP[k][i][j];
					}

					InterpolationWeightsLinear(
						vecPressureLevels[p],
						dataColumnP,
						kBegin,
						kEnd,
						dW);

					// Interpolate in the vertical
					dataOut[i][j] = 0.0;
					for (int k = kBegin; k < kEnd; k++) {
						dataOut[i][j] += dW[k] * dataZ[k][i][j];
					}
				}
				}

				// Write variable
				varOutZ->set_cur(t, p, 0, 0);
				varOutZ->put(&(dataOut[0][0]), 1, 1, nLat, nLon);

			}

			// Interpolate onto the physical surface
			if (fExtractSurface) {

				int kBegin = 0;
				int kEnd = 3;

				PolynomialInterp::LagrangianPolynomialCoeffs(
					3, dLev, dW, 0.0);

				// Loop thorugh all latlon indices
				for (int i = 0; i < nLat; i++) {
				for (int j = 0; j < nLon; j++) {

					// Interpolate in the vertical
					dataOut[i][j] = 0.0;
					for (int k = kBegin; k < kEnd; k++) {
						dataOut[i][j] += dW[k] * dataZ[k][i][j];
					}
				}
				}

				// Write variable
				varOutZs->set_cur(t, 0, 0);
				varOutZs->put(&(dataOut[0][0]), 1, nLat, nLon);

			}
		}

		// Extract total energy
		if (fExtractTotalEnergy) {
			Announce("Total Energy");

			// Zonal velocity
			DataArray3D<double> dataU(nLev, nLat, nLon);

			NcVar * varU = ncdf_in.get_var("U");
			varU->set_cur(t, 0, 0, 0);
			varU->get(&(dataU[0][0][0]), 1, nLev, nLat, nLon);

			// Meridional velocity
			DataArray3D<double> dataV(nLev, nLat, nLon);

			NcVar * varV = ncdf_in.get_var("V");
			varV->set_cur(t, 0, 0, 0);
			varV->get(&(dataV[0][0][0]), 1, nLev, nLat, nLon);

			// Vertical velocity
			DataArray3D<double> dataW(nLev, nLat, nLon);

			NcVar * varW = ncdf_in.get_var("W");
			varW->set_cur(t, 0, 0, 0);
			varW->get(&(dataW[0][0][0]), 1, nLev, nLat, nLon);

			// Calculate total energy
			double dTotalEnergy = 0.0;

			double dElementRefArea =
				dEarthRadius * dEarthRadius
				* M_PI / static_cast<double>(nLat)
				* 2.0 * M_PI / static_cast<double>(nLon);

			for (int k = 0; k < nLev; k++) {
			for (int i = 0; i < nLat; i++) {
			for (int j = 0; j < nLon; j++) {
				double dKineticEnergy =
					0.5 * dataRho[k][i][j] *
						( dataU[k][i][j] * dataU[k][i][j]
						+ dataV[k][i][j] * dataV[k][i][j]
						+ dataW[k][i][j] * dataW[k][i][j]);

				double dInternalEnergy =
					dataP[k][i][j] / (dGamma - 1.0);

				dTotalEnergy +=
					(dKineticEnergy + dInternalEnergy)
						* std::cos(M_PI * dLat[i] / 180.0) * dElementRefArea
						* (dZtop - dZs[i][j]) / static_cast<double>(nLev);
			}
			}
			}

			// Put total energy into file
			varEnergy->set_cur(t);
			varEnergy->put(&dTotalEnergy, 1);
		}

		AnnounceEndBlock("Done");
	}

	AnnounceEndBlock("Done");

} catch(Exception & e) {
	Announce(e.ToString().c_str());
}

	// Finalize MPI
	MPI_Finalize();
}
Exemple #6
0
void GridCartesianGLL::ApplyDSS(
	int iDataUpdate,
	DataType eDataType
) {
	// Exchange data between nodes
	Exchange(eDataType, iDataUpdate);

	// Post-process velocities across panel edges and
	// perform direct stiffness summation (DSS)
	for (int n = 0; n < GetActivePatchCount(); n++) {
		GridPatchCartesianGLL * pPatch =
			dynamic_cast<GridPatchCartesianGLL*>(GetActivePatch(n));

		const PatchBox & box = pPatch->GetPatchBox();

		// Patch-specific quantities
		int nElementCountA = pPatch->GetElementCountA();
		int nElementCountB = pPatch->GetElementCountB();

		// Apply panel transforms to velocity data
		if (eDataType == DataType_State) {
			pPatch->TransformHaloVelocities(iDataUpdate);
		}
		if (eDataType == DataType_TopographyDeriv) {
			pPatch->TransformTopographyDeriv();
		}

		// Loop through all components associated with this DataType
		int nComponents;
		if (eDataType == DataType_State) {
			nComponents = m_model.GetEquationSet().GetComponents();
		} else if (eDataType == DataType_Tracers) {
			nComponents = m_model.GetEquationSet().GetTracers();
		} else if (eDataType == DataType_Vorticity) {
			nComponents = 1;
		} else if (eDataType == DataType_Divergence) {
			nComponents = 1;
		} else if (eDataType == DataType_TopographyDeriv) {
			nComponents = 2;
		} else {
			_EXCEPTIONT("Invalid DataType");
		}

		// Apply BC only to state DSS
		if (eDataType == DataType_State) {
			pPatch->ApplyBoundaryConditions(iDataUpdate, DataType_State, n);
		}

		// Perform Direct Stiffness Summation (DSS)
		for (int c = 0; c < nComponents; c++) {

			// Obtain the array of working data
			int nRElements = GetRElements();

			DataArray3D<double> pDataUpdate;

			if ((eDataType == DataType_State) &&
				(GetVarLocation(c) == DataLocation_REdge)
			) {
				nRElements++;
			}
			if (eDataType == DataType_TopographyDeriv) {
				nRElements = 2;
			}

			pDataUpdate.SetSize(
				nRElements,
				box.GetATotalWidth(),
				box.GetBTotalWidth());

			// State data
			if (eDataType == DataType_State) {
				DataArray4D<double> & dState =
					pPatch->GetDataState(iDataUpdate, GetVarLocation(c));

				pDataUpdate.AttachToData(&(dState[c][0][0][0]));

			// Tracer data
			} else if (eDataType == DataType_Tracers) {
				DataArray4D<double> & dTracers =
					pPatch->GetDataTracers(iDataUpdate);

				pDataUpdate.AttachToData(&(dTracers[c][0][0][0]));

			// Vorticity data
			} else if (eDataType == DataType_Vorticity) {
				DataArray3D<double> & dVorticity =
					pPatch->GetDataVorticity();

				pDataUpdate.AttachToData(&(dVorticity[0][0][0]));

			// Divergence data
			} else if (eDataType == DataType_Divergence) {
				DataArray3D<double> & dDivergence =
					pPatch->GetDataDivergence();

				pDataUpdate.AttachToData(&(dDivergence[0][0][0]));

			// Topographic derivative data
			} else if (eDataType == DataType_TopographyDeriv) {
				DataArray3D<double> & dTopographyDeriv =
					pPatch->GetTopographyDeriv();

				pDataUpdate.AttachToData(&(dTopographyDeriv[0][0][0]));
			}

			// Averaging DSS across patch boundaries
			for (int k = 0; k < nRElements; k++) {

				// Average in the alpha direction
				for (int a = 0; a <= nElementCountA; a++) {
					int iA = a * m_nHorizontalOrder + box.GetHaloElements();

					// Averaging done at the corners of the panel
					int jBegin = box.GetBInteriorBegin()-1;
					int jEnd = box.GetBInteriorEnd()+1;

					// Perform averaging across edge of patch
					for (int j = jBegin; j < jEnd; j++) {
						pDataUpdate[k][iA][j] = 0.5 * (
							+ pDataUpdate[k][iA  ][j]
							+ pDataUpdate[k][iA-1][j]);

						pDataUpdate[k][iA-1][j] = pDataUpdate[k][iA][j];
					}
				}

				// Average in the beta direction
				for (int b = 0; b <= nElementCountB; b++) {
					int iB = b * m_nHorizontalOrder + box.GetHaloElements();

					// Averaging done at the corners of the panel
					int iBegin = box.GetAInteriorBegin()-1;
					int iEnd = box.GetAInteriorEnd()+1;

					for (int i = iBegin; i < iEnd; i++) {
						pDataUpdate[k][i][iB] = 0.5 * (
							+ pDataUpdate[k][i][iB  ]
							+ pDataUpdate[k][i][iB-1]);

						pDataUpdate[k][i][iB-1] = pDataUpdate[k][i][iB];
					}
				}
			}
		}
	}
}
Exemple #7
0
void GridPatchCSGLL::InterpolateData(
	DataType eDataType,
	const DataArray1D<double> & dREta,
	const DataArray1D<double> & dAlpha,
	const DataArray1D<double> & dBeta,
	const DataArray1D<int> & iPatch,
	DataArray3D<double> & dInterpData,
	DataLocation eOnlyVariablesAt,
	bool fIncludeReferenceState,
	bool fConvertToPrimitive
) {
	if ((dAlpha.GetRows() != dBeta.GetRows()) ||
		(dAlpha.GetRows() != iPatch.GetRows())
	) {
		_EXCEPTIONT("Point vectors must have equivalent length.");
	}

	// Vector for storage interpolated points
	DataArray1D<double> dAInterpCoeffs(m_nHorizontalOrder);
	DataArray1D<double> dBInterpCoeffs(m_nHorizontalOrder);
	DataArray1D<double> dADiffCoeffs(m_nHorizontalOrder);
	DataArray1D<double> dBDiffCoeffs(m_nHorizontalOrder);
	DataArray1D<double> dAInterpPt(m_nHorizontalOrder);

	// Physical constants
	const PhysicalConstants & phys = m_grid.GetModel().GetPhysicalConstants();

	// Perform interpolation on all variables
	int nComponents = 0;
	int nRElements = m_grid.GetRElements();

	// Discretization type
	Grid::VerticalDiscretization eVerticalDiscType =
		m_grid.GetVerticalDiscretization();

	// State Data: Perform interpolation on all variables
	if (eDataType == DataType_State) {
		nComponents = m_datavecStateNode[0].GetSize(0);
		nRElements = m_grid.GetRElements() + 1;

	// Tracer Data: Perform interpolation on all variables
	} else if (eDataType == DataType_Tracers) {
		nComponents = m_datavecTracers[0].GetSize(0);

	// Topography Data
	} else if (eDataType == DataType_Topography) {
		nComponents = 1;
		nRElements = 1;

	// Vorticity Data
	} else if (eDataType == DataType_Vorticity) {
		nComponents = 1;

	// Divergence Data
	} else if (eDataType == DataType_Divergence) {
		nComponents = 1;

	// Temperature Data
	} else if (eDataType == DataType_Temperature) {
		nComponents = 1;

	// Surface Pressure Data
	} else if (eDataType == DataType_SurfacePressure) {
		nComponents = 1;
		nRElements = 1;

	// 2D User Data
	} else if (eDataType == DataType_Auxiliary2D) {
		nComponents = m_dataUserData2D.GetSize(0);
		nRElements = 1;

	} else {
		_EXCEPTIONT("Invalid DataType");
	}

	// Buffer storage in column
	DataArray1D<double> dColumnDataOut(dREta.GetRows());

	// Loop through all components
	for (int c = 0; c < nComponents; c++) {

		DataLocation eDataLocation = DataLocation_Node;

		if (eDataType == DataType_State) {
			eDataLocation = m_grid.GetVarLocation(c);

			// Exclude variables not at the specified DataLocation
			if ((eOnlyVariablesAt != DataLocation_None) &&
			    (eOnlyVariablesAt != eDataLocation)
			) {
				continue;
			}

			// Adjust RElements depending on state data location
			if (eDataLocation == DataLocation_Node) {
				nRElements = m_grid.GetRElements();
			} else if (eDataLocation == DataLocation_REdge) {
				nRElements = m_grid.GetRElements() + 1;
			} else {
				_EXCEPTIONT("Invalid DataLocation");
			}
		}

		// Vertical interpolation operator
		LinearColumnInterpFEM opInterp;

		if (nRElements != 1) {

			// Finite element interpolation
			if (eVerticalDiscType ==
				Grid::VerticalDiscretization_FiniteElement
			) {
				if (eDataLocation == DataLocation_Node) {
					opInterp.Initialize(
						LinearColumnInterpFEM::InterpSource_Levels,
						m_nVerticalOrder,
						m_grid.GetREtaLevels(),
						m_grid.GetREtaInterfaces(),
						dREta);

				} else if (eDataLocation == DataLocation_REdge) {
					opInterp.Initialize(
						LinearColumnInterpFEM::InterpSource_Interfaces,
						m_nVerticalOrder,
						m_grid.GetREtaLevels(),
						m_grid.GetREtaInterfaces(),
						dREta);

				} else {
					_EXCEPTIONT("Invalid DataLocation");
				}

			// Finite volume interpolation
			} else if (
				eVerticalDiscType ==
				Grid::VerticalDiscretization_FiniteVolume
			) {
				if (eDataLocation == DataLocation_Node) {
					opInterp.Initialize(
						LinearColumnInterpFEM::InterpSource_Levels,
						1,
						m_grid.GetREtaLevels(),
						m_grid.GetREtaInterfaces(),
						dREta);

				} else if (eDataLocation == DataLocation_REdge) {
					opInterp.Initialize(
						LinearColumnInterpFEM::InterpSource_Interfaces,
						1,
						m_grid.GetREtaLevels(),
						m_grid.GetREtaInterfaces(),
						dREta);

				} else {
					_EXCEPTIONT("Invalid DataLocation");
				}

			// Invalid vertical discretization type
			} else {
				_EXCEPTIONT("Invalid VerticalDiscretization");
			}

		} else {
			opInterp.InitializeIdentity(1);
		}

		// Buffer storage in column
		DataArray1D<double> dColumnData(nRElements);

		// Get a pointer to the 3D data structure
		DataArray3D<double> pData;
		DataArray3D<double> pDataRef;

		pData.SetSize(
			nRElements,
			m_box.GetATotalWidth(),
			m_box.GetBTotalWidth());

		pDataRef.SetSize(
			nRElements,
			m_box.GetATotalWidth(),
			m_box.GetBTotalWidth());

		if (eDataType == DataType_State) {
			if (eDataLocation == DataLocation_Node) {
				pData.AttachToData(&(m_datavecStateNode[0][c][0][0][0]));
				pDataRef.AttachToData(&(m_dataRefStateNode[c][0][0][0]));
			} else if (eDataLocation == DataLocation_REdge) {
				pData.AttachToData(&(m_datavecStateREdge[0][c][0][0][0]));
				pDataRef.AttachToData(&(m_dataRefStateREdge[c][0][0][0]));
			} else {
				_EXCEPTIONT("Invalid DataLocation");
			}

		} else if (eDataType == DataType_Tracers) {
			pData.AttachToData(&(m_datavecTracers[0][c][0][0][0]));

		} else if (eDataType == DataType_Topography) {
			pData.AttachToData(&(m_dataTopography[0][0]));

		} else if (eDataType == DataType_Vorticity) {
			pData.AttachToData(&(m_dataVorticity[0][0][0]));

		} else if (eDataType == DataType_Divergence) {
			pData.AttachToData(&(m_dataDivergence[0][0][0]));

		} else if (eDataType == DataType_Temperature) {
			pData.AttachToData(&(m_dataTemperature[0][0][0]));

		} else if (eDataType == DataType_SurfacePressure) {
			pData.AttachToData(&(m_dataSurfacePressure[0][0]));

		} else if (eDataType == DataType_Auxiliary2D) {
			pData.AttachToData(&(m_dataUserData2D[c][0][0]));
		}

		// Loop throught all points
		for (int i = 0; i < dAlpha.GetRows(); i++) {

			// Element index
			if (iPatch[i] != GetPatchIndex()) {
				continue;
			}

			// Verify point lies within domain of patch
			const double Eps = 1.0e-10;
			if ((dAlpha[i] < m_dAEdge[m_box.GetAInteriorBegin()] - Eps) ||
				(dAlpha[i] > m_dAEdge[m_box.GetAInteriorEnd()] + Eps) ||
				(dBeta[i] < m_dBEdge[m_box.GetBInteriorBegin()] - Eps) ||
				(dBeta[i] > m_dBEdge[m_box.GetBInteriorEnd()] + Eps)
			) {
				_EXCEPTIONT("Point out of range");
			}

			// Determine finite element index
			int iA =
				(dAlpha[i] - m_dAEdge[m_box.GetAInteriorBegin()])
					/ GetElementDeltaA();

			int iB =
				(dBeta[i] - m_dBEdge[m_box.GetBInteriorBegin()])
					/ GetElementDeltaB();

			// Bound the index within the element
			if (iA < 0) {
				iA = 0;
			}
			if (iA >= (m_box.GetAInteriorWidth() / m_nHorizontalOrder)) {
				iA = m_box.GetAInteriorWidth() / m_nHorizontalOrder - 1;
			}
			if (iB < 0) {
				iB = 0;
			}
			if (iB >= (m_box.GetBInteriorWidth() / m_nHorizontalOrder)) {
				iB = m_box.GetBInteriorWidth() / m_nHorizontalOrder - 1;
			}

			iA = m_box.GetHaloElements() + iA * m_nHorizontalOrder;
			iB = m_box.GetHaloElements() + iB * m_nHorizontalOrder;

			// Compute interpolation coefficients
			PolynomialInterp::LagrangianPolynomialCoeffs(
				m_nHorizontalOrder,
				&(m_dAEdge[iA]),
				dAInterpCoeffs,
				dAlpha[i]);

			PolynomialInterp::LagrangianPolynomialCoeffs(
				m_nHorizontalOrder,
				&(m_dBEdge[iB]),
				dBInterpCoeffs,
				dBeta[i]);

			// Perform interpolation on all levels
			for (int k = 0; k < nRElements; k++) {

				dColumnData[k] = 0.0;

				// Rescale vertical velocity
				const int WIx = 3;
				if ((c == WIx) && (fConvertToPrimitive)) {
					if (m_grid.GetVarLocation(WIx) == DataLocation_REdge) {
						for (int m = 0; m < m_nHorizontalOrder; m++) {
						for (int n = 0; n < m_nHorizontalOrder; n++) {
							dColumnData[k] +=
								  dAInterpCoeffs[m]
								* dBInterpCoeffs[n]
								* pData[k][iA+m][iB+n]
								/ m_dataDerivRREdge[k][iA][iB][2];
						}
						}

					} else {
						for (int m = 0; m < m_nHorizontalOrder; m++) {
						for (int n = 0; n < m_nHorizontalOrder; n++) {
							dColumnData[k] +=
								  dAInterpCoeffs[m]
								* dBInterpCoeffs[n]
								* pData[k][iA+m][iB+n]
								/ m_dataDerivRNode[k][iA][iB][2];
						}
						}
					}

				} else {
					for (int m = 0; m < m_nHorizontalOrder; m++) {
					for (int n = 0; n < m_nHorizontalOrder; n++) {
						dColumnData[k] +=
							  dAInterpCoeffs[m]
							* dBInterpCoeffs[n]
							* pData[k][iA+m][iB+n];
					}
					}
				}

				// Do not include the reference state
				if ((eDataType == DataType_State) &&
					(!fIncludeReferenceState)
				) {
					for (int m = 0; m < m_nHorizontalOrder; m++) {
					for (int n = 0; n < m_nHorizontalOrder; n++) {
						dColumnData[k] -=
							  dAInterpCoeffs[m]
							* dBInterpCoeffs[n]
							* pDataRef[k][iA+m][iB+n];
					}
					}
				}
			}

			// Interpolate vertically
			opInterp.Apply(
				&(dColumnData[0]),
				&(dColumnDataOut[0]));

			// Store data
			for (int k = 0; k < dREta.GetRows(); k++) {
				dInterpData[c][k][i] = dColumnDataOut[k];
			}

		}
	}

	// Convert to primitive variables
	if ((eDataType == DataType_State) && (fConvertToPrimitive)) {
		for (int i = 0; i < dAlpha.GetRows(); i++) { 
			if (iPatch[i] != GetPatchIndex()) {
				continue;
			}

			for (int k = 0; k < dREta.GetRows(); k++) {
				double dUalpha =
					dInterpData[0][k][i] / phys.GetEarthRadius();
				double dUbeta =
					dInterpData[1][k][i] / phys.GetEarthRadius();

				CubedSphereTrans::CoVecTransRLLFromABP(
					tan(dAlpha[i]),
					tan(dBeta[i]),
					GetPatchBox().GetPanel(),
					dUalpha,
					dUbeta,
					dInterpData[0][k][i],
					dInterpData[1][k][i]);
			}
		}
	}
}