ERMsg C20thReanalysisProject::ReadData( CString filePath, CString varName, CData3Array& data)
{
//typedef boost::multi_array<int, 2> array_type;
ERMsg msg;
NcError::set_err( NcError::silent_nonfatal );
//CString filePath = m_path + "cccma_cgcm3_1-20c3m-run1-pr-1961-2000_monthly.nc";//GetFilePath(v, year, m);
NcFile file(filePath); //current year file
if( !file.is_valid() )
{
CString err;
err.FormatMessage(IDS_CMN_UNABLE_OPEN_READ, filePath);
msg.ajoute(err);
return msg;
}
NcVar* pVarData = file.get_var((LPCTSTR)varName);//is the varaible always at ffirst???
//CString varName = pVarData->name();
size_t sizeTime = pVarData->get_dim(0)->size();
size_t sizeY = pVarData->get_dim(1)->size();
size_t sizeX = pVarData->get_dim(2)->size();
float offset = pVarData->get_att("add_offset")->as_float(0);
float scaleFactor = pVarData->get_att("scale_factor")->as_float(0);
boost::multi_array<short, 3> tmp(boost::extents[sizeTime][sizeY][sizeX]);
ENSURE( pVarData->num_dims() == 3);
if( pVarData->get(&(tmp[0][0][0]), sizeTime, sizeY, sizeX) )
{
//tmp.extens()
//data.resize(sizeTime);
//data.resize(boost::extents[sizeTime][sizeY][sizeX]);
//apply offset and scale factor
for(size_t i=0; i<tmp.size(); i++)
for(size_t j=0; j<tmp[i].size(); j++)
for(size_t k=0; k<tmp[i][j].size(); k++)
data[i][j][k] = tmp[i][j][k]*scaleFactor+offset;
file.close();
}
else
{
msg.ajoute( "Unable to get NetCDFData");
}
return msg;
}
void DumpableNcFile::dumpvars( void )
{
int n;
static const char* types[] =
{"","byte","char","short","long","float","double"};
NcVar* vp;
for(n = 0; vp = get_var(n); n++) {
cout << "\t" << types[vp->type()] << " " << vp->name() ;
if (vp->num_dims() > 0) {
cout << "(";
for (int d = 0; d < vp->num_dims(); d++) {
NcDim* dim = vp->get_dim(d);
cout << dim->name();
if (d < vp->num_dims()-1)
cout << ", ";
}
cout << ")";
}
cout << " ;\n";
// now dump each of this variable's attributes
dumpatts(*vp);
}
}
void NetCdfConfigureDialog::getDimEdges(int dimId, unsigned &size, double &firstValue, double &lastValue)
{
if ((_currentFile->get_var(_currentVar->get_dim(dimId)->name())) != NULL)
{
NcVar *tmpVarOfDim = _currentFile->get_var(_currentVar->get_dim(dimId)->name());
if ((tmpVarOfDim->num_dims()) == 1)
{
int sizeOfDim = tmpVarOfDim->get_dim(0)->size();
size = sizeOfDim;
double arrayOfDimStart[1] = {0};
size_t edgeOfArray[1] = {1};
long edgeOrigin[1] = {0};
tmpVarOfDim->set_cur(edgeOrigin);
tmpVarOfDim->get(arrayOfDimStart,edgeOfArray);
firstValue = arrayOfDimStart[0];
double arrayOfDimEnd[1] = {0};
edgeOrigin[0] = sizeOfDim - 1;
tmpVarOfDim->set_cur(edgeOrigin);
tmpVarOfDim->get(arrayOfDimEnd,edgeOfArray);
lastValue = arrayOfDimEnd[0];
}
} else {
size = 0;
firstValue = 0;
lastValue = 0;
}
}
Var( const NcVar* var ):
m_var( var )
{
const int ndims = m_var->num_dims();
for ( int i = 0; i < ndims; i++ )
{
m_dims.push_back( new Dim( m_var->get_dim(i) ) );
}
}
std::vector<T> read_vector(NcFile &nc, std::string const &var_name)
{
// Read points vector
NcVar *vpoints = nc.get_var(var_name.c_str());
long npoints = vpoints->get_dim(0)->size();
std::vector<T> points(npoints);
vpoints->get(&points[0], npoints);
return points;
}
bool ReadAstro::open(const char *path)
{
assert(!m_ncfile);
m_ncfile = new NcFile(path, NcFile::ReadOnly);
if (!m_ncfile->is_valid())
{
close();
sendError("failed to open NetCDF file %s", path);
return false;
}
if (m_ncfile->get_format() == NcFile::BadFormat)
{
close();
sendError("bad NetCDF file");
return false;
}
fprintf(stderr, "dims=%d, vars=%d, attrs=%d\n",
m_ncfile->num_dims(), m_ncfile->num_vars(), m_ncfile->num_atts());
for (int i = 0; i < m_ncfile->num_dims(); ++i)
{
fprintf(stderr, "%s: %ld\n",
m_ncfile->get_dim(i)->name(),
m_ncfile->get_dim(i)->size());
}
for (int i = 0; i < m_ncfile->num_vars(); ++i)
{
fprintf(stderr, "%s: dims=%d atts=%d vals=%ld type=%d\n",
m_ncfile->get_var(i)->name(),
m_ncfile->get_var(i)->num_dims(),
m_ncfile->get_var(i)->num_atts(),
m_ncfile->get_var(i)->num_vals(),
m_ncfile->get_var(i)->type());
//int dims = m_ncfile->get_var(i)->num_dims();
NcVar *var = m_ncfile->get_var(i);
for (int j = 0; j < var->num_dims(); ++j)
{
fprintf(stderr, " %s: %ld edge=%ld\n",
var->get_dim(j)->name(),
var->get_dim(j)->size(),
var->edges()[j]);
}
}
for (int i = 0; i < m_ncfile->num_atts(); ++i)
{
fprintf(stderr, "%s\n", m_ncfile->get_att(i)->name());
}
return true;
}
int main(int argc, char *argv[])
{
NcFile at(atpath, NcFile::ReadOnly);
if(!at.is_valid() || at.num_dims() != 3 || at.num_vars() != 4) {
fprintf(stderr, "failed reading file: %s\n", atpath);
return 1;
}
NcVar* data = at.get_var("rhum");
if(!data->is_valid() || data->num_dims() != 3) {
fprintf(stderr, "rhum has incorrect dimensions");
return 1;
}
NcDim* time = data->get_dim(0);
int timecnt = time->size();
float *rhumd = new float[timecnt*LATS*LONS];
data->get(rhumd, timecnt, LATS, LONS);
float rhumdmon[12][LATS][LONS];
for(int i = 0; i<LATS; i++)
for(int j = 0; j<LONS; j++) {
float rhumdmoncnt[12];
for(int k = 0; k<12; k++) {
rhumdmon[k][i][j] = 0;
rhumdmoncnt[k] = 0;
}
for(int k = 0; k<timecnt; k++) {
double v = rhumd[(k*LATS+i)*LONS+j]*.1 + 3276.5;
if(v >= 0 && v <= 100) {
rhumdmon[k%12][i][j] += v;
rhumdmoncnt[k%12]++;
}
}
for(int k = 0; k<12; k++)
rhumdmon[k][i][j] /= rhumdmoncnt[k];
}
delete [] rhumd;
/* use a single byte instead of 2 to save memory,
resolution of 1/5th of a mm/day resolution */
uint8_t rhumbyte[12][LATS][LONS];
for(int i = 0; i<12; i++)
for(int j = 0; j<LATS; j++)
for(int k = 0; k<LONS; k++)
if(isnan(rhumdmon[i][j][k]) || fabs(rhumdmon[i][j][k]) > 100)
rhumbyte[i][j][k] = 255;
else
rhumbyte[i][j][k] = rhumdmon[i][j][k]*2.0;
fwrite(rhumbyte, sizeof rhumbyte, 1, stdout);
return 0;
}
int NetcdfSource::readMatrix(double *v, const QString& field)
{
/* For a variable from the netCDF file */
QByteArray bytes = field.toLatin1();
NcVar *var = _ncfile->get_var(bytes.constData()); // var is owned by _ncfile
if (!var) {
KST_DBG qDebug() << "Queried field " << field << " which can't be read" << endl;
return -1;
}
int xSize = var->get_dim(0)->size();
int ySize = var->get_dim(1)->size();
var->get(v, xSize, ySize);
return xSize * ySize;
}
CCatchmentSetupParams::CCatchmentSetupParams(const std::string & szFilenameForCatchment)
{
if(!boost::filesystem::exists(szFilenameForCatchment))
throw "szFilenameForCatchment doesn't exist";
pCatchmentDescriptionFile = new NcFile(szFilenameForCatchment.c_str());
if (!pCatchmentDescriptionFile->is_valid())
{
throw "Couldn't open file!";
}
NcVar * pReachIDs = pCatchmentDescriptionFile->get_var("rchid");
NcDim* numReaches = pReachIDs->get_dim(0);
const int nNumReaches = numReaches->size();
cout << "numReaches=" << nNumReaches << endl;
//std::vector<int> aReachIDs(nNumReaches, -1);
int anReaches[nNumReaches];
pReachIDs->get(anReaches, nNumReaches); //This is the mapping from rchid to nrch (=idx)
for(int nrch=0;nrch<nNumReaches;nrch++)
{
std::cout << "Reach: " << anReaches[nrch] << std::endl;
aSubcatchments[nrch] = new CSubcatchmentParams(nrch, anReaches[nrch], this);
}
int anDownstreamReaches[nNumReaches];
pCatchmentDescriptionFile->get_var("dsrch_nrch")->get(anDownstreamReaches, nNumReaches);
for(int nrch=0;nrch<nNumReaches;nrch++)
{
if(anDownstreamReaches[nrch]>=0)
{
std::cout << "Catchment " << nrch << " Downstream reach: " << anDownstreamReaches[nrch] << std::endl;
//aSubcatchments[nrch] = new CSubcatchmentParams(nrch, this);
aSubcatchments[nrch]->setDownstreamCatchment(aSubcatchments[anDownstreamReaches[nrch]]);
}
}
std::cout << std::endl;
}
const DataMatrix::DataInfo DataInterfaceNetCdfMatrix::dataInfo(const QString& matrix) const
{
if (!netcdf._matrixList.contains( matrix ) ) {
return DataMatrix::DataInfo();
}
QByteArray bytes = matrix.toLatin1();
NcVar *var = netcdf._ncfile->get_var(bytes.constData()); // var is owned by _ncfile
if (var->num_dims() != 2) {
return DataMatrix::DataInfo();
}
DataMatrix::DataInfo info;
info.samplesPerFrame = 1;
// TODO is this right?
info.xSize = var->get_dim(0)->size();
info.ySize = var->get_dim(1)->size();
return info;
}
blitz::Array<T,rank> read_blitz(NcFile &nc, std::string const &var_name)
{
// Read points vector
NcVar *vpoints = nc.get_var(var_name.c_str());
int ndims = vpoints->num_dims();
if (ndims != rank) {
fprintf(stderr, "NetCDF variable %s has rank %d, expected rank %d\n",
var_name.c_str(), ndims, rank);
throw std::exception();
}
blitz::TinyVector<int,rank> shape(0);
long counts[rank];
for (int i=0; i<rank; ++i) {
shape[i] = vpoints->get_dim(i)->size();
printf("read_blitz: shape[%d] = %d\n", i, shape[i]);
counts[i] = shape[i];
}
blitz::Array<T,rank> ret(shape);
for (int i=0; i<rank; ++i) printf("read_blitz: ret.extent(%d) = %d\n", i, ret.extent(i));
vpoints->get(ret.data(), counts);
return ret;
}
ERMsg CMADIS::ExtractLOC(const CString& filePath, CSCCallBack& callback)
{
ERMsg msg;
callback.SetCurrentDescription("Extract location");
callback.SetCurrentStepRange(0, 3, 1);
callback.SetStartNewStep();
CString outputFilePath(filePath);
UtilWin::SetFileExtension( outputFilePath, ".loc");
// CString outputFilePathGrid(filePath);
// UtilWin::SetFileExtension( outputFilePathGrid, ".bil");
// CShapeFileBase shapefile;
// shapefile.Read("C:\\ouranos_data\\MapInput\\canada.shp");
NcFile file(filePath);
if( !file.is_valid() )
{
CString err;
err.FormatMessage(IDS_CMN_UNABLE_OPEN_READ, filePath);
msg.ajoute(err);
}
//CMapBinaryFile grid;
// grid.SetProjection( GetDataGrid().GetPrj() );
// grid.SetNbCols(180);
// grid.SetNbRows(172);
// grid.SetBoundingBox(GetDataGrid());
// grid.SetNoData(-9999);
// grid.SetCellSizeX( 45000 );
// grid.SetCellSizeY( 45000 );
CStdioFileEx fileOut;
//msg += grid.Open(outputFilePathGrid, CGeoFileInterface::modeWrite);
msg += fileOut.Open( outputFilePath, CFile::modeCreate|CFile::modeWrite);
if(!msg)
return msg;
fileOut.WriteString("LOC_FILE 4 2,ID,Latitude,Longitude,Elevation,Slope(%),Orientation(°)\n");
//char providerId(recNum, maxProviderIdLen) ;
// providerId:long_name = "Data Provider station Id" ;
// providerId:reference = "station table" ;
//char stationId(recNum, maxStaIdLen) ;
// stationId:long_name = "alphanumeric station Id" ;
// stationId:reference = "station table" ;
//char handbook5Id(recNum, maxStaIdLen) ;
// handbook5Id:long_name = "Handbook5 Id (AFOS or SHEF id)" ;
// handbook5Id:reference = "station table" ;
//char stationName(recNum, maxNameLength) ;
// stationName:long_name = "alphanumeric station name" ;
// stationName:reference = "station table" ;
//float latitude(recNum) ;
// latitude:long_name = "latitude" ;
// latitude:units = "degree_north" ;
// latitude:_FillValue = 3.4028235e+038f ;
// latitude:missing_value = -9999.f ;
// latitude:reference = "station table" ;
// latitude:standard_name = "latitude" ;
//float longitude(recNum) ;
// longitude:long_name = "longitude" ;
// longitude:units = "degree_east" ;
// longitude:_FillValue = 3.4028235e+038f ;
// longitude:missing_value = -9999.f ;
// longitude:reference = "station table" ;
// longitude:standard_name = "longitude" ;
//float elevation(recNum) ;
// elevation:long_name = "elevation" ;
// elevation:units = "meter" ;
// elevation:_FillValue = 3.4028235e+038f ;
// elevation:missing_value = -9999.f ;
// elevation:reference = "station table" ;
// elevation:standard_name = "elevation" ;
//NcVar* pVarX = file.get_var("xc");
//NcVar* pVarY = file.get_var("yc");
NcVar* pVarLat = file.get_var("staLat");//file.get_var("latitude");
NcVar* pVarLon = file.get_var("staLon");//file.get_var("longitude");
NcVar* pVarElev = file.get_var("staElev");//file.get_var("elevation");
NcDim* pDim = pVarLat->get_dim(0);
//double offset = pVarElev->get_att("add_offset")->as_float(0);
//double scaleFactor = pVarElev->get_att("scale_factor")->as_float(0);
int nbRect = pDim->size();
//delete pDim; pDim = NULL;
//typedef boost::multi_array<char, 1> CDataCharArray;
typedef boost::multi_array<float, 1> CDataFloatArray;
//CDataFloatArray X(boost::extents[180]);
//CDataFloatArray Y(boost::extents[172]);
CDataFloatArray lat(boost::extents[nbRect]);
CDataFloatArray lon(boost::extents[nbRect]);
CDataFloatArray elev(boost::extents[nbRect]);
ASSERT( pVarLat->get_dim(0)->size() == pVarLon->get_dim(0)->size() );
ASSERT( pVarLat->get_dim(0)->size() == pVarElev->get_dim(0)->size() );
//pVarX->get(&(X[0]), 180);
//pVarY->get(&(Y[0]), 172);
pVarLat->get(&(lat[0]), nbRect);callback.StepIt();
pVarLon->get(&(lon[0]), nbRect);callback.StepIt();
pVarElev->get(&(elev[0]), nbRect);callback.StepIt();
//CProjection prj = GetDataGrid().GetPrj();
for(int i=0; i<nbRect; i++)
{
double Lat=0, Lon=0;
CString line;
line.Format("%d,%d,%f,%f,%.1f,0,0\n",i+1,i+1,lat[i],lon[i],elev[i]);
fileOut.WriteString(line);
}
fileOut.Close();
return msg;
}
int main(int argc, char** argv) {
NcError error(NcError::silent_nonfatal);
try {
// Input filename
std::string strInputFile;
// Output mesh filename
std::string strOutputFile;
// Polynomial degree per element
int nP = 2;
// Parse the command line
BeginCommandLine()
CommandLineString(strInputFile, "in", "");
CommandLineString(strOutputFile, "out", "");
//CommandLineInt(nP, "np", 2);
//CommandLineBool(fCGLL, "cgll");
ParseCommandLine(argc, argv);
EndCommandLine(argv)
// Check file names
if (strInputFile == "") {
std::cout << "ERROR: No input file specified" << std::endl;
return (-1);
}
if (strOutputFile == "") {
std::cout << "ERROR: No output file specified" << std::endl;
return (-1);
}
if (nP < 1) {
std::cout << "ERROR: --np must be >= 2" << std::endl;
return (-1);
}
AnnounceBanner();
// Load input mesh
AnnounceStartBlock("Loading input mesh");
Mesh meshIn(strInputFile);
meshIn.RemoveZeroEdges();
AnnounceEndBlock("Done");
// Construct edge map
AnnounceStartBlock("Constructing edge map");
meshIn.ConstructEdgeMap();
AnnounceEndBlock("Done");
// Build connectivity vector using edge map
AnnounceStartBlock("Constructing connectivity");
std::vector< std::set<int> > vecConnectivity;
int err = GenerateConnectivityData(meshIn, vecConnectivity);
if (err) return err;
AnnounceEndBlock("Done");
// Open output file
AnnounceStartBlock("Writing connectivity file");
NcFile ncmesh(strInputFile.c_str(), NcFile::ReadOnly);
NcVar * varLat = ncmesh.get_var("grid_center_lat");
NcVar * varLon = ncmesh.get_var("grid_center_lon");
// Check if center latitudes and longitudes are already available
DataArray1D<double> dAllLats;
DataArray1D<double> dAllLons;
bool fConvertLatToDegrees = true;
bool fConvertLonToDegrees = true;
if ((varLat == NULL) || (varLon == NULL)) {
Announce("grid_center_lat not found, recalculating face centers");
} else {
Announce("grid_center_lat found in file, loading values");
if (varLat->get_dim(0)->size() != vecConnectivity.size()) {
_EXCEPTIONT("grid_center_lat dimension mismatch");
}
if (varLon->get_dim(0)->size() != vecConnectivity.size()) {
_EXCEPTIONT("grid_center_lon dimension mismatch");
}
dAllLats.Allocate(vecConnectivity.size());
varLat->set_cur((long)0);
varLat->get(dAllLats, vecConnectivity.size());
NcAtt * attLatUnits = varLat->get_att("units");
std::string strLatUnits = attLatUnits->as_string(0);
if (strLatUnits == "degrees") {
fConvertLatToDegrees = false;
}
dAllLons.Allocate(vecConnectivity.size());
varLon->set_cur((long)0);
varLon->get(dAllLons, vecConnectivity.size());
NcAtt * attLonUnits = varLon->get_att("units");
std::string strLonUnits = attLonUnits->as_string(0);
if (strLonUnits == "degrees") {
fConvertLonToDegrees = false;
}
}
// Write connectiivty file
FILE * fp = fopen(strOutputFile.c_str(), "w");
fprintf(fp, "%lu\n", vecConnectivity.size());
for (size_t f = 0; f < vecConnectivity.size(); f++) {
double dLon;
double dLat;
if ((varLat == NULL) || (varLon == NULL)) {
Node nodeCentroid;
for (int i = 0; i < meshIn.faces[f].edges.size(); i++) {
nodeCentroid.x += meshIn.nodes[meshIn.faces[f][i]].x;
nodeCentroid.y += meshIn.nodes[meshIn.faces[f][i]].y;
nodeCentroid.z += meshIn.nodes[meshIn.faces[f][i]].z;
}
double dMagnitude = nodeCentroid.Magnitude();
nodeCentroid.x /= dMagnitude;
nodeCentroid.y /= dMagnitude;
nodeCentroid.z /= dMagnitude;
dLon = atan2(nodeCentroid.y, nodeCentroid.x);
dLat = asin(nodeCentroid.z);
if (dLon < 0.0) {
dLon += 2.0 * M_PI;
}
} else {
dLon = dAllLons[f];
dLat = dAllLats[f];
}
if (fConvertLonToDegrees) {
dLon *= 180.0 / M_PI;
}
if (fConvertLatToDegrees) {
dLat *= 180.0 / M_PI;
}
fprintf(fp, "%1.14f,", dLon);
fprintf(fp, "%1.14f,", dLat);
fprintf(fp, "%lu", vecConnectivity[f].size());
std::set<int>::const_iterator iter = vecConnectivity[f].begin();
for (; iter != vecConnectivity[f].end(); iter++) {
fprintf(fp, ",%i", *iter);
}
if (f != vecConnectivity.size()-1) {
fprintf(fp,"\n");
}
}
fclose(fp);
AnnounceEndBlock("Done");
// Announce
AnnounceBanner();
return (0);
} catch(Exception & e) {
Announce(e.ToString().c_str());
return (-1);
} catch(...) {
return (-2);
}
}
void ReadCFTimeDataFromNcFile(
NcFile * ncfile,
const std::string & strFilename,
std::vector<Time> & vecTimes,
bool fWarnOnMissingCalendar
) {
// Empty existing Time vector
vecTimes.clear();
// Get time dimension
NcDim * dimTime = ncfile->get_dim("time");
if (dimTime == NULL) {
_EXCEPTION1("Dimension \"time\" not found in file \"%s\"",
strFilename.c_str());
}
// Get time variable
NcVar * varTime = ncfile->get_var("time");
if (varTime == NULL) {
_EXCEPTION1("Variable \"time\" not found in file \"%s\"",
strFilename.c_str());
}
if (varTime->num_dims() != 1) {
_EXCEPTION1("Variable \"time\" has more than one dimension in file \"%s\"",
strFilename.c_str());
}
if (strcmp(varTime->get_dim(0)->name(), "time") != 0) {
_EXCEPTION1("Variable \"time\" does not have dimension \"time\" in file \"%s\"",
strFilename.c_str());
}
// Calendar attribute
NcAtt * attTimeCal = varTime->get_att("calendar");
std::string strCalendar;
if (attTimeCal == NULL) {
if (fWarnOnMissingCalendar) {
Announce("WARNING: Variable \"time\" is missing \"calendar\" attribute; assuming \"standard\"");
}
strCalendar = "standard";
} else {
strCalendar = attTimeCal->as_string(0);
}
Time::CalendarType eCalendarType =
Time::CalendarTypeFromString(strCalendar);
// Units attribute
NcAtt * attTimeUnits = varTime->get_att("units");
if (attTimeUnits == NULL) {
_EXCEPTION1("Variable \"time\" is missing \"units\" attribute in file \"%s\"",
strFilename.c_str());
}
std::string strTimeUnits = attTimeUnits->as_string(0);
// Load in time data
DataVector<int> vecTimeInt;
DataVector<float> vecTimeFloat;
DataVector<double> vecTimeDouble;
DataVector<ncint64> vecTimeInt64;
if (varTime->type() == ncInt) {
vecTimeInt.Initialize(dimTime->size());
varTime->set_cur((long)0);
varTime->get(&(vecTimeInt[0]), dimTime->size());
} else if (varTime->type() == ncFloat) {
vecTimeFloat.Initialize(dimTime->size());
varTime->set_cur((long)0);
varTime->get(&(vecTimeFloat[0]), dimTime->size());
} else if (varTime->type() == ncDouble) {
vecTimeDouble.Initialize(dimTime->size());
varTime->set_cur((long)0);
varTime->get(&(vecTimeDouble[0]), dimTime->size());
} else if (varTime->type() == ncInt64) {
vecTimeInt64.Initialize(dimTime->size());
varTime->set_cur((long)0);
varTime->get(&(vecTimeInt64[0]), dimTime->size());
} else {
_EXCEPTION1("Variable \"time\" has invalid type "
"(expected \"int\", \"int64\", \"float\" or \"double\")"
" in file \"%s\"", strFilename.c_str());
}
for (int t = 0; t < dimTime->size(); t++) {
Time time(eCalendarType);
if (varTime->type() == ncInt) {
time.FromCFCompliantUnitsOffsetInt(
strTimeUnits,
vecTimeInt[t]);
} else if (varTime->type() == ncFloat) {
time.FromCFCompliantUnitsOffsetDouble(
strTimeUnits,
static_cast<double>(vecTimeFloat[t]));
} else if (varTime->type() == ncDouble) {
time.FromCFCompliantUnitsOffsetDouble(
strTimeUnits,
vecTimeDouble[t]);
} else if (varTime->type() == ncInt64) {
time.FromCFCompliantUnitsOffsetInt(
strTimeUnits,
(int)(vecTimeInt64[t]));
}
vecTimes.push_back(time);
}
}
void CopyNcVar(
NcFile & ncIn,
NcFile & ncOut,
const std::string & strVarName,
bool fCopyAttributes,
bool fCopyData
) {
if (!ncIn.is_valid()) {
_EXCEPTIONT("Invalid input file specified");
}
if (!ncOut.is_valid()) {
_EXCEPTIONT("Invalid output file specified");
}
NcVar * var = ncIn.get_var(strVarName.c_str());
if (var == NULL) {
_EXCEPTION1("NetCDF file does not contain variable \"%s\"",
strVarName.c_str());
}
NcVar * varOut;
std::vector<NcDim *> dimOut;
dimOut.resize(var->num_dims());
std::vector<long> counts;
counts.resize(var->num_dims());
long nDataSize = 1;
for (int d = 0; d < var->num_dims(); d++) {
NcDim * dimA = var->get_dim(d);
dimOut[d] = ncOut.get_dim(dimA->name());
if (dimOut[d] == NULL) {
if (dimA->is_unlimited()) {
dimOut[d] = ncOut.add_dim(dimA->name());
} else {
dimOut[d] = ncOut.add_dim(dimA->name(), dimA->size());
}
if (dimOut[d] == NULL) {
_EXCEPTION2("Failed to add dimension \"%s\" (%i) to file",
dimA->name(), dimA->size());
}
}
if (dimOut[d]->size() != dimA->size()) {
if (dimA->is_unlimited() && !dimOut[d]->is_unlimited()) {
_EXCEPTION2("Mismatch between input file dimension \"%s\" and "
"output file dimension (UNLIMITED / %i)",
dimA->name(), dimOut[d]->size());
} else if (!dimA->is_unlimited() && dimOut[d]->is_unlimited()) {
_EXCEPTION2("Mismatch between input file dimension \"%s\" and "
"output file dimension (%i / UNLIMITED)",
dimA->name(), dimA->size());
} else if (!dimA->is_unlimited() && !dimOut[d]->is_unlimited()) {
_EXCEPTION3("Mismatch between input file dimension \"%s\" and "
"output file dimension (%i / %i)",
dimA->name(), dimA->size(), dimOut[d]->size());
}
}
counts[d] = dimA->size();
nDataSize *= counts[d];
}
// ncByte / ncChar type
if ((var->type() == ncByte) || (var->type() == ncChar)) {
DataVector<char> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
// ncShort type
if (var->type() == ncShort) {
DataVector<short> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
if (fCopyData) {
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
}
// ncInt type
if (var->type() == ncInt) {
DataVector<int> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
if (fCopyData) {
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
}
// ncFloat type
if (var->type() == ncFloat) {
DataVector<float> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
if (fCopyData) {
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
}
// ncDouble type
if (var->type() == ncDouble) {
DataVector<double> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
if (fCopyData) {
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
}
// ncInt64 type
if (var->type() == ncInt64) {
DataVector<ncint64> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
if (fCopyData) {
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
}
// Check output variable exists
if (varOut == NULL) {
_EXCEPTION1("Unable to create output variable \"%s\"",
var->name());
}
// Copy attributes
if (fCopyAttributes) {
CopyNcVarAttributes(var, varOut);
}
}
int main(int argc, char** argv)
{
if (!cmdline(argc, argv))
{
printhelp();
return EXIT_FAILURE;
}
NcFile infile(infilename.c_str(), NcFile::ReadOnly);
if (!infile.is_valid())
{
std::cerr << "Error: invalid input file -- '" << infilename << "'" << std::endl;
infile.close();
return EXIT_FAILURE;
}
NcFile outfile(outfilename.c_str(), NcFile::Replace);
if (!outfile.is_valid())
{
std::cerr << "Error: cannot open output file -- '" << outfilename << "'" << std::endl;
outfile.close();
return EXIT_FAILURE;
}
if (varstrings.size() == 0)
{
std::cerr << "Warning: no variables specified" << std::endl;
}
std::vector<NcVar*> invars;
for (std::vector<std::string>::const_iterator it = varstrings.begin();
it != varstrings.end(); ++it)
{
NcVar* var = infile.get_var((*it).c_str());
if (var == NULL)
{
std::cerr << "Error: " << *it << ": no such variable" << std::endl;
infile.close();
outfile.close();
return EXIT_FAILURE;
}
invars.push_back(var);
}
// extract the distinct set of dims
std::map<std::string, NcDim*> indims;
for (std::vector<NcVar*>::const_iterator it = invars.begin();
it != invars.end(); ++it)
{
NcVar* var = *it;
for (int i = 0; i < var->num_dims(); ++i)
{
NcDim* dim = var->get_dim(i);
indims[dim->name()] = dim;
}
}
// add dims to outfile
std::map<std::string, NcDim*> outdims;
for (std::map<std::string, NcDim*>::const_iterator it = indims.begin();
it != indims.end(); ++it)
{
NcDim* dim = (*it).second;
NcDim* outdim = NULL;
if (dim->is_unlimited())
{
outdim = outfile.add_dim(dim->name());
}
else
{
outdim = outfile.add_dim(dim->name(), dim->size());
}
if (outdim != NULL)
{
outdims[outdim->name()] = outdim;
}
}
// create variables
for (std::vector<NcVar*>::const_iterator it = invars.begin();
it != invars.end(); ++it)
{
NcVar* var = *it;
std::vector<const NcDim*> dims(var->num_dims());
for (int i = 0; i < var->num_dims(); ++i)
{
dims[i] = outdims[var->get_dim(i)->name()];
}
NcVar* outvar = outfile.add_var(var->name(), var->type(), var->num_dims(), &dims[0]);
// identify largest dim, if dim (nearly) exceeds main memory, split along that dim
int maxdim = -1;
long maxdimsize = 0;
long totallen = 1;
for (int i = 0; i < var->num_dims(); ++i)
{
NcDim* dim = var->get_dim(i);
if (dim->size() > maxdimsize)
{
maxdim = i;
maxdimsize = dim->size();
}
totallen *= dim->size();
}
// TODO: support other data types
totallen *= sizeof(float);
// TODO: configurable page size
const unsigned long pagesize = 1000000000;
#ifdef __linux__
struct sysinfo info;
sysinfo(&info);
if (pagesize >= info.freeram)
{
std::cerr << "Warning: page size exceeds free memory" << std::endl;
}
#endif
int numpages = 1;
long pagesizedim = var->get_dim(maxdim)->size();
if (totallen < pagesize)
{
}
else
{
long mul = 1;
for (int i = 0; i < var->num_dims(); ++i)
{
if (i != maxdim)
{
NcDim* dim = var->get_dim(i);
mul *= dim->size();
}
}
// TODO: support other data types
mul *= sizeof(float);
pagesizedim = pagesize / mul;
numpages = var->get_dim(maxdim)->size() / pagesizedim;
if (var->get_dim(maxdim)->size() % pagesizedim > 0)
{
++numpages;
}
}
std::vector< std::vector<long> > curvec;
std::vector< std::vector<long> > countsvec;
std::vector<long> lengths;
int pages = numpages > 0 ? numpages : 1;
for (int p = 0; p < pages; ++p)
{
long len = 1;
std::vector<long> cur;
std::vector<long> counts;
for (int i = 0; i < var->num_dims(); ++i)
{
NcDim* dim = var->get_dim(i);
long current = 0;
long count = dim->size();
if (i == maxdim)
{
current = pagesizedim * p;
count = pagesizedim;
if (p == pages -1)
{
if (dim->size() % pagesizedim != 0)
{
count = dim->size() % pagesizedim;
}
}
}
cur.push_back(current);
counts.push_back(count);
len *= count;
}
curvec.push_back(cur);
countsvec.push_back(counts);
lengths.push_back(len);
}
std::vector< std::vector<long> >::const_iterator it1;
std::vector< std::vector<long> >::const_iterator it2;
std::vector<long>::const_iterator it3;
for (it1 = curvec.begin(), it2 = countsvec.begin(), it3 = lengths.begin();
it1 != curvec.end() && it2 != countsvec.end() && it3 != lengths.end(); ++it1, ++it2, ++it3)
{
std::vector<long> cur = *it1;
std::vector<long> counts = *it2;
long len = *it3;
var->set_cur(&cur[0]);
outvar->set_cur(&cur[0]);
switch (outvar->type())
{
case ncByte:
{
ncbyte* barr = new ncbyte[len];
var->get(barr, &counts[0]);
outvar->put(barr, &counts[0]);
delete[] barr;
break;
}
case ncChar:
{
char* carr = new char[len];
var->get(carr, &counts[0]);
outvar->put(carr, &counts[0]);
delete[] carr;
break;
}
case ncShort:
{
short* sarr = new short[len];
var->get(sarr, &counts[0]);
outvar->put(sarr, &counts[0]);
delete[] sarr;
break;
}
case ncInt:
{
long* larr = new long[len];
var->get(larr, &counts[0]);
outvar->put(larr, &counts[0]);
delete[] larr;
break;
}
case ncFloat:
{
float* farr = new float[len];
var->get(farr, &counts[0]);
outvar->put(farr, &counts[0]);
delete[] farr;
break;
}
case ncDouble:
{
double* darr = new double[len];
var->get(darr, &counts[0]);
outvar->put(darr, &counts[0]);
delete[] darr;
break;
}
default:
break;
}
}
}
infile.close();
outfile.close();
return 0;
}
int main(int argc, char ** argv) {
MPI_Init(&argc, &argv);
NcError error(NcError::silent_nonfatal);
try {
// Input filename
std::string strInputFile;
// Output filename
std::string strOutputFile;
// Separate topography file
std::string strTopographyFile;
// List of variables to extract
std::string strVariables;
// Extract geopotential height
bool fGeopotentialHeight;
// Pressure levels to extract
std::string strPressureLevels;
// Height levels to extract
std::string strHeightLevels;
// Extract variables at the surface
bool fExtractSurface;
// Extract total energy
bool fExtractTotalEnergy;
// Parse the command line
BeginCommandLine()
CommandLineString(strInputFile, "in", "");
CommandLineString(strOutputFile, "out", "");
CommandLineString(strVariables, "var", "");
CommandLineBool(fGeopotentialHeight, "output_z");
CommandLineBool(fExtractTotalEnergy, "output_energy");
CommandLineString(strPressureLevels, "p", "");
CommandLineString(strHeightLevels, "z", "");
CommandLineBool(fExtractSurface, "surf");
ParseCommandLine(argc, argv);
EndCommandLine(argv)
AnnounceBanner();
// Check command line arguments
if (strInputFile == "") {
_EXCEPTIONT("No input file specified");
}
if (strOutputFile == "") {
_EXCEPTIONT("No output file specified");
}
if (strVariables == "") {
_EXCEPTIONT("No variables specified");
}
// Parse variable string
std::vector< std::string > vecVariableStrings;
ParseVariableList(strVariables, vecVariableStrings);
// Check variables
if (vecVariableStrings.size() == 0) {
_EXCEPTIONT("No variables specified");
}
// Parse pressure level string
std::vector<double> vecPressureLevels;
ParseLevelArray(strPressureLevels, vecPressureLevels);
int nPressureLevels = (int)(vecPressureLevels.size());
for (int k = 0; k < nPressureLevels; k++) {
if (vecPressureLevels[k] <= 0.0) {
_EXCEPTIONT("Non-positive pressure values not allowed");
}
}
// Parse height level string
std::vector<double> vecHeightLevels;
ParseLevelArray(strHeightLevels, vecHeightLevels);
int nHeightLevels = (int)(vecHeightLevels.size());
// Check pressure levels
if ((nPressureLevels == 0) &&
(nHeightLevels == 0) &&
(!fExtractSurface)
) {
_EXCEPTIONT("No pressure / height levels to process");
}
// Open input file
AnnounceStartBlock("Loading input file");
NcFile ncdf_in(strInputFile.c_str(), NcFile::ReadOnly);
if (!ncdf_in.is_valid()) {
_EXCEPTION1("Unable to open file \"%s\" for reading",
strInputFile.c_str());
}
// Load time array
Announce("Time");
NcVar * varTime = ncdf_in.get_var("time");
if (varTime == NULL) {
_EXCEPTION1("File \"%s\" does not contain variable \"time\"",
strInputFile.c_str());
}
int nTime = varTime->get_dim(0)->size();
DataArray1D<double> dTime(nTime);
varTime->set_cur((long)0);
varTime->get(&(dTime[0]), nTime);
// Load latitude array
Announce("Latitude");
NcVar * varLat = ncdf_in.get_var("lat");
if (varLat == NULL) {
_EXCEPTION1("File \"%s\" does not contain variable \"lat\"",
strInputFile.c_str());
}
int nLat = varLat->get_dim(0)->size();
DataArray1D<double> dLat(nLat);
varLat->set_cur((long)0);
varLat->get(&(dLat[0]), nLat);
// Load longitude array
Announce("Longitude");
NcVar * varLon = ncdf_in.get_var("lon");
if (varLon == NULL) {
_EXCEPTION1("File \"%s\" does not contain variable \"lon\"",
strInputFile.c_str());
}
int nLon = varLon->get_dim(0)->size();
DataArray1D<double> dLon(nLon);
varLon->set_cur((long)0);
varLon->get(&(dLon[0]), nLon);
// Load level array
Announce("Level");
NcVar * varLev = ncdf_in.get_var("lev");
if (varLev == NULL) {
_EXCEPTION1("File \"%s\" does not contain variable \"lev\"",
strInputFile.c_str());
}
int nLev = varLev->get_dim(0)->size();
DataArray1D<double> dLev(nLev);
varLev->set_cur((long)0);
varLev->get(&(dLev[0]), nLev);
// Load level interface array
Announce("Interface");
NcVar * varILev = ncdf_in.get_var("ilev");
int nILev = 0;
DataArray1D<double> dILev;
if (varILev == NULL) {
Announce("Warning: Variable \"ilev\" not found");
} else {
nILev = varILev->get_dim(0)->size();
if (nILev != nLev + 1) {
_EXCEPTIONT("Variable \"ilev\" must have size lev+1");
}
dILev.Allocate(nILev);
varILev->set_cur((long)0);
varILev->get(&(dILev[0]), nILev);
}
// Load topography
Announce("Topography");
NcVar * varZs = ncdf_in.get_var("Zs");
if (varZs == NULL) {
_EXCEPTION1("File \"%s\" does not contain variable \"Zs\"",
strInputFile.c_str());
}
DataArray2D<double> dZs(nLat, nLon);
varZs->set_cur((long)0, (long)0);
varZs->get(&(dZs[0][0]), nLat, nLon);
AnnounceEndBlock("Done");
// Open output file
AnnounceStartBlock("Constructing output file");
NcFile ncdf_out(strOutputFile.c_str(), NcFile::Replace);
if (!ncdf_out.is_valid()) {
_EXCEPTION1("Unable to open file \"%s\" for writing",
strOutputFile.c_str());
}
CopyNcFileAttributes(&ncdf_in, &ncdf_out);
// Output time array
Announce("Time");
NcDim * dimOutTime = ncdf_out.add_dim("time");
NcVar * varOutTime = ncdf_out.add_var("time", ncDouble, dimOutTime);
varOutTime->set_cur((long)0);
varOutTime->put(&(dTime[0]), nTime);
CopyNcVarAttributes(varTime, varOutTime);
// Output pressure array
NcDim * dimOutP = NULL;
NcVar * varOutP = NULL;
if (nPressureLevels > 0) {
Announce("Pressure");
dimOutP = ncdf_out.add_dim("p", nPressureLevels);
varOutP = ncdf_out.add_var("p", ncDouble, dimOutP);
varOutP->set_cur((long)0);
varOutP->put(&(vecPressureLevels[0]), nPressureLevels);
}
// Output height array
NcDim * dimOutZ = NULL;
NcVar * varOutZ = NULL;
if (nHeightLevels > 0) {
Announce("Height");
dimOutZ = ncdf_out.add_dim("z", nHeightLevels);
varOutZ = ncdf_out.add_var("z", ncDouble, dimOutZ);
varOutZ->set_cur((long)0);
varOutZ->put(&(vecHeightLevels[0]), nHeightLevels);
}
// Output latitude and longitude array
Announce("Latitude");
NcDim * dimOutLat = ncdf_out.add_dim("lat", nLat);
NcVar * varOutLat = ncdf_out.add_var("lat", ncDouble, dimOutLat);
varOutLat->set_cur((long)0);
varOutLat->put(&(dLat[0]), nLat);
CopyNcVarAttributes(varLat, varOutLat);
Announce("Longitude");
NcDim * dimOutLon = ncdf_out.add_dim("lon", nLon);
NcVar * varOutLon = ncdf_out.add_var("lon", ncDouble, dimOutLon);
varOutLon->set_cur((long)0);
varOutLon->put(&(dLon[0]), nLon);
CopyNcVarAttributes(varLon, varOutLon);
// Output topography
Announce("Topography");
NcVar * varOutZs = ncdf_out.add_var(
"Zs", ncDouble, dimOutLat, dimOutLon);
varOutZs->set_cur((long)0, (long)0);
varOutZs->put(&(dZs[0][0]), nLat, nLon);
AnnounceEndBlock("Done");
// Done
AnnounceEndBlock("Done");
// Load all variables
Announce("Loading variables");
std::vector<NcVar *> vecNcVar;
for (int v = 0; v < vecVariableStrings.size(); v++) {
vecNcVar.push_back(ncdf_in.get_var(vecVariableStrings[v].c_str()));
if (vecNcVar[v] == NULL) {
_EXCEPTION1("Unable to load variable \"%s\" from file",
vecVariableStrings[v].c_str());
}
}
// Physical constants
Announce("Initializing thermodynamic variables");
NcAtt * attEarthRadius = ncdf_in.get_att("earth_radius");
double dEarthRadius = attEarthRadius->as_double(0);
NcAtt * attRd = ncdf_in.get_att("Rd");
double dRd = attRd->as_double(0);
NcAtt * attCp = ncdf_in.get_att("Cp");
double dCp = attCp->as_double(0);
double dGamma = dCp / (dCp - dRd);
NcAtt * attP0 = ncdf_in.get_att("P0");
double dP0 = attP0->as_double(0);
double dPressureScaling = dP0 * std::pow(dRd / dP0, dGamma);
NcAtt * attZtop = ncdf_in.get_att("Ztop");
double dZtop = attZtop->as_double(0);
// Input data
DataArray3D<double> dataIn(nLev, nLat, nLon);
DataArray3D<double> dataInt(nILev, nLat, nLon);
// Output data
DataArray2D<double> dataOut(nLat, nLon);
// Pressure in column
DataArray1D<double> dataColumnP(nLev);
// Height in column
DataArray1D<double> dataColumnZ(nLev);
DataArray1D<double> dataColumnIZ(nILev);
// Column weights
DataArray1D<double> dW(nLev);
DataArray1D<double> dIW(nILev);
// Loop through all times, pressure levels and variables
AnnounceStartBlock("Interpolating");
// Add energy variable
NcVar * varEnergy;
if (fExtractTotalEnergy) {
varEnergy = ncdf_out.add_var("TE", ncDouble, dimOutTime);
}
// Create output pressure variables
std::vector<NcVar *> vecOutNcVarP;
if (nPressureLevels > 0) {
for (int v = 0; v < vecVariableStrings.size(); v++) {
vecOutNcVarP.push_back(
ncdf_out.add_var(
vecVariableStrings[v].c_str(), ncDouble,
dimOutTime, dimOutP, dimOutLat, dimOutLon));
// Copy attributes
CopyNcVarAttributes(vecNcVar[v], vecOutNcVarP[v]);
}
}
// Create output height variables
std::vector<NcVar *> vecOutNcVarZ;
if (nHeightLevels > 0) {
for (int v = 0; v < vecVariableStrings.size(); v++) {
std::string strVarName = vecVariableStrings[v];
if (nPressureLevels > 0) {
strVarName += "z";
}
vecOutNcVarZ.push_back(
ncdf_out.add_var(
strVarName.c_str(), ncDouble,
dimOutTime, dimOutZ, dimOutLat, dimOutLon));
// Copy attributes
CopyNcVarAttributes(vecNcVar[v], vecOutNcVarZ[v]);
}
}
// Create output surface variable
std::vector<NcVar *> vecOutNcVarS;
if (fExtractSurface) {
for (int v = 0; v < vecVariableStrings.size(); v++) {
std::string strVarName = vecVariableStrings[v];
strVarName += "S";
vecOutNcVarS.push_back(
ncdf_out.add_var(
strVarName.c_str(), ncDouble,
dimOutTime, dimOutLat, dimOutLon));
// Copy attributes
CopyNcVarAttributes(vecNcVar[v], vecOutNcVarS[v]);
}
}
// Loop over all times
for (int t = 0; t < nTime; t++) {
char szAnnounce[256];
sprintf(szAnnounce, "Time %i", t);
AnnounceStartBlock(szAnnounce);
// Rho
DataArray3D<double> dataRho(nLev, nLat, nLon);
NcVar * varRho = ncdf_in.get_var("Rho");
if (varRho == NULL) {
_EXCEPTIONT("Unable to load variable \"Rho\" from file");
}
varRho->set_cur(t, 0, 0, 0);
varRho->get(&(dataRho[0][0][0]), 1, nLev, nLat, nLon);
// Pressure
DataArray3D<double> dataP(nLev, nLat, nLon);
if (nPressureLevels != 0) {
NcVar * varP = ncdf_in.get_var("P");
if (varP == NULL) {
_EXCEPTIONT("Unable to load variable \"P\" from file");
}
varP->set_cur(t, 0, 0, 0);
varP->get(&(dataP[0][0][0]), 1, nLev, nLat, nLon);
}
/*
// Populate pressure array
if (nPressureLevels > 0) {
// Calculate pointwise pressure
for (int k = 0; k < nLev; k++) {
for (int i = 0; i < nLat; i++) {
for (int j = 0; j < nLon; j++) {
dataP[k][i][j] = dPressureScaling
* exp(log(dataRho[k][i][j] * dataP[k][i][j]) * dGamma);
}
}
}
}
*/
// Height everywhere
DataArray3D<double> dataZ(nLev, nLat, nLon);
DataArray3D<double> dataIZ;
if (nILev != 0) {
dataIZ.Allocate(nILev, nLat, nLon);
}
// Populate height array
if ((nHeightLevels > 0) || (fGeopotentialHeight)) {
for (int k = 0; k < nLev; k++) {
for (int i = 0; i < nLat; i++) {
for (int j = 0; j < nLon; j++) {
dataZ[k][i][j] = dZs[i][j] + dLev[k] * (dZtop - dZs[i][j]);
}
}
}
for (int k = 0; k < nILev; k++) {
for (int i = 0; i < nLat; i++) {
for (int j = 0; j < nLon; j++) {
dataIZ[k][i][j] = dZs[i][j] + dILev[k] * (dZtop - dZs[i][j]);
}
}
}
}
// Loop through all pressure levels and variables
for (int v = 0; v < vecNcVar.size(); v++) {
bool fOnInterfaces = false;
// Load in the data array
vecNcVar[v]->set_cur(t, 0, 0, 0);
if (vecNcVar[v]->get_dim(1)->size() == nLev) {
vecNcVar[v]->get(&(dataIn[0][0][0]), 1, nLev, nLat, nLon);
Announce("%s (n)", vecVariableStrings[v].c_str());
} else if (vecNcVar[v]->get_dim(1)->size() == nILev) {
vecNcVar[v]->get(&(dataInt[0][0][0]), 1, nILev, nLat, nLon);
fOnInterfaces = true;
Announce("%s (i)", vecVariableStrings[v].c_str());
} else {
_EXCEPTION1("Variable \"%s\" has invalid level dimension",
vecVariableStrings[v].c_str());
}
// At the physical surface
if (fExtractSurface) {
if (fOnInterfaces) {
for (int i = 0; i < nLat; i++) {
for (int j = 0; j < nLon; j++) {
dataOut[i][j] = dataInt[0][i][j];
}
}
} else {
int kBegin = 0;
int kEnd = 3;
PolynomialInterp::LagrangianPolynomialCoeffs(
3, dLev, dW, 0.0);
// Loop thorugh all latlon indices
for (int i = 0; i < nLat; i++) {
for (int j = 0; j < nLon; j++) {
// Interpolate in the vertical
dataOut[i][j] = 0.0;
for (int k = kBegin; k < kEnd; k++) {
dataOut[i][j] += dW[k] * dataIn[k][i][j];
}
}
}
}
// Write variable
vecOutNcVarS[v]->set_cur(t, 0, 0);
vecOutNcVarS[v]->put(&(dataOut[0][0]), 1, nLat, nLon);
}
// Loop through all pressure levels
for (int p = 0; p < nPressureLevels; p++) {
// Loop thorugh all latlon indices
for (int i = 0; i < nLat; i++) {
for (int j = 0; j < nLon; j++) {
// Store column pressure
for (int k = 0; k < nLev; k++) {
dataColumnP[k] = dataP[k][i][j];
}
// Find weights
int kBegin = 0;
int kEnd = 0;
// On a pressure surface
InterpolationWeightsLinear(
vecPressureLevels[p],
dataColumnP,
kBegin,
kEnd,
dW);
// Interpolate in the vertical
dataOut[i][j] = 0.0;
for (int k = kBegin; k < kEnd; k++) {
dataOut[i][j] += dW[k] * dataIn[k][i][j];
}
}
}
// Write variable
vecOutNcVarP[v]->set_cur(t, p, 0, 0);
vecOutNcVarP[v]->put(&(dataOut[0][0]), 1, 1, nLat, nLon);
}
// Loop through all height levels
for (int z = 0; z < nHeightLevels; z++) {
// Loop thorugh all latlon indices
for (int i = 0; i < nLat; i++) {
for (int j = 0; j < nLon; j++) {
// Find weights
int kBegin = 0;
int kEnd = 0;
// Interpolate from levels to z surfaces
if (!fOnInterfaces) {
for (int k = 0; k < nLev; k++) {
dataColumnZ[k] = dataZ[k][i][j];
}
InterpolationWeightsLinear(
vecHeightLevels[z],
dataColumnZ,
kBegin,
kEnd,
dW);
dataOut[i][j] = 0.0;
for (int k = kBegin; k < kEnd; k++) {
dataOut[i][j] += dW[k] * dataIn[k][i][j];
}
// Interpolate from interfaces to z surfaces
} else {
for (int k = 0; k < nILev; k++) {
dataColumnIZ[k] = dataIZ[k][i][j];
}
InterpolationWeightsLinear(
vecHeightLevels[z],
dataColumnIZ,
kBegin,
kEnd,
dIW);
dataOut[i][j] = 0.0;
for (int k = kBegin; k < kEnd; k++) {
dataOut[i][j] += dIW[k] * dataInt[k][i][j];
}
}
}
}
// Write variable
vecOutNcVarZ[v]->set_cur(t, z, 0, 0);
vecOutNcVarZ[v]->put(&(dataOut[0][0]), 1, 1, nLat, nLon);
}
}
// Output geopotential height
if (fGeopotentialHeight) {
Announce("Geopotential height");
// Output variables
NcVar * varOutZ;
NcVar * varOutZs;
if (nPressureLevels > 0) {
varOutZ = ncdf_out.add_var(
"PHIZ", ncDouble, dimOutTime, dimOutP, dimOutLat, dimOutLon);
}
if (fExtractSurface) {
varOutZs = ncdf_out.add_var(
"PHIZS", ncDouble, dimOutTime, dimOutLat, dimOutLon);
}
// Interpolate onto pressure levels
for (int p = 0; p < nPressureLevels; p++) {
// Loop thorugh all latlon indices
for (int i = 0; i < nLat; i++) {
for (int j = 0; j < nLon; j++) {
int kBegin = 0;
int kEnd = 0;
for (int k = 0; k < nLev; k++) {
dataColumnP[k] = dataP[k][i][j];
}
InterpolationWeightsLinear(
vecPressureLevels[p],
dataColumnP,
kBegin,
kEnd,
dW);
// Interpolate in the vertical
dataOut[i][j] = 0.0;
for (int k = kBegin; k < kEnd; k++) {
dataOut[i][j] += dW[k] * dataZ[k][i][j];
}
}
}
// Write variable
varOutZ->set_cur(t, p, 0, 0);
varOutZ->put(&(dataOut[0][0]), 1, 1, nLat, nLon);
}
// Interpolate onto the physical surface
if (fExtractSurface) {
int kBegin = 0;
int kEnd = 3;
PolynomialInterp::LagrangianPolynomialCoeffs(
3, dLev, dW, 0.0);
// Loop thorugh all latlon indices
for (int i = 0; i < nLat; i++) {
for (int j = 0; j < nLon; j++) {
// Interpolate in the vertical
dataOut[i][j] = 0.0;
for (int k = kBegin; k < kEnd; k++) {
dataOut[i][j] += dW[k] * dataZ[k][i][j];
}
}
}
// Write variable
varOutZs->set_cur(t, 0, 0);
varOutZs->put(&(dataOut[0][0]), 1, nLat, nLon);
}
}
// Extract total energy
if (fExtractTotalEnergy) {
Announce("Total Energy");
// Zonal velocity
DataArray3D<double> dataU(nLev, nLat, nLon);
NcVar * varU = ncdf_in.get_var("U");
varU->set_cur(t, 0, 0, 0);
varU->get(&(dataU[0][0][0]), 1, nLev, nLat, nLon);
// Meridional velocity
DataArray3D<double> dataV(nLev, nLat, nLon);
NcVar * varV = ncdf_in.get_var("V");
varV->set_cur(t, 0, 0, 0);
varV->get(&(dataV[0][0][0]), 1, nLev, nLat, nLon);
// Vertical velocity
DataArray3D<double> dataW(nLev, nLat, nLon);
NcVar * varW = ncdf_in.get_var("W");
varW->set_cur(t, 0, 0, 0);
varW->get(&(dataW[0][0][0]), 1, nLev, nLat, nLon);
// Calculate total energy
double dTotalEnergy = 0.0;
double dElementRefArea =
dEarthRadius * dEarthRadius
* M_PI / static_cast<double>(nLat)
* 2.0 * M_PI / static_cast<double>(nLon);
for (int k = 0; k < nLev; k++) {
for (int i = 0; i < nLat; i++) {
for (int j = 0; j < nLon; j++) {
double dKineticEnergy =
0.5 * dataRho[k][i][j] *
( dataU[k][i][j] * dataU[k][i][j]
+ dataV[k][i][j] * dataV[k][i][j]
+ dataW[k][i][j] * dataW[k][i][j]);
double dInternalEnergy =
dataP[k][i][j] / (dGamma - 1.0);
dTotalEnergy +=
(dKineticEnergy + dInternalEnergy)
* std::cos(M_PI * dLat[i] / 180.0) * dElementRefArea
* (dZtop - dZs[i][j]) / static_cast<double>(nLev);
}
}
}
// Put total energy into file
varEnergy->set_cur(t);
varEnergy->put(&dTotalEnergy, 1);
}
AnnounceEndBlock("Done");
}
AnnounceEndBlock("Done");
} catch(Exception & e) {
Announce(e.ToString().c_str());
}
// Finalize MPI
MPI_Finalize();
}
bool
NetworkObject::loadNetCDF(QString flnm)
{
m_fileName = flnm;
NcError err(NcError::verbose_nonfatal);
NcFile ncfFile(flnm.toLatin1().data(), NcFile::ReadOnly);
NcAtt *att;
NcVar *var;
// ---- get gridsize -----
att = ncfFile.get_att("gridsize");
m_nX = att->as_int(0);
m_nY = att->as_int(1);
m_nZ = att->as_int(2);
//------------------------
// ---- get vertex centers -----
var = ncfFile.get_var("vertex_centers");
if (!var)
var = ncfFile.get_var("vertex_center");
if (!var)
var = ncfFile.get_var("vertex_centres");
if (!var)
var = ncfFile.get_var("vertex_centre");
int nv = var->get_dim(0)->size();
float *vc = new float [3*nv];
var->get(vc, nv, 3);
m_vertexCenters.resize(nv);
for(int i=0; i<nv; i++)
m_vertexCenters[i] = Vec(vc[3*i+0], vc[3*i+1], vc[3*i+2]);
delete [] vc;
//------------------------
// ---- get edges -----
var = ncfFile.get_var("edge_neighbours");
int ne = var->get_dim(0)->size();
int *ed = new int [2*ne];
var->get(ed, ne, 2);
m_edgeNeighbours.resize(ne);
for(int i=0; i<ne; i++)
m_edgeNeighbours[i] = qMakePair(ed[2*i], ed[2*i+1]);
delete [] ed;
//------------------------
Vec bmin = m_vertexCenters[0];
Vec bmax = m_vertexCenters[0];
for(int i=0; i<m_vertexCenters.count(); i++)
{
bmin = StaticFunctions::minVec(bmin, m_vertexCenters[i]);
bmax = StaticFunctions::maxVec(bmax, m_vertexCenters[i]);
}
m_centroid = (bmin + bmax)/2;
m_enclosingBox[0] = Vec(bmin.x, bmin.y, bmin.z);
m_enclosingBox[1] = Vec(bmax.x, bmin.y, bmin.z);
m_enclosingBox[2] = Vec(bmax.x, bmax.y, bmin.z);
m_enclosingBox[3] = Vec(bmin.x, bmax.y, bmin.z);
m_enclosingBox[4] = Vec(bmin.x, bmin.y, bmax.z);
m_enclosingBox[5] = Vec(bmax.x, bmin.y, bmax.z);
m_enclosingBox[6] = Vec(bmax.x, bmax.y, bmax.z);
m_enclosingBox[7] = Vec(bmin.x, bmax.y, bmax.z);
// QStringList vatt, eatt;
int nvars = ncfFile.num_vars();
// for (int i=0; i < nvars; i++)
// {
// var = ncfFile.get_var(i);
// QString attname = var->name();
// attname.toLower();
// if (attname.contains("vertex_") &&
// ( attname != "vertex_centers" ||
// attname != "vertex_centres"))
// vatt.append(attname);
// else if (attname.contains("edge_") &&
// attname != "edge_neighbours")
// eatt.append(attname);
// }
m_vertexAttribute.clear();
m_edgeAttribute.clear();
m_vertexRadiusAttribute = -1;
m_edgeRadiusAttribute = -1;
int vri = 0;
int eri = 0;
for (int i=0; i < nvars; i++)
{
var = ncfFile.get_var(i);
QString attname = var->name();
attname.toLower();
if (attname.contains("vertex_") &&
attname != "vertex_center" &&
attname != "vertex_centre" &&
attname != "vertex_centers" &&
attname != "vertex_centres")
{
if (attname == "vertex_radius")
m_vertexRadiusAttribute = vri;
vri++;
QVector<float> val;
val.clear();
if (var->type() == ncByte || var->type() == ncChar)
{
uchar *v = new uchar[nv];
var->get((ncbyte *)v, nv);
for(int j=0; j<nv; j++)
val.append(v[j]);
delete [] v;
}
else if (var->type() == ncShort)
{
short *v = new short[nv];
var->get((short *)v, nv);
for(int j=0; j<nv; j++)
val.append(v[j]);
delete [] v;
}
else if (var->type() == ncInt)
{
int *v = new int[nv];
var->get((int *)v, nv);
for(int j=0; j<nv; j++)
val.append(v[j]);
delete [] v;
}
else if (var->type() == ncFloat)
{
float *v = new float[nv];
var->get((float *)v, nv);
for(int j=0; j<nv; j++)
val.append(v[j]);
delete [] v;
}
if (val.count() > 0)
m_vertexAttribute.append(qMakePair(attname, val));
}
else if (attname.contains("edge_") &&
attname != "edge_neighbours")
{
if (attname == "edge_radius")
m_edgeRadiusAttribute = eri;
eri++;
QVector<float> val;
val.clear();
if (var->type() == ncByte || var->type() == ncChar)
{
uchar *v = new uchar[ne];
var->get((ncbyte *)v, ne);
for(int j=0; j<ne; j++)
val.append(v[j]);
delete [] v;
}
else if (var->type() == ncShort)
{
short *v = new short[ne];
var->get((short *)v, ne);
for(int j=0; j<ne; j++)
val.append(v[j]);
delete [] v;
}
else if (var->type() == ncInt)
{
int *v = new int[ne];
var->get((int *)v, ne);
for(int j=0; j<ne; j++)
val.append(v[j]);
delete [] v;
}
else if (var->type() == ncFloat)
{
float *v = new float[ne];
var->get((float *)v, ne);
for(int j=0; j<ne; j++)
val.append(v[j]);
delete [] v;
}
if (val.count() > 0)
m_edgeAttribute.append(qMakePair(attname, val));
}
}
ncfFile.close();
if (!Global::batchMode())
{
QString str;
str = QString("Grid Size : %1 %2 %3\n").arg(m_nX).arg(m_nY).arg(m_nZ);
str += QString("Vertices : %1\n").arg(m_vertexCenters.count());
str += QString("Edges : %1\n").arg(m_edgeNeighbours.count());
str += QString("\n");
str += QString("Vertex Attributes : %1\n").arg(m_vertexAttribute.count());
for(int i=0; i<m_vertexAttribute.count(); i++)
str += QString(" %1\n").arg(m_vertexAttribute[i].first);
str += QString("\n");
str += QString("Edge Attributes : %1\n").arg(m_edgeAttribute.count());
for(int i=0; i<m_edgeAttribute.count(); i++)
str += QString(" %1\n").arg(m_edgeAttribute[i].first);
QMessageBox::information(0, "Network loaded", str);
}
m_Vatt = 0;
m_Eatt = 0;
m_Vminmax.clear();
m_Eminmax.clear();
m_userVminmax.clear();
m_userEminmax.clear();
for(int i=0; i<m_vertexAttribute.count(); i++)
{
float vmin = m_vertexAttribute[i].second[0];
float vmax = m_vertexAttribute[i].second[0];
for(int j=1; j<m_vertexAttribute[i].second.count(); j++)
{
vmin = qMin((float)m_vertexAttribute[i].second[j], vmin);
vmax = qMax((float)m_vertexAttribute[i].second[j], vmax);
}
m_Vminmax.append(qMakePair(vmin, vmax));
m_userVminmax.append(qMakePair((vmin+vmax)/2, vmax));
}
for(int i=0; i<m_edgeAttribute.count(); i++)
{
float emin = m_edgeAttribute[i].second[0];
float emax = m_edgeAttribute[i].second[0];
for(int j=1; j<m_edgeAttribute[i].second.count(); j++)
{
emin = qMin((float)m_edgeAttribute[i].second[j], emin);
emax = qMax((float)m_edgeAttribute[i].second[j], emax);
}
m_Eminmax.append(qMakePair(emin, emax));
m_userEminmax.append(qMakePair((emin+emax)/2, emax));
}
return true;
}
bool EpidemicDataSet::loadNetCdfFile(const char * filename)
{
#if USE_NETCDF // TODO: should handle this differently
// change netcdf library error behavior
NcError err(NcError::verbose_nonfatal);
// open the netcdf file
NcFile ncFile(filename, NcFile::ReadOnly);
if(!ncFile.is_valid())
{
put_flog(LOG_FATAL, "invalid file %s", filename);
return false;
}
// get dimensions
NcDim * timeDim = ncFile.get_dim("time");
NcDim * nodesDim = ncFile.get_dim("nodes");
NcDim * stratificationsDim = ncFile.get_dim("stratifications");
if(timeDim == NULL || nodesDim == NULL || stratificationsDim == NULL)
{
put_flog(LOG_FATAL, "could not find a required dimension");
return false;
}
numTimes_ = timeDim->size();
// make sure we have the expected number of nodes
if(nodesDim->size() != numNodes_)
{
put_flog(LOG_FATAL, "got %i nodes, expected %i", nodesDim->size(), numNodes_);
return false;
}
put_flog(LOG_DEBUG, "file contains %i timesteps, %i nodes", numTimes_, numNodes_);
// make sure number of stratifications matches our expectation...
int numExpectedStratifications = 1;
for(unsigned int i=0; i<NUM_STRATIFICATION_DIMENSIONS; i++)
{
numExpectedStratifications *= stratifications_[i].size();
}
if(stratificationsDim->size() != numExpectedStratifications)
{
put_flog(LOG_FATAL, "got %i stratifications, expected %i", stratificationsDim->size(), numExpectedStratifications);
return false;
}
// get all float variables with dimensions (time, nodes, stratifications)
for(int i=0; i<ncFile.num_vars(); i++)
{
NcVar * ncVar = ncFile.get_var(i);
if(ncVar->num_dims() == 3 && ncVar->type() == ncFloat && strcmp(ncVar->get_dim(0)->name(), "time") == 0 && strcmp(ncVar->get_dim(1)->name(), "nodes") == 0 && strcmp(ncVar->get_dim(2)->name(), "stratifications") == 0)
{
put_flog(LOG_INFO, "found variable: %s", ncVar->name());
// full shape
blitz::TinyVector<int, 2+NUM_STRATIFICATION_DIMENSIONS> shape;
shape(0) = numTimes_;
shape(1) = numNodes_;
for(int j=0; j<NUM_STRATIFICATION_DIMENSIONS; j++)
{
shape(2 + j) = stratifications_[j].size();
}
blitz::Array<float, 2+NUM_STRATIFICATION_DIMENSIONS> var((float *)ncVar->values()->base(), shape, blitz::duplicateData);
variables_[std::string(ncVar->name())].reference(var);
}
}
#endif
return true;
}
eavlNetCDFImporter::eavlNetCDFImporter(const string &filename)
{
file = new NcFile(filename.c_str(), NcFile::ReadOnly);
if (!file->is_valid())
{
THROW(eavlException,"Couldn't open file!\n");
}
if (debugoutput) cerr << "num_dims="<<file->num_dims()<<endl;
if (debugoutput) cerr << "num_vars="<<file->num_vars()<<endl;
if (debugoutput) cerr << "num_atts="<<file->num_atts()<<endl;
for (int i=0; i<file->num_dims(); i++)
{
NcDim *d = file->get_dim(i);
if (debugoutput) cerr << " dim["<<i<<"]: name="<<d->name()<<" size="<<d->size()<<endl;
}
for (int i=0; i<file->num_atts(); i++)
{
NcAtt *a = file->get_att(i);
if (debugoutput) cerr << " att["<<i<<"]: name="<<a->name()<<" numvals="<<a->num_vals()<<endl;
}
bool found_grid = false;
for (int i=0; i<file->num_vars(); i++)
{
NcVar *v = file->get_var(i);
if (debugoutput)
{
cerr << " var["<<i<<"]: name="<<v->name();
cerr << " ndims="<<v->num_dims();
cerr << " dims = ";
for (int j=0; j<v->num_dims(); j++)
{
cerr << v->get_dim(j)->name();
if (j<v->num_dims()-1)
cerr << "*";
}
cerr << endl;
}
// Here's the condition for what we're going to use;
// we only support one mesh for the moment, so we're picking one.
// Also, the netcdf files we have have the time dim size as "1"
if (v->num_dims() == 4 && string(v->get_dim(0)->name())=="time")
{
if (!found_grid)
{
dims.push_back(v->get_dim(1));
dims.push_back(v->get_dim(2));
dims.push_back(v->get_dim(3));
found_grid = true;
vars.push_back(v);
if (debugoutput) cerr << " * using as first real var\n";
}
else
{
if (string(v->get_dim(1)->name()) == dims[0]->name() &&
string(v->get_dim(2)->name()) == dims[1]->name() &&
string(v->get_dim(3)->name()) == dims[2]->name())
{
vars.push_back(v);
if (debugoutput) cerr << " * using as another var; matches the first real one's dims\n";
}
}
}
}
}