template<class T> static bool load_nc_array(const NcFile& ncf, const string& name, vector<T>& dest, bool required = true, int offset = 0, int count = -1)
{
NcVar *v = load_nc_variable(ncf, name.c_str(), required);
if (v)
{
vector<long> offsets = list_of(offset).repeat(v->num_dims()-1, 0);
v->set_cur(&offsets.front());
vector<long> counts (v->num_dims());
long* shape = v->edges();
transform(shape, shape + v->num_dims(), offsets.begin(), counts.begin(), minus<long>());
delete shape;
if (count > 0)
{
counts[0] = count;
}
dest.resize(product(counts));
bool success = v->get(&dest.front(), &counts.front());
if (!success)
{
dest.resize(0);
check(!required, string("NetcdfDataset::load_nc_array<") + typeid(T).name() + "> " + name + '\n' + "failed with offset " + str(offsets) + ", counts " + str(counts));
}
return success;
}
return false;
}
void DumpableNcFile::dumpvars( void )
{
int n;
static const char* types[] =
{"","byte","char","short","long","float","double"};
NcVar* vp;
for(n = 0; vp = get_var(n); n++) {
cout << "\t" << types[vp->type()] << " " << vp->name() ;
if (vp->num_dims() > 0) {
cout << "(";
for (int d = 0; d < vp->num_dims(); d++) {
NcDim* dim = vp->get_dim(d);
cout << dim->name();
if (d < vp->num_dims()-1)
cout << ", ";
}
cout << ")";
}
cout << " ;\n";
// now dump each of this variable's attributes
dumpatts(*vp);
}
}
bool NetcdfSource::initFile() {
_ncfile = new NcFile(_filename.toUtf8().data(), NcFile::ReadOnly);
if (!_ncfile->is_valid()) {
qDebug() << _filename << ": failed to open in initFile()" << endl;
return false;
}
KST_DBG qDebug() << _filename << ": building field list" << endl;
_fieldList.clear();
_fieldList += "INDEX";
int nb_vars = _ncfile->num_vars();
KST_DBG qDebug() << nb_vars << " vars found in total" << endl;
_maxFrameCount = 0;
for (int i = 0; i < nb_vars; i++) {
NcVar *var = _ncfile->get_var(i);
if (var->num_dims() == 0) {
_scalarList += var->name();
} else if (var->num_dims() == 1) {
_fieldList += var->name();
int fc = var->num_vals() / var->rec_size();
_maxFrameCount = qMax(_maxFrameCount, fc);
_frameCounts[var->name()] = fc;
} else if (var->num_dims() == 2) {
_matrixList += var->name();
}
}
// Get strings
int globalAttributesNb = _ncfile->num_atts();
for (int i = 0; i < globalAttributesNb; ++i) {
// Get only first value, should be enough for a start especially as strings are complete
NcAtt *att = _ncfile->get_att(i);
if (att) {
QString attrName = QString(att->name());
char *attString = att->as_string(0);
QString attrValue = QString(att->as_string(0));
delete[] attString;
//TODO port
//KstString *ms = new KstString(KstObjectTag(attrName, tag()), this, attrValue);
_stringList += attrName;
}
delete att;
}
// TODO update(); // necessary? slows down initial loading
return true;
}
void NetCdfConfigureDialog::getDimEdges(int dimId, unsigned &size, double &firstValue, double &lastValue)
{
if ((_currentFile->get_var(_currentVar->get_dim(dimId)->name())) != NULL)
{
NcVar *tmpVarOfDim = _currentFile->get_var(_currentVar->get_dim(dimId)->name());
if ((tmpVarOfDim->num_dims()) == 1)
{
int sizeOfDim = tmpVarOfDim->get_dim(0)->size();
size = sizeOfDim;
double arrayOfDimStart[1] = {0};
size_t edgeOfArray[1] = {1};
long edgeOrigin[1] = {0};
tmpVarOfDim->set_cur(edgeOrigin);
tmpVarOfDim->get(arrayOfDimStart,edgeOfArray);
firstValue = arrayOfDimStart[0];
double arrayOfDimEnd[1] = {0};
edgeOrigin[0] = sizeOfDim - 1;
tmpVarOfDim->set_cur(edgeOrigin);
tmpVarOfDim->get(arrayOfDimEnd,edgeOfArray);
lastValue = arrayOfDimEnd[0];
}
} else {
size = 0;
firstValue = 0;
lastValue = 0;
}
}
FieldPtr FileArome::getFieldCore(std::string iVariable, int iTime) const {
// Not cached, retrieve data
NcVar* var = getVar(iVariable);
int nLat = mNLat;
int nLon = mNLon;
int numDims = var->num_dims();
long* count;
long totalCount = nLat*nLon;
if(numDims == 4) {
// Variable has a surface dimension
count = new long[4];
count[0] = 1;
count[1] = 1;
count[2] = nLat;
count[3] = nLon;
var->set_cur(iTime, 0, 0, 0);
}
else if(numDims == 3) {
count = new long[3];
count[0] = 1;
count[1] = nLat;
count[2] = nLon;
var->set_cur(iTime, 0, 0);
}
else {
std::stringstream ss;
ss << "Cannot read variable '" << iVariable << "' from '" << getFilename() << "'";
Util::error(ss.str());
}
float* values = new float[nLat*nLon];
var->get(values, count);
float MV = getMissingValue(var);
float offset = getOffset(var);
float scale = getScale(var);
int index = 0;
FieldPtr field = getEmptyField();
for(int lat = 0; lat < nLat; lat++) {
for(int lon = 0; lon < nLon; lon++) {
float value = values[index];
assert(index < totalCount);
if(value == MV) {
// Field has missing value indicator and the value is missing
// Save values using our own internal missing value indicator
value = Util::MV;
}
else {
value = scale*values[index] + offset;
}
(*field)(lat,lon,0) = value;
index++;
}
}
delete[] values;
delete[] count;
return field;
}
void NetCdfConfigureDialog::setVariableSelect()
{
for (int i=0; i<(_currentFile->num_vars()); i++)
{
NcVar *focusedVar = _currentFile->get_var(i);
if (focusedVar->num_dims() > 0) comboBoxVariable->addItem(focusedVar->name());
}
}
Var( const NcVar* var ):
m_var( var )
{
const int ndims = m_var->num_dims();
for ( int i = 0; i < ndims; i++ )
{
m_dims.push_back( new Dim( m_var->get_dim(i) ) );
}
}
ERMsg C20thReanalysisProject::ReadData( CString filePath, CString varName, CData3Array& data)
{
//typedef boost::multi_array<int, 2> array_type;
ERMsg msg;
NcError::set_err( NcError::silent_nonfatal );
//CString filePath = m_path + "cccma_cgcm3_1-20c3m-run1-pr-1961-2000_monthly.nc";//GetFilePath(v, year, m);
NcFile file(filePath); //current year file
if( !file.is_valid() )
{
CString err;
err.FormatMessage(IDS_CMN_UNABLE_OPEN_READ, filePath);
msg.ajoute(err);
return msg;
}
NcVar* pVarData = file.get_var((LPCTSTR)varName);//is the varaible always at ffirst???
//CString varName = pVarData->name();
size_t sizeTime = pVarData->get_dim(0)->size();
size_t sizeY = pVarData->get_dim(1)->size();
size_t sizeX = pVarData->get_dim(2)->size();
float offset = pVarData->get_att("add_offset")->as_float(0);
float scaleFactor = pVarData->get_att("scale_factor")->as_float(0);
boost::multi_array<short, 3> tmp(boost::extents[sizeTime][sizeY][sizeX]);
ENSURE( pVarData->num_dims() == 3);
if( pVarData->get(&(tmp[0][0][0]), sizeTime, sizeY, sizeX) )
{
//tmp.extens()
//data.resize(sizeTime);
//data.resize(boost::extents[sizeTime][sizeY][sizeX]);
//apply offset and scale factor
for(size_t i=0; i<tmp.size(); i++)
for(size_t j=0; j<tmp[i].size(); j++)
for(size_t k=0; k<tmp[i][j].size(); k++)
data[i][j][k] = tmp[i][j][k]*scaleFactor+offset;
file.close();
}
else
{
msg.ajoute( "Unable to get NetCDFData");
}
return msg;
}
bool ReadAstro::open(const char *path)
{
assert(!m_ncfile);
m_ncfile = new NcFile(path, NcFile::ReadOnly);
if (!m_ncfile->is_valid())
{
close();
sendError("failed to open NetCDF file %s", path);
return false;
}
if (m_ncfile->get_format() == NcFile::BadFormat)
{
close();
sendError("bad NetCDF file");
return false;
}
fprintf(stderr, "dims=%d, vars=%d, attrs=%d\n",
m_ncfile->num_dims(), m_ncfile->num_vars(), m_ncfile->num_atts());
for (int i = 0; i < m_ncfile->num_dims(); ++i)
{
fprintf(stderr, "%s: %ld\n",
m_ncfile->get_dim(i)->name(),
m_ncfile->get_dim(i)->size());
}
for (int i = 0; i < m_ncfile->num_vars(); ++i)
{
fprintf(stderr, "%s: dims=%d atts=%d vals=%ld type=%d\n",
m_ncfile->get_var(i)->name(),
m_ncfile->get_var(i)->num_dims(),
m_ncfile->get_var(i)->num_atts(),
m_ncfile->get_var(i)->num_vals(),
m_ncfile->get_var(i)->type());
//int dims = m_ncfile->get_var(i)->num_dims();
NcVar *var = m_ncfile->get_var(i);
for (int j = 0; j < var->num_dims(); ++j)
{
fprintf(stderr, " %s: %ld edge=%ld\n",
var->get_dim(j)->name(),
var->get_dim(j)->size(),
var->edges()[j]);
}
}
for (int i = 0; i < m_ncfile->num_atts(); ++i)
{
fprintf(stderr, "%s\n", m_ncfile->get_att(i)->name());
}
return true;
}
int main(int argc, char *argv[])
{
NcFile at(atpath, NcFile::ReadOnly);
if(!at.is_valid() || at.num_dims() != 3 || at.num_vars() != 4) {
fprintf(stderr, "failed reading file: %s\n", atpath);
return 1;
}
NcVar* data = at.get_var("rhum");
if(!data->is_valid() || data->num_dims() != 3) {
fprintf(stderr, "rhum has incorrect dimensions");
return 1;
}
NcDim* time = data->get_dim(0);
int timecnt = time->size();
float *rhumd = new float[timecnt*LATS*LONS];
data->get(rhumd, timecnt, LATS, LONS);
float rhumdmon[12][LATS][LONS];
for(int i = 0; i<LATS; i++)
for(int j = 0; j<LONS; j++) {
float rhumdmoncnt[12];
for(int k = 0; k<12; k++) {
rhumdmon[k][i][j] = 0;
rhumdmoncnt[k] = 0;
}
for(int k = 0; k<timecnt; k++) {
double v = rhumd[(k*LATS+i)*LONS+j]*.1 + 3276.5;
if(v >= 0 && v <= 100) {
rhumdmon[k%12][i][j] += v;
rhumdmoncnt[k%12]++;
}
}
for(int k = 0; k<12; k++)
rhumdmon[k][i][j] /= rhumdmoncnt[k];
}
delete [] rhumd;
/* use a single byte instead of 2 to save memory,
resolution of 1/5th of a mm/day resolution */
uint8_t rhumbyte[12][LATS][LONS];
for(int i = 0; i<12; i++)
for(int j = 0; j<LATS; j++)
for(int k = 0; k<LONS; k++)
if(isnan(rhumdmon[i][j][k]) || fabs(rhumdmon[i][j][k]) > 100)
rhumbyte[i][j][k] = 255;
else
rhumbyte[i][j][k] = rhumdmon[i][j][k]*2.0;
fwrite(rhumbyte, sizeof rhumbyte, 1, stdout);
return 0;
}
const DataMatrix::DataInfo DataInterfaceNetCdfMatrix::dataInfo(const QString& matrix) const
{
if (!netcdf._matrixList.contains( matrix ) ) {
return DataMatrix::DataInfo();
}
QByteArray bytes = matrix.toLatin1();
NcVar *var = netcdf._ncfile->get_var(bytes.constData()); // var is owned by _ncfile
if (var->num_dims() != 2) {
return DataMatrix::DataInfo();
}
DataMatrix::DataInfo info;
info.samplesPerFrame = 1;
// TODO is this right?
info.xSize = var->get_dim(0)->size();
info.ySize = var->get_dim(1)->size();
return info;
}
blitz::Array<T,rank> read_blitz(NcFile &nc, std::string const &var_name)
{
// Read points vector
NcVar *vpoints = nc.get_var(var_name.c_str());
int ndims = vpoints->num_dims();
if (ndims != rank) {
fprintf(stderr, "NetCDF variable %s has rank %d, expected rank %d\n",
var_name.c_str(), ndims, rank);
throw std::exception();
}
blitz::TinyVector<int,rank> shape(0);
long counts[rank];
for (int i=0; i<rank; ++i) {
shape[i] = vpoints->get_dim(i)->size();
printf("read_blitz: shape[%d] = %d\n", i, shape[i]);
counts[i] = shape[i];
}
blitz::Array<T,rank> ret(shape);
for (int i=0; i<rank; ++i) printf("read_blitz: ret.extent(%d) = %d\n", i, ret.extent(i));
vpoints->get(ret.data(), counts);
return ret;
}
void FileArome::writeCore(std::vector<Variable::Type> iVariables) {
writeTimes();
writeReferenceTime();
writeGlobalAttributes();
for(int v = 0; v < iVariables.size(); v++) {
Variable::Type varType = iVariables[v];
std::string variable = getVariableName(varType);
NcVar* var;
if(hasVariableCore(varType)) {
var = getVar(variable);
}
else {
// Create variable
if(0) {
NcDim* dTime = getDim("time");
NcDim* dSurface = getDim("height0");
NcDim* dLon = getDim("x");
NcDim* dLat = getDim("y");
var = mFile.add_var(variable.c_str(), ncFloat, dTime, dSurface, dLat, dLon);
}
else {
NcDim* dTime = getDim("time");
NcDim* dLon = getDim("x");
NcDim* dLat = getDim("y");
var = mFile.add_var(variable.c_str(), ncFloat, dTime, dLat, dLon);
}
}
float MV = getMissingValue(var); // The output file's missing value indicator
for(int t = 0; t < mNTime; t++) {
float offset = getOffset(var);
float scale = getScale(var);
FieldPtr field = getField(varType, t);
if(field != NULL) { // TODO: Can't be null if coming from reference
float* values = new float[mNLat*mNLon];
int index = 0;
for(int lat = 0; lat < mNLat; lat++) {
for(int lon = 0; lon < mNLon; lon++) {
float value = (*field)(lat,lon,0);
if(!Util::isValid(value)) {
// Field has missing value indicator and the value is missing
// Save values using the file's missing indicator value
value = MV;
}
else {
value = ((*field)(lat,lon,0) - offset)/scale;
}
values[index] = value;
index++;
}
}
int numDims = var->num_dims();
if(numDims == 4) {
var->set_cur(t, 0, 0, 0);
var->put(values, 1, 1, mNLat, mNLon);
}
else if(numDims == 3) {
var->set_cur(t, 0, 0);
var->put(values, 1, mNLat, mNLon);
}
else {
std::stringstream ss;
ss << "Cannot write variable '" << variable << "' from '" << getFilename() << "'";
Util::error(ss.str());
}
setAttribute(var, "coordinates", "longitude latitude");
setAttribute(var, "units", Variable::getUnits(varType));
setAttribute(var, "standard_name", Variable::getStandardName(varType));
delete[] values;
}
}
setMissingValue(var, MV);
}
}
void ReadCFTimeDataFromNcFile(
NcFile * ncfile,
const std::string & strFilename,
std::vector<Time> & vecTimes,
bool fWarnOnMissingCalendar
) {
// Empty existing Time vector
vecTimes.clear();
// Get time dimension
NcDim * dimTime = ncfile->get_dim("time");
if (dimTime == NULL) {
_EXCEPTION1("Dimension \"time\" not found in file \"%s\"",
strFilename.c_str());
}
// Get time variable
NcVar * varTime = ncfile->get_var("time");
if (varTime == NULL) {
_EXCEPTION1("Variable \"time\" not found in file \"%s\"",
strFilename.c_str());
}
if (varTime->num_dims() != 1) {
_EXCEPTION1("Variable \"time\" has more than one dimension in file \"%s\"",
strFilename.c_str());
}
if (strcmp(varTime->get_dim(0)->name(), "time") != 0) {
_EXCEPTION1("Variable \"time\" does not have dimension \"time\" in file \"%s\"",
strFilename.c_str());
}
// Calendar attribute
NcAtt * attTimeCal = varTime->get_att("calendar");
std::string strCalendar;
if (attTimeCal == NULL) {
if (fWarnOnMissingCalendar) {
Announce("WARNING: Variable \"time\" is missing \"calendar\" attribute; assuming \"standard\"");
}
strCalendar = "standard";
} else {
strCalendar = attTimeCal->as_string(0);
}
Time::CalendarType eCalendarType =
Time::CalendarTypeFromString(strCalendar);
// Units attribute
NcAtt * attTimeUnits = varTime->get_att("units");
if (attTimeUnits == NULL) {
_EXCEPTION1("Variable \"time\" is missing \"units\" attribute in file \"%s\"",
strFilename.c_str());
}
std::string strTimeUnits = attTimeUnits->as_string(0);
// Load in time data
DataVector<int> vecTimeInt;
DataVector<float> vecTimeFloat;
DataVector<double> vecTimeDouble;
DataVector<ncint64> vecTimeInt64;
if (varTime->type() == ncInt) {
vecTimeInt.Initialize(dimTime->size());
varTime->set_cur((long)0);
varTime->get(&(vecTimeInt[0]), dimTime->size());
} else if (varTime->type() == ncFloat) {
vecTimeFloat.Initialize(dimTime->size());
varTime->set_cur((long)0);
varTime->get(&(vecTimeFloat[0]), dimTime->size());
} else if (varTime->type() == ncDouble) {
vecTimeDouble.Initialize(dimTime->size());
varTime->set_cur((long)0);
varTime->get(&(vecTimeDouble[0]), dimTime->size());
} else if (varTime->type() == ncInt64) {
vecTimeInt64.Initialize(dimTime->size());
varTime->set_cur((long)0);
varTime->get(&(vecTimeInt64[0]), dimTime->size());
} else {
_EXCEPTION1("Variable \"time\" has invalid type "
"(expected \"int\", \"int64\", \"float\" or \"double\")"
" in file \"%s\"", strFilename.c_str());
}
for (int t = 0; t < dimTime->size(); t++) {
Time time(eCalendarType);
if (varTime->type() == ncInt) {
time.FromCFCompliantUnitsOffsetInt(
strTimeUnits,
vecTimeInt[t]);
} else if (varTime->type() == ncFloat) {
time.FromCFCompliantUnitsOffsetDouble(
strTimeUnits,
static_cast<double>(vecTimeFloat[t]));
} else if (varTime->type() == ncDouble) {
time.FromCFCompliantUnitsOffsetDouble(
strTimeUnits,
vecTimeDouble[t]);
} else if (varTime->type() == ncInt64) {
time.FromCFCompliantUnitsOffsetInt(
strTimeUnits,
(int)(vecTimeInt64[t]));
}
vecTimes.push_back(time);
}
}
void CopyNcVar(
NcFile & ncIn,
NcFile & ncOut,
const std::string & strVarName,
bool fCopyAttributes,
bool fCopyData
) {
if (!ncIn.is_valid()) {
_EXCEPTIONT("Invalid input file specified");
}
if (!ncOut.is_valid()) {
_EXCEPTIONT("Invalid output file specified");
}
NcVar * var = ncIn.get_var(strVarName.c_str());
if (var == NULL) {
_EXCEPTION1("NetCDF file does not contain variable \"%s\"",
strVarName.c_str());
}
NcVar * varOut;
std::vector<NcDim *> dimOut;
dimOut.resize(var->num_dims());
std::vector<long> counts;
counts.resize(var->num_dims());
long nDataSize = 1;
for (int d = 0; d < var->num_dims(); d++) {
NcDim * dimA = var->get_dim(d);
dimOut[d] = ncOut.get_dim(dimA->name());
if (dimOut[d] == NULL) {
if (dimA->is_unlimited()) {
dimOut[d] = ncOut.add_dim(dimA->name());
} else {
dimOut[d] = ncOut.add_dim(dimA->name(), dimA->size());
}
if (dimOut[d] == NULL) {
_EXCEPTION2("Failed to add dimension \"%s\" (%i) to file",
dimA->name(), dimA->size());
}
}
if (dimOut[d]->size() != dimA->size()) {
if (dimA->is_unlimited() && !dimOut[d]->is_unlimited()) {
_EXCEPTION2("Mismatch between input file dimension \"%s\" and "
"output file dimension (UNLIMITED / %i)",
dimA->name(), dimOut[d]->size());
} else if (!dimA->is_unlimited() && dimOut[d]->is_unlimited()) {
_EXCEPTION2("Mismatch between input file dimension \"%s\" and "
"output file dimension (%i / UNLIMITED)",
dimA->name(), dimA->size());
} else if (!dimA->is_unlimited() && !dimOut[d]->is_unlimited()) {
_EXCEPTION3("Mismatch between input file dimension \"%s\" and "
"output file dimension (%i / %i)",
dimA->name(), dimA->size(), dimOut[d]->size());
}
}
counts[d] = dimA->size();
nDataSize *= counts[d];
}
// ncByte / ncChar type
if ((var->type() == ncByte) || (var->type() == ncChar)) {
DataVector<char> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
// ncShort type
if (var->type() == ncShort) {
DataVector<short> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
if (fCopyData) {
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
}
// ncInt type
if (var->type() == ncInt) {
DataVector<int> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
if (fCopyData) {
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
}
// ncFloat type
if (var->type() == ncFloat) {
DataVector<float> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
if (fCopyData) {
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
}
// ncDouble type
if (var->type() == ncDouble) {
DataVector<double> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
if (fCopyData) {
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
}
// ncInt64 type
if (var->type() == ncInt64) {
DataVector<ncint64> data;
data.Initialize(nDataSize);
varOut =
ncOut.add_var(
var->name(), var->type(),
dimOut.size(), (const NcDim**)&(dimOut[0]));
if (varOut == NULL) {
_EXCEPTION1("Cannot create variable \"%s\"", var->name());
}
if (fCopyData) {
var->get(&(data[0]), &(counts[0]));
varOut->put(&(data[0]), &(counts[0]));
}
}
// Check output variable exists
if (varOut == NULL) {
_EXCEPTION1("Unable to create output variable \"%s\"",
var->name());
}
// Copy attributes
if (fCopyAttributes) {
CopyNcVarAttributes(var, varOut);
}
}
int main(int argc, char** argv)
{
if (!cmdline(argc, argv))
{
printhelp();
return EXIT_FAILURE;
}
NcFile infile(infilename.c_str(), NcFile::ReadOnly);
if (!infile.is_valid())
{
std::cerr << "Error: invalid input file -- '" << infilename << "'" << std::endl;
infile.close();
return EXIT_FAILURE;
}
NcFile outfile(outfilename.c_str(), NcFile::Replace);
if (!outfile.is_valid())
{
std::cerr << "Error: cannot open output file -- '" << outfilename << "'" << std::endl;
outfile.close();
return EXIT_FAILURE;
}
if (varstrings.size() == 0)
{
std::cerr << "Warning: no variables specified" << std::endl;
}
std::vector<NcVar*> invars;
for (std::vector<std::string>::const_iterator it = varstrings.begin();
it != varstrings.end(); ++it)
{
NcVar* var = infile.get_var((*it).c_str());
if (var == NULL)
{
std::cerr << "Error: " << *it << ": no such variable" << std::endl;
infile.close();
outfile.close();
return EXIT_FAILURE;
}
invars.push_back(var);
}
// extract the distinct set of dims
std::map<std::string, NcDim*> indims;
for (std::vector<NcVar*>::const_iterator it = invars.begin();
it != invars.end(); ++it)
{
NcVar* var = *it;
for (int i = 0; i < var->num_dims(); ++i)
{
NcDim* dim = var->get_dim(i);
indims[dim->name()] = dim;
}
}
// add dims to outfile
std::map<std::string, NcDim*> outdims;
for (std::map<std::string, NcDim*>::const_iterator it = indims.begin();
it != indims.end(); ++it)
{
NcDim* dim = (*it).second;
NcDim* outdim = NULL;
if (dim->is_unlimited())
{
outdim = outfile.add_dim(dim->name());
}
else
{
outdim = outfile.add_dim(dim->name(), dim->size());
}
if (outdim != NULL)
{
outdims[outdim->name()] = outdim;
}
}
// create variables
for (std::vector<NcVar*>::const_iterator it = invars.begin();
it != invars.end(); ++it)
{
NcVar* var = *it;
std::vector<const NcDim*> dims(var->num_dims());
for (int i = 0; i < var->num_dims(); ++i)
{
dims[i] = outdims[var->get_dim(i)->name()];
}
NcVar* outvar = outfile.add_var(var->name(), var->type(), var->num_dims(), &dims[0]);
// identify largest dim, if dim (nearly) exceeds main memory, split along that dim
int maxdim = -1;
long maxdimsize = 0;
long totallen = 1;
for (int i = 0; i < var->num_dims(); ++i)
{
NcDim* dim = var->get_dim(i);
if (dim->size() > maxdimsize)
{
maxdim = i;
maxdimsize = dim->size();
}
totallen *= dim->size();
}
// TODO: support other data types
totallen *= sizeof(float);
// TODO: configurable page size
const unsigned long pagesize = 1000000000;
#ifdef __linux__
struct sysinfo info;
sysinfo(&info);
if (pagesize >= info.freeram)
{
std::cerr << "Warning: page size exceeds free memory" << std::endl;
}
#endif
int numpages = 1;
long pagesizedim = var->get_dim(maxdim)->size();
if (totallen < pagesize)
{
}
else
{
long mul = 1;
for (int i = 0; i < var->num_dims(); ++i)
{
if (i != maxdim)
{
NcDim* dim = var->get_dim(i);
mul *= dim->size();
}
}
// TODO: support other data types
mul *= sizeof(float);
pagesizedim = pagesize / mul;
numpages = var->get_dim(maxdim)->size() / pagesizedim;
if (var->get_dim(maxdim)->size() % pagesizedim > 0)
{
++numpages;
}
}
std::vector< std::vector<long> > curvec;
std::vector< std::vector<long> > countsvec;
std::vector<long> lengths;
int pages = numpages > 0 ? numpages : 1;
for (int p = 0; p < pages; ++p)
{
long len = 1;
std::vector<long> cur;
std::vector<long> counts;
for (int i = 0; i < var->num_dims(); ++i)
{
NcDim* dim = var->get_dim(i);
long current = 0;
long count = dim->size();
if (i == maxdim)
{
current = pagesizedim * p;
count = pagesizedim;
if (p == pages -1)
{
if (dim->size() % pagesizedim != 0)
{
count = dim->size() % pagesizedim;
}
}
}
cur.push_back(current);
counts.push_back(count);
len *= count;
}
curvec.push_back(cur);
countsvec.push_back(counts);
lengths.push_back(len);
}
std::vector< std::vector<long> >::const_iterator it1;
std::vector< std::vector<long> >::const_iterator it2;
std::vector<long>::const_iterator it3;
for (it1 = curvec.begin(), it2 = countsvec.begin(), it3 = lengths.begin();
it1 != curvec.end() && it2 != countsvec.end() && it3 != lengths.end(); ++it1, ++it2, ++it3)
{
std::vector<long> cur = *it1;
std::vector<long> counts = *it2;
long len = *it3;
var->set_cur(&cur[0]);
outvar->set_cur(&cur[0]);
switch (outvar->type())
{
case ncByte:
{
ncbyte* barr = new ncbyte[len];
var->get(barr, &counts[0]);
outvar->put(barr, &counts[0]);
delete[] barr;
break;
}
case ncChar:
{
char* carr = new char[len];
var->get(carr, &counts[0]);
outvar->put(carr, &counts[0]);
delete[] carr;
break;
}
case ncShort:
{
short* sarr = new short[len];
var->get(sarr, &counts[0]);
outvar->put(sarr, &counts[0]);
delete[] sarr;
break;
}
case ncInt:
{
long* larr = new long[len];
var->get(larr, &counts[0]);
outvar->put(larr, &counts[0]);
delete[] larr;
break;
}
case ncFloat:
{
float* farr = new float[len];
var->get(farr, &counts[0]);
outvar->put(farr, &counts[0]);
delete[] farr;
break;
}
case ncDouble:
{
double* darr = new double[len];
var->get(darr, &counts[0]);
outvar->put(darr, &counts[0]);
delete[] darr;
break;
}
default:
break;
}
}
}
infile.close();
outfile.close();
return 0;
}
bool EpidemicDataSet::loadNetCdfFile(const char * filename)
{
#if USE_NETCDF // TODO: should handle this differently
// change netcdf library error behavior
NcError err(NcError::verbose_nonfatal);
// open the netcdf file
NcFile ncFile(filename, NcFile::ReadOnly);
if(!ncFile.is_valid())
{
put_flog(LOG_FATAL, "invalid file %s", filename);
return false;
}
// get dimensions
NcDim * timeDim = ncFile.get_dim("time");
NcDim * nodesDim = ncFile.get_dim("nodes");
NcDim * stratificationsDim = ncFile.get_dim("stratifications");
if(timeDim == NULL || nodesDim == NULL || stratificationsDim == NULL)
{
put_flog(LOG_FATAL, "could not find a required dimension");
return false;
}
numTimes_ = timeDim->size();
// make sure we have the expected number of nodes
if(nodesDim->size() != numNodes_)
{
put_flog(LOG_FATAL, "got %i nodes, expected %i", nodesDim->size(), numNodes_);
return false;
}
put_flog(LOG_DEBUG, "file contains %i timesteps, %i nodes", numTimes_, numNodes_);
// make sure number of stratifications matches our expectation...
int numExpectedStratifications = 1;
for(unsigned int i=0; i<NUM_STRATIFICATION_DIMENSIONS; i++)
{
numExpectedStratifications *= stratifications_[i].size();
}
if(stratificationsDim->size() != numExpectedStratifications)
{
put_flog(LOG_FATAL, "got %i stratifications, expected %i", stratificationsDim->size(), numExpectedStratifications);
return false;
}
// get all float variables with dimensions (time, nodes, stratifications)
for(int i=0; i<ncFile.num_vars(); i++)
{
NcVar * ncVar = ncFile.get_var(i);
if(ncVar->num_dims() == 3 && ncVar->type() == ncFloat && strcmp(ncVar->get_dim(0)->name(), "time") == 0 && strcmp(ncVar->get_dim(1)->name(), "nodes") == 0 && strcmp(ncVar->get_dim(2)->name(), "stratifications") == 0)
{
put_flog(LOG_INFO, "found variable: %s", ncVar->name());
// full shape
blitz::TinyVector<int, 2+NUM_STRATIFICATION_DIMENSIONS> shape;
shape(0) = numTimes_;
shape(1) = numNodes_;
for(int j=0; j<NUM_STRATIFICATION_DIMENSIONS; j++)
{
shape(2 + j) = stratifications_[j].size();
}
blitz::Array<float, 2+NUM_STRATIFICATION_DIMENSIONS> var((float *)ncVar->values()->base(), shape, blitz::duplicateData);
variables_[std::string(ncVar->name())].reference(var);
}
}
#endif
return true;
}
eavlNetCDFImporter::eavlNetCDFImporter(const string &filename)
{
file = new NcFile(filename.c_str(), NcFile::ReadOnly);
if (!file->is_valid())
{
THROW(eavlException,"Couldn't open file!\n");
}
if (debugoutput) cerr << "num_dims="<<file->num_dims()<<endl;
if (debugoutput) cerr << "num_vars="<<file->num_vars()<<endl;
if (debugoutput) cerr << "num_atts="<<file->num_atts()<<endl;
for (int i=0; i<file->num_dims(); i++)
{
NcDim *d = file->get_dim(i);
if (debugoutput) cerr << " dim["<<i<<"]: name="<<d->name()<<" size="<<d->size()<<endl;
}
for (int i=0; i<file->num_atts(); i++)
{
NcAtt *a = file->get_att(i);
if (debugoutput) cerr << " att["<<i<<"]: name="<<a->name()<<" numvals="<<a->num_vals()<<endl;
}
bool found_grid = false;
for (int i=0; i<file->num_vars(); i++)
{
NcVar *v = file->get_var(i);
if (debugoutput)
{
cerr << " var["<<i<<"]: name="<<v->name();
cerr << " ndims="<<v->num_dims();
cerr << " dims = ";
for (int j=0; j<v->num_dims(); j++)
{
cerr << v->get_dim(j)->name();
if (j<v->num_dims()-1)
cerr << "*";
}
cerr << endl;
}
// Here's the condition for what we're going to use;
// we only support one mesh for the moment, so we're picking one.
// Also, the netcdf files we have have the time dim size as "1"
if (v->num_dims() == 4 && string(v->get_dim(0)->name())=="time")
{
if (!found_grid)
{
dims.push_back(v->get_dim(1));
dims.push_back(v->get_dim(2));
dims.push_back(v->get_dim(3));
found_grid = true;
vars.push_back(v);
if (debugoutput) cerr << " * using as first real var\n";
}
else
{
if (string(v->get_dim(1)->name()) == dims[0]->name() &&
string(v->get_dim(2)->name()) == dims[1]->name() &&
string(v->get_dim(3)->name()) == dims[2]->name())
{
vars.push_back(v);
if (debugoutput) cerr << " * using as another var; matches the first real one's dims\n";
}
}
}
}
}