Example #1
0
template<class T> static bool load_nc_array(const NcFile& ncf, const string& name, vector<T>& dest, bool required = true, int offset = 0, int count = -1)
{
	NcVar *v = load_nc_variable(ncf, name.c_str(), required);
	if (v)
	{
		vector<long> offsets = list_of(offset).repeat(v->num_dims()-1, 0);
		v->set_cur(&offsets.front());
		vector<long> counts (v->num_dims());
		long* shape = v->edges();
		transform(shape, shape + v->num_dims(), offsets.begin(), counts.begin(), minus<long>());
		delete shape;
		if (count > 0)
		{
			counts[0] = count;
		}
		dest.resize(product(counts));
		bool success = v->get(&dest.front(), &counts.front());
		if (!success)
		{
			dest.resize(0);
			check(!required, string("NetcdfDataset::load_nc_array<") + typeid(T).name() + "> " + name + '\n' + "failed with offset " + str(offsets) + ", counts " + str(counts));
		}
		return success;
	}
	return false;
}
Example #2
0
void DumpableNcFile::dumpvars( void )
{
    int n;
    static const char* types[] =
      {"","byte","char","short","long","float","double"};
    NcVar* vp;

    for(n = 0; vp = get_var(n); n++) {
	cout << "\t" << types[vp->type()] << " " << vp->name() ;

	if (vp->num_dims() > 0) {
	    cout << "(";
	    for (int d = 0; d < vp->num_dims(); d++) {
		NcDim* dim = vp->get_dim(d);
		cout << dim->name();
		if (d < vp->num_dims()-1)
		  cout << ", ";		  
	    }
	    cout << ")";
	}
	cout << " ;\n";
	// now dump each of this variable's attributes
	dumpatts(*vp);
    }
}
Example #3
0
bool NetcdfSource::initFile() {
  _ncfile = new NcFile(_filename.toUtf8().data(), NcFile::ReadOnly);
  if (!_ncfile->is_valid()) {
      qDebug() << _filename << ": failed to open in initFile()" << endl;
      return false;
    }

  KST_DBG qDebug() << _filename << ": building field list" << endl;
  _fieldList.clear();
  _fieldList += "INDEX";

  int nb_vars = _ncfile->num_vars();
  KST_DBG qDebug() << nb_vars << " vars found in total" << endl;

  _maxFrameCount = 0;

  for (int i = 0; i < nb_vars; i++) {
    NcVar *var = _ncfile->get_var(i);
    if (var->num_dims() == 0) {
      _scalarList += var->name();
    } else if (var->num_dims() == 1) {
      _fieldList += var->name();
      int fc = var->num_vals() / var->rec_size();
      _maxFrameCount = qMax(_maxFrameCount, fc);
      _frameCounts[var->name()] = fc;
    } else if (var->num_dims() == 2) {
      _matrixList += var->name();
    }
  }

  // Get strings
  int globalAttributesNb = _ncfile->num_atts();
  for (int i = 0; i < globalAttributesNb; ++i) {
    // Get only first value, should be enough for a start especially as strings are complete
    NcAtt *att = _ncfile->get_att(i);
    if (att) {
      QString attrName = QString(att->name());
      char *attString = att->as_string(0);
      QString attrValue = QString(att->as_string(0));
      delete[] attString;
      //TODO port
      //KstString *ms = new KstString(KstObjectTag(attrName, tag()), this, attrValue);
      _stringList += attrName;
    }
    delete att;
  }

  // TODO update(); // necessary?  slows down initial loading
  return true;
}
Example #4
0
void NetCdfConfigureDialog::getDimEdges(int dimId, unsigned &size, double &firstValue, double &lastValue)
{
    if ((_currentFile->get_var(_currentVar->get_dim(dimId)->name())) != NULL)
    {
        NcVar *tmpVarOfDim = _currentFile->get_var(_currentVar->get_dim(dimId)->name());
        if ((tmpVarOfDim->num_dims()) == 1)
        {
            int sizeOfDim = tmpVarOfDim->get_dim(0)->size();
            size = sizeOfDim;
            double arrayOfDimStart[1] = {0};
            size_t edgeOfArray[1] = {1};
            long edgeOrigin[1] = {0};
            tmpVarOfDim->set_cur(edgeOrigin);
            tmpVarOfDim->get(arrayOfDimStart,edgeOfArray);
            firstValue = arrayOfDimStart[0];
            double arrayOfDimEnd[1] = {0};
            edgeOrigin[0] = sizeOfDim - 1;
            tmpVarOfDim->set_cur(edgeOrigin);
            tmpVarOfDim->get(arrayOfDimEnd,edgeOfArray);
            lastValue = arrayOfDimEnd[0];
        }
    } else {
        size = 0;
        firstValue = 0;
        lastValue = 0;
    }
}
Example #5
0
FieldPtr FileArome::getFieldCore(std::string iVariable, int iTime) const {
   // Not cached, retrieve data
   NcVar* var = getVar(iVariable);
   int nLat  = mNLat;
   int nLon  = mNLon;

   int numDims = var->num_dims();

   long* count;
   long totalCount = nLat*nLon;
   if(numDims == 4) {
      // Variable has a surface dimension
      count = new long[4];
      count[0] = 1;
      count[1] = 1;
      count[2] = nLat;
      count[3] = nLon;
      var->set_cur(iTime, 0, 0, 0);
   }
   else if(numDims == 3) {
      count = new long[3];
      count[0] = 1;
      count[1] = nLat;
      count[2] = nLon;
      var->set_cur(iTime, 0, 0);
   }
   else {
      std::stringstream ss;
      ss << "Cannot read variable '" << iVariable << "' from '" << getFilename() << "'";
      Util::error(ss.str());
   }
   float* values = new float[nLat*nLon];
   var->get(values, count);
   float MV = getMissingValue(var);

   float offset = getOffset(var);
   float scale = getScale(var);
   int index = 0;
   FieldPtr field = getEmptyField();
   for(int lat = 0; lat < nLat; lat++) {
      for(int lon = 0; lon < nLon; lon++) {
         float value = values[index];
         assert(index < totalCount);
         if(value == MV) {
            // Field has missing value indicator and the value is missing
            // Save values using our own internal missing value indicator
            value = Util::MV;
         }
         else {
            value = scale*values[index] + offset;
         }
         (*field)(lat,lon,0) = value;
         index++;
      }
   }
   delete[] values;
   delete[] count;
   return field;
}
Example #6
0
void NetCdfConfigureDialog::setVariableSelect()
{
    for (int i=0; i<(_currentFile->num_vars()); i++)
    {
        NcVar *focusedVar = _currentFile->get_var(i);
        if (focusedVar->num_dims() > 0) comboBoxVariable->addItem(focusedVar->name());
    }
}
Example #7
0
 Var( const NcVar* var ):
     m_var( var )
 {
     const int ndims = m_var->num_dims();
     for ( int i = 0; i < ndims; i++ )
     {
         m_dims.push_back( new Dim( m_var->get_dim(i) ) );
     }
 }
ERMsg C20thReanalysisProject::ReadData( CString filePath, CString varName, CData3Array& data)
{
	//typedef boost::multi_array<int, 2> array_type;


	ERMsg msg;
	
	NcError::set_err( NcError::silent_nonfatal );

  
	//CString filePath = m_path + "cccma_cgcm3_1-20c3m-run1-pr-1961-2000_monthly.nc";//GetFilePath(v, year, m);
	NcFile file(filePath);	//current year file

	if( !file.is_valid() )
	{
		CString err;
		err.FormatMessage(IDS_CMN_UNABLE_OPEN_READ, filePath);
		msg.ajoute(err);
		return msg;
	}

	NcVar* pVarData = file.get_var((LPCTSTR)varName);//is the varaible always at ffirst???
	
	//CString varName = pVarData->name();
	
	size_t sizeTime = pVarData->get_dim(0)->size();
	size_t sizeY = pVarData->get_dim(1)->size();
	size_t sizeX = pVarData->get_dim(2)->size();
	float offset = pVarData->get_att("add_offset")->as_float(0);
	float scaleFactor = pVarData->get_att("scale_factor")->as_float(0);
	
	boost::multi_array<short, 3> tmp(boost::extents[sizeTime][sizeY][sizeX]);

	ENSURE( pVarData->num_dims() == 3);
	
	if( pVarData->get(&(tmp[0][0][0]), sizeTime, sizeY, sizeX) )
	{
		//tmp.extens()
		//data.resize(sizeTime);
		//data.resize(boost::extents[sizeTime][sizeY][sizeX]);
		//apply offset and scale factor
		for(size_t i=0; i<tmp.size(); i++)
			for(size_t j=0; j<tmp[i].size(); j++)
				for(size_t k=0; k<tmp[i][j].size(); k++)
					data[i][j][k] = tmp[i][j][k]*scaleFactor+offset;

		file.close();
	}
	else
	{
		msg.ajoute( "Unable to get NetCDFData");
	}

	return msg;
}
Example #9
0
bool ReadAstro::open(const char *path)
{
    assert(!m_ncfile);
    m_ncfile = new NcFile(path, NcFile::ReadOnly);
    if (!m_ncfile->is_valid())
    {
        close();
        sendError("failed to open NetCDF file %s", path);
        return false;
    }

    if (m_ncfile->get_format() == NcFile::BadFormat)
    {
        close();
        sendError("bad NetCDF file");
        return false;
    }

    fprintf(stderr, "dims=%d, vars=%d, attrs=%d\n",
            m_ncfile->num_dims(), m_ncfile->num_vars(), m_ncfile->num_atts());

    for (int i = 0; i < m_ncfile->num_dims(); ++i)
    {
        fprintf(stderr, "%s: %ld\n",
                m_ncfile->get_dim(i)->name(),
                m_ncfile->get_dim(i)->size());
    }

    for (int i = 0; i < m_ncfile->num_vars(); ++i)
    {
        fprintf(stderr, "%s: dims=%d atts=%d vals=%ld type=%d\n",
                m_ncfile->get_var(i)->name(),
                m_ncfile->get_var(i)->num_dims(),
                m_ncfile->get_var(i)->num_atts(),
                m_ncfile->get_var(i)->num_vals(),
                m_ncfile->get_var(i)->type());
        //int dims = m_ncfile->get_var(i)->num_dims();
        NcVar *var = m_ncfile->get_var(i);
        for (int j = 0; j < var->num_dims(); ++j)
        {
            fprintf(stderr, "   %s: %ld edge=%ld\n",
                    var->get_dim(j)->name(),
                    var->get_dim(j)->size(),
                    var->edges()[j]);
        }
    }

    for (int i = 0; i < m_ncfile->num_atts(); ++i)
    {
        fprintf(stderr, "%s\n", m_ncfile->get_att(i)->name());
    }

    return true;
}
int main(int argc, char *argv[])
{
    NcFile at(atpath, NcFile::ReadOnly);
    if(!at.is_valid() || at.num_dims() != 3 || at.num_vars() != 4) {
        fprintf(stderr, "failed reading file: %s\n", atpath);
        return 1;
    }

    NcVar* data = at.get_var("rhum");
    if(!data->is_valid() || data->num_dims() != 3) {
        fprintf(stderr, "rhum has incorrect dimensions");
        return 1;
    }

    NcDim* time = data->get_dim(0);
    int timecnt = time->size();
    float  *rhumd = new float[timecnt*LATS*LONS];
    data->get(rhumd, timecnt, LATS, LONS);

    float  rhumdmon[12][LATS][LONS];
    for(int i = 0; i<LATS; i++)
        for(int j = 0; j<LONS; j++) {
            float rhumdmoncnt[12];
            for(int k = 0; k<12; k++) {
                rhumdmon[k][i][j] = 0;
                rhumdmoncnt[k] = 0;
            }
            for(int k = 0; k<timecnt; k++) {
                double v = rhumd[(k*LATS+i)*LONS+j]*.1 + 3276.5;
                if(v >= 0 && v <= 100) {
                    rhumdmon[k%12][i][j] += v;
                    rhumdmoncnt[k%12]++;
                }
            }
            for(int k = 0; k<12; k++)
                rhumdmon[k][i][j] /= rhumdmoncnt[k];
        }
    delete [] rhumd;

    /* use a single byte instead of 2 to save memory,
       resolution of 1/5th of a mm/day resolution */
    uint8_t rhumbyte[12][LATS][LONS];
    for(int i = 0; i<12; i++)
        for(int j = 0; j<LATS; j++)
            for(int k = 0; k<LONS; k++)
                if(isnan(rhumdmon[i][j][k]) || fabs(rhumdmon[i][j][k]) > 100)
                    rhumbyte[i][j][k] = 255;
                else
                    rhumbyte[i][j][k] = rhumdmon[i][j][k]*2.0;
    
    fwrite(rhumbyte, sizeof rhumbyte, 1, stdout);
    return 0;
}
Example #11
0
const DataMatrix::DataInfo DataInterfaceNetCdfMatrix::dataInfo(const QString& matrix) const
{
  if (!netcdf._matrixList.contains( matrix ) ) {
    return DataMatrix::DataInfo();
  }

  QByteArray bytes = matrix.toLatin1();
  NcVar *var = netcdf._ncfile->get_var(bytes.constData());  // var is owned by _ncfile

  if (var->num_dims() != 2) {
    return DataMatrix::DataInfo();
  }

  DataMatrix::DataInfo info;
  info.samplesPerFrame = 1;
  // TODO is this right?
  info.xSize = var->get_dim(0)->size();
  info.ySize = var->get_dim(1)->size();

  return info;
}
Example #12
0
blitz::Array<T,rank> read_blitz(NcFile &nc, std::string const &var_name)
{
	// Read points vector
	NcVar *vpoints = nc.get_var(var_name.c_str());
	int ndims = vpoints->num_dims();
	if (ndims != rank) {
		fprintf(stderr, "NetCDF variable %s has rank %d, expected rank %d\n",
			var_name.c_str(), ndims, rank);
		throw std::exception();
	}

	blitz::TinyVector<int,rank> shape(0);
	long counts[rank];
	for (int i=0; i<rank; ++i) {
		shape[i] = vpoints->get_dim(i)->size();
printf("read_blitz: shape[%d] = %d\n", i, shape[i]);
		counts[i] = shape[i];
	}

	blitz::Array<T,rank> ret(shape);
for (int i=0; i<rank; ++i) printf("read_blitz: ret.extent(%d) = %d\n", i, ret.extent(i));
	vpoints->get(ret.data(), counts);
	return ret;
}
Example #13
0
void FileArome::writeCore(std::vector<Variable::Type> iVariables) {
   writeTimes();
   writeReferenceTime();
   writeGlobalAttributes();
   for(int v = 0; v < iVariables.size(); v++) {
      Variable::Type varType = iVariables[v];
      std::string variable = getVariableName(varType);
      NcVar* var;
      if(hasVariableCore(varType)) {
         var = getVar(variable);
      }
      else {
         // Create variable
         if(0) {
            NcDim* dTime    = getDim("time");
            NcDim* dSurface = getDim("height0");
            NcDim* dLon     = getDim("x");
            NcDim* dLat     = getDim("y");
            var = mFile.add_var(variable.c_str(), ncFloat, dTime, dSurface, dLat, dLon);
         }
         else {
            NcDim* dTime    = getDim("time");
            NcDim* dLon     = getDim("x");
            NcDim* dLat     = getDim("y");
            var = mFile.add_var(variable.c_str(), ncFloat, dTime, dLat, dLon);
         }
      }
      float MV = getMissingValue(var); // The output file's missing value indicator
      for(int t = 0; t < mNTime; t++) {
         float offset = getOffset(var);
         float scale = getScale(var);
         FieldPtr field = getField(varType, t);
         if(field != NULL) { // TODO: Can't be null if coming from reference
            float* values = new float[mNLat*mNLon];

            int index = 0;
            for(int lat = 0; lat < mNLat; lat++) {
               for(int lon = 0; lon < mNLon; lon++) {
                  float value = (*field)(lat,lon,0);
                  if(!Util::isValid(value)) {
                     // Field has missing value indicator and the value is missing
                     // Save values using the file's missing indicator value
                     value = MV;
                  }
                  else {
                     value = ((*field)(lat,lon,0) - offset)/scale;
                  }
                  values[index] = value;
                  index++;
               }
            }
            int numDims = var->num_dims();
            if(numDims == 4) {
               var->set_cur(t, 0, 0, 0);
               var->put(values, 1, 1, mNLat, mNLon);
            }
            else if(numDims == 3) {
               var->set_cur(t, 0, 0);
               var->put(values, 1, mNLat, mNLon);
            }
            else {
               std::stringstream ss;
               ss << "Cannot write variable '" << variable << "' from '" << getFilename() << "'";
               Util::error(ss.str());
            }
            setAttribute(var, "coordinates", "longitude latitude");
            setAttribute(var, "units", Variable::getUnits(varType));
            setAttribute(var, "standard_name", Variable::getStandardName(varType));
            delete[] values;
         }
      }
      setMissingValue(var, MV);
   }
}
void ReadCFTimeDataFromNcFile(
	NcFile * ncfile,
	const std::string & strFilename,
	std::vector<Time> & vecTimes,
	bool fWarnOnMissingCalendar
) {
	// Empty existing Time vector
	vecTimes.clear();

	// Get time dimension
	NcDim * dimTime = ncfile->get_dim("time");
	if (dimTime == NULL) {
		_EXCEPTION1("Dimension \"time\" not found in file \"%s\"",
			strFilename.c_str());
	}

	// Get time variable
	NcVar * varTime = ncfile->get_var("time");
	if (varTime == NULL) {
		_EXCEPTION1("Variable \"time\" not found in file \"%s\"",
			strFilename.c_str());
	}
	if (varTime->num_dims() != 1) {
		_EXCEPTION1("Variable \"time\" has more than one dimension in file \"%s\"",
			strFilename.c_str());
	}
	if (strcmp(varTime->get_dim(0)->name(), "time") != 0) {
		_EXCEPTION1("Variable \"time\" does not have dimension \"time\" in file \"%s\"",
			strFilename.c_str());
	}

	// Calendar attribute
	NcAtt * attTimeCal = varTime->get_att("calendar");
	std::string strCalendar;
	if (attTimeCal == NULL) {
		if (fWarnOnMissingCalendar) {
			Announce("WARNING: Variable \"time\" is missing \"calendar\" attribute; assuming \"standard\"");
		}
		strCalendar = "standard";
	} else {
		strCalendar = attTimeCal->as_string(0);
	}
	Time::CalendarType eCalendarType =
		Time::CalendarTypeFromString(strCalendar);

	// Units attribute
	NcAtt * attTimeUnits = varTime->get_att("units");
	if (attTimeUnits == NULL) {
		_EXCEPTION1("Variable \"time\" is missing \"units\" attribute in file \"%s\"",
			strFilename.c_str());
	}
	std::string strTimeUnits = attTimeUnits->as_string(0);

	// Load in time data
	DataVector<int> vecTimeInt;
	DataVector<float> vecTimeFloat;
	DataVector<double> vecTimeDouble;
	DataVector<ncint64> vecTimeInt64;

	if (varTime->type() == ncInt) {
		vecTimeInt.Initialize(dimTime->size());
		varTime->set_cur((long)0);
		varTime->get(&(vecTimeInt[0]), dimTime->size());

	} else if (varTime->type() == ncFloat) {
		vecTimeFloat.Initialize(dimTime->size());
		varTime->set_cur((long)0);
		varTime->get(&(vecTimeFloat[0]), dimTime->size());

	} else if (varTime->type() == ncDouble) {
		vecTimeDouble.Initialize(dimTime->size());
		varTime->set_cur((long)0);
		varTime->get(&(vecTimeDouble[0]), dimTime->size());

	} else if (varTime->type() == ncInt64) {
		vecTimeInt64.Initialize(dimTime->size());
		varTime->set_cur((long)0);
		varTime->get(&(vecTimeInt64[0]), dimTime->size());

	} else {
		_EXCEPTION1("Variable \"time\" has invalid type "
			"(expected \"int\", \"int64\", \"float\" or \"double\")"
			" in file \"%s\"", strFilename.c_str());
	}

	for (int t = 0; t < dimTime->size(); t++) {
		Time time(eCalendarType);
		if (varTime->type() == ncInt) {
			time.FromCFCompliantUnitsOffsetInt(
				strTimeUnits,
				vecTimeInt[t]);

		} else if (varTime->type() == ncFloat) {
			time.FromCFCompliantUnitsOffsetDouble(
				strTimeUnits,
				static_cast<double>(vecTimeFloat[t]));

		} else if (varTime->type() == ncDouble) {
			time.FromCFCompliantUnitsOffsetDouble(
				strTimeUnits,
				vecTimeDouble[t]);

		} else if (varTime->type() == ncInt64) {
			time.FromCFCompliantUnitsOffsetInt(
				strTimeUnits,
				(int)(vecTimeInt64[t]));

		}

		vecTimes.push_back(time);
	}
}
void CopyNcVar(
	NcFile & ncIn,
	NcFile & ncOut,
	const std::string & strVarName,
	bool fCopyAttributes,
	bool fCopyData
) {
	if (!ncIn.is_valid()) {
		_EXCEPTIONT("Invalid input file specified");
	}
	if (!ncOut.is_valid()) {
		_EXCEPTIONT("Invalid output file specified");
	}
	NcVar * var = ncIn.get_var(strVarName.c_str());
	if (var == NULL) {
		_EXCEPTION1("NetCDF file does not contain variable \"%s\"",
			strVarName.c_str());
	}

	NcVar * varOut;

	std::vector<NcDim *> dimOut;
	dimOut.resize(var->num_dims());

	std::vector<long> counts;
	counts.resize(var->num_dims());

	long nDataSize = 1;

	for (int d = 0; d < var->num_dims(); d++) {
		NcDim * dimA = var->get_dim(d);

		dimOut[d] = ncOut.get_dim(dimA->name());

		if (dimOut[d] == NULL) {
			if (dimA->is_unlimited()) {
				dimOut[d] = ncOut.add_dim(dimA->name());
			} else {
				dimOut[d] = ncOut.add_dim(dimA->name(), dimA->size());
			}

			if (dimOut[d] == NULL) {
				_EXCEPTION2("Failed to add dimension \"%s\" (%i) to file",
					dimA->name(), dimA->size());
			}
		}
		if (dimOut[d]->size() != dimA->size()) {
			if (dimA->is_unlimited() && !dimOut[d]->is_unlimited()) {
				_EXCEPTION2("Mismatch between input file dimension \"%s\" and "
					"output file dimension (UNLIMITED / %i)",
					dimA->name(), dimOut[d]->size());
			} else if (!dimA->is_unlimited() && dimOut[d]->is_unlimited()) {
				_EXCEPTION2("Mismatch between input file dimension \"%s\" and "
					"output file dimension (%i / UNLIMITED)",
					dimA->name(), dimA->size());
			} else if (!dimA->is_unlimited() && !dimOut[d]->is_unlimited()) {
				_EXCEPTION3("Mismatch between input file dimension \"%s\" and "
					"output file dimension (%i / %i)",
					dimA->name(), dimA->size(), dimOut[d]->size());
			}
		}

		counts[d] = dimA->size();
		nDataSize *= counts[d];
	}

	// ncByte / ncChar type
	if ((var->type() == ncByte) || (var->type() == ncChar)) {
		DataVector<char> data;
		data.Initialize(nDataSize);

		varOut =
			ncOut.add_var(
				var->name(), var->type(),
				dimOut.size(), (const NcDim**)&(dimOut[0]));

		if (varOut == NULL) {
			_EXCEPTION1("Cannot create variable \"%s\"", var->name());
		}

		var->get(&(data[0]), &(counts[0]));
		varOut->put(&(data[0]), &(counts[0]));
	}

	// ncShort type
	if (var->type() == ncShort) {
		DataVector<short> data;
		data.Initialize(nDataSize);

		varOut =
			ncOut.add_var(
				var->name(), var->type(),
				dimOut.size(), (const NcDim**)&(dimOut[0]));

		if (varOut == NULL) {
			_EXCEPTION1("Cannot create variable \"%s\"", var->name());
		}

		if (fCopyData) {
			var->get(&(data[0]), &(counts[0]));
			varOut->put(&(data[0]), &(counts[0]));
		}
	}

	// ncInt type
	if (var->type() == ncInt) {
		DataVector<int> data;
		data.Initialize(nDataSize);

		varOut =
			ncOut.add_var(
				var->name(), var->type(),
				dimOut.size(), (const NcDim**)&(dimOut[0]));

		if (varOut == NULL) {
			_EXCEPTION1("Cannot create variable \"%s\"", var->name());
		}

		if (fCopyData) {
			var->get(&(data[0]), &(counts[0]));
			varOut->put(&(data[0]), &(counts[0]));
		}
	}

	// ncFloat type
	if (var->type() == ncFloat) {
		DataVector<float> data;
		data.Initialize(nDataSize);

		varOut =
			ncOut.add_var(
				var->name(), var->type(),
				dimOut.size(), (const NcDim**)&(dimOut[0]));

		if (varOut == NULL) {
			_EXCEPTION1("Cannot create variable \"%s\"", var->name());
		}

		if (fCopyData) {
			var->get(&(data[0]), &(counts[0]));
			varOut->put(&(data[0]), &(counts[0]));
		}
	}

	// ncDouble type
	if (var->type() == ncDouble) {
		DataVector<double> data;
		data.Initialize(nDataSize);

		varOut =
			ncOut.add_var(
				var->name(), var->type(),
				dimOut.size(), (const NcDim**)&(dimOut[0]));

		if (varOut == NULL) {
			_EXCEPTION1("Cannot create variable \"%s\"", var->name());
		}

		if (fCopyData) {
			var->get(&(data[0]), &(counts[0]));
			varOut->put(&(data[0]), &(counts[0]));
		}
	}

	// ncInt64 type
	if (var->type() == ncInt64) {
		DataVector<ncint64> data;
		data.Initialize(nDataSize);

		varOut =
			ncOut.add_var(
				var->name(), var->type(),
				dimOut.size(), (const NcDim**)&(dimOut[0]));

		if (varOut == NULL) {
			_EXCEPTION1("Cannot create variable \"%s\"", var->name());
		}

		if (fCopyData) {
			var->get(&(data[0]), &(counts[0]));
			varOut->put(&(data[0]), &(counts[0]));
		}
	}

	// Check output variable exists
	if (varOut == NULL) {
		_EXCEPTION1("Unable to create output variable \"%s\"",
			var->name());
	}

	// Copy attributes
	if (fCopyAttributes) {
		CopyNcVarAttributes(var, varOut);
	}
}
Example #16
0
int main(int argc, char** argv)
{
    if (!cmdline(argc, argv))
    {
        printhelp();
        return EXIT_FAILURE;
    }

    NcFile infile(infilename.c_str(), NcFile::ReadOnly);
    if (!infile.is_valid())
    {
        std::cerr << "Error: invalid input file -- '" << infilename << "'" << std::endl;
        infile.close();
        return EXIT_FAILURE;
    }

    NcFile outfile(outfilename.c_str(), NcFile::Replace);
    if (!outfile.is_valid())
    {
        std::cerr << "Error: cannot open output file -- '" << outfilename << "'" << std::endl;
        outfile.close();
        return EXIT_FAILURE;
    }

    if (varstrings.size() == 0)
    {
        std::cerr << "Warning: no variables specified" << std::endl;
    }

    std::vector<NcVar*> invars;
    for (std::vector<std::string>::const_iterator it = varstrings.begin();
         it != varstrings.end(); ++it)
    {
        NcVar* var = infile.get_var((*it).c_str());
        if (var == NULL)
        {
            std::cerr << "Error: " << *it << ": no such variable" << std::endl;
            infile.close();
            outfile.close();
            return EXIT_FAILURE;
        }
        invars.push_back(var);
    }

    // extract the distinct set of dims
    std::map<std::string, NcDim*> indims;
    for (std::vector<NcVar*>::const_iterator it = invars.begin();
         it != invars.end(); ++it)
    {
        NcVar* var = *it;
        for (int i = 0; i < var->num_dims(); ++i)
        {
            NcDim* dim = var->get_dim(i);
            indims[dim->name()] = dim;
        }
    }

    // add dims to outfile
    std::map<std::string, NcDim*> outdims;
    for (std::map<std::string, NcDim*>::const_iterator it = indims.begin();
         it != indims.end(); ++it)
    {
        NcDim* dim = (*it).second;
        NcDim* outdim = NULL;
        if (dim->is_unlimited())
        {
            outdim = outfile.add_dim(dim->name());
        }
        else
        {
            outdim = outfile.add_dim(dim->name(), dim->size());
        }

        if (outdim != NULL)
        {
            outdims[outdim->name()] = outdim;
        }
    }

    // create variables
    for (std::vector<NcVar*>::const_iterator it = invars.begin();
         it != invars.end(); ++it)
    {
        NcVar* var = *it;
        std::vector<const NcDim*> dims(var->num_dims());
        for (int i = 0; i < var->num_dims(); ++i)
        {
            dims[i] = outdims[var->get_dim(i)->name()];
        }
        NcVar* outvar = outfile.add_var(var->name(), var->type(), var->num_dims(), &dims[0]);

        // identify largest dim, if dim (nearly) exceeds main memory, split along that dim
        int maxdim = -1;
        long maxdimsize = 0;
        long totallen = 1;
        for (int i = 0; i < var->num_dims(); ++i)
        {
            NcDim* dim = var->get_dim(i);
            if (dim->size() > maxdimsize)
            {
                maxdim = i;
                maxdimsize = dim->size();
            }
            totallen *= dim->size();
        }

        // TODO: support other data types
        totallen *= sizeof(float);

        // TODO: configurable page size
        const unsigned long pagesize = 1000000000;
#ifdef __linux__
        struct sysinfo info;
        sysinfo(&info);
        if (pagesize >= info.freeram)
        {
            std::cerr << "Warning: page size exceeds free memory" << std::endl;
        }
#endif

        int numpages = 1;
        long pagesizedim = var->get_dim(maxdim)->size();
        if (totallen < pagesize)
        {
        }
        else
        {
            long mul = 1;
            for (int i = 0; i < var->num_dims(); ++i)
            {
                if (i != maxdim)
                {
                    NcDim* dim = var->get_dim(i);
                    mul *= dim->size();
                }
            }
            // TODO: support other data types
            mul *= sizeof(float);

            pagesizedim = pagesize / mul;
            numpages = var->get_dim(maxdim)->size() / pagesizedim;
            if (var->get_dim(maxdim)->size() % pagesizedim > 0)
            {
                ++numpages;
            }
        }

        std::vector< std::vector<long> > curvec;
        std::vector< std::vector<long> > countsvec;
        std::vector<long> lengths;

        int pages = numpages > 0 ? numpages : 1;
        for (int p = 0; p < pages; ++p)
        {
            long len = 1;
            std::vector<long> cur;
            std::vector<long> counts;
            for (int i = 0; i < var->num_dims(); ++i)
            {
                NcDim* dim = var->get_dim(i);
                long current = 0;
                long count = dim->size();
                if (i == maxdim)
                {
                    current = pagesizedim * p;
                    count = pagesizedim;
                    if (p == pages -1)
                    {
                        if (dim->size() % pagesizedim != 0)
                        {
                            count = dim->size() % pagesizedim;
                        }
                    }
                }
                cur.push_back(current);
                counts.push_back(count);
                len *= count;
            }
            curvec.push_back(cur);
            countsvec.push_back(counts);
            lengths.push_back(len);
        }

        std::vector< std::vector<long> >::const_iterator it1;
        std::vector< std::vector<long> >::const_iterator it2;
        std::vector<long>::const_iterator it3;

        for (it1 = curvec.begin(), it2 = countsvec.begin(), it3 = lengths.begin();
             it1 != curvec.end() && it2 != countsvec.end() && it3 != lengths.end(); ++it1, ++it2, ++it3)
        {
            std::vector<long> cur = *it1;
            std::vector<long> counts = *it2;
            long len = *it3;

            var->set_cur(&cur[0]);
            outvar->set_cur(&cur[0]);
            switch (outvar->type())
            {
            case ncByte:
            {
                ncbyte* barr = new ncbyte[len];
                var->get(barr, &counts[0]);
                outvar->put(barr, &counts[0]);
                delete[] barr;
                break;
            }
            case ncChar:
            {
                char* carr = new char[len];
                var->get(carr, &counts[0]);
                outvar->put(carr, &counts[0]);
                delete[] carr;
                break;
            }
            case ncShort:
            {
                short* sarr = new short[len];
                var->get(sarr, &counts[0]);
                outvar->put(sarr, &counts[0]);
                delete[] sarr;
                break;
            }
            case ncInt:
            {
                long* larr = new long[len];
                var->get(larr, &counts[0]);
                outvar->put(larr, &counts[0]);
                delete[] larr;
                break;
            }
            case ncFloat:
            {
                float* farr = new float[len];
                var->get(farr, &counts[0]);
                outvar->put(farr, &counts[0]);
                delete[] farr;
                break;
            }
            case ncDouble:
            {
                double* darr = new double[len];
                var->get(darr, &counts[0]);
                outvar->put(darr, &counts[0]);
                delete[] darr;
                break;
            }
            default:
                break;
            }
        }
    }

    infile.close();
    outfile.close();
    return 0;
}
bool EpidemicDataSet::loadNetCdfFile(const char * filename)
{
#if USE_NETCDF // TODO: should handle this differently
    // change netcdf library error behavior
    NcError err(NcError::verbose_nonfatal);

    // open the netcdf file
    NcFile ncFile(filename, NcFile::ReadOnly);

    if(!ncFile.is_valid())
    {
        put_flog(LOG_FATAL, "invalid file %s", filename);
        return false;
    }

    // get dimensions
    NcDim * timeDim = ncFile.get_dim("time");
    NcDim * nodesDim = ncFile.get_dim("nodes");
    NcDim * stratificationsDim = ncFile.get_dim("stratifications");

    if(timeDim == NULL || nodesDim == NULL || stratificationsDim == NULL)
    {
        put_flog(LOG_FATAL, "could not find a required dimension");
        return false;
    }

    numTimes_ = timeDim->size();

    // make sure we have the expected number of nodes
    if(nodesDim->size() != numNodes_)
    {
        put_flog(LOG_FATAL, "got %i nodes, expected %i", nodesDim->size(), numNodes_);
        return false;
    }

    put_flog(LOG_DEBUG, "file contains %i timesteps, %i nodes", numTimes_, numNodes_);

    // make sure number of stratifications matches our expectation...
    int numExpectedStratifications = 1;

    for(unsigned int i=0; i<NUM_STRATIFICATION_DIMENSIONS; i++)
    {
        numExpectedStratifications *= stratifications_[i].size();
    }

    if(stratificationsDim->size() != numExpectedStratifications)
    {
        put_flog(LOG_FATAL, "got %i stratifications, expected %i", stratificationsDim->size(), numExpectedStratifications);
        return false;
    }

    // get all float variables with dimensions (time, nodes, stratifications)
    for(int i=0; i<ncFile.num_vars(); i++)
    {
        NcVar * ncVar = ncFile.get_var(i);

        if(ncVar->num_dims() == 3 && ncVar->type() == ncFloat && strcmp(ncVar->get_dim(0)->name(), "time") == 0 && strcmp(ncVar->get_dim(1)->name(), "nodes") == 0 && strcmp(ncVar->get_dim(2)->name(), "stratifications") == 0)
        {
            put_flog(LOG_INFO, "found variable: %s", ncVar->name());

            // full shape
            blitz::TinyVector<int, 2+NUM_STRATIFICATION_DIMENSIONS> shape;
            shape(0) = numTimes_;
            shape(1) = numNodes_;

            for(int j=0; j<NUM_STRATIFICATION_DIMENSIONS; j++)
            {
                shape(2 + j) = stratifications_[j].size();
            }

            blitz::Array<float, 2+NUM_STRATIFICATION_DIMENSIONS> var((float *)ncVar->values()->base(), shape, blitz::duplicateData);

            variables_[std::string(ncVar->name())].reference(var);
        }
    }
#endif
    return true;
}
Example #18
0
eavlNetCDFImporter::eavlNetCDFImporter(const string &filename)
{
    file = new NcFile(filename.c_str(), NcFile::ReadOnly);
     
    if (!file->is_valid())
    {
        THROW(eavlException,"Couldn't open file!\n");
    }

    if (debugoutput) cerr << "num_dims="<<file->num_dims()<<endl;
    if (debugoutput) cerr << "num_vars="<<file->num_vars()<<endl;
    if (debugoutput) cerr << "num_atts="<<file->num_atts()<<endl;

    for (int i=0; i<file->num_dims(); i++)
    {
        NcDim *d = file->get_dim(i);
        if (debugoutput) cerr << "  dim["<<i<<"]: name="<<d->name()<<" size="<<d->size()<<endl;
    }

    for (int i=0; i<file->num_atts(); i++)
    {
        NcAtt *a = file->get_att(i);
        if (debugoutput) cerr << "  att["<<i<<"]: name="<<a->name()<<" numvals="<<a->num_vals()<<endl;
    }

    bool found_grid = false;

    for (int i=0; i<file->num_vars(); i++)
    {
        NcVar *v = file->get_var(i);
        if (debugoutput) 
        {
            cerr << "  var["<<i<<"]: name="<<v->name();
            cerr << "  ndims="<<v->num_dims();
            cerr << "  dims = ";
            for (int j=0; j<v->num_dims(); j++)
            {
                cerr << v->get_dim(j)->name();
                if (j<v->num_dims()-1)
                    cerr << "*";
            }
            cerr << endl;
        }

        // Here's the condition for what we're going to use;
        // we only support one mesh for the moment, so we're picking one.
        // Also, the netcdf files we have have the time dim size as "1"
        if (v->num_dims() == 4 && string(v->get_dim(0)->name())=="time")
        {
            if (!found_grid)
            {
                dims.push_back(v->get_dim(1));
                dims.push_back(v->get_dim(2));
                dims.push_back(v->get_dim(3));
                found_grid = true;
                vars.push_back(v);
                if (debugoutput) cerr << "     * using as first real var\n";
            }
            else
            {
                if (string(v->get_dim(1)->name()) == dims[0]->name() &&
                    string(v->get_dim(2)->name()) == dims[1]->name() &&
                    string(v->get_dim(3)->name()) == dims[2]->name())
                {
                    vars.push_back(v);
                    if (debugoutput) cerr << "     * using as another var; matches the first real one's dims\n";
                }
            }
        }

    }
}