void bi::SMCNetCDFBuffer::map() {
std::vector<int> dimids;
lwVar = nc_inq_varid(ncid, "logweight");
BI_ERROR_MSG(lwVar >= 0, "No variable logweight in file " << file);
dimids = nc_inq_vardimid(ncid, lwVar);
BI_ERROR_MSG(dimids.size() == 1,
"Variable logweight has " << dimids.size() << " dimensions, should have 1, in file " << file);
BI_ERROR_MSG(dimids[0] == npDim,
"Only dimension of variable logweight should be np, in file " << file);
}
void bi::ParticleMCMCNetCDFBuffer::create(const long P, const long T) {
int id, i;
VarType type;
Var* var;
Dim* dim;
ncFile->add_att(PACKAGE_TARNAME "_schema", "ParticleMCMC");
ncFile->add_att(PACKAGE_TARNAME "_schema_version", 1);
ncFile->add_att(PACKAGE_TARNAME "_version", PACKAGE_VERSION);
/* dimensions */
nrDim = createDim("nr", T);
for (i = 0; i < m.getNumDims(); ++i) {
dim = m.getDim(i);
nDims.push_back(createDim(dim->getName().c_str(), dim->getSize()));
}
npDim = createDim("np", P);
/* time variable */
tVar = ncFile->add_var("time", netcdf_real, nrDim);
BI_ERROR_MSG(tVar != NULL && tVar->is_valid(), "Could not create time variable");
/* other variables */
for (i = 0; i < NUM_VAR_TYPES; ++i) {
type = static_cast<VarType>(i);
vars[type].resize(m.getNumVars(type), NULL);
if (type == D_VAR || type == R_VAR || type == P_VAR) {
for (id = 0; id < (int)vars[type].size(); ++id) {
var = m.getVar(type, id);
if (var->hasOutput()) {
if (type == P_VAR) {
vars[type][id] = createVar(var, false, true);
} else {
vars[type][id] = createVar(var, !var->getOutputOnce(), true);
}
}
}
}
}
llVar = ncFile->add_var("loglikelihood", netcdf_real, npDim);
BI_ERROR_MSG(llVar != NULL && llVar->is_valid(),
"Could not create loglikelihood variable");
lpVar = ncFile->add_var("logprior", netcdf_real, npDim);
BI_ERROR_MSG(lpVar != NULL && lpVar->is_valid(),
"Could not create logprior variable");
}
void bi::SMC2NetCDFBuffer::create(const long P, const long T) {
ncFile->add_att(PACKAGE_TARNAME "_schema", "SMC2");
ncFile->add_att(PACKAGE_TARNAME "_schema_version", 1);
ncFile->add_att(PACKAGE_TARNAME "_version", PACKAGE_VERSION);
lwVar = ncFile->add_var("logweight", netcdf_real, npDim);
BI_ERROR_MSG(lwVar != NULL && lwVar->is_valid(),
"Could not create logweight variable");
}
int bi::InputNetCDFBuffer::mapCoordDim(int ncVar) {
int ncDim, j = 0;
/* check dimensions */
std::vector<int> dimids = nc_inq_vardimid(ncid, ncVar);
ncDim = dimids[j];
if (ncDim >= 0 && ncDim == nsDim) {
ncDim = dimids[j++];
}
BI_ERROR_MSG(ncDim >= 0 && dimids.size() <= 3,
"Coordinate variable " << nc_inq_varname(ncid, ncVar) << " has invalid dimensions, must have optional ns dimension followed by one or two arbitrary dimensions");
return ncDim;
}
void bi::MCMCNetCDFBuffer::map() {
std::vector<int> dimids;
llVar = nc_inq_varid(ncid, "loglikelihood");
BI_ERROR_MSG(llVar >= 0, "No variable loglikelihood in file " << file);
dimids = nc_inq_vardimid(ncid, llVar);
BI_ERROR_MSG(dimids.size() == 1u,
"Variable loglikelihood has " << dimids.size() << " dimensions, should have 1, in file " << file);
BI_ERROR_MSG(dimids[0] == npDim,
"Only dimension of variable loglikelihood should be np, in file " << file);
lpVar = nc_inq_varid(ncid, "logprior");
BI_ERROR_MSG(lpVar >= 0, "No variable logprior in file " << file);
dimids = nc_inq_vardimid(ncid, lpVar);
BI_ERROR_MSG(dimids.size() == 1u,
"Variable logprior has " << dimids.size() << " dimensions, should have 1, in file " << file);
BI_ERROR_MSG(dimids[0] == npDim,
"Only dimension of variable logprior should be np, in file " << file);
}
void bi::OptimiserNetCDFBuffer::map() {
std::string name;
std::vector<int> dimids;
/* function value variable */
valueVar = nc_inq_varid(ncid, "optimiser.value");
BI_ERROR_MSG(valueVar >= 0, "No variable optimiser.value in file " << file);
dimids = nc_inq_vardimid(ncid, valueVar);
BI_ERROR_MSG(dimids.size() == 1,
"Variable optimiser.value has " << dimids.size() << " dimensions, should have 1, in file " << file);
BI_ERROR_MSG(dimids[0] == npDim,
"Only dimension of variable optimiser.value should be np, in file " << file);
/* size variable */
sizeVar = nc_inq_varid(ncid, "optimiser.size");
BI_ERROR_MSG(sizeVar >= 0, "No variable optimiser.size in file " << file);
dimids = nc_inq_vardimid(ncid, sizeVar);
BI_ERROR_MSG(dimids.size() == 1,
"Variable optimiser.size has " << dimids.size() << " dimensions, should have 1, in file " << file);
BI_ERROR_MSG(dimids[0] == npDim,
"Only dimension of variable optimiser.size should be np, in file " << file);
}
void bi::ParticleFilterNetCDFBuffer::map() {
std::vector<int> dimids;
aVar = nc_inq_varid(ncid, "ancestor");
BI_ERROR_MSG(aVar >= 0, "No variable ancestor in file " << file);
dimids = nc_inq_vardimid(ncid, aVar);
if (schema == FLEXI) {
BI_ERROR_MSG(dimids.size() == 1u,
"Variable ancestor has " << dimids.size() << " dimensions, should have 1, in file " << file);
BI_ERROR_MSG(dimids[0] == nrpDim,
"Only dimension of variable ancestor should be nrp, in file " << file);
} else {
BI_ERROR_MSG(dimids.size() == 2u,
"Variable ancestor has " << dimids.size() << " dimensions, should have 2, in file " << file);
BI_ERROR_MSG(dimids[0] == nrDim,
"First dimension of variable ancestor should be nr, in file " << file);
BI_ERROR_MSG(dims[1] == npDim,
"Second dimension of variable ancestor should be np, in file " << file);
}
lwVar = nc_inq_varid(ncid, "logweight");
BI_ERROR_MSG(lwVar >= 0, "No variable logweight in file " << file);
dimids = nc_inq_vardimid(ncid, lwVar);
if (schema == FLEXI) {
BI_ERROR_MSG(dimids.size() == 1u,
"Variable logweight has " << dimids.size() << " dimensions, should have 1, in file " << file);
BI_ERROR_MSG(dimids[0] == nrpDim,
"Only dimension of variable logweight should be nrp, in file " << file);
} else {
BI_ERROR_MSG(dimids.size() == 2u,
"Variable logweight has " << dimids.size() << " dimensions, should have 2, in file " << file);
BI_ERROR_MSG(dimids[0] == nrDim,
"First dimension of variable logweight should be nr, in file " << file);
BI_ERROR_MSG(dimids[1] == npDim,
"Second dimension of variable logweight should be np, in file " << file);
}
llVar = nc_inq_varid(ncid, "loglikelihood");
BI_ERROR_MSG(llVar >= 0, "No variable loglikelihood in file " << file);
dimids = nc_inq_vardimid(ncid, llVar);
BI_ERROR_MSG(dimids.size() == 0u,
"Variable loglikelihood has " << dimids.size() << " dimensions, should have 0, in file " << file);
}
std::pair<int,int> bi::InputNetCDFBuffer::mapVarDim(const Var* var) {
/* pre-condition */
BI_ASSERT(var != NULL);
const VarType type = var->getType();
Dim* dim;
int ncVar, ncDim, i, j = 0, k = -1;
std::vector<int> dimids;
BI_UNUSED bool canHaveTime, canHaveP;
canHaveTime = type == D_VAR || type == R_VAR || type == F_VAR
|| type == O_VAR;
ncVar = nc_inq_varid(ncid, var->getInputName());
if (ncVar >= 0) {
dimids = nc_inq_vardimid(ncid, ncVar);
/* check for ns-dimension */
if (nsDim >= 0 && j < static_cast<int>(dimids.size())
&& dimids[j] == nsDim) {
++j;
}
/* check for record dimension */
if (j < static_cast<int>(dimids.size())) {
BOOST_AUTO(iter, std::find(recDims.begin(), recDims.end(), dimids[j]));
if (iter != recDims.end()) {
k = std::distance(recDims.begin(), iter);
++j;
}
}
/* check for np-dimension */
if (npDim >= 0 && j < static_cast<int>(dimids.size())
&& dimids[j] == npDim) {
++j;
} else if (j < static_cast<int>(dimids.size())) {
/* check for model dimensions */
for (i = var->getNumDims() - 1;
i >= 0 && j < static_cast<int>(dimids.size()); --i, ++j) {
dim = var->getDim(i);
ncDim = dimids[j];
BI_ERROR_MSG(dim->getName().compare(nc_inq_dimname(ncid, ncDim)) == 0,
"Dimension " << j << " of variable " << var->getName() << " should be " << dim->getName() << " not " << nc_inq_dimname(ncid, ncDim) << ", in file " << file);
BI_ERROR_MSG(k < 0 || coordVars[k] < 0,
"Variable " << nc_inq_varname(ncid, ncVar) << " has both dense and sparse definitions, in file " << file);
}
BI_ERROR_MSG(i == -1,
"Variable " << nc_inq_varname(ncid, ncVar) << " is missing one or more dimensions, in file " << file);
/* check again for np dimension */
if (npDim >= 0 && j < static_cast<int>(dimids.size())
&& dimids[j] == npDim) {
++j;
}
}
BI_ERROR_MSG(j == static_cast<int>(dimids.size()),
"Variable " << nc_inq_varname(ncid, ncVar) << " has extra dimensions, in file " << file);
}
return std::make_pair(k, ncVar);
}
void bi::InputNetCDFBuffer::map() {
int ncDim, ncVar;
Var* var;
std::string name;
VarType type;
int i, k, id;
/* ns dimension */
nsDim = nc_inq_dimid(ncid, "ns");
if (nsDim >= 0) {
BI_ERROR_MSG(ns < (int )nc_inq_dimlen(ncid, nsDim),
"Given index " << ns << " outside range of ns dimension");
}
/* np dimension */
npDim = nc_inq_dimid(ncid, "np");
if (npDim >= 0) {
BI_ERROR_MSG(np < 0 || np < (int )nc_inq_dimlen(ncid, npDim),
"Given index " << np << " outside range of np dimension");
}
/* record dimensions, time and coordinate variables */
int nvars = nc_inq_nvars(ncid);
for (i = 0; i < nvars; ++i) {
ncVar = i;
name = nc_inq_varname(ncid, i);
if (name.find("time") == 0) {
/* is a time variable */
ncDim = mapTimeDim(ncVar);
if (ncDim >= 0) {
BOOST_AUTO(iter, std::find(recDims.begin(), recDims.end(), ncDim));
if (iter == recDims.end()) {
/* newly encountered record dimension */
recDims.push_back(ncDim);
timeVars.push_back(ncVar);
coordVars.push_back(-1);
} else {
/* record dimension encountered before */
k = std::distance(recDims.begin(), iter);
BI_ASSERT_MSG(timeVars[k] < 0,
"Time variables " << nc_inq_varname(ncid, timeVars[k]) << " and " << name << " cannot share the same record dimension " << nc_inq_dimname(ncid, *iter));
timeVars[k] = ncVar;
}
}
} else if (name.find("coord") == 0) {
/* is a coordinate variable */
ncDim = mapCoordDim(ncVar);
if (ncDim >= 0) {
BOOST_AUTO(iter, std::find(recDims.begin(), recDims.end(), ncDim));
if (iter == recDims.end()) {
/* newly encountered record dimension */
recDims.push_back(ncDim);
timeVars.push_back(-1);
coordVars.push_back(ncVar);
} else {
/* record dimension encountered before */
k = std::distance(recDims.begin(), iter);
BI_ASSERT_MSG(coordVars[k] < 0,
"Coordinate variables " << nc_inq_varname(ncid, coordVars[k]) << " and " << name << " cannot share the same record dimension " << nc_inq_dimname(ncid, *iter));
coordVars[k] = ncVar;
}
}
}
}
/* model variables */
for (i = 0; i < NUM_VAR_TYPES; ++i) {
type = static_cast<VarType>(i);
/* initialise NetCDF variables for this type */
vars[type].resize(m.getNumVars(type), -1);
/* map model variables */
for (id = 0; id < m.getNumVars(type); ++id) {
var = m.getVar(type, id);
if (var->hasInput()) {
BOOST_AUTO(pair, mapVarDim(var));
k = pair.first;
ncVar = pair.second;
if (ncVar >= 0) {
vars[type][id] = ncVar;
}
modelVars.insert(std::make_pair(k, var));
}
}
}
/* preload random access tables */
std::multimap<real,int> seq;
std::vector<size_t> starts(recDims.size(), 0), lens(recDims.size(), 0);
real tnxt;
for (k = 0; k < int(recDims.size()); ++k) {
if (timeVars[k] >= 0 && modelVars.count(k) > 0) {
/* ^ ignores record dimensions with no associated time or model
* variables */
readTime(timeVars[k], starts[k], &lens[k], &tnxt);
seq.insert(std::make_pair(tnxt, k));
}
}
while (!seq.empty()) {
/* next in time */
tnxt = seq.begin()->first;
k = seq.begin()->second;
seq.erase(seq.begin());
ncDim = recDims[k];
ncVar = timeVars[k];
if (times.empty() || times.back() != tnxt) {
times.push_back(tnxt);
recStarts.push_back(std::vector < size_t > (recDims.size(), 0));
recLens.push_back(std::vector < size_t > (recDims.size(), 0));
}
recStarts.back()[k] = starts[k];
recLens.back()[k] = lens[k];
/* read next time and range for this time variable */
starts[k] += lens[k];
if (starts[k] < nc_inq_dimlen(ncid, ncDim)) {
/* more to come on this record dimension */
readTime(ncVar, starts[k], &lens[k], &tnxt);
seq.insert(std::make_pair(tnxt, k));
}
}
}
void bi::ParticleMCMCNetCDFBuffer::map(const long P, const long T) {
std::string name;
int id, i;
VarType type;
Var* var;
Dim* dim;
/* dimensions */
BI_ERROR_MSG(hasDim("nr"), "File must have nr dimension");
nrDim = mapDim("nr", T);
for (i = 0; i < m.getNumDims(); ++i) {
dim = m.getDim(i);
BI_ERROR_MSG(hasDim(dim->getName().c_str()), "File must have " <<
dim->getName() << " dimension");
nDims.push_back(mapDim(dim->getName().c_str(), dim->getSize()));
}
BI_ERROR_MSG(hasDim("np"), "File must have np dimension");
npDim = mapDim("np", P);
/* time variable */
tVar = ncFile->get_var("time");
BI_ERROR_MSG(tVar != NULL && tVar->is_valid(),
"File does not contain variable time");
BI_ERROR_MSG(tVar->num_dims() == 1, "Variable time has " << tVar->num_dims() <<
" dimensions, should have 1");
BI_ERROR_MSG(tVar->get_dim(0) == nrDim, "Dimension 0 of variable time should be nr");
/* other variables */
for (i = 0; i < NUM_VAR_TYPES; ++i) {
type = static_cast<VarType>(i);
if (type == D_VAR || type == R_VAR || type == P_VAR) {
vars[type].resize(m.getNumVars(type), NULL);
for (id = 0; id < m.getNumVars(type); ++id) {
var = m.getVar(type, id);
if (hasVar(var->getOutputName().c_str())) {
vars[type][id] = mapVar(m.getVar(type, id));
}
}
}
}
llVar = ncFile->get_var("loglikelihood");
BI_ERROR_MSG(llVar != NULL && llVar->is_valid(),
"File does not contain variable loglikelihood");
BI_ERROR_MSG(llVar->num_dims() == 1, "Variable loglikelihood has " <<
llVar->num_dims() << " dimensions, should have 1");
BI_ERROR_MSG(llVar->get_dim(0) == npDim,
"Dimension 0 of variable loglikelihood should be np");
lpVar = ncFile->get_var("logprior");
BI_ERROR_MSG(lpVar != NULL && lpVar->is_valid(),
"File does not contain variable logprior");
BI_ERROR_MSG(lpVar->num_dims() == 1, "Variable logprior has " <<
lpVar->num_dims() << " dimensions, should have 1");
BI_ERROR_MSG(lpVar->get_dim(0) == npDim,
"Dimension 0 of variable logprior should be np");
}
void bi::InputNullBuffer::readMask(const size_t k, const VarType type,
Mask<ON_HOST>& mask) {
BI_ERROR_MSG(false, "time index outside valid range");
}
void bi::SMC2NetCDFBuffer::map(const long P, const long T) {
lwVar = ncFile->get_var("logweight");
BI_ERROR_MSG(lwVar != NULL && lwVar->is_valid(),
"Could not create logweight variable");
}
