/**
* Validate the parameters passed in from the configuration file
*/
void Cinitial::DoValidate() {
LOG_TRACE();
min_age_ = model_->min_age();
max_age_ = model_->max_age();
if (max_age_ < min_age_)
LOG_ERROR_P(PARAM_MIN_AGE) << "(" << min_age_ << ") cannot be less than the max age(" << max_age_ << ")";
column_count_ = (max_age_ - min_age_) + 2;
/**
* Convert the string values to doubles and load them in to a table.
*/
vector<vector<string>>& data = n_table_->data();
unsigned row_number = 1;
for (auto row : data) {
string row_label = row[0];
// CHeck that it is a valid category
bool check_combined = model_->categories()->IsCombinedLabels(row_label);
LOG_FINEST() << "Checking row with label = " << row_label;
if (find(category_labels_.begin(),category_labels_.end(), row_label )== category_labels_.end())
LOG_FATAL_P(PARAM_N) << " Could not find '" << row_label << "' in the categories supplied, please make sure that categories supplied is the same as the row labels.";
if (check_combined) {
vector<string> split_category_labels;
boost::split(split_category_labels, row_label, boost::is_any_of("+"));
unsigned category_iter = 0;
for (const string& split_category_label : split_category_labels) {
if (!model_->categories()->IsValid(split_category_label)) {
LOG_FATAL_P(PARAM_N)<< ": The category " << split_category_label << " is not a valid category.";
}
++category_iter;
}
} else {
if (!model_->categories()->IsValid(row_label))
LOG_FATAL_P(PARAM_N) << ": The category " << row_label << " is not a valid category.";
}
// convert to lower case
row_label = utilities::ToLowercase(row_label);
if (row.size() != column_count_)
LOG_ERROR_P(PARAM_N) << "the " << row_number << "the row has " << row.size() << " values but " << column_count_ << " values are expected";
if (n_.find(row_label) != n_.end())
LOG_ERROR_P(PARAM_N) << "the category " << row_label << " is defined more than once. You can only define a category once";
for (unsigned i = 1; i < row.size(); ++i) {
Double temp = Double();
if (!utilities::To<Double>(row[i], temp))
LOG_ERROR_P(PARAM_N) << "value (" << row[i] << ") in row " << row_number << " is not a valid numeric";
n_[row_label].push_back(temp);
}
row_number++;
}
}
void Project::Build() {
string error = "";
if (!model_->objects().VerfiyAddressableForUse(parameter_, addressable::kProject, error)) {
LOG_FATAL_P(PARAM_PARAMETER) << "could not be verified for use in a @project block. Error was " << error;
}
addressable::Type addressable_type = model_->objects().GetAddressableType(parameter_);
switch(addressable_type) {
case addressable::kInvalid:
LOG_CODE_ERROR() << "Invalid addressable type: " << parameter_;
break;
case addressable::kSingle:
LOG_FINEST() << "applying projection for parameter " << parameter_ << " is an single type";
DoUpdateFunc_ = &Project::SetSingleValue;
addressable_ = model_->objects().GetAddressable(parameter_);
original_value_ = *addressable_;
break;
case addressable::kVector:
LOG_FINEST() << "applying projection for parameter " << parameter_ << " is a vector";
addressable_vector_ = model_->objects().GetAddressableVector(parameter_);
DoUpdateFunc_ = &Project::SetVectorValue;
break;
case addressable::kUnsignedMap:
LOG_FINEST() << "applying projection for parameter " << parameter_ << " is an unsigned map";
DoUpdateFunc_ = &Project::SetMapValue;
addressable_map_ = model_->objects().GetAddressableUMap(parameter_);
break;
default:
LOG_ERROR() << "The addressable you have provided for use in a projection: " << parameter_ << " is not a type that is supported for projection modification";
break;
}
DoBuild();
}
/**
* Build all of the runtime relationships required for
* this observation
*/
void Observation::Build() {
LOG_TRACE();
likelihood_ = model_->managers().likelihood()->GetOrCreateLikelihood(model_, label_, likelihood_type_);
if (!likelihood_) {
LOG_FATAL_P(PARAM_LIKELIHOOD) << "(" << likelihood_type_ << ") could not be found or constructed.";
return;
}
likelihood_->set_multiplier(likelihood_multiplier_);
likelihood_->set_error_value_multiplier(error_value_multiplier_);
if (std::find(allowed_likelihood_types_.begin(), allowed_likelihood_types_.end(), likelihood_->type()) == allowed_likelihood_types_.end()) {
string allowed = boost::algorithm::join(allowed_likelihood_types_, ", ");
LOG_FATAL_P(PARAM_LIKELIHOOD) << ": likelihood " << likelihood_->type() << " is not supported by the " << type_ << " observation."
<< " Allowed types are: " << allowed;
}
DoBuild();
}
/**
* Get the score for this penalty
*
* @return Penalty score
*/
Double VectorSmoothing::GetScore() {
vector<Double> values;
if (estimable_vector_ != 0)
values.assign((*estimable_vector_).begin(), (*estimable_vector_).end());
else if (estimable_map_ != 0) {
for (auto iter : (*estimable_map_))
values.push_back(iter.second);
} else
LOG_CODE_ERROR() << "(estimable_map_ != 0) && (estimable_map_ != 0)";
if(upper_ == lower_ && upper_ == 0u) {
upper_ = values.size();
lower_ = 1;
}
if(upper_ == lower_)
LOG_FATAL_P(PARAM_UPPER_BOUND) << "Lower and upper bound cannot be equal";
if (upper_ > values.size())
LOG_FATAL_P(PARAM_UPPER_BOUND) << "The last element must not be greater than size of vector";
if (lower_ < 1)
LOG_FATAL_P(PARAM_LOWER_BOUND) << "The first element must not be less than 1";
if (r_ >= (upper_ - lower_))
LOG_FATAL_P(PARAM_R) << PARAM_R << " R cannot be greater than or equal to size of vector - 1";
Double score = 0.0;
if (log_scale_) {
for (Double& value : values)
value = log(value);
}
for (unsigned i = 1; i <= r_; ++i) {
for(unsigned j = (lower_ - 1); j <= ((upper_ - 1) - i); ++j) {
values[j] = values[j + 1] - values[j];
}
values[(upper_ - 1) - i + 1] = 0;
}
for (unsigned k = (lower_ - 1); k <= (upper_ - 1); ++k)
score += values[k] * values[k];
return score * multiplier_;
}
/**
* Build our process proportions by category
*/
void ProcessProportionsByCategory::DoBuild() {
ProportionsByCategory::DoBuild();
if (process_proportion_ < 0.0 || process_proportion_ > 1.0)
LOG_ERROR_P(PARAM_PROCESS_PROPORTION) << ": process_proportion (" << AS_DOUBLE(process_proportion_) << ") must be between 0.0 and 1.0";
proportion_of_time_ = process_proportion_;
auto time_step = model_->managers().time_step()->GetTimeStep(time_step_label_);
if (!time_step) {
LOG_FATAL_P(PARAM_TIME_STEP) << time_step_label_ << " could not be found. Have you defined it?";
} else {
for (unsigned year : years_)
time_step->SubscribeToProcess(this, year, process_label_);
}
}
/**
* Build any runtime relationships we may have and ensure
* the labels for other objects are valid.
*/
void ProcessRemovalsByLength::DoBuild() {
partition_ = CombinedCategoriesPtr(new niwa::partition::accessors::CombinedCategories(model_, category_labels_));
cached_partition_ = CachedCombinedCategoriesPtr(new niwa::partition::accessors::cached::CombinedCategories(model_, category_labels_));
// if (ageing_error_label_ != "")
// LOG_CODE_ERROR() << "ageing error has not been implemented for the proportions at age observation";
length_results_.resize(number_bins_ * category_labels_.size(), 0.0);
auto time_step = model_->managers().time_step()->GetTimeStep(time_step_label_);
if (!time_step) {
LOG_FATAL_P(PARAM_TIME_STEP)<< time_step_label_ << " could not be found. Have you defined it?";
} else {
auto process = time_step->SubscribeToProcess(this, years_, process_label_);
mortality_instantaneous_ = dynamic_cast<MortalityInstantaneous*>(process);
}
if (!mortality_instantaneous_)
LOG_ERROR_P(PARAM_PROCESS) << "This observation can only be used for Process of type = " << PARAM_MORTALITY_INSTANTANEOUS;
// Need to split the categories if any are combined for checking
vector<string> temp_split_category_labels, split_category_labels;
for (const string& category_label : category_labels_) {
boost::split(temp_split_category_labels, category_label, boost::is_any_of("+"));
for (const string& split_category_label : temp_split_category_labels) {
split_category_labels.push_back(split_category_label);
}
}
// Need to make this a vector so its compatible with the function couldn't be bothered templating sorry
vector<string> methods;
methods.push_back(method_);
// Do some checks so that the observation and process are compatible
if (!mortality_instantaneous_->check_methods_for_removal_obs(methods))
LOG_ERROR_P(PARAM_METHOD_OF_REMOVAL) << "could not find all these methods in the instantaneous_mortality process labeled " << process_label_
<< " please check that the methods are compatible with this process";
if (!mortality_instantaneous_->check_categories_in_methods_for_removal_obs(methods, split_category_labels))
LOG_ERROR_P(PARAM_CATEGORIES) << "could not find all these categories in the instantaneous_mortality process labeled " << process_label_
<< " please check that the categories are compatible with this process";
if (!mortality_instantaneous_->check_years_in_methods_for_removal_obs(years_, methods))
LOG_ERROR_P(PARAM_YEARS) << "could not find catches in all years in the instantaneous_mortality process labeled " << process_label_
<< " please check that the years are compatible with this process";
}
/**
* Build any runtime relationships we may have and ensure
* the labels for other objects are valid.
*/
void ProportionsMigrating::DoBuild() {
partition_ = CombinedCategoriesPtr(new niwa::partition::accessors::CombinedCategories(model_, category_labels_));
cached_partition_ = CachedCombinedCategoriesPtr(new niwa::partition::accessors::cached::CombinedCategories(model_, category_labels_));
// Create a pointer to misclassification matrix
if( ageing_error_label_ != "") {
ageing_error_ = model_->managers().ageing_error()->GetAgeingError(ageing_error_label_);
if (!ageing_error_)
LOG_ERROR_P(PARAM_AGEING_ERROR) << "(" << ageing_error_label_ << ") could not be found. Have you defined it?";
}
age_results_.resize(age_spread_ * category_labels_.size(), 0.0);
TimeStep* time_step = model_->managers().time_step()->GetTimeStep(time_step_label_);
if (!time_step) {
LOG_FATAL_P(PARAM_TIME_STEP) << time_step_label_ << " could not be found. Have you defined it?";
} else
time_step->SubscribeToProcess(this, years_, process_label_);
}
/**
* Build any runtime relationships
* - Build the partition accessor
* - Build our list of selectivities
* - Build our ratios for the number of time steps
*/
void TagLoss::DoBuild() {
partition_.Init(category_labels_);
for (string label : selectivity_names_) {
Selectivity* selectivity = model_->managers().selectivity()->GetSelectivity(label);
if (!selectivity)
LOG_ERROR_P(PARAM_SELECTIVITIES) << ": selectivity " << label << " does not exist. Have you defined it?";
selectivities_.push_back(selectivity);
}
/**
* Organise our time step ratios. Each time step can
* apply a different ratio of M so here we want to verify
* we have enough and re-scale them to 1.0
*/
vector<TimeStep*> time_steps = model_->managers().time_step()->ordered_time_steps();
LOG_FINEST() << "time_steps.size(): " << time_steps.size();
vector<unsigned> active_time_steps;
for (unsigned i = 0; i < time_steps.size(); ++i) {
if (time_steps[i]->HasProcess(label_))
active_time_steps.push_back(i);
}
if (ratios_.size() == 0) {
for (unsigned i : active_time_steps)
time_step_ratios_[i] = 1.0;
} else {
if (ratios_.size() != active_time_steps.size())
LOG_FATAL_P(PARAM_TIME_STEP_RATIO) << " length (" << ratios_.size()
<< ") does not match the number of time steps this process has been assigned to (" << active_time_steps.size() << ")";
for (Double value : ratios_) {
if (value < 0.0 || value > 1.0)
LOG_ERROR_P(PARAM_TIME_STEP_RATIO) << " value (" << value << ") must be between 0.0 (inclusive) and 1.0 (inclusive)";
}
for (unsigned i = 0; i < ratios_.size(); ++i)
time_step_ratios_[active_time_steps[i]] = ratios_[i];
}
}
/**
* Validate configuration file parameters
*/
void ProcessRemovalsByLength::DoValidate() {
// How many elements are expected in our observed table;
if (length_plus_) {
number_bins_ = length_bins_.size();
} else {
number_bins_ = length_bins_.size() - 1;
}
for (auto year : years_) {
if ((year < model_->start_year()) || (year > model_->final_year()))
LOG_ERROR_P(PARAM_YEARS) << "Years can't be less than start_year (" << model_->start_year() << "), or greater than final_year (" << model_->final_year() << "). Please fix this.";
}
map<unsigned, vector<Double>> error_values_by_year;
map<unsigned, vector<Double>> obs_by_year;
/**
* Do some simple checks
* e.g Validate that the length_bins are strictly increasing
*/
for (unsigned length = 0; length < length_bins_.size(); ++length) {
if (length_bins_[length] < 0.0)
if (length_bins_[length] > length_bins_[length + 1])
LOG_ERROR_P(PARAM_LENGTH_BINS) << ": Length bins must be strictly increasing " << length_bins_[length] << " is greater than " << length_bins_[length + 1];
}
if (process_error_values_.size() != 0 && process_error_values_.size() != years_.size()) {
LOG_ERROR_P(PARAM_PROCESS_ERRORS) << " number of values provied (" << process_error_values_.size() << ") does not match the number of years provided (" << years_.size() << ")";
}
for (Double process_error : process_error_values_) {
if (process_error < 0.0)
LOG_ERROR_P(PARAM_PROCESS_ERRORS) << ": process_error (" << AS_DOUBLE(process_error) << ") cannot be less than 0.0";
}
if (process_error_values_.size() != 0)
process_errors_by_year_ = utilities::Map::create(years_, process_error_values_);
if (delta_ < 0.0)
LOG_ERROR_P(PARAM_DELTA) << ": delta (" << AS_DOUBLE(delta_) << ") cannot be less than 0.0";
/**
* Validate the number of obs provided matches age spread * category_labels * years
* This is because we'll have 1 set of obs per category collection provided.
* categories male+female male = 2 collections
*/
unsigned obs_expected = number_bins_ * category_labels_.size() + 1;
vector<vector<string>>& obs_data = obs_table_->data();
if (obs_data.size() != years_.size()) {
LOG_ERROR_P(PARAM_OBS) << " has " << obs_data.size() << " rows defined, but we expected " << years_.size() << " to match the number of years provided";
}
for (vector<string>& obs_data_line : obs_data) {
if (obs_data_line.size() != obs_expected) {
LOG_ERROR_P(PARAM_OBS) << " has " << obs_data_line.size() << " values defined, but we expected " << obs_expected << " to match the number bins * categories + 1 (for year)";
}
unsigned year = 0;
if (!utilities::To<unsigned>(obs_data_line[0], year))
LOG_ERROR_P(PARAM_OBS) << " value " << obs_data_line[0] << " could not be converted in to an unsigned integer. It should be the year for this line";
if (std::find(years_.begin(), years_.end(), year) == years_.end())
LOG_ERROR_P(PARAM_OBS) << " value " << year << " is not a valid year for this observation";
for (unsigned i = 1; i < obs_data_line.size(); ++i) {
Double value = 0;
if (!utilities::To<Double>(obs_data_line[i], value))
LOG_ERROR_P(PARAM_OBS) << " value (" << obs_data_line[i] << ") could not be converted to a double";
obs_by_year[year].push_back(value);
}
if (obs_by_year[year].size() != obs_expected - 1)
LOG_FATAL_P(PARAM_OBS)<< "you supplied " << obs_by_year[year].size() << " lengths, but we expected " << obs_expected -1 << " can you please sort this out. Chairs";
}
/**
* Build our error value map
*/
vector<vector<string>>& error_values_data = error_values_table_->data();
if (error_values_data.size() != years_.size()) {
LOG_FATAL_P(PARAM_ERROR_VALUES)<< " has " << error_values_data.size() << " rows defined, but we expected " << years_.size()
<< " to match the number of years provided";
}
for (vector<string>& error_values_data_line : error_values_data) {
if (error_values_data_line.size() != 2 && error_values_data_line.size() != obs_expected) {
LOG_ERROR_P(PARAM_ERROR_VALUES) << " has " << error_values_data_line.size() << " values defined, but we expected " << obs_expected << " to match the number bins * categories + 1 (for year)";
}
unsigned year = 0;
if (!utilities::To<unsigned>(error_values_data_line[0], year))
LOG_FATAL_P(PARAM_ERROR_VALUES)<< " value " << error_values_data_line[0] << " could not be converted in to an unsigned integer. It should be the year for this line";
if (std::find(years_.begin(), years_.end(), year) == years_.end())
LOG_FATAL_P(PARAM_ERROR_VALUES)<< " value " << year << " is not a valid year for this observation";
for (unsigned i = 1; i < error_values_data_line.size(); ++i) {
Double value = 0;
if (!utilities::To<Double>(error_values_data_line[i], value))
LOG_FATAL_P(PARAM_ERROR_VALUES)<< " value (" << error_values_data_line[i] << ") could not be converted to a double";
if (likelihood_type_ == PARAM_LOGNORMAL && value <= 0.0) {
LOG_ERROR_P(PARAM_ERROR_VALUES) << ": error_value (" << AS_DOUBLE(value) << ") cannot be equal to or less than 0.0";
} else if (likelihood_type_ == PARAM_MULTINOMIAL && value < 0.0) {
LOG_ERROR_P(PARAM_ERROR_VALUES) << ": error_value (" << AS_DOUBLE(value) << ") cannot be less than 0.0";
}
error_values_by_year[year].push_back(value);
}
if (error_values_by_year[year].size() == 1) {
error_values_by_year[year].assign(obs_expected - 1, error_values_by_year[year][0]);
}
if (error_values_by_year[year].size() != obs_expected - 1)
LOG_FATAL_P(PARAM_ERROR_VALUES)<< "We counted " << error_values_by_year[year].size() << " error values by year but expected " << obs_expected -1 << " based on the obs table";
}
/**
* Build our proportions and error values for use in the observation
* If the proportions for a given observation do not sum to 1.0
* and is off by more than the tolerance rescale them.
*/
Double value = 0.0;
for (auto iter = obs_by_year.begin(); iter != obs_by_year.end(); ++iter) {
Double total = 0.0;
for (unsigned i = 0; i < category_labels_.size(); ++i) {
for (unsigned j = 0; j < number_bins_; ++j) {
unsigned obs_index = i * number_bins_ + j;
value = iter->second[obs_index];
Double error_value = error_values_by_year[iter->first][obs_index];
error_values_[iter->first][category_labels_[i]].push_back(error_value);
proportions_[iter->first][category_labels_[i]].push_back(value);
total += value;
}
}
if (fabs(1.0 - total) > tolerance_) {
LOG_ERROR_P(PARAM_OBS) << ": obs sum total (" << total << ") for year (" << iter->first << ") exceeds tolerance (" << tolerance_ << ") from 1.0";
}
}
}
/**
* Build our parameters
*/
void ElementDifference::DoBuild() {
LOG_TRACE();
string error = "";
if (!model_->objects().VerfiyAddressableForUse(second_parameter_, addressable::kLookup, error)) {
LOG_FATAL_P(PARAM_SECOND_PARAMETER) << "could not be verified for use in additional_prior.element_difference. Error was " << error;
}
error = "";
if (!model_->objects().VerfiyAddressableForUse(parameter_, addressable::kLookup, error)) {
LOG_FATAL_P(PARAM_PARAMETER) << "could not be verified for use in additional_prior.element_difference. Error was " << error;
}
// first parameter
addressable::Type addressable_type = model_->objects().GetAddressableType(parameter_);
LOG_FINEST() << "addressable type = " << addressable_type;
switch(addressable_type) {
case addressable::kInvalid:
LOG_CODE_ERROR() << "Invalid addressable type: " << parameter_;
break;
case addressable::kMultiple:
addressable_ptr_vector_ = model_->objects().GetAddressables(parameter_);
break;
case addressable::kVector:
addressable_vector_ = model_->objects().GetAddressableVector(parameter_);
break;
case addressable::kUnsignedMap:
addressable_map_ = model_->objects().GetAddressableUMap(parameter_);
break;
case addressable::kSingle:
addressable_ = model_->objects().GetAddressable(parameter_);
break;
default:
LOG_ERROR() << "The addressable you have provided for use in a additional priors: " << parameter_
<< " is not a type that is supported for vector smoothing additional priors";
break;
}
// Get second parameter estimates
addressable_type = model_->objects().GetAddressableType(second_parameter_);
LOG_FINEST() << "addressable type = " << addressable_type;
switch(addressable_type) {
case addressable::kInvalid:
LOG_CODE_ERROR() << "Invalid addressable type: " << second_parameter_;
break;
case addressable::kMultiple:
second_addressable_ptr_vector_ = model_->objects().GetAddressables(second_parameter_);
break;
case addressable::kVector:
second_addressable_vector_ = model_->objects().GetAddressableVector(second_parameter_);
break;
case addressable::kUnsignedMap:
second_addressable_map_ = model_->objects().GetAddressableUMap(second_parameter_);
break;
case addressable::kSingle:
second_addressable_ = model_->objects().GetAddressable(second_parameter_);
break;
default:
LOG_ERROR() << "The addressable you have provided for use in a additional priors: " << second_parameter_
<< " is not a type that is supported for difference element additional priors";
break;
}
// Check the two parameters are the same length
vector<Double> values;
vector<Double> second_values;
// Load first parameter
if (addressable_vector_ != nullptr)
values.assign((*addressable_vector_).begin(), (*addressable_vector_).end());
else if (addressable_ptr_vector_ != nullptr) {
for (auto ptr : (*addressable_ptr_vector_))
values.push_back((*ptr));
} else if (addressable_map_ != nullptr) {
for (auto iter : (*addressable_map_))
values.push_back(iter.second);
} else if (addressable_ != nullptr) {
values.push_back((*addressable_));
} else
LOG_CODE_ERROR() << "(addressable_map_ != 0) && (addressable_vector_ != 0)";
// Load second parameter
if (second_addressable_vector_ != nullptr)
second_values.assign((*second_addressable_vector_).begin(), (*second_addressable_vector_).end());
else if (second_addressable_ptr_vector_ != nullptr) {
for (auto ptr : (*second_addressable_ptr_vector_))
second_values.push_back((*ptr));
} else if (second_addressable_map_ != nullptr) {
for (auto iter : (*second_addressable_map_))
second_values.push_back(iter.second);
} else if (second_addressable_ != nullptr) {
second_values.push_back((*second_addressable_));
} else
LOG_CODE_ERROR() << "(second_addressable_map_ != 0) && (second_addressable_vector_ != 0) && (second_addressable_ != 0)";
if(second_values.size() != values.size())
LOG_ERROR_P(PARAM_SECOND_PARAMETER) << "The parameters are not the same size, which they need to be, the second parameter has " << second_values.size() << " elements where as, the first parameter has " << values.size() << " elements";
}
