void set_extra_passes (double extra_passes)
{
LOOM_ASSERT_LE(0, extra_passes);
LOOM_ASSERT_LE(extra_passes, max_extra_passes);
add_rate_ = 1.0 + extra_passes;
remove_rate_ = extra_passes;
LOOM_ASSERT(remove_rate_ < add_rate_, "underflow");
}
void generate_rows (
const protobuf::Config::Generate & config,
CrossCat & cross_cat,
Assignments & assignments,
const char * rows_out,
rng_t & rng)
{
const size_t kind_count = cross_cat.kinds.size();
const size_t row_count = config.row_count();
const float density = config.density();
LOOM_ASSERT_LE(0.0, density);
LOOM_ASSERT_LE(density, 1.0);
VectorFloat scores;
std::vector<ProductModel::Value> partial_values(kind_count);
protobuf::Row row;
protobuf::OutFile rows(rows_out);
for (auto & kind : cross_cat.kinds) {
kind.model.realize(rng);
}
cross_cat.schema.clear(* row.mutable_diff());
ProductValue & full_value = * row.mutable_diff()->mutable_pos();
for (size_t id = 0; id < row_count; ++id) {
assignments.rowids().try_push(id);
for (size_t k = 0; k < kind_count; ++k) {
auto & kind = cross_cat.kinds[k];
ProductModel & model = kind.model;
auto & mixture = kind.mixture;
ProductValue & value = partial_values[k];
auto & groupids = assignments.groupids(k);
scores.resize(mixture.clustering.counts().size());
mixture.clustering.score_value(model.clustering, scores);
distributions::scores_to_probs(scores);
const VectorFloat & probs = scores;
auto & observed = * value.mutable_observed();
ValueSchema::clear(observed);
observed.set_sparsity(ProductModel::Value::Observed::DENSE);
const size_t feature_count = kind.featureids.size();
for (size_t f = 0; f < feature_count; ++f) {
observed.add_dense(
distributions::sample_bernoulli(rng, density));
}
size_t groupid = mixture.sample_value(model, probs, value, rng);
model.add_value(value, rng);
mixture.add_value(model, groupid, value, rng);
groupids.push(groupid);
}
row.set_id(id);
cross_cat.splitter.join(full_value, partial_values);
rows.write_stream(row);
}
}
AcceleratingSchedule (const protobuf::Config::Schedule & config) :
extra_passes_(config.extra_passes()),
small_data_size_(config.small_data_size()),
big_data_size_(config.big_data_size())
{
LOOM_ASSERT_LE(0, small_data_size_);
LOOM_ASSERT_LE(small_data_size_, big_data_size_);
LOOM_ASSERT(
std::isfinite(big_data_size_) or std::isinf(small_data_size_),
"big_data_size infinite but small_data_size finite");
}
PipelineQueue (size_t size, size_t stage_count) :
envelopes_(size + 1),
size_plus_one_(size + 1),
stage_count_(stage_count),
consumer_counts_(stage_count, 0),
position_(0),
guards_()
{
LOOM_ASSERT_LE(1, stage_count_);
LOOM_ASSERT_LE(1 + stage_count_, PipelineState::max_stage_count);
for (size_t i = 0; i < stage_count_; ++i) {
guards_[i].init(i, 0);
}
guards_[stage_count_].init(stage_count_, 1);
PipelineGuard & guard = guards_[stage_count_];
for (size_t i = 0; i < size_plus_one_; ++i) {
guard.unsafe_set_ready(envelopes_[i].state);
}
assert_ready();
}
static pair_t create_state (uint_fast64_t stage_number, count_t count)
{
LOOM_ASSERT_LT(stage_number, max_stage_count);
LOOM_ASSERT_LE(count, 0xFFFFUL);
return _state(stage_number, count);
}