void ProgramProfiler::log_stats(const std::map<const void *, size_t> &linenos) {
std::unique_lock<std::mutex> lock(s_mutex);
for (ProgramProfiler *instance : s_instances) {
instance->unsafe_report();
}
std::vector<LogLine> log_lines;
double total_sec = 0;
for (auto &pair : s_stats) {
size_t lineno = map_find(linenos, pair.first);
Stat &stat = pair.second;
log_lines.push_back({stat, lineno});
total_sec += stat.time * 1e-6;
stat.count = 0;
stat.time = 0;
}
std::sort(log_lines.begin(), log_lines.end());
POMAGMA_INFO("Profile of VirtualMachine programs:");
POMAGMA_INFO(" Line Calls Percent Total sec Per call sec");
POMAGMA_INFO("----- ----------- ------- ----------- ------------");
for (const auto &line : log_lines) {
double time_sec = line.stat.time * 1e-6;
double percent = 100 * time_sec / total_sec;
POMAGMA_INFO(std::fixed << std::setprecision(2) << std::setw(5)
<< std::right << line.lineno << std::setw(12)
<< std::right << line.stat.count << std::setw(8)
<< std::right << percent << std::setw(12)
<< time_sec << std::setw(10)
<< (time_sec / line.stat.count));
}
}
// FIXME HACK this depends on link order, dying if log has been destroyed
~CleanupProfiler ()
{
unsigned long task_count = s_counts.size();
POMAGMA_INFO("Task Id\tCount\tElapsed sec");
for (unsigned long i = 0; i < task_count; ++i) {
POMAGMA_INFO(
std::setw(4) << i <<
std::setw(8) << s_counts[i].load() <<
std::setw(16) << (s_elapsed[i].load() * 1e-6));
}
}
size_t batch_simplify(
Structure & structure,
const std::vector<std::string> & routes,
const char * source_file,
const char * destin_file)
{
POMAGMA_INFO("simplifying expressions");
POMAGMA_ASSERT(
std::string(source_file) != std::string(destin_file),
"source and destin cannot be the same");
SimplifyParser parser(structure.signature(), routes);
std::ofstream destin(destin_file);
POMAGMA_ASSERT(destin, "failed to open " << destin_file);
destin << "# expressions simplifed by pomagma\n";
size_t line_count = 0;
for (LineParser iter(source_file); iter.ok(); iter.next()) {
const std::string & expression = * iter;
POMAGMA_DEBUG("simplifying " << expression);
// simplify relations like EQUAL I APP APP S K K
parser.begin(expression);
std::string type = parser.parse_token();
SimplifyTerm lhs = parser.parse_term();
SimplifyTerm rhs = parser.parse_term();
parser.end();
destin << type << " " << lhs.route << " " << rhs.route << "\n";
++line_count;
}
return line_count;
}
void Carrier::validate () const
{
UniqueLock lock(m_mutex);
POMAGMA_INFO("Validating Carrier");
m_support.validate();
size_t actual_item_count = 0;
size_t actual_rep_count = 0;
for (Ob i = 1; i <= item_dim(); ++i) {
Ob rep = m_reps[i].load();
if (contains(i)) {
POMAGMA_ASSERT(rep, "supported object has no rep: " << i);
POMAGMA_ASSERT(rep <= i, "rep out of order: " << rep << "," << i);
++actual_item_count;
if (rep == i) {
++actual_rep_count;
}
} else {
POMAGMA_ASSERT(rep == 0, "unsupported object has rep: " << i);
}
}
POMAGMA_ASSERT_EQ(item_count(), actual_item_count);
POMAGMA_ASSERT_EQ(rep_count(), actual_rep_count);
}
DenseSet Router::find_defined() const {
POMAGMA_INFO("Finding defined obs");
DenseSet defined(m_carrier.item_dim());
DenseSet undefined(m_carrier.item_dim());
undefined = m_carrier.support();
bool changed = true;
while (changed) {
changed = false;
POMAGMA_DEBUG("accumulating route probabilities");
undefined -= defined;
for (auto iter = undefined.iter(); iter.ok(); iter.next()) {
Ob ob = *iter;
if (defines(defined, ob)) {
defined.insert(ob);
changed = true;
break;
}
}
}
return defined;
}
void Server::serve (const char * address)
{
POMAGMA_INFO("Starting server");
zmq::context_t context(1);
zmq::socket_t socket(context, ZMQ_REP);
socket.bind(address);
while (true) {
POMAGMA_DEBUG("waiting for request");
zmq::message_t raw_request;
socket.recv(& raw_request);
POMAGMA_DEBUG("parsing request");
messaging::AnalystRequest request;
request.ParseFromArray(raw_request.data(), raw_request.size());
messaging::AnalystResponse response = handle(* this, request);
POMAGMA_DEBUG("serializing response");
std::string response_str;
response.SerializeToString(& response_str);
const size_t size = response_str.length();
zmq::message_t raw_response(size);
memcpy(raw_response.data(), response_str.c_str(), size);
POMAGMA_DEBUG("sending response");
socket.send(raw_response);
}
}
size_t Approximator::test_less() {
POMAGMA_INFO("Testing LESS closure");
size_t fail_count = 0;
const size_t item_dim = m_item_dim;
#pragma omp parallel
{
Approximation actual(item_dim, m_top, m_bot);
DenseSet temp_set(m_item_dim);
#pragma omp for schedule(dynamic, 1)
for (Ob x = 1; x <= item_dim; ++x) {
Approximation expected(x, m_less);
actual = expected;
close(actual, temp_set);
if (actual != expected) {
#pragma omp atomic
fail_count += 1;
}
}
}
if (fail_count) {
POMAGMA_WARN("LESS failed " << fail_count << " cases");
}
return fail_count;
}
void SymmetricFunction::validate() const {
POMAGMA_INFO("Validating SymmetricFunction");
m_lines.validate();
POMAGMA_DEBUG("validating line-value consistency");
for (size_t i = 1; i <= item_dim(); ++i)
for (size_t j = i; j <= item_dim(); ++j) {
auto val_iter = m_values.find(assert_sorted_pair(i, j));
if (not(support().contains(i) and support().contains(j))) {
POMAGMA_ASSERT(val_iter == m_values.end(),
"found unsupported lhs, rhs: " << i << ',' << j);
} else if (val_iter != m_values.end()) {
POMAGMA_ASSERT(defined(i, j),
"found undefined value: " << i << ',' << j);
Ob val = val_iter->second;
POMAGMA_ASSERT(val, "found zero value: " << i << ',' << j);
POMAGMA_ASSERT(support().contains(val),
"found unsupported value: " << i << ',' << j);
} else {
POMAGMA_ASSERT(not defined(i, j),
"found defined null value: " << i << ',' << j);
}
}
}
std::vector<float> Router::measure_probs(float reltol) const {
POMAGMA_INFO("Measuring ob probs");
const size_t item_count = m_carrier.item_count();
std::vector<float> probs(1 + item_count, 0);
const float max_increase = 1.0 + reltol;
bool changed = true;
while (changed) {
changed = false;
POMAGMA_DEBUG("accumulating route probabilities");
// The following three cannot be mixed: openmp, gcc, fork.
// see http://bisqwit.iki.fi/story/howto/openmp/#OpenmpAndFork
//# pragma omp parallel for schedule(dynamic, 1)
for (size_t i = 0; i < item_count; ++i) {
Ob ob = 1 + i;
float prob = 0;
for (const Segment& segment : iter_val(ob)) {
prob += get_prob(segment, probs);
}
if (prob > probs[ob] * max_increase) {
//#pragma omp atomic
changed = true;
}
probs[ob] = prob; // relaxed memory order
}
}
return probs;
}
void Router::update_probs(std::vector<float>& probs, float reltol) const {
POMAGMA_INFO("Updating ob probs");
const size_t item_count = m_carrier.item_count();
POMAGMA_ASSERT_EQ(probs.size(), 1 + item_count);
const float max_increase = 1.0 + reltol;
bool changed = true;
while (changed) {
changed = false;
POMAGMA_DEBUG("accumulating route probabilities");
#pragma omp parallel for schedule(dynamic, 1)
for (size_t i = 0; i < item_count; ++i) {
Ob ob = 1 + i;
float& prob = probs[ob];
float temp_prob = 0;
for (const Segment& segment : iter_val(ob)) {
temp_prob += get_prob(segment, probs);
}
if (temp_prob > prob * max_increase) {
changed = true;
}
prob = temp_prob;
}
}
}
void Structure::assert_valid() {
POMAGMA_INFO("Validating solver::Structure");
// Check atoms.
POMAGMA_ASSERT(term_arity(TermAtom::TOP) == TermArity::TOP, "Missing TOP");
POMAGMA_ASSERT(term_arity(TermAtom::BOT) == TermArity::BOT, "Missing BOT");
POMAGMA_ASSERT(term_arity(TermAtom::I) == TermArity::I, "Missing I");
POMAGMA_ASSERT(term_arity(TermAtom::K) == TermArity::K, "Missing K");
POMAGMA_ASSERT(term_arity(TermAtom::B) == TermArity::B, "Missing B");
POMAGMA_ASSERT(term_arity(TermAtom::C) == TermArity::C, "Missing C");
POMAGMA_ASSERT(term_arity(TermAtom::S) == TermArity::S, "Missing S");
// Check terms.
const Term max_term = term_arity_.size() - 1;
for (Term term = 1; term <= max_term; ++term) {
const TermArity arity = term_arity(term);
switch (arity) {
case TermArity::IVAR: {
const unsigned rank = ivar_arg(term);
POMAGMA_ASSERT_EQ(term, ivar(rank));
break;
}
case TermArity::NVAR: {
const std::string& name = nvar_arg(term);
POMAGMA_ASSERT_EQ(term, nvar(name));
break;
}
case TermArity::APP: {
Term lhs;
Term rhs;
std::tie(lhs, rhs) = app_arg(term);
POMAGMA_ASSERT_EQ(term, app(lhs, rhs));
break;
}
case TermArity::JOIN: {
Term lhs;
Term rhs;
std::tie(lhs, rhs) = join_arg(term);
POMAGMA_ASSERT_EQ(term, join(lhs, rhs));
break;
}
default:
break;
}
}
// Check literals.
const Literal max_lit = less_arg_.size() - 1;
for (Literal lit = 1; lit <= max_lit; ++lit) {
Term lhs;
Term rhs;
std::tie(lhs, rhs) = literal_arg(lit);
POMAGMA_ASSERT_EQ(lit, less(lhs, rhs));
std::tie(lhs, rhs) = literal_arg(-lit);
POMAGMA_ASSERT_EQ(-lit, nless(lhs, rhs));
}
}
void Router::update_weights(
const std::vector<float>& probs,
const std::unordered_map<std::string, size_t>& symbol_counts,
const std::unordered_map<Ob, size_t>& ob_counts,
std::vector<float>& symbol_weights, std::vector<float>& ob_weights,
float reltol) const {
POMAGMA_INFO("Updating weights");
const size_t symbol_count = m_types.size();
const size_t ob_count = m_carrier.item_count();
POMAGMA_ASSERT_EQ(probs.size(), 1 + ob_count);
POMAGMA_ASSERT_EQ(symbol_weights.size(), symbol_count);
POMAGMA_ASSERT_EQ(ob_weights.size(), 1 + ob_count);
const float max_increase = 1.0 + reltol;
std::vector<float> temp_symbol_weights(symbol_weights.size());
std::vector<float> temp_ob_weights(ob_weights.size());
update_weights_loop : {
POMAGMA_DEBUG("distributing route weight");
std::fill(temp_symbol_weights.begin(), temp_symbol_weights.end(), 0);
for (size_t i = 0; i < symbol_count; ++i) {
temp_symbol_weights[i] = map_get(symbol_counts, m_types[i].name, 0);
}
std::fill(temp_ob_weights.begin(), temp_ob_weights.end(), 0);
for (const auto& pair : ob_counts) {
temp_ob_weights[pair.first] = pair.second;
}
#pragma omp parallel for schedule(dynamic, 1)
for (size_t i = 0; i < ob_count; ++i) {
Ob ob = 1 + i;
const float weight = ob_weights[ob] / probs[ob];
for (const Segment& segment : iter_val(ob)) {
float part = weight * get_prob(segment, probs);
add_weight(part, segment, temp_symbol_weights, temp_ob_weights);
}
}
std::swap(symbol_weights, temp_symbol_weights);
std::swap(ob_weights, temp_ob_weights);
for (size_t i = 0; i < symbol_count; ++i) {
if (symbol_weights[i] > temp_symbol_weights[i] * max_increase) {
goto update_weights_loop;
}
}
for (size_t i = 0; i < ob_count; ++i) {
Ob ob = 1 + i;
if (ob_weights[ob] > temp_ob_weights[ob] * max_increase) {
goto update_weights_loop;
}
}
}
}
void Server::serve(const char* address) {
void* context;
void* socket;
zmq_msg_t message;
POMAGMA_INFO("Starting server");
POMAGMA_ASSERT_C((context = zmq_ctx_new()));
POMAGMA_ASSERT_C((socket = zmq_socket(context, ZMQ_REP)));
POMAGMA_ASSERT_C(0 == zmq_bind(socket, address));
for (m_serving = true; m_serving;) {
POMAGMA_DEBUG("waiting for request");
POMAGMA_ASSERT_C(0 == zmq_msg_init(&message));
POMAGMA_ASSERT_C(-1 != zmq_msg_recv(&message, socket, 0));
POMAGMA_DEBUG("parsing request");
protobuf::CartographerRequest request;
bool parsed = request.ParseFromArray(zmq_msg_data(&message),
zmq_msg_size(&message));
POMAGMA_ASSERT(parsed, "Failed to parse request");
POMAGMA_ASSERT_C(0 == zmq_msg_close(&message));
protobuf::CartographerResponse response = handle(*this, request);
POMAGMA_DEBUG("serializing response");
std::string response_str;
response.SerializeToString(&response_str);
const int size = response_str.length();
POMAGMA_ASSERT_C(0 == zmq_msg_init(&message));
POMAGMA_ASSERT_C(0 == zmq_msg_init_size(&message, size));
memcpy(zmq_msg_data(&message), response_str.c_str(), size);
POMAGMA_DEBUG("sending response");
POMAGMA_ASSERT_C(size == zmq_msg_send(&message, socket, 0));
POMAGMA_ASSERT_C(0 == zmq_msg_close(&message));
}
POMAGMA_INFO("stopping server");
int linger_ms = 0;
POMAGMA_ASSERT_C(
0 == zmq_setsockopt(socket, ZMQ_LINGER, &linger_ms, sizeof(linger_ms)));
POMAGMA_ASSERT_C(0 == zmq_close(socket));
POMAGMA_ASSERT_C(0 == zmq_ctx_destroy(context));
}
void NullaryFunction::validate () const
{
SharedLock lock(m_mutex);
POMAGMA_INFO("Validating NullaryFunction");
Ob value = m_value;
if (value) {
POMAGMA_ASSERT(support().contains(value),
"unsupported value: " << value);
}
}
size_t Approximator::test() {
POMAGMA_INFO("Testing approximator");
size_t fail_count = 0;
fail_count += test_less();
for (auto pair : m_structure.signature().binary_functions()) {
fail_count += test_function(pair.first, *pair.second);
}
for (auto pair : m_structure.signature().symmetric_functions()) {
fail_count += test_function(pair.first, *pair.second);
}
if (fail_count) {
POMAGMA_WARN("Failed approximator test");
} else {
POMAGMA_INFO("Passed approximator test");
}
return fail_count;
}
void insert_nullary_functions ()
{
const auto & functions = signature.nullary_functions();
POMAGMA_INFO("Inserting " << functions.size() << " nullary functions");
for (auto pair : functions) {
NullaryFunction * fun = pair.second;
if (not fun->find()) {
Ob val = carrier.try_insert();
POMAGMA_ASSERT(val, "no space to insert nullary functions");
fun->insert(val);
}
}
}
void BinaryRelation::validate_disjoint(const BinaryRelation& other) const {
POMAGMA_INFO("Validating disjoint pair of BinaryRelations");
// validate supports agree
POMAGMA_ASSERT_EQ(support().item_dim(), other.support().item_dim());
POMAGMA_ASSERT_EQ(support().count_items(), other.support().count_items());
POMAGMA_ASSERT(support() == other.support(),
"BinaryRelation supports differ");
// validate disjointness
DenseSet this_set(item_dim(), nullptr);
DenseSet other_set(item_dim(), nullptr);
for (auto i = support().iter(); i.ok(); i.next()) {
this_set.init(m_lines.Lx(*i));
other_set.init(other.m_lines.Lx(*i));
POMAGMA_ASSERT(this_set.disjoint(other_set),
"BinaryRelations intersect at row " << *i);
}
}
void test_merge (Carrier & carrier, rng_t & rng)
{
POMAGMA_INFO("Checking unsafe_merge");
const DenseSet & support = carrier.support();
size_t merge_count = 0;
g_merge_count = 0;
std::bernoulli_distribution randomly_merge(0.1);
for (auto rep_iter = support.iter(); rep_iter.ok(); rep_iter.next())
for (auto dep_iter = support.iter(); dep_iter.ok(); dep_iter.next()) {
Ob dep = carrier.find(*dep_iter);
Ob rep = carrier.find(*rep_iter);
if ((rep < dep) and randomly_merge(rng)) {
carrier.merge(dep, rep);
++merge_count;
break;
}
}
POMAGMA_ASSERT_EQ(merge_count, g_merge_count);
}
void remove_deps (Carrier & carrier, Function & fun)
{
POMAGMA_INFO("Merging deps");
const DenseSet & support = carrier.support();
bool merged;
do {
merged = false;
for (auto iter = support.iter(); iter.ok(); iter.next()) {
Ob dep = *iter;
if (carrier.find(dep) != dep) {
fun.unsafe_merge(dep);
carrier.unsafe_remove(dep);
merged = true;
}
}
} while (merged);
fun.update_values();
POMAGMA_ASSERT_EQ(carrier.rep_count(), carrier.item_count());
fun.validate();
}
Structure::Structure() {
POMAGMA_INFO("Initializing solver::Structure");
// Initialize vectors for 1-based indexing.
term_arity_.resize(1);
less_arg_.resize(1);
// Initialize atoms.
new_term(TermArity::TOP);
new_term(TermArity::BOT);
new_term(TermArity::I);
new_term(TermArity::K);
new_term(TermArity::B);
new_term(TermArity::C);
new_term(TermArity::S);
if (POMAGMA_DEBUG_LEVEL) {
assert_valid();
}
}
void BinaryRelation::validate () const
{
POMAGMA_INFO("Validating BinaryRelation");
m_lines.validate();
size_t num_pairs = 0;
DenseSet Lx(round_item_dim(), nullptr);
DenseSet Rx(round_item_dim(), nullptr);
for (Ob i = 1; i <= item_dim(); ++i) {
bool sup_i = supports(i);
Lx.init(m_lines.Lx(i));
for (Ob j = 1; j <= item_dim(); ++j) {
bool sup_ij = sup_i and supports(j);
Rx.init(m_lines.Rx(j));
bool Lx_ij = Lx.contains(j);
bool Rx_ij = Rx.contains(i);
num_pairs += Rx_ij;
POMAGMA_ASSERT(Lx_ij == Rx_ij,
"Lx,Rx disagree at " << i << "," << j
<< ", Lx is " << Lx_ij << ", Rx is " << Rx_ij );
POMAGMA_ASSERT(sup_ij or not Lx_ij,
"Lx unsupported at " << i << "," << j );
POMAGMA_ASSERT(sup_ij or not Rx_ij,
"Rx unsupported at " << i << "," << j );
}
}
size_t true_size = count_pairs();
POMAGMA_ASSERT(num_pairs == true_size,
"incorrect number of pairs: "
<< num_pairs << " should be " << true_size);
}
void Router::fit_language(
const std::unordered_map<std::string, size_t>& symbol_counts,
const std::unordered_map<Ob, size_t>& ob_counts, float reltol) {
POMAGMA_INFO("Fitting language");
const size_t item_count = m_carrier.item_count();
std::vector<float> ob_probs(1 + item_count, 0);
std::vector<float> ob_weights(1 + item_count, 0);
std::vector<float> symbol_weights(m_types.size(), 0);
POMAGMA_ASSERT_EQ(m_types.size(), m_language.size());
const float max_increase = 1.0 + reltol;
bool changed = true;
while (changed) {
changed = false;
update_probs(ob_probs, reltol);
update_weights(ob_probs, symbol_counts, ob_counts, symbol_weights,
ob_weights, reltol);
POMAGMA_DEBUG("optimizing language");
float total_weight = 0;
for (float weight : symbol_weights) {
total_weight += weight;
}
for (size_t i = 0; i < m_types.size(); ++i) {
SegmentType& type = m_types[i];
float new_prob = symbol_weights[i] / total_weight;
float old_prob = type.prob;
type.prob = new_prob;
m_language[type.name] = new_prob;
if (new_prob > old_prob * max_increase) {
changed = true;
}
}
}
}
void Server::serve(const char* address) {
void* context;
void* socket;
zmq_msg_t message;
POMAGMA_INFO("Starting server");
POMAGMA_ASSERT_C((context = zmq_ctx_new()));
POMAGMA_ASSERT_C((socket = zmq_socket(context, ZMQ_REP)));
POMAGMA_ASSERT_C(0 == zmq_bind(socket, address));
while (true) {
POMAGMA_DEBUG("waiting for request");
POMAGMA_ASSERT_C(0 == zmq_msg_init(&message));
POMAGMA_ASSERT_C(-1 != zmq_msg_recv(&message, socket, 0));
POMAGMA_DEBUG("parsing request");
protobuf::AnalystRequest request;
bool parsed = request.ParseFromArray(zmq_msg_data(&message),
zmq_msg_size(&message));
POMAGMA_ASSERT(parsed, "Failed to parse request");
POMAGMA_ASSERT_C(0 == zmq_msg_close(&message));
protobuf::AnalystResponse response = handle(*this, request);
POMAGMA_DEBUG("serializing response");
std::string response_str;
response.SerializeToString(&response_str);
const int size = response_str.length();
POMAGMA_ASSERT_C(0 == zmq_msg_init(&message));
POMAGMA_ASSERT_C(0 == zmq_msg_init_size(&message, size));
memcpy(zmq_msg_data(&message), response_str.c_str(), size);
POMAGMA_DEBUG("sending response");
POMAGMA_ASSERT_C(size == zmq_msg_send(&message, socket, 0));
POMAGMA_ASSERT_C(0 == zmq_msg_close(&message));
}
}
std::vector<Ob> conjecture_diverge(Structure& structure,
const std::vector<float>& probs,
const std::vector<std::string>& routes,
const char* conjectures_file) {
POMAGMA_INFO("Conjecturing divergent terms");
const Carrier& carrier = structure.carrier();
const BinaryRelation& NLESS = structure.binary_relation("NLESS");
const Ob BOT = structure.nullary_function("BOT").find();
const Ob TOP = structure.nullary_function("TOP").find();
POMAGMA_DEBUG("collecting conjectures");
DenseSet conjecture_set(carrier.item_count());
conjecture_set.set_diff(carrier.support(), NLESS.get_Rx_set(BOT));
POMAGMA_ASSERT(conjecture_set.contains(BOT), "BOT not conjectured");
POMAGMA_ASSERT(not conjecture_set.contains(TOP), "TOP conjectured");
conjecture_set.remove(BOT);
std::vector<Ob> conjectures;
for (auto iter = conjecture_set.iter(); iter.ok(); iter.next()) {
Ob ob = *iter;
conjectures.push_back(ob);
}
POMAGMA_DEBUG("sorting " << conjectures.size() << " conjectures");
std::sort(conjectures.begin(), conjectures.end(),
[&](const Ob& x, const Ob& y) { return probs[x] > probs[y]; });
POMAGMA_DEBUG("writing conjectures to " << conjectures_file);
std::ofstream file(conjectures_file, std::ios::out | std::ios::trunc);
POMAGMA_ASSERT(file, "failed to open " << conjectures_file);
file << "# divergence conjectures generated by pomagma";
for (auto ob : conjectures) {
file << "\nEQUAL BOT " << routes[ob];
}
return conjectures;
}
void NullaryFunction::log_stats (const std::string & prefix) const
{
if (not m_value.load()) {
POMAGMA_INFO(prefix << " undefined");
}
}
size_t Approximator::test_function(const std::string& name,
const Function& fun) {
POMAGMA_INFO("Testing " << name << " approximation");
size_t ob_fail_count = 0;
size_t upper_fail_count = 0;
size_t upper_extra_count = 0;
size_t upper_missing_count = 0;
size_t lower_fail_count = 0;
size_t lower_extra_count = 0;
size_t lower_missing_count = 0;
const size_t item_dim = m_item_dim;
#pragma omp parallel
{
DenseSet temp_set(item_dim);
#pragma omp for schedule(dynamic, 1)
for (Ob x = 1; x <= item_dim; ++x) {
Approximation approx_x(x, m_less);
approx_x.ob = 0;
for (auto iter = fun.iter_lhs(x); iter.ok(); iter.next()) {
Ob y = *iter;
Approximation approx_y(y, m_less);
approx_y.ob = 0;
Ob xy = fun.find(x, y);
Approximation expected(xy, m_less);
Approximation actual = find(fun, approx_x, approx_y);
if (actual.ob != expected.ob) {
#pragma omp atomic
ob_fail_count += 1;
}
if (actual.upper != expected.upper) {
#pragma omp atomic
upper_fail_count += 1;
temp_set.set_diff(actual.upper, expected.upper);
if (size_t count = temp_set.count_items()) {
#pragma omp atomic
upper_extra_count += count;
}
temp_set.set_diff(expected.upper, actual.upper);
if (size_t count = temp_set.count_items()) {
#pragma omp atomic
upper_missing_count += count;
}
}
if (actual.lower != expected.lower) {
#pragma omp atomic
lower_fail_count += 1;
temp_set.set_diff(actual.lower, expected.lower);
if (size_t count = temp_set.count_items()) {
#pragma omp atomic
lower_extra_count += count;
}
temp_set.set_diff(expected.lower, actual.lower);
if (size_t count = temp_set.count_items()) {
#pragma omp atomic
lower_missing_count += count;
}
}
}
}
}
if (ob_fail_count) {
POMAGMA_WARN(name << " ob failed " << ob_fail_count << " cases");
}
if (upper_missing_count or upper_extra_count) {
POMAGMA_WARN(name << " upper has " << upper_missing_count
<< " missing and " << upper_extra_count
<< " extra obs in " << upper_fail_count << " cases");
}
if (lower_missing_count or lower_extra_count) {
POMAGMA_WARN(name << " lower has " << lower_missing_count
<< " missing and " << lower_extra_count
<< " extra obs in " << lower_fail_count << " cases");
}
return ob_fail_count + upper_fail_count + lower_fail_count;
}
static protobuf::CartographerResponse handle(
Server& server, protobuf::CartographerRequest& request) {
POMAGMA_INFO("Handling request");
Timer timer;
protobuf::CartographerResponse response;
if (request.has_crop()) {
server.crop(request.crop().headroom());
response.mutable_crop();
}
if (request.has_declare()) {
for (const auto& name : request.declare().nullary_functions()) {
server.declare(name);
}
response.mutable_declare();
}
if (request.has_assume()) {
const std::string& facts_in = request.assume().facts_in();
auto counts = server.assume(facts_in);
response.mutable_assume()->set_pos_count(counts["pos"]);
response.mutable_assume()->set_neg_count(counts["neg"]);
response.mutable_assume()->set_merge_count(counts["merge"]);
response.mutable_assume()->set_ignored_count(counts["ignored"]);
}
if (request.has_infer()) {
const size_t priority = request.infer().priority();
const size_t theorem_count = server.infer(priority);
response.mutable_infer()->set_theorem_count(theorem_count);
}
if (request.has_execute()) {
server.execute(request.execute().program());
response.mutable_execute();
}
if (request.has_aggregate()) {
server.aggregate(request.aggregate().survey_in());
response.mutable_aggregate();
}
if (request.has_validate()) {
server.validate();
response.mutable_validate();
}
if (request.has_info()) {
const auto info = server.info();
response.mutable_info()->set_item_count(info.item_count);
}
if (request.has_dump()) {
server.dump(request.dump().world_out());
response.mutable_dump();
}
if (request.trim_size() > 0) {
std::vector<Server::TrimTask> tasks(request.trim_size());
for (int i = 0; i < request.trim_size(); ++i) {
const auto& task = request.trim(i);
tasks[i].size = task.size();
tasks[i].temperature = task.temperature();
tasks[i].filename = task.filename();
response.add_trim();
}
server.trim(tasks);
}
if (request.has_conjecture()) {
const std::string& diverge_out = request.conjecture().diverge_out();
const std::string& equal_out = request.conjecture().equal_out();
const size_t max_count = request.conjecture().max_count();
auto counts = server.conjecture(diverge_out, equal_out, max_count);
response.mutable_conjecture()->set_diverge_count(counts["diverge"]);
response.mutable_conjecture()->set_equal_count(counts["equal"]);
}
if (request.has_stop()) {
server.stop();
response.mutable_stop();
}
POMAGMA_INFO("Handled request in " << timer.elapsed() << " sec");
return response;
}
static protobuf::AnalystResponse handle(Server& server,
protobuf::AnalystRequest& request) {
POMAGMA_INFO("Handling request");
protobuf::AnalystResponse response;
typedef protobuf::AnalystResponse::Trool Trool;
if (request.has_id()) {
response.set_id(request.id());
}
if (request.has_test_inference()) {
size_t fail_count = server.test_inference();
response.mutable_test_inference()->set_fail_count(fail_count);
}
if (request.has_simplify()) {
size_t code_count = request.simplify().codes_size();
for (size_t i = 0; i < code_count; ++i) {
const std::string& code = request.simplify().codes(i);
std::string result = server.simplify(code);
response.mutable_simplify()->add_codes(result);
}
}
if (request.has_validate()) {
size_t code_count = request.validate().codes_size();
for (size_t i = 0; i < code_count; ++i) {
const std::string& code = request.validate().codes(i);
auto validity = server.validate(code);
auto& result = *response.mutable_validate()->add_results();
result.set_is_top(static_cast<Trool>(validity.is_top));
result.set_is_bot(static_cast<Trool>(validity.is_bot));
result.set_pending(false);
}
}
if (request.has_validate_corpus()) {
size_t line_count = request.validate_corpus().lines_size();
std::vector<Corpus::LineOf<std::string>> lines(line_count);
for (size_t i = 0; i < line_count; ++i) {
const auto& line = request.validate_corpus().lines(i);
if (line.has_name()) {
lines[i].maybe_name = line.name();
}
lines[i].body = line.code();
}
const auto validities = server.validate_corpus(lines);
auto& responses = *response.mutable_validate_corpus();
for (const auto& pair : validities) {
auto& result = *responses.add_results();
result.set_is_top(static_cast<Trool>(pair.validity.is_top));
result.set_is_bot(static_cast<Trool>(pair.validity.is_bot));
result.set_pending(pair.pending);
}
}
if (request.has_get_histogram()) {
const Corpus::Histogram& histogram = server.get_histogram();
auto& response_histogram =
*response.mutable_get_histogram()->mutable_histogram();
for (const auto& pair : histogram.obs) {
auto& term = *response_histogram.add_terms();
term.set_ob(pair.first);
term.set_count(pair.second);
}
for (const auto& pair : histogram.symbols) {
auto& term = *response_histogram.add_terms();
term.set_name(pair.first);
term.set_count(pair.second);
}
}
if (request.has_fit_language()) {
std::unordered_map<std::string, float> language;
if (request.fit_language().has_histogram()) {
Corpus::Histogram histogram;
const auto& request_histogram = request.fit_language().histogram();
size_t terms_size = request_histogram.terms_size();
for (size_t i = 0; i < terms_size; ++i) {
const auto& term = request_histogram.terms(i);
if (term.has_ob()) {
histogram.obs[term.ob()] = term.count();
} else {
histogram.symbols[term.name()] = term.count();
}
}
language = server.fit_language(histogram);
} else {
language = server.fit_language(server.get_histogram());
}
auto& response_fit_language = *response.mutable_fit_language();
for (const auto& pair : language) {
auto& symbol = *response_fit_language.add_symbols();
symbol.set_name(pair.first);
symbol.set_prob(pair.second);
}
}
if (request.has_solve()) {
size_t max_solutions = std::numeric_limits<size_t>::max();
if (request.solve().has_max_solutions()) {
max_solutions = request.solve().max_solutions();
}
if (max_solutions > 0) {
Server::SolutionSet solutions =
server.solve(request.solve().program(), max_solutions);
auto& response_solve = *response.mutable_solve();
for (const auto& solution : solutions.necessary) {
response_solve.add_necessary(solution);
}
for (const auto& solution : solutions.possible) {
response_solve.add_possible(solution);
}
} else {
response.add_error_log(
"expected request.solve.max_solutions > 0; actual 0");
}
}
if (request.has_validate_facts()) {
const auto& facts = request.validate_facts().facts();
const std::vector<std::string> polish_facts(facts.begin(), facts.end());
const auto result = server.validate_facts(polish_facts);
response.mutable_validate_facts()->set_result(
static_cast<Trool>(result));
}
for (const std::string& message : server.flush_errors()) {
response.add_error_log(message);
}
return response;
}
std::vector<std::string> Router::find_routes() const {
POMAGMA_INFO("Routing all obs");
const size_t item_count = m_carrier.item_count();
std::vector<float> best_probs(1 + item_count, 0);
std::vector<Segment> best_segments(1 + item_count);
bool changed = true;
while (changed) {
changed = false;
POMAGMA_DEBUG("finding best local routes");
//#pragma omp parallel for schedule(dynamic, 1)
for (size_t i = 0; i < item_count; ++i) {
Ob ob = 1 + i;
float& best_prob = best_probs[ob];
Segment& best_segment = best_segments[ob];
bool best_changed = false;
for (const Segment& segment : iter_val(ob)) {
float prob = get_prob(segment, best_probs);
if (unlikely(std::make_pair(-prob, segment) <
std::make_pair(-best_prob, best_segment))) {
best_prob = prob; // relaxed memory order
best_segment = segment; // relaxed memory order
best_changed = true;
}
}
if (best_changed) {
//#pragma omp atomic
changed = true;
}
}
}
POMAGMA_DEBUG("scheduling route building");
std::vector<Ob> schedule;
schedule.reserve(item_count);
for (auto iter = m_carrier.iter(); iter.ok(); iter.next()) {
Ob ob = *iter;
POMAGMA_ASSERT_LT(0, best_probs[ob]);
schedule.push_back(ob);
}
std::sort(schedule.begin(), schedule.end(), [&](const Ob& x, const Ob& y) {
return best_probs[x] > best_probs[y];
});
POMAGMA_DEBUG("building full routes");
std::vector<std::string> routes(1 + item_count);
for (Ob ob : schedule) {
const Segment& segment = best_segments[ob];
const SegmentType& type = m_types[segment.type];
switch (type.arity) {
case NULLARY: {
routes[ob] = type.name;
} break;
case UNARY: {
const auto& arg = routes[segment.arg1];
POMAGMA_ASSERT(not arg.empty(), "unknown arg route");
routes[ob] = type.name + " " + arg;
} break;
case BINARY: {
const auto& lhs = routes[segment.arg1];
const auto& rhs = routes[segment.arg2];
POMAGMA_ASSERT(not lhs.empty(), "unknown lhs route");
POMAGMA_ASSERT(not rhs.empty(), "unknown rhs route");
routes[ob] = type.name + " " + lhs + " " + rhs;
} break;
}
}
return routes;
}
Router::Router(const Signature& signature,
const std::unordered_map<std::string, float>& language)
: m_carrier(*signature.carrier()),
m_language(language),
m_value_index(signature.carrier()->item_count()) {
POMAGMA_INFO("Building router indices");
for (auto pair : signature.nullary_functions()) {
const auto& name = pair.first;
const auto& fun = *pair.second;
if (m_language.find(name) != m_language.end()) {
const TypeId type = new_type(NULLARY, name);
if (Ob val = fun.find()) {
m_segments.push_back(Segment(type, val));
}
}
}
for (auto pair : signature.injective_functions()) {
const auto& name = pair.first;
const auto& fun = *pair.second;
if (m_language.find(name) != m_language.end()) {
const TypeId type = new_type(UNARY, name);
for (auto iter = fun.iter(); iter.ok(); iter.next()) {
Ob arg = *iter;
Ob val = fun.find(arg);
m_segments.push_back(Segment(type, val, arg));
}
}
}
for (auto pair : signature.binary_functions()) {
const auto& name = pair.first;
const auto& fun = *pair.second;
if (m_language.find(name) != m_language.end()) {
const TypeId type = new_type(BINARY, name);
for (auto iter = m_carrier.iter(); iter.ok(); iter.next()) {
Ob lhs = *iter;
for (auto iter = fun.iter_lhs(lhs); iter.ok(); iter.next()) {
Ob rhs = *iter;
Ob val = fun.find(lhs, rhs);
m_segments.push_back(Segment(type, val, lhs, rhs));
}
}
}
}
for (auto pair : signature.symmetric_functions()) {
const auto& name = pair.first;
const auto& fun = *pair.second;
if (m_language.find(name) != m_language.end()) {
const TypeId type = new_type(BINARY, name);
for (auto iter = m_carrier.iter(); iter.ok(); iter.next()) {
Ob lhs = *iter;
for (auto iter = fun.iter_lhs(lhs); iter.ok(); iter.next()) {
Ob rhs = *iter;
Ob val = fun.find(lhs, rhs);
m_segments.push_back(Segment(type, val, lhs, rhs));
}
}
}
}
std::sort(m_segments.begin(), m_segments.end());
// assume all values are reached by at least one segment
m_value_index.resize(1 + m_carrier.item_count(), 0);
for (size_t i = 0; i < m_segments.size(); ++i) {
m_value_index[m_segments[i].val] = i;
}
}
