void handle_delete(DBClientBase *client, DBOperation *operation)
{
BSONObj result;
bool success;
try {
success = client->runCommand(
m_db,
BSON("findandmodify" << "astron.objects" <<
"query" << BSON(
"_id" << operation->doid()) <<
"remove" << true),
result);
} catch(mongo::DBException &e) {
m_log->error() << "Unexpected error while deleting "
<< operation->doid() << ": " << e.what() << endl;
operation->on_failure();
return;
}
m_log->trace() << "handle_delete: got response: "
<< result << endl;
// If the findandmodify command failed, there wasn't anything there
// to delete in the first place.
if(!success || result["value"].isNull()) {
m_log->error() << "Tried to delete non-existent doid "
<< operation->doid() << endl;
operation->on_failure();
return;
}
free_doid(client, operation->doid());
operation->on_complete();
}
inline bool load(doid_t do_id, YAML::Node &document)
{
ifstream stream(filename(do_id));
document = YAML::Load(stream);
if(!document.IsDefined() || document.IsNull())
{
m_log->error() << "obj-" << do_id << " does not exist in database." << endl;
return false;
}
if(!document["class"].IsDefined() || document["class"].IsNull())
{
m_log->error() << filename(do_id) << " does not contain the 'class' key." << endl;
return false;
}
if(!document["fields"].IsDefined() || document["fields"].IsNull())
{
m_log->error() << filename(do_id) << " does not contain the 'fields' key." << endl;
return false;
}
// Read object's DistributedClass
string dc_name = document["class"].as<string>();
if(!g_dcf->get_class_by_name(dc_name))
{
m_log->error() << "Class '" << dc_name << "', loaded from '" << filename(do_id)
<< "', does not exist." << endl;
return false;
}
return true;
}
bool ConfigGroup::validate(ConfigNode node)
{
if(!node.IsMap())
{
if(m_name.length() > 0)
{
config_log.error() << "Section '" << m_path
<< "' has key/value config variables.\n";
}
else
{
config_log.error() << "Config sections must be at root of config file.\n";
}
return false;
}
bool ok = true;
for(auto it = node.begin(); it != node.end(); ++it)
{
string key = it->first.as<std::string>();
auto found_var = m_variables.find(key);
if(found_var != m_variables.end())
{
rtest r = found_var->second;
if(!r(node))
{
config_log.info() << "In Section '" << m_path << "', attribute '"
<< key << "' did not match constraint (see error).\n";
ok = false;
}
continue;
}
auto found_grp = m_children.find(key);
if(found_grp != m_children.end())
{
if(!found_grp->second->validate(node[key]))
{
ok = false;
}
continue;
}
if(m_name.length() > 0)
{
config_log.error() << "Section '" << m_path
<< "' has no attribute named '" << key << "'.\n";
}
else
{
config_log.error() << "Section '" << key << "' is not a valid config category.\n";
}
ok = false;
}
return ok;
}
bool KeyedConfigList::validate(ConfigNode node)
{
if(!node.IsSequence())
{
config_log.error() << "Section '" << m_path
<< "' expects a list of values.\n";
return false;
}
int n = -1;
bool ok = true;
for(auto it = node.begin(); it != node.end(); ++it)
{
n += 1;
if(!it->IsMap())
{
config_log.error() << "The " << n << "th item of section '" << m_path
<< "' does not have key/value config variables.\n";
ok = false;
continue;
}
ConfigNode element = *it;
if(!element[m_key])
{
config_log.error() << "The " << n << "th item of section '" << m_path
<< "' did not specify the attribute '" << m_key << "'.\n";
ok = false;
continue;
}
string key = element[m_key].as<std::string>();
auto found_grp = m_children.find(key);
if(found_grp == m_children.end())
{
config_log.error() << "The value '" << key << "' is not valid for attribute '"
<< m_key << "' of section '" << m_path << "'.\n";
print_keys();
ok = false;
continue;
}
if(!found_grp->second->validate(element))
{
ok = false;
}
}
return ok;
}
void handle_create(DBClientBase *client, DBOperation *operation)
{
// First, let's convert the requested object into BSON; this way, if
// a failure happens, it happens before we waste a doid.
BSONObjBuilder fields;
try {
for(auto it = operation->set_fields().begin();
it != operation->set_fields().end();
++it) {
DatagramPtr dg = Datagram::create();
dg->add_data(it->second);
DatagramIterator dgi(dg);
fields << it->first->get_name()
<< bamboo2bson(it->first->get_type(), dgi)["_"];
}
} catch(mongo::DBException &e) {
m_log->error() << "While formatting "
<< operation->dclass()->get_name()
<< " for insertion: " << e.what() << endl;
operation->on_failure();
return;
}
doid_t doid = assign_doid(client);
if(doid == INVALID_DO_ID) {
// The error will already have been emitted at this point, so
// all that's left for us to do is fail silently:
operation->on_failure();
return;
}
BSONObj b = BSON("_id" << doid <<
"dclass" << operation->dclass()->get_name() <<
"fields" << fields.obj());
m_log->trace() << "Inserting new " << operation->dclass()->get_name()
<< "(" << doid << "): " << b << endl;
try {
client->insert(m_obj_collection, b);
} catch(mongo::DBException &e) {
m_log->error() << "Cannot insert new "
<< operation->dclass()->get_name()
<< "(" << doid << "): " << e.what() << endl;
operation->on_failure();
return;
}
operation->on_complete(doid);
}
// This is used when the monotonic counter is exhausted:
doid_t assign_doid_reuse(DBClientBase *client)
{
BSONObj result;
bool success = client->runCommand(
m_db,
BSON("findandmodify" << "astron.globals" <<
"query" << BSON(
"_id" << "GLOBALS" <<
"doid.free.0" << BSON("$exists" << true)
) <<
"update" << BSON(
"$pop" << BSON("doid.free" << -1)
)), result);
// If the findandmodify command failed, the document either doesn't
// exist, or we ran out of reusable doids.
if(!success || result["value"].isNull()) {
m_log->error() << "Could not allocate a reused DOID!" << endl;
return INVALID_DO_ID;
}
m_log->trace() << "assign_doid_reuse: got globals element: "
<< result << endl;
// Otherwise, use the first one:
doid_t doid;
const BSONElement &element = result["value"]["doid"]["free"];
if(sizeof(doid) == sizeof(long long)) {
doid = element.Array()[0].Long();
} else if(sizeof(doid) == sizeof(int)) {
doid = element.Array()[0].Int();
}
return doid;
}
void handle_get(DBClientBase *client, DBOperation *operation)
{
BSONObj obj;
try {
obj = client->findOne(m_obj_collection,
BSON("_id" << operation->doid()));
} catch(mongo::DBException &e) {
m_log->error() << "Unexpected error occurred while trying to"
" retrieve object with DOID "
<< operation->doid() << ": " << e.what() << endl;
operation->on_failure();
return;
}
if(obj.isEmpty()) {
m_log->warning() << "Got queried for non-existent object with DOID "
<< operation->doid() << endl;
operation->on_failure();
return;
}
DBObjectSnapshot *snap = format_snapshot(operation->doid(), obj);
if(!snap || !operation->verify_class(snap->m_dclass)) {
operation->on_failure();
} else {
operation->on_complete(snap);
}
}
void run_thread()
{
unique_lock<mutex> guard(m_lock);
DBClientBase *client = new_connection();
while(true) {
if(!client->isStillConnected()) {
m_log->error() << "A thread lost its connection, reconnecting..." << endl;
delete client;
client = new_connection();
}
if(m_operation_queue.size() > 0) {
DBOperation *op = m_operation_queue.front();
m_operation_queue.pop();
guard.unlock();
handle_operation(client, op);
guard.lock();
} else if(m_shutdown) {
break;
} else {
m_cv.wait(guard);
}
}
delete client;
}
// Get a DBObjectSnapshot from a MongoDB BSON object; returns NULL if failure.
DBObjectSnapshot *format_snapshot(doid_t doid, const BSONObj &obj)
{
m_log->trace() << "Formatting database snapshot of " << doid << ": "
<< obj << endl;
try {
string dclass_name = obj["dclass"].String();
const dclass::Class *dclass = g_dcf->get_class_by_name(dclass_name);
if(!dclass) {
m_log->error() << "Encountered unknown database object: "
<< dclass_name << "(" << doid << ")" << endl;
return NULL;
}
BSONObj fields = obj["fields"].Obj();
DBObjectSnapshot *snap = new DBObjectSnapshot();
snap->m_dclass = dclass;
for(auto it = fields.begin(); it.more(); ++it) {
const char *name = (*it).fieldName();
const dclass::Field *field = dclass->get_field_by_name(name);
if(!field) {
m_log->warning() << "Encountered unexpected field " << name
<< " while formatting " << dclass_name
<< "(" << doid << "); ignored." << endl;
continue;
}
{
DatagramPtr dg = Datagram::create();
bson2bamboo(field->get_type(), *it, *dg);
snap->m_fields[field].resize(dg->size());
memcpy(snap->m_fields[field].data(), dg->get_data(), dg->size());
}
}
return snap;
} catch(mongo::DBException &e) {
m_log->error() << "Unexpected error while trying to format"
" database snapshot for " << doid << ": "
<< e.what() << endl;
return NULL;
}
}
vector<uint8_t> read_yaml_field(const Field* field, YAML::Node node, doid_t id)
{
bool error;
string packed_data = dclass::parse_value(field->get_type(), node.as<string>(), error);
if(error)
{
m_log->error() << "Failed parsing value for field '" << field->get_name()
<< "' of object " << id << "' from database.\n";
return vector<uint8_t>();
}
vector<uint8_t> result(packed_data.begin(), packed_data.end());
return result;
}
// This returns a DOID to the free list:
void free_doid(DBClientBase *client, doid_t doid)
{
m_log->trace() << "Returning doid " << doid << " to the free pool..." << endl;
try {
client->update(
m_global_collection,
BSON("_id" << "GLOBALS"),
BSON("$push" << BSON("doid.free" << doid)));
} catch(mongo::DBException &e) {
m_log->error() << "Could not return doid " << doid
<< " to free pool: " << e.what() << endl;
}
}
bool ConfigList::validate(ConfigNode node)
{
if(!node.IsSequence())
{
config_log.error() << "Section '" << m_path
<< "' expects a list of values.\n";
return false;
}
bool ok = true;
for(auto it = node.begin(); it != node.end(); ++it)
{
if(!ConfigGroup::validate(*it))
{
ok = false;
}
}
return ok;
}
// This function is used by handle_create to get a fresh DOID assignment.
doid_t assign_doid(DBClientBase *client)
{
try {
if(!m_monotonic_exhausted) {
doid_t doid = assign_doid_monotonic(client);
if(doid == INVALID_DO_ID) {
m_monotonic_exhausted = true;
} else {
return doid;
}
}
// We've exhausted our supply of doids from the monotonic counter.
// We must now resort to pulling things out of the free list:
return assign_doid_reuse(client);
} catch(mongo::DBException &e) {
m_log->error() << "Unexpected error occurred while trying to"
" allocate a new DOID: " << e.what() << endl;
return INVALID_DO_ID;
}
}
void get_all_from_table(doid_t id, const Class* dcc, FieldValues &fields)
{
string value;
indicator ind;
for(unsigned int i = 0; i < dcc->get_num_fields(); ++i) {
const Field* field = dcc->get_field(i);
if(field->has_keyword("db")) {
m_sql << "SELECT " << field->get_name() << " FROM fields_" << dcc->get_name()
<< " WHERE object_id=" << id << ";", into(value, ind);
if(ind == i_ok) {
bool parse_err;
string packed_data = parse_value(field->get_type(), value, parse_err);
if(parse_err) {
m_log->error() << "Failed parsing value for field '" << field->get_name()
<< "' of object " << id << "' from database.\n";
continue;
}
fields[field] = vector<uint8_t>(packed_data.begin(), packed_data.end());
}
}
}
}
void config_error(const std::string& msg)
{
config_log.error() << msg << "\n";
}
virtual void handle_datagram(Datagram &in_dg, DatagramIterator &dgi)
{
channel_t sender = dgi.read_uint64();
unsigned short msg_type = dgi.read_uint16();
switch(msg_type)
{
case DBSERVER_CREATE_STORED_OBJECT:
{
unsigned int context = dgi.read_uint32();
// Start response with generic header
Datagram resp;
resp.add_server_header(sender, m_control_channel, DBSERVER_CREATE_STORED_OBJECT_RESP);
resp.add_uint32(context);
// Get DistributedClass
unsigned short dc_id = dgi.read_uint16();
DCClass *dcc = gDCF->get_class(dc_id);
if(!dcc)
{
m_log->error() << "Invalid DCClass when creating object: #" << dc_id << std::endl;
resp.add_uint32(0);
send(resp);
return;
}
// Unpack fields to be passed to database
DatabaseObject dbo(dc_id);
unsigned short field_count = dgi.read_uint16();
m_log->spam() << "Unpacking fields..." << std::endl;
try
{
for(unsigned int i = 0; i < field_count; ++i)
{
unsigned short field_id = dgi.read_uint16();
DCField *field = dcc->get_field_by_index(field_id);
if(field)
{
if(field->is_db())
{
dgi.unpack_field(field, dbo.fields[field]);
}
else
{
std::string tmp;
dgi.unpack_field(field, tmp);
}
}
}
}
catch(std::exception &e)
{
m_log->error() << "Error while unpacking fields, msg may be truncated. e.what(): "
<< e.what() << std::endl;
resp.add_uint32(0);
send(resp);
return;
}
// Check for required fields, and populate with defaults
m_log->spam() << "Checking all required fields exist..." << std::endl;
for(int i = 0; i < dcc->get_num_inherited_fields(); ++i)
{
DCField *field = dcc->get_inherited_field(i);
if(field->is_required() && field->is_db() && !field->as_molecular_field())
{
if(dbo.fields.find(field) == dbo.fields.end())
{
if(!field->has_default_value())
{
m_log->error() << "Field " << field->get_name() << " missing when trying to create "
"object of type " << dcc->get_name();
resp.add_uint32(0);
send(resp);
return;
}
else
{
dbo.fields[field] = field->get_default_value();
}
}
}
}
// Create object in database
m_log->spam() << "Creating stored object..." << std::endl;
unsigned int do_id = m_db_engine->create_object(dbo);
if(do_id == 0 || do_id < m_min_id || do_id > m_max_id)
{
m_log->error() << "Ran out of DistributedObject ids while creating new object." << std::endl;
resp.add_uint32(0);
send(resp);
return;
}
m_log->spam() << "... created with ID: " << do_id << std::endl;
resp.add_uint32(do_id);
send(resp);
}
break;
case DBSERVER_SELECT_STORED_OBJECT_ALL:
{
unsigned int context = dgi.read_uint32();
Datagram resp;
resp.add_server_header(sender, m_control_channel, DBSERVER_SELECT_STORED_OBJECT_ALL_RESP);
resp.add_uint32(context);
unsigned int do_id = dgi.read_uint32();
DatabaseObject dbo;
if(m_db_engine->get_object(do_id, dbo))
{
resp.add_uint8(1);
resp.add_uint16(dbo.dc_id);
resp.add_uint16(dbo.fields.size());
for(auto it = dbo.fields.begin(); it != dbo.fields.end(); ++it)
{
resp.add_uint16(it->first->get_number());
resp.add_data(it->second);
}
}
else
{
resp.add_uint8(0);
}
send(resp);
}
break;
case DBSERVER_DELETE_STORED_OBJECT:
{
if(dgi.read_uint32() == DBSERVER_DELETE_STORED_OBJECT_VERIFY_CODE)
{
unsigned int do_id = dgi.read_uint32();
m_db_engine->delete_object(do_id);
m_log->debug() << "Deleted object with ID: " << do_id << std::endl;
}
else
{
m_log->warning() << "Wrong delete verify code." << std::endl;
}
}
break;
default:
m_log->error() << "Recieved unknown MsgType: " << msg_type << std::endl;
};
}
void handle_modify(DBClientBase *client, DBOperation *operation)
{
// First, we have to format our findandmodify.
BSONObjBuilder sets;
bool has_sets = false;
BSONObjBuilder unsets;
bool has_unsets = false;
for(auto it = operation->set_fields().begin();
it != operation->set_fields().end(); ++it) {
stringstream fieldname;
fieldname << "fields." << it->first->get_name();
if(it->second.empty()) {
unsets << fieldname.str() << true;
has_unsets = true;
} else {
DatagramPtr dg = Datagram::create();
dg->add_data(it->second);
DatagramIterator dgi(dg);
sets << fieldname.str() << bamboo2bson(it->first->get_type(), dgi)["_"];
has_sets = true;
}
}
BSONObjBuilder updates_b;
if(has_sets) {
updates_b << "$set" << sets.obj();
}
if(has_unsets) {
updates_b << "$unset" << unsets.obj();
}
BSONObj updates = updates_b.obj();
// Also format any criteria for the change:
BSONObjBuilder query_b;
query_b << "_id" << operation->doid();
for(auto it = operation->criteria_fields().begin();
it != operation->criteria_fields().end(); ++it) {
stringstream fieldname;
fieldname << "fields." << it->first->get_name();
if(it->second.empty()) {
query_b << fieldname.str() << BSON("$exists" << false);
} else {
DatagramPtr dg = Datagram::create();
dg->add_data(it->second);
DatagramIterator dgi(dg);
query_b << fieldname.str() << bamboo2bson(it->first->get_type(), dgi)["_"];
}
}
BSONObj query = query_b.obj();
m_log->trace() << "Performing updates to " << operation->doid()
<< ": " << updates << endl;
m_log->trace() << "Query is: " << query << endl;
BSONObj result;
bool success;
try {
success = client->runCommand(
m_db,
BSON("findandmodify" << "astron.objects"
<< "query" << query
<< "update" << updates),
result);
} catch(mongo::DBException &e) {
m_log->error() << "Unexpected error while modifying "
<< operation->doid() << ": " << e.what() << endl;
operation->on_failure();
return;
}
m_log->trace() << "Update result: " << result << endl;
BSONObj obj;
if(!success || result["value"].isNull()) {
// Okay, something didn't work right. If we had criteria, let's
// try to fetch the object without the criteria to see if it's a
// criteria mismatch or a missing DOID.
if(!operation->criteria_fields().empty()) {
try {
obj = client->findOne(m_obj_collection,
BSON("_id" << operation->doid()));
} catch(mongo::DBException &e) {
m_log->error() << "Unexpected error while modifying "
<< operation->doid() << ": " << e.what() << endl;
operation->on_failure();
return;
}
if(!obj.isEmpty()) {
// There's the problem. Now we can send back a snapshot:
DBObjectSnapshot *snap = format_snapshot(operation->doid(), obj);
if(snap && operation->verify_class(snap->m_dclass)) {
operation->on_criteria_mismatch(snap);
return;
} else {
// Something else weird happened with our snapshot;
// either the class wasn't recognized or it was the
// wrong class. Either way, an error has been logged,
// and we need to fail the operation.
operation->on_failure();
return;
}
}
}
// Nope, not that. We're missing the DOID.
m_log->error() << "Attempted to modify unknown DOID: "
<< operation->doid() << endl;
operation->on_failure();
return;
}
// If we've gotten to this point: Hooray! The change has gone
// through to the database.
// Let's, however, double-check our changes. (Specifically, we should
// run verify_class so that we know the frontend is happy with what
// kind of object we just modified.)
obj = result["value"].Obj();
try {
string dclass_name = obj["dclass"].String();
const dclass::Class *dclass = g_dcf->get_class_by_name(dclass_name);
if(!dclass) {
m_log->error() << "Encountered unknown database object: "
<< dclass_name << "(" << operation->doid() << ")" << endl;
} else if(operation->verify_class(dclass)) {
// Yep, it all checks out. Complete the operation:
operation->on_complete();
return;
}
} catch(mongo::DBException &e) { }
// If we've gotten here, something is seriously wrong. We've just
// mucked with an object without knowing the consequences! What have
// we done?! We've created an abomination in the database! Kill it!
// Kill it with fire!
// All we really can do to mitigate this is scream at the user (which
// the above verification has already done by now) and revert the
// object back to how it was when we found it.
// NOTE: This DOES have the potential for data loss, because we're
// wiping out any changes that conceivably could have happened
// between the findandmodify and now. In dev environments, (which we
// are probably in right now, if other components are making
// outlandish requests like this) this shouldn't be a huge issue.
m_log->trace() << "Reverting changes made to " << operation->doid() << endl;
try {
client->update(
m_obj_collection,
BSON("_id" << operation->doid()),
obj);
} catch(mongo::DBException &e) {
// Wow, we REALLY fail at life.
m_log->error() << "Could not revert corrupting changes to "
<< operation->doid() << ": " << e.what() << endl;
}
operation->on_failure();
}
