void handle_delete(DBClientBase *client, DBOperation *operation) { BSONObj result; bool success; try { success = client->runCommand( m_db, BSON("findandmodify" << "astron.objects" << "query" << BSON( "_id" << operation->doid()) << "remove" << true), result); } catch(mongo::DBException &e) { m_log->error() << "Unexpected error while deleting " << operation->doid() << ": " << e.what() << endl; operation->on_failure(); return; } m_log->trace() << "handle_delete: got response: " << result << endl; // If the findandmodify command failed, there wasn't anything there // to delete in the first place. if(!success || result["value"].isNull()) { m_log->error() << "Tried to delete non-existent doid " << operation->doid() << endl; operation->on_failure(); return; } free_doid(client, operation->doid()); operation->on_complete(); }
inline bool load(doid_t do_id, YAML::Node &document) { ifstream stream(filename(do_id)); document = YAML::Load(stream); if(!document.IsDefined() || document.IsNull()) { m_log->error() << "obj-" << do_id << " does not exist in database." << endl; return false; } if(!document["class"].IsDefined() || document["class"].IsNull()) { m_log->error() << filename(do_id) << " does not contain the 'class' key." << endl; return false; } if(!document["fields"].IsDefined() || document["fields"].IsNull()) { m_log->error() << filename(do_id) << " does not contain the 'fields' key." << endl; return false; } // Read object's DistributedClass string dc_name = document["class"].as<string>(); if(!g_dcf->get_class_by_name(dc_name)) { m_log->error() << "Class '" << dc_name << "', loaded from '" << filename(do_id) << "', does not exist." << endl; return false; } return true; }
bool ConfigGroup::validate(ConfigNode node) { if(!node.IsMap()) { if(m_name.length() > 0) { config_log.error() << "Section '" << m_path << "' has key/value config variables.\n"; } else { config_log.error() << "Config sections must be at root of config file.\n"; } return false; } bool ok = true; for(auto it = node.begin(); it != node.end(); ++it) { string key = it->first.as<std::string>(); auto found_var = m_variables.find(key); if(found_var != m_variables.end()) { rtest r = found_var->second; if(!r(node)) { config_log.info() << "In Section '" << m_path << "', attribute '" << key << "' did not match constraint (see error).\n"; ok = false; } continue; } auto found_grp = m_children.find(key); if(found_grp != m_children.end()) { if(!found_grp->second->validate(node[key])) { ok = false; } continue; } if(m_name.length() > 0) { config_log.error() << "Section '" << m_path << "' has no attribute named '" << key << "'.\n"; } else { config_log.error() << "Section '" << key << "' is not a valid config category.\n"; } ok = false; } return ok; }
bool KeyedConfigList::validate(ConfigNode node) { if(!node.IsSequence()) { config_log.error() << "Section '" << m_path << "' expects a list of values.\n"; return false; } int n = -1; bool ok = true; for(auto it = node.begin(); it != node.end(); ++it) { n += 1; if(!it->IsMap()) { config_log.error() << "The " << n << "th item of section '" << m_path << "' does not have key/value config variables.\n"; ok = false; continue; } ConfigNode element = *it; if(!element[m_key]) { config_log.error() << "The " << n << "th item of section '" << m_path << "' did not specify the attribute '" << m_key << "'.\n"; ok = false; continue; } string key = element[m_key].as<std::string>(); auto found_grp = m_children.find(key); if(found_grp == m_children.end()) { config_log.error() << "The value '" << key << "' is not valid for attribute '" << m_key << "' of section '" << m_path << "'.\n"; print_keys(); ok = false; continue; } if(!found_grp->second->validate(element)) { ok = false; } } return ok; }
void handle_create(DBClientBase *client, DBOperation *operation) { // First, let's convert the requested object into BSON; this way, if // a failure happens, it happens before we waste a doid. BSONObjBuilder fields; try { for(auto it = operation->set_fields().begin(); it != operation->set_fields().end(); ++it) { DatagramPtr dg = Datagram::create(); dg->add_data(it->second); DatagramIterator dgi(dg); fields << it->first->get_name() << bamboo2bson(it->first->get_type(), dgi)["_"]; } } catch(mongo::DBException &e) { m_log->error() << "While formatting " << operation->dclass()->get_name() << " for insertion: " << e.what() << endl; operation->on_failure(); return; } doid_t doid = assign_doid(client); if(doid == INVALID_DO_ID) { // The error will already have been emitted at this point, so // all that's left for us to do is fail silently: operation->on_failure(); return; } BSONObj b = BSON("_id" << doid << "dclass" << operation->dclass()->get_name() << "fields" << fields.obj()); m_log->trace() << "Inserting new " << operation->dclass()->get_name() << "(" << doid << "): " << b << endl; try { client->insert(m_obj_collection, b); } catch(mongo::DBException &e) { m_log->error() << "Cannot insert new " << operation->dclass()->get_name() << "(" << doid << "): " << e.what() << endl; operation->on_failure(); return; } operation->on_complete(doid); }
// This is used when the monotonic counter is exhausted: doid_t assign_doid_reuse(DBClientBase *client) { BSONObj result; bool success = client->runCommand( m_db, BSON("findandmodify" << "astron.globals" << "query" << BSON( "_id" << "GLOBALS" << "doid.free.0" << BSON("$exists" << true) ) << "update" << BSON( "$pop" << BSON("doid.free" << -1) )), result); // If the findandmodify command failed, the document either doesn't // exist, or we ran out of reusable doids. if(!success || result["value"].isNull()) { m_log->error() << "Could not allocate a reused DOID!" << endl; return INVALID_DO_ID; } m_log->trace() << "assign_doid_reuse: got globals element: " << result << endl; // Otherwise, use the first one: doid_t doid; const BSONElement &element = result["value"]["doid"]["free"]; if(sizeof(doid) == sizeof(long long)) { doid = element.Array()[0].Long(); } else if(sizeof(doid) == sizeof(int)) { doid = element.Array()[0].Int(); } return doid; }
void handle_get(DBClientBase *client, DBOperation *operation) { BSONObj obj; try { obj = client->findOne(m_obj_collection, BSON("_id" << operation->doid())); } catch(mongo::DBException &e) { m_log->error() << "Unexpected error occurred while trying to" " retrieve object with DOID " << operation->doid() << ": " << e.what() << endl; operation->on_failure(); return; } if(obj.isEmpty()) { m_log->warning() << "Got queried for non-existent object with DOID " << operation->doid() << endl; operation->on_failure(); return; } DBObjectSnapshot *snap = format_snapshot(operation->doid(), obj); if(!snap || !operation->verify_class(snap->m_dclass)) { operation->on_failure(); } else { operation->on_complete(snap); } }
void run_thread() { unique_lock<mutex> guard(m_lock); DBClientBase *client = new_connection(); while(true) { if(!client->isStillConnected()) { m_log->error() << "A thread lost its connection, reconnecting..." << endl; delete client; client = new_connection(); } if(m_operation_queue.size() > 0) { DBOperation *op = m_operation_queue.front(); m_operation_queue.pop(); guard.unlock(); handle_operation(client, op); guard.lock(); } else if(m_shutdown) { break; } else { m_cv.wait(guard); } } delete client; }
// Get a DBObjectSnapshot from a MongoDB BSON object; returns NULL if failure. DBObjectSnapshot *format_snapshot(doid_t doid, const BSONObj &obj) { m_log->trace() << "Formatting database snapshot of " << doid << ": " << obj << endl; try { string dclass_name = obj["dclass"].String(); const dclass::Class *dclass = g_dcf->get_class_by_name(dclass_name); if(!dclass) { m_log->error() << "Encountered unknown database object: " << dclass_name << "(" << doid << ")" << endl; return NULL; } BSONObj fields = obj["fields"].Obj(); DBObjectSnapshot *snap = new DBObjectSnapshot(); snap->m_dclass = dclass; for(auto it = fields.begin(); it.more(); ++it) { const char *name = (*it).fieldName(); const dclass::Field *field = dclass->get_field_by_name(name); if(!field) { m_log->warning() << "Encountered unexpected field " << name << " while formatting " << dclass_name << "(" << doid << "); ignored." << endl; continue; } { DatagramPtr dg = Datagram::create(); bson2bamboo(field->get_type(), *it, *dg); snap->m_fields[field].resize(dg->size()); memcpy(snap->m_fields[field].data(), dg->get_data(), dg->size()); } } return snap; } catch(mongo::DBException &e) { m_log->error() << "Unexpected error while trying to format" " database snapshot for " << doid << ": " << e.what() << endl; return NULL; } }
vector<uint8_t> read_yaml_field(const Field* field, YAML::Node node, doid_t id) { bool error; string packed_data = dclass::parse_value(field->get_type(), node.as<string>(), error); if(error) { m_log->error() << "Failed parsing value for field '" << field->get_name() << "' of object " << id << "' from database.\n"; return vector<uint8_t>(); } vector<uint8_t> result(packed_data.begin(), packed_data.end()); return result; }
// This returns a DOID to the free list: void free_doid(DBClientBase *client, doid_t doid) { m_log->trace() << "Returning doid " << doid << " to the free pool..." << endl; try { client->update( m_global_collection, BSON("_id" << "GLOBALS"), BSON("$push" << BSON("doid.free" << doid))); } catch(mongo::DBException &e) { m_log->error() << "Could not return doid " << doid << " to free pool: " << e.what() << endl; } }
bool ConfigList::validate(ConfigNode node) { if(!node.IsSequence()) { config_log.error() << "Section '" << m_path << "' expects a list of values.\n"; return false; } bool ok = true; for(auto it = node.begin(); it != node.end(); ++it) { if(!ConfigGroup::validate(*it)) { ok = false; } } return ok; }
// This function is used by handle_create to get a fresh DOID assignment. doid_t assign_doid(DBClientBase *client) { try { if(!m_monotonic_exhausted) { doid_t doid = assign_doid_monotonic(client); if(doid == INVALID_DO_ID) { m_monotonic_exhausted = true; } else { return doid; } } // We've exhausted our supply of doids from the monotonic counter. // We must now resort to pulling things out of the free list: return assign_doid_reuse(client); } catch(mongo::DBException &e) { m_log->error() << "Unexpected error occurred while trying to" " allocate a new DOID: " << e.what() << endl; return INVALID_DO_ID; } }
void get_all_from_table(doid_t id, const Class* dcc, FieldValues &fields) { string value; indicator ind; for(unsigned int i = 0; i < dcc->get_num_fields(); ++i) { const Field* field = dcc->get_field(i); if(field->has_keyword("db")) { m_sql << "SELECT " << field->get_name() << " FROM fields_" << dcc->get_name() << " WHERE object_id=" << id << ";", into(value, ind); if(ind == i_ok) { bool parse_err; string packed_data = parse_value(field->get_type(), value, parse_err); if(parse_err) { m_log->error() << "Failed parsing value for field '" << field->get_name() << "' of object " << id << "' from database.\n"; continue; } fields[field] = vector<uint8_t>(packed_data.begin(), packed_data.end()); } } } }
void config_error(const std::string& msg) { config_log.error() << msg << "\n"; }
virtual void handle_datagram(Datagram &in_dg, DatagramIterator &dgi) { channel_t sender = dgi.read_uint64(); unsigned short msg_type = dgi.read_uint16(); switch(msg_type) { case DBSERVER_CREATE_STORED_OBJECT: { unsigned int context = dgi.read_uint32(); // Start response with generic header Datagram resp; resp.add_server_header(sender, m_control_channel, DBSERVER_CREATE_STORED_OBJECT_RESP); resp.add_uint32(context); // Get DistributedClass unsigned short dc_id = dgi.read_uint16(); DCClass *dcc = gDCF->get_class(dc_id); if(!dcc) { m_log->error() << "Invalid DCClass when creating object: #" << dc_id << std::endl; resp.add_uint32(0); send(resp); return; } // Unpack fields to be passed to database DatabaseObject dbo(dc_id); unsigned short field_count = dgi.read_uint16(); m_log->spam() << "Unpacking fields..." << std::endl; try { for(unsigned int i = 0; i < field_count; ++i) { unsigned short field_id = dgi.read_uint16(); DCField *field = dcc->get_field_by_index(field_id); if(field) { if(field->is_db()) { dgi.unpack_field(field, dbo.fields[field]); } else { std::string tmp; dgi.unpack_field(field, tmp); } } } } catch(std::exception &e) { m_log->error() << "Error while unpacking fields, msg may be truncated. e.what(): " << e.what() << std::endl; resp.add_uint32(0); send(resp); return; } // Check for required fields, and populate with defaults m_log->spam() << "Checking all required fields exist..." << std::endl; for(int i = 0; i < dcc->get_num_inherited_fields(); ++i) { DCField *field = dcc->get_inherited_field(i); if(field->is_required() && field->is_db() && !field->as_molecular_field()) { if(dbo.fields.find(field) == dbo.fields.end()) { if(!field->has_default_value()) { m_log->error() << "Field " << field->get_name() << " missing when trying to create " "object of type " << dcc->get_name(); resp.add_uint32(0); send(resp); return; } else { dbo.fields[field] = field->get_default_value(); } } } } // Create object in database m_log->spam() << "Creating stored object..." << std::endl; unsigned int do_id = m_db_engine->create_object(dbo); if(do_id == 0 || do_id < m_min_id || do_id > m_max_id) { m_log->error() << "Ran out of DistributedObject ids while creating new object." << std::endl; resp.add_uint32(0); send(resp); return; } m_log->spam() << "... created with ID: " << do_id << std::endl; resp.add_uint32(do_id); send(resp); } break; case DBSERVER_SELECT_STORED_OBJECT_ALL: { unsigned int context = dgi.read_uint32(); Datagram resp; resp.add_server_header(sender, m_control_channel, DBSERVER_SELECT_STORED_OBJECT_ALL_RESP); resp.add_uint32(context); unsigned int do_id = dgi.read_uint32(); DatabaseObject dbo; if(m_db_engine->get_object(do_id, dbo)) { resp.add_uint8(1); resp.add_uint16(dbo.dc_id); resp.add_uint16(dbo.fields.size()); for(auto it = dbo.fields.begin(); it != dbo.fields.end(); ++it) { resp.add_uint16(it->first->get_number()); resp.add_data(it->second); } } else { resp.add_uint8(0); } send(resp); } break; case DBSERVER_DELETE_STORED_OBJECT: { if(dgi.read_uint32() == DBSERVER_DELETE_STORED_OBJECT_VERIFY_CODE) { unsigned int do_id = dgi.read_uint32(); m_db_engine->delete_object(do_id); m_log->debug() << "Deleted object with ID: " << do_id << std::endl; } else { m_log->warning() << "Wrong delete verify code." << std::endl; } } break; default: m_log->error() << "Recieved unknown MsgType: " << msg_type << std::endl; }; }
void handle_modify(DBClientBase *client, DBOperation *operation) { // First, we have to format our findandmodify. BSONObjBuilder sets; bool has_sets = false; BSONObjBuilder unsets; bool has_unsets = false; for(auto it = operation->set_fields().begin(); it != operation->set_fields().end(); ++it) { stringstream fieldname; fieldname << "fields." << it->first->get_name(); if(it->second.empty()) { unsets << fieldname.str() << true; has_unsets = true; } else { DatagramPtr dg = Datagram::create(); dg->add_data(it->second); DatagramIterator dgi(dg); sets << fieldname.str() << bamboo2bson(it->first->get_type(), dgi)["_"]; has_sets = true; } } BSONObjBuilder updates_b; if(has_sets) { updates_b << "$set" << sets.obj(); } if(has_unsets) { updates_b << "$unset" << unsets.obj(); } BSONObj updates = updates_b.obj(); // Also format any criteria for the change: BSONObjBuilder query_b; query_b << "_id" << operation->doid(); for(auto it = operation->criteria_fields().begin(); it != operation->criteria_fields().end(); ++it) { stringstream fieldname; fieldname << "fields." << it->first->get_name(); if(it->second.empty()) { query_b << fieldname.str() << BSON("$exists" << false); } else { DatagramPtr dg = Datagram::create(); dg->add_data(it->second); DatagramIterator dgi(dg); query_b << fieldname.str() << bamboo2bson(it->first->get_type(), dgi)["_"]; } } BSONObj query = query_b.obj(); m_log->trace() << "Performing updates to " << operation->doid() << ": " << updates << endl; m_log->trace() << "Query is: " << query << endl; BSONObj result; bool success; try { success = client->runCommand( m_db, BSON("findandmodify" << "astron.objects" << "query" << query << "update" << updates), result); } catch(mongo::DBException &e) { m_log->error() << "Unexpected error while modifying " << operation->doid() << ": " << e.what() << endl; operation->on_failure(); return; } m_log->trace() << "Update result: " << result << endl; BSONObj obj; if(!success || result["value"].isNull()) { // Okay, something didn't work right. If we had criteria, let's // try to fetch the object without the criteria to see if it's a // criteria mismatch or a missing DOID. if(!operation->criteria_fields().empty()) { try { obj = client->findOne(m_obj_collection, BSON("_id" << operation->doid())); } catch(mongo::DBException &e) { m_log->error() << "Unexpected error while modifying " << operation->doid() << ": " << e.what() << endl; operation->on_failure(); return; } if(!obj.isEmpty()) { // There's the problem. Now we can send back a snapshot: DBObjectSnapshot *snap = format_snapshot(operation->doid(), obj); if(snap && operation->verify_class(snap->m_dclass)) { operation->on_criteria_mismatch(snap); return; } else { // Something else weird happened with our snapshot; // either the class wasn't recognized or it was the // wrong class. Either way, an error has been logged, // and we need to fail the operation. operation->on_failure(); return; } } } // Nope, not that. We're missing the DOID. m_log->error() << "Attempted to modify unknown DOID: " << operation->doid() << endl; operation->on_failure(); return; } // If we've gotten to this point: Hooray! The change has gone // through to the database. // Let's, however, double-check our changes. (Specifically, we should // run verify_class so that we know the frontend is happy with what // kind of object we just modified.) obj = result["value"].Obj(); try { string dclass_name = obj["dclass"].String(); const dclass::Class *dclass = g_dcf->get_class_by_name(dclass_name); if(!dclass) { m_log->error() << "Encountered unknown database object: " << dclass_name << "(" << operation->doid() << ")" << endl; } else if(operation->verify_class(dclass)) { // Yep, it all checks out. Complete the operation: operation->on_complete(); return; } } catch(mongo::DBException &e) { } // If we've gotten here, something is seriously wrong. We've just // mucked with an object without knowing the consequences! What have // we done?! We've created an abomination in the database! Kill it! // Kill it with fire! // All we really can do to mitigate this is scream at the user (which // the above verification has already done by now) and revert the // object back to how it was when we found it. // NOTE: This DOES have the potential for data loss, because we're // wiping out any changes that conceivably could have happened // between the findandmodify and now. In dev environments, (which we // are probably in right now, if other components are making // outlandish requests like this) this shouldn't be a huge issue. m_log->trace() << "Reverting changes made to " << operation->doid() << endl; try { client->update( m_obj_collection, BSON("_id" << operation->doid()), obj); } catch(mongo::DBException &e) { // Wow, we REALLY fail at life. m_log->error() << "Could not revert corrupting changes to " << operation->doid() << ": " << e.what() << endl; } operation->on_failure(); }