inline bool load(doid_t do_id, YAML::Node &document)
{
ifstream stream(filename(do_id));
document = YAML::Load(stream);
if(!document.IsDefined() || document.IsNull())
{
m_log->error() << "obj-" << do_id << " does not exist in database." << endl;
return false;
}
if(!document["class"].IsDefined() || document["class"].IsNull())
{
m_log->error() << filename(do_id) << " does not contain the 'class' key." << endl;
return false;
}
if(!document["fields"].IsDefined() || document["fields"].IsNull())
{
m_log->error() << filename(do_id) << " does not contain the 'fields' key." << endl;
return false;
}
// Read object's DistributedClass
string dc_name = document["class"].as<string>();
if(!g_dcf->get_class_by_name(dc_name))
{
m_log->error() << "Class '" << dc_name << "', loaded from '" << filename(do_id)
<< "', does not exist." << endl;
return false;
}
return true;
}
void set_field(doid_t do_id, const Field* field, const val_t &value)
{
m_log->trace() << "Setting field on obj-" << do_id << endl;
YAML::Node document;
if(!load(do_id, document))
{
return;
}
// Get the fields from the file that are not being updated
const Class* dcc = g_dcf->get_class_by_name(document["class"].as<string>());
ObjectData dbo(dcc->get_id());
YAML::Node existing = document["fields"];
for(auto it = existing.begin(); it != existing.end(); ++it)
{
const Field* field = dcc->get_field_by_name(it->first.as<string>());
if(!field)
{
m_log->warning() << "Field '" << it->first.as<string>()
<< "', loaded from '" << filename(do_id)
<< "', does not exist." << endl;
continue;
}
vector<uint8_t> value = read_yaml_field(field, it->second, do_id);
if(value.size() > 0)
{
dbo.fields[field] = value;
}
}
dbo.fields[field] = value;
write_yaml_object(do_id, dcc, dbo);
}
GLDisplay gglCreateDisplay(::Display* display,int screen)
{
// Load and bind DLL functions
if (!_hGL) {
static LogCategory log("/Gfx/Device/GL");
_hGL = LoadLibrary("libGL.so");
if (!_hGL) {
log.print("GLDevice: OpenGL dll failed to load");
return false;
}
if (!gglBindCoreFunctions(_hGL.ptr(),&_LibraryFunctions)) {
log.print("GLDevice: Failed to bind all core functions");
return false;
}
_GGLptr = &_LibraryFunctions;
log.print("OpenGL library loaded");
}
//
GGLDisplay* dp = new GGLDisplay;
dp->display = display;
dp->screen = screen;
gglBindGLX(display,screen,&dp->glx);
return dp;
}
bool get_fields(doid_t do_id, const vector<const Field*> &fields, map_t &values)
{
m_log->trace() << "Getting fields on obj-" << do_id << endl;
YAML::Node document;
if(!load(do_id, document))
{
return false;
}
// Get the fields from the file that are not being updated
for(auto it = fields.begin(); it != fields.end(); ++it)
{
const Field* field = *it;
m_log->trace() << "Searching for field: " << field->get_name() << endl;
YAML::Node existing = document["fields"];
for(auto it2 = existing.begin(); it2 != existing.end(); ++it2)
{
if(it2->first.as<string>() == field->get_name())
{
vector<uint8_t> value = read_yaml_field(field, it2->second, do_id);
if(value.size() > 0)
{
values[*it] = value;
m_log->trace() << "Found requested field: " + field->get_name() << endl;
}
}
}
}
return true;
}
void handle_delete(DBClientBase *client, DBOperation *operation)
{
BSONObj result;
bool success;
try {
success = client->runCommand(
m_db,
BSON("findandmodify" << "astron.objects" <<
"query" << BSON(
"_id" << operation->doid()) <<
"remove" << true),
result);
} catch(mongo::DBException &e) {
m_log->error() << "Unexpected error while deleting "
<< operation->doid() << ": " << e.what() << endl;
operation->on_failure();
return;
}
m_log->trace() << "handle_delete: got response: "
<< result << endl;
// If the findandmodify command failed, there wasn't anything there
// to delete in the first place.
if(!success || result["value"].isNull()) {
m_log->error() << "Tried to delete non-existent doid "
<< operation->doid() << endl;
operation->on_failure();
return;
}
free_doid(client, operation->doid());
operation->on_complete();
}
// This is used when the monotonic counter is exhausted:
doid_t assign_doid_reuse(DBClientBase *client)
{
BSONObj result;
bool success = client->runCommand(
m_db,
BSON("findandmodify" << "astron.globals" <<
"query" << BSON(
"_id" << "GLOBALS" <<
"doid.free.0" << BSON("$exists" << true)
) <<
"update" << BSON(
"$pop" << BSON("doid.free" << -1)
)), result);
// If the findandmodify command failed, the document either doesn't
// exist, or we ran out of reusable doids.
if(!success || result["value"].isNull()) {
m_log->error() << "Could not allocate a reused DOID!" << endl;
return INVALID_DO_ID;
}
m_log->trace() << "assign_doid_reuse: got globals element: "
<< result << endl;
// Otherwise, use the first one:
doid_t doid;
const BSONElement &element = result["value"]["doid"]["free"];
if(sizeof(doid) == sizeof(long long)) {
doid = element.Array()[0].Long();
} else if(sizeof(doid) == sizeof(int)) {
doid = element.Array()[0].Int();
}
return doid;
}
void handle_get(DBClientBase *client, DBOperation *operation)
{
BSONObj obj;
try {
obj = client->findOne(m_obj_collection,
BSON("_id" << operation->doid()));
} catch(mongo::DBException &e) {
m_log->error() << "Unexpected error occurred while trying to"
" retrieve object with DOID "
<< operation->doid() << ": " << e.what() << endl;
operation->on_failure();
return;
}
if(obj.isEmpty()) {
m_log->warning() << "Got queried for non-existent object with DOID "
<< operation->doid() << endl;
operation->on_failure();
return;
}
DBObjectSnapshot *snap = format_snapshot(operation->doid(), obj);
if(!snap || !operation->verify_class(snap->m_dclass)) {
operation->on_failure();
} else {
operation->on_complete(snap);
}
}
bool get_field(doid_t do_id, const Field* field, val_t &value)
{
m_log->trace() << "Getting field on obj-" << do_id << endl;
YAML::Node document;
if(!load(do_id, document))
{
return false;
}
// Get the fields from the file that are not being updated
YAML::Node node = document["fields"][field->get_name()];
if(!node.IsDefined() || node.IsNull())
{
return false;
}
m_log->trace() << "Found requested field: " + field->get_name() << endl;
value = read_yaml_field(field, node, do_id);
if(value.size() > 0)
{
return true;
}
return false;
}
void check_class(uint16_t id, string name, unsigned long hash)
{
DCClass* dcc = g_dcf->get_class(id);
if(name != dcc->get_name())
{
// TODO: Try and update the database instead of exiting
m_log->fatal() << "Class name '" << dcc->get_name() << "' from DCFile does not match"
" name '" << name << "' in database, for dc_id " << id << endl;
m_log->fatal() << "Database must be rebuilt." << endl;
exit(1);
}
HashGenerator gen;
dcc->generate_hash(gen);
if(hash != gen.get_hash())
{
// TODO: Try and update the database instead of exiting
m_log->fatal() << "Class hash '" << gen.get_hash() << "' from DCFile does not match"
" hash '" << hash << "' in database, for dc_id " << id << endl;
m_log->fatal() << "Database must be rebuilt." << endl;
exit(1);
}
// TODO: Check class_fields table exists
}
bool ConfigGroup::validate(ConfigNode node)
{
if(!node.IsMap())
{
if(m_name.length() > 0)
{
config_log.error() << "Section '" << m_path
<< "' has key/value config variables.\n";
}
else
{
config_log.error() << "Config sections must be at root of config file.\n";
}
return false;
}
bool ok = true;
for(auto it = node.begin(); it != node.end(); ++it)
{
string key = it->first.as<std::string>();
auto found_var = m_variables.find(key);
if(found_var != m_variables.end())
{
rtest r = found_var->second;
if(!r(node))
{
config_log.info() << "In Section '" << m_path << "', attribute '"
<< key << "' did not match constraint (see error).\n";
ok = false;
}
continue;
}
auto found_grp = m_children.find(key);
if(found_grp != m_children.end())
{
if(!found_grp->second->validate(node[key]))
{
ok = false;
}
continue;
}
if(m_name.length() > 0)
{
config_log.error() << "Section '" << m_path
<< "' has no attribute named '" << key << "'.\n";
}
else
{
config_log.error() << "Section '" << key << "' is not a valid config category.\n";
}
ok = false;
}
return ok;
}
LogCategory* LoggerDB::createCategoryLocked(
LoggerNameMap& loggersByName,
StringPiece name,
LogCategory* parent) {
auto uptr = std::make_unique<LogCategory>(name, parent);
LogCategory* logger = uptr.get();
auto ret = loggersByName.emplace(logger->getName(), std::move(uptr));
DCHECK(ret.second);
return logger;
}
void handle_create(DBClientBase *client, DBOperation *operation)
{
// First, let's convert the requested object into BSON; this way, if
// a failure happens, it happens before we waste a doid.
BSONObjBuilder fields;
try {
for(auto it = operation->set_fields().begin();
it != operation->set_fields().end();
++it) {
DatagramPtr dg = Datagram::create();
dg->add_data(it->second);
DatagramIterator dgi(dg);
fields << it->first->get_name()
<< bamboo2bson(it->first->get_type(), dgi)["_"];
}
} catch(mongo::DBException &e) {
m_log->error() << "While formatting "
<< operation->dclass()->get_name()
<< " for insertion: " << e.what() << endl;
operation->on_failure();
return;
}
doid_t doid = assign_doid(client);
if(doid == INVALID_DO_ID) {
// The error will already have been emitted at this point, so
// all that's left for us to do is fail silently:
operation->on_failure();
return;
}
BSONObj b = BSON("_id" << doid <<
"dclass" << operation->dclass()->get_name() <<
"fields" << fields.obj());
m_log->trace() << "Inserting new " << operation->dclass()->get_name()
<< "(" << doid << "): " << b << endl;
try {
client->insert(m_obj_collection, b);
} catch(mongo::DBException &e) {
m_log->error() << "Cannot insert new "
<< operation->dclass()->get_name()
<< "(" << doid << "): " << e.what() << endl;
operation->on_failure();
return;
}
operation->on_complete(doid);
}
bool KeyedConfigList::validate(ConfigNode node)
{
if(!node.IsSequence())
{
config_log.error() << "Section '" << m_path
<< "' expects a list of values.\n";
return false;
}
int n = -1;
bool ok = true;
for(auto it = node.begin(); it != node.end(); ++it)
{
n += 1;
if(!it->IsMap())
{
config_log.error() << "The " << n << "th item of section '" << m_path
<< "' does not have key/value config variables.\n";
ok = false;
continue;
}
ConfigNode element = *it;
if(!element[m_key])
{
config_log.error() << "The " << n << "th item of section '" << m_path
<< "' did not specify the attribute '" << m_key << "'.\n";
ok = false;
continue;
}
string key = element[m_key].as<std::string>();
auto found_grp = m_children.find(key);
if(found_grp == m_children.end())
{
config_log.error() << "The value '" << key << "' is not valid for attribute '"
<< m_key << "' of section '" << m_path << "'.\n";
print_keys();
ok = false;
continue;
}
if(!found_grp->second->validate(element))
{
ok = false;
}
}
return ok;
}
void setup()
{
mdperf_log.info() << "Creating random data..." << std::endl;
data = new uint8_t[MD_PERF_DATASIZE];
data[0] = MD_PERF_NUM_DEST_CHANNELS;
for(uint32_t i = 1; i < MD_PERF_DATASIZE; ++i) {
data[i] = rand() % 256;
}
mdperf_log.info() << "Creating MDPerformanceParticipants" << std::endl;
m_participants = new MDPerformanceParticipant*[MD_PERF_NUM_PARTICIPANTS];
for(uint32_t i = 0; i < MD_PERF_NUM_PARTICIPANTS; ++i) {
m_participants[i] = new MDPerformanceParticipant;
}
}
void check_class(uint16_t id, string name)
{
const Class* dcc = g_dcf->get_class_by_id(id);
if(name != dcc->get_name()) {
// TODO: Try and update the database instead of exiting
m_log->fatal() << "Class name '" << dcc->get_name() << "' from File does not match"
" name '" << name << "' in database, for dc_id " << id << endl;
m_log->fatal() << "Database must be rebuilt." << endl;
astron_shutdown(1);
}
// TODO: Check class_fields table exists
}
// This returns a DOID to the free list:
void free_doid(DBClientBase *client, doid_t doid)
{
m_log->trace() << "Returning doid " << doid << " to the free pool..." << endl;
try {
client->update(
m_global_collection,
BSON("_id" << "GLOBALS"),
BSON("$push" << BSON("doid.free" << doid)));
} catch(mongo::DBException &e) {
m_log->error() << "Could not return doid " << doid
<< " to free pool: " << e.what() << endl;
}
}
DatabaseServer(RoleConfig roleconfig) : Role(roleconfig),
m_db_engine(DBEngineFactory::singleton.instantiate(
engine_type.get_rval(roleconfig),
roleconfig["engine"],
min_id.get_rval(roleconfig),
max_id.get_rval(roleconfig))),
m_control_channel(control_channel.get_rval(roleconfig)),
m_min_id(min_id.get_rval(roleconfig)),
m_max_id(max_id.get_rval(roleconfig))
{
// Initialize DatabaseServer log
std::stringstream log_title;
log_title << "Database(" << m_control_channel << ")";
m_log = new LogCategory("db", log_title.str());
// Check to see the engine was instantiated
if(!m_db_engine)
{
m_log->fatal() << "No database engine of type '" << engine_type.get_rval(roleconfig) << "' exists." << std::endl;
exit(1);
}
// Listen on control channel
subscribe_channel(m_control_channel);
}
doid_t assign_doid_monotonic(DBClientBase *client)
{
BSONObj result;
bool success = client->runCommand(
m_db,
BSON("findandmodify" << "astron.globals" <<
"query" << BSON(
"_id" << "GLOBALS" <<
"doid.monotonic" << GTE << m_min_id <<
"doid.monotonic" << LTE << m_max_id
) <<
"update" << BSON(
"$inc" << BSON("doid.monotonic" << 1)
)), result);
// If the findandmodify command failed, the document either doesn't
// exist, or we ran out of monotonic doids.
if(!success || result["value"].isNull()) {
return INVALID_DO_ID;
}
m_log->trace() << "assign_doid_monotonic: got globals element: "
<< result << endl;
doid_t doid;
const BSONElement &element = result["value"]["doid"]["monotonic"];
if(sizeof(doid) == sizeof(long long)) {
doid = element.Long();
} else if(sizeof(doid) == sizeof(int)) {
doid = element.Int();
}
return doid;
}
void run_thread()
{
unique_lock<mutex> guard(m_lock);
DBClientBase *client = new_connection();
while(true) {
if(!client->isStillConnected()) {
m_log->error() << "A thread lost its connection, reconnecting..." << endl;
delete client;
client = new_connection();
}
if(m_operation_queue.size() > 0) {
DBOperation *op = m_operation_queue.front();
m_operation_queue.pop();
guard.unlock();
handle_operation(client, op);
guard.lock();
} else if(m_shutdown) {
break;
} else {
m_cv.wait(guard);
}
}
delete client;
}
void cleanup()
{
mdperf_log.info() << "Cleaning up..." << std::endl;
for(uint32_t i = 0; i < MD_PERF_NUM_PARTICIPANTS; ++i) {
delete m_participants[i];
}
delete [] m_participants;
delete [] data;
}
void speed_test()
{
mdperf_log.info() << "Starting speed test I..." << std::endl;
clock_t startTime = clock();
while((clock() - startTime) / CLOCKS_PER_SEC < MD_PERF_TIME) {
for(uint32_t i = 0; i < MD_PERF_NUM_PARTICIPANTS; ++i) {
m_participants[i]->spam();
}
}
mdperf_log.info() << "Test over. Averaging messages..." << std::endl;
double num_messages = 0;
for(uint32_t i = 0; i < MD_PERF_NUM_PARTICIPANTS; ++i) {
num_messages += double(m_participants[i]->num_messages) / double(MD_PERF_NUM_PARTICIPANTS);
}
mdperf_log.info() << "An average of " << num_messages << " messages were processed. "
"this comes out to be " << num_messages / MD_PERF_TIME << " messages/second" << std::endl;
}
void delete_object(doid_t do_id)
{
m_log->debug() << "Deleting file: " << filename(do_id) << endl;
if(!remove(filename(do_id).c_str()))
{
m_free_ids.insert(m_free_ids.end(), do_id);
update_info();
}
}
bool set_field_if_equals(doid_t do_id, const Field* field, const val_t &equal, val_t &value)
{
m_log->trace() << "Setting field if equal on obj-" << do_id << endl;
YAML::Node document;
if(!load(do_id, document))
{
value = vector<uint8_t>();
return false;
}
// Get current field values from the file
const Class* dcc = g_dcf->get_class_by_name(document["class"].as<string>());
ObjectData dbo(dcc->get_id());
YAML::Node existing = document["fields"];
for(auto it = existing.begin(); it != existing.end(); ++it)
{
const Field* field = dcc->get_field_by_name(it->first.as<string>());
if(!field)
{
m_log->warning() << "Field '" << it->first.as<string>()
<< "', loaded from '" << filename(do_id)
<< "', does not exist." << endl;
continue;
}
vector<uint8_t> value = read_yaml_field(field, it->second, do_id);
if(value.size() > 0)
{
dbo.fields[field] = value;
}
}
auto found = dbo.fields.find(field);
if(found == dbo.fields.end() || found->second != equal)
{
value = dbo.fields[field];
return false;
}
dbo.fields[field] = value;
write_yaml_object(do_id, dcc, dbo);
return true;
}
MDPerformanceTest()
{
mdperf_log.info() << "Starting perf test..." << std::endl;
setup();
speed_test();
cleanup();
setup();
speed_test_no_memcpy();
cleanup();
}
// Get a DBObjectSnapshot from a MongoDB BSON object; returns NULL if failure.
DBObjectSnapshot *format_snapshot(doid_t doid, const BSONObj &obj)
{
m_log->trace() << "Formatting database snapshot of " << doid << ": "
<< obj << endl;
try {
string dclass_name = obj["dclass"].String();
const dclass::Class *dclass = g_dcf->get_class_by_name(dclass_name);
if(!dclass) {
m_log->error() << "Encountered unknown database object: "
<< dclass_name << "(" << doid << ")" << endl;
return NULL;
}
BSONObj fields = obj["fields"].Obj();
DBObjectSnapshot *snap = new DBObjectSnapshot();
snap->m_dclass = dclass;
for(auto it = fields.begin(); it.more(); ++it) {
const char *name = (*it).fieldName();
const dclass::Field *field = dclass->get_field_by_name(name);
if(!field) {
m_log->warning() << "Encountered unexpected field " << name
<< " while formatting " << dclass_name
<< "(" << doid << "); ignored." << endl;
continue;
}
{
DatagramPtr dg = Datagram::create();
bson2bamboo(field->get_type(), *it, *dg);
snap->m_fields[field].resize(dg->size());
memcpy(snap->m_fields[field].data(), dg->get_data(), dg->size());
}
}
return snap;
} catch(mongo::DBException &e) {
m_log->error() << "Unexpected error while trying to format"
" database snapshot for " << doid << ": "
<< e.what() << endl;
return NULL;
}
}
void KeyedConfigList::print_keys()
{
ostream& out = config_log.info();
out << "Expected value in '" << m_name << "',\n"
<< " Candidates for attribute '" << m_key << "' are:\n";
for(auto it = m_children.begin(); it != m_children.end(); ++it)
{
out << " " << it->second->get_name() << "\n";
}
out << "\n";
}
void LogCategory::updateEffectiveLevel(LogLevel newEffectiveLevel) {
auto oldEffectiveLevel =
effectiveLevel_.exchange(newEffectiveLevel, std::memory_order_acq_rel);
// Break out early if the value did not change.
if (newEffectiveLevel == oldEffectiveLevel) {
return;
}
// Update all of the values in xlogLevels_
for (auto* levelPtr : xlogLevels_) {
levelPtr->store(newEffectiveLevel, std::memory_order_release);
}
// Update all children loggers
LogCategory* child = firstChild_;
while (child != nullptr) {
child->parentLevelUpdated(newEffectiveLevel);
child = child->nextSibling_;
}
}
void ConfigGroup::add_variable(const string& varname, rtest r)
{
bool inserted = m_variables.insert(pair<string,rtest>(varname, r)).second;
if(!inserted)
{
config_log.fatal() << "Duplicate ConfigVariable name (" << varname << ") in ConfigGroup '"
<< m_name << ".'\n\tPlease submit a bug/issue to Astron with your"
<< " CMakeCache and this ouput.\n";
exit(1);
// TODO: Produce a warning for the developer or something...
}
}
DBClientBase *new_connection()
{
string error;
DBClientBase *connection;
if(!(connection = m_connection_string.connect(error))) {
m_log->fatal() << "Connection failure: " << error << endl;
exit(1);
}
return connection;
}
void gglBindGLX(::Display* display,int screen,GLXExtensionFlags* glx)
{
// Check GLX version and glx extensions
int glxMajor,glxMinor;
glXQueryVersion(display,&glxMajor,&glxMinor);
const char* glxExtensions = glXQueryExtensionsString(display,screen);
static LogCategory log("/Gfx/Device/GL");
log.print(format("GLX Version: %d.%d",glxMajor,glxMinor));
Assert(glxMajor == 1 && glxMinor >= 1,"GLXBind: Need GLX version 1.1 or greater");
#define GL_GROUP_BEGIN(name) \
glx->has_##name = hasVersion(#name,"GLX_VERSION",glxMajor,glxMinor) || \
hasExtension(#name,glxExtensions);
#define GL_FUNCTION(fn_name, fn_return, fn_args)
#define GL_GROUP_END()
#include "../generated/glxefn.h"
#undef GL_FUNCTION
#undef GL_GROUP_BEGIN
#undef GL_GROUP_END
}
