T_Status C_DDI_Skills::Output_Entry(T_XML_Node root, T_Skill_Info * info)
{
T_Status status = LWD_ERROR;
T_Int32S result;
T_Glyph skill_id[UNIQUE_ID_LENGTH+1], buffer[500];
/* Generate a unique id for the class - if we can't, get out
*/
if (!Generate_Thing_Id(skill_id, "sk", info->name, &m_id_list)) {
sprintf(buffer, "Error generating unique id for skill %s\n", info->name);
Log_Message(buffer);
goto cleanup_exit;
}
info->id = m_pool->Acquire(skill_id);
/* Now that all our required information has been discerned, we can go
ahead and create the node
*/
result = Create_Thing_Node(root, m_term, "Skill", info->name, skill_id, info->description,
UNIQUENESS_UNIQUE, info->source, &info->node, info->is_partial);
if (x_Trap_Opt(result != 0))
goto cleanup_exit;
/* Add this skill to our internal list of skills
*/
Mapping_Add("skill", info->name, m_pool->Acquire(skill_id));
/* If we don't have a password, we can't get any more data
*/
if (info->is_partial) {
status = SUCCESS;
goto cleanup_exit;
}
/* Add a link for this skill's linked attribute
*/
Output_Link(info->node, "attribute", info->attribute, info, "attr");
status = SUCCESS;
cleanup_exit:
x_Status_Return(status);
}
static T_Status Finish_Document(T_XML_Document document, T_Glyph_Ptr output_folder,
T_Glyph_Ptr filename)
{
T_Int32S result;
T_Glyph buffer[MAX_FILE_NAME+1];
sprintf(buffer, "%s" DIR "%s", output_folder, filename);
Backup_Existing(buffer);
/* Output the document into our output folder
*/
result = XML_Write_Document(document, buffer, false, true);
if (x_Trap_Opt(result != 0)) {
sprintf(buffer, "Could not write %s XML document.", filename);
Log_Message(buffer);
x_Status_Return(LWD_ERROR);
}
x_Status_Return_Success();
}
void KeyspaceConn::Init(KeyspaceDB* kdb_, KeyspaceServer* server_)
{
Log_Trace();
TCPConn<KEYSPACE_BUF_SIZE>::Init();
if (!running)
Log_Trace("KeyspaceConn::Init(): running == false");
running = true;
KeyspaceService::Init(kdb_);
server = server_;
closeAfterSend = false;
bytesRead = 0;
Endpoint endpoint;
socket.GetEndpoint(endpoint);
endpoint.ToString(endpointString);
Log_Message("[%s] Keyspace: client connected", endpointString);
}
T_Void_Ptr Regex_Create(T_Glyph_Ptr pattern)
{
T_Regexp * rx;
int result;
T_Glyph message[500];
rx = new T_Regexp;
if (x_Trap_Opt(rx == NULL))
return(NULL);
result = regcomp(&rx->rx, pattern, REG_EXTENDED);
if (x_Trap_Opt(result != 0)) {
sprintf(message, "Couldn't create regular expression: error %d\n", result);
Log_Message(message, TRUE);
delete rx;
return(NULL);
}
return(rx);
}
void ShardMigrationWriter::Begin(ClusterMessage& request)
{
ConfigState* configState;
ConfigShard* configShard;
ConfigShardServer* configShardServer;
ASSERT(!isActive);
ASSERT(cursor == NULL);
configState = shardServer->GetConfigState();
configShard = configState->GetShard(request.srcShardID);
ASSERT(configShard != NULL);
configShardServer = configState->GetShardServer(request.nodeID);
ASSERT(configShardServer != NULL);
quorumProcessor = shardServer->GetQuorumProcessor(configShard->quorumID);
ASSERT(quorumProcessor != NULL);
isActive = true;
nodeID = request.nodeID;
quorumID = request.quorumID;
srcShardID = request.srcShardID;
dstShardID = request.dstShardID;
bytesTotal = environment->GetSize(QUORUM_DATABASE_DATA_CONTEXT, srcShardID);
startTime = NowClock();
CONTEXT_TRANSPORT->AddNode(nodeID, configShardServer->endpoint);
Log_Debug("ShardMigrationWriter::Begin() nodeID = %U", nodeID);
Log_Debug("ShardMigrationWriter::Begin() quorumID = %U", quorumID);
Log_Debug("ShardMigrationWriter::Begin() srcShardID = %U", srcShardID);
Log_Debug("ShardMigrationWriter::Begin() dstShardID = %U", dstShardID);
Log_Message("Migrating shard %U into quorum %U as shard %U (sending)",
srcShardID, quorumID, dstShardID);
sendFirst = true;
EventLoop::Add(&onTimeout);
CONTEXT_TRANSPORT->RegisterWriteReadyness(nodeID, MFUNC(ShardMigrationWriter, OnWriteReadyness));
}
T_Status C_DDI_Backgrounds::Output_Entry(T_XML_Node root, T_Background_Info * info)
{
T_Status status = LWD_ERROR;
T_Int32S result;
T_Glyph background_id[UNIQUE_ID_LENGTH+1], buffer[10000];
/* Generate a unique id for the background - if we can't, get out
*/
if (!Generate_Thing_Id(background_id, "bg", info->name, &m_id_list)) {
sprintf(buffer, "Error generating unique id for background %s\n", info->name);
Log_Message(buffer);
goto cleanup_exit;
}
info->id = m_pool->Acquire(background_id);
/* Now that all our required information has been discerned, we can go
ahead and create the node
*/
result = Create_Thing_Node(root, m_term, "Background", info->name, background_id, info->description,
UNIQUENESS_UNIQUE, info->source, &info->node, info->is_partial);
if (x_Trap_Opt(result != 0))
goto cleanup_exit;
if ((info->type != NULL) && (info->type[0] != '\0'))
Output_Tag(info->node, "BackType", info->type, info, "backtype", NULL, true, true);
if ((info->campaign != NULL) && (info->campaign[0] != '\0'))
Output_Tag(info->node, "BackCamp", info->campaign, info, "backcamp", NULL, true, true);
Output_Options(info, true, &m_id_list, m_pool);
Output_Options(info, false, &m_id_list, m_pool);
/* Add this path to our internal list of backgrounds
*/
Mapping_Add("background", info->name, m_pool->Acquire(background_id));
status = SUCCESS;
cleanup_exit:
x_Status_Return(status);
}
static void Output_Key_Skills(T_Ritual_Info * info)
{
T_Int32U i, length, temp_length, chunk_count;
T_Glyph_Ptr ptr, temp, chunks[1000];
T_Glyph buffer[100000], message[1000];
if ((info->keyskill == NULL) || (info->keyskill[0] == '\0')) {
sprintf(message, "No key skill present for ritual %s\n", info->name);
Log_Message(message);
return;
}
/* First, check for '(no check)' at the end
*/
temp = "(no check)";
temp_length = strlen(temp);
ptr = info->keyskill;
length = strlen(ptr);
if (temp_length < length) {
ptr = info->keyskill + length - temp_length;
if (strcmp(ptr, temp) == 0) {
Output_Tag(info->node, "RitualSkl", "NoCheck", info);
while ((ptr > info->keyskill) && (ptr[-1] == ' '))
ptr--;
*ptr = '\0';
}
}
chunk_count = x_Array_Size(chunks);
Split_To_Array(buffer, chunks, &chunk_count, info->keyskill, ", ");
for (i = 0; i < chunk_count; i++) {
/* If this is 'or', ignore it
*/
if (strcmp(chunks[i], "or") == 0)
continue;
Output_Tag(info->node, "RitualSkl", chunks[i], info, "skill");
}
}
void ConfigQuorumContext::OnPaxosMessage(ReadBuffer buffer)
{
PaxosMessage msg;
if (!isReplicationActive)
return;
msg.Read(buffer);
if (msg.IsPaxosRequest() || msg.IsPaxosResponse() || msg.IsLearn())
{
if (!quorum.IsMember(msg.nodeID))
{
Log_Message("Dropping paxos msg from %U because that node is not a quourm member", msg.nodeID);
OnMessageProcessed();
return;
}
}
RegisterPaxosID(msg.paxosID);
replicatedLog.RegisterPaxosID(msg.paxosID, msg.nodeID);
replicatedLog.OnMessage(msg);
}
static void Delete_Generated_Files(T_Glyph_Ptr output_folder)
{
T_Glyph choice;
T_Filename wildcard;
/* Make sure the user knows what they're doing.
*/
printf("The following files will be deleted:\n");
strcpy(wildcard, output_folder);
strcat(wildcard, DIR "ddi_*.*");
FileSys_List_Files(wildcard);
printf("\nTo continue, press 'y', or 'n' to cancel.\n\n");
choice = Get_Character();
if (tolower(choice) != 'y') {
printf("Cancelled.\n\n");
return;
}
/* Delete all files in the folder with "ddi_" at the start of the name
*/
FileSys_Delete_Files(wildcard);
Log_Message("All output files deleted.\n", true);
}
void ReplicatedLog::TryAppendDummy()
{
Log_Trace();
proposer.SetUseTimeouts(true);
if (proposer.IsActive() || proposer.IsLearnSent())
{
appendDummyNext = true;
return;
}
if (waitingOnAppend)
{
appendDummyNext = true;
Log_Message("TryAppendDummy called, but waitingOnAppend = true");
return;
}
Append(dummy);
#ifdef RLOG_DEBUG_MESSAGES
Log_Debug("Appending DUMMY!");
#endif
}
T_Status C_DDI_Rituals::Output_Entry(T_XML_Node root, T_Ritual_Info * info)
{
T_Status status = LWD_ERROR;
T_Int32S result;
T_Glyph_Ptr ptr, src;
bool is_level;
T_Glyph ritual_id[UNIQUE_ID_LENGTH+1], buffer[5000];
/* Generate a unique id for the ritual - if we can't, get out
*/
if (!Generate_Thing_Id(ritual_id, "rt", info->name, &m_id_list)) {
sprintf(buffer, "Error generating unique id for ritual %s\n", info->name);
Log_Message(buffer);
goto cleanup_exit;
}
info->id = m_pool->Acquire(ritual_id);
/* Now that all our required information has been discerned, we can go
ahead and create the node
*/
result = Create_Thing_Node(root, m_term, "Ritual", info->name, ritual_id, info->description,
UNIQUENESS_NONE, info->source, &info->node, info->is_partial);
if (x_Trap_Opt(result != 0))
goto cleanup_exit;
is_level = false;
if (!x_Is_Digit(info->level[0]))
ptr = info->level;
else {
is_level = true;
Output_Tag(info->node, "ReqLevel", As_Int(atoi(info->level)), info);
ptr = strchr(info->level, ' ');
while ((ptr != NULL) && (*ptr == ' '))
ptr++;
}
if ((ptr != NULL) && (ptr[0] != '\0'))
Output_Field(info->node, "ritLevText", ptr, info);
else if (!is_level) {
sprintf(buffer, "No ritual level found for ritual %s\n", info->name);
Log_Message(buffer);
}
Output_Field(info->node, "ritCompCst", info->componentcost, info);
Output_Key_Skills(info);
/* If we don't have a password, we can't get any more data
*/
if (info->is_partial) {
status = SUCCESS;
goto cleanup_exit;
}
Split_To_Dynamic_Tags(info->node, info, info->category, ",", "RitualCat", "ritualcat");
Output_Field(info->node, "ritTime", info->time, info);
if ((info->duration != NULL) && (info->duration[0] != '\0'))
Output_Field(info->node, "ritLasts", info->duration, info);
if (x_Is_Digit(info->price[0])) {
strcpy(buffer, info->price);
ptr = buffer;
src = buffer;
while (*src != '\0') {
if (*src != ',')
*ptr++ = *src;
src++;
}
*ptr = '\0';
Output_Field(info->node, "ritPrice", As_Int(atoi(buffer)), info);
}
else if ((strcmp(info->price, "Unique") != 0) && (strcmp(info->price, "-") != 0) &&
(strcmp(info->price, "None") != 0)) {
sprintf(buffer, "Invalid price '%s' found for ritual %s\n", info->price, info->name);
Log_Message(buffer);
goto cleanup_exit;
}
status = SUCCESS;
cleanup_exit:
x_Status_Return(status);
}
static T_XML_Document Load_Mappings(T_Glyph_Ptr folder)
{
long result;
T_XML_Document document;
T_XML_Node root, map, tuple;
T_Glyph_Ptr ptr;
T_Mapping * mapping;
T_Tuple t;
T_Filename filename;
/* Load the mappings XML file
*/
sprintf(filename, "%s" DIR "ddidownloader" DIR "mapping.xml", folder);
result = XML_Read_Document(&document, &l_mapping, filename);
if (x_Trap_Opt(result != 0))
return(NULL);
result = XML_Get_Document_Node(document, &root);
if (x_Trap_Opt(result != 0))
return(NULL);
/* Go through all our mappings
*/
result = XML_Get_First_Named_Child(root, "map", &map);
while (result == 0) {
/* Get the unique id for this mapping
*/
ptr = (T_Glyph_Ptr) XML_Get_Attribute_Pointer(map, "name");
if (!UniqueId_Is_Valid(ptr)) {
Log_Message("**** Invalid unique id found for mapping!\n", true);
goto next_map;
}
/* Allocate a new mapping structure
*/
mapping = new T_Mapping;
if (x_Trap_Opt(mapping == NULL))
goto next_map;
mapping->id = UniqueId_From_Text(ptr);
/* Add each tuple to the list
*/
result = XML_Get_First_Named_Child(map, "tuple", &tuple);
while (result == 0) {
t.a = (T_Glyph_Ptr) XML_Get_Attribute_Pointer(tuple, "a");
if (x_Trap_Opt((t.a == NULL) || (t.a[0] == '\0'))) {
Log_Message("**** Empty tuple found!\n", true);
goto next_tuple;
}
t.b = (T_Glyph_Ptr) XML_Get_Attribute_Pointer(tuple, "b");
if ((t.b != NULL) && (t.b[0] == '\0'))
t.b = NULL;
t.c = (T_Glyph_Ptr) XML_Get_Attribute_Pointer(tuple, "c");
if ((t.c != NULL) && (t.c[0] == '\0'))
t.c = NULL;
mapping->list.push_back(t);
next_tuple:
result = XML_Get_Next_Named_Child(map, &tuple);
}
/* Add our new mapping structure to the main list
*/
l_mappings.push_back(mapping);
next_map:
result = XML_Get_Next_Named_Child(root, &map);
}
/* Return the completed XML document, since it's already storing all the
strings we're using - this way we don't have to allocate new space
for them, we just throw the document away before we exit
*/
return(document);
}
int main(int argc,char ** argv)
{
T_Status status;
T_Int32S result = 0;
bool use_cache, is_command_line, is_clear = true;
E_Query_Mode mode;
T_Filename output_folder, folder, logfile;
T_XML_Document mappings = NULL;
T_Glyph xml_errors[1000], email[1000], password[1000];
/* On windows, get the current directory to use as the output folder. On the
mac, the current directory could well be the user's home directory, so
get the directory we were launched from
*/
#ifdef _WIN32
FileSys_Get_Current_Directory(output_folder);
#elif defined(_OSX)
strcpy(output_folder, argv[0]);
T_Glyph_Ptr ptr;
ptr = strrchr(output_folder, DIR[0]);
if (x_Trap_Opt(ptr == NULL)) {
printf("Error getting path of executable.");
exit(1);
}
*++ptr = '\0';
#else
#error Unknown platform!
#endif
/* Set up our static output stream for logging
*/
sprintf(logfile, "%s" DIR "output.txt", output_folder);
ofstream output(logfile,ios::out);
if (output.good())
Initialize_Helper(&output);
else {
Log_Message("Error opening output log file.", true);
goto cleanup_exit;
}
/* Set up an error buffer for our XML library
*/
XML_Set_Error_Buffer(xml_errors);
/* Initialize our unique id mechanisms
*/
UniqueId_Initialize();
/* Verify that we have write privileges in our output folder - if not, then
downloading stuff would be a bit of a waste of time (this can happen if
the downloader needs to be run with administrator rights).
*/
if (!FileSys_Verify_Write_Privileges(output_folder)) {
status = LWD_ERROR;
goto cleanup_exit;
}
/* Load up our mappings from the mapping XML file
*/
mappings = Load_Mappings(output_folder);
if (x_Trap_Opt(mappings == NULL)) {
Log_Message("Could not open mapping file.\n", true);
goto cleanup_exit;
}
/* Set up some sensible defaults
*/
email[0] = '\0';
password[0] = '\0';
use_cache = false;
/* If our first parameter is the secret "don't delete stuff" code, unset
the flag that causes everything to be deleted
*/
if ((argc > 1) && (stricmp(argv[1], "-nodelete") == 0))
is_clear = false;
/* Ask the user how the program is going to run - if we're told to exit,
just get out now
*/
mode = Query_Mode(argc, argv, &is_command_line);
if (mode == e_mode_exit)
goto cleanup_exit;
/* If we want to download with no password, do nothing - this is the
normal way we'll work
*/
if (mode == e_mode_no_password)
/* do nothing */;
/* If we just want to reprocess stuff without downloading, set our 'use cache'
flag, and don't clear the data after processing it
*/
else if (mode == e_mode_process) {
use_cache = true;
is_clear = false;
}
/* If we just want to append extensions, do that
*/
else if (mode == e_mode_append) {
Get_Temporary_Folder(folder);
Append_Extensions(folder, output_folder);
status = SUCCESS;
goto cleanup_exit;
}
/* If we want to download with no password, get the username and password
*/
else if (mode == e_mode_password) {
Get_Credentials(email, password);
if ((email[0] == '\0') || (password[0] == '\0')) {
Log_Message("Empty username or password - cancelling download.\n", true);
goto cleanup_exit;
}
}
/* Restore mode
*/
else if (mode == e_mode_restore) {
Restore_Old_Files(output_folder);
Log_Message("\nThe old versions of your files were restored.\nYou can use this command again to switch back to the \"new\" versions.\n", true);
goto cleanup_exit;
}
/* If we just want to delete stuff, delete it
*/
else if (mode == e_mode_delete) {
Delete_Generated_Files(output_folder);
goto cleanup_exit;
}
/* Otherwise, something is wrong
*/
else {
x_Trap_Opt(1);
Log_Message("Unrecognised option - exiting.\n", true);
goto cleanup_exit;
}
/* Crawl everything
*/
status = Crawl_Data(use_cache, email, password, output_folder, is_clear);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
if (status == SUCCESS)
Log_Message("\n\nImport complete. Reload the game system to see the new content.\n\n", true);
else
Log_Message("\n\nThere were one or more problems with the import. Please see the manual for more details.\n\n", true);
/* wrapup everything
*/
cleanup_exit:
for (map_iter it = l_mappings.begin(); it != l_mappings.end(); ++it)
delete *it;
if (mappings != NULL)
XML_Destroy_Document(mappings);
Shutdown_Helper();
/* The window doesn't auto-close on OS X, so we don't need this
*/
#ifndef _OSX
if (!is_command_line) {
printf("Press any key to quit.\n\n");
Get_Character();
}
#endif
if (!x_Is_Success(status))
result = -1;
return(result);
}
static void Fixup_Wizard_Powers(T_Glyph_Ptr folder)
{
T_Int32U count;
long result;
T_XML_Node root_old, root_new, node, tag;
T_XML_Document doc_old = NULL, doc_new = NULL;
bool is_match;
T_XML_Iter iter;
T_Glyph buffer[500];
T_Filename file_old, file_new;
T_XML_Vector nodes;
/* Open the previous version of the powers file - if we don't have one,
there's nothing to restore.
*/
sprintf(file_old, "%s" DIR "%s%s", folder, POWERS_FILENAME, BACKUP_EXTENSION);
if (!FileSys_Does_File_Exist(file_old))
goto cleanup_exit;
result = XML_Read_Document(&doc_old, &l_data, file_old);
if (x_Trap_Opt(result != 0)) {
Log_Message("Could not read old powers XML document.\n", true);
goto cleanup_exit;
}
result = XML_Get_Document_Node(doc_old, &root_old);
if (x_Trap_Opt(result != 0))
goto cleanup_exit;
/* Open the new powers file
*/
sprintf(file_new, "%s" DIR "%s", folder, POWERS_FILENAME);
if (!FileSys_Does_File_Exist(file_new))
goto cleanup_exit;
result = XML_Read_Document(&doc_new, &l_data, file_new);
if (x_Trap_Opt(result != 0)) {
Log_Message("Could not read new powers XML document.\n", true);
goto cleanup_exit;
}
result = XML_Get_Document_Node(doc_new, &root_new);
if (x_Trap_Opt(result != 0))
goto cleanup_exit;
/* Go through the old powers file, finding all powers (things from the
"Power" compset) for the Wizard class (with a "PowerClass.clsWizard"
tag). Add their nodes to a list.
*/
Log_Message("Copying missing powers from old powers file...", true);
result = XML_Get_First_Named_Child_With_Attr(root_old, "thing", "compset", "Power", &node);
while (result == 0) {
is_match = false;
result = XML_Get_First_Named_Child_With_Attr(node, "tag", "group", "PowerClass", &tag);
while (result == 0) {
XML_Read_Text_Attribute(tag, "tag", buffer);
if (strcmp("clsWizard", buffer) == 0) {
is_match = true;
break;
}
result = XML_Get_Next_Named_Child_With_Attr(node, &tag);
}
if (is_match)
nodes.push_back(node);
result = XML_Get_Next_Named_Child_With_Attr(root_old, &node);
}
/* For each node in our list, make sure a node with the same id doesn't
exist in the new powers file, since we don't want to introduce
duplicates.
*/
count = 0;
for (iter = nodes.begin(); iter != nodes.end(); iter++) {
XML_Read_Text_Attribute(*iter, "id", buffer);
result = XML_Get_First_Named_Child_With_Attr(root_new, "thing", "id", buffer, &node);
if (result == 0)
continue;
/* If it doesn't, copy the node from the old file to the new file. This
lets someone keep their wizard powers, even though the new data
doesn't include them.
*/
result = XML_Create_Child(root_new, "thing", &node);
if (x_Trap_Opt(result != 0))
continue;
result = XML_Duplicate_Node(*iter, &node);
x_Trap_Opt(result != 0);
count++;
}
/* Write out the new contents of the new powers file.
*/
sprintf(buffer, " done (%lu powers restored).\n", count);
Log_Message(buffer, true);
result = XML_Write_Document(doc_new, file_new, false, true);
if (x_Trap_Opt(result != 0))
Log_Message("Could not write XML document.", true);
/* We're finished with the XML documents.
*/
cleanup_exit:
if (doc_old != NULL)
XML_Destroy_Document(doc_old);
if (doc_new != NULL)
XML_Destroy_Document(doc_new);
}
void Session::LogNoexcept( std::string message )
{
write_message_noexcept( Log_Message( log_prefix + message ) ) ;
}
void Session::Log( std::wstring message )
{
WriteMessage( Log_Message( log_prefix_w + message ) ) ;
}
static void Output_Options(T_Background_Info * info, bool is_skill, T_List_Ids * id_list, C_Pool * pool)
{
T_Glyph backup;
T_Glyph_Ptr text, temp, end, table, search;
bool is_skip;
T_Glyph buffer[500];
if (info->description == NULL)
return;
if (is_skill) {
search = "Associated Skills: {/i}";
table = "skill";
}
else {
search = "Associated Languages: {/i}";
table = "language";
}
text = strstr(info->description, search);
if (text == NULL)
return;
text += strlen(search);
while (x_Is_Space(*text))
text++;
/* Parse out the next skill name
*/
while (true) {
end = strchr(text, ',');
end = Get_Earliest(end, strchr(text, '.'));
end = Get_Earliest(end, strchr(text, '{'));
if (end != NULL) {
backup = *end;
*end = '\0';
}
/* Note that Output_Language takes care of checking whether the
language exists for us
*/
if (!is_skill) {
is_skip = (Mapping_Find("bglangskip", text, false, false) != NULL);
if (!is_skip)
Output_Language(info->node, text, info, id_list, pool, "User", "BackLang");
}
else {
temp = Mapping_Find(table, text, false, true);
if (temp != NULL)
Output_Tag(info->node, "BackSkill", temp, info);
else {
sprintf(buffer, "Skill not found for backgrounds: %s\n", text);
Log_Message(buffer);
}
}
if (end != NULL)
*end = backup;
if ((end == NULL) || (*end == '{') || (*end == '\0'))
return;
/* Skip over spaces, commas, etc, to next skill
*/
text = end + 1;
while (x_Is_Space(*text) || x_Is_Punct(*text))
text++;
/* If we're at the end of the string, or if we hit a newline, we're
done
*/
if ((*text == '\0') || (*text == '{'))
return;
}
}
static T_Status Crawl_Data(bool use_cache, T_Glyph_Ptr email, T_Glyph_Ptr password,
T_Filename output_folder, bool is_clear)
{
T_Status status;
T_WWW internet = NULL;
bool is_exists;
T_Glyph_Ptr path_ptr;
T_Filename folder;
T_XML_Document doc_languages = NULL;
T_XML_Document doc_wepprops = NULL;
T_XML_Document doc_sources = NULL;
C_Pool pool(10000000,10000000);
C_DDI_Deities deities(&pool);
C_DDI_Skills skills(&pool);
C_DDI_Rituals rituals(&pool);
C_DDI_Items items(&pool);
C_DDI_Classes classes(&pool);
C_DDI_Races races(&pool);
C_DDI_Paragons paragons(&pool);
C_DDI_Epics epics(&pool);
C_DDI_Feats feats(&pool);
C_DDI_Powers powers(&pool);
C_DDI_Monsters monsters(&pool);
C_DDI_Backgrounds backgrounds(&pool);
vector<C_DDI_Common *> list;
vector<C_DDI_Single_Common *> singles;
/* Add all of our downloader classes to the list
*/
list.push_back(&deities);
list.push_back(&skills);
list.push_back(&rituals);
list.push_back(&items);
list.push_back(&classes);
list.push_back(&races);
list.push_back(¶gons);
list.push_back(&epics);
list.push_back(&feats);
list.push_back(&powers);
//Monsters aren't supported yet...
#ifdef NOTYET
list.push_back(&monsters);
#endif
list.push_back(&backgrounds);
/* Find a temporary folder to use
*/
Get_Temporary_Folder(folder);
path_ptr = folder + strlen(folder);
/* Search for the folder and create it if it doesn't exist
*/
is_exists = FileSys_Does_Folder_Exist(folder);
if (!is_exists) {
status = FileSys_Create_Directory(folder);
if (x_Trap_Opt(!x_Is_Success(status))) {
Log_Message("Couldn't create temporary folder.");
x_Status_Return(LWD_ERROR);
}
}
/* If we're not using cached copies, log in to D&D insider (if we don't
have a username and password, this just establishes the connection)
*/
if (!use_cache) {
Log_Message("Connecting to server...\n", true);
/* Open a connection to the server first
*/
status = WWW_HTTP_Open(&internet,LOGIN_URL,NULL,NULL,NULL);
if (x_Trap_Opt(!x_Is_Success(status))) {
Log_Message("Couldn't open connection to login server!\n", true);
goto cleanup_exit;
}
status = Login(internet, email, password, true);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
/* If we're not using the cache, we want to make sure we have fresh copies
of everything, so delete the files in our temporary folder
*/
if (!use_cache) {
strcpy(path_ptr,"*.*");
FileSys_Delete_Files(folder);
*path_ptr = '\0';
}
/* Create XML documents for deities languages - the crawlers can add their
own entries for these during processing
*/
status = Create_Document(&doc_languages, &l_language_root, "language",
DTD_Get_Data());
if (!x_Is_Success(status))
goto cleanup_exit;
status = Create_Document(&doc_wepprops, &l_wepprop_root, "weapon property",
DTD_Get_Augmentation());
if (!x_Is_Success(status))
goto cleanup_exit;
status = Create_Document(&doc_sources, &l_source_root, "source",
DTD_Get_Augmentation());
if (!x_Is_Success(status))
goto cleanup_exit;
/* First, download all the index pages if we need to
*/
if (!use_cache) {
for (ddi_iter it = list.begin(); it != list.end(); ++it) {
status = (*it)->Download_Index(folder, internet);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
for (single_iter it = singles.begin(); it != singles.end(); ++it) {
status = (*it)->Download(folder, internet);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
}
/* Read them in
*/
for (ddi_iter it = list.begin(); it != list.end(); ++it) {
status = (*it)->Read_Index(folder);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
for (single_iter it = singles.begin(); it != singles.end(); ++it) {
status = (*it)->Read(folder);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
/* Now download the rest of the content all at once
NOTE: As of March 2011, there's no further content for download unless
you have a password.
*/
if (!use_cache && l_is_password) {
for (ddi_iter it = list.begin(); it != list.end(); ++it) {
status = (*it)->Download_Content(folder, internet);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
/* It's possible that some downloads were screwed up due to the
servers acting weirdly. If so, they were added to a failed list,
so try them again
*/
Retry_Failed_Downloads(internet);
}
/* Read it in and process it
*/
for (ddi_iter it = list.begin(); it != list.end(); ++it) {
status = (*it)->Read_Content(folder);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
/* Process all our stuff
*/
for (ddi_iter it = list.begin(); it != list.end(); ++it) {
status = (*it)->Process();
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
for (single_iter it = singles.begin(); it != singles.end(); ++it) {
status = (*it)->Process();
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
/* Post-process all our stuff - this requires it to have been output first,
and puts finishing touches on everything
*/
for (ddi_iter it = list.begin(); it != list.end(); ++it) {
status = (*it)->Post_Process(folder);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
for (single_iter it = singles.begin(); it != singles.end(); ++it) {
status = (*it)->Post_Process(folder);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
/* Output our languages to the temporary folder, so we can append
extensions to them later
*/
status = Finish_Document(doc_languages, folder, LANGUAGE_FILENAME);
if (!x_Is_Success(status))
goto cleanup_exit;
/* Now append any fixup extensions that are needed to compensate for the
data being terrible
*/
Append_Extensions(folder, output_folder);
/* Load the old "powers" XML document and try and copy any wizard powers
out of it
*/
//We don't need to do this any more, but keep the code around just in case...
// Fixup_Wizard_Powers(output_folder);
/* Sources and weapon properties don't need any extensions, so they can go
directly into the output folder - plus they're augmentation files,
which aren't messed with anyway
*/
status = Finish_Document(doc_sources, output_folder, SOURCE_FILENAME);
if (!x_Is_Success(status))
goto cleanup_exit;
status = Finish_Document(doc_wepprops, output_folder, WEPPROP_FILENAME);
if (!x_Is_Success(status))
goto cleanup_exit;
/* wrapup everything
*/
cleanup_exit:
l_language_root = NULL;
if (doc_languages != NULL)
XML_Destroy_Document(doc_languages);
l_wepprop_root = NULL;
if (doc_wepprops != NULL)
XML_Destroy_Document(doc_wepprops);
l_source_root = NULL;
if (doc_sources != NULL)
XML_Destroy_Document(doc_sources);
if (internet != NULL)
WWW_Close_Server(internet);
/* Delete everything in our folder if required
*/
if (is_clear) {
strcpy(path_ptr,"*.*");
FileSys_Delete_Files(folder);
*path_ptr = '\0';
}
x_Status_Return(status);
}
int main(int argc, char* argv[])
{
enum { single, replicated, missing } mode;
int logTargets;
const char* user;
char buf[4096];
bool deleteDB;
bool firstRun;
firstRun = true;
mode = missing;
if (argc == 1)
{
fprintf(stderr, "You did not specify a config file!\n");
fprintf(stderr, "Starting in single mode with defaults...\n");
fprintf(stderr, "Using database.dir = '%s'\n", DATABASE_CONFIG_DIR);
mode = single;
}
else if (argc == 2)
{
if (!Config::Init(argv[1]))
STOP_FAIL("Cannot open config file", 1);
}
else
{
fprintf(stderr, "usage: %s <config-file>\n", argv[0]);
STOP_FAIL("invalid arguments", 1);
}
if (strcmp("single", Config::GetValue("mode", "")) == 0)
mode = single;
else if (strcmp("replicated", Config::GetValue("mode", "")) == 0)
mode = replicated;
else if (mode == missing)
{
fprintf(stderr, "specify mode = single or mode = replicated\n");
STOP_FAIL("invalid configuration file", 1);
}
logTargets = 0;
if (Config::GetListNum("log.targets") == 0)
logTargets = LOG_TARGET_STDOUT;
for (int i = 0; i < Config::GetListNum("log.targets"); i++)
{
if (strcmp(Config::GetListValue("log.targets", i, ""), "file") == 0)
{
logTargets |= LOG_TARGET_FILE;
Log_SetOutputFile(Config::GetValue("log.file", NULL),
Config::GetBoolValue("log.truncate", false));
}
if (strcmp(Config::GetListValue("log.targets", i, NULL), "stdout") == 0)
logTargets |= LOG_TARGET_STDOUT;
if (strcmp(Config::GetListValue("log.targets", i, NULL), "stderr") == 0)
logTargets |= LOG_TARGET_STDERR;
}
Log_SetTarget(logTargets);
Log_SetTrace(Config::GetBoolValue("log.trace", false));
Log_SetTimestamping(Config::GetBoolValue("log.timestamping", false));
Log_Message(VERSION_FMT_STRING " started");
run:
{
if (!IOProcessor::Init(Config::GetIntValue("io.maxfd", 1024)))
STOP_FAIL("Cannot initalize IOProcessor!", 1);
// after io is initialized, drop root rights
user = Config::GetValue("daemon.user", NULL);
if (!ChangeUser(user))
STOP_FAIL(rprintf("Cannot setuid to %s", user), 1);
DatabaseConfig dbConfig;
dbConfig.dir = Config::GetValue("database.dir", DATABASE_CONFIG_DIR);
dbConfig.pageSize = Config::GetIntValue("database.pageSize", DATABASE_CONFIG_PAGE_SIZE);
dbConfig.cacheSize = Config::GetIntValue("database.cacheSize", DATABASE_CONFIG_CACHE_SIZE);
dbConfig.logBufferSize = Config::GetIntValue("database.logBufferSize", DATABASE_CONFIG_LOG_BUFFER_SIZE);
dbConfig.checkpointTimeout = Config::GetIntValue("database.checkpointTimeout", DATABASE_CONFIG_CHECKPOINT_TIMEOUT);
dbConfig.verbose = Config::GetBoolValue("database.verbose", DATABASE_CONFIG_VERBOSE);
dbConfig.directDB = Config::GetBoolValue("database.directDB", DATABASE_CONFIG_DIRECT_DB);
dbConfig.txnNoSync = Config::GetBoolValue("database.txnNoSync", DATABASE_CONFIG_TXN_NOSYNC);
dbConfig.txnWriteNoSync = Config::GetBoolValue("database.txnWriteNoSync", DATABASE_CONFIG_TXN_WRITE_NOSYNC);
if (Config::GetBoolValue("database.warmCache", true) && firstRun)
WarmCache((char*)dbConfig.dir, dbConfig.cacheSize);
if (firstRun)
Log_Message("Opening database...");
if (!database.Init(dbConfig))
STOP_FAIL("Cannot initialize database!", 1);
if (firstRun)
Log_Message("Database opened");
dbWriter.Init(1);
dbReader.Init(Config::GetIntValue("database.numReaders", 20));
if (!RCONF->Init())
STOP_FAIL("Cannot initialize paxos!", 1);
KeyspaceDB* kdb;
if (mode == replicated)
{
RLOG->Init(Config::GetBoolValue("paxos.useSoftClock", true));
kdb = new ReplicatedKeyspaceDB;
}
else
{
kdb = new SingleKeyspaceDB;
}
kdb->Init();
HttpServer protoHttp;
HttpKeyspaceHandler httpKeyspaceHandler(kdb);
int httpPort = Config::GetIntValue("http.port", 8080);
if (httpPort)
{
protoHttp.Init(httpPort);
protoHttp.RegisterHandler(&httpKeyspaceHandler);
}
KeyspaceServer protoKeyspace;
protoKeyspace.Init(kdb, Config::GetIntValue("keyspace.port", 7080));
EventLoop::Init();
EventLoop::Run();
EventLoop::Shutdown();
if (mode == replicated)
deleteDB = ((ReplicatedKeyspaceDB*)kdb)->DeleteDB();
else
deleteDB = false;
protoKeyspace.Shutdown();
protoHttp.Shutdown();
kdb->Shutdown();
delete kdb;
dbReader.Shutdown();
dbWriter.Shutdown();
RLOG->Shutdown();
database.Shutdown();
IOProcessor::Shutdown();
if (mode == replicated && deleteDB)
{
// snprintf(buf, SIZE(buf), "%s/__*", dbConfig.dir);
// DeleteWC(buf);
snprintf(buf, SIZE(buf), "%s/log*", dbConfig.dir);
DeleteWC(buf);
snprintf(buf, SIZE(buf), "%s/keyspace", dbConfig.dir);
DeleteWC(buf);
#ifdef _WIN32
MSleep(3000); // otherwise Windows won't let use reuse the same ports
#endif
firstRun = false;
goto run;
}
}
Log_Message("Keyspace shutting down.");
Config::Shutdown();
Log_Shutdown();
}
int DatabaseSetTest(TestConfig& conf)
{
int numTest;
Stopwatch sw;
Table* table;
Transaction* tx;
bool ret;
int limit = 16*KB;
int sum;
if (conf.argc < 5)
{
Log_Message("\n\tusage: %s <keySize> <valueSize>", conf.typeString);
return 1;
}
Log_SetTrace(true);
if (DatabaseSetup())
{
Log_Message("Cannot initialize database!", 1);
return 1;
}
table = database.GetTable("keyspace");
if (!table)
{
Log_Message("Cannot initialize table!", 1);
return 1;
}
conf.SetKeySize(atoi(conf.argv[3]));
conf.SetValueSize(atoi(conf.argv[4]));
Log_Message("Test type = %s, keySize = %d, valueSize = %d",
conf.typeString, conf.keySize, conf.valueSize);
tx = NULL;
tx = new Transaction(table);
sw.Start();
tx->Begin();
sw.Stop();
sum = 0;
numTest = conf.datasetTotal / conf.valueSize;
for (int i = 0; i < numTest; i++)
{
if (conf.rndkey)
GenRandomString(conf.key, conf.keySize);
else
conf.key.Writef("key%B:%d", conf.padding.length, conf.padding.buffer, i);
sw.Start();
ret = table->Set(tx, conf.key, conf.value);
sw.Stop();
if (!ret)
{
Log_Message("Test failed, ret = %s (%s failed after %d)", ret ? "true" : "false", conf.typeString, i);
return 1;
}
sum += conf.keySize + conf.valueSize;
if (sum > limit)
{
sw.Start();
tx->Commit();
sw.Stop();
double mbps = sum / (sw.elapsed / 1000.0) / 1000000;
Log_Message("num = %d, elapsed = %ld, thruput = %lf MB/s", i, sw.elapsed, mbps);
sw.Reset();
sw.Start();
tx->Begin();
sw.Stop();
sum = 0;
}
}
sw.Start();
tx->Commit();
sw.Stop();
double mbps = (conf.valueSize + conf.keySize) * numTest / (sw.elapsed / 1000.0) / 1000000;
Log_Message("Test succeeded, %s/sec = %lf (num = %d, elapsed = %ld, thruput = %lf MB/s)", conf.typeString, numTest / (sw.elapsed / 1000.0), numTest, sw.elapsed, mbps);
return 0;
}
void ConfigServerApp::OnLogStatTimer()
{
char humanBuf[5];
Log_Message("=== ConfigServer stats ==>");
Log_Message("SDBP active: %u, inactive: %u", sdbpServer.GetNumActiveConns(), sdbpServer.GetNumInactiveConns());
Log_Message("HTTP active: %u, inactive: %u", httpServer.GetNumActiveConns(), httpServer.GetNumInactiveConns());
Log_Message("Cluster active: %u, write readyness: %u", CONTEXT_TRANSPORT->GetNumConns(), CONTEXT_TRANSPORT->GetNumWriteReadyness());
Log_Message("Heartbeats: %u", configServer.GetHeartbeatManager()->GetNumHeartbeats());
Log_Message("Primary leases: %u", configServer.GetPrimaryLeaseManager()->GetNumPrimaryLeases());
Log_Message("Config messages: %u", configServer.GetQuorumProcessor()->GetNumConfigMessages());
Log_Message("Requests: %u", configServer.GetQuorumProcessor()->GetNumRequests());
Log_Message("Listen requests: %u", configServer.GetQuorumProcessor()->GetNumListenRequests());
Log_Message("Timers: %u", EventLoop::GetNumTimers());
Log_Message("Page cache size: %s, num. pages: %u", HumanBytes(StoragePageCache::GetSize(), humanBuf), StoragePageCache::GetNumPages());
Log_Message("Request cache free list size: %u", REQUEST_CACHE->GetNumFreeRequests());
Log_Message("=== ConfigServer stats <==");
EventLoop::Add(&logStatTimer);
}
bool ClusterConnection::OnMessage(ReadBuffer& msg)
{
uint64_t otherNodeID;
uint64_t otherClusterID;
ReadBuffer buffer;
ClusterConnection* dup;
int read;
//Log_Debug("ClusterConnection::OnMessage");
if (progress == ClusterConnection::INCOMING)
{
// we have no incoming connections if we don't have a nodeID
ASSERT(transport->IsAwaitingNodeID() == false);
// the node at the other end is awaiting its nodeID
if (msg.GetCharAt(0) == '*')
{
read = msg.Readf("*:%#R", &buffer);
if (read != (int) msg.GetLength())
{
// protocol error
GetSocket().GetEndpoint(endpoint);
Log_Message("[%s] Cluster protocol error, disconnecting...", endpoint.ToString());
transport->DeleteConnection(this);
return true;
}
if (!endpoint.Set(buffer, true))
{
Log_Message("[%R] Cluster invalid network address", &buffer);
transport->DeleteConnection(this);
return true;
}
progress = ClusterConnection::AWAITING_NODEID;
Log_Trace("Conn is awaiting nodeID at %s", endpoint.ToString());
transport->AddConnection(this);
if (transport->OnAwaitingNodeID(endpoint))
{
Log_Trace();
transport->DeleteConnection(this);
return true;
}
if (nodeID == UNDEFINED_NODEID)
Log_Message("[%s] Cluster unknown node connected <=", endpoint.ToString());
else
Log_Message("[%s] Cluster node %U connected <=", endpoint.ToString(), nodeID);
return false;
}
// both ends have nodeIDs
read = msg.Readf("%U:%U:%#R", &otherClusterID, &otherNodeID, &buffer);
if (read != (int) msg.GetLength())
{
// protocol error
GetSocket().GetEndpoint(endpoint);
Log_Message("[%s] Cluster protocol error, disconnecting...", endpoint.ToString());
transport->DeleteConnection(this);
return true;
}
if (otherClusterID > 0)
{
if (transport->GetClusterID() == 0)
{
// the other side has a clusterID, I don't, so set mine
// if I'm a config server:
// the clusterID also needs to be set in REPLICATON_CONFIG,
// this is set in ConfigServer::OnConnectionReady()
// if I'm a shard server:
// the controllers will send a SetNodeID message, which
// contains the clusterID, so I'll be fine
transport->SetClusterID(otherClusterID);
}
else
{
if (otherClusterID != transport->GetClusterID())
{
Log_Message("[%R] Cluster invalid configuration, disconnecting...", &buffer);
Log_Debug("mine: %U != controller %U", transport->GetClusterID(), otherClusterID);
transport->DeleteConnection(this); // drop this
return true;
}
}
}
dup = transport->GetConnection(otherNodeID);
if (dup)
{
// if the other connections isn't ready yet, drop it
// OR
// the other connection is ready
// in this case, kill the connection that was initiated by higher nodeID
// in other words, since this is an incoming connection:
// if nodeID[of initiator] > transport->GetSelfNodeID(): drop
if (dup->progress != READY || otherNodeID > transport->GetSelfNodeID())
{
Log_Trace("delete dup");
transport->DeleteConnection(dup); // drop dup
}
else if (otherNodeID != transport->GetSelfNodeID())
{
Log_Trace("delete this");
transport->DeleteConnection(this); // drop this
return true;
}
}
progress = ClusterConnection::READY;
SetNodeID(otherNodeID);
if (!endpoint.Set(buffer, true))
{
Log_Message("[%R] Cluster invalid network address", &buffer);
transport->DeleteConnection(this);
return true;
}
// check if the other side is not sending its localhost address, when they are on
// different nodes
if (endpoint.GetAddress() == Endpoint::GetLoopbackAddress() &&
transport->GetSelfEndpoint().GetAddress() != Endpoint::GetLoopbackAddress())
{
Log_Message("[%R] Cluster invalid network address", &buffer);
transport->DeleteConnection(this);
return true;
}
Log_Trace("Conn READY to node %U at %s", nodeID, endpoint.ToString());
if (nodeID != transport->GetSelfNodeID())
{
if (nodeID == UNDEFINED_NODEID)
Log_Message("[%s] Cluster unknown node connected <=", endpoint.ToString());
else
Log_Message("[%s] Cluster node %U connected <=", endpoint.ToString(), nodeID);
}
transport->AddConnection(this);
transport->OnWriteReadyness(this);
transport->OnConnectionReady(nodeID, endpoint);
}
else if (progress == ClusterConnection::OUTGOING)
ASSERT_FAIL();
else
transport->OnMessage(nodeID, msg); // pass msg to upper layer
return false;
}
static T_Status Restore_Old_File(T_Glyph_Ptr filename, T_File_Attribute attributes,
T_Void_Ptr context)
{
T_Status status;
T_Glyph_Ptr base_filename, output_folder = (T_Glyph_Ptr) context;
T_Filename file_old, file_new, file_temp;
T_Glyph buffer[1000];
if ((attributes & e_filetype_directory) != 0)
x_Status_Return_Success();
/* Get the base filename to use for this file
*/
base_filename = strrchr(filename, DIR[0]);
if (x_Trap_Opt(base_filename == NULL))
x_Status_Return(LWD_ERROR);
base_filename++;
if (x_Trap_Opt(*base_filename == '\0'))
x_Status_Return(LWD_ERROR);
/* Get the temporary filename we'll use for stuff
*/
sprintf(file_temp, "%s" DIR "temporary.ignore", output_folder);
/* Get filenames for our source, and target files. The source is the
one we just found; the target is that with ".saved" chopped off the
end.
*/
sprintf(file_old, "%s" DIR "%s", output_folder, base_filename);
strcpy(file_new, file_old);
file_new[strlen(file_new) - strlen(BACKUP_EXTENSION)] = '\0';
sprintf(buffer, "Restoring %s...\n", base_filename);
Log_Message(buffer, true);
/* We want to swap the two files, so that the new is replaced by the
old.
*/
FileSys_Delete_File(file_temp, FALSE);
if (FileSys_Does_File_Exist(file_new)) {
status = FileSys_Copy_File(file_temp, file_new, FALSE);
if (!x_Is_Success(status)) {
sprintf(buffer, "Couldn't make a backup of %s!\n", file_new);
Log_Message(buffer, true);
}
}
if (FileSys_Does_File_Exist(file_old)) {
status = FileSys_Copy_File(file_new, file_old, FALSE);
if (!x_Is_Success(status)) {
sprintf(buffer, "Couldn't restore old file %s!\n", file_old);
Log_Message(buffer, true);
}
}
if (FileSys_Does_File_Exist(file_temp)) {
status = FileSys_Copy_File(file_old, file_temp, FALSE);
if (!x_Is_Success(status)) {
sprintf(buffer, "Couldn't restore backup of new file %s!\n", file_new);
Log_Message(buffer, true);
}
}
FileSys_Delete_File(file_temp, FALSE);
x_Status_Return_Success();
}
