T_Status C_DDI_Rituals::Parse_Index_Cell(T_XML_Node node, T_Ritual_Info * info,
T_Int32U subtype)
{
T_Status status;
status = Get_Required_Child_PCDATA(&info->level, node, "Level");
if (!x_Is_Success(status))
x_Status_Return(status);
status = Get_Required_Child_PCDATA(&info->price, node, "Price");
if (!x_Is_Success(status))
x_Status_Return(status);
/* If our level is '0', that's obviously nonsense so get
rid of it
*/
if ((info->level[0] == '0') && (info->level[1] == '\0'))
info->level = "";
status = Get_Required_Child_PCDATA(&info->componentcost, node, "ComponentCost");
if (!x_Is_Success(status))
x_Status_Return(status);
status = Get_Required_Child_PCDATA(&info->keyskill, node, "KeySkillDescription");
if (!x_Is_Success(status))
x_Status_Return(status);
x_Status_Return_Success();
}
static T_Status Login(T_WWW internet, T_Glyph_Ptr email, T_Glyph_Ptr password, bool is_report_success)
{
static T_Glyph_Ptr l_last_email = NULL;
static T_Glyph_Ptr l_last_password = NULL;
T_Status status;
T_Glyph_Ptr ptr, newline, invalid;
/* Save our last email and password if given, or re-use them if we weren't
passed in an email or password
*/
if (email != NULL)
l_last_email = email;
else
email = l_last_email;
if (password != NULL)
l_last_password = password;
else
password = l_last_password;
/* If we don't have an email or password, we can't log in
*/
if ((email[0] == '\0') || (password[0] == '\0'))
x_Status_Return_Success();
/* Set up our arrays of name/value parameters to use for the login POST
*/
T_Glyph_Ptr names[] = { "email", "password" };
T_Glyph_Ptr values[] = { email, password };
/* Logging in is a simple POST of our username & password to the D&DI server
*/
status = WWW_HTTP_Post(internet,LOGIN_URL,x_Array_Size(names),names,values,&ptr);
if (x_Trap_Opt(!x_Is_Success(status))) {
Log_Message("Couldn't make login HTTP post!\n", true);
x_Status_Return(status);
}
/* If the string 'invalid' appears in the first line of our response,
it's a failed login.
*/
newline = strchr(ptr, '\n');
if (newline == NULL) {
Log_Message("\n**** Login error - could not log in to D&D insider.\n", true);
x_Status_Return(LWD_ERROR);
}
invalid = strstr(ptr, "invalid");
if ((invalid != NULL) && (invalid < newline)) {
Log_Message("\n**** Login error - username / password not recognised.\n", true);
x_Status_Return(LWD_ERROR);
}
Log_Message("Login succeeded.\n", is_report_success);
l_is_password = true;
x_Status_Return_Success();
}
T_Status C_DDI_Deities::Parse_Index_Cell(T_XML_Node node, T_Deity_Info * info,
T_Int32U subtype)
{
T_Status status;
status = Get_Required_Child_PCDATA(&info->alignment, node, "Alignment");
if (!x_Is_Success(status))
x_Status_Return(status);
x_Status_Return_Success();
}
static void Append_Extensions(T_Filename temp_folder, T_Filename output_folder)
{
T_Status status;
T_Filename input_wildcard;
/* Go through our data files and post-process them
*/
sprintf(input_wildcard, "%s" DIR "*.dat", temp_folder);
status = FileSys_Enumerate_Matching_Files(input_wildcard, Append_Extension, output_folder);
if (x_Trap_Opt(status == WARN_CORE_FILE_NOT_FOUND)) {
printf("<No files to append to!>\n");
return;
}
x_Trap_Opt(!x_Is_Success(status));
}
static void Restore_Old_Files(T_Glyph_Ptr output_folder)
{
T_Status status;
T_Filename wildcard;
/* Go through our data files and post-process them
*/
sprintf(wildcard, "%s" DIR "ddi_*" BACKUP_EXTENSION, output_folder);
status = FileSys_Enumerate_Matching_Files(wildcard, Restore_Old_File, output_folder);
if (x_Trap_Opt(status == WARN_CORE_FILE_NOT_FOUND)) {
Log_Message("No old files found to restore!\n", true);
return;
}
x_Trap_Opt(!x_Is_Success(status));
}
/* Note - this function should be resilient to stuff in the HTML changing, so
if we don't find what we expect, we should just return successfully and
not complain about it.
*/
T_Status C_DDI_Deities::Parse_Entry_Details(T_Glyph_Ptr contents, T_Deity_Info * info,
T_Glyph_Ptr ptr, T_Glyph_Ptr checkpoint,
vector<T_Deity_Info> * extras)
{
T_Status status;
T_Glyph_Ptr end;
ptr = Extract_Text(&info->flavor, m_pool, ptr, checkpoint, "<b>Type: </b>", "<br");
ptr = Extract_Text(&info->alignment, m_pool, ptr, checkpoint, "<b>Alignment :</b>", "</b><br");
ptr = Extract_Text(&info->gender, m_pool, ptr, checkpoint, "<b>Gender:</b>", "<br");
ptr = Extract_Text(&info->sphere, m_pool, ptr, checkpoint, "<b>Sphere:</b>", "<br");
ptr = Extract_Text(&info->dominion, m_pool, ptr, checkpoint, "<b>Dominion:</b>", "<br");
ptr = Extract_Text(&info->priests, m_pool, ptr, checkpoint, "<b>Priests:</b>", "<br");
ptr = Extract_Text(&info->adjective, m_pool, ptr, checkpoint, "<b>Adjective:</b>", "<br");
/* The description text appears after the end of the paragraph node
*/
ptr = Find_Text(ptr, "</p>", checkpoint, true, false);
if (x_Trap_Opt(ptr == NULL))
x_Status_Return_Success();
end = Find_Text(ptr, "<p>", checkpoint, false, false);
if (x_Trap_Opt(end == NULL))
x_Status_Return_Success();
*end = '\0';
status = Parse_Description_Text(&info->description, ptr, m_pool);
if (x_Trap_Opt(!x_Is_Success(status)))
x_Status_Return(status);
*end = '<';
x_Status_Return_Success();
}
/* Note - this function should be resilient to stuff in the HTML changing, so
if we don't find what we expect, we should just return successfully and
not complain about it.
*/
T_Status C_DDI_Rituals::Parse_Entry_Details(T_Glyph_Ptr contents, T_Ritual_Info * info,
T_Glyph_Ptr ptr, T_Glyph_Ptr checkpoint,
vector<T_Ritual_Info> * extras)
{
T_Status status;
T_Glyph_Ptr end;
T_Glyph_Ptr starts[] = { "???", };
T_Glyph_Ptr ends[] = { "<br", "</p", "</span" };
ptr = Extract_Text(&info->flavor, m_pool, ptr, checkpoint, "<span class=\"flavor\">", "</span>");
Strip_Bad_Characters(info->flavor);
/* This was parsed from the index above, but we might as well grab it again
here, in case there's more information
*/
starts[0] = "<b>Component Cost</b>:";
ptr = Extract_Text(&info->componentcost, m_pool, ptr, checkpoint, starts, x_Array_Size(starts),
ends, x_Array_Size(ends));
Strip_Bad_Characters(info->componentcost);
/* Although we've parsed the skill out of the index, replace it if we can -
the one on the actual page has more data (e.g. "no check" info).
*/
starts[0] = "<b>Key Skill</b>:";
ptr = Extract_Text(&info->keyskill, m_pool, ptr, checkpoint, starts, x_Array_Size(starts),
ends, x_Array_Size(ends));
Strip_Bad_Characters(info->keyskill);
/* Although we've parsed the level out of the index, replace it if we can -
the one on the actual page has more data.
*/
starts[0] = "<b>Level</b>:";
ptr = Extract_Text(&info->level, m_pool, ptr, checkpoint, starts, x_Array_Size(starts),
ends, x_Array_Size(ends));
Strip_Bad_Characters(info->level);
starts[0] = "<b>Category</b>:";
ptr = Extract_Text(&info->category, m_pool, ptr, checkpoint, starts, x_Array_Size(starts),
ends, x_Array_Size(ends));
Strip_Bad_Characters(info->category);
starts[0] = "<b>Time</b>:";
ptr = Extract_Text(&info->time, m_pool, ptr, checkpoint, starts, x_Array_Size(starts),
ends, x_Array_Size(ends));
Strip_Bad_Characters(info->time);
starts[0] = "<b>Duration</b>:";
ptr = Extract_Text(&info->duration, m_pool, ptr, checkpoint, starts, x_Array_Size(starts),
ends, x_Array_Size(ends));
Strip_Bad_Characters(info->duration);
starts[0] = "<b>Prerequisite</b>:";
ptr = Extract_Text(&info->prerequisite, m_pool, ptr, checkpoint, starts, x_Array_Size(starts),
ends, x_Array_Size(ends));
Strip_Bad_Characters(info->prerequisite);
/* Parse out any paragraph close tag we encounter here
*/
if (strnicmp(ptr, "</p>", 4) == 0)
ptr += 4;
/* Now parse the rest of the description - it might end in a </p> tag, or
it might not! Who knows!
*/
end = Find_Text(ptr, "</p>", checkpoint, false, false);
if (end == NULL)
end = checkpoint;
*end = '\0';
status = Parse_Description_Text(&info->description, ptr, m_pool);
if (x_Trap_Opt(!x_Is_Success(status)))
x_Status_Return(status);
*end = '<';
x_Status_Return_Success();
}
static T_Status Restore_Old_File(T_Glyph_Ptr filename, T_File_Attribute attributes,
T_Void_Ptr context)
{
T_Status status;
T_Glyph_Ptr base_filename, output_folder = (T_Glyph_Ptr) context;
T_Filename file_old, file_new, file_temp;
T_Glyph buffer[1000];
if ((attributes & e_filetype_directory) != 0)
x_Status_Return_Success();
/* Get the base filename to use for this file
*/
base_filename = strrchr(filename, DIR[0]);
if (x_Trap_Opt(base_filename == NULL))
x_Status_Return(LWD_ERROR);
base_filename++;
if (x_Trap_Opt(*base_filename == '\0'))
x_Status_Return(LWD_ERROR);
/* Get the temporary filename we'll use for stuff
*/
sprintf(file_temp, "%s" DIR "temporary.ignore", output_folder);
/* Get filenames for our source, and target files. The source is the
one we just found; the target is that with ".saved" chopped off the
end.
*/
sprintf(file_old, "%s" DIR "%s", output_folder, base_filename);
strcpy(file_new, file_old);
file_new[strlen(file_new) - strlen(BACKUP_EXTENSION)] = '\0';
sprintf(buffer, "Restoring %s...\n", base_filename);
Log_Message(buffer, true);
/* We want to swap the two files, so that the new is replaced by the
old.
*/
FileSys_Delete_File(file_temp, FALSE);
if (FileSys_Does_File_Exist(file_new)) {
status = FileSys_Copy_File(file_temp, file_new, FALSE);
if (!x_Is_Success(status)) {
sprintf(buffer, "Couldn't make a backup of %s!\n", file_new);
Log_Message(buffer, true);
}
}
if (FileSys_Does_File_Exist(file_old)) {
status = FileSys_Copy_File(file_new, file_old, FALSE);
if (!x_Is_Success(status)) {
sprintf(buffer, "Couldn't restore old file %s!\n", file_old);
Log_Message(buffer, true);
}
}
if (FileSys_Does_File_Exist(file_temp)) {
status = FileSys_Copy_File(file_old, file_temp, FALSE);
if (!x_Is_Success(status)) {
sprintf(buffer, "Couldn't restore backup of new file %s!\n", file_new);
Log_Message(buffer, true);
}
}
FileSys_Delete_File(file_temp, FALSE);
x_Status_Return_Success();
}
static T_Status Crawl_Data(bool use_cache, T_Glyph_Ptr email, T_Glyph_Ptr password,
T_Filename output_folder, bool is_clear)
{
T_Status status;
T_WWW internet = NULL;
bool is_exists;
T_Glyph_Ptr path_ptr;
T_Filename folder;
T_XML_Document doc_languages = NULL;
T_XML_Document doc_wepprops = NULL;
T_XML_Document doc_sources = NULL;
C_Pool pool(10000000,10000000);
C_DDI_Deities deities(&pool);
C_DDI_Skills skills(&pool);
C_DDI_Rituals rituals(&pool);
C_DDI_Items items(&pool);
C_DDI_Classes classes(&pool);
C_DDI_Races races(&pool);
C_DDI_Paragons paragons(&pool);
C_DDI_Epics epics(&pool);
C_DDI_Feats feats(&pool);
C_DDI_Powers powers(&pool);
C_DDI_Monsters monsters(&pool);
C_DDI_Backgrounds backgrounds(&pool);
vector<C_DDI_Common *> list;
vector<C_DDI_Single_Common *> singles;
/* Add all of our downloader classes to the list
*/
list.push_back(&deities);
list.push_back(&skills);
list.push_back(&rituals);
list.push_back(&items);
list.push_back(&classes);
list.push_back(&races);
list.push_back(¶gons);
list.push_back(&epics);
list.push_back(&feats);
list.push_back(&powers);
//Monsters aren't supported yet...
#ifdef NOTYET
list.push_back(&monsters);
#endif
list.push_back(&backgrounds);
/* Find a temporary folder to use
*/
Get_Temporary_Folder(folder);
path_ptr = folder + strlen(folder);
/* Search for the folder and create it if it doesn't exist
*/
is_exists = FileSys_Does_Folder_Exist(folder);
if (!is_exists) {
status = FileSys_Create_Directory(folder);
if (x_Trap_Opt(!x_Is_Success(status))) {
Log_Message("Couldn't create temporary folder.");
x_Status_Return(LWD_ERROR);
}
}
/* If we're not using cached copies, log in to D&D insider (if we don't
have a username and password, this just establishes the connection)
*/
if (!use_cache) {
Log_Message("Connecting to server...\n", true);
/* Open a connection to the server first
*/
status = WWW_HTTP_Open(&internet,LOGIN_URL,NULL,NULL,NULL);
if (x_Trap_Opt(!x_Is_Success(status))) {
Log_Message("Couldn't open connection to login server!\n", true);
goto cleanup_exit;
}
status = Login(internet, email, password, true);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
/* If we're not using the cache, we want to make sure we have fresh copies
of everything, so delete the files in our temporary folder
*/
if (!use_cache) {
strcpy(path_ptr,"*.*");
FileSys_Delete_Files(folder);
*path_ptr = '\0';
}
/* Create XML documents for deities languages - the crawlers can add their
own entries for these during processing
*/
status = Create_Document(&doc_languages, &l_language_root, "language",
DTD_Get_Data());
if (!x_Is_Success(status))
goto cleanup_exit;
status = Create_Document(&doc_wepprops, &l_wepprop_root, "weapon property",
DTD_Get_Augmentation());
if (!x_Is_Success(status))
goto cleanup_exit;
status = Create_Document(&doc_sources, &l_source_root, "source",
DTD_Get_Augmentation());
if (!x_Is_Success(status))
goto cleanup_exit;
/* First, download all the index pages if we need to
*/
if (!use_cache) {
for (ddi_iter it = list.begin(); it != list.end(); ++it) {
status = (*it)->Download_Index(folder, internet);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
for (single_iter it = singles.begin(); it != singles.end(); ++it) {
status = (*it)->Download(folder, internet);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
}
/* Read them in
*/
for (ddi_iter it = list.begin(); it != list.end(); ++it) {
status = (*it)->Read_Index(folder);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
for (single_iter it = singles.begin(); it != singles.end(); ++it) {
status = (*it)->Read(folder);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
/* Now download the rest of the content all at once
NOTE: As of March 2011, there's no further content for download unless
you have a password.
*/
if (!use_cache && l_is_password) {
for (ddi_iter it = list.begin(); it != list.end(); ++it) {
status = (*it)->Download_Content(folder, internet);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
/* It's possible that some downloads were screwed up due to the
servers acting weirdly. If so, they were added to a failed list,
so try them again
*/
Retry_Failed_Downloads(internet);
}
/* Read it in and process it
*/
for (ddi_iter it = list.begin(); it != list.end(); ++it) {
status = (*it)->Read_Content(folder);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
/* Process all our stuff
*/
for (ddi_iter it = list.begin(); it != list.end(); ++it) {
status = (*it)->Process();
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
for (single_iter it = singles.begin(); it != singles.end(); ++it) {
status = (*it)->Process();
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
/* Post-process all our stuff - this requires it to have been output first,
and puts finishing touches on everything
*/
for (ddi_iter it = list.begin(); it != list.end(); ++it) {
status = (*it)->Post_Process(folder);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
for (single_iter it = singles.begin(); it != singles.end(); ++it) {
status = (*it)->Post_Process(folder);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
}
/* Output our languages to the temporary folder, so we can append
extensions to them later
*/
status = Finish_Document(doc_languages, folder, LANGUAGE_FILENAME);
if (!x_Is_Success(status))
goto cleanup_exit;
/* Now append any fixup extensions that are needed to compensate for the
data being terrible
*/
Append_Extensions(folder, output_folder);
/* Load the old "powers" XML document and try and copy any wizard powers
out of it
*/
//We don't need to do this any more, but keep the code around just in case...
// Fixup_Wizard_Powers(output_folder);
/* Sources and weapon properties don't need any extensions, so they can go
directly into the output folder - plus they're augmentation files,
which aren't messed with anyway
*/
status = Finish_Document(doc_sources, output_folder, SOURCE_FILENAME);
if (!x_Is_Success(status))
goto cleanup_exit;
status = Finish_Document(doc_wepprops, output_folder, WEPPROP_FILENAME);
if (!x_Is_Success(status))
goto cleanup_exit;
/* wrapup everything
*/
cleanup_exit:
l_language_root = NULL;
if (doc_languages != NULL)
XML_Destroy_Document(doc_languages);
l_wepprop_root = NULL;
if (doc_wepprops != NULL)
XML_Destroy_Document(doc_wepprops);
l_source_root = NULL;
if (doc_sources != NULL)
XML_Destroy_Document(doc_sources);
if (internet != NULL)
WWW_Close_Server(internet);
/* Delete everything in our folder if required
*/
if (is_clear) {
strcpy(path_ptr,"*.*");
FileSys_Delete_Files(folder);
*path_ptr = '\0';
}
x_Status_Return(status);
}
int main(int argc,char ** argv)
{
T_Status status;
T_Int32S result = 0;
bool use_cache, is_command_line, is_clear = true;
E_Query_Mode mode;
T_Filename output_folder, folder, logfile;
T_XML_Document mappings = NULL;
T_Glyph xml_errors[1000], email[1000], password[1000];
/* On windows, get the current directory to use as the output folder. On the
mac, the current directory could well be the user's home directory, so
get the directory we were launched from
*/
#ifdef _WIN32
FileSys_Get_Current_Directory(output_folder);
#elif defined(_OSX)
strcpy(output_folder, argv[0]);
T_Glyph_Ptr ptr;
ptr = strrchr(output_folder, DIR[0]);
if (x_Trap_Opt(ptr == NULL)) {
printf("Error getting path of executable.");
exit(1);
}
*++ptr = '\0';
#else
#error Unknown platform!
#endif
/* Set up our static output stream for logging
*/
sprintf(logfile, "%s" DIR "output.txt", output_folder);
ofstream output(logfile,ios::out);
if (output.good())
Initialize_Helper(&output);
else {
Log_Message("Error opening output log file.", true);
goto cleanup_exit;
}
/* Set up an error buffer for our XML library
*/
XML_Set_Error_Buffer(xml_errors);
/* Initialize our unique id mechanisms
*/
UniqueId_Initialize();
/* Verify that we have write privileges in our output folder - if not, then
downloading stuff would be a bit of a waste of time (this can happen if
the downloader needs to be run with administrator rights).
*/
if (!FileSys_Verify_Write_Privileges(output_folder)) {
status = LWD_ERROR;
goto cleanup_exit;
}
/* Load up our mappings from the mapping XML file
*/
mappings = Load_Mappings(output_folder);
if (x_Trap_Opt(mappings == NULL)) {
Log_Message("Could not open mapping file.\n", true);
goto cleanup_exit;
}
/* Set up some sensible defaults
*/
email[0] = '\0';
password[0] = '\0';
use_cache = false;
/* If our first parameter is the secret "don't delete stuff" code, unset
the flag that causes everything to be deleted
*/
if ((argc > 1) && (stricmp(argv[1], "-nodelete") == 0))
is_clear = false;
/* Ask the user how the program is going to run - if we're told to exit,
just get out now
*/
mode = Query_Mode(argc, argv, &is_command_line);
if (mode == e_mode_exit)
goto cleanup_exit;
/* If we want to download with no password, do nothing - this is the
normal way we'll work
*/
if (mode == e_mode_no_password)
/* do nothing */;
/* If we just want to reprocess stuff without downloading, set our 'use cache'
flag, and don't clear the data after processing it
*/
else if (mode == e_mode_process) {
use_cache = true;
is_clear = false;
}
/* If we just want to append extensions, do that
*/
else if (mode == e_mode_append) {
Get_Temporary_Folder(folder);
Append_Extensions(folder, output_folder);
status = SUCCESS;
goto cleanup_exit;
}
/* If we want to download with no password, get the username and password
*/
else if (mode == e_mode_password) {
Get_Credentials(email, password);
if ((email[0] == '\0') || (password[0] == '\0')) {
Log_Message("Empty username or password - cancelling download.\n", true);
goto cleanup_exit;
}
}
/* Restore mode
*/
else if (mode == e_mode_restore) {
Restore_Old_Files(output_folder);
Log_Message("\nThe old versions of your files were restored.\nYou can use this command again to switch back to the \"new\" versions.\n", true);
goto cleanup_exit;
}
/* If we just want to delete stuff, delete it
*/
else if (mode == e_mode_delete) {
Delete_Generated_Files(output_folder);
goto cleanup_exit;
}
/* Otherwise, something is wrong
*/
else {
x_Trap_Opt(1);
Log_Message("Unrecognised option - exiting.\n", true);
goto cleanup_exit;
}
/* Crawl everything
*/
status = Crawl_Data(use_cache, email, password, output_folder, is_clear);
if (x_Trap_Opt(!x_Is_Success(status)))
goto cleanup_exit;
if (status == SUCCESS)
Log_Message("\n\nImport complete. Reload the game system to see the new content.\n\n", true);
else
Log_Message("\n\nThere were one or more problems with the import. Please see the manual for more details.\n\n", true);
/* wrapup everything
*/
cleanup_exit:
for (map_iter it = l_mappings.begin(); it != l_mappings.end(); ++it)
delete *it;
if (mappings != NULL)
XML_Destroy_Document(mappings);
Shutdown_Helper();
/* The window doesn't auto-close on OS X, so we don't need this
*/
#ifndef _OSX
if (!is_command_line) {
printf("Press any key to quit.\n\n");
Get_Character();
}
#endif
if (!x_Is_Success(status))
result = -1;
return(result);
}