void Checks_Versification::verses (string bible, int book, int chapter, vector <int> verses)
{
// Get verses in this chapter according to the versification system for the Bible.
Database_Versifications database_versifications;
string versification = Database_Config_Bible::getVersificationSystem (bible);
if (versification.empty ()) versification = english ();
vector <int> standardVerses = database_versifications.getVerses (versification, book, chapter);
// Look for missing and extra verses.
vector <int> absentVerses = filter_string_array_diff (standardVerses, verses);
vector <int> extraVerses = filter_string_array_diff (verses, standardVerses);
Database_Check database_check;
for (auto verse : absentVerses) {
database_check.recordOutput (bible, book, chapter, verse, "This verse is missing according to the versification system");
}
for (auto verse : extraVerses) {
//if ((chapter == 0) && (verse == 0)) continue;
database_check.recordOutput (bible, book, chapter, verse, "This verse is extra according to the versification system");
}
// Look for verses out of order.
int previousVerse = 0;
for (unsigned int i = 0; i < verses.size(); i++) {
int verse = verses[i];
if (i > 0) {
if (verse != (previousVerse + 1)) {
database_check.recordOutput (bible, book, chapter, verse, "The verse is out of sequence");
}
}
previousVerse = verse;
}
}
void Checks_Versification::chapters (string bible, int book, vector <int> chapters)
{
Database_Versifications database_versifications;
string versification = Database_Config_Bible::getVersificationSystem (bible);
if (versification.empty ()) versification = english ();
vector <int> standardChapters = database_versifications.getChapters (versification, book, true);
vector <int> absentChapters = filter_string_array_diff (standardChapters, chapters);
vector <int> extraChapters = filter_string_array_diff (chapters, standardChapters);
Database_Check database_check;
for (auto chapter : absentChapters) {
database_check.recordOutput (bible, book, chapter, 1, "This chapter is missing");
}
for (auto chapter : extraChapters) {
database_check.recordOutput (bible, book, chapter, 1, "This chapter is extra");
}
}
// Helper function.
// $user: The user whose changes are being processed.
// $recipients: The users who opted to receive online notifications of any contributors.
void changes_process_identifiers (Webserver_Request * request,
string user,
vector <string> recipients,
string bible,
int book, int chapter,
int oldId, int newId,
string & email)
{
if (oldId != 0) {
recipients = filter_string_array_diff (recipients, {user});
Database_Modifications database_modifications;
string stylesheet = Database_Config_Bible::getExportStylesheet (bible);
Database_Modifications_Text old_chapter_text = database_modifications.getUserChapter (user, bible, book, chapter, oldId);
string old_chapter_usfm = old_chapter_text.oldtext;
Database_Modifications_Text new_chapter_text = database_modifications.getUserChapter (user, bible, book, chapter, newId);
string new_chapter_usfm = new_chapter_text.newtext;
//int timestamp = database_modifications.getUserTimestamp (user, bible, book, chapter, newId);
vector <int> old_verse_numbers = usfm_get_verse_numbers (old_chapter_usfm);
vector <int> new_verse_numbers = usfm_get_verse_numbers (new_chapter_usfm);
vector <int> verses = old_verse_numbers;
verses.insert (verses.end (), new_verse_numbers.begin (), new_verse_numbers.end ());
verses = array_unique (verses);
sort (verses.begin(), verses.end());
for (auto verse : verses) {
string old_verse_usfm = usfm_get_verse_text (old_chapter_usfm, verse);
string new_verse_usfm = usfm_get_verse_text (new_chapter_usfm, verse);
if (old_verse_usfm != new_verse_usfm) {
Filter_Text filter_text_old = Filter_Text (bible);
Filter_Text filter_text_new = Filter_Text (bible);
filter_text_old.html_text_standard = new Html_Text (translate("Bible"));
filter_text_new.html_text_standard = new Html_Text (translate("Bible"));
filter_text_old.text_text = new Text_Text ();
filter_text_new.text_text = new Text_Text ();
filter_text_old.addUsfmCode (old_verse_usfm);
filter_text_new.addUsfmCode (new_verse_usfm);
filter_text_old.run (stylesheet);
filter_text_new.run (stylesheet);
string old_html = filter_text_old.html_text_standard->getInnerHtml ();
string new_html = filter_text_new.html_text_standard->getInnerHtml ();
string old_text = filter_text_old.text_text->get ();
string new_text = filter_text_new.text_text->get ();
if (old_text != new_text) {
string modification = filter_diff_diff (old_text, new_text);
email += "<div>";
email += filter_passage_display (book, chapter, convert_to_string (verse));
email += " ";
email += modification;
email += "</div>";
if (request->database_config_user()->getUserUserChangesNotificationsOnline (user)) {
database_modifications.recordNotification ({user}, changes_personal_category (), bible, book, chapter, verse, old_html, modification, new_html);
}
for (auto recipient : recipients) {
if (recipient == user) continue;
database_modifications.recordNotification ({recipient}, user, bible, book, chapter, verse, old_html, modification, new_html);
}
}
}
}
}
}
void Checks_Versification::books (string bible, vector <int> books)
{
Database_Versifications database_versifications;
string versification = Database_Config_Bible::getVersificationSystem (bible);
if (versification.empty ()) versification = english ();
vector <int> standardBooks = database_versifications.getBooks (versification);
vector <int> absentBooks = filter_string_array_diff (standardBooks, books);
vector <int> extraBooks = filter_string_array_diff (books, standardBooks);
Database_Check database_check;
for (auto book : absentBooks) {
database_check.recordOutput (bible, book, 1, 1, "This book is absent from the Bible");
}
for (auto book : extraBooks) {
database_check.recordOutput (bible, book, 1, 1, "This book is extra in the Bible");
}
}
string xrefs_translate (void * webserver_request)
{
Webserver_Request * request = (Webserver_Request *) webserver_request;
string sourceBible = request->database_config_user()->getSourceXrefBible ();
string targetBible = request->database_config_user()->getTargetXrefBible ();
// Save book / abbreviation pair.
if (request->post.count ("save")) {
string fullname = request->post ["fullname"];
string abbreviation = request->post ["abbreviation"];
string abbreviations = Database_Config_Bible::getBookAbbreviations (targetBible);
abbreviations = filter_abbreviations_add (abbreviations, fullname, abbreviation);
Database_Config_Bible::setBookAbbreviations (targetBible, abbreviations);
}
string abbrevs = Database_Config_Bible::getBookAbbreviations (sourceBible);
vector <pair <int, string> > sourceAbbreviations = filter_abbreviations_read (abbrevs);
vector <int> sourceBooks;
for (auto element : sourceAbbreviations) sourceBooks.push_back (element.first);
abbrevs = Database_Config_Bible::getBookAbbreviations (targetBible);
vector <pair <int, string> > targetAbbreviations = filter_abbreviations_read (abbrevs);
vector <int> targetBooks;
for (auto element : targetAbbreviations) targetBooks.push_back (element.first);
vector <int> unknown_books = filter_string_array_diff (sourceBooks, targetBooks);
unknown_books = array_unique (unknown_books);
if (unknown_books.empty ()) {
redirect_browser (request, xrefs_clear_url ());
return "";
}
string page;
Assets_Header header = Assets_Header (translate("Cross references"), webserver_request);
page = header.run ();
Assets_View view;
view.set_variable ("remaining", convert_to_string (unknown_books.size () - 1));
string bookname = Database_Books::getEnglishFromId (unknown_books [0]);
view.set_variable ("bookname", bookname);
page += view.render ("xrefs", "translate");
page += Assets_Page::footer ();
return page;
}
void Editor_Export::closeElementNode (xmlNodePtr node)
{
// The tag and class names of this element node.
string tagName ((char *) node->name);
string className;
xmlChar * property = xmlGetProp (node, BAD_CAST "class");
if (property) {
className = (char *) property;
xmlFree (property);
}
if (tagName == "p")
{
// While editing it happens that the p element does not have a class.
// Use the 'p' class in such cases.
if (className == "") className = "p";
if (noteOpeners.find (className) != noteOpeners.end()) {
// Deal with note closers.
currentLine += usfm_get_closing_usfm (className);
} else {
// Normally a p element closes the USFM line.
flushLine ();
mono = false;
// Clear active character styles.
characterStyles.clear();
}
}
if (tagName == "span")
{
// Do nothing for monospace elements, because the USFM would be the text nodes only.
if (mono) return;
// Do nothing without a class.
if (className.empty()) return;
// Do nothing if no endmarkers are supposed to be produced.
if (suppressEndMarkers.find (className) != suppressEndMarkers.end()) return;
// Add closing USFM, optionally closing embedded tags in reverse order.
vector <string> classes = filter_string_explode (className, ' ');
characterStyles = filter_string_array_diff (characterStyles, classes);
reverse (classes.begin(), classes.end());
for (unsigned int offset = 0; offset < classes.size(); offset++) {
bool embedded = (classes.size () > 1) && (offset == 0);
if (!characterStyles.empty ()) embedded = true;
currentLine += usfm_get_closing_usfm (classes [offset], embedded);
}
}
if (tagName == "a")
{
// Do nothing for note citations in the text.
}
}
// Returns the verse numbers in the string of $usfm code at line number $line_number.
vector <int> usfm_linenumber_to_versenumber (string usfm, unsigned int line_number)
{
vector <int> verse_number = {0}; // Initial verse number.
vector <string> lines = filter_string_explode (usfm, '\n');
for (unsigned int i = 0; i < lines.size(); i++) {
if (i <= line_number) {
vector <int> verse_numbers = usfm_get_verse_numbers (lines[i]);
if (verse_numbers.size() >= 2) {
verse_number = filter_string_array_diff (verse_numbers, {0});
}
}
}
return verse_number;
}
string search_search2 (void * webserver_request)
{
Webserver_Request * request = (Webserver_Request *) webserver_request;
Database_Volatile database_volatile = Database_Volatile ();
string siteUrl = config_logic_site_url ();
string bible = request->database_config_user()->getBible ();
if (request->query.count ("bible")) bible = request->query ["bible"];
bool hit_is_set = request->query.count ("h");
bool query_is_set = request->query.count ("q");
int identifier = convert_to_int (request->query ["i"]);
string query = request->query ["q"];
string hit = request->query ["h"];
// Get one search hit.
if (hit_is_set) {
// Retrieve the search parameters from the volatile database.
string query = database_volatile.getValue (identifier, "query");
//bool casesensitive = convert_to_bool (database_volatile.getValue (identifier, "casesensitive"));
bool plaintext = convert_to_bool (database_volatile.getValue (identifier, "plaintext"));
// Get the Bible and passage for this identifier.
Passage details = Passage::from_text (hit);
string bible = details.bible;
int book = details.book;
int chapter = details.chapter;
string verse = details.verse;
// Get the plain text or USFM.
string text;
if (plaintext) {
text = search_logic_get_bible_verse_text (bible, book, chapter, convert_to_int (verse));
} else {
text = search_logic_get_bible_verse_usfm (bible, book, chapter, convert_to_int (verse));
}
// Format it.
string link = filter_passage_link_for_opening_editor_at (book, chapter, verse);
text = filter_string_markup_words ({query}, text);
string output = "<div>" + link + " " + text + "</div>";
// Output to browser.
return output;
}
// Perform the initial search.
if (query_is_set) {
// Get extra search parameters and store them all in the volatile database.
bool casesensitive = (request->query ["c"] == "true");
bool plaintext = (request->query ["p"] == "true");
bool currentbook = (request->query ["b"] == "true");
string sharing = request->query ["s"];
database_volatile.setValue (identifier, "query", query);
database_volatile.setValue (identifier, "casesensitive", convert_to_string (casesensitive));
database_volatile.setValue (identifier, "plaintext", convert_to_string (plaintext));
// Deal with case sensitivity.
// Deal with whether to search the plain text, or the raw USFM.
// Fetch the initial set of hits.
vector <Passage> passages;
if (plaintext) {
if (casesensitive) {
passages = search_logic_search_bible_text_case_sensitive (bible, query);
} else {
passages = search_logic_search_bible_text (bible, query);
}
} else {
if (casesensitive) {
passages = search_logic_search_bible_usfm_case_sensitive (bible, query);
} else {
passages = search_logic_search_bible_usfm (bible, query);
}
}
// Deal with possible searching in the current book only.
if (currentbook) {
int book = Ipc_Focus::getBook (request);
vector <Passage> bookpassages;
for (auto & passage : passages) {
if (book == passage.book) {
bookpassages.push_back (passage);
}
}
passages = bookpassages;
}
// Deal with how to share the results.
vector <string> hits;
for (auto & passage : passages) {
hits.push_back (passage.to_text ());
}
if (sharing != "load") {
vector <string> loaded_hits = filter_string_explode (database_volatile.getValue (identifier, "hits"), '\n');
if (sharing == "add") {
hits.insert (hits.end(), loaded_hits.begin(), loaded_hits.end());
}
if (sharing == "remove") {
hits = filter_string_array_diff (loaded_hits, hits);
}
if (sharing == "intersect") {
hits = array_intersect (loaded_hits, hits);
}
hits = array_unique (hits);
}
// Generate one string from the hits.
string output = filter_string_implode (hits, "\n");
// Store search hits in the volatile database.
database_volatile.setValue (identifier, "hits", output);
// Output results.
return output;
}
// Build the advanced search page.
string page;
Assets_Header header = Assets_Header (translate("Search"), request);
header.setNavigator ();
header.addBreadCrumb (menu_logic_search_menu (), menu_logic_search_text ());
page = header.run ();
Assets_View view;
view.set_variable ("bible", bible);
string script = "var searchBible = \"" + bible + "\";";
view.set_variable ("script", script);
page += view.render ("search", "search2");
page += Assets_Page::footer ();
return page;
}
void sendreceive_changes ()
{
if (sendreceive_changes_watchdog) {
int time = filter_date_seconds_since_epoch ();
if (time < (sendreceive_changes_watchdog + 900)) {
Database_Logs::log (sendreceive_changes_text () + translate("Still busy"), Filter_Roles::translator ());
return;
}
Database_Logs::log (sendreceive_changes_text () + translate("Watchdog timeout"), Filter_Roles::translator ());
}
sendreceive_changes_kick_watchdog ();
config_globals_syncing_changes = true;
Database_Logs::log (sendreceive_changes_sendreceive_text (), Filter_Roles::translator ());
Webserver_Request request;
Sync_Logic sync_logic = Sync_Logic (&request);
Database_Modifications database_modifications;
if (!database_modifications.healthy ()) {
Database_Logs::log (sendreceive_changes_text () + translate("Recreate damaged modifications database"), Filter_Roles::translator ());
database_modifications.erase ();
database_modifications.create ();
}
string response = client_logic_connection_setup ();
int iresponse = convert_to_int (response);
if (iresponse < Filter_Roles::guest () || iresponse > Filter_Roles::admin ()) {
Database_Logs::log (sendreceive_changes_text () + translate("Failure to initiate connection"), Filter_Roles::translator ());
send_receive_changes_done ();
return;
}
// Set the correct user in the session: The sole user on the Client.
vector <string> users = request.database_users ()->getUsers ();
if (users.empty ()) {
Database_Logs::log (translate("No user found"), Filter_Roles::translator ());
send_receive_changes_done ();
return;
}
string user = users [0];
request.session_logic ()->setUsername (user);
string password = request.database_users ()->getmd5 (user);
// The basic request to be POSTed to the server.
// It contains the user's credentials.
map <string, string> post;
post ["u"] = bin2hex (user);
post ["p"] = password;
post ["l"] = convert_to_string (request.database_users ()->getUserLevel (user));
// Error variables.
string error;
bool communication_errors = false;
// Server URL to call.
string address = Database_Config_General::getServerAddress ();
int port = Database_Config_General::getServerPort ();
string url = client_logic_url (address, port, sync_changes_url ());
// Send the removed change notifications to the server.
vector <int> ids = request.database_config_user ()->getRemovedChanges ();
if (!ids.empty ()) Database_Logs::log (sendreceive_changes_text () + "Sending removed notifications: " + convert_to_string (ids.size()), Filter_Roles::translator ());
for (auto & id : ids) {
post ["a"] = convert_to_string (Sync_Logic::changes_delete_modification);
post ["i"] = convert_to_string (id);
response = sync_logic.post (post, url, error);
if (!error.empty ()) {
communication_errors = true;
Database_Logs::log (sendreceive_changes_text () + "Failure sending removed notification: " + error, Filter_Roles::translator ());
}
else {
request.database_config_user ()->removeRemovedChange (id);
}
}
if (communication_errors) {
Database_Logs::log (sendreceive_changes_text () + translate("Not downloading change notifications due to communication error"), Filter_Roles::translator ());
send_receive_changes_done ();
return;
}
// Compare the total checksum for the change notifications for the active user on client and server.
// Checksum is cached for future re-use.
// Take actions based on that.
string client_checksum = request.database_config_user ()->getChangeNotificationsChecksum ();
if (client_checksum.empty ()) {
client_checksum = Sync_Logic::changes_checksum (user);
request.database_config_user ()->setChangeNotificationsChecksum (client_checksum);
}
string server_checksum;
post ["a"] = convert_to_string (Sync_Logic::changes_get_checksum);
response = sync_logic.post (post, url, error);
if (!error.empty ()) {
Database_Logs::log (sendreceive_changes_text () + "Failure receiving checksum: " + error, Filter_Roles::translator ());
send_receive_changes_done ();
return;
}
server_checksum = response;
if (client_checksum == server_checksum) {
Database_Logs::log (sendreceive_changes_up_to_date_text (), Filter_Roles::translator ());
send_receive_changes_done ();
return;
}
// Get all identifiers for the notifications on the server for the user.
// Get the identifiers on the client.
vector <int> client_identifiers = database_modifications.getNotificationIdentifiers (user, false);
vector <int> server_identifiers;
post ["a"] = convert_to_string (Sync_Logic::changes_get_identifiers);
response = sync_logic.post (post, url, error);
if (!error.empty ()) {
Database_Logs::log (sendreceive_changes_text () + "Failure receiving identifiers: " + error, Filter_Roles::translator ());
send_receive_changes_done ();
return;
}
{
vector <string> ids = filter_string_explode (response, '\n');
for (auto & id : ids) server_identifiers.push_back (convert_to_int (id));
}
// Any identifiers on the client, but not on the server, remove them from the client.
vector <int> remove_identifiers = filter_string_array_diff (client_identifiers, server_identifiers);
for (auto & id : remove_identifiers) {
database_modifications.deleteNotification (id);
request.database_config_user ()->setChangeNotificationsChecksum ("");
Database_Logs::log (sendreceive_changes_text () + "Removing notification: " + convert_to_string (id), Filter_Roles::translator ());
}
// Any identifiers on the server, but not on the client, download them from the server.
vector <int> download_identifiers = filter_string_array_diff (server_identifiers, client_identifiers);
for (auto & id : download_identifiers) {
sendreceive_changes_kick_watchdog ();
Database_Logs::log (sendreceive_changes_text () + "Downloading notification: " + convert_to_string (id), Filter_Roles::translator ());
post ["a"] = convert_to_string (Sync_Logic::changes_get_modification);
post ["i"] = convert_to_string (id);
response = sync_logic.post (post, url, error);
if (!error.empty ()) {
Database_Logs::log (sendreceive_changes_text () + "Failure downloading notification: " + error, Filter_Roles::translator ());
}
else {
// The server has put all bits together, one bit per line.
vector <string> lines = filter_string_explode (response, '\n');
string category;
if (!lines.empty ()) {
category = lines [0];
lines.erase (lines.begin ());
}
string bible;
if (!lines.empty ()) {
bible = lines [0];
lines.erase (lines.begin ());
}
int book = 0;
if (!lines.empty ()) {
book = convert_to_int (lines [0]);
lines.erase (lines.begin ());
}
int chapter = 0;
if (!lines.empty ()) {
chapter = convert_to_int (lines [0]);
lines.erase (lines.begin ());
}
int verse = 0;
if (!lines.empty ()) {
verse = convert_to_int (lines [0]);
lines.erase (lines.begin ());
}
string oldtext;
if (!lines.empty ()) {
oldtext = lines [0];
lines.erase (lines.begin ());
}
string modification;
if (!lines.empty ()) {
modification = lines [0];
lines.erase (lines.begin ());
}
string newtext;
if (!lines.empty ()) {
newtext = lines [0];
lines.erase (lines.begin ());
}
database_modifications.storeClientNotification (id, user, category, bible, book, chapter, verse, oldtext, modification, newtext);
request.database_config_user ()->setChangeNotificationsChecksum ("");
}
}
// Done.
Database_Logs::log (sendreceive_changes_text () + "Ready", Filter_Roles::translator ());
send_receive_changes_done ();
}
void sendreceive_resources ()
{
if (sendreceive_resources_watchdog) {
int time = filter_date_seconds_since_epoch ();
if (time < (sendreceive_resources_watchdog + 900)) {
return;
}
Database_Logs::log ("Resources: " + translate("Watchdog timeout"), Filter_Roles::translator ());
sendreceive_resources_done ();
}
// If any of the prioritized synchronization tasks run, postpone the current task and do not start it.
if (sendreceive_logic_prioritized_task_is_active ()) {
sendreceive_resources_done ();
this_thread::sleep_for (chrono::seconds (5));
tasks_logic_queue (SYNCRESOURCES);
return;
}
sendreceive_resources_interrupt = false;
// If there's nothing to cache, bail out.
vector <string> resources = Database_Config_General::getResourcesToCache ();
if (resources.empty ()) return;
sendreceive_resources_kick_watchdog ();
// Error counter.
int error_count = 0;
// Resource to cache.
string resource = resources [0];
// Erase the two older storage locations that were used to cache resources in earlier versions of Bibledit.
{
Database_OfflineResources database_offlineresources;
Database_UsfmResources database_usfmresources;
database_offlineresources.erase (resource);
database_usfmresources.deleteResource (resource);
}
Database_Logs::log ("Starting to install resource:" " " + resource, Filter_Roles::consultant ());
Database_Versifications database_versifications;
vector <int> books = database_versifications.getMaximumBooks ();
for (auto & book : books) {
sendreceive_resources_delay_during_prioritized_tasks ();
if (sendreceive_resources_interrupt) continue;
// Database layout is per book: Create a database for this book.
Database_Cache::create (resource, book);
// Last downloaded passage in a previous download operation.
int last_downloaded_passage = 0;
{
pair <int, int> progress = Database_Cache::progress (resource, book);
int chapter = progress.first;
int verse = progress.second;
Passage passage ("", book, chapter, convert_to_string (verse));
last_downloaded_passage = filter_passage_to_integer (passage);
}
// List of passages recorded in the database that had errors on a previous download operation.
vector <int> previous_errors;
{
vector <pair <int, int> > errors = Database_Cache::errors (resource, book);
for (auto & element : errors) {
int chapter = element.first;
int verse = element.second;
Passage passage ("", book, chapter, convert_to_string (verse));
int numeric_error = filter_passage_to_integer (passage);
previous_errors.push_back (numeric_error);
}
}
string bookName = Database_Books::getEnglishFromId (book);
vector <int> chapters = database_versifications.getMaximumChapters (book);
for (auto & chapter : chapters) {
sendreceive_resources_delay_during_prioritized_tasks ();
if (sendreceive_resources_interrupt) continue;
bool downloaded = false;
string message = resource + ": " + bookName + " chapter " + convert_to_string (chapter);
vector <int> verses = database_versifications.getMaximumVerses (book, chapter);
for (auto & verse : verses) {
sendreceive_resources_delay_during_prioritized_tasks ();
if (sendreceive_resources_interrupt) continue;
// Numeric representation of passage to deal with.
Passage passage ("", book, chapter, convert_to_string (verse));
int numeric_passage = filter_passage_to_integer (passage);
// Conditions to download this verse:
// 1. The passage is past the last downloaded passage.
bool download_verse_past = numeric_passage > last_downloaded_passage;
// 2. The passage was recorded as an error in a previous download operation.
bool download_verse_error = in_array (numeric_passage, previous_errors);
// Whether to download the verse.
if (download_verse_past || download_verse_error) {
// Fetch the text for the passage.
bool server_is_installing_module = false;
int wait_iterations = 0;
string html, error;
do {
// Fetch this resource from the server.
string address = Database_Config_General::getServerAddress ();
int port = Database_Config_General::getServerPort ();
// If the client has not been connected to a cloud instance,
// fetch the resource from the Bibledit Cloud demo.
if (!client_logic_client_enabled ()) {
address = demo_address ();
port = demo_port ();
}
string url = client_logic_url (address, port, sync_resources_url ());
url = filter_url_build_http_query (url, "r", filter_url_urlencode (resource));
url = filter_url_build_http_query (url, "b", convert_to_string (book));
url = filter_url_build_http_query (url, "c", convert_to_string (chapter));
url = filter_url_build_http_query (url, "v", convert_to_string (verse));
error.clear ();
html = filter_url_http_get (url, error);
server_is_installing_module = (html == sword_logic_installing_module_text ());
if (server_is_installing_module) {
Database_Logs::log ("Waiting while Bibledit Cloud installs the requested SWORD module");
this_thread::sleep_for (chrono::seconds (60));
wait_iterations++;
}
} while (server_is_installing_module && (wait_iterations < 5));
// Record the passage as having been done in case it was a regular download,
// rather than one to retry a previous download error.
if (download_verse_past) Database_Cache::progress (resource, book, chapter, verse);
// Clear the registered error in case the verse download corrects it.
if (download_verse_error) Database_Cache::error (resource, book, chapter, verse, false);
if (error.empty ()) {
// Cache the verse data.
if (!Database_Cache::exists (resource, book)) Database_Cache::create (resource, book);
Database_Cache::cache (resource, book, chapter, verse, html);
} else {
// Record an error.
Database_Cache::error (resource, book, chapter, verse, true);
if (message.find (error) == string::npos) {
message.append ("; " + error);
}
error_count++;
this_thread::sleep_for (chrono::seconds (1));
}
downloaded = true;
}
sendreceive_resources_kick_watchdog ();
}
message += "; done";
if (downloaded) Database_Logs::log (message, Filter_Roles::manager ());
}
}
// Done.
if (error_count) {
string msg = "Error count while downloading resource: " + convert_to_string (error_count);
Database_Logs::log (msg, Filter_Roles::consultant ());
}
Database_Logs::log ("Completed installing resource:" " " + resource, Filter_Roles::consultant ());
// In case of too many errors, schedule the resource download again.
bool re_schedule_download = false;
if (error_count) {
if (!sendreceive_resources_interrupt) {
re_schedule_download = true;
Database_Logs::log ("Errors: Re-scheduling resource installation", Filter_Roles::consultant ());
}
}
// Store new download schedule.
resources = Database_Config_General::getResourcesToCache ();
resources = filter_string_array_diff (resources, {resource});
if (re_schedule_download) {
resources.push_back (resource);
}
Database_Config_General::setResourcesToCache (resources);
sendreceive_resources_done ();
sendreceive_resources_interrupt = false;
// If there's another resource waiting to be cached, schedule it for caching.
if (!resources.empty ()) tasks_logic_queue (SYNCRESOURCES);
}
void sendreceive_usfmresources ()
{
mutex_sendreceive_usfmresources.lock ();
bool bail_out = sendreceive_usfmresources_running;
mutex_sendreceive_usfmresources.unlock ();
if (bail_out) return;
mutex_sendreceive_usfmresources.lock ();
sendreceive_usfmresources_running = true;
mutex_sendreceive_usfmresources.unlock ();
Database_UsfmResources database_usfmresources = Database_UsfmResources ();
Webserver_Request request;
Sync_Logic sync_logic = Sync_Logic (&request);
Database_Logs::log (sendreceive_usfmresources_sendreceive_text (), Filter_Roles::translator ());
string address = Database_Config_General::getServerAddress ();
int port = Database_Config_General::getServerPort ();
string url = client_logic_url (address, port, sync_usfmresources_url ());
map <string, string> post;
string error;
string response;
// Request the checksum of all USFM resources from the server.
// Compare it with the local checksum.
// If the two match: Ready.
post ["a"] = convert_to_string (Sync_Logic::usfmresources_get_total_checksum);
response = sync_logic.post (post, url, error);
if (!error.empty ()) {
Database_Logs::log (sendreceive_usfmresources_text () + "Failure getting total checksum: " + error, Filter_Roles::translator ());
sendreceive_usfmresources_done ();
return;
}
string checksum = Sync_Logic::usfm_resources_checksum ();
if (response == checksum) {
Database_Logs::log (sendreceive_usfmresources_up_to_date_text (), Filter_Roles::translator ());
sendreceive_usfmresources_done ();
return;
}
// Request a list of all USFM resources available on the server.
post ["a"] = convert_to_string (Sync_Logic::usfmresources_get_resources);
response = sync_logic.post (post, url, error);
if (!error.empty ()) {
Database_Logs::log (sendreceive_usfmresources_text () + "Failure getting resources: " + error, Filter_Roles::translator ());
sendreceive_usfmresources_done ();
return;
}
vector <string> server_resources = filter_string_explode (response, '\n');
// Delete any resource on the local client but not on the server.
vector <string> client_resources = database_usfmresources.getResources ();
vector <string> resources = filter_string_array_diff (client_resources, server_resources);
for (auto & resource : resources) {
database_usfmresources.deleteResource (resource);
}
// Deal with each USFM resource individually.
for (auto & resource : server_resources) {
// Request the checksum of the resources from the server.
// Compare it with the checksum of the local resource.
// If they match: Go to the next resource.
post ["a"] = convert_to_string (Sync_Logic::usfmresources_get_resource_checksum);
post ["r"] = resource;
response = sync_logic.post (post, url, error);
if (!error.empty ()) {
Database_Logs::log (sendreceive_usfmresources_text () + "Failure getting checksum of resource: " + error, Filter_Roles::translator ());
sendreceive_usfmresources_done ();
return;
}
checksum = Sync_Logic::usfm_resource_checksum (resource);
if (response == checksum) {
continue;
}
// Request a list of all books in the resource on the server.
post ["a"] = convert_to_string (Sync_Logic::usfmresources_get_books);
post ["r"] = resource;
response = sync_logic.post (post, url, error);
if (!error.empty ()) {
Database_Logs::log (sendreceive_usfmresources_text () + "Failure getting books of resource: " + error, Filter_Roles::translator ());
sendreceive_usfmresources_done ();
return;
}
vector <int> server_books;
vector <string> sbooks = filter_string_explode (response, '\n');
for (auto & book : sbooks) server_books.push_back (convert_to_int (book));
// Delete any books from the client that are not on the server.
vector <int> client_books = database_usfmresources.getBooks (resource);
vector <int> books = filter_string_array_diff (client_books, server_books);
for (auto & book : books) {
database_usfmresources.deleteBook (resource, book);
}
// Deal with each book individually.
for (auto & book : server_books) {
// Request checksum of this book,
// compare it with the local checksum,
// and skip the book if the checksums match.
post ["a"] = convert_to_string (Sync_Logic::usfmresources_get_book_checksum);
post ["r"] = resource;
post ["b"] = convert_to_string (book);
response = sync_logic.post (post, url, error);
if (!error.empty ()) {
Database_Logs::log (sendreceive_usfmresources_text () + "Failure getting checksum of resource book: " + error, Filter_Roles::translator ());
sendreceive_usfmresources_done ();
return;
}
checksum = Sync_Logic::usfm_resource_book_checksum (resource, book);
if (response == checksum) {
continue;
}
string bookname = Database_Books::getEnglishFromId (book);
Database_Logs::log (sendreceive_usfmresources_text () + "Synchronizing " + resource + " " + bookname, Filter_Roles::translator ());
// Retrieve a list of chapters in the $book from the server.
post ["a"] = convert_to_string (Sync_Logic::usfmresources_get_chapters);
post ["r"] = resource,
post ["b"] = convert_to_string (book);
response = sync_logic.post (post, url, error);
if (!error.empty ()) {
Database_Logs::log (sendreceive_usfmresources_text () + "Failure getting chapters of resource book: " + error, Filter_Roles::translator ());
sendreceive_usfmresources_done ();
return;
}
vector <int> server_chapters;
vector <string> schapters = filter_string_explode (response, '\n');
for (auto & chapter : schapters) server_chapters.push_back (convert_to_int (chapter));
// Delete local chapters not found on the server.
vector <int> client_chapters = database_usfmresources.getChapters (resource, book);
vector <int> chapters = filter_string_array_diff (client_chapters, server_chapters);
for (auto & chapter : chapters) {
database_usfmresources.deleteChapter (resource, book, chapter);
}
// Go through each chapter individually.
for (auto & chapter : server_chapters) {
// Get the checksum of the chapter as it is on the server.
post ["a"] = convert_to_string (Sync_Logic::usfmresources_get_chapter_checksum);
post ["r"] = resource;
post ["b"] = convert_to_string (book);
post ["c"] = convert_to_string (chapter);
response = sync_logic.post (post, url, error);
if (!error.empty ()) {
Database_Logs::log (sendreceive_usfmresources_text () + "Failure getting checksum of resource chapter: " + error, Filter_Roles::translator ());
sendreceive_usfmresources_done ();
return;
}
checksum = Sync_Logic::usfm_resource_chapter_checksum (resource, book, chapter);
if (response == checksum) {
continue;
}
// Download the chapter from the server, and store it locally on the client.
post ["a"] = convert_to_string (Sync_Logic::usfmresources_get_chapter);
post ["r"] = resource;
post ["b"] = convert_to_string (book);
post ["c"] = convert_to_string (chapter);
response = sync_logic.post (post, url, error);
if (!error.empty ()) {
Database_Logs::log (sendreceive_usfmresources_text () + "Failure downloading resource chapter: " + error, Filter_Roles::translator ());
sendreceive_usfmresources_done ();
return;
}
database_usfmresources.storeChapter (resource, book, chapter, response);
}
}
}
// Done.
Database_Logs::log (sendreceive_usfmresources_text () + "Now up to date", Filter_Roles::translator ());
sendreceive_usfmresources_done ();
}