string render_journal_entry (string entry) { // Remove the user's level. entry.erase (0, 2); // Extract and remove the seconds since the Unix epoch. time_t seconds = convert_to_int (entry); entry.erase (0, 11); // Localize the seconds. seconds = filter_date_local_seconds (seconds); // Sanitize HTML. entry = filter_string_sanitize_html (entry); // Convert \n to <br> entry = filter_string_str_replace ("\n", "<br>", entry); // Convert the seconds into a human readable time. string timestamp = filter_string_fill (convert_to_string (filter_date_numerical_hour (seconds)), 2, '0'); timestamp.append (":"); timestamp.append (filter_string_fill (convert_to_string (filter_date_numerical_minute (seconds)), 2, '0')); timestamp.append (":"); timestamp.append (filter_string_fill (convert_to_string (filter_date_numerical_second (seconds)), 2, '0')); // Done. return timestamp + " | " + entry; }
void timer_index () { bool client = client_logic_client_enabled (); int previous_second = -1; int previous_minute = -1; int previous_fraction = -1; while (config_globals_running) { try { // Wait shortly. this_thread::sleep_for (chrono::milliseconds (100)); // Wait tilll the data structures have been initialized. if (!config_globals_data_initialized) continue; // The current time, localized. int seconds_since_epoch = filter_date_seconds_since_epoch (); int local_seconds = filter_date_local_seconds (seconds_since_epoch); int second = filter_date_numerical_second (local_seconds); int minute = filter_date_numerical_minute (local_seconds); int hour = filter_date_numerical_hour (local_seconds); int weekday = filter_date_numerical_week_day (local_seconds); // Run once per second. if (second == previous_second) continue; previous_second = second; // Every second: Deal with queued and/or active tasks. tasks_run_check (); // Run the part below every so many seconds. int fraction = second / 5; if (fraction != previous_fraction) { previous_fraction = fraction; } // Run the part below once per minute. if (minute == previous_minute) continue; previous_minute = minute; // Every minute send out queued email, except in client mode. if (!client) tasks_logic_queue (SENDEMAIL); // Check for new mail every five minutes. // Do not check more often with gmail else the account may be shut down. if ((!client) && ((minute % 5) == 0)) { tasks_logic_queue (RECEIVEEMAIL); } // At the nineth minute after every full hour rotate the journal. // The nine is chosen, because the journal rotation will summarize the send/receive messages. // In case send/receive happens every five minutes, it is expected that under normal circumstances // the whole process of sending/receivning will be over, so summarization can then be done. if (minute == 9) tasks_logic_queue (ROTATEJOURNAL); // Client sends/receives Bibles and Consultation. sendreceive_queue_sync (minute); // Sending and receiving Bibles to and from the git repository. // On a production website running on an inexpensive virtual private server // with 512 Mbyte of memory and a fast network connection, // sending and receiving two Bibles takes more than 15 minutes when there are many changes. bool sendreceive = ((hour == 0) && (minute == 5)); bool repeat = ((minute % 5) == 0); if (sendreceive || repeat) { sendreceive_queue_all (sendreceive); } // Deal with the changes in the Bible made per user. // Deal with notifications for the daily changes in the Bibles. // This takes a few minutes on a production machine with two Bibles and changes in several chapters. // It runs in a server configuration, not on a client. if (!client) { if ((hour == 0) && (minute == 20)) { changes_logic_start (); } } // Run the checks on the Bibles. // This takes 15 minutes on a production machine with two Bibles. if (!client) { if ((hour == 0) && (minute == 30)) { checks_logic_start_all (); } } // Database maintenance and trimming. // It takes a few minutes on a production machine. if ((hour == 0) && (minute == 50)) { tasks_logic_queue (MAINTAINDATABASE); } // Export the Bibles to the various output formats. // This may take an hour on a production machine. // This hour was in PHP. In C++ it is much faster. if (!client) { if ((hour == 1) && (minute == 10)) { Export_Logic::scheduleAll (); } } // Delete temporal files older than a few days. if ((hour == 2) && (minute == 0)) { tasks_logic_queue (CLEANTMPFILES); } // Re-index Bibles and notes. // Only update missing indexes. if ((hour == 2) && (minute == 0)) { Database_State::create (); Database_Config_General::setIndexBibles (true); tasks_logic_queue (REINDEXBIBLES); Database_Config_General::setIndexNotes (true); tasks_logic_queue (REINDEXNOTES); } // Actions for a demo installation. if (minute == 10) { if (config_logic_demo_enabled ()) { tasks_logic_queue (CLEANDEMO); } } // Sprint burndown. // It runs every hour in the Cloud. // The script itself determines what to do at which hour of the day or day of the week or day of the month. if (!client) { if (minute == 5) { tasks_logic_queue (SPRINTBURNDOWN); } } // Quit at midnight if flag is set. if (config_globals_quit_at_midnight) { if (hour == 0) { if (minute == 1) { if (!Database_Config_General::getJustStarted ()) { if (tasks_run_active_count ()) { Database_Logs::log ("Server is due to restart itself but does not because of active jobs"); } else { Database_Logs::log ("Server restarts itself"); exit (0); } } } // Clear flag in preparation of restart next minute. // This flag also has the purpose of ensuring the server restarts once during that minute, // rather than restarting repeatedly many times during that minute. if (minute == 0) { Database_Config_General::setJustStarted (false); } } } // Email notes statistics to the users. if (!client) { if ((hour == 3) && (minute == 0)) { tasks_logic_queue (NOTESSTATISTICS); } } // Update SWORD stuff once a week. if (weekday == 1) { // Refresh module list. if ((!client) && (hour == 3) && (minute == 5)) { tasks_logic_queue (REFRESHSWORDMODULES); } // Update installed SWORD modules, shortly after the module list has been refreshed. if ((!client) && (hour == 3) && (minute == 15)) { tasks_logic_queue (UPDATESWORDMODULES); } } // The Cloud updates the list of USFM resources once a week. if (weekday == 1) { if ((!client) && (hour == 3) && (minute == 10)) { tasks_logic_queue (LISTUSFMRESOURCES); } } } catch (exception & e) { Database_Logs::log (e.what ()); } catch (exception * e) { Database_Logs::log (e->what ()); } catch (...) { Database_Logs::log ("A general internal error occurred in the timers"); } } }
void changes_modifications () { Database_Logs::log ("Change notifications: Generating", Filter_Roles::translator ()); // Notifications are not available to clients for the duration of processing them. config_globals_change_notifications_available = false; // Data objects. Webserver_Request request; Database_Modifications database_modifications; // Check on the health of the modifications database and (re)create it if needed. if (!database_modifications.healthy ()) database_modifications.erase (); database_modifications.create (); // Create online change notifications for users who made changes in Bibles // through the web editor or through a client. // It runs before the team changes. // This produces the desired order of the notifications in the GUI. // At the same time, produce change statistics per user. // Get the users who will receive the changes entered by the contributors. vector <string> recipients; { vector <string> users = request.database_users ()->getUsers (); for (auto & user : users) { if (request.database_config_user ()->getContributorChangesNotificationsOnline (user)) { recipients.push_back (user); } } } // Storage for the statistics. map <string, int> user_change_statistics; float modification_time_total = 0; int modification_time_count = 0; vector <string> users = database_modifications.getUserUsernames (); if (!users.empty ()) Database_Logs::log ("Change notifications: Per user", Filter_Roles::translator ()); for (auto user : users) { // Total changes made by this user. int change_count = 0; // Go through the Bibles changed by the current user. vector <string> bibles = database_modifications.getUserBibles (user); for (auto bible : bibles) { // Body of the email to be sent. string email = "<p>" + translate("You have entered the changes below in a Bible editor.") + " " + translate ("You may check if it made its way into the Bible text.") + "</p>"; size_t empty_email_length = email.length (); // Go through the books in that Bible. vector <int> books = database_modifications.getUserBooks (user, bible); for (auto book : books) { // Go through the chapters in that book. vector <int> chapters = database_modifications.getUserChapters (user, bible, book); for (auto chapter : chapters) { // Get the sets of identifiers for that chapter, and set some variables. vector <Database_Modifications_Id> IdSets = database_modifications.getUserIdentifiers (user, bible, book, chapter); //int referenceOldId = 0; int referenceNewId = 0; int newId = 0; int lastNewId = 0; bool restart = true; // Go through the sets of identifiers. for (auto IdSet : IdSets) { int oldId = IdSet.oldid; newId = IdSet.newid; if (restart) { changes_process_identifiers (&request, user, recipients, bible, book, chapter, referenceNewId, newId, email, change_count, modification_time_total, modification_time_count); //referenceOldId = oldId; referenceNewId = newId; lastNewId = newId; restart = false; continue; } if (oldId == lastNewId) { lastNewId = newId; } else { restart = true; } } // Process the last set of identifiers. changes_process_identifiers (&request, user, recipients, bible, book, chapter, referenceNewId, newId, email, change_count, modification_time_total, modification_time_count); } } // Check whether there's any email to be sent. if (email.length () != empty_email_length) { // Send the user email with the user's personal changes if the user opted to receive it. if (request.database_config_user()->getUserUserChangesNotification (user)) { string subject = translate("Changes you entered in") + " " + bible; if (!client_logic_client_enabled ()) email_schedule (user, subject, email); } } } // Store change statistics for this user. user_change_statistics [user] = change_count; // Clear the user's changes in the database. database_modifications.clearUserUser (user); // Clear checksum cache. request.database_config_user ()->setUserChangeNotificationsChecksum (user, ""); } // Generate the notifications, online and by email, // for the changes in the Bibles entered by anyone since the previous notifications were generated. vector <string> bibles = database_modifications.getTeamDiffBibles (); for (auto bible : bibles) { string stylesheet = Database_Config_Bible::getExportStylesheet (bible); vector <string> changeNotificationUsers; vector <string> users = request.database_users ()->getUsers (); for (auto user : users) { if (access_bible_read (&request, bible, user)) { if (request.database_config_user()->getUserGenerateChangeNotifications (user)) { changeNotificationUsers.push_back (user); } } } users.clear (); // The number of changes processed so far for this Bible. int processedChangesCount = 0; // The files get stored at http://site.org:<port>/revisions/<Bible>/<date> int seconds = filter_date_seconds_since_epoch (); string timepath; timepath.append (convert_to_string (filter_date_numerical_year (seconds))); timepath.append ("-"); timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_month (seconds)), 2, '0')); timepath.append ("-"); timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_month_day (seconds)), 2, '0')); timepath.append (" "); timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_hour (seconds)), 2, '0')); timepath.append (":"); timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_minute (seconds)), 2, '0')); timepath.append (":"); timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_second (seconds)), 2, '0')); string directory = filter_url_create_root_path ("revisions", bible, timepath); filter_url_mkdir (directory); // Produce the USFM and html files. filter_diff_produce_verse_level (bible, directory); // Create online page with changed verses. string versesoutputfile = filter_url_create_path (directory, "changed_verses.html"); filter_diff_run_file (filter_url_create_path (directory, "verses_old.txt"), filter_url_create_path (directory, "verses_new.txt"), versesoutputfile); // Storage for body of the email with the changes. vector <string> email_changes; // Generate the online change notifications. vector <int> books = database_modifications.getTeamDiffBooks (bible); for (auto book : books) { vector <int> chapters = database_modifications.getTeamDiffChapters (bible, book); for (auto chapter : chapters) { Database_Logs::log ("Change notifications: " + bible + " " + filter_passage_display (book, chapter, ""), Filter_Roles::translator ()); string old_chapter_usfm = database_modifications.getTeamDiff (bible, book, chapter); string new_chapter_usfm = request.database_bibles()->getChapter (bible, book, chapter); vector <int> old_verse_numbers = usfm_get_verse_numbers (old_chapter_usfm); vector <int> new_verse_numbers = usfm_get_verse_numbers (new_chapter_usfm); vector <int> verses = old_verse_numbers; verses.insert (verses.end (), new_verse_numbers.begin (), new_verse_numbers.end ()); verses = array_unique (verses); sort (verses.begin (), verses.end()); for (auto verse : verses) { string old_verse_usfm = usfm_get_verse_text (old_chapter_usfm, verse); string new_verse_usfm = usfm_get_verse_text (new_chapter_usfm, verse); if (old_verse_usfm != new_verse_usfm) { processedChangesCount++; // In case of too many change notifications, processing them would take too much time, so take a few shortcuts. string old_html = "<p>" + old_verse_usfm + "</p>"; string new_html = "<p>" + new_verse_usfm + "</p>"; string old_text = old_verse_usfm; string new_text = new_verse_usfm; if (processedChangesCount < 800) { Filter_Text filter_text_old = Filter_Text (bible); Filter_Text filter_text_new = Filter_Text (bible); filter_text_old.html_text_standard = new Html_Text (""); filter_text_new.html_text_standard = new Html_Text (""); filter_text_old.text_text = new Text_Text (); filter_text_new.text_text = new Text_Text (); filter_text_old.addUsfmCode (old_verse_usfm); filter_text_new.addUsfmCode (new_verse_usfm); filter_text_old.run (stylesheet); filter_text_new.run (stylesheet); old_html = filter_text_old.html_text_standard->getInnerHtml (); new_html = filter_text_new.html_text_standard->getInnerHtml (); old_text = filter_text_old.text_text->get (); new_text = filter_text_new.text_text->get (); } string modification = filter_diff_diff (old_text, new_text); database_modifications.recordNotification (changeNotificationUsers, changes_bible_category (), bible, book, chapter, verse, old_html, modification, new_html); string passage = filter_passage_display (book, chapter, convert_to_string (verse)) + ": "; if (old_text != new_text) { email_changes.push_back (passage + modification); } else { email_changes.push_back (translate ("The following passage has no change in the text.") + " " + translate ("The change is in the formatting only.") + " " + translate ("The USFM code is given for reference.")); email_changes.push_back (passage); email_changes.push_back (translate ("Old code:") + " " + old_verse_usfm); email_changes.push_back (translate ("New code:") + " " + new_verse_usfm); } } } // Delete the diff data for this chapter, for two reasons: // 1. New diffs for this chapter can be stored straightaway. // 2. In case of large amounts of diff data, and this function gets killed, // then the next time it runs again, it will continue from where it was killed. database_modifications.deleteTeamDiffChapter (bible, book, chapter); } } // Email the changes to the subscribed users. if (!email_changes.empty ()) { string body; for (auto & line : email_changes) { body.append ("<div>"); body.append (line); body.append ("</div>\n"); } string subject = translate("Recent changes:") + " " + bible; vector <string> users = request.database_users ()->getUsers (); for (auto & user : users) { if (request.database_config_user()->getUserBibleChangesNotification (user)) { if (access_bible_read (&request, bible, user)) { if (!client_logic_client_enabled ()) { email_schedule (user, subject, body); } } } } } } // Index the data and remove expired notifications. Database_Logs::log ("Change notifications: Indexing", Filter_Roles::translator ()); database_modifications.indexTrimAllNotifications (); // Remove expired downloadable revisions. string directory = filter_url_create_root_path ("revisions"); int now = filter_date_seconds_since_epoch (); bibles = filter_url_scandir (directory); for (auto &bible : bibles) { string folder = filter_url_create_path (directory, bible); int time = filter_url_file_modification_time (folder); int days = (now - time) / 86400; if (days > 31) { filter_url_rmdir (folder); } else { vector <string> revisions = filter_url_scandir (folder); for (auto & revision : revisions) { string path = filter_url_create_path (folder, revision); int time = filter_url_file_modification_time (path); int days = (now - time) / 86400; if (days > 31) { filter_url_rmdir (path); Database_Logs::log ("Removing expired downloadable revision notification: " + bible + " " + revision, Filter_Roles::translator ()); } } } } // Clear checksum caches. users = request.database_users ()->getUsers (); for (auto user : users) { request.database_config_user ()->setUserChangeNotificationsChecksum (user, ""); } // Vacuum the modifications index, as it might have been updated. database_modifications.vacuum (); // Make the notifications available again to clients. config_globals_change_notifications_available = true; // Store the statistics in the database. if (modification_time_count) { // Take average timestamp of all timestamps. int timestamp = round (modification_time_total / modification_time_count); for (auto & element : user_change_statistics) { // Store dated change statistics per user. string user = element.first; int count = element.second; Database_Statistics::store_changes (timestamp, user, count); } } Database_Logs::log ("Change notifications: Ready", Filter_Roles::translator ()); }