string render_journal_entry (string entry)
{
// Remove the user's level.
entry.erase (0, 2);
// Extract and remove the seconds since the Unix epoch.
time_t seconds = convert_to_int (entry);
entry.erase (0, 11);
// Localize the seconds.
seconds = filter_date_local_seconds (seconds);
// Sanitize HTML.
entry = filter_string_sanitize_html (entry);
// Convert \n to <br>
entry = filter_string_str_replace ("\n", "<br>", entry);
// Convert the seconds into a human readable time.
string timestamp = filter_string_fill (convert_to_string (filter_date_numerical_hour (seconds)), 2, '0');
timestamp.append (":");
timestamp.append (filter_string_fill (convert_to_string (filter_date_numerical_minute (seconds)), 2, '0'));
timestamp.append (":");
timestamp.append (filter_string_fill (convert_to_string (filter_date_numerical_second (seconds)), 2, '0'));
// Done.
return timestamp + " | " + entry;
}
void timer_index ()
{
bool client = client_logic_client_enabled ();
int previous_second = -1;
int previous_minute = -1;
int previous_fraction = -1;
while (config_globals_running) {
try {
// Wait shortly.
this_thread::sleep_for (chrono::milliseconds (100));
// Wait tilll the data structures have been initialized.
if (!config_globals_data_initialized) continue;
// The current time, localized.
int seconds_since_epoch = filter_date_seconds_since_epoch ();
int local_seconds = filter_date_local_seconds (seconds_since_epoch);
int second = filter_date_numerical_second (local_seconds);
int minute = filter_date_numerical_minute (local_seconds);
int hour = filter_date_numerical_hour (local_seconds);
int weekday = filter_date_numerical_week_day (local_seconds);
// Run once per second.
if (second == previous_second) continue;
previous_second = second;
// Every second: Deal with queued and/or active tasks.
tasks_run_check ();
// Run the part below every so many seconds.
int fraction = second / 5;
if (fraction != previous_fraction) {
previous_fraction = fraction;
}
// Run the part below once per minute.
if (minute == previous_minute) continue;
previous_minute = minute;
// Every minute send out queued email, except in client mode.
if (!client) tasks_logic_queue (SENDEMAIL);
// Check for new mail every five minutes.
// Do not check more often with gmail else the account may be shut down.
if ((!client) && ((minute % 5) == 0)) {
tasks_logic_queue (RECEIVEEMAIL);
}
// At the nineth minute after every full hour rotate the journal.
// The nine is chosen, because the journal rotation will summarize the send/receive messages.
// In case send/receive happens every five minutes, it is expected that under normal circumstances
// the whole process of sending/receivning will be over, so summarization can then be done.
if (minute == 9) tasks_logic_queue (ROTATEJOURNAL);
// Client sends/receives Bibles and Consultation.
sendreceive_queue_sync (minute);
// Sending and receiving Bibles to and from the git repository.
// On a production website running on an inexpensive virtual private server
// with 512 Mbyte of memory and a fast network connection,
// sending and receiving two Bibles takes more than 15 minutes when there are many changes.
bool sendreceive = ((hour == 0) && (minute == 5));
bool repeat = ((minute % 5) == 0);
if (sendreceive || repeat) {
sendreceive_queue_all (sendreceive);
}
// Deal with the changes in the Bible made per user.
// Deal with notifications for the daily changes in the Bibles.
// This takes a few minutes on a production machine with two Bibles and changes in several chapters.
// It runs in a server configuration, not on a client.
if (!client) {
if ((hour == 0) && (minute == 20)) {
changes_logic_start ();
}
}
// Run the checks on the Bibles.
// This takes 15 minutes on a production machine with two Bibles.
if (!client) {
if ((hour == 0) && (minute == 30)) {
checks_logic_start_all ();
}
}
// Database maintenance and trimming.
// It takes a few minutes on a production machine.
if ((hour == 0) && (minute == 50)) {
tasks_logic_queue (MAINTAINDATABASE);
}
// Export the Bibles to the various output formats.
// This may take an hour on a production machine.
// This hour was in PHP. In C++ it is much faster.
if (!client) {
if ((hour == 1) && (minute == 10)) {
Export_Logic::scheduleAll ();
}
}
// Delete temporal files older than a few days.
if ((hour == 2) && (minute == 0)) {
tasks_logic_queue (CLEANTMPFILES);
}
// Re-index Bibles and notes.
// Only update missing indexes.
if ((hour == 2) && (minute == 0)) {
Database_State::create ();
Database_Config_General::setIndexBibles (true);
tasks_logic_queue (REINDEXBIBLES);
Database_Config_General::setIndexNotes (true);
tasks_logic_queue (REINDEXNOTES);
}
// Actions for a demo installation.
if (minute == 10) {
if (config_logic_demo_enabled ()) {
tasks_logic_queue (CLEANDEMO);
}
}
// Sprint burndown.
// It runs every hour in the Cloud.
// The script itself determines what to do at which hour of the day or day of the week or day of the month.
if (!client) {
if (minute == 5) {
tasks_logic_queue (SPRINTBURNDOWN);
}
}
// Quit at midnight if flag is set.
if (config_globals_quit_at_midnight) {
if (hour == 0) {
if (minute == 1) {
if (!Database_Config_General::getJustStarted ()) {
if (tasks_run_active_count ()) {
Database_Logs::log ("Server is due to restart itself but does not because of active jobs");
} else {
Database_Logs::log ("Server restarts itself");
exit (0);
}
}
}
// Clear flag in preparation of restart next minute.
// This flag also has the purpose of ensuring the server restarts once during that minute,
// rather than restarting repeatedly many times during that minute.
if (minute == 0) {
Database_Config_General::setJustStarted (false);
}
}
}
// Email notes statistics to the users.
if (!client) {
if ((hour == 3) && (minute == 0)) {
tasks_logic_queue (NOTESSTATISTICS);
}
}
// Update SWORD stuff once a week.
if (weekday == 1) {
// Refresh module list.
if ((!client) && (hour == 3) && (minute == 5)) {
tasks_logic_queue (REFRESHSWORDMODULES);
}
// Update installed SWORD modules, shortly after the module list has been refreshed.
if ((!client) && (hour == 3) && (minute == 15)) {
tasks_logic_queue (UPDATESWORDMODULES);
}
}
// The Cloud updates the list of USFM resources once a week.
if (weekday == 1) {
if ((!client) && (hour == 3) && (minute == 10)) {
tasks_logic_queue (LISTUSFMRESOURCES);
}
}
} catch (exception & e) {
Database_Logs::log (e.what ());
} catch (exception * e) {
Database_Logs::log (e->what ());
} catch (...) {
Database_Logs::log ("A general internal error occurred in the timers");
}
}
}
void changes_modifications ()
{
Database_Logs::log ("Change notifications: Generating", Filter_Roles::translator ());
// Notifications are not available to clients for the duration of processing them.
config_globals_change_notifications_available = false;
// Data objects.
Webserver_Request request;
Database_Modifications database_modifications;
// Check on the health of the modifications database and (re)create it if needed.
if (!database_modifications.healthy ()) database_modifications.erase ();
database_modifications.create ();
// Create online change notifications for users who made changes in Bibles
// through the web editor or through a client.
// It runs before the team changes.
// This produces the desired order of the notifications in the GUI.
// At the same time, produce change statistics per user.
// Get the users who will receive the changes entered by the contributors.
vector <string> recipients;
{
vector <string> users = request.database_users ()->getUsers ();
for (auto & user : users) {
if (request.database_config_user ()->getContributorChangesNotificationsOnline (user)) {
recipients.push_back (user);
}
}
}
// Storage for the statistics.
map <string, int> user_change_statistics;
float modification_time_total = 0;
int modification_time_count = 0;
vector <string> users = database_modifications.getUserUsernames ();
if (!users.empty ()) Database_Logs::log ("Change notifications: Per user", Filter_Roles::translator ());
for (auto user : users) {
// Total changes made by this user.
int change_count = 0;
// Go through the Bibles changed by the current user.
vector <string> bibles = database_modifications.getUserBibles (user);
for (auto bible : bibles) {
// Body of the email to be sent.
string email = "<p>" + translate("You have entered the changes below in a Bible editor.") + " " + translate ("You may check if it made its way into the Bible text.") + "</p>";
size_t empty_email_length = email.length ();
// Go through the books in that Bible.
vector <int> books = database_modifications.getUserBooks (user, bible);
for (auto book : books) {
// Go through the chapters in that book.
vector <int> chapters = database_modifications.getUserChapters (user, bible, book);
for (auto chapter : chapters) {
// Get the sets of identifiers for that chapter, and set some variables.
vector <Database_Modifications_Id> IdSets = database_modifications.getUserIdentifiers (user, bible, book, chapter);
//int referenceOldId = 0;
int referenceNewId = 0;
int newId = 0;
int lastNewId = 0;
bool restart = true;
// Go through the sets of identifiers.
for (auto IdSet : IdSets) {
int oldId = IdSet.oldid;
newId = IdSet.newid;
if (restart) {
changes_process_identifiers (&request, user, recipients, bible, book, chapter, referenceNewId, newId, email, change_count, modification_time_total, modification_time_count);
//referenceOldId = oldId;
referenceNewId = newId;
lastNewId = newId;
restart = false;
continue;
}
if (oldId == lastNewId) {
lastNewId = newId;
} else {
restart = true;
}
}
// Process the last set of identifiers.
changes_process_identifiers (&request, user, recipients, bible, book, chapter, referenceNewId, newId, email, change_count, modification_time_total, modification_time_count);
}
}
// Check whether there's any email to be sent.
if (email.length () != empty_email_length) {
// Send the user email with the user's personal changes if the user opted to receive it.
if (request.database_config_user()->getUserUserChangesNotification (user)) {
string subject = translate("Changes you entered in") + " " + bible;
if (!client_logic_client_enabled ()) email_schedule (user, subject, email);
}
}
}
// Store change statistics for this user.
user_change_statistics [user] = change_count;
// Clear the user's changes in the database.
database_modifications.clearUserUser (user);
// Clear checksum cache.
request.database_config_user ()->setUserChangeNotificationsChecksum (user, "");
}
// Generate the notifications, online and by email,
// for the changes in the Bibles entered by anyone since the previous notifications were generated.
vector <string> bibles = database_modifications.getTeamDiffBibles ();
for (auto bible : bibles) {
string stylesheet = Database_Config_Bible::getExportStylesheet (bible);
vector <string> changeNotificationUsers;
vector <string> users = request.database_users ()->getUsers ();
for (auto user : users) {
if (access_bible_read (&request, bible, user)) {
if (request.database_config_user()->getUserGenerateChangeNotifications (user)) {
changeNotificationUsers.push_back (user);
}
}
}
users.clear ();
// The number of changes processed so far for this Bible.
int processedChangesCount = 0;
// The files get stored at http://site.org:<port>/revisions/<Bible>/<date>
int seconds = filter_date_seconds_since_epoch ();
string timepath;
timepath.append (convert_to_string (filter_date_numerical_year (seconds)));
timepath.append ("-");
timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_month (seconds)), 2, '0'));
timepath.append ("-");
timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_month_day (seconds)), 2, '0'));
timepath.append (" ");
timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_hour (seconds)), 2, '0'));
timepath.append (":");
timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_minute (seconds)), 2, '0'));
timepath.append (":");
timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_second (seconds)), 2, '0'));
string directory = filter_url_create_root_path ("revisions", bible, timepath);
filter_url_mkdir (directory);
// Produce the USFM and html files.
filter_diff_produce_verse_level (bible, directory);
// Create online page with changed verses.
string versesoutputfile = filter_url_create_path (directory, "changed_verses.html");
filter_diff_run_file (filter_url_create_path (directory, "verses_old.txt"), filter_url_create_path (directory, "verses_new.txt"), versesoutputfile);
// Storage for body of the email with the changes.
vector <string> email_changes;
// Generate the online change notifications.
vector <int> books = database_modifications.getTeamDiffBooks (bible);
for (auto book : books) {
vector <int> chapters = database_modifications.getTeamDiffChapters (bible, book);
for (auto chapter : chapters) {
Database_Logs::log ("Change notifications: " + bible + " " + filter_passage_display (book, chapter, ""), Filter_Roles::translator ());
string old_chapter_usfm = database_modifications.getTeamDiff (bible, book, chapter);
string new_chapter_usfm = request.database_bibles()->getChapter (bible, book, chapter);
vector <int> old_verse_numbers = usfm_get_verse_numbers (old_chapter_usfm);
vector <int> new_verse_numbers = usfm_get_verse_numbers (new_chapter_usfm);
vector <int> verses = old_verse_numbers;
verses.insert (verses.end (), new_verse_numbers.begin (), new_verse_numbers.end ());
verses = array_unique (verses);
sort (verses.begin (), verses.end());
for (auto verse : verses) {
string old_verse_usfm = usfm_get_verse_text (old_chapter_usfm, verse);
string new_verse_usfm = usfm_get_verse_text (new_chapter_usfm, verse);
if (old_verse_usfm != new_verse_usfm) {
processedChangesCount++;
// In case of too many change notifications, processing them would take too much time, so take a few shortcuts.
string old_html = "<p>" + old_verse_usfm + "</p>";
string new_html = "<p>" + new_verse_usfm + "</p>";
string old_text = old_verse_usfm;
string new_text = new_verse_usfm;
if (processedChangesCount < 800) {
Filter_Text filter_text_old = Filter_Text (bible);
Filter_Text filter_text_new = Filter_Text (bible);
filter_text_old.html_text_standard = new Html_Text ("");
filter_text_new.html_text_standard = new Html_Text ("");
filter_text_old.text_text = new Text_Text ();
filter_text_new.text_text = new Text_Text ();
filter_text_old.addUsfmCode (old_verse_usfm);
filter_text_new.addUsfmCode (new_verse_usfm);
filter_text_old.run (stylesheet);
filter_text_new.run (stylesheet);
old_html = filter_text_old.html_text_standard->getInnerHtml ();
new_html = filter_text_new.html_text_standard->getInnerHtml ();
old_text = filter_text_old.text_text->get ();
new_text = filter_text_new.text_text->get ();
}
string modification = filter_diff_diff (old_text, new_text);
database_modifications.recordNotification (changeNotificationUsers, changes_bible_category (), bible, book, chapter, verse, old_html, modification, new_html);
string passage = filter_passage_display (book, chapter, convert_to_string (verse)) + ": ";
if (old_text != new_text) {
email_changes.push_back (passage + modification);
} else {
email_changes.push_back (translate ("The following passage has no change in the text.") + " " + translate ("The change is in the formatting only.") + " " + translate ("The USFM code is given for reference."));
email_changes.push_back (passage);
email_changes.push_back (translate ("Old code:") + " " + old_verse_usfm);
email_changes.push_back (translate ("New code:") + " " + new_verse_usfm);
}
}
}
// Delete the diff data for this chapter, for two reasons:
// 1. New diffs for this chapter can be stored straightaway.
// 2. In case of large amounts of diff data, and this function gets killed,
// then the next time it runs again, it will continue from where it was killed.
database_modifications.deleteTeamDiffChapter (bible, book, chapter);
}
}
// Email the changes to the subscribed users.
if (!email_changes.empty ()) {
string body;
for (auto & line : email_changes) {
body.append ("<div>");
body.append (line);
body.append ("</div>\n");
}
string subject = translate("Recent changes:") + " " + bible;
vector <string> users = request.database_users ()->getUsers ();
for (auto & user : users) {
if (request.database_config_user()->getUserBibleChangesNotification (user)) {
if (access_bible_read (&request, bible, user)) {
if (!client_logic_client_enabled ()) {
email_schedule (user, subject, body);
}
}
}
}
}
}
// Index the data and remove expired notifications.
Database_Logs::log ("Change notifications: Indexing", Filter_Roles::translator ());
database_modifications.indexTrimAllNotifications ();
// Remove expired downloadable revisions.
string directory = filter_url_create_root_path ("revisions");
int now = filter_date_seconds_since_epoch ();
bibles = filter_url_scandir (directory);
for (auto &bible : bibles) {
string folder = filter_url_create_path (directory, bible);
int time = filter_url_file_modification_time (folder);
int days = (now - time) / 86400;
if (days > 31) {
filter_url_rmdir (folder);
} else {
vector <string> revisions = filter_url_scandir (folder);
for (auto & revision : revisions) {
string path = filter_url_create_path (folder, revision);
int time = filter_url_file_modification_time (path);
int days = (now - time) / 86400;
if (days > 31) {
filter_url_rmdir (path);
Database_Logs::log ("Removing expired downloadable revision notification: " + bible + " " + revision, Filter_Roles::translator ());
}
}
}
}
// Clear checksum caches.
users = request.database_users ()->getUsers ();
for (auto user : users) {
request.database_config_user ()->setUserChangeNotificationsChecksum (user, "");
}
// Vacuum the modifications index, as it might have been updated.
database_modifications.vacuum ();
// Make the notifications available again to clients.
config_globals_change_notifications_available = true;
// Store the statistics in the database.
if (modification_time_count) {
// Take average timestamp of all timestamps.
int timestamp = round (modification_time_total / modification_time_count);
for (auto & element : user_change_statistics) {
// Store dated change statistics per user.
string user = element.first;
int count = element.second;
Database_Statistics::store_changes (timestamp, user, count);
}
}
Database_Logs::log ("Change notifications: Ready", Filter_Roles::translator ());
}
