// Prepares a sample Bible.
// The output of this is supposed to be manually put into the source tree, folder "samples".
// This will be used to quickly create a sample Bible, that is fast, even on mobile devices.
void demo_prepare_sample_bible ()
{
Database_Bibles database_bibles;
// Remove the Bible to remove all stuff that might have been in it.
database_bibles.deleteBible (demo_sample_bible_name ());
search_logic_delete_bible (demo_sample_bible_name ());
// Create a new one.
database_bibles.createBible (demo_sample_bible_name ());
// Location of the USFM files for the sample Bible.
string directory = filter_url_create_root_path ("demo");
vector <string> files = filter_url_scandir (directory);
for (auto file : files) {
// Only process the USFM files.
if (filter_url_get_extension (file) == "usfm") {
cout << file << endl;
// Read the USFM.
file = filter_url_create_path (directory, file);
string usfm = filter_url_file_get_contents (file);
usfm = filter_string_str_replace (" ", " ", usfm);
// Import the USFM into the Bible.
vector <BookChapterData> book_chapter_data = usfm_import (usfm, styles_logic_standard_sheet ());
for (auto data : book_chapter_data) {
Bible_Logic::storeChapter (demo_sample_bible_name (), data.book, data.chapter, data.data);
}
}
}
// Clean the destination location for the Bible.
string destination = sample_bible_bible_path ();
filter_url_rmdir (destination);
// Copy the Bible data to the destination.
string source = database_bibles.bibleFolder (demo_sample_bible_name ());
filter_url_dir_cp (source, destination);
// Clean the destination location for the Bible search index.
destination = sample_bible_index_path ();
filter_url_rmdir (destination);
// Create destination location.
filter_url_mkdir (destination);
// Copy the index files over to the destination.
source = search_logic_index_folder ();
files = filter_url_scandir (source);
for (auto file : files) {
if (file.find (demo_sample_bible_name ()) != string::npos) {
string source_file = filter_url_create_path (source, file);
string destination_file = filter_url_create_path (destination, file);
filter_url_file_cp (source_file, destination_file);
}
}
}
void tmp_tmp ()
{
Database_Logs::log ("Removing expired temporal files", Filter_Roles::admin ());
int expired = filter_date_seconds_since_epoch () - (3600 * 24 * 3);
string directory = filter_url_create_root_path ("tmp");
vector <string> names = filter_url_scandir (directory);
for (auto & name : names) {
if (name.find ("tmp.") == 0) continue;
string filename = filter_url_create_path (directory, name);
int mtime = filter_url_filemtime (filename);
if (mtime < expired) {
filter_url_rmdir (filename);
filter_url_unlink (filename);
}
}
}
// Uncompresses a .tar.gz archive identified by $file.
// Returns the path to the folder it created.
string filter_archive_untar_gzip (string file)
{
file = filter_url_escape_shell_argument (file);
string folder = filter_url_tempfile ();
filter_url_mkdir (folder);
folder.append (DIRECTORY_SEPARATOR);
string logfile = filter_url_tempfile () + ".log";
string command = "cd " + folder + " && tar zxf " + file + " > " + logfile + " 2>&1";
int return_var = system (command.c_str());
if (return_var != 0) {
filter_url_rmdir (folder);
folder.clear();
string errors = filter_url_file_get_contents (logfile);
Database_Logs::log (errors);
}
return folder;
}
// Removes directory recursively.
void filter_url_rmdir (string directory)
{
DIR *dir;
struct dirent *entry;
char path[PATH_MAX];
dir = opendir(directory.c_str());
if (dir == NULL) return;
while ((entry = readdir (dir)) != NULL) {
if (strcmp (entry->d_name, ".") && strcmp (entry->d_name, "..")) {
snprintf (path, (size_t) PATH_MAX, "%s/%s", directory.c_str(), entry->d_name);
if (entry->d_type == DT_DIR) {
filter_url_rmdir (path);
}
remove (path);
}
}
closedir(dir);
remove (directory.c_str());
}
// Uncompresses a zip archive identified by $file.
// Returns the path to the folder it created.
string filter_archive_unzip (string file)
{
string folder = filter_url_tempfile ();
filter_url_mkdir (folder);
folder.append (DIRECTORY_SEPARATOR);
string logfile = filter_url_tempfile () + ".log";
file = filter_url_escape_shell_argument (file);
string command = "unzip -o -d " + folder + " " + file + " > " + logfile + " 2>&1";
int return_var = system (command.c_str());
if (return_var != 0) {
filter_url_rmdir (folder);
folder.clear();
string errors = filter_url_file_get_contents (logfile);
Database_Logs::log (errors);
} else {
// Set free permissions after unzipping.
command = "chmod -R 0777 " + folder;
int result = system (command.c_str ());
(void) result;
}
return folder;
}
void Database_Config_Bible::remove (string bible)
{
string folder = file (bible);
filter_url_rmdir (folder);
}
void database_privileges_client_remove (const string & user)
{
string path = database_privileges_client_path (user);
path = filter_url_dirname (path);
filter_url_rmdir (path);
}
string bible_manage (void * webserver_request)
{
Webserver_Request * request = (Webserver_Request *) webserver_request;
string page;
Assets_Header header = Assets_Header (translate("Bibles"), webserver_request);
header.addBreadCrumb (menu_logic_settings_menu (), menu_logic_settings_text ());
page = header.run ();
Assets_View view;
string success_message;
string error_message;
// New Bible handler.
if (request->query.count ("new")) {
Dialog_Entry dialog_entry = Dialog_Entry ("manage", translate("Please enter a name for the new empty Bible"), "", "new", "");
page += dialog_entry.run ();
return page;
}
if (request->post.count ("new")) {
string bible = request->post ["entry"];
vector <string> bibles = request->database_bibles ()->getBibles ();
if (find (bibles.begin(), bibles.end(), bible) != bibles.end()) {
error_message = translate("This Bible already exists");
} else {
request->database_bibles ()->createBible (bible);
// Check / grant access.
if (!access_bible_write (request, bible)) {
string me = request->session_logic ()->currentUser ();
Database_Privileges::setBible (me, bible, true);
}
success_message = translate("The Bible was created");
// Creating a Bible removes any Sample Bible that might have been there.
if (!config_logic_demo_enabled ()) {
request->database_bibles ()->deleteBible (demo_sample_bible_name ());
search_logic_delete_bible (demo_sample_bible_name ());
}
}
}
// Copy Bible handler.
if (request->query.count ("copy")) {
string copy = request->query["copy"];
Dialog_Entry dialog_entry = Dialog_Entry ("manage", translate("Please enter a name for where to copy the Bible to"), "", "", "A new Bible will be created with the given name, and the current Bible copied to it");
dialog_entry.add_query ("origin", copy);
page += dialog_entry.run ();
return page;
}
if (request->query.count ("origin")) {
string origin = request->query["origin"];
if (request->post.count ("entry")) {
string destination = request->post["entry"];
vector <string> bibles = request->database_bibles ()->getBibles ();
if (find (bibles.begin(), bibles.end(), destination) != bibles.end()) {
error_message = translate("Cannot copy the Bible because the destination Bible already exists.");
} else {
// User needs read access to the original.
if (access_bible_read (request, origin)) {
request->database_bibles ()->createBible (destination);
vector <int> books = request->database_bibles ()->getBooks (origin);
for (auto & book : books) {
vector <int> chapters = request->database_bibles ()->getChapters (origin, book);
for (auto & chapter : chapters) {
string data = request->database_bibles ()->getChapter (origin, book, chapter);
Bible_Logic::storeChapter (destination, book, chapter, data);
}
}
success_message = translate("The Bible was copied.");
// Check / grant access to destination Bible.
if (!access_bible_write (request, destination)) {
string me = request->session_logic ()->currentUser ();
Database_Privileges::setBible (me, destination, true);
}
// Creating a Bible removes any Sample Bible that might have been there.
if (!config_logic_demo_enabled ()) {
request->database_bibles ()->deleteBible (demo_sample_bible_name ());
search_logic_delete_bible (demo_sample_bible_name ());
}
}
}
}
}
// Delete Bible handler.
if (request->query.count ("delete")) {
string bible = request->query ["delete"];
string confirm = request->query ["confirm"];
if (confirm == "yes") {
// User needs write access for delete operation.
if (access_bible_write (request, bible)) {
Bible_Logic::deleteBible (bible);
string gitdirectory = filter_git_directory (bible);
if (file_exists (gitdirectory)) {
filter_url_rmdir (gitdirectory);
}
// Remove associated settings and privileges.
Database_Privileges::removeBible (bible);
Database_Config_Bible::remove (bible);
} else {
page += Assets_Page::error ("Insufficient privileges to complete action");
}
}
if (confirm == "") {
Dialog_Yes dialog_yes = Dialog_Yes ("manage", translate("Would you like to delete this Bible?") + " (" + bible + ")");
dialog_yes.add_query ("delete", bible);
page += dialog_yes.run ();
return page;
}
}
view.set_variable ("success_message", success_message);
view.set_variable ("error_message", error_message);
vector <string> bibles = access_bible_bibles (request);
string bibleblock;
for (auto & bible : bibles) {
bibleblock.append ("<li><a href=\"settings?bible=" + bible + "\">" + bible + "</a></li>\n");
}
view.set_variable ("bibleblock", bibleblock);
if (!client_logic_client_enabled ()) view.enable_zone ("server");
page += view.render ("bible", "manage");
page += Assets_Page::footer ();
return page;
}
void export_index ()
{
Database_Bibles database_bibles;
vector <string> bibles = database_bibles.getBibles ();
// Go through all sub directories of the exports directory.
// Remove subdirectories if their corresponding Bible no longer exists in the system.
string directory = Export_Logic::mainDirectory ();
vector <string> files = filter_url_scandir (directory);
for (auto & file : files) {
if (in_array (file, bibles)) continue;
filter_url_rmdir (filter_url_create_path (directory, file));
Database_Logs::log ("Removing exported Bible " + file, Filter_Roles::admin ());
}
// Schedule the relevant Bibles for export.
for (auto bible : bibles) {
if (Database_State::getExport (bible, 0, Export_Logic::export_needed)) {
Database_State::clearExport (bible, 0, Export_Logic::export_needed);
vector <int> books = database_bibles.getBooks (bible);
// Book 0 flags export of the whole Bible (this is not relevant to all export types).
books.push_back (0);
for (auto book : books) {
for (int format = Export_Logic::export_needed + 1; format < Export_Logic::export_end; format++) {
Database_State::setExport (bible, book, format);
}
}
if (Database_Config_Bible::getExportWebDuringNight (bible)) {
Export_Logic::scheduleWeb (bible);
Export_Logic::scheduleWebIndex (bible);
}
if (Database_Config_Bible::getExportHtmlDuringNight (bible)) {
Export_Logic::scheduleHtml (bible);
}
if (Database_Config_Bible::getExportUsfmDuringNight (bible)) {
Export_Logic::scheduleUsfm (bible);
}
if (Database_Config_Bible::getExportTextDuringNight (bible)) {
Export_Logic::scheduleTextAndBasicUsfm (bible);
}
if (Database_Config_Bible::getExportOdtDuringNight (bible)) {
Export_Logic::scheduleOpenDocument (bible);
}
if (Database_Config_Bible::getGenerateInfoDuringNight (bible)) {
Export_Logic::scheduleInfo (bible);
}
if (Database_Config_Bible::getExportESwordDuringNight (bible)) {
Export_Logic::scheduleESword (bible);
}
if (Database_Config_Bible::getExportOnlineBibleDuringNight (bible)) {
Export_Logic::scheduleOnlineBible (bible);
}
}
}
}
void changes_modifications ()
{
Database_Logs::log ("Change notifications: Generating", Filter_Roles::translator ());
// Notifications are not available to clients for the duration of processing them.
config_globals_change_notifications_available = false;
// Data objects.
Webserver_Request request;
Database_Modifications database_modifications;
// Check on the health of the modifications database and (re)create it if needed.
if (!database_modifications.healthy ()) database_modifications.erase ();
database_modifications.create ();
// Create online change notifications for users who made changes in Bibles
// through the web editor or through a client.
// It runs before the team changes.
// This produces the desired order of the notifications in the GUI.
// At the same time, produce change statistics per user.
// Get the users who will receive the changes entered by the contributors.
vector <string> recipients;
{
vector <string> users = request.database_users ()->getUsers ();
for (auto & user : users) {
if (request.database_config_user ()->getContributorChangesNotificationsOnline (user)) {
recipients.push_back (user);
}
}
}
// Storage for the statistics.
map <string, int> user_change_statistics;
float modification_time_total = 0;
int modification_time_count = 0;
vector <string> users = database_modifications.getUserUsernames ();
if (!users.empty ()) Database_Logs::log ("Change notifications: Per user", Filter_Roles::translator ());
for (auto user : users) {
// Total changes made by this user.
int change_count = 0;
// Go through the Bibles changed by the current user.
vector <string> bibles = database_modifications.getUserBibles (user);
for (auto bible : bibles) {
// Body of the email to be sent.
string email = "<p>" + translate("You have entered the changes below in a Bible editor.") + " " + translate ("You may check if it made its way into the Bible text.") + "</p>";
size_t empty_email_length = email.length ();
// Go through the books in that Bible.
vector <int> books = database_modifications.getUserBooks (user, bible);
for (auto book : books) {
// Go through the chapters in that book.
vector <int> chapters = database_modifications.getUserChapters (user, bible, book);
for (auto chapter : chapters) {
// Get the sets of identifiers for that chapter, and set some variables.
vector <Database_Modifications_Id> IdSets = database_modifications.getUserIdentifiers (user, bible, book, chapter);
//int referenceOldId = 0;
int referenceNewId = 0;
int newId = 0;
int lastNewId = 0;
bool restart = true;
// Go through the sets of identifiers.
for (auto IdSet : IdSets) {
int oldId = IdSet.oldid;
newId = IdSet.newid;
if (restart) {
changes_process_identifiers (&request, user, recipients, bible, book, chapter, referenceNewId, newId, email, change_count, modification_time_total, modification_time_count);
//referenceOldId = oldId;
referenceNewId = newId;
lastNewId = newId;
restart = false;
continue;
}
if (oldId == lastNewId) {
lastNewId = newId;
} else {
restart = true;
}
}
// Process the last set of identifiers.
changes_process_identifiers (&request, user, recipients, bible, book, chapter, referenceNewId, newId, email, change_count, modification_time_total, modification_time_count);
}
}
// Check whether there's any email to be sent.
if (email.length () != empty_email_length) {
// Send the user email with the user's personal changes if the user opted to receive it.
if (request.database_config_user()->getUserUserChangesNotification (user)) {
string subject = translate("Changes you entered in") + " " + bible;
if (!client_logic_client_enabled ()) email_schedule (user, subject, email);
}
}
}
// Store change statistics for this user.
user_change_statistics [user] = change_count;
// Clear the user's changes in the database.
database_modifications.clearUserUser (user);
// Clear checksum cache.
request.database_config_user ()->setUserChangeNotificationsChecksum (user, "");
}
// Generate the notifications, online and by email,
// for the changes in the Bibles entered by anyone since the previous notifications were generated.
vector <string> bibles = database_modifications.getTeamDiffBibles ();
for (auto bible : bibles) {
string stylesheet = Database_Config_Bible::getExportStylesheet (bible);
vector <string> changeNotificationUsers;
vector <string> users = request.database_users ()->getUsers ();
for (auto user : users) {
if (access_bible_read (&request, bible, user)) {
if (request.database_config_user()->getUserGenerateChangeNotifications (user)) {
changeNotificationUsers.push_back (user);
}
}
}
users.clear ();
// The number of changes processed so far for this Bible.
int processedChangesCount = 0;
// The files get stored at http://site.org:<port>/revisions/<Bible>/<date>
int seconds = filter_date_seconds_since_epoch ();
string timepath;
timepath.append (convert_to_string (filter_date_numerical_year (seconds)));
timepath.append ("-");
timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_month (seconds)), 2, '0'));
timepath.append ("-");
timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_month_day (seconds)), 2, '0'));
timepath.append (" ");
timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_hour (seconds)), 2, '0'));
timepath.append (":");
timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_minute (seconds)), 2, '0'));
timepath.append (":");
timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_second (seconds)), 2, '0'));
string directory = filter_url_create_root_path ("revisions", bible, timepath);
filter_url_mkdir (directory);
// Produce the USFM and html files.
filter_diff_produce_verse_level (bible, directory);
// Create online page with changed verses.
string versesoutputfile = filter_url_create_path (directory, "changed_verses.html");
filter_diff_run_file (filter_url_create_path (directory, "verses_old.txt"), filter_url_create_path (directory, "verses_new.txt"), versesoutputfile);
// Storage for body of the email with the changes.
vector <string> email_changes;
// Generate the online change notifications.
vector <int> books = database_modifications.getTeamDiffBooks (bible);
for (auto book : books) {
vector <int> chapters = database_modifications.getTeamDiffChapters (bible, book);
for (auto chapter : chapters) {
Database_Logs::log ("Change notifications: " + bible + " " + filter_passage_display (book, chapter, ""), Filter_Roles::translator ());
string old_chapter_usfm = database_modifications.getTeamDiff (bible, book, chapter);
string new_chapter_usfm = request.database_bibles()->getChapter (bible, book, chapter);
vector <int> old_verse_numbers = usfm_get_verse_numbers (old_chapter_usfm);
vector <int> new_verse_numbers = usfm_get_verse_numbers (new_chapter_usfm);
vector <int> verses = old_verse_numbers;
verses.insert (verses.end (), new_verse_numbers.begin (), new_verse_numbers.end ());
verses = array_unique (verses);
sort (verses.begin (), verses.end());
for (auto verse : verses) {
string old_verse_usfm = usfm_get_verse_text (old_chapter_usfm, verse);
string new_verse_usfm = usfm_get_verse_text (new_chapter_usfm, verse);
if (old_verse_usfm != new_verse_usfm) {
processedChangesCount++;
// In case of too many change notifications, processing them would take too much time, so take a few shortcuts.
string old_html = "<p>" + old_verse_usfm + "</p>";
string new_html = "<p>" + new_verse_usfm + "</p>";
string old_text = old_verse_usfm;
string new_text = new_verse_usfm;
if (processedChangesCount < 800) {
Filter_Text filter_text_old = Filter_Text (bible);
Filter_Text filter_text_new = Filter_Text (bible);
filter_text_old.html_text_standard = new Html_Text ("");
filter_text_new.html_text_standard = new Html_Text ("");
filter_text_old.text_text = new Text_Text ();
filter_text_new.text_text = new Text_Text ();
filter_text_old.addUsfmCode (old_verse_usfm);
filter_text_new.addUsfmCode (new_verse_usfm);
filter_text_old.run (stylesheet);
filter_text_new.run (stylesheet);
old_html = filter_text_old.html_text_standard->getInnerHtml ();
new_html = filter_text_new.html_text_standard->getInnerHtml ();
old_text = filter_text_old.text_text->get ();
new_text = filter_text_new.text_text->get ();
}
string modification = filter_diff_diff (old_text, new_text);
database_modifications.recordNotification (changeNotificationUsers, changes_bible_category (), bible, book, chapter, verse, old_html, modification, new_html);
string passage = filter_passage_display (book, chapter, convert_to_string (verse)) + ": ";
if (old_text != new_text) {
email_changes.push_back (passage + modification);
} else {
email_changes.push_back (translate ("The following passage has no change in the text.") + " " + translate ("The change is in the formatting only.") + " " + translate ("The USFM code is given for reference."));
email_changes.push_back (passage);
email_changes.push_back (translate ("Old code:") + " " + old_verse_usfm);
email_changes.push_back (translate ("New code:") + " " + new_verse_usfm);
}
}
}
// Delete the diff data for this chapter, for two reasons:
// 1. New diffs for this chapter can be stored straightaway.
// 2. In case of large amounts of diff data, and this function gets killed,
// then the next time it runs again, it will continue from where it was killed.
database_modifications.deleteTeamDiffChapter (bible, book, chapter);
}
}
// Email the changes to the subscribed users.
if (!email_changes.empty ()) {
string body;
for (auto & line : email_changes) {
body.append ("<div>");
body.append (line);
body.append ("</div>\n");
}
string subject = translate("Recent changes:") + " " + bible;
vector <string> users = request.database_users ()->getUsers ();
for (auto & user : users) {
if (request.database_config_user()->getUserBibleChangesNotification (user)) {
if (access_bible_read (&request, bible, user)) {
if (!client_logic_client_enabled ()) {
email_schedule (user, subject, body);
}
}
}
}
}
}
// Index the data and remove expired notifications.
Database_Logs::log ("Change notifications: Indexing", Filter_Roles::translator ());
database_modifications.indexTrimAllNotifications ();
// Remove expired downloadable revisions.
string directory = filter_url_create_root_path ("revisions");
int now = filter_date_seconds_since_epoch ();
bibles = filter_url_scandir (directory);
for (auto &bible : bibles) {
string folder = filter_url_create_path (directory, bible);
int time = filter_url_file_modification_time (folder);
int days = (now - time) / 86400;
if (days > 31) {
filter_url_rmdir (folder);
} else {
vector <string> revisions = filter_url_scandir (folder);
for (auto & revision : revisions) {
string path = filter_url_create_path (folder, revision);
int time = filter_url_file_modification_time (path);
int days = (now - time) / 86400;
if (days > 31) {
filter_url_rmdir (path);
Database_Logs::log ("Removing expired downloadable revision notification: " + bible + " " + revision, Filter_Roles::translator ());
}
}
}
}
// Clear checksum caches.
users = request.database_users ()->getUsers ();
for (auto user : users) {
request.database_config_user ()->setUserChangeNotificationsChecksum (user, "");
}
// Vacuum the modifications index, as it might have been updated.
database_modifications.vacuum ();
// Make the notifications available again to clients.
config_globals_change_notifications_available = true;
// Store the statistics in the database.
if (modification_time_count) {
// Take average timestamp of all timestamps.
int timestamp = round (modification_time_total / modification_time_count);
for (auto & element : user_change_statistics) {
// Store dated change statistics per user.
string user = element.first;
int count = element.second;
Database_Statistics::store_changes (timestamp, user, count);
}
}
Database_Logs::log ("Change notifications: Ready", Filter_Roles::translator ());
}
