// Prepares a sample Bible. // The output of this is supposed to be manually put into the source tree, folder "samples". // This will be used to quickly create a sample Bible, that is fast, even on mobile devices. void demo_prepare_sample_bible () { Database_Bibles database_bibles; // Remove the Bible to remove all stuff that might have been in it. database_bibles.deleteBible (demo_sample_bible_name ()); search_logic_delete_bible (demo_sample_bible_name ()); // Create a new one. database_bibles.createBible (demo_sample_bible_name ()); // Location of the USFM files for the sample Bible. string directory = filter_url_create_root_path ("demo"); vector <string> files = filter_url_scandir (directory); for (auto file : files) { // Only process the USFM files. if (filter_url_get_extension (file) == "usfm") { cout << file << endl; // Read the USFM. file = filter_url_create_path (directory, file); string usfm = filter_url_file_get_contents (file); usfm = filter_string_str_replace (" ", " ", usfm); // Import the USFM into the Bible. vector <BookChapterData> book_chapter_data = usfm_import (usfm, styles_logic_standard_sheet ()); for (auto data : book_chapter_data) { Bible_Logic::storeChapter (demo_sample_bible_name (), data.book, data.chapter, data.data); } } } // Clean the destination location for the Bible. string destination = sample_bible_bible_path (); filter_url_rmdir (destination); // Copy the Bible data to the destination. string source = database_bibles.bibleFolder (demo_sample_bible_name ()); filter_url_dir_cp (source, destination); // Clean the destination location for the Bible search index. destination = sample_bible_index_path (); filter_url_rmdir (destination); // Create destination location. filter_url_mkdir (destination); // Copy the index files over to the destination. source = search_logic_index_folder (); files = filter_url_scandir (source); for (auto file : files) { if (file.find (demo_sample_bible_name ()) != string::npos) { string source_file = filter_url_create_path (source, file); string destination_file = filter_url_create_path (destination, file); filter_url_file_cp (source_file, destination_file); } } }
// If $command and $parameters are queued as a task, the function returns true. // Else it returns false. // It looks for an exact match. // Parameters left out are not checked. bool tasks_logic_queued (string command, vector <string> parameters) { // The lines to look for consist of the command followed by the parameters. vector <string> search (parameters); search.insert (search.begin (), command); // Go through all queued tasks. vector <string> files = filter_url_scandir (tasks_logic_folder ()); for (auto & file : files) { // Read the task's contents. string contents = filter_url_file_get_contents (filter_url_create_path (tasks_logic_folder (), file)); vector <string> lines = filter_string_explode (contents, '\n'); if (lines.empty ()) return false; // Look for a match. bool match = true; for (size_t i = 0; i < search.size (); i++) { if (i < lines.size ()) { if (search [i] != lines[i]) match = false; } else { match = false; } } if (match) return true; } // No match found. return false; }
// Clears all journal entries. void Database_Logs::clear () { string directory = folder (); vector <string> files = filter_url_scandir (directory); for (auto file : files) { filter_url_unlink (filter_url_create_path (directory, file)); } log ("The journal was cleared"); }
int main (int argc, char **argv) { if (argc) {}; if (argv[0]) {}; string consultations_in = "/home/2000/consultations"; string consultations_out = "/home/2000/2000/consultations"; vector <int> identifiers; vector <string> bits1 = filter_url_scandir (consultations_in); for (auto & bit1 : bits1) { if (bit1.length () != 3) continue; string directory_in = consultations_in + "/" + bit1; string directory_out = consultations_out + "/" + bit1; mkdir (directory_out.c_str(), 0777); vector <string> bits2 = filter_url_scandir (directory_in); for (auto & bit2 : bits2) { if (bit2.length () != 3) continue; string directory_in = consultations_in + "/" + bit1 + "/" + bit2; string directory_out = consultations_out + "/" + bit1 + "/" + bit2; mkdir (directory_out.c_str(), 0777); vector <string> bits3 = filter_url_scandir (directory_in); for (auto & bit3 : bits3) { if (bit3.length () != 3) continue; if (bit3[0] != '0') continue; string directory_in = consultations_in + "/" + bit1 + "/" + bit2 + "/" + bit3; string directory_out = consultations_out + "/" + bit1 + "/" + bit2 + "/" + bit3; mkdir (directory_out.c_str(), 0777); int identifier = stoi (bit1 + bit2 + bit3); identifiers.push_back (identifier); string command = "cp " + directory_in + "/* " + directory_out; system (command.c_str ()); cout << command << endl; // Todo } } } cout << identifiers.size () << endl; return EXIT_SUCCESS; }
// Recursively scans a directory for directories and files. void filter_url_recursive_scandir (string folder, vector <string> & paths) { vector <string> files = filter_url_scandir (folder); for (auto & file : files) { string path = filter_url_create_path (folder, file); paths.push_back (path); if (filter_url_is_dir (path)) { filter_url_recursive_scandir (path, paths); } } }
void search_logic_delete_chapter (string bible, int book, int chapter) { string fragment = search_logic_chapter_file (bible, book, chapter); fragment = filter_url_basename (fragment); vector <string> files = filter_url_scandir (search_logic_index_folder ()); for (auto & file : files) { if (file.find (fragment) == 0) { string path = filter_url_create_path (search_logic_index_folder (), file); filter_url_unlink (path); } } }
string index_listing (void * webserver_request, string url) { string page; page = Assets_Page::header ("Bibledit", webserver_request); // No breadcrumbs because the user can arrive here from more than one place. Assets_View view; url = filter_url_urldecode (url); url = filter_url_create_path ("", url); url = filter_string_str_replace ("\\", "/", url); view.set_variable ("url", url); string parent = filter_url_dirname_web (url); if (parent.length () > 1) { view.enable_zone ("parent"); view.set_variable ("parent", parent); } string directory = filter_url_create_root_path (url); if (!file_or_dir_exists (directory) || filter_url_is_dir (directory)) { vector <string> files = filter_url_scandir (directory); for (auto & file : files) { string path = filter_url_create_path (directory, file); string line; line.append ("<tr>"); line.append ("<td>"); line.append ("<a href=\"" + filter_url_create_path (url, file) + "\">"); line.append (file); line.append ("</a>"); line.append ("</td>"); line.append ("<td>"); if (!filter_url_is_dir (path)) { line.append (convert_to_string (filter_url_filesize (path))); } line.append ("</td>"); line.append ("</tr>"); file = line; } string listing = filter_string_implode (files, "\n"); if (listing.empty ()) listing = translate ("No files in this folder"); else { listing.insert (0, "<table>"); listing.append ("</table>"); } view.set_variable ("listing", listing); } else { string filename = filter_url_create_root_path (url); return filter_url_file_get_contents (filename); } page += view.render ("index", "listing"); page += Assets_Page::footer (); return page; }
vector <string> Fonts_Logic::getFonts () { vector <string> files = filter_url_scandir (folder()); vector <string> fonts; for (auto & file : files) { string suffix = filter_url_get_extension (file); if (suffix == "txt") continue; if (suffix == "html") continue; if (suffix == "h") continue; if (suffix == "cpp") continue; if (suffix == "o") continue; fonts.push_back (file); } return fonts; }
// Gets journal entry more recent than "filename". // Updates "filename" to the item it got. string Database_Logs::getNext (string &filename) { string directory = folder (); vector <string> files = filter_url_scandir (directory); for (unsigned int i = 0; i < files.size (); i++) { string file = files [i]; if (file > filename) { filename = file; string path = filter_url_create_path (directory, file); string contents = filter_url_file_get_contents (path); return contents; } } return ""; }
void tmp_tmp () { Database_Logs::log ("Removing expired temporal files", Filter_Roles::admin ()); int expired = filter_date_seconds_since_epoch () - (3600 * 24 * 3); string directory = filter_url_create_root_path ("tmp"); vector <string> names = filter_url_scandir (directory); for (auto & name : names) { if (name.find ("tmp.") == 0) continue; string filename = filter_url_create_path (directory, name); int mtime = filter_url_filemtime (filename); if (mtime < expired) { filter_url_rmdir (filename); filter_url_unlink (filename); } } }
// Copies the search index of Bible $original to Bible $destination. void search_logic_copy_bible (string original, string destination) { string original_fragment = search_logic_bible_fragment (original); original_fragment = filter_url_basename (original_fragment); string destination_fragment = search_logic_bible_fragment (destination); destination_fragment = filter_url_basename (destination_fragment); vector <string> files = filter_url_scandir (search_logic_index_folder ()); for (auto & file : files) { if (file.find (original_fragment) == 0) { string original_path = filter_url_create_path (search_logic_index_folder (), file); string destination_file = destination_fragment + file.substr (original_fragment.length ()); string destination_path = filter_url_create_path (search_logic_index_folder (), destination_file); filter_url_file_cp (original_path, destination_path); } } }
// Get the names of the available default versification systems that come with Bibledit. vector <string> versification_logic_names () { vector <string> names; string directory = filter_url_create_root_path ("versification"); vector <string> files = filter_url_scandir (directory); for (auto file : files) { if (filter_url_get_extension (file) == "txt") { // Remove the dot and extension. file = file.substr (0, file.length () - 4); // Change underscores to spaces for the names. file = filter_string_str_replace ("_", " ", file); names.push_back (file); } } return names; }
// Copies the entire directory $input to a directory named $output. // It will recursively copy the inner directories also. void filter_url_dir_cp (const string & input, const string & output) { // Create the output directory. filter_url_mkdir (output); // Check on all files in the input directory. vector <string> files = filter_url_scandir (input); for (auto & file : files) { string input_path = filter_url_create_path (input, file); string output_path = filter_url_create_path (output, file); if (filter_url_is_dir (input_path)) { // Create output directory. filter_url_mkdir (output_path); // Handle the new input directory. filter_url_dir_cp (input_path, output_path); } else { // Copy input file to output. filter_url_file_cp (input_path, output_path); } } }
// Get the logbook entries. vector <string> Database_Logs::get (string & lastfilename) { lastfilename = "0"; // Read entries from the filesystem. vector <string> entries; string directory = folder (); vector <string> files = filter_url_scandir (directory); for (unsigned int i = 0; i < files.size(); i++) { string file = files [i]; string path = filter_url_create_path (directory, file); string contents = filter_url_file_get_contents (path); entries.push_back (contents); // Last second gets updated based on the filename. lastfilename = file; } // Done. return entries; }
void changes_modifications () { Database_Logs::log ("Change notifications: Generating", Filter_Roles::translator ()); // Notifications are not available to clients for the duration of processing them. config_globals_change_notifications_available = false; // Data objects. Webserver_Request request; Database_Modifications database_modifications; // Check on the health of the modifications database and (re)create it if needed. if (!database_modifications.healthy ()) database_modifications.erase (); database_modifications.create (); // Create online change notifications for users who made changes in Bibles // through the web editor or through a client. // It runs before the team changes. // This produces the desired order of the notifications in the GUI. // At the same time, produce change statistics per user. // Get the users who will receive the changes entered by the contributors. vector <string> recipients; { vector <string> users = request.database_users ()->getUsers (); for (auto & user : users) { if (request.database_config_user ()->getContributorChangesNotificationsOnline (user)) { recipients.push_back (user); } } } // Storage for the statistics. map <string, int> user_change_statistics; float modification_time_total = 0; int modification_time_count = 0; vector <string> users = database_modifications.getUserUsernames (); if (!users.empty ()) Database_Logs::log ("Change notifications: Per user", Filter_Roles::translator ()); for (auto user : users) { // Total changes made by this user. int change_count = 0; // Go through the Bibles changed by the current user. vector <string> bibles = database_modifications.getUserBibles (user); for (auto bible : bibles) { // Body of the email to be sent. string email = "<p>" + translate("You have entered the changes below in a Bible editor.") + " " + translate ("You may check if it made its way into the Bible text.") + "</p>"; size_t empty_email_length = email.length (); // Go through the books in that Bible. vector <int> books = database_modifications.getUserBooks (user, bible); for (auto book : books) { // Go through the chapters in that book. vector <int> chapters = database_modifications.getUserChapters (user, bible, book); for (auto chapter : chapters) { // Get the sets of identifiers for that chapter, and set some variables. vector <Database_Modifications_Id> IdSets = database_modifications.getUserIdentifiers (user, bible, book, chapter); //int referenceOldId = 0; int referenceNewId = 0; int newId = 0; int lastNewId = 0; bool restart = true; // Go through the sets of identifiers. for (auto IdSet : IdSets) { int oldId = IdSet.oldid; newId = IdSet.newid; if (restart) { changes_process_identifiers (&request, user, recipients, bible, book, chapter, referenceNewId, newId, email, change_count, modification_time_total, modification_time_count); //referenceOldId = oldId; referenceNewId = newId; lastNewId = newId; restart = false; continue; } if (oldId == lastNewId) { lastNewId = newId; } else { restart = true; } } // Process the last set of identifiers. changes_process_identifiers (&request, user, recipients, bible, book, chapter, referenceNewId, newId, email, change_count, modification_time_total, modification_time_count); } } // Check whether there's any email to be sent. if (email.length () != empty_email_length) { // Send the user email with the user's personal changes if the user opted to receive it. if (request.database_config_user()->getUserUserChangesNotification (user)) { string subject = translate("Changes you entered in") + " " + bible; if (!client_logic_client_enabled ()) email_schedule (user, subject, email); } } } // Store change statistics for this user. user_change_statistics [user] = change_count; // Clear the user's changes in the database. database_modifications.clearUserUser (user); // Clear checksum cache. request.database_config_user ()->setUserChangeNotificationsChecksum (user, ""); } // Generate the notifications, online and by email, // for the changes in the Bibles entered by anyone since the previous notifications were generated. vector <string> bibles = database_modifications.getTeamDiffBibles (); for (auto bible : bibles) { string stylesheet = Database_Config_Bible::getExportStylesheet (bible); vector <string> changeNotificationUsers; vector <string> users = request.database_users ()->getUsers (); for (auto user : users) { if (access_bible_read (&request, bible, user)) { if (request.database_config_user()->getUserGenerateChangeNotifications (user)) { changeNotificationUsers.push_back (user); } } } users.clear (); // The number of changes processed so far for this Bible. int processedChangesCount = 0; // The files get stored at http://site.org:<port>/revisions/<Bible>/<date> int seconds = filter_date_seconds_since_epoch (); string timepath; timepath.append (convert_to_string (filter_date_numerical_year (seconds))); timepath.append ("-"); timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_month (seconds)), 2, '0')); timepath.append ("-"); timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_month_day (seconds)), 2, '0')); timepath.append (" "); timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_hour (seconds)), 2, '0')); timepath.append (":"); timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_minute (seconds)), 2, '0')); timepath.append (":"); timepath.append (filter_string_fill (convert_to_string (filter_date_numerical_second (seconds)), 2, '0')); string directory = filter_url_create_root_path ("revisions", bible, timepath); filter_url_mkdir (directory); // Produce the USFM and html files. filter_diff_produce_verse_level (bible, directory); // Create online page with changed verses. string versesoutputfile = filter_url_create_path (directory, "changed_verses.html"); filter_diff_run_file (filter_url_create_path (directory, "verses_old.txt"), filter_url_create_path (directory, "verses_new.txt"), versesoutputfile); // Storage for body of the email with the changes. vector <string> email_changes; // Generate the online change notifications. vector <int> books = database_modifications.getTeamDiffBooks (bible); for (auto book : books) { vector <int> chapters = database_modifications.getTeamDiffChapters (bible, book); for (auto chapter : chapters) { Database_Logs::log ("Change notifications: " + bible + " " + filter_passage_display (book, chapter, ""), Filter_Roles::translator ()); string old_chapter_usfm = database_modifications.getTeamDiff (bible, book, chapter); string new_chapter_usfm = request.database_bibles()->getChapter (bible, book, chapter); vector <int> old_verse_numbers = usfm_get_verse_numbers (old_chapter_usfm); vector <int> new_verse_numbers = usfm_get_verse_numbers (new_chapter_usfm); vector <int> verses = old_verse_numbers; verses.insert (verses.end (), new_verse_numbers.begin (), new_verse_numbers.end ()); verses = array_unique (verses); sort (verses.begin (), verses.end()); for (auto verse : verses) { string old_verse_usfm = usfm_get_verse_text (old_chapter_usfm, verse); string new_verse_usfm = usfm_get_verse_text (new_chapter_usfm, verse); if (old_verse_usfm != new_verse_usfm) { processedChangesCount++; // In case of too many change notifications, processing them would take too much time, so take a few shortcuts. string old_html = "<p>" + old_verse_usfm + "</p>"; string new_html = "<p>" + new_verse_usfm + "</p>"; string old_text = old_verse_usfm; string new_text = new_verse_usfm; if (processedChangesCount < 800) { Filter_Text filter_text_old = Filter_Text (bible); Filter_Text filter_text_new = Filter_Text (bible); filter_text_old.html_text_standard = new Html_Text (""); filter_text_new.html_text_standard = new Html_Text (""); filter_text_old.text_text = new Text_Text (); filter_text_new.text_text = new Text_Text (); filter_text_old.addUsfmCode (old_verse_usfm); filter_text_new.addUsfmCode (new_verse_usfm); filter_text_old.run (stylesheet); filter_text_new.run (stylesheet); old_html = filter_text_old.html_text_standard->getInnerHtml (); new_html = filter_text_new.html_text_standard->getInnerHtml (); old_text = filter_text_old.text_text->get (); new_text = filter_text_new.text_text->get (); } string modification = filter_diff_diff (old_text, new_text); database_modifications.recordNotification (changeNotificationUsers, changes_bible_category (), bible, book, chapter, verse, old_html, modification, new_html); string passage = filter_passage_display (book, chapter, convert_to_string (verse)) + ": "; if (old_text != new_text) { email_changes.push_back (passage + modification); } else { email_changes.push_back (translate ("The following passage has no change in the text.") + " " + translate ("The change is in the formatting only.") + " " + translate ("The USFM code is given for reference.")); email_changes.push_back (passage); email_changes.push_back (translate ("Old code:") + " " + old_verse_usfm); email_changes.push_back (translate ("New code:") + " " + new_verse_usfm); } } } // Delete the diff data for this chapter, for two reasons: // 1. New diffs for this chapter can be stored straightaway. // 2. In case of large amounts of diff data, and this function gets killed, // then the next time it runs again, it will continue from where it was killed. database_modifications.deleteTeamDiffChapter (bible, book, chapter); } } // Email the changes to the subscribed users. if (!email_changes.empty ()) { string body; for (auto & line : email_changes) { body.append ("<div>"); body.append (line); body.append ("</div>\n"); } string subject = translate("Recent changes:") + " " + bible; vector <string> users = request.database_users ()->getUsers (); for (auto & user : users) { if (request.database_config_user()->getUserBibleChangesNotification (user)) { if (access_bible_read (&request, bible, user)) { if (!client_logic_client_enabled ()) { email_schedule (user, subject, body); } } } } } } // Index the data and remove expired notifications. Database_Logs::log ("Change notifications: Indexing", Filter_Roles::translator ()); database_modifications.indexTrimAllNotifications (); // Remove expired downloadable revisions. string directory = filter_url_create_root_path ("revisions"); int now = filter_date_seconds_since_epoch (); bibles = filter_url_scandir (directory); for (auto &bible : bibles) { string folder = filter_url_create_path (directory, bible); int time = filter_url_file_modification_time (folder); int days = (now - time) / 86400; if (days > 31) { filter_url_rmdir (folder); } else { vector <string> revisions = filter_url_scandir (folder); for (auto & revision : revisions) { string path = filter_url_create_path (folder, revision); int time = filter_url_file_modification_time (path); int days = (now - time) / 86400; if (days > 31) { filter_url_rmdir (path); Database_Logs::log ("Removing expired downloadable revision notification: " + bible + " " + revision, Filter_Roles::translator ()); } } } } // Clear checksum caches. users = request.database_users ()->getUsers (); for (auto user : users) { request.database_config_user ()->setUserChangeNotificationsChecksum (user, ""); } // Vacuum the modifications index, as it might have been updated. database_modifications.vacuum (); // Make the notifications available again to clients. config_globals_change_notifications_available = true; // Store the statistics in the database. if (modification_time_count) { // Take average timestamp of all timestamps. int timestamp = round (modification_time_total / modification_time_count); for (auto & element : user_change_statistics) { // Store dated change statistics per user. string user = element.first; int count = element.second; Database_Statistics::store_changes (timestamp, user, count); } } Database_Logs::log ("Change notifications: Ready", Filter_Roles::translator ()); }
void Database_Logs::rotate () { // Remove the database that was used in older versions of Bibledit. // Since February 2016 Bibledit no longer uses a database for storing the journal. // Reasons that a database is no longer used: // 1. Simpler system. // 2. Android has VACUUM errors due to a locked database. string old_database_file = database_sqlite_file ("logs2"); if (file_exists (old_database_file)) { filter_url_unlink (old_database_file); } // Use a mechanism that handles huge amounts of entries. // The PHP function scandir choked on this or took a very long time. // The PHP functions opendir / readdir / closedir handled it better. // But now, in C++, with the new journal mechanism, this is no longer relevant. string directory = folder (); vector <string> files = filter_url_scandir (directory); // Timestamp for removing older records, depending on whether it's a tiny journal. #ifdef HAVE_TINYJOURNAL int oldtimestamp = filter_date_seconds_since_epoch () - (14400); #else int oldtimestamp = filter_date_seconds_since_epoch () - (6 * 86400); #endif // Limit the available the journal entrie count in the filesystem. // This speeds up subsequent reading of the Journal by the users. // In previous versions of Bibledit, there were certain conditions // that led to an infinite loop, as had been noticed at times, // and this quickly exhausted the available inodes on the filesystem. #ifdef HAVE_TINYJOURNAL int limitfilecount = files.size () - 200; #else int limitfilecount = files.size () - 2000; #endif bool filtered_entries = false; for (unsigned int i = 0; i < files.size(); i++) { string path = filter_url_create_path (directory, files [i]); // Limit the number of journal entries. if ((int)i < limitfilecount) { filter_url_unlink (path); continue; } // Remove expired entries. int timestamp = convert_to_int (files [i].substr (0, 10)); if (timestamp < oldtimestamp) { filter_url_unlink (path); continue; } // Filtering of certain entries. string entry = filter_url_file_get_contents (path); if (journal_logic_filter_entry (entry)) { filtered_entries = true; filter_url_unlink (path); continue; } } if (filtered_entries) { log (journal_logic_filtered_message ()); } }
void export_index () { Database_Bibles database_bibles; vector <string> bibles = database_bibles.getBibles (); // Go through all sub directories of the exports directory. // Remove subdirectories if their corresponding Bible no longer exists in the system. string directory = Export_Logic::mainDirectory (); vector <string> files = filter_url_scandir (directory); for (auto & file : files) { if (in_array (file, bibles)) continue; filter_url_rmdir (filter_url_create_path (directory, file)); Database_Logs::log ("Removing exported Bible " + file, Filter_Roles::admin ()); } // Schedule the relevant Bibles for export. for (auto bible : bibles) { if (Database_State::getExport (bible, 0, Export_Logic::export_needed)) { Database_State::clearExport (bible, 0, Export_Logic::export_needed); vector <int> books = database_bibles.getBooks (bible); // Book 0 flags export of the whole Bible (this is not relevant to all export types). books.push_back (0); for (auto book : books) { for (int format = Export_Logic::export_needed + 1; format < Export_Logic::export_end; format++) { Database_State::setExport (bible, book, format); } } if (Database_Config_Bible::getExportWebDuringNight (bible)) { Export_Logic::scheduleWeb (bible); Export_Logic::scheduleWebIndex (bible); } if (Database_Config_Bible::getExportHtmlDuringNight (bible)) { Export_Logic::scheduleHtml (bible); } if (Database_Config_Bible::getExportUsfmDuringNight (bible)) { Export_Logic::scheduleUsfm (bible); } if (Database_Config_Bible::getExportTextDuringNight (bible)) { Export_Logic::scheduleTextAndBasicUsfm (bible); } if (Database_Config_Bible::getExportOdtDuringNight (bible)) { Export_Logic::scheduleOpenDocument (bible); } if (Database_Config_Bible::getGenerateInfoDuringNight (bible)) { Export_Logic::scheduleInfo (bible); } if (Database_Config_Bible::getExportESwordDuringNight (bible)) { Export_Logic::scheduleESword (bible); } if (Database_Config_Bible::getExportOnlineBibleDuringNight (bible)) { Export_Logic::scheduleOnlineBible (bible); } } } }