void notes_convert_database_to_plain_files () { // Bail out if there's no database to convert. ustring database_filename = gw_build_filename(Directories->get_notes(), "notes.sql2"); if (!g_file_test(database_filename.c_str(), G_FILE_TEST_IS_REGULAR)) { return; } // Progress window. ProgressWindow progresswindow (_("Converting project notes"), false); // Access the database. sqlite3 *db; int rc; char *error = NULL; try { rc = sqlite3_open(database_filename.c_str(), &db); if (rc) throw runtime_error(sqlite3_errmsg(db)); sqlite3_busy_timeout(db, 1000); { // Read how many notes there are. SqliteReader reader(0); char *sql; sql = g_strdup_printf("select count(*) from notes;"); rc = sqlite3_exec(db, sql, reader.callback, &reader, &error); g_free(sql); if (rc != SQLITE_OK) throw runtime_error(error); if (!reader.ustring0.empty()) { progresswindow.set_iterate (0, 1, convert_to_int (reader.ustring0[0])); } } { SqliteReader reader(0); rc = sqlite3_exec(db, "select id, ref_osis, project, category, note, created, modified, user, logbook from notes;", reader.callback, &reader, &error); if (rc != SQLITE_OK) throw runtime_error(error); for (unsigned int i = 0; i < reader.ustring0.size(); i++) { progresswindow.iterate (); notes_store_one_in_file(convert_to_int (reader.ustring0[i]), // ID. reader.ustring4[i], // Note. reader.ustring2[i], // Project. reader.ustring1[i], // References. reader.ustring3[i], // Category. convert_to_int (reader.ustring5[i]), // Date created. reader.ustring7[i], // User. convert_to_int (reader.ustring6[i]), // Date modified. reader.ustring8[i]); // Logbook. } } // Delete the notes database that was converted. unix_unlink (database_filename.c_str()); } catch(exception & ex) { gw_critical(ex.what()); } sqlite3_close(db); }
void notes_create_index () // This creates an index for the notes stored in plain files. { ProgressWindow progresswindow (_("Creating notes index"), false); // Remove any old index. unix_unlink (notes_index_filename ().c_str()); // Create index database. sqlite3 *db; int rc; char *error = NULL; try { // Connect to the index database. rc = sqlite3_open(notes_index_filename ().c_str(), &db); if (rc) throw runtime_error(sqlite3_errmsg(db)); sqlite3_busy_timeout(db, 1000); // Create the notes table. char *sql; sql = g_strdup_printf("create table notes (id integer, reference text, project text, category text, casefolded text, created integer, modified integer);"); rc = sqlite3_exec(db, sql, NULL, NULL, &error); g_free(sql); // Fast writing. sqlite3_exec(db, "PRAGMA synchronous=OFF;", NULL, NULL, NULL); // Create an index for all the note files. The index will speed up note selection. ReadFiles note_files (notes_shared_storage_folder (), "", ""); progresswindow.set_iterate (0, 1, note_files.files.size()); for (unsigned int i = 0; i < note_files.files.size(); i++) { progresswindow.iterate (); notes_store_index_entry (db, convert_to_int (note_files.files[i])); } } catch(exception & ex) { gw_critical(ex.what()); unix_unlink (notes_index_filename ().c_str()); } // Close connection. sqlite3_close(db); }
void kjv_import_sword (const ustring& textfile, const ustring& database) { // Show the progress. KJV has 31102 verses. ProgressWindow progresswindow (_("Importing King James Bible"), false); progresswindow.set_iterate (0, 1, 31102); gchar * contents; g_file_get_contents(textfile.c_str(), &contents, NULL, NULL); if (!contents) return; // Create the database, put it in fast mode. unix_unlink (database.c_str()); sqlite3 *db; sqlite3_open(database.c_str(), &db); sqlite3_exec(db, "create table kjv (book integer, chapter integer, verse integer, item integer, fragment text, lemma text);", NULL, NULL, NULL); sqlite3_exec(db, "PRAGMA synchronous=OFF;", NULL, NULL, NULL); // Parse input. xmlParserInputBufferPtr inputbuffer; inputbuffer = xmlParserInputBufferCreateMem(contents, strlen (contents), XML_CHAR_ENCODING_NONE); xmlTextReaderPtr reader = xmlNewTextReader(inputbuffer, NULL); if (reader) { bool within_relevant_element = false; Reference reference (0, 0, "0"); unsigned int item_number = 0; ustring textfragment; ustring lemmata; while ((xmlTextReaderRead(reader) == 1)) { switch (xmlTextReaderNodeType(reader)) { case XML_READER_TYPE_ELEMENT: { xmlChar *element_name = xmlTextReaderName(reader); // Deal with a verse element. if (!xmlStrcmp(element_name, BAD_CAST "verse")) { progresswindow.iterate(); char *attribute; attribute = (char *)xmlTextReaderGetAttribute(reader, BAD_CAST "osisID"); if (attribute) { Parse parse (attribute, false, "."); if (parse.words.size() == 3) { reference.assign(books_osis_to_id (parse.words[0]), // book convert_to_int (parse.words[1]), // chapter parse.words[2]); // verse } else { gw_critical (attribute); } free(attribute); } item_number = 0; } // Deal with a w element. if (!xmlStrcmp(element_name, BAD_CAST "w")) { within_relevant_element = true; item_number++; textfragment.clear(); lemmata.clear(); char *attribute; attribute = (char *)xmlTextReaderGetAttribute(reader, BAD_CAST "lemma"); if (attribute) { lemmata = attribute; free(attribute); } } break; } case XML_READER_TYPE_TEXT: { if (within_relevant_element) { xmlChar *text = xmlTextReaderValue(reader); if (text) { textfragment = (const char *)text; xmlFree(text); textfragment = textfragment.casefold(); } } break; } case XML_READER_TYPE_END_ELEMENT: { xmlChar *element_name = xmlTextReaderName(reader); if (!xmlStrcmp(element_name, BAD_CAST "w")) { replace_text (lemmata, "strong:", ""); char *sql; sql = g_strdup_printf("insert into kjv values (%d, %d, %d, %d, '%s', '%s');", reference.book_get(), reference.chapter_get(), convert_to_int (reference.verse_get()), item_number, double_apostrophy (textfragment).c_str(), lemmata.c_str()); sqlite3_exec(db, sql, NULL, NULL, NULL); g_free(sql); within_relevant_element = false; } break; } } } } if (reader) xmlFreeTextReader(reader); if (inputbuffer) xmlFreeParserInputBuffer(inputbuffer); // Close database. sqlite3_close(db); // Free xml data. g_free(contents); }
void import_bibleworks_text_file (const ustring& file, const ustring& bible, vector <ustring>& messages) // Imports a bibleworks text file. { // Read the file. ReadText rt (file, true, false); // If there's nothing to import, bail out. if (rt.lines.empty()) { messages.push_back ("The file is empty"); } // Divide the input into separate bits for each book. vector <VectorUstring> bookdata; if (messages.empty()) { try { ustring previousbook; vector < ustring > booklines; de_byte_order_mark (rt.lines[0]); for (unsigned int i = 0; i < rt.lines.size(); i++) { ustring currentbook = rt.lines[i].substr(0, 3); if (i == 0) { previousbook = currentbook; } if (currentbook != previousbook) { bookdata.push_back (booklines); booklines.clear(); previousbook = currentbook; } booklines.push_back(rt.lines[i]); } bookdata.push_back (booklines); } catch(exception & ex) { messages.push_back(ex.what()); } } // Import each book. if (messages.empty ()) { ProgressWindow progresswindow ("Importing", false); progresswindow.set_iterate (0, 1, bookdata.size()); for (unsigned int i = 0; i < bookdata.size(); i++) { progresswindow.iterate (); try { // Input and output data. vector <ustring> rawlines = bookdata[i]; vector <ustring> usfmlines; // Get the name of the book. unsigned int book_id = books_bibleworks_to_id(rawlines[0].substr(0, 3)); if (book_id == 0) { messages.push_back ("Unknown book: " + rawlines[0]); return; } // Store USFM id. ustring usfmid = books_id_to_paratext (book_id); usfmlines.push_back("\\id " + usfmid); // Convert the BibleWorks lines to USFM code. ustring previouschapter = "0"; for (unsigned int i = 0; i < rawlines.size(); i++) { // Convert chapter information. ustring line = rawlines[i]; line.erase(0, 4); ustring currentchapter = number_in_string(line); line.erase(0, currentchapter.length() + 1); if (currentchapter != previouschapter) { usfmlines.push_back("\\c " + currentchapter); usfmlines.push_back("\\p"); previouschapter = currentchapter; } // Convert verse data. usfmlines.push_back("\\v " + line); } // Store into the Bible. CategorizeChapterVerse ccv(usfmlines); project_store_book(bible, book_id, ccv); } catch(exception & ex) { messages.push_back(ex.what()); } } } }
void mechon_mamre_action_page (HtmlWriter2& htmlwriter) { htmlwriter.heading_open (3); htmlwriter.text_add (_("Hebrew import from Mechon Mamre")); htmlwriter.heading_close (); vector <ustring> messages; bool keep_going = true; // Locate the downloaded file. ustring ct005zipfilename = gw_build_filename (g_get_home_dir (), "ct005.zip"); messages.push_back (_("Looking for file ") + ct005zipfilename); if (!g_file_test (ct005zipfilename.c_str(), G_FILE_TEST_IS_REGULAR)) { ct005zipfilename.clear(); } if (ct005zipfilename.empty()) { ct005zipfilename = gw_build_filename (g_get_home_dir (), "Desktop", "ct005.zip"); messages.push_back (_("Looking for file ") + ct005zipfilename); if (!g_file_test (ct005zipfilename.c_str(), G_FILE_TEST_IS_REGULAR)) { ct005zipfilename.clear(); } } if (ct005zipfilename.empty()) { ct005zipfilename = gw_build_filename (g_get_home_dir (), "Downloads", "ct005.zip"); messages.push_back (_("Looking for file ") + ct005zipfilename); if (!g_file_test (ct005zipfilename.c_str(), G_FILE_TEST_IS_REGULAR)) { ct005zipfilename.clear(); } } if (ct005zipfilename.empty()) { messages.push_back (_("Can't find Hebrew input file")); keep_going = false; } if (keep_going) { messages.push_back (_("Using file ") + ct005zipfilename); } // Unpack the zipped file. ustring directory; if (keep_going) { directory = gw_build_filename (Directories->get_temp (), "uncompress"); unix_rmdir (directory); gw_mkdir_with_parents (directory); if (!uncompress (ct005zipfilename, directory)) { messages.push_back (_("Could not unpack the file")); keep_going = false; } messages.push_back (_("Unpacking into folder ") + directory); } // Show the readme file. if (keep_going) { ustring readmefile = gw_build_filename (directory, "readme.txt"); ReadText rt (readmefile, true, true); ustring line; for (unsigned int i = 0; i < rt.lines.size(); i++) { if (rt.lines[i].empty()) { if (!line.empty()) { messages.push_back (line); line.clear(); } } else { line.append (rt.lines[i] + " "); } } messages.push_back (line); } // Look for the directory where all the html files reside. if (keep_going) { directory = gw_build_filename (directory, "c", "ct"); if (!g_file_test (directory.c_str(), G_FILE_TEST_IS_DIR)) { messages.push_back (_("Can't find data in directory ") + directory); } messages.push_back (_("Looking for data in directory ") + directory); } // Get a list of the html files that have the data. vector <ustring> files; if (keep_going) { ReadFiles rf (directory, "c", ".htm"); for (unsigned int i = 0; i < rf.files.size(); i++) { ustring filename = gw_build_filename (directory, rf.files[i]); // Check on a few characteristics. if (mechon_mamre_copyright(filename)) { unsigned int digitcount = digit_count_in_string(rf.files[i]); if ((digitcount == 3) || (digitcount == 4)) { files.push_back(filename); } } } } // Create a new Bible into which to import the data. ustring bible = _("Hebrew Mechon Mamre"); if (keep_going) { if (project_exists (bible)) { messages.push_back (_("A Bible already exists by this name: ") + bible); keep_going = false; } else { project_create_restore (bible, ""); messages.push_back (_("Creating a new Bible called \"") + bible + "\""); // Make a couple of settings. extern Settings * settings; ProjectConfiguration * projectconfig = settings->projectconfig (bible); projectconfig->versification_set ("Original"); projectconfig->editable_set (false); projectconfig->right_to_left_set (true); projectconfig->spelling_check_set (false); } } // Store all the chapters 0 in each book. if (keep_going) { vector <unsigned int> books = books_type_to_ids(btOldTestament); ProgressWindow progresswindow (_("Creating books"), false); progresswindow.set_iterate (0, 1, books.size()); for (unsigned int bk = 0; bk < books.size(); bk++) { progresswindow.iterate (); vector <ustring> usfm; usfm.push_back ("\\id " + books_id_to_paratext (books[bk])); CategorizeChapterVerse ccv (usfm); project_store_chapter (bible, books[bk], ccv); } } // Store all the chapters. if (keep_going) { ProgressWindow progresswindow (_("Importing chapters"), false); progresswindow.set_iterate (0, 1, files.size()); for (unsigned int i = 0; i < files.size(); i++) { progresswindow.iterate (); unsigned int book = 0; unsigned int chapter = 0; mechon_mamre_extract_book_chapter (files[i], book, chapter); vector <ustring> contents = mechon_mamre_extract_contents (files[i], chapter); CategorizeChapterVerse ccv (contents); project_store_chapter (bible, book, ccv); messages.push_back (_("Importing ") + books_id_to_localname (book) + " " + convert_to_string (chapter) + _(" from file ") + files[i]); } } // Write accumulated messages. htmlwriter.heading_open (3); if (keep_going) { htmlwriter.text_add (_("Success! Bible was created: ") + bible); } else { htmlwriter.text_add (_("Error!")); } htmlwriter.heading_close (); if (keep_going) { htmlwriter.paragraph_open (); htmlwriter.text_add (_("To view the Hebrew text, open the Bible in the editor. Optionally set the font for better display of the Hebrew text. A donation made to Mechon Mamre will support their work.")); htmlwriter.paragraph_close (); } for (unsigned int i = 0; i < messages.size(); i++) { htmlwriter.paragraph_open (); htmlwriter.text_add (messages[i]); htmlwriter.paragraph_close (); } // Write OK. htmlwriter.paragraph_open (); htmlwriter.hyperlink_add ("ok", _("Ok")); htmlwriter.paragraph_close (); }