void notes_convert_database_to_plain_files ()
{
// Bail out if there's no database to convert.
ustring database_filename = gw_build_filename(Directories->get_notes(), "notes.sql2");
if (!g_file_test(database_filename.c_str(), G_FILE_TEST_IS_REGULAR)) {
return;
}
// Progress window.
ProgressWindow progresswindow (_("Converting project notes"), false);
// Access the database.
sqlite3 *db;
int rc;
char *error = NULL;
try {
rc = sqlite3_open(database_filename.c_str(), &db);
if (rc) throw runtime_error(sqlite3_errmsg(db));
sqlite3_busy_timeout(db, 1000);
{
// Read how many notes there are.
SqliteReader reader(0);
char *sql;
sql = g_strdup_printf("select count(*) from notes;");
rc = sqlite3_exec(db, sql, reader.callback, &reader, &error);
g_free(sql);
if (rc != SQLITE_OK) throw runtime_error(error);
if (!reader.ustring0.empty()) {
progresswindow.set_iterate (0, 1, convert_to_int (reader.ustring0[0]));
}
}
{
SqliteReader reader(0);
rc = sqlite3_exec(db, "select id, ref_osis, project, category, note, created, modified, user, logbook from notes;", reader.callback, &reader, &error);
if (rc != SQLITE_OK) throw runtime_error(error);
for (unsigned int i = 0; i < reader.ustring0.size(); i++) {
progresswindow.iterate ();
notes_store_one_in_file(convert_to_int (reader.ustring0[i]), // ID.
reader.ustring4[i], // Note.
reader.ustring2[i], // Project.
reader.ustring1[i], // References.
reader.ustring3[i], // Category.
convert_to_int (reader.ustring5[i]), // Date created.
reader.ustring7[i], // User.
convert_to_int (reader.ustring6[i]), // Date modified.
reader.ustring8[i]); // Logbook.
}
}
// Delete the notes database that was converted.
unix_unlink (database_filename.c_str());
}
catch(exception & ex) {
gw_critical(ex.what());
}
sqlite3_close(db);
}
void notes_create_index ()
// This creates an index for the notes stored in plain files.
{
ProgressWindow progresswindow (_("Creating notes index"), false);
// Remove any old index.
unix_unlink (notes_index_filename ().c_str());
// Create index database.
sqlite3 *db;
int rc;
char *error = NULL;
try {
// Connect to the index database.
rc = sqlite3_open(notes_index_filename ().c_str(), &db);
if (rc)
throw runtime_error(sqlite3_errmsg(db));
sqlite3_busy_timeout(db, 1000);
// Create the notes table.
char *sql;
sql = g_strdup_printf("create table notes (id integer, reference text, project text, category text, casefolded text, created integer, modified integer);");
rc = sqlite3_exec(db, sql, NULL, NULL, &error);
g_free(sql);
// Fast writing.
sqlite3_exec(db, "PRAGMA synchronous=OFF;", NULL, NULL, NULL);
// Create an index for all the note files. The index will speed up note selection.
ReadFiles note_files (notes_shared_storage_folder (), "", "");
progresswindow.set_iterate (0, 1, note_files.files.size());
for (unsigned int i = 0; i < note_files.files.size(); i++) {
progresswindow.iterate ();
notes_store_index_entry (db, convert_to_int (note_files.files[i]));
}
}
catch(exception & ex) {
gw_critical(ex.what());
unix_unlink (notes_index_filename ().c_str());
}
// Close connection.
sqlite3_close(db);
}
void kjv_import_sword (const ustring& textfile, const ustring& database)
{
// Show the progress. KJV has 31102 verses.
ProgressWindow progresswindow (_("Importing King James Bible"), false);
progresswindow.set_iterate (0, 1, 31102);
gchar * contents;
g_file_get_contents(textfile.c_str(), &contents, NULL, NULL);
if (!contents)
return;
// Create the database, put it in fast mode.
unix_unlink (database.c_str());
sqlite3 *db;
sqlite3_open(database.c_str(), &db);
sqlite3_exec(db, "create table kjv (book integer, chapter integer, verse integer, item integer, fragment text, lemma text);", NULL, NULL, NULL);
sqlite3_exec(db, "PRAGMA synchronous=OFF;", NULL, NULL, NULL);
// Parse input.
xmlParserInputBufferPtr inputbuffer;
inputbuffer = xmlParserInputBufferCreateMem(contents, strlen (contents), XML_CHAR_ENCODING_NONE);
xmlTextReaderPtr reader = xmlNewTextReader(inputbuffer, NULL);
if (reader) {
bool within_relevant_element = false;
Reference reference (0, 0, "0");
unsigned int item_number = 0;
ustring textfragment;
ustring lemmata;
while ((xmlTextReaderRead(reader) == 1)) {
switch (xmlTextReaderNodeType(reader)) {
case XML_READER_TYPE_ELEMENT:
{
xmlChar *element_name = xmlTextReaderName(reader);
// Deal with a verse element.
if (!xmlStrcmp(element_name, BAD_CAST "verse")) {
progresswindow.iterate();
char *attribute;
attribute = (char *)xmlTextReaderGetAttribute(reader, BAD_CAST "osisID");
if (attribute) {
Parse parse (attribute, false, ".");
if (parse.words.size() == 3) {
reference.assign(books_osis_to_id (parse.words[0]), // book
convert_to_int (parse.words[1]), // chapter
parse.words[2]); // verse
} else {
gw_critical (attribute);
}
free(attribute);
}
item_number = 0;
}
// Deal with a w element.
if (!xmlStrcmp(element_name, BAD_CAST "w")) {
within_relevant_element = true;
item_number++;
textfragment.clear();
lemmata.clear();
char *attribute;
attribute = (char *)xmlTextReaderGetAttribute(reader, BAD_CAST "lemma");
if (attribute) {
lemmata = attribute;
free(attribute);
}
}
break;
}
case XML_READER_TYPE_TEXT:
{
if (within_relevant_element) {
xmlChar *text = xmlTextReaderValue(reader);
if (text) {
textfragment = (const char *)text;
xmlFree(text);
textfragment = textfragment.casefold();
}
}
break;
}
case XML_READER_TYPE_END_ELEMENT:
{
xmlChar *element_name = xmlTextReaderName(reader);
if (!xmlStrcmp(element_name, BAD_CAST "w")) {
replace_text (lemmata, "strong:", "");
char *sql;
sql = g_strdup_printf("insert into kjv values (%d, %d, %d, %d, '%s', '%s');",
reference.book_get(), reference.chapter_get(), convert_to_int (reference.verse_get()),
item_number,
double_apostrophy (textfragment).c_str(), lemmata.c_str());
sqlite3_exec(db, sql, NULL, NULL, NULL);
g_free(sql);
within_relevant_element = false;
}
break;
}
}
}
}
if (reader)
xmlFreeTextReader(reader);
if (inputbuffer)
xmlFreeParserInputBuffer(inputbuffer);
// Close database.
sqlite3_close(db);
// Free xml data.
g_free(contents);
}
void import_bibleworks_text_file (const ustring& file, const ustring& bible, vector <ustring>& messages)
// Imports a bibleworks text file.
{
// Read the file.
ReadText rt (file, true, false);
// If there's nothing to import, bail out.
if (rt.lines.empty()) {
messages.push_back ("The file is empty");
}
// Divide the input into separate bits for each book.
vector <VectorUstring> bookdata;
if (messages.empty()) {
try {
ustring previousbook;
vector < ustring > booklines;
de_byte_order_mark (rt.lines[0]);
for (unsigned int i = 0; i < rt.lines.size(); i++) {
ustring currentbook = rt.lines[i].substr(0, 3);
if (i == 0) {
previousbook = currentbook;
}
if (currentbook != previousbook) {
bookdata.push_back (booklines);
booklines.clear();
previousbook = currentbook;
}
booklines.push_back(rt.lines[i]);
}
bookdata.push_back (booklines);
}
catch(exception & ex) {
messages.push_back(ex.what());
}
}
// Import each book.
if (messages.empty ()) {
ProgressWindow progresswindow ("Importing", false);
progresswindow.set_iterate (0, 1, bookdata.size());
for (unsigned int i = 0; i < bookdata.size(); i++) {
progresswindow.iterate ();
try {
// Input and output data.
vector <ustring> rawlines = bookdata[i];
vector <ustring> usfmlines;
// Get the name of the book.
unsigned int book_id = books_bibleworks_to_id(rawlines[0].substr(0, 3));
if (book_id == 0) {
messages.push_back ("Unknown book: " + rawlines[0]);
return;
}
// Store USFM id.
ustring usfmid = books_id_to_paratext (book_id);
usfmlines.push_back("\\id " + usfmid);
// Convert the BibleWorks lines to USFM code.
ustring previouschapter = "0";
for (unsigned int i = 0; i < rawlines.size(); i++) {
// Convert chapter information.
ustring line = rawlines[i];
line.erase(0, 4);
ustring currentchapter = number_in_string(line);
line.erase(0, currentchapter.length() + 1);
if (currentchapter != previouschapter) {
usfmlines.push_back("\\c " + currentchapter);
usfmlines.push_back("\\p");
previouschapter = currentchapter;
}
// Convert verse data.
usfmlines.push_back("\\v " + line);
}
// Store into the Bible.
CategorizeChapterVerse ccv(usfmlines);
project_store_book(bible, book_id, ccv);
}
catch(exception & ex) {
messages.push_back(ex.what());
}
}
}
}
void mechon_mamre_action_page (HtmlWriter2& htmlwriter)
{
htmlwriter.heading_open (3);
htmlwriter.text_add (_("Hebrew import from Mechon Mamre"));
htmlwriter.heading_close ();
vector <ustring> messages;
bool keep_going = true;
// Locate the downloaded file.
ustring ct005zipfilename = gw_build_filename (g_get_home_dir (), "ct005.zip");
messages.push_back (_("Looking for file ") + ct005zipfilename);
if (!g_file_test (ct005zipfilename.c_str(), G_FILE_TEST_IS_REGULAR)) {
ct005zipfilename.clear();
}
if (ct005zipfilename.empty()) {
ct005zipfilename = gw_build_filename (g_get_home_dir (), "Desktop", "ct005.zip");
messages.push_back (_("Looking for file ") + ct005zipfilename);
if (!g_file_test (ct005zipfilename.c_str(), G_FILE_TEST_IS_REGULAR)) {
ct005zipfilename.clear();
}
}
if (ct005zipfilename.empty()) {
ct005zipfilename = gw_build_filename (g_get_home_dir (), "Downloads", "ct005.zip");
messages.push_back (_("Looking for file ") + ct005zipfilename);
if (!g_file_test (ct005zipfilename.c_str(), G_FILE_TEST_IS_REGULAR)) {
ct005zipfilename.clear();
}
}
if (ct005zipfilename.empty()) {
messages.push_back (_("Can't find Hebrew input file"));
keep_going = false;
}
if (keep_going) {
messages.push_back (_("Using file ") + ct005zipfilename);
}
// Unpack the zipped file.
ustring directory;
if (keep_going) {
directory = gw_build_filename (Directories->get_temp (), "uncompress");
unix_rmdir (directory);
gw_mkdir_with_parents (directory);
if (!uncompress (ct005zipfilename, directory)) {
messages.push_back (_("Could not unpack the file"));
keep_going = false;
}
messages.push_back (_("Unpacking into folder ") + directory);
}
// Show the readme file.
if (keep_going) {
ustring readmefile = gw_build_filename (directory, "readme.txt");
ReadText rt (readmefile, true, true);
ustring line;
for (unsigned int i = 0; i < rt.lines.size(); i++) {
if (rt.lines[i].empty()) {
if (!line.empty()) {
messages.push_back (line);
line.clear();
}
} else {
line.append (rt.lines[i] + " ");
}
}
messages.push_back (line);
}
// Look for the directory where all the html files reside.
if (keep_going) {
directory = gw_build_filename (directory, "c", "ct");
if (!g_file_test (directory.c_str(), G_FILE_TEST_IS_DIR)) {
messages.push_back (_("Can't find data in directory ") + directory);
}
messages.push_back (_("Looking for data in directory ") + directory);
}
// Get a list of the html files that have the data.
vector <ustring> files;
if (keep_going) {
ReadFiles rf (directory, "c", ".htm");
for (unsigned int i = 0; i < rf.files.size(); i++) {
ustring filename = gw_build_filename (directory, rf.files[i]);
// Check on a few characteristics.
if (mechon_mamre_copyright(filename)) {
unsigned int digitcount = digit_count_in_string(rf.files[i]);
if ((digitcount == 3) || (digitcount == 4)) {
files.push_back(filename);
}
}
}
}
// Create a new Bible into which to import the data.
ustring bible = _("Hebrew Mechon Mamre");
if (keep_going) {
if (project_exists (bible)) {
messages.push_back (_("A Bible already exists by this name: ") + bible);
keep_going = false;
} else {
project_create_restore (bible, "");
messages.push_back (_("Creating a new Bible called \"") + bible + "\"");
// Make a couple of settings.
extern Settings * settings;
ProjectConfiguration * projectconfig = settings->projectconfig (bible);
projectconfig->versification_set ("Original");
projectconfig->editable_set (false);
projectconfig->right_to_left_set (true);
projectconfig->spelling_check_set (false);
}
}
// Store all the chapters 0 in each book.
if (keep_going) {
vector <unsigned int> books = books_type_to_ids(btOldTestament);
ProgressWindow progresswindow (_("Creating books"), false);
progresswindow.set_iterate (0, 1, books.size());
for (unsigned int bk = 0; bk < books.size(); bk++) {
progresswindow.iterate ();
vector <ustring> usfm;
usfm.push_back ("\\id " + books_id_to_paratext (books[bk]));
CategorizeChapterVerse ccv (usfm);
project_store_chapter (bible, books[bk], ccv);
}
}
// Store all the chapters.
if (keep_going) {
ProgressWindow progresswindow (_("Importing chapters"), false);
progresswindow.set_iterate (0, 1, files.size());
for (unsigned int i = 0; i < files.size(); i++) {
progresswindow.iterate ();
unsigned int book = 0;
unsigned int chapter = 0;
mechon_mamre_extract_book_chapter (files[i], book, chapter);
vector <ustring> contents = mechon_mamre_extract_contents (files[i], chapter);
CategorizeChapterVerse ccv (contents);
project_store_chapter (bible, book, ccv);
messages.push_back (_("Importing ") + books_id_to_localname (book) + " " + convert_to_string (chapter) + _(" from file ") + files[i]);
}
}
// Write accumulated messages.
htmlwriter.heading_open (3);
if (keep_going) {
htmlwriter.text_add (_("Success! Bible was created: ") + bible);
} else {
htmlwriter.text_add (_("Error!"));
}
htmlwriter.heading_close ();
if (keep_going) {
htmlwriter.paragraph_open ();
htmlwriter.text_add (_("To view the Hebrew text, open the Bible in the editor. Optionally set the font for better display of the Hebrew text. A donation made to Mechon Mamre will support their work."));
htmlwriter.paragraph_close ();
}
for (unsigned int i = 0; i < messages.size(); i++) {
htmlwriter.paragraph_open ();
htmlwriter.text_add (messages[i]);
htmlwriter.paragraph_close ();
}
// Write OK.
htmlwriter.paragraph_open ();
htmlwriter.hyperlink_add ("ok", _("Ok"));
htmlwriter.paragraph_close ();
}
