QString Import::toString(bool addSemicolon, bool skipAlias) const
{
QString result = QLatin1String("import ");
if (isFileImport())
result += '"' + file() + '"';
else if (isLibraryImport())
result += url();
else
return QString();
if (hasVersion())
result += ' ' + version();
if (hasAlias() && !skipAlias)
result += " as " + alias();
if (addSemicolon)
result += ';';
return result;
}
bool CSwordModuleInfo::hasIndex() {
//this will return true only
//if the index exists and has correct version information for both index and module
QDir d;
if (!d.exists( getModuleStandardIndexLocation() )) {
return false;
}
//first check if the index version and module version are ok
QSettings module_config(getModuleBaseIndexLocation() + QString("/bibletime-index.conf"), QSettings::IniFormat);
if (hasVersion()) {
if (module_config.value("module-version") != QString(config(CSwordModuleInfo::ModuleVersion)) ) {
return false;
}
}
if (module_config.value("index-version") != QString::number( INDEX_VERSION )) {
qDebug("%s: INDEX_VERSION is not compatible with this version of BibleTime.", name().toUtf8().constData());
return false;
}
//then check if the index is there
return lucene::index::IndexReader::indexExists(getModuleStandardIndexLocation().toAscii().constData());
}
QString CSwordModuleInfo::aboutText() const {
QString text;
text += "<table>";
text += QString("<tr><td><b>%1</b></td><td>%2</td><tr>")
.arg(tr("Version"))
.arg(hasVersion() ? config(CSwordModuleInfo::ModuleVersion) : tr("unknown"));
text += QString("<tr><td><b>%1</b></td><td>%2</td></tr>")
.arg(tr("Markup"))
.arg(!QString(m_module->getConfigEntry("SourceType")).isEmpty() ? QString(m_module->
getConfigEntry("SourceType")) : tr("unknown"));
text += QString("<tr><td><b>%1</b></td><td>%2</td></tr>")
.arg(tr("Location"))
.arg(config(CSwordModuleInfo::AbsoluteDataPath));
text += QString("<tr><td><b>%1</b></td><td>%2</td></tr>")
.arg(tr("Language"))
.arg(language()->translatedName());
if (m_module->getConfigEntry("Category"))
text += QString("<tr><td><b>%1</b></td><td>%2</td></tr>")
.arg(tr("Category"))
.arg(m_module->getConfigEntry("Category"));
if (m_module->getConfigEntry("LCSH"))
text += QString("<tr><td><b>%1</b></td><td>%2</td></tr>")
.arg(tr("LCSH"))
.arg(m_module->getConfigEntry("LCSH"));
text += QString("<tr><td><b>%1</b></td><td>%2</td></tr>")
.arg(tr("Writable"))
.arg(isWritable() ? tr("yes") : tr("no"));
if (isEncrypted())
text += QString("<tr><td><b>%1</b></td><td>%2</td></tr>")
.arg(tr("Unlock key"))
.arg(config(CSwordModuleInfo::CipherKey));
QString options;
unsigned int opts;
for (opts = CSwordModuleInfo::filterTypesMIN; opts <= CSwordModuleInfo::filterTypesMAX; ++opts) {
if (has(static_cast < CSwordModuleInfo::FilterTypes > (opts))) {
if (!options.isEmpty()) {
options += QString::fromLatin1(", ");
}
options += CSwordBackend::translatedOptionName(static_cast < CSwordModuleInfo::FilterTypes > (opts));
}
}
if (!options.isEmpty()) {
text += QString("<tr><td><b>%1</b></td><td>%2</td></tr>")
.arg(tr("Features"))
.arg(options);
}
text += "</table><hr>";
if (category() == Cult) //clearly say the module contains cult/questionable materials
text += QString("<br/><b>%1</b><br/><br/>")
.arg(tr("Take care, this work contains cult / questionable material!"));
text += QString("<b>%1:</b><br/>%2</font>")
.arg(tr("About"))
.arg(config(AboutInformation));
typedef QList<CSwordModuleInfo::ConfigEntry> ListConfigEntry;
ListConfigEntry entries;
entries.append(DistributionLicense);
entries.append(DistributionSource);
entries.append(DistributionNotes);
entries.append(TextSource);
entries.append(CopyrightNotes);
entries.append(CopyrightHolder);
entries.append(CopyrightDate);
entries.append(CopyrightContactName);
entries.append(CopyrightContactAddress);
entries.append(CopyrightContactEmail);
typedef QMap<CSwordModuleInfo::ConfigEntry, QString> MapConfigEntry;
MapConfigEntry entryMap;
entryMap[DistributionLicense] = tr("Distribution license");
entryMap[DistributionSource] = tr("Distribution source");
entryMap[DistributionNotes] = tr("Distribution notes");
entryMap[TextSource] = tr("Text source");
entryMap[CopyrightNotes] = tr("Copyright notes");
entryMap[CopyrightHolder] = tr("Copyright holder");
entryMap[CopyrightDate] = tr("Copyright date");
entryMap[CopyrightContactName] = tr("Copyright contact name");
entryMap[CopyrightContactAddress] = tr("Copyright contact address");
entryMap[CopyrightContactEmail] = tr("Copyright contact email");
text += ("<hr><table>");
for (ListConfigEntry::iterator it(entries.begin()); it != entries.end(); ++it) {
QString t( config(*it) );
if (!t.isEmpty()) {
text += QString("<tr><td><b>%1</b></td><td>%2</td></tr>")
.arg(entryMap[*it])
.arg(config(*it));
}
}
text += "</table></font>";
return text;
}
void CSwordModuleInfo::buildIndex() {
m_cancelIndexing = false;
try {
//Without this we don't get strongs, lemmas, etc
backend()->setFilterOptions ( CBTConfig::getFilterOptionDefaults() );
//make sure we reset all important filter options which influcence the plain filters.
// turn on these options, they are needed for the EntryAttributes population
backend()->setOption( CSwordModuleInfo::strongNumbers, true );
backend()->setOption( CSwordModuleInfo::morphTags, true );
backend()->setOption( CSwordModuleInfo::footnotes, true );
backend()->setOption( CSwordModuleInfo::headings, true );
// we don't want the following in the text, the do not carry searchable information
backend()->setOption( CSwordModuleInfo::morphSegmentation, false );
backend()->setOption( CSwordModuleInfo::scriptureReferences, false );
backend()->setOption( CSwordModuleInfo::redLetterWords, false );
// do not use any stop words
const TCHAR* stop_words[] = { NULL };
lucene::analysis::standard::StandardAnalyzer an( (const TCHAR**)stop_words );
QString index = getModuleStandardIndexLocation();
QDir dir("/");
dir.mkpath( getGlobalBaseIndexLocation() );
dir.mkpath( getModuleBaseIndexLocation() );
dir.mkpath( getModuleStandardIndexLocation() );
if (lucene::index::IndexReader::indexExists(index.toAscii().constData())) {
if (lucene::index::IndexReader::isLocked(index.toAscii().constData()) ) {
lucene::index::IndexReader::unlock(index.toAscii().constData());
}
}
boost::scoped_ptr<lucene::index::IndexWriter> writer( new lucene::index::IndexWriter(index.toAscii().constData(), &an, true) ); //always create a new index
writer->setMaxFieldLength(BT_MAX_LUCENE_FIELD_LENGTH);
writer->setUseCompoundFile(true); //merge segments into a single file
writer->setMinMergeDocs(1000);
*m_module = sword::TOP;
unsigned long verseLowIndex = m_module->Index();
*m_module = sword::BOTTOM;
unsigned long verseHighIndex = m_module->Index();
//verseLowIndex is not 0 in all cases (i.e. NT-only modules)
unsigned long verseIndex = verseLowIndex + 1;
unsigned long verseSpan = verseHighIndex - verseLowIndex;
//Index() is not implemented properly for lexicons, so we use a
//workaround.
if (type() == CSwordModuleInfo::Lexicon) {
verseIndex = 0;
verseLowIndex = 0;
verseSpan = ((CSwordLexiconModuleInfo*)this)->entries()->size();
}
emit indexingProgress(0);
sword::SWKey* key = m_module->getKey();
//VerseKey for bibles
sword::VerseKey* vk = dynamic_cast<sword::VerseKey*>(key);
if (vk) {
// we have to be sure to insert the english key into the index, otherwise we'd be in trouble if the language changes
vk->setLocale("en_US");
//If we have a verse based module, we want to include the pre-chapter etc. headings in the search
vk->Headings(1);
}
//holds UTF-8 data and is faster than QString.
QByteArray textBuffer;
// we start with the first module entry, key is automatically updated
// because key is a pointer to the modules key
m_module->setSkipConsecutiveLinks(true);
wchar_t wcharBuffer[BT_MAX_LUCENE_FIELD_LENGTH + 1];
for (*m_module = sword::TOP; !(m_module->Error()) && !m_cancelIndexing; (*m_module)++) {
// Also index Chapter 0 and Verse 0, because they might have information in the entry attributes
// We used to just put their content into the textBuffer and continue to the next verse, but
// with entry attributes this doesn't work any more.
// Hits in the search dialog will show up as 1:1 (instead of 0)
boost::scoped_ptr<lucene::document::Document> doc(new lucene::document::Document());
//index the key
lucene_utf8towcs(wcharBuffer, key->getText(), BT_MAX_LUCENE_FIELD_LENGTH);
//doc->add(*lucene::document::Field::UnIndexed((const TCHAR*)_T("key"), (const TCHAR*)wcharBuffer));
doc->add(*(new lucene::document::Field((const TCHAR*)_T("key"), (const TCHAR*)wcharBuffer, lucene::document::Field::STORE_YES | lucene::document::Field::INDEX_NO)));
// index the main text
//at this point we have to make sure we disabled the strongs and the other options
//so the plain filters won't include the numbers somehow.
lucene_utf8towcs(wcharBuffer, (const char*) textBuffer.append(m_module->StripText()), BT_MAX_LUCENE_FIELD_LENGTH);
doc->add(*(new lucene::document::Field((const TCHAR*)_T("content"), (const TCHAR*)wcharBuffer, lucene::document::Field::STORE_NO | lucene::document::Field::INDEX_TOKENIZED)));
textBuffer.resize(0); //clean up
// index attributes
sword::AttributeList::iterator attListI;
sword::AttributeValue::iterator attValueI;
// Footnotes
for (attListI = m_module->getEntryAttributes()["Footnote"].begin();
attListI != m_module->getEntryAttributes()["Footnote"].end();
attListI++) {
lucene_utf8towcs(wcharBuffer, attListI->second["body"], BT_MAX_LUCENE_FIELD_LENGTH);
//doc->add(*lucene::document::Field::UnStored((const TCHAR*)_T("footnote"), wcharBuffer));
doc->add(*(new lucene::document::Field((const TCHAR*)_T("footnote"), (const TCHAR*)wcharBuffer, lucene::document::Field::STORE_NO | lucene::document::Field::INDEX_TOKENIZED)));
} // for attListI
// Headings
for (attValueI = m_module->getEntryAttributes()["Heading"]["Preverse"].begin();
attValueI != m_module->getEntryAttributes()["Heading"]["Preverse"].end();
attValueI++) {
lucene_utf8towcs(wcharBuffer, attValueI->second, BT_MAX_LUCENE_FIELD_LENGTH);
//doc->add(*lucene::document::Field::UnStored((const TCHAR*)_T("heading"), wcharBuffer));
doc->add(*(new lucene::document::Field((const TCHAR*)_T("heading"), (const TCHAR*)wcharBuffer, lucene::document::Field::STORE_NO | lucene::document::Field::INDEX_TOKENIZED)));
} // for attValueI
// Strongs/Morphs
for (attListI = m_module->getEntryAttributes()["Word"].begin();
attListI != m_module->getEntryAttributes()["Word"].end();
attListI++) {
// for each attribute
if (attListI->second["LemmaClass"] == "strong") {
lucene_utf8towcs(wcharBuffer, attListI->second["Lemma"], BT_MAX_LUCENE_FIELD_LENGTH);
//doc->add(*lucene::document::Field::UnStored((const TCHAR*)_T("strong"), wcharBuffer));
doc->add(*(new lucene::document::Field((const TCHAR*)_T("strong"), (const TCHAR*)wcharBuffer, lucene::document::Field::STORE_NO | lucene::document::Field::INDEX_TOKENIZED)));
//qWarning("Adding strong %s", attListI->second["Lemma"].c_str());
}
if (attListI->second.find("Morph") != attListI->second.end()) {
lucene_utf8towcs(wcharBuffer, attListI->second["Morph"], BT_MAX_LUCENE_FIELD_LENGTH);
//doc->add(*lucene::document::Field::UnStored((const TCHAR*)_T("morph"), wcharBuffer));
doc->add(*(new lucene::document::Field((const TCHAR*)_T("morph"), (const TCHAR*)wcharBuffer, lucene::document::Field::STORE_NO | lucene::document::Field::INDEX_TOKENIZED)));
}
} // for attListI
writer->addDocument(doc.get());
//Index() is not implemented properly for lexicons, so we use a
//workaround.
if (type() == CSwordModuleInfo::Lexicon) {
verseIndex++;
}
else {
verseIndex = m_module->Index();
}
if (verseIndex % 200 == 0) {
int indexingProgressValue;
if (verseSpan == 0) { //prevent division by zero
//m_indexingProgress.setValue( QVariant(0) );
indexingProgressValue = 0;
}
else {
//m_indexingProgress.setValue( QVariant((int)((100*(verseIndex-verseLowIndex))/(verseHighIndex-verseLowIndex))) );
indexingProgressValue = (int)((100 * (verseIndex - verseLowIndex)) / (verseSpan));
}
//m_indexingProgress.activate();
emit indexingProgress(indexingProgressValue);
}
}
if (!m_cancelIndexing) {
writer->optimize();
}
writer->close();
if (m_cancelIndexing) {
deleteIndex();
m_cancelIndexing = false;
}
else {
QSettings module_config(getModuleBaseIndexLocation() + QString("/bibletime-index.conf"), QSettings::IniFormat);
if (hasVersion()) module_config.setValue("module-version", config(CSwordModuleInfo::ModuleVersion) );
module_config.setValue("index-version", INDEX_VERSION);
emit hasIndexChanged(true);
}
}
catch (...) {
qWarning("CLucene exception occurred while indexing");
util::showWarning(0, QCoreApplication::tr("Indexing aborted"), QCoreApplication::tr("An internal error occurred while building the index."));
deleteIndex();
m_cancelIndexing = false;
}
}
