void IWORKStyleRefContext::endOfElement()
{
const boost::optional<std::string> dummyIdent;
const boost::optional<IWORKPropertyMap> dummyProps;
// TODO: need to get the style
switch (m_id)
{
case IWORKToken::NS_URI_SF | IWORKToken::cell_style_ref :
getCollector()->collectStyle(IWORKStylePtr_t(), m_anonymous);
break;
case IWORKToken::NS_URI_SF | IWORKToken::characterstyle_ref :
{
IWORKStylePtr_t style;
if (getRef())
{
const IWORKStyleMap_t::const_iterator it = getDictionary().m_characterStyles.find(get(getRef()));
if (getDictionary().m_characterStyles.end() != it)
style = it->second;
}
getCollector()->collectStyle(style, m_anonymous);
break;
}
case IWORKToken::NS_URI_SF | IWORKToken::liststyle_ref :
getCollector()->collectStyle(IWORKStylePtr_t(), m_anonymous);
break;
case IWORKToken::NS_URI_SF | IWORKToken::paragraphstyle_ref :
{
IWORKStylePtr_t style;
if (getRef())
{
const IWORKStyleMap_t::const_iterator it = getDictionary().m_paragraphStyles.find(get(getRef()));
if (getDictionary().m_paragraphStyles.end() != it)
style = it->second;
}
getCollector()->collectStyle(style, m_anonymous);
break;
}
case IWORKToken::NS_URI_SF | IWORKToken::vector_style_ref :
getCollector()->collectStyle(IWORKStylePtr_t(), m_anonymous);
break;
default :
break;
}
}
already_AddRefd<iface::cellml_services::MaLaESTransform>
CDA_DictionaryGenerator::getMalTransform()
throw(std::exception&)
{
RETURN_INTO_OBJREF(malDict, iface::cellml_services::LanguageDictionary,
getDictionary(L"http://www.cellml.org/CeLEDS/MaLaES/1.0#"));
if (malDict == NULL)
return NULL;
uint32_t i;
std::wstring MalString(L"");
RETURN_INTO_OBJREF(entries, iface::dom::NodeList,
malDict->getMappings());
for (i=0; i < entries->length(); i++)
{
RETURN_INTO_OBJREF(currentNode, iface::dom::Node, entries->item(i));
DECLARE_QUERY_INTERFACE_OBJREF(currentEl, currentNode, dom::Element);
if (currentEl == NULL)
continue;
// Get attributes
RETURN_INTO_WSTRING(keyName,
currentEl->getAttribute(L"keyname"));
RETURN_INTO_WSTRING(precedence,
currentEl->getAttribute(L"precedence"));
// Create Mal string
MalString.append(keyName);
MalString.append(L": ");
if (precedence != L"")
{
MalString.append(L"#prec[");
MalString.append(precedence);
MalString.append(L"]");
}
RETURN_INTO_WSTRING(tc, getTextContents(currentNode));
RETURN_INTO_WSTRING(ptc, padMalString(tc.c_str()));
MalString.append(ptc);
MalString.append(L"\r\n");
}
// create transformer
RETURN_INTO_OBJREF(mb, iface::cellml_services::MaLaESBootstrap,
CreateMaLaESBootstrap());
try
{
return mb->compileTransformer(MalString.c_str());
}
catch (...)
{
return NULL;
}
}
void KEY1ParserState::storeCurrentPlaceholder()
{
if (!m_isBodyContentOpened && !m_isTitleContentOpened)
return;
bool isTitle=m_isTitleContentOpened;
m_isBodyContentOpened = m_isTitleContentOpened = false;
if (!m_enableCollector)
return;
if (m_currentText && !m_currentText->empty())
{
if (isTitle && getDictionary().getTitlePlaceholder())
getDictionary().getTitlePlaceholder()->m_text=m_currentText;
else if (!isTitle && getDictionary().getBodyPlaceholder())
getDictionary().getBodyPlaceholder()->m_text=m_currentText;
else
{
ETONYEK_DEBUG_MSG(("KEY1ParserState::storeCurrentPlaceholder: oops can not store the text zone\n"));
}
}
m_currentText.reset();
}
int main(int argc, char *argv[])
{
QApplication a(argc, argv);
QVector<train_tweet> train_sample=getTrainSample("/Users/v144/Documents/workspace/C++/build-tweets-Desktop_Qt_5_6_0_clang_64bit-Debug/bank_train_2016.xml");
QMap<QString,dictionaryWord> dict=getDictionary(train_sample);
QVector<train_tweet> etalon_sample=getEtalonSample("/Users/v144/Documents/workspace/C++/build-tweets-Desktop_Qt_5_6_0_clang_64bit-Debug/banks_test_etalon.xml");
isValidDictionary(dict,etalon_sample);
return a.exec();
}
bool BencodeParser::parse(const QByteArray &content)
{
if (content.isEmpty()) {
errString = QString("No content");
return false;
}
this->content = content;
index = 0;
infoStart = 0;
infoLength = 0;
return getDictionary(&dictionaryValue);
}
vector<unsigned char> SuffixModelTrieBinaryFileCreator::binarySuffixModelNode(SuffixModelNode * _node, int _parentId)
{
vector<unsigned char> _result;
_result.push_back(getDictionary()->getDictionaryTools()->wcharToChar(_node->getCharacter()));
vector<unsigned char> binParentId = getDictionary()->getDictionaryTools()->intToCharVector3(_parentId);
_result.insert(_result.end(), binParentId.begin(), binParentId.end());
if (max_frequency_size < (int) _node->getFeatureFrequencyMap().size())
{
max_frequency_size = (int) _node->getFeatureFrequencyMap().size();
wcout << "update max_frequency_size = " << max_frequency_size << endl;
}
map<int, int> _frequencyMap = _node->getFeatureFrequencyMap();
vector<unsigned char> binSizeOfFrequencyMap = getDictionary()->getDictionaryTools()->intToCharVector2((int) _frequencyMap.size());
_result.insert(_result.end(), binSizeOfFrequencyMap.begin(), binSizeOfFrequencyMap.end());
if (!_frequencyMap.size())
{
return _result;
}
map<int, int>::iterator iter;
for (iter = _frequencyMap.begin(); iter != _frequencyMap.end(); ++iter)
{
int _featureId = iter->first;
vector<unsigned char> binFeatureId = getDictionary()->getDictionaryTools()->intToCharVector3(_featureId);
_result.insert(_result.end(), binFeatureId.begin(), binFeatureId.end());
int _frequency = iter->second;
if (max_frequency < _frequency)
{
max_frequency = _frequency;
wcout << "update max_frequency = " << max_frequency << endl;
}
vector<unsigned char> binFrequency = getDictionary()->getDictionaryTools()->intToCharVector2(_frequency);
_result.insert(_result.end(), binFrequency.begin(), binFrequency.end());
}
return _result;
}
/**
* @brief Saves the Python dictionary containing the user-language_pair settings to a file
*
* pythonInit() must have been called before or an error will occur (the module is not loaded)
*/
void saveDictionary(void){
PyObject *pFunc, *pArgs;
if(getDictionary() != Py_None && getDictionary() != NULL){
if (files_module != NULL) {
pFunc = PyObject_GetAttrString(files_module, "save");
if (pFunc) {
pArgs = PyTuple_New(0);
PyObject_CallObject(pFunc, pArgs);
}
else {
return;
}
Py_XDECREF(pFunc);
}
else {
notify_error("Module: \'apertiumFiles\' is not loaded");
return;
}
}
}
/**
* @brief Returns the language stored for a user in the preferences file
*
* pythonInit() must have been called before or an error will occur (the module is not loaded)
* @param user Name of the user to look for
* @param direction Direction to look for the user in ("incoming" or "outgoing")
* @param key Language to look for ("source" or "target")
* @return The language if the call was successful, or "None" otherwise
*/
char* dictionaryGetUserLanguage(const char *user, const char* direction, const char* key){
char* user_lang;
PyObject *dictionary;
if((dictionary = getDictionary()) == Py_None){
return "None";
}
user_lang = PyBytes_AsString(PyDict_GetItemString(
PyDict_GetItemString(PyDict_GetItemString(dictionary, direction), user),key));
Py_XDECREF(dictionary);
return user_lang;
}
bool BencodeParser::getDictionary(QMap<QByteArray, QVariant> *dictionary)
{
const int contentSize = content.size();
if (content.at(index) != 'd')
return false;
QMap<QByteArray, QVariant> tmp;
++index;
do {
if (content.at(index) == 'e') {
++index;
break;
}
QByteArray key;
if (!getByteString(&key))
break;
if (key == "info")
infoStart = index;
qint64 number;
QByteArray byteString;
QList<QVariant> tmpList;
QMap<QByteArray, QVariant> dictionary;
if (getInteger(&number))
tmp.insert(key, number);
else if (getByteString(&byteString))
tmp.insert(key, byteString);
else if (getList(&tmpList))
tmp.insert(key, tmpList);
else if (getDictionary(&dictionary))
tmp.insert(key, QVariant::fromValue<QMap<QByteArray, QVariant> >(dictionary));
else {
errString = QString("error at index %1").arg(index);
return false;
}
if (key == "info")
infoLength = index - infoStart;
} while (index < contentSize);
if (dictionary)
*dictionary = tmp;
return true;
}
/**
* @brief Checks whether the dictionary contains language pair information for a given user
*
* pythonInit() must have been called before or an error will occur (the module is not loaded)
* @param user Name of the user to look for
* @param direction Direction to look for the user in ("incoming" or "outgoing")
* @return 1 if there is a language pair for the user, or 0 otherwise
*/
int dictionaryHasUser(const char* user, const char* direction){
int has_user;
PyObject *dictionary;
if((dictionary = getDictionary()) == Py_None){
return 0;
}
has_user = PyDict_Contains(
PyDict_GetItemString(dictionary, direction),
PyUnicode_FromString(user));
Py_XDECREF(dictionary);
return has_user;
}
/*****************************************************************************
* ~Ndb();
*
* Remark: Disconnect with the database.
*****************************************************************************/
Ndb::~Ndb()
{
DBUG_ENTER("Ndb::~Ndb()");
DBUG_PRINT("enter",("this: 0x%lx", (long) this));
if (m_sys_tab_0)
getDictionary()->removeTableGlobal(*m_sys_tab_0, 0);
assert(theImpl->m_ev_op == 0); // user should return NdbEventOperation's
for (NdbEventOperationImpl *op= theImpl->m_ev_op; op; op=op->m_next)
{
if (op->m_state == NdbEventOperation::EO_EXECUTING && op->stop())
g_eventLogger.error("stopping NdbEventOperation failed in Ndb destructor");
op->m_magic_number= 0;
}
doDisconnect();
/* Disconnect from transporter to stop signals from coming in */
if (theImpl->m_transporter_facade != NULL && theNdbBlockNumber > 0) {
theImpl->m_transporter_facade->close(theNdbBlockNumber, theFirstTransId);
}
delete theEventBuffer;
releaseTransactionArrays();
delete []theConnectionArray;
if(theCommitAckSignal != NULL) {
delete theCommitAckSignal;
theCommitAckSignal = NULL;
}
delete theImpl;
#ifdef POORMANSPURIFY
#ifdef POORMANSGUI
ndbout << "cnewSignals=" << cnewSignals << endl;
ndbout << "cfreeSignals=" << cfreeSignals << endl;
ndbout << "cgetSignals=" << cgetSignals << endl;
ndbout << "creleaseSignals=" << creleaseSignals << endl;
#endif
// Poor mans purifier
assert(cnewSignals == cfreeSignals);
assert(cgetSignals == creleaseSignals);
#endif
DBUG_VOID_RETURN;
}
void BasicHDT::fillHeader(string& baseUri) {
string formatNode = "_:format";
string dictNode = "_:dictionary";
string triplesNode = "_:triples";
string statisticsNode = "_:statistics";
string publicationInfoNode = "_:publicationInformation";
uint64_t origSize = header->getPropertyLong(statisticsNode.c_str(), HDTVocabulary::ORIGINAL_SIZE.c_str());
header->clear();
// BASE
header->insert(baseUri, HDTVocabulary::RDF_TYPE, HDTVocabulary::HDT_DATASET);
// VOID
header->insert(baseUri, HDTVocabulary::RDF_TYPE, HDTVocabulary::VOID_DATASET);
header->insert(baseUri, HDTVocabulary::VOID_TRIPLES, triples->getNumberOfElements());
header->insert(baseUri, HDTVocabulary::VOID_PROPERTIES, dictionary->getNpredicates());
header->insert(baseUri, HDTVocabulary::VOID_DISTINCT_SUBJECTS, dictionary->getNsubjects());
header->insert(baseUri, HDTVocabulary::VOID_DISTINCT_OBJECTS, dictionary->getNobjects());
// TODO: Add more VOID Properties. E.g. void:classes
// Structure
header->insert(baseUri, HDTVocabulary::HDT_STATISTICAL_INFORMATION, statisticsNode);
header->insert(baseUri, HDTVocabulary::HDT_PUBLICATION_INFORMATION, publicationInfoNode);
header->insert(baseUri, HDTVocabulary::HDT_FORMAT_INFORMATION, formatNode);
header->insert(formatNode, HDTVocabulary::HDT_DICTIONARY, dictNode);
header->insert(formatNode, HDTVocabulary::HDT_TRIPLES, triplesNode);
// Dictionary
dictionary->populateHeader(*header, dictNode);
// Triples
triples->populateHeader(*header, triplesNode);
// Sizes
header->insert(statisticsNode, HDTVocabulary::ORIGINAL_SIZE, origSize);
header->insert(statisticsNode, HDTVocabulary::HDT_SIZE, getDictionary()->size() + getTriples()->size());
// Current time
time_t now;
char date[40];
time(&now);
struct tm* today = localtime(&now);
strftime(date, 40, "%Y-%m-%dT%H:%M:%S%z", today);
header->insert(publicationInfoNode, HDTVocabulary::DUBLIN_CORE_ISSUED, date);
}
/**
* @brief Removes all the entries from the dictionary related to the given user
*
* pythonInit() must have been called before or an error will occur (the module is not loaded)
* @param user Name of the user whose entries will be removed
* @return 1 on success, or 0 otherwise
*/
int dictionaryRemoveUserEntries(const char* user){
PyObject *dictionary;
if((dictionary = getDictionary()) == Py_None){
return 0;
}
if(PyDict_Contains(PyDict_GetItemString(dictionary, "incoming"), PyUnicode_FromString(user))){
PyDict_DelItemString(PyDict_GetItemString(dictionary, "incoming"), user);
}
if(PyDict_Contains(PyDict_GetItemString(dictionary, "outgoing"), PyUnicode_FromString(user))){
PyDict_DelItemString(PyDict_GetItemString(dictionary, "outgoing"), user);
}
setDictionary(dictionary);
Py_XDECREF(dictionary);
return 1;
}
bool BencodeParser::getList(QList<QVariant> *list)
{
const int contentSize = content.size();
if (content.at(index) != 'l')
return false;
QList<QVariant> tmp;
++index;
do {
if (content.at(index) == 'e') {
++index;
break;
}
qint64 number;
QByteArray byteString;
QList<QVariant> tmpList;
QMap<QByteArray, QVariant> dictionary;
if (getInteger(&number))
tmp << number;
else if (getByteString(&byteString))
tmp << byteString;
else if (getList(&tmpList))
tmp << tmpList;
else if (getDictionary(&dictionary))
tmp << QVariant::fromValue<QMap<QByteArray, QVariant> >(dictionary);
else {
errString = QString("error at index %1").arg(index);
return false;
}
} while (index < contentSize);
if (list)
*list = tmp;
return true;
}
/*
* Bind the application dictionary to the cache
*/
static void initializeCacheDictionary()
{
RsslDataDictionary *dictionary;
if (cacheInfo.useCache && cacheInfo.cacheHandle)
{
dictionary = getDictionary();
if (dictionary)
{
if (rsslPayloadCacheSetDictionary(cacheInfo.cacheHandle, dictionary, cacheInfo.cacheDictionaryKey, &cacheInfo.cacheErrorInfo)
!= RSSL_RET_SUCCESS)
{
printf("Error: Failed to bind RDM Field dictionary to cache.\n\tError (%d): %s\n",
cacheInfo.cacheErrorInfo.rsslErrorId, cacheInfo.cacheErrorInfo.text);
cacheInfo.useCache = RSSL_FALSE;
}
}
else
{
printf("Error: No RDM Field dictionary for cache.\n");
cacheInfo.useCache = RSSL_FALSE;
}
}
}
int Ngn_Hashcat::run ()
{
Application& app = Application::instance();
DJob *job = DJob::Instance();
string cmd;
string hashcat(dengine->getBinaryPath("hashcat"));
int eCode(-1);
// clean results
_results = string("");
// grab settings from job
setAttackMode(job->getAttackMode());
setHashType(job->getHashType());
setMask(job->getMask());
setRules(job->getRules());
setDictionary(job->getDictionary());
setPot("disthc.pot");
// clean pot before working
File f(getPot());
if(f.exists()) {
f.remove();
}
// setup command prefix (format command takes 7 args max)
cmd = format("%s -o %s -s %lu -l %u",
hashcat,
getPot(),
job->getChunk(),
job->getChunkSize()
);
// Attack modes:
// 0 = Straight
// 1 = Combination
// 2 = Toggle-Case
// 3 = Brute-force
// 4 = Permutation
// 5 = Table-Lookup
// if mask minimum set, apply it
if(job->getMaskMin())
{
cmd = format("%s --pw-min %d",
cmd,
job->getMaskMin()
);
}
// if mask maximum set, apply it
if(job->getMaskMax())
{
cmd = format("%s --pw-min %d",
cmd,
job->getMaskMax()
);
}
// discover attack mode and create command to execute
switch(getAttackMode())
{
case 3:
cmd = format("%s -a3 -m %d %s %s %s",
cmd,
getHashType(),
getFlags(),
getHashFile(),
getMask()
);
break;
default:
// default command uses attack mode 0
cmd = format("%s -m %d %s %s %s %s",
cmd,
getHashType(),
getFlags(),
getHashFile(),
getDictionary(),
getRules()
);
}
if(DEBUG) app.logger().information(format("%%Running command: %s", cmd));
// check for ghosts, and run as appropriate
if(isGhost())
{
app.logger().information("~~~ A ghost is loose! ~~~");
app.logger().information(" .-.");
app.logger().information(" (o o) boo!");
app.logger().information(" \\| O \\/");
app.logger().information(" \\ \\ ");
app.logger().information(" `~~~' ");
}
else
{
// run hashcat! :)
// TODO change this over to use Poco Processes
eCode = system(cmd.c_str());
// check for results
if(f.exists()) {
FileInputStream fis(getPot());
//std::ifstream in(pot,std::ios::in);
string line;
while(fis >> line) {
_results.append(line + "\n");
}
}
// TODO might take this out?
// see if it's worth it to just display hashcout output during
// execution
// if enabled, print pot to screen
// if(false) {
// app.logger().information("\n=== Recovered Hashes ===");
// if(!_results.empty()) app.logger().information(_results);
// app.logger().information("========================");
// }
}
void SuffixModelTrieBinaryFileCreator::buildBuffer(void)
{
wcout << "SuffixModelTrieBinaryFileCreator build buffer ...";
buffer = new unsigned char[N_GRAM_MAX_BUFFER_SIZE];
bufferSize = 0;
wcout << "numberOfNodes = " << numberOfNodes << endl;
vector<unsigned char> binNumberOfNode = getDictionary()->getDictionaryTools()->intToCharVector3(numberOfNodes);
writeToBuffer(binNumberOfNode);
queue<SuffixModelNode*> nodeQueue = queue<SuffixModelNode*>();
nodeQueue.push(root);
int currentNodeId = -1;
int _count = 1;
// write root
vector<unsigned char> binRoot = binarySuffixModelNode(root, currentNodeId);
writeToBuffer(binRoot);
while (!nodeQueue.empty())
{
SuffixModelNode* currentNode = nodeQueue.front();
nodeQueue.pop();
currentNodeId++;
vector<SuffixModelNode*> childrenNodes = currentNode->getChildrenNode();
for (int i = 0; i < (int) childrenNodes.size(); ++i)
{
SuffixModelNode* childNode = childrenNodes.at(i);
nodeQueue.push(childNode);
_count++;
// write node
vector<unsigned char> binCurrentNode = binarySuffixModelNode(childNode, currentNodeId);
writeToBuffer(binCurrentNode);
}
}
wcout << " Node count = " << _count << endl;
_count = 0;
// featureIdFrequency
wcout << "featureIdFrequency.size = " << (int) featureIdFrequency.size() << endl;
vector<unsigned char> binFeatureIdFrequencySize = getDictionary()->getDictionaryTools()->intToCharVector2((int) featureIdFrequency.size());
writeToBuffer(binFeatureIdFrequencySize);
map<int, int>::iterator iter;
for (iter = featureIdFrequency.begin(); iter != featureIdFrequency.end(); ++iter)
{
int _featureId = iter->first;
vector<unsigned char> binFeatureId = getDictionary()->getDictionaryTools()->intToCharVector3(_featureId);
writeToBuffer(binFeatureId);
int _frequency = iter->second;
vector<unsigned char> binFrequency = getDictionary()->getDictionaryTools()->intToCharVector3(_frequency);
writeToBuffer(binFrequency);
_count++;
if (max_feature_frequency < _frequency)
{
max_feature_frequency = _frequency;
//wcout << "update max_frequency = " << max_frequency << endl;
}
}
wcout << "Done! Count = " << _count << endl;
wcout << "max_frequency_size = " << max_frequency_size << endl;
wcout << "max_frequency = " << max_frequency << endl;
wcout << "max_feature_frequency = " << max_feature_frequency << endl;
}
/*****************************************************************************
* ~Ndb();
*
* Remark: Disconnect with the database.
*****************************************************************************/
Ndb::~Ndb()
{
DBUG_ENTER("Ndb::~Ndb()");
DBUG_PRINT("enter",("this: 0x%lx", (long) this));
if (m_sys_tab_0)
getDictionary()->removeTableGlobal(*m_sys_tab_0, 0);
if (theImpl->m_ev_op != 0)
{
g_eventLogger->warning("Deleting Ndb-object with NdbEventOperation still"
" active");
printf("this: %p NdbEventOperation(s): ", this);
for (NdbEventOperationImpl *op= theImpl->m_ev_op; op; op=op->m_next)
{
printf("%p ", op);
}
printf("\n");
fflush(stdout);
}
assert(theImpl->m_ev_op == 0); // user should return NdbEventOperation's
for (NdbEventOperationImpl *op= theImpl->m_ev_op; op; op=op->m_next)
{
if (op->m_state == NdbEventOperation::EO_EXECUTING && op->stop())
g_eventLogger->error("stopping NdbEventOperation failed in Ndb destructor");
op->m_magic_number= 0;
}
doDisconnect();
/* Disconnect from transporter to stop signals from coming in */
theImpl->close();
delete theEventBuffer;
releaseTransactionArrays();
delete []theConnectionArray;
delete []theConnectionArrayLast;
if(theCommitAckSignal != NULL){
delete theCommitAckSignal;
theCommitAckSignal = NULL;
}
theImpl->m_ndb_cluster_connection.unlink_ndb_object(this);
delete theImpl;
#ifdef POORMANSPURIFY
#ifdef POORMANSGUI
ndbout << "cnewSignals=" << cnewSignals << endl;
ndbout << "cfreeSignals=" << cfreeSignals << endl;
ndbout << "cgetSignals=" << cgetSignals << endl;
ndbout << "creleaseSignals=" << creleaseSignals << endl;
#endif
// Poor mans purifier
assert(cnewSignals == cfreeSignals);
assert(cgetSignals == creleaseSignals);
#endif
DBUG_VOID_RETURN;
}
/*
* Publically visable - used by all non-admin domain handlers to output field lists
*
* Decodes the field entry data and prints out the field entry data
* with help of the dictionary. Returns success if decoding succeeds
* or failure if decoding fails.
* fEntry - The field entry data
* dIter - The decode iterator
*/
RsslRet decodeFieldEntry(RsslFieldEntry* fEntry, RsslDecodeIterator *dIter)
{
RsslRet ret = 0;
RsslDataType dataType = RSSL_DT_UNKNOWN;
RsslUInt64 fidUIntValue = 0;
RsslInt64 fidIntValue = 0;
RsslFloat tempFloat = 0;
RsslDouble tempDouble = 0;
RsslReal fidRealValue = RSSL_INIT_REAL;
RsslEnum fidEnumValue;
RsslFloat fidFloatValue = 0;
RsslDouble fidDoubleValue = 0;
RsslQos fidQosValue = RSSL_INIT_QOS;
RsslDateTime fidDateTimeValue;
RsslState fidStateValue;
RsslBuffer fidBufferValue;
RsslBuffer fidDateTimeBuf;
RsslBuffer fidRealBuf;
RsslBuffer fidStateBuf;
RsslBuffer fidQosBuf;
RsslDataDictionary* dictionary = getDictionary();
RsslDictionaryEntry* dictionaryEntry = NULL;
/* get dictionary entry */
if (!dictionary->entriesArray)
{
dumpHexBuffer(&fEntry->encData);
return RSSL_RET_SUCCESS;
}
else
dictionaryEntry = dictionary->entriesArray[fEntry->fieldId];
/* return if no entry found */
if (!dictionaryEntry)
{
printf("\tFid %d not found in dictionary\n", fEntry->fieldId);
dumpHexBuffer(&fEntry->encData);
return RSSL_RET_SUCCESS;
}
/* print out fid name */
printf("\t%-20s", dictionaryEntry->acronym.data);
/* decode and print out fid value */
dataType = dictionaryEntry->rwfType;
switch (dataType)
{
case RSSL_DT_UINT:
if ((ret = rsslDecodeUInt(dIter, &fidUIntValue)) == RSSL_RET_SUCCESS)
{
printf(""RTR_LLU"\n", fidUIntValue);
}
else if (ret != RSSL_RET_BLANK_DATA)
{
printf("rsslDecodeUInt() failed with return code: %d\n", ret);
return ret;
}
break;
case RSSL_DT_INT:
if ((ret = rsslDecodeInt(dIter, &fidIntValue)) == RSSL_RET_SUCCESS)
{
printf(""RTR_LLD"\n", fidIntValue);
}
else if (ret != RSSL_RET_BLANK_DATA)
{
printf("rsslDecodeInt() failed with return code: %d\n", ret);
return ret;
}
break;
case RSSL_DT_FLOAT:
if ((ret = rsslDecodeFloat(dIter, &fidFloatValue)) == RSSL_RET_SUCCESS)
{
printf("%f\n", fidFloatValue);
}
else if (ret != RSSL_RET_BLANK_DATA)
{
printf("rsslDecodeFloat() failed with return code: %d\n", ret);
return ret;
}
break;
case RSSL_DT_DOUBLE:
if ((ret = rsslDecodeDouble(dIter, &fidDoubleValue)) == RSSL_RET_SUCCESS)
{
printf("%f\n", fidDoubleValue);
}
else if (ret != RSSL_RET_BLANK_DATA)
{
printf("rsslDecodeDouble() failed with return code: %d\n", ret);
return ret;
}
break;
case RSSL_DT_REAL:
if ((ret = rsslDecodeReal(dIter, &fidRealValue)) == RSSL_RET_SUCCESS)
{
fidRealBuf.data = (char*)alloca(35);
fidRealBuf.length = 35;
rsslRealToString(&fidRealBuf, &fidRealValue);
printf("%s\n", fidRealBuf.data);
}
else if (ret != RSSL_RET_BLANK_DATA)
{
printf("rsslDecodeReal() failed with return code: %d\n", ret);
return ret;
}
break;
case RSSL_DT_ENUM:
if ((ret = rsslDecodeEnum(dIter, &fidEnumValue)) == RSSL_RET_SUCCESS)
{
RsslEnumType *pEnumType = getFieldEntryEnumType(dictionaryEntry, fidEnumValue);
if (pEnumType)
printf("%.*s(%d)\n", pEnumType->display.length, pEnumType->display.data, fidEnumValue);
else
printf("%d\n", fidEnumValue);
}
else if (ret != RSSL_RET_BLANK_DATA)
{
printf("rsslDecodeEnum() failed with return code: %d\n", ret);
return ret;
}
break;
case RSSL_DT_DATE:
if ((ret = rsslDecodeDate(dIter, &fidDateTimeValue.date)) == RSSL_RET_SUCCESS)
{
fidDateTimeBuf.data = (char*)alloca(30);
fidDateTimeBuf.length = 30;
rsslDateTimeToString(&fidDateTimeBuf, RSSL_DT_DATE, &fidDateTimeValue);
printf("%s\n", fidDateTimeBuf.data);
}
else if (ret != RSSL_RET_BLANK_DATA)
{
printf("rsslDecodeDate() failed with return code: %d\n", ret);
return ret;
}
break;
case RSSL_DT_TIME:
if ((ret = rsslDecodeTime(dIter, &fidDateTimeValue.time)) == RSSL_RET_SUCCESS)
{
fidDateTimeBuf.data = (char*)alloca(30);
fidDateTimeBuf.length = 30;
rsslDateTimeToString(&fidDateTimeBuf, RSSL_DT_TIME, &fidDateTimeValue);
printf("%s\n", fidDateTimeBuf.data);
}
else if (ret != RSSL_RET_BLANK_DATA)
{
printf("rsslDecodeTime() failed with return code: %d\n", ret);
return ret;
}
break;
case RSSL_DT_DATETIME:
if ((ret = rsslDecodeDateTime(dIter, &fidDateTimeValue)) == RSSL_RET_SUCCESS)
{
fidDateTimeBuf.data = (char*)alloca(50);
fidDateTimeBuf.length = 50;
rsslDateTimeToString(&fidDateTimeBuf, RSSL_DT_DATETIME, &fidDateTimeValue);
printf("%s\n", fidDateTimeBuf.data);
}
else if (ret != RSSL_RET_BLANK_DATA)
{
printf("rsslDecodeDateTime() failed with return code: %d\n", ret);
return ret;
}
break;
case RSSL_DT_QOS:
if((ret = rsslDecodeQos(dIter, &fidQosValue)) == RSSL_RET_SUCCESS) {
fidQosBuf.data = (char*)alloca(100);
fidQosBuf.length = 100;
rsslQosToString(&fidQosBuf, &fidQosValue);
printf("%s\n", fidQosBuf.data);
}
else if (ret != RSSL_RET_BLANK_DATA)
{
printf("rsslDecodeQos() failed with return code: %d\n", ret);
return ret;
}
break;
case RSSL_DT_STATE:
if((ret = rsslDecodeState(dIter, &fidStateValue)) == RSSL_RET_SUCCESS) {
int stateBufLen = 80;
if (fidStateValue.text.data)
stateBufLen += fidStateValue.text.length;
fidStateBuf.data = (char*)alloca(stateBufLen);
fidStateBuf.length = stateBufLen;
rsslStateToString(&fidStateBuf, &fidStateValue);
printf("%.*s\n", fidStateBuf.length, fidStateBuf.data);
}
else if (ret != RSSL_RET_BLANK_DATA)
{
printf("rsslDecodeState() failed with return code: %d\n", ret);
return ret;
}
break;
/*For an example of array decoding, see fieldListEncDec.c*/
case RSSL_DT_ARRAY:
break;
case RSSL_DT_BUFFER:
case RSSL_DT_ASCII_STRING:
case RSSL_DT_UTF8_STRING:
case RSSL_DT_RMTES_STRING:
if((ret = rsslDecodeBuffer(dIter, &fidBufferValue)) == RSSL_RET_SUCCESS)
{
printf("%.*s\n", fidBufferValue.length, fidBufferValue.data);
}
else if (ret != RSSL_RET_BLANK_DATA)
{
printf("rsslDecodeBuffer() failed with return code: %d\n", ret);
return ret;
}
break;
default:
printf("Unsupported data type (%d) for fid value\n", dataType);
break;
}
if (ret == RSSL_RET_BLANK_DATA)
{
printf("<blank data>\n");
}
return RSSL_RET_SUCCESS;
}
/**
* Load dictionary from binary file
*/
void SuffixModelTrieBinaryFileReader::loadFromBinaryFile(string _filePath)
{
ifstream f(_filePath.c_str(), ios::in|ios::binary|ios::ate);
char* buffer;
if (f.is_open())
{
// get size of file
int size = (int) f.tellg();
// jump to begin of file
f.seekg(0, ios::beg);
//============= Read SuffixModelNode(s) ============================================================================================
// allocate buffer
buffer = new char[size];
// read file
f.read(buffer, size);
// close file
f.close();
// set offset begin of buffer
int offset = 0;
// convert 3 bytes to number of NodeVer3
numberOfNodes = (unsigned char) buffer[offset] * 65536 + (unsigned char) buffer[offset + 1] * 256 + (unsigned char) buffer[offset + 2];
offset += 3;
// read list of SuffixModelNode
SuffixModelNodeList = vector<SuffixModelNode*>();
for (int _nodeId = 0; _nodeId < numberOfNodes; ++_nodeId)
{
// convert first byte to wchar_t
wchar_t _character = getDictionary()->getDictionaryTools()->charToWchar((unsigned char) buffer[offset]);
// convert 3 remaining bytes to _parentId
int _parentId = (unsigned char) buffer[offset + 1] * 65536 + (unsigned char) buffer[offset + 2] * 256 + (unsigned char) buffer[offset + 3];
offset += 4;
// create new SuffixModelNode
SuffixModelNode* _node = new SuffixModelNode(_character);
// addChildNode
if (_parentId >= 0 && _parentId < _nodeId)
{
SuffixModelNodeList.at(_parentId)->addChildNode(_node);
}
// size of frequency map
int _sizeOfFrequencyMap = (unsigned char) buffer[offset] * 256 + (unsigned char) buffer[offset + 1];
offset += 2;
map<int, int> _featureFrequencyMap = map<int, int>();
for (int i = 0; i < _sizeOfFrequencyMap; ++i)
{
int _featureId = (unsigned char) buffer[offset] * 65536 + (unsigned char) buffer[offset + 1] * 256 + (unsigned char) buffer[offset + 2];
offset += 3;
int _frequency = (unsigned char) buffer[offset] * 256 + (unsigned char) buffer[offset + 1];
offset += 2;
_featureFrequencyMap.insert(pair<int, int>(_featureId, _frequency));
}
_node->setFeatureFrequencyMap(_featureFrequencyMap);
SuffixModelNodeList.push_back(_node);
}
root = SuffixModelNodeList.at(0);
//============= Read featureIdFrequency ============================================================================================
// convert 3 bytes to number of SuffixModelNode
int featureIdFrequencySize = (unsigned char) buffer[offset] * 256 + (unsigned char) buffer[offset + 1];
offset += 2;
for (int i = 0; i < featureIdFrequencySize; ++i)
{
int _featureId = (unsigned char) buffer[offset] * 65536 + (unsigned char) buffer[offset + 1] * 256 + (unsigned char) buffer[offset + 2];
offset += 3;
int _frequency = (unsigned char) buffer[offset] * 65536 + (unsigned char) buffer[offset + 1] * 256 + (unsigned char) buffer[offset + 2];
offset += 3;
featureIdFrequency.insert(pair<int, int>(_featureId, _frequency));
}
delete[] buffer;
buffer = NULL;
}
else
{
throw FILE_NOT_FOUND_ERROR_CODE;
//wcout << "### Error ### : loadFromBinaryFile -> Unable to open file";
}
}
/**
* Load dictionary from binary file
*/
void DictionaryTrieBinaryFileReader::loadFromBinaryFile(string _filePath)
{
bool debug = false;
// wcout << "loadFromBinaryFile" << endl;
ifstream f(_filePath.c_str(), ios::in|ios::binary|ios::ate);
unsigned char* buffer;
if (f.is_open())
{
// get size of file
int size = (int) f.tellg();
// jump to begin of file
f.seekg(0, ios::beg);
//============= Read NodeVer3s ============================================================================================
// allocate buffer
buffer = new unsigned char[size];
// read file
f.read((char*) buffer, size);
// close file
f.close();
// set offset begin of buffer
int offset = 0;
// convert 3 bytes to number of NodeVer3
numberOfNodes = buffer[offset] * 65536 + buffer[offset + 1] * 256 + buffer[offset + 2];
if (debug)
{
wcout << "numberOfNodes = " << numberOfNodes << endl;
}
offset += 3;
// read list of NodeVer3
DictionaryNodeList = vector<DictionaryNode*>();
for (int _nodeId = 0; _nodeId < numberOfNodes; ++_nodeId)
{
// convert first byte to wchar_t
wchar_t _character = getDictionary()->getDictionaryTools()->charToWchar(buffer[offset]);
// convert 3 remaining bytes to _parentId
int _parentId = buffer[offset + 1] * 65536 + buffer[offset + 2] * 256 + buffer[offset + 3];
offset += 4;
// if (_nodeId == 35627)
// {
// wcout << "NodeId = " << _nodeId << " Char = " << _character << " ParentId = " << _parentId << endl;
// }
// create new NodeVer3
DictionaryNode* _node = new DictionaryNode();
_node->setCharacter(_character);
if (_parentId < 0 || _parentId >= _nodeId)
{
//wcout << "### Error ### : loadFromBinaryFile -> id = " << _nodeId << " parentId = " << _parentId << endl;
_node->setParentNode(NULL); // root
}
else
{
_node->setParentNode(DictionaryNodeList.at(_parentId));
DictionaryNodeList.at(_parentId)->addChild(_node);
}
DictionaryNodeList.push_back(_node);
}
root = DictionaryNodeList.at(0);
root->setParentNode(NULL);
if (debug)
{
wcout << endl << "OK 1 : numberOfNodes = " << numberOfNodes << endl;
}
//================ Read NodeModels =========================================================================================
// read number of NodeModel
// convert 3 bytes to number of NodeModel
int numberOfNodeModel = buffer[offset] * 65536 + buffer[offset + 1] * 256 + buffer[offset + 2];
if (debug)
{
wcout << "numberOfNodeModel = " << numberOfNodeModel << endl;
}
offset += 3;
// read list of NodeModel
nodeModelMap = map<int, DictionaryNodeModel*>();
for (int i = 0; i < numberOfNodeModel; ++i)
{
// convert first 2 bytes to modelId
int _modelId = buffer[offset] * 256 + buffer[offset + 1];
// convert the next 3 bytes to _lemmaId
int _lemmaId = buffer[offset + 2] * 65536 + buffer[offset + 3] * 256 + buffer[offset + 4];
// convert 3 remaining bytes to _nodeId
int _nodeVer3Id = buffer[offset + 5] * 65536 + buffer[offset + 6] * 256 + buffer[offset + 7];
offset += 8;
// create new NodeModel
if (nodeModelMap.count(_lemmaId))
{
//wcout << "##### TrieVer5::loadFromBinaryFile -> Duplicate : lemmaId = " << _lemmaId << " modelId = " << _modelId << endl;
}
else
{
// create a NodeModel
DictionaryNodeModel* _nodeModel = new DictionaryNodeModel(_modelId, _lemmaId);
// add NodeModel to NodeVer3
DictionaryNodeList.at(_nodeVer3Id)->addNodeModel(_nodeModel);
// map lemmaId to NodeModel
nodeModelMap.insert(pair<int, DictionaryNodeModel*>(_lemmaId, _nodeModel));
// if (_modelId == 872)
// {
// wcout << "NodeId == " << _nodeVer3Id << " ModelId = " << _modelId << endl;
// }
}
}
if (debug)
{
wcout << "OK 2 : numberOfNodeModel = " << numberOfNodeModel << endl;
}
//================ Read links =========================================================================================
// read number of links
// convert 3 bytes to number of links
int numberOfLinks = buffer[offset] * 65536 + buffer[offset + 1] * 256 + buffer[offset + 2];
offset += 3;
if (debug)
{
wcout << "numberOfLinks = " << numberOfLinks << endl;
}
// read links
for (int i = 0; i < numberOfLinks; ++i)
{
// convert the first 3 bytes to _fromLemmaId
int _fromLemmaId = buffer[offset] * 65536 + buffer[offset + 1] * 256 + buffer[offset + 2];
// convert the 3 remaining bytes to _toLemmaId
int _toLemmaId = buffer[offset + 3] * 65536 + buffer[offset + 4] * 256 + buffer[offset + 5];
offset += 6;
addLinkByLemmaIds(_fromLemmaId, _toLemmaId);
}
if (debug)
{
wcout << "OK 3" << endl;
}
//================ Read TrieModels =========================================================================================
// read number of TrieModel
// convert 2 bytes to number of TrieModel
int numberOfModels = buffer[offset] * 256 + buffer[offset + 1];
offset += 2;
if (debug)
{
wcout << "Number of TrieModels = " << numberOfModels << endl;
}
// read TrieModels
for (int i = 1; i <= numberOfModels; ++i)
{
// create a TrieModel from string sufixesAttrs
DictionaryTrieModel* _trieModel = new DictionaryTrieModel();
// convert 1 bytes to numberOfTrieModelElement
int numberOfTrieModelElement = buffer[offset];
offset += 1;
// wcout << "TM#" << i << ": elements=" << numberOfTrieModelElement << endl;
for (int j = 0; j < numberOfTrieModelElement; ++j)
{
DictionaryTrieModelElement* _modelElement = new DictionaryTrieModelElement();
// convert 1 byte to suffix's length
int suffixLength = buffer[offset];
offset += 1;
wstring _suffix = L"";
// read suffix
for (int k = 0; k < suffixLength; ++k)
{
_suffix.push_back(getDictionary()->getDictionaryTools()->charToWchar(buffer[offset + k]));
}
offset += suffixLength;
// wcout << "_suffix = " << _suffix << " ";
// set suffix for DictionaryTrieModelElement
_modelElement->setSuffix(_suffix);
// convert 1 byte to beginWithPo
int _beginWithPo = buffer[offset];
offset += 1;
// wcout << "Po = " << _beginWithPo << " ";
// set beginWithPo for DictionaryTrieModelElement
_modelElement->setBeginWithPo(_beginWithPo == 1);
// convert 2 byte to featureListId
int _featureListId = buffer[offset] * 256 + buffer[offset + 1];
offset += 2;
// wcout << "_featureListId = " << _featureListId << endl;
// set featureListId for DictionaryTrieModelElement
_modelElement->setFeatureListId(_featureListId);
// add DictionaryTrieModelElement to DictionaryTrieModel
_trieModel->addDictionaryTrieModelElement(_modelElement);
}
// map modelIndex to trieModel
mapModelIndexToTrieModel(i, _trieModel);
}
if (debug)
{
wcout << "OK 4" << endl;
}
//================ Read featureListMap =========================================================================================
// read number of FeatureList
// convert 2 bytes to number of FeatureList
int numberOfFeatureList = buffer[offset] * 256 + buffer[offset + 1];
if (debug)
{
wcout << "numberOfFeatureList = " << numberOfFeatureList << endl;
}
offset += 2;
for (int i = 1; i <= numberOfFeatureList; ++i)
{
// read number of features in list
int numberOfFeature = buffer[offset];
// wcout << " numberOfFeature = " << numberOfFeature << ": ";
offset += 1;
vector<int> featureIdList = vector<int>();
// read features
for (int j = 0; j < numberOfFeature; ++j)
{
// convert 1 byte to featureId
int featureId = buffer[offset];
// wcout << featureId << "; ";
offset += 1;
// add featureId to featureIdList
featureIdList.push_back(featureId);
}
// wcout << endl;
// insert featureIdList to featureListMap
featureListMap.insert(pair<int, vector<int> >(i, featureIdList));
}
if (debug)
{
wcout << "OK 5" << endl;
}
//================ Read featureMap =========================================================================================
// read number of features
// convert 1 bytes to number of FeatureList
featureIdMap.clear();
idFeatureMap.clear();
int _numberOfFeature = buffer[offset];
if (debug)
{
wcout << "_numberOfFeature = " << _numberOfFeature << endl;
}
offset += 1;
for (int i = 1; i <= _numberOfFeature; ++i)
{
// short feature
// convert 1 byte to feature's length
int _short_feature_length = buffer[offset];
offset += 1;
// read feature
wstring _short_feature = L"";
for (int j = 0; j < _short_feature_length; ++j)
{
//wcout << buffer[offset + j] << "; ";
_short_feature.push_back(getDictionary()->getDictionaryTools()->charToWchar(buffer[offset + j]));
}
//wcout << endl;
offset += _short_feature_length;
// insert _feature to featureMap
featureIdMap.insert(pair<wstring, int>(_short_feature, i));
idFeatureMap.insert(pair<int, wstring>(i, _short_feature));
if (debug)
{
wcout << "Short feature (BIN) #" << i << ": (" << _short_feature_length << ") " << _short_feature << endl;
}
// long feature
// convert 1 byte to feature's length
int _long_feature_length = buffer[offset];
offset += 1;
// read feature
wstring _long_feature = L"";
for (int j = 0; j < _long_feature_length; ++j)
{
//wcout << buffer[offset + j] << "; ";
_long_feature.push_back(getDictionary()->getDictionaryTools()->charToWchar(buffer[offset + j]));
}
//wcout << endl;
offset += _long_feature_length;
// insert _feature to featureMap
featureMap.insert(pair<int, wstring>(i, _long_feature));
if (debug)
{
wcout << "Long feature (BIN) #" << i << ": (" << _long_feature_length << ") " << _long_feature << endl;
}
}
if (debug)
{
wcout << "OK 6" << endl;
}
//================ Loading done =========================================================================================
delete[] buffer;
buffer = NULL;
if (debug)
{
wcout << "... loadFromBinaryFile done!" << endl;
}
}
else
{
throw FILE_NOT_FOUND_ERROR_CODE;
wcout << "### Error ### : loadFromBinaryFile -> Unable to open file" << endl;
}
}
