// **************************************************************************
// Function: LoadDictionary
// Purpose: This function loads a dictionary file
// Parameters: dictionaryfile - filename of the dictionary file
// eraseold - true ... deletes all existing words in the dictionary
// false ... does not delete existing words in the dictionary
// Returns: 0 ... error
// 1 ... OK
// **************************************************************************
int DICTIONARY::LoadDictionary(const char *dictionaryfile, bool eraseold)
{
char line[MAX_WORDLENGTH];
FILE *fp;
int i;
if (eraseold) DeleteWords();
fp=fopen(dictionaryfile, "rb");
if (!fp) return(0);
while (!feof(fp))
{
fgets(line, MAX_WORDLENGTH, fp);
line[MAX_WORDLENGTH-1]=0;
for (i=0; i<(int)strlen(line); i++)
if ((line[i] == ' ') || (line[i] == '\r') || (line[i] == '\n'))
line[i]=0;
AddWord(line);
}
fclose(fp);
return(1);
}
void Phrase::Append(const Phrase &endPhrase)
{
for (size_t i = 0; i < endPhrase.GetSize(); i++) {
AddWord(endPhrase.GetWord(i));
}
}
/* Create a new, blank dictionary
*/
void Niall_NewDictionary(void)
{
ClearDictionary();
WordList=AddWord("");
if(!WordList) Niall_Error("Out of memory.");
}
/*-----------------------------------------------------------------------------
* add a string.string = wlen + buffer
*-----------------------------------------------------------------------------*/
int AddString(char **p, int *piLenLeft, char *pBuf, uint16_t wBufLen)
{
int iLeftLen = 5000;
char *pCur = NULL;
if(AddWord(p, piLenLeft,wBufLen) != 0)
return -1;
pCur = *p;
if(piLenLeft)
iLeftLen = *piLenLeft;
if(iLeftLen < wBufLen)
return -2;
memcpy(pCur, pBuf, wBufLen);
pCur += wBufLen;
iLeftLen -= wBufLen;
*p = pCur;
if(piLenLeft)
*piLenLeft = iLeftLen;
return 0;
}
Phrase::Phrase(const vector< const Word* > &mergeWords)
{
m_words.reserve(mergeWords.size());
for (size_t currPos = 0 ; currPos < mergeWords.size() ; currPos++) {
AddWord(*mergeWords[currPos]);
}
}
void WordFinder::AddWords( std::vector<std::string> words )
{
for (int i = 0; i < words.size(); i++)
{
AddWord(words[i], i+1);
}
}
Phrase::Phrase(FactorDirection direction, const vector< const Word* > &mergeWords)
:m_direction(direction)
{
m_words.reserve(mergeWords.size());
for (size_t currPos = 0 ; currPos < mergeWords.size() ; currPos++) {
AddWord(*mergeWords[currPos]);
}
}
/* Load a dictionary
*/
void Niall_LoadDictionary(char *FileName)
{
char Buffer[BUFSIZ];
FILE *fHandle;
int nWords,i,d;
int nAsocs,j,w,p,k;
WORD *Word;
fHandle=fopen(FileName,"r");
if(!fHandle)
{
Niall_Warning("File %s not found.",FileName);
return;
}
fscanf(fHandle,"%s %d\n",Buffer,&nWords);
if((strcmp(Buffer,FILE_ID))||(nWords<2))
{
Niall_Warning("File %s is not a valid Niall file.",FileName);
return;
}
ClearDictionary();
for(i=0;i<nWords;i++)
{
fscanf(fHandle,"%4d: %s %d|",&d,Buffer,&nAsocs);
if((d!=i)||(nAsocs<1))
{
Niall_Warning("Word %d is corrupted.",i);
Niall_NewDictionary();
return;
}
if(Buffer[0]=='>')
{
if(i!=0)
{
Niall_Warning("Word %d is corrupted.",i);
Niall_NewDictionary();
return;
}
Buffer[0]=0;
}
Word=AddWord(Buffer);
if(Word==NULL) Niall_Error("Out of memory.");
for(j=0;j<nAsocs;)
{
fscanf(fHandle," %d(%d)",&w,&p);
if(w>=nWords)
{
Niall_Warning("Word %d/Assoc %d is corrupted.",i,j);
Niall_NewDictionary();
return;
}
for(k=0;k<p;k++) Associate(Word,w);
j+=p;
}
}
}
void TSwSet::LoadFromFile(const TStr& FNm) {
TStr FileStr = TStr::LoadTxt(FNm);
FileStr.DelChAll('\r');
TStrV WordV; FileStr.SplitOnAllCh('\n', WordV);
for (int WordN = 0; WordN < WordV.Len(); WordN++) {
const TStr& WordStr = WordV[WordN];
if (!IsIn(WordStr)) { AddWord(WordStr); }
}
}
/**
*push消息的组包
*/
int ComposePkg(tASecMsgHead &stHead, tASecPushMsgReq &stReq, uint8_t *pPkg, int& iPkgBufSize)
{
//数据预处理.做个保护措施
char szPkgBuf[2048];
char *pCur = szPkgBuf;
bzero(szPkgBuf, sizeof(szPkgBuf));
*pCur = 0x02;
pCur++;
tASecMsgHead* pHead = (tASecMsgHead*)pCur;
pCur+=sizeof(tASecMsgHead);
memcpy(pHead, &stHead, sizeof(stHead));
if(stReq.usENLen > sizeof(stReq.aucEN)
|| stReq.usReservedLen > sizeof(stReq.aucReserved))
{
return -1;
}
AddWord(&pCur, NULL, stReq.usENLen);
AddFixData(&pCur, NULL, (char*)stReq.aucEN, stReq.usENLen);
AddWord(&pCur, NULL, stReq.usReservedLen);
AddFixData(&pCur, NULL, (char*)stReq.aucReserved, stReq.usReservedLen);
*pCur = 0x03;
pCur++;
int iPkgLen = pCur-szPkgBuf;
if (iPkgLen > iPkgBufSize)
{
return -2;
}
pHead->usLength = iPkgLen;
Hton(pHead);
memcpy(pPkg, szPkgBuf, iPkgLen);
iPkgBufSize=iPkgLen;
return 0;
}
void wxHtmlWinParser::AddPreBlock(const wxString& text)
{
if ( text.find('\t') != wxString::npos )
{
wxString text2;
text2.reserve(text.length());
const wxString::const_iterator end = text.end();
wxString::const_iterator copyFrom = text.begin();
size_t pos = 0;
int posColumn = m_posColumn;
for ( wxString::const_iterator i = copyFrom; i != end; ++i, ++pos )
{
if ( *i == '\t' )
{
if ( copyFrom != i )
text2.append(copyFrom, i);
const unsigned SPACES_PER_TAB = 8;
const size_t expandTo = SPACES_PER_TAB - posColumn % SPACES_PER_TAB;
text2.append(expandTo, ' ');
posColumn += expandTo;
copyFrom = i + 1;
}
else
{
++posColumn;
}
}
if ( copyFrom != text.end() )
text2.append(copyFrom, text.end());
AddWord(new wxHtmlWordWithTabsCell(text2, text, m_posColumn, *(GetDC())));
m_posColumn = posColumn;
}
else
{
// no special formatting needed
AddWord(text);
m_posColumn += text.length();
}
}
NS_IMETHODIMP
mozSpellChecker::AddWordToPersonalDictionary(const nsAString &aWord)
{
nsresult res;
PRUnichar empty=0;
if (!mPersonalDictionary)
return NS_ERROR_NULL_POINTER;
res = mPersonalDictionary->AddWord(PromiseFlatString(aWord).get(),&empty);
return res;
}
VOID AinMessage::AddStr(LPCSTR pStr)
{
if(NULL == pStr)
{
assert(false);
return;
}
WORD wStrLen = (WORD)strlen(pStr);
AddWord(wStrLen);
AddEx(pStr, wStrLen);
}
void Phrase::PrependWord(const Word &newWord)
{
AddWord();
// shift
for (size_t pos = GetSize() - 1; pos >= 1; --pos) {
const Word &word = m_words[pos - 1];
m_words[pos] = word;
}
m_words[0] = newWord;
}
void CMyFirewallDlg::OnBnClickedBwordAdd()
{
UpdateData();
WordFilter wf;
wf.id = m_idWord++;
char ascii[10];
wcstombs( ascii, m_word, 10 );
strcpy(wf.word, ascii);
if( AddWord(wf) ) //send the rule
m_listWords.InsertItem(LVIF_TEXT|LVIF_PARAM, m_listWords.GetItemCount(), m_word, 0, 0, 0, wf.id);
m_word = "";
UpdateData(false);
}
void BadWords::Permute(const std::string& suffix,
const std::vector<std::string>& parts,
const std::vector<const std::vector<std::string> *>& values,
int index, bool standalone, bool wildcards) {
for (int i = 0; i < values[index]->size(); i++) {
std::string word = parts[index] + (*values[index])[i] + suffix;
if (index > 0) {
Permute(word, parts, values, index - 1, standalone, wildcards);
} else {
AddWord(word, standalone, wildcards);
}
}
}
void AddText (TraverseCtx &Ctx, const CString &sText)
{
enum States
{
stateInText,
stateInDelimeter,
stateDone,
};
// Parse the text into words
char *pPos = sText.GetASCIIZPointer();
char *pStart = pPos;
int iState = stateInText;
CString sWord;
while (iState != stateDone)
{
bool bText = strIsAlphaNumeric(pPos);
switch (iState)
{
case stateInText:
{
if (!bText)
{
AddWord(Ctx, CString(pStart, pPos - pStart));
iState = stateInDelimeter;
}
break;
}
case stateInDelimeter:
{
if (bText)
{
pStart = pPos;
iState = stateInText;
}
break;
}
}
// Next
if (*pPos == '\0')
iState = stateDone;
else
pPos++;
}
}
void Phrase::CreateFromStringNewFormat(FactorDirection direction
, const std::vector<FactorType> &factorOrder
, const std::string &phraseString
, const std::string &factorDelimiter
, Word &lhs)
{
m_arity = 0;
// parse
vector<string> annotatedWordVector;
Tokenize(annotatedWordVector, phraseString);
// KOMMA|none ART|Def.Z NN|Neut.NotGen.Sg VVFIN|none
// to
// "KOMMA|none" "ART|Def.Z" "NN|Neut.NotGen.Sg" "VVFIN|none"
for (size_t phrasePos = 0 ; phrasePos < annotatedWordVector.size() - 1 ; phrasePos++)
{
string &annotatedWord = annotatedWordVector[phrasePos];
bool isNonTerminal;
if (annotatedWord.substr(0, 1) == "[" && annotatedWord.substr(annotatedWord.size()-1, 1) == "]")
{ // non-term
isNonTerminal = true;
size_t nextPos = annotatedWord.find("[", 1);
assert(nextPos != string::npos);
if (direction == Input)
annotatedWord = annotatedWord.substr(1, nextPos - 2);
else
annotatedWord = annotatedWord.substr(nextPos + 1, annotatedWord.size() - nextPos - 2);
m_arity++;
}
else
{
isNonTerminal = false;
}
Word &word = AddWord();
word.CreateFromString(direction, factorOrder, annotatedWord, isNonTerminal);
}
// lhs
string &annotatedWord = annotatedWordVector.back();
assert(annotatedWord.substr(0, 1) == "[" && annotatedWord.substr(annotatedWord.size()-1, 1) == "]");
annotatedWord = annotatedWord.substr(1, annotatedWord.size() - 2);
lhs.CreateFromString(direction, factorOrder, annotatedWord, true);
assert(lhs.IsNonTerminal());
}
void Phrase::InitStartEndWord()
{
FactorCollection &factorCollection = FactorCollection::Instance();
Word startWord(Input);
const Factor *factor = factorCollection.AddFactor(Input, 0, BOS_); // TODO - non-factored
startWord.SetFactor(0, factor);
PrependWord(startWord);
Word endWord(Input);
factor = factorCollection.AddFactor(Input, 0, EOS_); // TODO - non-factored
endWord.SetFactor(0, factor);
AddWord(endWord);
}
void BacaDict(Tree *T)
{
char s[20];
/* ALGORITMA */
STARTKATA("Dictionary.txt");
CreateTree(T);
SetRoot(T,'\0', false);
while(!EndKata)
{
KataToString(s);
AddWord(T, s);
ADVKATA();
}
}
string alienOrder(vector<string>& words) {
int n=words.size();
if (!n) return "";
next.clear();
degree.clear();
AddWord(words[0]);
for (int i=1; i<n; ++i) {
AddWord(words[i]);
Diff(words[i-1], words[i]);
}
queue<char> q;
for (const auto& kv : degree) {
if (kv.second==0) {
q.push(kv.first);
}
}
string res;
while (!q.empty()) {
res += q.front();
for (char c : next[q.front()]) {
if ((--degree[c]) == 0) {
q.push(c);
}
}
q.pop();
}
for (const auto& kv : degree) {
if (kv.second > 0) return "";
}
return res;
}
bool IfExsist(Info *L,char *s)//搜索是否存在
{
node *search;
for(search=L->head->next;search!=NULL;search=search->next)
{
if(strcmp(search->word,s)==0)//如果相等！
{
search->num++;
L->num++;
return true;
}
}
AddWord(L,s);
return false;
}
int CNetData::AddString(const char *pszString, int iSize)
{
int iLen = Strnlen(pszString, iSize-1);
if(m_iPos + (int)sizeof(unsigned short) + iLen > m_iSize)
return -1;
if(-1 == AddWord(iLen))
return -1;
memcpy(m_pBuf + m_iPos, pszString, iLen);
m_iPos += iLen;
return m_iPos;
}
CPVT_WordPlace CPDF_VariableText::InsertWord(const CPVT_WordPlace& place,
uint16_t word,
int32_t charset,
const CPVT_WordProps* pWordProps) {
int32_t nTotlaWords = GetTotalWords();
if (m_nLimitChar > 0 && nTotlaWords >= m_nLimitChar)
return place;
if (m_nCharArray > 0 && nTotlaWords >= m_nCharArray)
return place;
CPVT_WordPlace newplace = place;
newplace.nWordIndex++;
if (m_bRichText) {
CPVT_WordProps* pNewProps =
pWordProps ? new CPVT_WordProps(*pWordProps) : new CPVT_WordProps();
pNewProps->nFontIndex =
GetWordFontIndex(word, charset, pWordProps->nFontIndex);
return AddWord(newplace, CPVT_WordInfo(word, charset, -1, pNewProps));
}
int32_t nFontIndex =
GetSubWord() > 0 ? GetDefaultFontIndex()
: GetWordFontIndex(word, charset, GetDefaultFontIndex());
return AddWord(newplace, CPVT_WordInfo(word, charset, nFontIndex, nullptr));
}
INT32 CNetData::AddString(const CHAR *pszString, INT32 iSize)
{
static const INT32 BUFLEN = 4096;
size_t iLen = 0;
//去掉libiconv库的依赖，避免iphone可能的移植问题
/*
if ((m_iCharsetNet >= 0) || (m_iCharsetHost >= 0))
{
if (m_iCharsetNet != m_iCharsetHost)
{
static char szBuffer[BUFLEN];
INT32 iResult = 0;
iLen = BUFLEN;
iResult = SDIconvCovert(g_szCharset[m_iCharsetNet],
g_szCharset[m_iCharsetHost], const_cast<char*>(pszString), iSize, szBuffer, iLen);
if (0 != iResult)
{
return -1;
}
if((INT32)(m_iPos + (INT32)sizeof(UINT16) + iLen) > m_iSize)
{
return -1;
}
if(-1 == AddWord((UINT16)iLen))
{
return -1;
}
memcpy(m_pBuf + m_iPos, szBuffer, iLen);
m_iPos += iLen;
return m_iPos;
}
}
*/
iLen = Strnlen(pszString, iSize-1);
if((INT32)(m_iPos + (INT32)sizeof(UINT16) + iLen) > m_iSize)
return -1;
if(-1 == AddWord((UINT16)iLen))
return -1;
memcpy(m_pBuf + m_iPos, pszString, iLen);
m_iPos += iLen;
return m_iPos;
}
void CorpusReader::FillDocument(const proto_doc& source, const IntIntMap& vocab,
const set<int>& pos, const IntIntMap& synsets,
BaseDoc* doc) {
doc->time_ = Time(source.date().year(), source.date().month(),
source.date().day());
int sentences_seen = 0;
for (int ss = 0; ss < source.sentences_size(); ++ss) {
const lib_corpora_proto::Document_Sentence sent = source.sentences(ss);
int words_added = 0;
for (int ww = 0; ww < sent.words_size(); ++ww) {
const lib_corpora_proto::Document_Sentence_Word word = sent.words(ww);
if (AddWord(word, sentences_seen, vocab, pos, synsets, doc))
++words_added;
}
if (words_added > 0) ++sentences_seen;
}
}
UINT64 TargetPhrase::ReadFromFile(std::fstream &fileTP)
{
UINT64 bytesRead = 0;
fileTP.seekg(m_filePos);
UINT64 numWords;
fileTP.read((char*) &numWords, sizeof(UINT64));
bytesRead += sizeof(UINT64);
for (size_t ind = 0; ind < numWords; ++ind) {
Word *word = new Word();
bytesRead += word->ReadFromFile(fileTP);
AddWord(word);
}
return bytesRead;
}
void Phrase::CreateFromString(const std::vector<FactorType> &factorOrder
, const vector< vector<string> > &phraseVector)
{
FactorCollection &factorCollection = FactorCollection::Instance();
for (size_t phrasePos = 0 ; phrasePos < phraseVector.size() ; phrasePos++)
{
// add word this phrase
Word &word = AddWord();
for (size_t currFactorIndex= 0 ; currFactorIndex < factorOrder.size() ; currFactorIndex++)
{
FactorType factorType = factorOrder[currFactorIndex];
const string &factorStr = phraseVector[phrasePos][currFactorIndex];
const Factor *factor = factorCollection.AddFactor(m_direction, factorType, factorStr);
word[factorType] = factor;
}
}
}
void wxHtmlWinParser::FlushWordBuf(wxChar *buf, int& len)
{
buf[len] = 0;
for ( int i = 0; i < len; i++ )
{
if ( buf[i] == CUR_NBSP_VALUE )
buf[i] = ' ';
}
#if !wxUSE_UNICODE
if (m_EncConv)
m_EncConv->Convert(buf);
#endif
AddWord(wxString(buf, len));
len = 0;
}
WORDS *AddFile(WORDS *wbase, char *fnam)
{
int i, len;
len = strlen(fnam);
assert(len > 2);
if (fnam[len-1] = 'c' && fnam[len-2] == '.');
{
for (i=len-3; i > 0; i--)
{
if (fnam[i] == ' ' || fnam[i] == '/' || fnam[i] == '\\')
{
i++;
break;
}
}
wbase = AddWord(wbase, fnam+i, len-i-2);
}
return(wbase);
}
