// langId is language of the query
long long getSynBaseHash64 ( char *qstr , uint8_t langId ) {
Words ww;
ww.set3 ( qstr );
long nw = ww.getNumWords();
long long *wids = ww.getWordIds();
//char **wptrs = ww.getWords();
//long *wlens = ww.getWordLens();
long long baseHash64 = 0LL;
Synonyms syn;
// assume english if unknown to fix 'pandora's tower'
// vs 'pandoras tower' where both words are in both
// english and german so langid is unknown
if ( langId == langUnknown ) langId = langEnglish;
// . store re-written query into here then hash that string
// . this way we can get rid of spaces
//char rebuf[1024];
//char *p = rebuf;
//if ( strstr(qstr,"cheatcodes") )
// log("hey");
// for deduping
HashTableX dups;
if ( ! dups.set ( 8,0,1024,NULL,0,false,0,"qhddup") ) return false;
// scan the words
for ( long i = 0 ; i < nw ; i++ ) {
// skip if not alnum
if ( ! wids[i] ) continue;
// get its synonyms into tmpBuf
char tmpBuf[TMPSYNBUFSIZE];
// . assume niceness of 0 for now
// . make sure to get all synsets!! ('love' has two synsets)
long naids = syn.getSynonyms (&ww,i,langId,tmpBuf,0);
// term freq algo
//long pop = g_speller.getPhrasePopularity(NULL,
// wids[i],
// true,
// langId);
// is it a queryStopWord like "the" or "and"?
bool isQueryStop = ::isQueryStopWord(NULL,0,wids[i]);
// a more restrictive list
bool isStop = ::isStopWord(NULL,0,wids[i]);
if ( ::isCommonQueryWordInEnglish(wids[i]) ) isStop = true;
// find the smallest one
unsigned long long min = wids[i];
//char *minWordPtr = wptrs[i];
//long minWordLen = wlens[i];
// declare up here since we have a goto below
long j;
// add to table too
if ( dups.isInTable ( &min ) ) goto gotdup;
// add to it
if ( ! dups.addKey ( &min ) ) return false;
// now scan the synonyms, they do not include "min" in them
for ( j = 0 ; j < naids ; j++ ) {
// get it
unsigned long long aid64;
aid64 = (unsigned long long)syn.m_aids[j];
// if any syn already hashed then skip it and count
// as a repeated term. we have to do it this way
// rather than just getting the minimum synonym
// word id, because 'love' has two synsets and
// 'like', a synonym of 'love' only has one synset
// and they end up having different minimum synonym
// word ids!!!
if ( dups.isInTable ( &aid64 ) ) break;
// add it. this could fail!
if ( ! dups.addKey ( &aid64 ) ) return false;
// set it?
if ( aid64 >= min ) continue;
// got a new min
min = aid64;
//minWordPtr = syn.m_termPtrs[j];
//minWordLen = syn.m_termLens[j];
// get largest term freq of all synonyms
//long pop2 = g_speller.getPhrasePopularity(NULL,aid64,
// true,langId);
//if ( pop2 > pop ) pop = pop2;
}
// early break out means a hit in dups table
if ( j < naids ) {
gotdup:
// do not count as repeat if query stop word
// because they often repeat
if ( isQueryStop ) continue;
// count # of repeated word forms
//nrwf++;
continue;
}
// hash that now
// do not include stop words in synbasehash so
// 'search the web' != 'search web'
if ( ! isStop ) {
// no! make it order independent so 'search the web'
// equals 'web the search' and 'engine search'
// equals 'search engine'
//baseHash64 <<= 1LL;
baseHash64 ^= min;
}
// count it, but only if not a query stop word like "and"
// or "the" or "a". # of unique word forms.
//if ( ! isQueryStop ) nuwf++;
// get term freq
//if ( pop > maxPop ) maxPop = pop;
// control word?
//if ( wids[i] == cw1 ) ncwf++;
}
return baseHash64;
}
// . return the score of the highest-scoring window containing match #m
// . window is defined by the half-open interval [a,b) where a and b are
// word #'s in the Words array indicated by match #m
// . return -1 and set g_errno on error
int64_t Summary::getBestWindow ( Matches *matches, int32_t mm, int32_t *lasta,
int32_t *besta, int32_t *bestb, char *gotIt,
char *retired, int32_t maxExcerptLen ) {
// get the window around match #mm
Match *m = &matches->m_matches[mm];
// what is the word # of match #mm?
int32_t matchWordNum = m->m_wordNum;
// what Words/Pos/Bits classes is this match in?
Words *words = m->m_words;
Section **sp = NULL;
int32_t *pos = m->m_pos->m_pos;
// use "m_swbits" not "m_bits", that is what Bits::setForSummary() uses
const swbit_t *bb = m->m_bits->m_swbits;
// shortcut
if ( m->m_sections ) {
sp = m->m_sections->m_sectionPtrs;
}
int32_t nw = words->getNumWords();
int64_t *wids = words->getWordIds();
nodeid_t *tids = words->getTagIds();
// . sanity check
// . this prevents a core i've seen
if ( matchWordNum >= nw ) {
log("summary: got overflow condition for q=%s",m_q->m_orig);
// assume no best window
*besta = -1;
*bestb = -1;
*lasta = matchWordNum;
return 0;
}
// . we NULLify the section ptrs if we already used the word in another summary.
int32_t badFlags = SEC_SCRIPT|SEC_STYLE|SEC_SELECT|SEC_IN_TITLE;
if ( (bb[matchWordNum] & D_USED) || ( sp && (sp[matchWordNum]->m_flags & badFlags) ) ) {
// assume no best window
*besta = -1;
*bestb = -1;
*lasta = matchWordNum;
return 0;
}
// . "a" is the left fence post of the window (it is a word # in Words)
// . go to the left as far as we can
// . thus we decrement "a"
int32_t a = matchWordNum;
// "posa" is the character position of the END of word #a
int32_t posa = pos[a+1];
int32_t firstFrag = -1;
bool startOnQuote = false;
bool goodStart = false;
int32_t wordCount = 0;
// . decrease "a" as int32_t as we stay within maxNumCharsPerLine
// . avoid duplicating windows by using "lasta", the last "a" of the
// previous call to getBestWindow(). This can happen if our last
// central query term was close to this one.
for ( ; a > 0 && posa - pos[a-1] < maxExcerptLen && a > *lasta; a-- ) {
// . don't include any "dead zone",
// . dead zones have already been used for the summary, and
// we are getting a second/third/... excerpt here now then
// stop if its the start of a sentence, too
// stop before title word
if ( (bb[a-1] & D_USED) || (bb[a] & D_STARTS_SENTENCE) || ( bb[a-1] & D_IN_TITLE )) {
goodStart = true;
break;
}
// don't go beyond an LI, TR, P tag
if ( tids && ( tids[a-1] == TAG_LI ||
tids[a-1] == TAG_TR ||
tids[a-1] == TAG_P ||
tids[a-1] == TAG_DIV ) ) {
goodStart = true;
break;
}
// stop if its the start of a quoted sentence
if ( a+1<nw && (bb[a+1] & D_IN_QUOTES) &&
words->getWord(a)[0] == '\"' ){
startOnQuote = true;
goodStart = true;
break;
}
// find out the first instance of a fragment (comma, etc)
// watch out! because frag also means 's' in there's
if ( ( bb[a] & D_STARTS_FRAG ) && !(bb[a-1] & D_IS_STRONG_CONNECTOR) && firstFrag == -1 ) {
firstFrag = a;
}
if ( wids[a] ) {
wordCount++;
}
}
// if didn't find a good start, then start at the start of the frag
if ( !goodStart && firstFrag != -1 ) {
a = firstFrag;
}
// don't let punct or tag word start a line, unless a quote
if ( a < matchWordNum && !wids[a] && words->getWord(a)[0] != '\"' ){
while ( a < matchWordNum && !wids[a] ) a++;
// do not break right after a "strong connector", like
// apostrophe
while ( a < matchWordNum && a > 0 &&
( bb[a-1] & D_IS_STRONG_CONNECTOR ) )
a++;
// don't let punct or tag word start a line
while ( a < matchWordNum && !wids[a] ) a++;
}
// remember, b is not included in the summary, the summary is [a,b-1]
// remember to include all words in a matched phrase
int32_t b = matchWordNum + m->m_numWords ;
int32_t endQuoteWordNum = -1;
int32_t numTagsCrossed = 0;
for ( ; b <= nw; b++ ) {
if ( b == nw ) {
break;
}
if ( pos[b+1] - pos[a] >= maxExcerptLen ) {
break;
}
if ( startOnQuote && words->getWord(b)[0] == '\"' ) {
endQuoteWordNum = b;
}
// don't include any dead zone, those are already-used samples
if ( bb[b] & D_USED ) {
break;
}
// stop on a title word
if ( bb[b] & D_IN_TITLE ) {
break;
}
if ( wids[b] ) {
wordCount++;
}
// don't go beyond an LI or TR backtag
if ( tids && ( tids[b] == (BACKBIT|TAG_LI) ||
tids[b] == (BACKBIT|TAG_TR) ) ) {
numTagsCrossed++;
// try to have atleast 10 words in the summary
if ( wordCount > 10 ) {
break;
}
}
// go beyond a P or DIV backtag in case the earlier char is a
// ':'. This came from a special case for wikipedia pages
// eg. http://en.wikipedia.org/wiki/Flyover
if ( tids && ( tids[b] == (BACKBIT|TAG_P) ||
tids[b] == (BACKBIT|TAG_DIV) )) {
numTagsCrossed++;
// try to have atleast 10 words in the summary
if ( wordCount > 10 && words->getWord(b-1)[0] != ':' ) {
break;
}
}
}
// don't end on a lot of punct words
if ( b > matchWordNum && !wids[b-1]){
// remove more than one punct words. if we're ending on a quote
// keep it
while ( b > matchWordNum && !wids[b-2] && endQuoteWordNum != -1 && b > endQuoteWordNum ) {
b--;
}
// do not break right after a "strong connector", like apostrophe
while ( b > matchWordNum && (bb[b-2] & D_IS_STRONG_CONNECTOR) ) {
b--;
}
}
Match *ms = matches->m_matches;
// make m_matches.m_matches[mi] the first match in our [a,b) window
int32_t mi ;
// . the match at the center of the window is match #"mm", so that
// matches->m_matches[mm] is the Match class
// . set "mi" to it and back up "mi" as int32_t as >= a
for ( mi = mm ; mi > 0 && ms[mi-1].m_wordNum >=a ; mi-- )
;
// now get the score of this excerpt. Also mark all the represented
// query words. Mark the represented query words in the array that
// comes to us. also mark how many times the same word is repeated in
// this summary.
int64_t score = 0LL;
// is a url contained in the summary, that looks bad! punish!
bool hasUrl = false;
// the word count we did above was just an approximate. count it right
wordCount = 0;
// for debug
//char buf[5000];
//char *xp = buf;
SafeBuf xp;
// wtf?
if ( b > nw ) {
b = nw;
}
// first score from the starting match down to a, including match
for ( int32_t i = a ; i < b ; i++ ) {
// debug print out
if ( g_conf.m_logDebugSummary ) {
int32_t len = words->getWordLen(i);
char cs;
for (int32_t k=0;k<len; k+=cs ) {
const char *c = words->getWord(i)+k;
cs = getUtf8CharSize(c);
if ( is_binary_utf8 ( c ) ) {
continue;
}
xp.safeMemcpy ( c , cs );
xp.nullTerm();
}
}
// skip if in bad section, marquee, select, script, style
if ( sp && (sp[i]->m_flags & badFlags) ) {
continue;
}
// don't count just numeric words
if ( words->isNum(i) ) {
continue;
}
// check if there is a url. best way to check for '://'
if ( wids && !wids[i] ) {
const char *wrd = words->getWord(i);
int32_t wrdLen = words->getWordLen(i);
if ( wrdLen == 3 && wrd[0] == ':' && wrd[1] == '/' && wrd[2] == '/' ) {
hasUrl = true;
}
}
// skip if not wid
if ( ! wids[i] ) {
continue;
}
// just make every word 100 pts
int32_t t = 100;
// penalize it if in one of these sections
if ( bb[i] & ( D_IN_PARENS | D_IN_SUP | D_IN_LIST ) ) {
t /= 2;
}
// boost it if in bold or italics
if ( bb[i] & D_IN_BOLDORITALICS ) {
t *= 2;
}
// add the score for this word
score += t;
// print the score, "t"
if ( g_conf.m_logDebugSummary ) {
xp.safePrintf("(%" PRId32")",t);
}
// count the alpha words we got
wordCount++;
// if no matches left, skip
if ( mi >= matches->m_numMatches ) {
continue;
}
// get the match
Match *next = &ms[mi];
// skip if not a match
if ( i != next->m_wordNum ) {
continue;
}
// must be a match in this class
if ( next->m_words != words ) {
continue;
}
// advance it
mi++;
// which query word # does it match
int32_t qwn = next->m_qwordNum;
if ( qwn < 0 || qwn >= m_q->m_numWords ){g_process.shutdownAbort(true);}
// undo old score
score -= t;
// add 100000 per match
t = 100000;
// weight based on tf, goes from 0.1 to 1.0
t = (int32_t)((float)t * m_wordWeights [ qwn ]);
// if it is a query stop word, make it 10000 pts
if ( m_q->m_qwords[qwn].m_isQueryStopWord ) {
t = 0;//10000;
}
// penalize it if in one of these sections
if ( bb[i] & ( D_IN_PARENS | D_IN_SUP | D_IN_LIST ) ) {
t /= 2;
}
if ( gotIt[qwn] > 0 ) {
// have we matched it in this [a,b) already?
if ( gotIt[qwn] == 1 ) {
t /= 15;
} else {
// if we have more than 2 matches in the same window,
// it may not give a good summary. give a heavy penalty
t -= 200000;
}
} else if ( retired [qwn] > 0 ) {
// have we matched it already in a winning window?
t /= 12;
}
// add it back
score += t;
if ( g_conf.m_logDebugSummary ) {
xp.safePrintf ("[%" PRId32"]{qwn=%" PRId32",ww=%f}",t,qwn,
m_wordWeights[qwn]);
}
// inc the query word count for this window
if ( gotIt[qwn] < 100 ) {
gotIt[qwn]++;
}
}
int32_t oldScore = score;
// apply the bonus if it starts or a sentence
// only apply if the score is positive and if the wordcount is decent
if ( score > 0 && wordCount > 7 ){
// a match can give us 10k to 100k pts based on the tf weights
// so we don't want to overwhelm that too much, so let's make
// this a 20k bonus if it starts a sentence
if ( bb[a] & D_STARTS_SENTENCE ) {
score += 8000;
} else if ( bb[a] & D_STARTS_FRAG ) {
// likewise, a fragment, like after a comma
score += 4000;
}
// 1k if the match word is very close to the
// start of a sentence, lets say 3 alphawords
if ( matchWordNum - a < 7 ) {
score += 1000;
}
}
// a summary isn't really a summary if its less than 7 words.
// reduce the score, but still give it a decent score.
// minus 5M.
if ( wordCount < 7 ) {
score -= 20000;
}
// summaries that cross a lot of tags are usually bad, penalize them
if ( numTagsCrossed > 1 ) {
score -= (numTagsCrossed * 20000);
}
if ( hasUrl ) {
score -= 8000;
}
// show it
if ( g_conf.m_logDebugSummary ) {
log(LOG_DEBUG, "sum: score=%08" PRId32" prescore=%08" PRId32" a=%05" PRId32" b=%05" PRId32" %s",
(int32_t)score,oldScore,(int32_t)a,(int32_t)b,
xp.getBufStart());
}
// set lasta, besta, bestb
*lasta = a;
*besta = a;
*bestb = b;
return score;
}