bool sendTurkPageReply ( State60 *st ) {
XmlDoc *xd = &st->m_xd;
//char *content = xd->ptr_utf8Content;
//int32_t contentLen = xd->size_utf8Content - 1;
// count the total number of EventDesc classes for all evids
//char *evd = xd->ptr_eventData;
//EventDisplay *ed = (EventDisplay *)evd;
//char *addr = evd + (int32_t)ed->m_addr;
//char timeZoneOffset = getTimeZoneFromAddr ( addr );
// in case getSections() block come right back in
xd->setCallback ( st , xdcallback );
// . set niceness to 1 so all this processing doesn't slow queries down
// . however, g_niceness should still be zero... hmmm...
xd->m_niceness = 1;
// default to 1 niceness
st->m_niceness = 1;
// now set the sections class
Sections *ss = xd->getSections();
// now for each section with alnum text, telescope up as far as
// possible without containing anymore alnum text than what it
// contained. set SEC_CONTROL bit. such sections will have the
// 2 green/blue dots, that are used for turning on/off title/desc.
// but really the indians will only turn off sections that should
// not have a title/desc.
for ( Section *si = ss->m_rootSection ; si ; si = si->m_next ) {
// breathe
QUICKPOLL(st->m_niceness);
// skip if does not have text
if ( si->m_firstWordPos < 0 ) continue;
// otherwise, find biggest parent that contains just that text
Section *p = si->m_parent;
Section *last = si;
for ( ; p ; p = p->m_parent ) {
if ( p->m_firstWordPos != si->m_firstWordPos ) break;
if ( p->m_lastWordPos != si->m_lastWordPos ) break;
last = p;
}
// set that bit then
last->m_flags |= SEC_CONTROL;
// and speed up the loop
si = last;
}
// * now each SEC_CONTROL sections have a fence activated by a turker
// * an event title or description can not span a fence. it must be
// confined within a fence. however, it is allowed to include
// title or description from a "title section".
// * hold shift down to designate as title section when clicking it
// * show the raw text of each event changing as you fence
// sections in or out. show in a right frame.
// * show list of events on page in the top frame. can toggle them
// all individually.
// * and remove no-display from all tags so we can see everything.
// * highlight addresses, not just dates.
// * each section hash has its own unique bg color when activated
// * with a single click, completely reject an event because:
// contains bad time, address, title or desc. specify which so
// we can improve our algo.
// * when selecting an individual event, scroll to its tod...
// * remove all color from webpage that we can so our colors show up
// * remove all imgs. just src them to dev null.
// * allow for entering a custom title for an event or all events
// that are or will ever appear on the page.
// * when displaying the text of the events, use hyphens to
// delineate the section topology. strike out text as a section
// fence is activated.
// * when a section is activated is it easier to just redownload
// the whole text of the page? maybe just the text frame?
// * clicking on an individual sentence section should just remove
// that sentence. that is kinda a special content hash removal
// tag. like "Click here for video."
// * when an event id is selected i guess activate its bgcolor to
// be light blue for all sentences currently in the event that
// are not in activated sections. (make exception for designated
// title sections). so we need multiple tags for each events
// sentence div section. if sentence is split use multiple div tags
// then to keep the order. so each event sentence would have
// <div ev1=1 ev2=1 ev10=1>...</div> if it is in event ids 1,2 and
// 10. that way we can activate it when one of those event ids is
// activated.
SafeBuf sb;
// int16_tcuts
if ( ! xd->m_wordsValid ) { char *xx=NULL;*xx=0; }
Words *words = &xd->m_words;
int32_t nw = words->getNumWords();
char **wptrs = words->getWords();
int32_t *wlens = words->getWordLens();
nodeid_t *tids = words->getTagIds();
// a special array for printing </div> tags
char *endCounts = (char *)mcalloc ( nw ,"endcounts");
if ( ! endCounts ) return sendErrorReply ( st , g_errno );
//
// now loop over all the words. if word starts a section that has
// SEC_CONTROL bit set, and print out the section hash and a color
// tag to be activated if the turkey activates us.
// CAUTION: word may start multiple sections.
//
for ( int32_t i = 0 ; i < nw ; i++ ) {
// get section ptr
Section *sj = ss->m_sectionPtrs[i];
// sanity check. sj must be first section ptr that starts @ a
if ( sj && sj->m_a==i && sj->m_prev && sj->m_prev->m_a==i ) {
char *xx=NULL;*xx=0; }
// . does word #i start a section?
// . if section is control, print out the control
while ( sj && sj->m_a == i ) {
// print this section's hash
if ( sj->m_flags & SEC_CONTROL) {
// after the turkeys have made all the edits
// they need to submit the changes they made.
// how can we get that data sent back to the
// back end? we need to send back the colors
// of the sections that have been activated
// i guess. just do a loop over them.
sb.safePrintf("<div nobreak gbsecid=%"UINT32" "
"bgcolor=#%"XINT32" "
"onclick=gbtogglecolor()>",
(uint32_t)sj->m_tagHash,
(uint32_t)sj->m_tagHash);
// sanity check
if ( sj->m_b < 0 ) { char *xx=NULL;*xx=0; }
if ( sj->m_b > nw ) { char *xx=NULL;*xx=0; }
// and inc the /div count for that word
endCounts[sj->m_b-1]++;
}
// try next section too
sj = sj->m_next;
}
// if this is a tag, remove any coloring
if ( tids[i] ) {
}
// print the word, be it a tag, alnum, punct
sb.safeMemcpy ( wptrs[i] , wlens[i] );
// end a div tag?
if ( ! endCounts[i] ) continue;
// might be many so loop it
for ( int32_t j = 0 ; j < endCounts[i] ; j++ )
sb.safePrintf("</div>");
}
return false;
}
// . return the score of the highest-scoring window containing match #m
// . window is defined by the half-open interval [a,b) where a and b are
// word #'s in the Words array indicated by match #m
// . return -1 and set g_errno on error
int64_t Summary::getBestWindow ( Matches *matches, int32_t mm, int32_t *lasta,
int32_t *besta, int32_t *bestb, char *gotIt,
char *retired, int32_t maxExcerptLen ) {
// get the window around match #mm
Match *m = &matches->m_matches[mm];
// what is the word # of match #mm?
int32_t matchWordNum = m->m_wordNum;
// what Words/Pos/Bits classes is this match in?
Words *words = m->m_words;
Section **sp = NULL;
int32_t *pos = m->m_pos->m_pos;
// use "m_swbits" not "m_bits", that is what Bits::setForSummary() uses
const swbit_t *bb = m->m_bits->m_swbits;
// shortcut
if ( m->m_sections ) {
sp = m->m_sections->m_sectionPtrs;
}
int32_t nw = words->getNumWords();
int64_t *wids = words->getWordIds();
nodeid_t *tids = words->getTagIds();
// . sanity check
// . this prevents a core i've seen
if ( matchWordNum >= nw ) {
log("summary: got overflow condition for q=%s",m_q->m_orig);
// assume no best window
*besta = -1;
*bestb = -1;
*lasta = matchWordNum;
return 0;
}
// . we NULLify the section ptrs if we already used the word in another summary.
int32_t badFlags = SEC_SCRIPT|SEC_STYLE|SEC_SELECT|SEC_IN_TITLE;
if ( (bb[matchWordNum] & D_USED) || ( sp && (sp[matchWordNum]->m_flags & badFlags) ) ) {
// assume no best window
*besta = -1;
*bestb = -1;
*lasta = matchWordNum;
return 0;
}
// . "a" is the left fence post of the window (it is a word # in Words)
// . go to the left as far as we can
// . thus we decrement "a"
int32_t a = matchWordNum;
// "posa" is the character position of the END of word #a
int32_t posa = pos[a+1];
int32_t firstFrag = -1;
bool startOnQuote = false;
bool goodStart = false;
int32_t wordCount = 0;
// . decrease "a" as int32_t as we stay within maxNumCharsPerLine
// . avoid duplicating windows by using "lasta", the last "a" of the
// previous call to getBestWindow(). This can happen if our last
// central query term was close to this one.
for ( ; a > 0 && posa - pos[a-1] < maxExcerptLen && a > *lasta; a-- ) {
// . don't include any "dead zone",
// . dead zones have already been used for the summary, and
// we are getting a second/third/... excerpt here now then
// stop if its the start of a sentence, too
// stop before title word
if ( (bb[a-1] & D_USED) || (bb[a] & D_STARTS_SENTENCE) || ( bb[a-1] & D_IN_TITLE )) {
goodStart = true;
break;
}
// don't go beyond an LI, TR, P tag
if ( tids && ( tids[a-1] == TAG_LI ||
tids[a-1] == TAG_TR ||
tids[a-1] == TAG_P ||
tids[a-1] == TAG_DIV ) ) {
goodStart = true;
break;
}
// stop if its the start of a quoted sentence
if ( a+1<nw && (bb[a+1] & D_IN_QUOTES) &&
words->getWord(a)[0] == '\"' ){
startOnQuote = true;
goodStart = true;
break;
}
// find out the first instance of a fragment (comma, etc)
// watch out! because frag also means 's' in there's
if ( ( bb[a] & D_STARTS_FRAG ) && !(bb[a-1] & D_IS_STRONG_CONNECTOR) && firstFrag == -1 ) {
firstFrag = a;
}
if ( wids[a] ) {
wordCount++;
}
}
// if didn't find a good start, then start at the start of the frag
if ( !goodStart && firstFrag != -1 ) {
a = firstFrag;
}
// don't let punct or tag word start a line, unless a quote
if ( a < matchWordNum && !wids[a] && words->getWord(a)[0] != '\"' ){
while ( a < matchWordNum && !wids[a] ) a++;
// do not break right after a "strong connector", like
// apostrophe
while ( a < matchWordNum && a > 0 &&
( bb[a-1] & D_IS_STRONG_CONNECTOR ) )
a++;
// don't let punct or tag word start a line
while ( a < matchWordNum && !wids[a] ) a++;
}
// remember, b is not included in the summary, the summary is [a,b-1]
// remember to include all words in a matched phrase
int32_t b = matchWordNum + m->m_numWords ;
int32_t endQuoteWordNum = -1;
int32_t numTagsCrossed = 0;
for ( ; b <= nw; b++ ) {
if ( b == nw ) {
break;
}
if ( pos[b+1] - pos[a] >= maxExcerptLen ) {
break;
}
if ( startOnQuote && words->getWord(b)[0] == '\"' ) {
endQuoteWordNum = b;
}
// don't include any dead zone, those are already-used samples
if ( bb[b] & D_USED ) {
break;
}
// stop on a title word
if ( bb[b] & D_IN_TITLE ) {
break;
}
if ( wids[b] ) {
wordCount++;
}
// don't go beyond an LI or TR backtag
if ( tids && ( tids[b] == (BACKBIT|TAG_LI) ||
tids[b] == (BACKBIT|TAG_TR) ) ) {
numTagsCrossed++;
// try to have atleast 10 words in the summary
if ( wordCount > 10 ) {
break;
}
}
// go beyond a P or DIV backtag in case the earlier char is a
// ':'. This came from a special case for wikipedia pages
// eg. http://en.wikipedia.org/wiki/Flyover
if ( tids && ( tids[b] == (BACKBIT|TAG_P) ||
tids[b] == (BACKBIT|TAG_DIV) )) {
numTagsCrossed++;
// try to have atleast 10 words in the summary
if ( wordCount > 10 && words->getWord(b-1)[0] != ':' ) {
break;
}
}
}
// don't end on a lot of punct words
if ( b > matchWordNum && !wids[b-1]){
// remove more than one punct words. if we're ending on a quote
// keep it
while ( b > matchWordNum && !wids[b-2] && endQuoteWordNum != -1 && b > endQuoteWordNum ) {
b--;
}
// do not break right after a "strong connector", like apostrophe
while ( b > matchWordNum && (bb[b-2] & D_IS_STRONG_CONNECTOR) ) {
b--;
}
}
Match *ms = matches->m_matches;
// make m_matches.m_matches[mi] the first match in our [a,b) window
int32_t mi ;
// . the match at the center of the window is match #"mm", so that
// matches->m_matches[mm] is the Match class
// . set "mi" to it and back up "mi" as int32_t as >= a
for ( mi = mm ; mi > 0 && ms[mi-1].m_wordNum >=a ; mi-- )
;
// now get the score of this excerpt. Also mark all the represented
// query words. Mark the represented query words in the array that
// comes to us. also mark how many times the same word is repeated in
// this summary.
int64_t score = 0LL;
// is a url contained in the summary, that looks bad! punish!
bool hasUrl = false;
// the word count we did above was just an approximate. count it right
wordCount = 0;
// for debug
//char buf[5000];
//char *xp = buf;
SafeBuf xp;
// wtf?
if ( b > nw ) {
b = nw;
}
// first score from the starting match down to a, including match
for ( int32_t i = a ; i < b ; i++ ) {
// debug print out
if ( g_conf.m_logDebugSummary ) {
int32_t len = words->getWordLen(i);
char cs;
for (int32_t k=0;k<len; k+=cs ) {
const char *c = words->getWord(i)+k;
cs = getUtf8CharSize(c);
if ( is_binary_utf8 ( c ) ) {
continue;
}
xp.safeMemcpy ( c , cs );
xp.nullTerm();
}
}
// skip if in bad section, marquee, select, script, style
if ( sp && (sp[i]->m_flags & badFlags) ) {
continue;
}
// don't count just numeric words
if ( words->isNum(i) ) {
continue;
}
// check if there is a url. best way to check for '://'
if ( wids && !wids[i] ) {
const char *wrd = words->getWord(i);
int32_t wrdLen = words->getWordLen(i);
if ( wrdLen == 3 && wrd[0] == ':' && wrd[1] == '/' && wrd[2] == '/' ) {
hasUrl = true;
}
}
// skip if not wid
if ( ! wids[i] ) {
continue;
}
// just make every word 100 pts
int32_t t = 100;
// penalize it if in one of these sections
if ( bb[i] & ( D_IN_PARENS | D_IN_SUP | D_IN_LIST ) ) {
t /= 2;
}
// boost it if in bold or italics
if ( bb[i] & D_IN_BOLDORITALICS ) {
t *= 2;
}
// add the score for this word
score += t;
// print the score, "t"
if ( g_conf.m_logDebugSummary ) {
xp.safePrintf("(%" PRId32")",t);
}
// count the alpha words we got
wordCount++;
// if no matches left, skip
if ( mi >= matches->m_numMatches ) {
continue;
}
// get the match
Match *next = &ms[mi];
// skip if not a match
if ( i != next->m_wordNum ) {
continue;
}
// must be a match in this class
if ( next->m_words != words ) {
continue;
}
// advance it
mi++;
// which query word # does it match
int32_t qwn = next->m_qwordNum;
if ( qwn < 0 || qwn >= m_q->m_numWords ){g_process.shutdownAbort(true);}
// undo old score
score -= t;
// add 100000 per match
t = 100000;
// weight based on tf, goes from 0.1 to 1.0
t = (int32_t)((float)t * m_wordWeights [ qwn ]);
// if it is a query stop word, make it 10000 pts
if ( m_q->m_qwords[qwn].m_isQueryStopWord ) {
t = 0;//10000;
}
// penalize it if in one of these sections
if ( bb[i] & ( D_IN_PARENS | D_IN_SUP | D_IN_LIST ) ) {
t /= 2;
}
if ( gotIt[qwn] > 0 ) {
// have we matched it in this [a,b) already?
if ( gotIt[qwn] == 1 ) {
t /= 15;
} else {
// if we have more than 2 matches in the same window,
// it may not give a good summary. give a heavy penalty
t -= 200000;
}
} else if ( retired [qwn] > 0 ) {
// have we matched it already in a winning window?
t /= 12;
}
// add it back
score += t;
if ( g_conf.m_logDebugSummary ) {
xp.safePrintf ("[%" PRId32"]{qwn=%" PRId32",ww=%f}",t,qwn,
m_wordWeights[qwn]);
}
// inc the query word count for this window
if ( gotIt[qwn] < 100 ) {
gotIt[qwn]++;
}
}
int32_t oldScore = score;
// apply the bonus if it starts or a sentence
// only apply if the score is positive and if the wordcount is decent
if ( score > 0 && wordCount > 7 ){
// a match can give us 10k to 100k pts based on the tf weights
// so we don't want to overwhelm that too much, so let's make
// this a 20k bonus if it starts a sentence
if ( bb[a] & D_STARTS_SENTENCE ) {
score += 8000;
} else if ( bb[a] & D_STARTS_FRAG ) {
// likewise, a fragment, like after a comma
score += 4000;
}
// 1k if the match word is very close to the
// start of a sentence, lets say 3 alphawords
if ( matchWordNum - a < 7 ) {
score += 1000;
}
}
// a summary isn't really a summary if its less than 7 words.
// reduce the score, but still give it a decent score.
// minus 5M.
if ( wordCount < 7 ) {
score -= 20000;
}
// summaries that cross a lot of tags are usually bad, penalize them
if ( numTagsCrossed > 1 ) {
score -= (numTagsCrossed * 20000);
}
if ( hasUrl ) {
score -= 8000;
}
// show it
if ( g_conf.m_logDebugSummary ) {
log(LOG_DEBUG, "sum: score=%08" PRId32" prescore=%08" PRId32" a=%05" PRId32" b=%05" PRId32" %s",
(int32_t)score,oldScore,(int32_t)a,(int32_t)b,
xp.getBufStart());
}
// set lasta, besta, bestb
*lasta = a;
*besta = a;
*bestb = b;
return score;
}