bool JsonItem::getCompoundName ( SafeBuf &nameBuf ) {
// reset, but don't free mem etc. just set m_length to 0
nameBuf.reset();
// get its full compound name like "meta.twitter.title"
JsonItem *p = this;//ji;
char *lastName = NULL;
char *nameArray[20];
int32_t numNames = 0;
for ( ; p ; p = p->m_parent ) {
// empty name?
if ( ! p->m_name ) continue;
if ( ! p->m_name[0] ) continue;
// dup? can happen with arrays. parent of string
// in object, has same name as his parent, the
// name of the array. "dupname":[{"a":"b"},{"c":"d"}]
if ( p->m_name == lastName ) continue;
// update
lastName = p->m_name;
// add it up
nameArray[numNames++] = p->m_name;
// breach?
if ( numNames < 15 ) continue;
log("build: too many names in json tag");
break;
}
// assemble the names in reverse order which is correct order
for ( int32_t i = 1 ; i <= numNames ; i++ ) {
// copy into our safebuf
if ( ! nameBuf.safeStrcpy ( nameArray[numNames-i]) )
return false;
// separate names with periods
if ( ! nameBuf.pushChar('.') ) return false;
}
// remove last period
nameBuf.removeLastChar('.');
// and null terminate
if ( ! nameBuf.nullTerm() ) return false;
// change all :'s in names to .'s since : is reserved!
char *px = nameBuf.getBufStart();
for ( ; *px ; px++ ) if ( *px == ':' ) *px = '.';
return true;
}
// . the url being reuqested
// . removes &code= facebook cruft
bool HttpRequest::getCurrentUrl ( SafeBuf &cu ) {
// makre sure we got enough room
if ( ! cu.reserve ( m_hostLen + 64 + m_plen + 1 + 1 ) ) return false;
// need a "Host: "
char *host = m_host;
if ( ! host ) host = APPSUBDOMAIN;
cu.safePrintf("http");
if ( m_isSSL ) cu.pushChar('s');
cu.safePrintf("://%s",host);
char *path = m_path;
long plen = m_plen;
if ( ! path ) {
path = "/";
plen = 1;
}
// . scan path and change \0 back to = or &
// . similar logic in HttpServer.cpp for logging!
char *dst = cu.getBuf();
char *src = path;
char *srcEnd = path + plen;
char dd = '=';
for ( ; src < srcEnd ; src++ , dst++ ) {
*dst = *src;
if ( *src ) continue;
*dst = dd;
if ( dd == '=' ) dd = '&';
else dd = '=';
}
*dst = '\0';
// cut it off at facebook's &code=
char *buf = cu.getBufStart();
char *code = strstr( buf,"&code=");
// fix for eventguru.com/blog.html?code=
if ( ! code ) code = strstr(buf,"?code=");
// hack that off if there
if ( code ) {
*code = '\0';
dst = code;
}
// update length
cu.setLength( dst - cu.getBufStart() );
return true;
}
// . returns false if blocked, true otherwise
// . sets g_errno on error
// . called either from
// 1) doDocIdSplitLoop
// 2) or getDocIds2() if only 1 docidsplit
bool Msg39::getLists () {
if ( m_debug ) m_startTime = gettimeofdayInMilliseconds();
// . ask Indexdb for the IndexLists we need for these termIds
// . each rec in an IndexList is a termId/score/docId tuple
//
// restrict to docid range?
//
// . get the docid start and end
// . do docid paritioning so we can send to all hosts
// in the network, not just one stripe
int64_t docIdStart = 0;
int64_t docIdEnd = MAX_DOCID;
// . restrict to this docid?
// . will really make gbdocid:| searches much faster!
int64_t dr = m_tmpq.m_docIdRestriction;
if ( dr ) {
docIdStart = dr;
docIdEnd = dr + 1;
}
// . override
// . this is set from Msg39::doDocIdSplitLoop() to compute
// search results in stages, so that we do not load massive
// termlists into memory and got OOM (out of memory)
if ( m_r->m_minDocId != -1 ) docIdStart = m_r->m_minDocId;
if ( m_r->m_maxDocId != -1 ) docIdEnd = m_r->m_maxDocId+1;
// if we have twins, then make sure the twins read different
// pieces of the same docid range to make things 2x faster
//bool useTwins = false;
//if ( g_hostdb.getNumStripes() == 2 ) useTwins = true;
//if ( useTwins ) {
// int64_t delta2 = ( docIdEnd - docIdStart ) / 2;
// if ( m_r->m_stripe == 0 ) docIdEnd = docIdStart + delta2;
// else docIdStart = docIdStart + delta2;
//}
// new striping logic:
int32_t numStripes = g_hostdb.getNumStripes();
int64_t delta2 = ( docIdEnd - docIdStart ) / numStripes;
int32_t stripe = g_hostdb.getMyHost()->m_stripe;
docIdStart += delta2 * stripe; // is this right?
docIdEnd = docIdStart + delta2;
// add 1 to be safe so we don't lose a docid
docIdEnd++;
// TODO: add triplet support later for this to split the
// read 3 ways. 4 ways for quads, etc.
//if ( g_hostdb.getNumStripes() >= 3 ) { char *xx=NULL;*xx=0;}
// do not go over MAX_DOCID because it gets masked and
// ends up being 0!!! and we get empty lists
if ( docIdEnd > MAX_DOCID ) docIdEnd = MAX_DOCID;
// remember so Msg2.cpp can use them to restrict the termlists
// from "whiteList" as well
m_docIdStart = docIdStart;
m_docIdEnd = docIdEnd;
//
// set startkey/endkey for each term/termlist
//
for ( int32_t i = 0 ; i < m_tmpq.getNumTerms() ; i++ ) {
// breathe
QUICKPOLL ( m_r->m_niceness );
// int16_tcuts
QueryTerm *qterm = &m_tmpq.m_qterms[i];
char *sk = qterm->m_startKey;
char *ek = qterm->m_endKey;
// get the term id
int64_t tid = m_tmpq.getTermId(i);
// if only 1 stripe
//if ( g_hostdb.getNumStripes() == 1 ) {
// docIdStart = 0;
// docIdEnd = MAX_DOCID;
//}
// debug
if ( m_debug )
log("query: setting sk/ek for docids %"INT64""
" to %"INT64" for termid=%"INT64""
, docIdStart
, docIdEnd
, tid
);
// store now in qterm
g_posdb.makeStartKey ( sk , tid , docIdStart );
g_posdb.makeEndKey ( ek , tid , docIdEnd );
qterm->m_ks = sizeof(POSDBKEY);//key144_t);
}
// debug msg
if ( m_debug || g_conf.m_logDebugQuery ) {
for ( int32_t i = 0 ; i < m_tmpq.getNumTerms() ; i++ ) {
// get the term in utf8
//char bb[256];
QueryTerm *qt = &m_tmpq.m_qterms[i];
//utf16ToUtf8(bb, 256, qt->m_term, qt->m_termLen);
char *tpc = qt->m_term + qt->m_termLen;
char tmp = *tpc;
*tpc = '\0';
char sign = qt->m_termSign;
if ( sign == 0 ) sign = '0';
QueryWord *qw = qt->m_qword;
int32_t wikiPhrId = qw->m_wikiPhraseId;
if ( m_tmpq.isPhrase(i) ) wikiPhrId = 0;
char leftwikibigram = 0;
char rightwikibigram = 0;
if ( qt->m_leftPhraseTerm &&
qt->m_leftPhraseTerm->m_isWikiHalfStopBigram )
leftwikibigram = 1;
if ( qt->m_rightPhraseTerm &&
qt->m_rightPhraseTerm->m_isWikiHalfStopBigram )
rightwikibigram = 1;
/*
char c = m_tmpq.getTermSign(i);
char tt[512];
int32_t ttlen = m_tmpq.getTermLen(i);
if ( ttlen > 254 ) ttlen = 254;
if ( ttlen < 0 ) ttlen = 0;
// old:painful: convert each term from unicode to ascii
gbmemcpy ( tt , m_tmpq.getTerm(i) , ttlen );
*/
int32_t isSynonym = 0;
QueryTerm *st = qt->m_synonymOf;
if ( st ) isSynonym = true;
SafeBuf sb;
// now we can display it
//tt[ttlen]='\0';
//if ( c == '\0' ) c = ' ';
sb.safePrintf(
"query: msg39: [%"PTRFMT"] "
"query term #%"INT32" \"%s\" "
"phr=%"INT32" termId=%"UINT64" rawTermId=%"UINT64" "
//"estimatedTermFreq=%"INT64" (+/- ~16000) "
"tfweight=%.02f "
"sign=%c "
"numPlusses=%hhu "
"required=%"INT32" "
"fielcode=%"INT32" "
"ebit=0x%0"XINT64" "
"impBits=0x%0"XINT64" "
"wikiphrid=%"INT32" "
"leftwikibigram=%"INT32" "
"rightwikibigram=%"INT32" "
//"range.startTermNum=%hhi range.endTermNum=%hhi "
//"minRecSizes=%"INT32" "
"readSizeInBytes=%"INT32" "
//"ebit=0x%"XINT64" "
//"impBits=0x%"XINT64" "
"hc=%"INT32" "
"component=%"INT32" "
"otermLen=%"INT32" "
"isSynonym=%"INT32" "
"querylangid=%"INT32" " ,
(PTRTYPE)this ,
i ,
qt->m_term,//bb ,
(int32_t)m_tmpq.isPhrase (i) ,
m_tmpq.getTermId (i) ,
m_tmpq.getRawTermId (i) ,
((float *)m_r->ptr_termFreqWeights)[i] ,
sign , //c ,
0 ,
(int32_t)qt->m_isRequired,
(int32_t)qt->m_fieldCode,
(int64_t)qt->m_explicitBit ,
(int64_t)qt->m_implicitBits ,
wikiPhrId,
(int32_t)leftwikibigram,
(int32_t)rightwikibigram,
((int32_t *)m_r->ptr_readSizes)[i] ,
//(int64_t)m_tmpq.m_qterms[i].m_explicitBit ,
//(int64_t)m_tmpq.m_qterms[i].m_implicitBits ,
(int32_t)m_tmpq.m_qterms[i].m_hardCount ,
(int32_t)m_tmpq.m_componentCodes[i],
(int32_t)m_tmpq.getTermLen(i) ,
isSynonym,
(int32_t)m_tmpq.m_langId ); // ,tt
// put it back
*tpc = tmp;
if ( st ) {
int32_t stnum = st - m_tmpq.m_qterms;
sb.safePrintf("synofterm#=%"INT32"",stnum);
//sb.safeMemcpy(st->m_term,st->m_termLen);
sb.pushChar(' ');
sb.safePrintf("synwid0=%"INT64" ",qt->m_synWids0);
sb.safePrintf("synwid1=%"INT64" ",qt->m_synWids1);
sb.safePrintf("synalnumwords=%"INT32" ",
qt->m_numAlnumWordsInSynonym);
// like for synonym "nj" it's base,
// "new jersey" has 2 alnum words!
sb.safePrintf("synbasealnumwords=%"INT32" ",
qt->m_numAlnumWordsInBase);
}
logf(LOG_DEBUG,"%s",sb.getBufStart());
}
m_tmpq.printBooleanTree();
}
// timestamp log
if ( m_debug )
log(LOG_DEBUG,"query: msg39: [%"PTRFMT"] "
"Getting %"INT32" index lists ",
(PTRTYPE)this,m_tmpq.getNumTerms());
// . now get the index lists themselves
// . return if it blocked
// . not doing a merge (last parm) means that the lists we receive
// will be an appending of a bunch of lists so keys won't be in order
// . merging is uneccessary for us here because we hash the keys anyway
// . and merging takes up valuable cpu time
// . caution: the index lists returned from Msg2 are now compressed
// . now i'm merging because it's 10 times faster than hashing anyway
// and the reply buf should now always be <= minRecSizes so we can
// pre-allocate one better, and, 3) this should fix the yahoo.com
// reindex bug
char rdbId = RDB_POSDB;
// . TODO: MDW: fix
// . partap says there is a bug in this??? we can't cache UOR'ed lists?
bool checkCache = false;
// split is us????
//int32_t split = g_hostdb.m_myHost->m_group;
int32_t split = g_hostdb.m_myHost->m_shardNum;
// call msg2
if ( ! m_msg2.getLists ( rdbId ,
m_r->m_collnum,//m_r->ptr_coll ,
m_r->m_maxAge ,
m_r->m_addToCache ,
//m_tmpq.m_qterms ,
&m_tmpq,
m_r->ptr_whiteList,
// we need to restrict docid range for
// whitelist as well! this is from
// doDocIdSplitLoop()
m_docIdStart,
m_docIdEnd,
// how much of each termlist to read in bytes
(int32_t *)m_r->ptr_readSizes ,
//m_tmpq.getNumTerms() , // numLists
// 1-1 with query terms
m_lists ,
this ,
controlLoopWrapper,//gotListsWrapper ,
m_r ,
m_r->m_niceness ,
true , // do merge?
m_debug ,
NULL , // best hostids
m_r->m_restrictPosdbForQuery ,
split ,
checkCache )) {
m_blocked = true;
return false;
}
// error?
//if ( g_errno ) {
// log("msg39: Had error getting termlists2: %s.",
// mstrerror(g_errno));
// // don't bail out here because we are in docIdSplitLoop()
// //sendReply (m_slot,this,NULL,0,0,true);
// return true;
//}
//return gotLists ( true );
return true;
}
bool sendPageThesaurus( TcpSocket *s, HttpRequest *r ) {
SafeBuf p;
char getBuf[64]; // holds extra values for GET method
char formBuf[256]; // holds extra values for forms
snprintf(getBuf, 64, "c=%s",
r->getString("c", 0, ""));
snprintf(formBuf, 256,
"<input type=hidden name=\"c\" value=\"%s\">",
//"<input type=hidden name=\"pwd\" value=\"%s\">",
r->getString("c", 0, ""));
g_pages.printAdminTop( &p, s, r);
if (r->getLong("cancel", 0) != 0) {
g_thesaurus.cancelRebuild();
p.safePrintf("<br><br>\n");
p.safePrintf(
"<center><b><font color=#ff0000>"
"rebuild canceled"
"</font></b></center>");
}
if (r->getLong("rebuild", 0) != 0) {
bool full = r->getLong("full", 0);
p.safePrintf("<br><br>\n");
if (g_thesaurus.rebuild(0, full)) {
p.safePrintf(
"<center><b><font color=#ff0000>"
"error starting rebuild, check log for details"
"</font></b></center>");
} else {
p.safePrintf(
"<center><b><font color=#ff0000>"
"rebuild started"
"</font></b></center>");
}
}
if (r->getLong("rebuildaff", 0) != 0) {
bool full = r->getLong("full", 0);
p.safePrintf("<br><br>\n");
if (g_thesaurus.rebuildAffinity(0, full)) {
p.safePrintf(
"<center><b><font color=#ff0000>"
"error starting rebuild, check log for details"
"</font></b></center>");
} else {
p.safePrintf(
"<center><b><font color=#ff0000>"
"rebuild started"
"</font></b></center>");
}
}
if (r->getLong("distribute", 0) != 0) {
char cmd[1024];
p.safePrintf("<br><br>\n");
if (g_thesaurus.m_affinityState) {
p.safePrintf(
"<center><b><font color=#ff0000>"
"cannot distribute during rebuild"
"</font></b></center>");
} else {
for ( long i = 0; i < g_hostdb.getNumHosts() ; i++ ) {
Host *h = g_hostdb.getHost(i);
snprintf(cmd, 512,
"rcp -r "
"./dict/thesaurus.* "
"%s:%s/dict/ &",
iptoa(h->m_ip),
h->m_dir);
log(LOG_INFO, "admin: %s", cmd);
system( cmd );
}
p.safePrintf(
"<center><b><font color=#ff0000>"
"data distributed"
"</font></b></center>");
}
}
if (r->getLong("reload", 0) != 0) {
p.safePrintf("<br><br>\n");
if (r->getLong("cast", 0) != 0) {
p.safePrintf(
"<center><b><font color=#ff0000>"
"reload command broadcast"
"</font></b></center>");
} else if (g_thesaurus.init()) {
p.safePrintf(
"<center><b><font color=#ff0000>"
"thesaurus data reloaded"
"</font></b></center>");
} else {
p.safePrintf(
"<center><b><font color=#ff0000>"
"error reloading thesaurus data"
"</font></b></center>");
}
}
long manualAddLen = 0;
char *manualAdd = NULL;
SafeBuf manualAddBuf;
if ((manualAdd = r->getString("manualadd", &manualAddLen))) {
trimWhite(manualAdd);
manualAddLen = gbstrlen(manualAdd);
File manualFile;
manualFile.set(g_hostdb.m_dir, "dict/thesaurus-manual.txt");
if (manualFile.open(O_WRONLY | O_CREAT | O_TRUNC) &&
(manualFile.write(manualAdd, manualAddLen, 0) ==
manualAddLen)) {
char newl = '\n'; // for write()
if (manualAdd[manualAddLen-1] != '\n')
manualFile.write(&newl, 1, manualAddLen);
p.safePrintf(
"<center><b><font color=#ff0000>"
"updated manual add file sucessfully"
"</font></b></center>");
} else {
p.safePrintf(
"<center><b><font color=#ff0000>"
"error writing manual add file"
"</font></b></center>");
}
} else {
char ff[PATH_MAX];
snprintf(ff, PATH_MAX, "%sdict/thesaurus-manual.txt",
g_hostdb.m_dir);
if (manualAddBuf.fillFromFile(ff)) {
if (*(manualAddBuf.getBuf()-1) != '\n')
manualAddBuf.pushChar('\n');
manualAdd = manualAddBuf.getBufStart();
manualAddLen = manualAddBuf.length();
}
}
long affinityAddLen = 0;
char *affinityAdd = NULL;
SafeBuf affinityAddBuf;
if ((affinityAdd = r->getString("affinityadd", &affinityAddLen))) {
trimWhite(affinityAdd);
affinityAddLen = gbstrlen(affinityAdd);
File affinityFile;
affinityFile.set(g_hostdb.m_dir,
"dict/thesaurus-affinity.txt");
if (affinityFile.open(O_WRONLY | O_CREAT | O_TRUNC) &&
(affinityFile.write(affinityAdd, affinityAddLen, 0) ==
affinityAddLen)) {
char newl = '\n'; // for write()
if (affinityAdd[affinityAddLen-1] != '\n')
affinityFile.write(&newl, 1, affinityAddLen);
p.safePrintf(
"<center><b><font color=#ff0000>"
"updated affinity add file sucessfully"
"</font></b></center>");
} else {
p.safePrintf(
"<center><b><font color=#ff0000>"
"error writing affinity add file"
"</font></b></center>");
}
} else {
char ff[PATH_MAX];
snprintf(ff, PATH_MAX, "%sdict/thesaurus-affinity.txt",
g_hostdb.m_dir);
if (affinityAddBuf.fillFromFile(ff)) {
if (*(affinityAddBuf.getBuf()-1) != '\n')
affinityAddBuf.pushChar('\n');
affinityAdd = affinityAddBuf.getBufStart();
affinityAddLen = affinityAddBuf.length();
}
}
char *syn = r->getString("synonym");
long len = 0;
if (syn) len = gbstrlen(syn);
if (len) {
SynonymInfo info;
bool r = g_thesaurus.getAllInfo(syn, &info, len, SYNBIT_ALL);
p.safePrintf("<br><br>\n");
p.safePrintf (
"<table cellpadding=4 width=100%% bgcolor=#%s border=1>"
"<tr>"
"<td colspan=2 bgcolor=#%s>"
"<center><b>Synonym List (%ld)</b></center>"
"</td>"
"</tr>\n",
LIGHT_BLUE, DARK_BLUE, info.m_numSyns);
if (r) {
p.safePrintf("<tr>"
"<td align=right><tt>%s</tt></td>"
"<td align=left>"
"<tt>1.000/%08lX (1.000/%08lX)</tt>"
"</td>"
"</tr>\n", syn, MAX_AFFINITY, MAX_AFFINITY);
for (long i = 0; i < info.m_numSyns; i++) {
// get the reverse affinity as well
long aff = g_thesaurus.getAffinity(
info.m_syn[i], syn,
info.m_len[i], len);
p.safePrintf(
"<tr>"
"<td width=40%% align=right>"
"<tt>");
p.safeMemcpy(info.m_syn[i], info.m_len[i]);
p.safePrintf("</tt>"
"</td>"
"<td width=60%% align=left>"
"<tt>");
if (info.m_affinity[i] >= 0) {
p.safePrintf("%0.3f/%08lX ",
(float)info.m_affinity[i]
/ MAX_AFFINITY,
info.m_affinity[i]);
} else {
p.safePrintf("u ");
}
if (aff >= 0) {
p.safePrintf("(%0.3f/%08lX) ",
(float)aff / MAX_AFFINITY,
aff);
} else {
p.safePrintf("(u) ");
}
p.safePrintf("(%ld) (%ld) (%ld) (%ld) "
"(%lld) (%lld)",
(long)info.m_type[i], (long)info.m_sort[i],
info.m_firstId[i], info.m_lastId[i],
info.m_leftSynHash[i],
info.m_rightSynHash[i]);
for (int j = info.m_firstId[i];
j <= info.m_lastId[i];
j++) {
p.safePrintf(" (%lld)",
info.m_termId[j]);
}
p.safePrintf(
"</tt>"
"</td>"
"</tr>\n");
}
p.safePrintf("</table>");
} else {
p.safePrintf("<tr>"
"<td align=center><font color=#FF0000>"
"synonym not found: %s"
"</font></td>"
"</tr>\n",
syn);
}
}
p.safePrintf ( "<br><br>\n" );
p.safePrintf (
"<table cellpadding=4 width=100%% bgcolor=#%s border=1>"
"<tr>"
"<td colspan=2 bgcolor=#%s>"
"<center><b>Thesaurus Controls"
"</b></center></td>"
"</tr>\n",
LIGHT_BLUE, DARK_BLUE);
p.safePrintf (
"<tr>"
"<td width=37%%><b>rebuild all data</b><br>"
"<font size=1>"
"rebuilds synonyms and then begins the rebuild process for "
"affinity data; this should only be run on one host, as the "
"data is copied when the process is finished; full rebuild "
"does not use existing affinity data"
"</font>"
"</td>"
"<td width=12%% bgcolor=#0000ff>"
"<center><b><a href=\"/master/thesaurus?rebuild=1&%s\">"
"rebuild all data</a> <a href=\"/master/thesaurus?"
"rebuild=1&full=1&%s\">(full)</a></b></center>"
"</td>"
"</tr>\n", getBuf, getBuf);
p.safePrintf (
"<tr>"
"<td width=37%%><b>distribute data</b><br>"
"<font size=1>"
"distributes all thesaurus data to all hosts, this is "
"normally done automatically but if there was a problem "
"with the copy, this lets you do it manually"
"</font>"
"</td>"
"<td width=12%% bgcolor=#0000ff>"
"<center><b><a href=\"/master/thesaurus?distribute=1&%s\">"
"distribute data</a></b></center>"
"</td>"
"</tr>\n", getBuf);
p.safePrintf (
"<tr>"
"<td width=37%%><b>reload data</b><br>"
"<font size=1>"
"reloads the synonyms and affinity table on this host only"
"</font>"
"</td>"
"<td width=12%% bgcolor=#0000ff>"
"<center><b>"
"<a href=\"/master/thesaurus?reload=1&cast=0&%s\">"
"reload data</a></b></center>"
"</td>"
"</tr>\n", getBuf);
p.safePrintf (
"<tr>"
"<td width=37%%><b>reload data (all hosts)</b><br>"
"<font size=1>"
"reloads the synonyms and affinity table on all hosts"
"</font>"
"</td>"
"<td width=12%% bgcolor=#0000ff>"
"<center><b>"
"<a href=\"/master/thesaurus?reload=1&cast=1&%s\">"
"reload data (all hosts)</a></b></center>"
"</td>"
"</tr>\n", getBuf);
p.safePrintf (
"<tr>"
"<td width=37%%><b>list synonyms</b><br>"
"<font size=1>"
"enter a word here to list all synonym entries and their "
"affinities"
"</font>"
"</td>"
"<td width=12%%>"
"<form action=\"/master/thesaurus>\">"
"<input type=text name=synonym size=20>"
"<input type=submit value=Submit>"
"%s"
"</form></td>"
"</tr>\n", formBuf);
p.safePrintf (
"<tr>"
"<td colspan=2 bgcolor=#%s>"
"<center><b>Affinity Controls"
"</b></center></td>"
"</tr>\n",
DARK_BLUE);
p.safePrintf (
"<tr>"
"<td width=37%%><b>cancel running rebuild</b><br>"
"<font size=1>"
"cancels the rebuild and throws all intermediate data away"
"</font>"
"</td>"
"<td width=12%% bgcolor=#0000ff>"
"<center><b><a href=\"/master/thesaurus?cancel=1&%s\">"
"cancel running rebuild</a></b></center>"
"</td>"
"</tr>\n", getBuf);
p.safePrintf (
"<tr>"
"<td width=37%%><b>rebuild affinity only</b><br>"
"<font size=1>"
"begins the rebuild process for affinity data, has no "
"effect if a rebuild is already in progress; full rebuild "
"does not reuse existing affinity data"
"</font>"
"</td>"
"<td width=12%% bgcolor=#0000ff>"
"<center><b><a href=\"/master/thesaurus?rebuildaff=1&%s\">"
"rebuild affinity</a> <a href=\"/master/thesaurus?"
"rebuildaff=1&full=1&%s\">(full)</a></b></center>"
"</td>"
"</tr>\n", getBuf, getBuf);
p.safePrintf (
"<tr>"
"<td colspan=2 bgcolor=#%s>"
"<center><b>Manual File Controls"
"</b></td>"
"</tr>\n",
DARK_BLUE);
p.safePrintf (
"<tr>"
"<td align=center colspan=2>");
p.safePrintf(
"<b>manually added pairs</b><br>\n"
"<font size=1>place word pairs here that should be linked "
"as synonyms, one pair per line, seperated by a pipe '|' "
"character, optionally followed by another pipe and a type "
"designation; any badly formatted lines will be silently "
"ignored</font><br>\n"
"<form action=\"/master/thesaurus\" method=post>"
"<textarea name=\"manualadd\" rows=20 cols=80>");
if (manualAdd && manualAddLen) {
p.htmlEncode(manualAdd, manualAddLen, true);
}
p.safePrintf (
"</textarea><br>"
"<input type=submit value=Submit>"
"<input type=reset value=Reset>"
"%s"
"</form></td>"
"</tr>\n",
formBuf);
p.safePrintf (
"<tr>"
"<td align=center colspan=2>"
"<b>affinity value overrides</b><br>\n"
"<font size=1>place word/phrase pairs here that should have "
"there affinity values overridden, format is "
"\"word1|word2|value\", where value is a floating point, "
"integer (either decimal or hex), or the word \"max\"; "
"any badly formatted lines will be silently ignored; note "
"that these pairs will only work if the thesaurus otherwise "
"has an entry for them, so add them to the manual add file "
"above if need be</font><br>\n"
"<form action=\"/master/thesaurus\" method=post>"
"<textarea name=\"affinityadd\" rows=20 cols=80>");
if (affinityAdd && affinityAddLen) {
p.htmlEncode(affinityAdd, affinityAddLen, true);
}
p.safePrintf (
"</textarea><br>"
"<input type=submit value=Submit>"
"<input type=reset value=Reset>"
"%s"
"</form></td>"
"</tr>\n",
formBuf);
p.safePrintf ( "</table>\n" );
p.safePrintf ( "<br><br>\n" );
p.safePrintf (
"<table cellpadding=4 width=100%% bgcolor=#%s border=1>"
"<tr>"
"<td colspan=2 bgcolor=#%s>"
"<center><b>Affinity Builder Status"
"</b></td>"
"</tr>\n",
LIGHT_BLUE, DARK_BLUE);
long long a, b, c, d, e, f, g, h, i, j, k;
StateAffinity *aff = g_thesaurus.m_affinityState;
if (!aff) {
p.safePrintf (
"<tr><td colspan=2>"
"<center><b>Not running</b></center>"
"</td></tr>\n");
a = b = c = d = e = f = g = h = i = j = k = 0;
} else {
a = aff->m_oldTable->getNumSlotsUsed();
b = aff->m_oldTable->getNumSlotsUsed() - aff->m_n;
c = aff->m_n;
d = (gettimeofdayInMilliseconds() - aff->m_time) / 1000;
if (!d || !(c / d)) {
e = 0;
} else {
e = b / (c / d);
}
f = aff->m_sent;
g = aff->m_recv;
h = aff->m_errors;
i = aff->m_old;
j = aff->m_cache;
k = aff->m_hitsTable.getNumSlotsUsed();
}
p.safePrintf (
"<tr><td><b># of total pairs</b></td>"
"<td>%lli</td></tr>\n"
"<tr><td><b># of pairs remaining</b></td>"
"<td>%lli</td></tr>\n"
"<tr><td><b># of pairs processed</b></td>"
"<td>%lli</td></tr>\n"
"<tr><td><b>elapsed time in seconds</b></td>"
"<td>%lli</td></tr>\n"
"<tr><td><b>estimated remaining time in seconds</b></td>"
"<td>%lli</td></tr>\n"
"<tr><td><b># of requests sent</b></td>"
"<td>%lli</td></tr>\n"
"<tr><td><b># of requests received</b></td>"
"<td>%lli</td></tr>\n"
"<tr><td><b># of request errors</b></td>"
"<td>%lli</td></tr>\n"
"<tr><td><b># of old values reused</b></td>"
"<td>%lli</td></tr>\n"
"<tr><td><b># of cache hits</b></td>"
"<td>%lli</td></tr>\n"
"<tr><td><b>cache size</b></td>"
"<td>%lli</td></tr>\n",
a, b, c, d, e, f, g, h, i, j, k);
p.safePrintf ( "</table>\n" );
return g_httpServer.sendDynamicPage ( s, p.getBufStart(), p.length() );
}
bool HttpMime::addCookieHeader(const char *cookieJar, const char *url, SafeBuf *sb) {
Url tmpUrl;
tmpUrl.set(url);
SafeBuf tmpSb;
size_t cookieJarLen = strlen(cookieJar);
const char *lineStartPos = cookieJar;
const char *lineEndPos = NULL;
while ((lineEndPos = (const char*)memchr(lineStartPos, '\n', cookieJarLen - (lineStartPos - cookieJar))) != NULL) {
const char *currentPos = lineStartPos;
const char *tabPos = NULL;
unsigned fieldCount = 0;
bool skipCookie = false;
const char *domain = NULL;
int32_t domainLen = 0;
while (fieldCount < 5 && (tabPos = (const char*)memchr(currentPos, '\t', lineEndPos - currentPos)) != NULL) {
switch (fieldCount) {
case 0:
// domain
domain = currentPos;
domainLen = tabPos - currentPos;
break;
case 1:
// flag
if (memcmp(currentPos, "TRUE", 4) == 0) {
// allow subdomain
if (tmpUrl.getHostLen() >= domainLen) {
if (!endsWith(tmpUrl.getHost(), tmpUrl.getHostLen(), domain, domainLen)) {
// doesn't end with domain - ignore cookie
skipCookie = true;
break;
}
} else {
skipCookie = true;
break;
}
} else {
// only specific domain
if (tmpUrl.getHostLen() != domainLen || strncasecmp(domain, tmpUrl.getHost(), domainLen) != 0) {
// non-matching domain - ignore cookie
skipCookie = true;
break;
}
}
break;
case 2: {
// path
const char *path = currentPos;
int32_t pathLen = tabPos - currentPos;
if (strncasecmp(path, tmpUrl.getPath(), pathLen) == 0) {
if (tmpUrl.getPathLen() != pathLen) {
if (path[pathLen - 1] != '/' && tmpUrl.getPath()[tmpUrl.getPathLen() - 1] != '/') {
// non-matching path - ignore cookie
skipCookie = true;
break;
}
}
} else {
// non-matching path - ignore cookie
skipCookie = true;
break;
}
} break;
case 3:
// secure
break;
case 4:
// expiration
break;
}
currentPos = tabPos + 1;
++fieldCount;
}
if (!skipCookie) {
tmpSb.safeMemcpy(currentPos, lineEndPos - currentPos);
tmpSb.pushChar(';');
}
lineStartPos = lineEndPos + 1;
}
// we don't need to care about the last line (we always end on \n)
if (tmpSb.length() > 0) {
sb->safeStrcpy("Cookie: ");
sb->safeMemcpy(&tmpSb);
sb->safeStrcpy("\r\n");
}
return true;
}
