// draw a HORIZONTAL line in html
void Statsdb::drawLine3 ( SafeBuf &sb ,
long x1 ,
long x2 ,
long fy1 ,
long color ,
long width ) {
// do not draw repeats in the case we have a ton of points to plot
long key32 ;
key32 = hash32h ( x1 , 0 );
key32 = hash32h ( x2 , key32);
key32 = hash32h ( fy1 , key32);
key32 = hash32h ( color , key32);
key32 = hash32h ( width , key32);
if ( m_dupTable.isInTable(&key32) ) return;
m_dupTable.addKey(&key32);
sb.safePrintf("<div style=\"position:absolute;"
"left:%li;"
"bottom:%li;"
"background-color:#%lx;"
"z-index:-5;"
"min-height:%lipx;"
"min-width:%lipx;\"></div>\n"
, x1 + m_bx
, (fy1 - width/2) + m_by
, color
, width
, x2 - x1
);
}
// draw a HORIZONTAL line in html
void Statsdb::drawLine3 ( SafeBuf &sb ,
int32_t x1 ,
int32_t x2 ,
int32_t fy1 ,
int32_t color ,
int32_t width ) {
// do not draw repeats in the case we have a ton of points to plot
int32_t key32 ;
key32 = hash32h ( x1 , 0 );
key32 = hash32h ( x2 , key32);
key32 = hash32h ( fy1 , key32);
key32 = hash32h ( color , key32);
key32 = hash32h ( width , key32);
if ( m_dupTable.isInTable(&key32) ) return;
m_dupTable.addKey(&key32);
sb.safePrintf("<div style=\"position:absolute;"
"left:%" PRId32";"
"bottom:%" PRId32";"
"background-color:#%" PRIx32";"
"z-index:-5;"
"min-height:%" PRId32"px;"
"min-width:%" PRId32"px;\""
" class=\"color-%" PRIx32"\"></div>\n"
, x1 + m_bx
, (fy1 - width/2) + m_by
, color
, width
, x2 - x1
, color
);
}
iconv_t gbiconv_open( char *tocode, char *fromcode) {
// get hash for to/from
uint32_t hash1 = hash32Lower_a(tocode, gbstrlen(tocode), 0);
uint32_t hash2 = hash32Lower_a(fromcode, gbstrlen(fromcode),0);
uint32_t hash = hash32h(hash1, hash2);
g_errno = 0;
iconv_t *convp = (iconv_t *)s_convTable.getValue(&hash);
iconv_t conv = NULL;
if ( convp ) conv = *convp;
//log(LOG_DEBUG, "uni: convertor %s -> %s from hash 0x%"XINT32": 0x%"XINT32"",
// fromcode, tocode,
// hash, conv);
if (!conv){
//log(LOG_DEBUG, "uni: Allocating new convertor for "
// "%s to %s (hash: 0x%"XINT32")",
// fromcode, tocode,hash);
conv = iconv_open(tocode, fromcode);
if (conv == (iconv_t) -1) {
log(LOG_WARN, "uni: failed to open converter for "
"%s to %s: %s (%d)", fromcode, tocode,
strerror(errno), errno);
// need to stop if necessary converters don't open
//char *xx=NULL; *xx = 0;
g_errno = errno;
if (errno == EINVAL)
g_errno = EBADCHARSET;
return conv;
}
// add mem to table to keep track
g_mem.addMem((void*)conv, 52, "iconv", 1);
// cache convertor
s_convTable.addKey(&hash, &conv);
//log(LOG_DEBUG, "uni: Saved convertor 0x%"INT32" under hash 0x%"XINT32"",
// conv, hash);
}
else{
// reset convertor
char *dummy = NULL;
size_t dummy2 = 0;
// JAB: warning abatement
//size_t res = iconv(conv,NULL,NULL,&dummy,&dummy2);
iconv(conv,NULL,NULL,&dummy,&dummy2);
}
return conv;
}
u_int96_t hash96 ( int32_t h1 , u_int96_t h2 ) {
h2.n0 = hash64 ( h1 , h2.n0 );
h2.n1 = hash32h ( h1 , h2.n1 );
return h2;
}
u_int96_t hash96 ( u_int96_t h1 , u_int96_t h2 ) {
h1.n0 = hash64 ( h1.n0 , h2.n0 );
h1.n1 = hash32h ( h1.n1 , h2.n1 );
return h1;
}
// returns false if blocked, true otherwise
bool Statsdb::gifLoop ( ) {
// shortcut
Msg5 *m = &m_msg5;
//#ifndef _USEPLOTTER_
//return true;
//#endif
// loop over all the lists in the time range, [m_t1,m_t2]
for ( ; ! m_done ; ) {
if ( ! m->getList ( (char)RDB_STATSDB ,
"statsdb" , // coll
&m_list ,
(char *)&m_startKey ,
(char *)&m_endKey ,
32000 , // requested scan size
true , // include tree?
false , // add to cache?
0 , // max cache age
0 , // start file number
-1 , // number of files
NULL , // state
gotListWrapper, // callback
m_niceness , // niceness
false , // do error correction?
NULL , // cache key pointer
0 , // # retries
-1 , // max # retries
true , // compensate for merge?
-1 , // sync point
NULL ) ) // msg5b
return false;
// . process list
// . returns false with g_errno set on error
if ( ! processList() ) return true;
}
// define time delta - commented out because it's currently not used.
long dt = m_t2 - m_t1;
//#ifdef _USEPLOTTER_
// gif size
//char tmp[64];
// dimensions of the gif
//sprintf ( tmp , "%lix%li", (long)DX+m_bx*2 , (long)DY+m_by*2 );
//GIFPlotter::parampl ( "BITMAPSIZE" , (void *)tmp );
// create one
//GIFPlotter plotter ( NULL , m_fd , NULL );
// open it
//plotter.openpl ( );
// define the space with boundaries 100 unit wide boundaries
//plotter.space ( 0 , 0 , DX + m_bx * 2 , DY + m_by * 2 );
// line thickness in user coordinates (pixels for us)
//plotter.linewidth ( 1 );
// set bg color to gray (r/g/b)
//plotter.bgcolor ( 0xd600 , 0xce00 , 0xd600 );
// erase Plotter's graphics display
//plotter.erase ();
// draw axises in black
//plotter.pencolorname ("black");
//
// main graphing window
//
m_gw.safePrintf("<div style=\"position:relative;"
"background-color:#c0c0c0;"
//"overflow-y:hidden;"
"overflow-x:hidden;"
"z-index:-10;"
// the tick marks we print below are based on it
// being a window of the last 20 seconds... and using
// DX pixels
"min-width:%lipx;"
"min-height:%lipx;"
//"width:100%%;"
//"min-height:600px;"
"margin-top:10px;"
"margin-bottom:10px;"
"margin-right:10px;"
"margin-left:10px;\">"
,(long)DX + 2 *m_bx
,(long)DY + 2*m_by);
// draw the x-axis
//plotter.line ( m_bx , m_by , DX + m_bx , m_by );
// 10 x-axis tick marks
for ( int x = DX/20 ; x <= DX ; x += DX/20 ) {
// tick mark
//plotter.line ( x , -20 , x , 20 );
m_gw.safePrintf("<div style=\"position:absolute;"
"left:%li;"
"bottom:0;"
"background-color:#000000;"
"z-index:110;"
"min-height:20px;"
"min-width:3px;\"></div>\n"
, m_bx + (long)x-1
);
long xv = (long)(dt * (long long)x/(long long)DX)-(long)dt;
// LABEL
m_gw.safePrintf("<div style=\"position:absolute;"
"left:%li;"
"bottom:20;"
//"background-color:#000000;"
"z-index:110;"
"min-height:20px;"
"min-width:3px;\">%lis</div>\n"
, (long)x-10 + m_bx
// the label:
, xv
);
}
HashTableX tmpht;
tmpht.set(4,0,0,NULL,0,false,m_niceness,"statsparms");
long col = 0;
m_sb2->safePrintf("<table border=1 width=100%%>\n");
// label offset to prevent collisions of superimposing multiple
// graph calbrations
long zoff = 0;
//
// point to the triplets in m_sb1's buffer (x,y,c)
//
char *p = m_sb1.getBufStart();
char *pend = p + m_sb1.length();
for ( ; p < pend ; p += 12 ) {
// breathe
QUICKPOLL ( m_niceness );
// get graph hash of this point
long gh = *(long *)(p +8);
// if we already did this graph, skip it
if ( tmpht.isInTable ( &gh ) ) continue;
// . graph this single graph of this color
// . returns ptr to first point of different color!
plotGraph ( p , pend , gh , m_gw , zoff );
// prevent collisions
zoff += 20;
// get the label based on graphHash
Label *bb = getLabel ( gh );
// add to key
if ( col == 0 )
m_sb2->safePrintf("<tr>");
m_sb2->safePrintf("<td bgcolor=#%06lx> </td>"
"<td>%s</td>\n",
bb->m_color ,
bb->m_keyDesc );
if ( col == 1 )
m_sb2->safePrintf("</tr>\n");
// inc column and wrap
if ( ++col >= 2 ) col = 0;
// . do not re-display
// . TODO: deal with error
tmpht.addKey ( &gh );
}
// clear that up
m_sb1.reset();
// now plot the events, horizontal line segments like the performance
// graph uses
for ( long i = 0 ; i < m_ht3.m_numSlots ; i++ ) {
// breathe
QUICKPOLL ( m_niceness );
// skip if slot empty
if ( ! m_ht3.m_flags[i] ) continue;
// get the offset into m_sb3
long offset = *(long *)m_ht3.getValueFromSlot(i);
// get buf start
char *bufStart = m_sb3.getBufStart();
// get the ptr
EventPoint *pp = (EventPoint *)(bufStart + offset);
// get name of parm
Parm *m = g_parms.getParmFromParmHash ( pp->m_parmHash );
// make sure we got it
if ( ! m ) {
log("statsdb: unrecognized parm hash = %li",
pp->m_parmHash);
continue;
//char *xx=NULL;*xx=0; }
}
// set the line width
//plotter.linewidth ( pp->m_thickness );
// get parm hash
long colorHash = pp->m_parmHash;
// add in old/new values to make it different
colorHash = hash32h ( (long)pp->m_oldVal , colorHash );
colorHash = hash32h ( (long)pp->m_newVal , colorHash );
// . get color
// . is really the parm hash in disguise
long c1 = colorHash & 0x00ffffff;
// use the color specified from addStat_r() for this line/pt
//plotter.pencolor ( ((c1 >> 16) & 0xff) << 8 ,
// ((c1 >> 8) & 0xff) << 8 ,
// ((c1 >> 0) & 0xff) << 8 );
long x1 = pp->m_a;
long x2 = pp->m_b;
long y1 = *(long *)m_ht3.getKey(i); // i value
// ensure at least 3 units wide for visibility
if ( x2 < x1 + 10 ) x2 = x1 + 10;
// . flip the y so we don't have to scroll the browser down
// . DY does not include the axis and tick marks
//long fy1 = DY - y1 + m_by ;
// plot it
//plotter.line ( x1 , fy1 , x2 , fy1 );
drawLine3 ( m_gw , x1 , x2 , y1 , c1 , pp->m_thickness );
// add to map key? only if we haven't already
if ( tmpht.isInTable ( &colorHash ) ) continue;
// add it
if ( col == 0 )
m_sb2->safePrintf("<tr>");
char *title = "unknown parm";
if ( m ) title = m->m_title;
m_sb2->safePrintf("<td bgcolor=#%06lx> </td>",c1);
// print the parm name and old/new values
m_sb2->safePrintf("<td><b>%s</b>",title);
if ( pp->m_oldVal != pp->m_newVal )
m_sb2->safePrintf(" (%.02f -> %.02f)",
pp->m_oldVal,pp->m_newVal);
m_sb2->safePrintf("</td>");
if ( col == 1 )
m_sb2->safePrintf("</tr>\n");
// inc column and wrap
if ( ++col >= 2 ) col = 0;
// . do not re-display
// . TODO: deal with error
tmpht.addKey ( &colorHash ) ;
}
m_sb2->safePrintf("</table>\n");
// clear that up
m_ht3.reset();
m_sb3.reset();
// and stat states
m_ht0.reset();
m_sb0.reset();
// all done free some mem
m_sb1.reset();
//m_sb2.reset();
//
// but not m_sb2 cuz that has the html in it!!
//
// all done
//if ( plotter.closepl () < 0 )
// log("admin: Could not close performance graph object.");
// close the file
//fclose ( m_fd );
//#endif
// close main graphing window
m_gw.safePrintf("</div>\n");
return true;
}
bool Statsdb::init ( ) {
// 20 pixel borders
m_bx = 10;
m_by = 40;
// keep it at least at 20MB otherwise it is filling up the tree
// constantly and dumping
long maxTreeMem = g_conf.m_statsdbMaxTreeMem;
if ( maxTreeMem < 10000000 ) maxTreeMem = 10000000;
long nodeSize = sizeof(StatData)+8+12+4 + sizeof(collnum_t);
// for debug
//nodeSize = 50000;
// . We take a snapshot of g_stats every minute.
// . Each sample struct taken from g_stats ranges from 1k - 2k
// after compression depending on the state of the
// all errors arrays.
uint32_t maxTreeNodes = maxTreeMem / nodeSize;
uint32_t maxCacheNodes = g_conf.m_statsdbMaxCacheMem / nodeSize;
// assume low niceness
m_niceness = 0;
// init the label table
static char s_buf[576];
if ( ! m_labelTable.set(4,4,64,s_buf,576,false,0,"statcolors") )
return false;
// stock the table
long n = (long)sizeof(s_labels)/ sizeof(Label);
for ( long i = 0 ; i < n ; i++ ) {
Label *bb = &s_labels[i];
// hash the label
bb->m_labelHash = hash32n ( bb->m_label );
// then incorporate the bool parm
bb->m_graphHash = hash32h( bb->m_labelHash , bb->m_graphType );
// add it to labeltable... why???
if ( ! m_labelTable.addKey (&bb->m_graphHash,&bb ) ) {
char *xx=NULL;*xx=0; }
}
// sanity test
//Stat ts;
//ts.setKey ( 0x123456789LL , 0x7654321 );
//if ( ts.getTime1() != 0x123456789LL ) { char *xx=NULL;*xx=0; }
//if ( ts.getLabelHash() != 0x7654321 ) { char *xx=NULL;*xx=0; }
//ts.setKey ( 1268261684329LL , -246356284 );
//if ( ts.getTime1() != 1268261684329LL ) { char *xx=NULL;*xx=0; }
//if ( ts.getLabelHash() != -246356284 ) { char *xx=NULL;*xx=0; }
// call this twice per second
if ( ! g_loop.registerSleepCallback(500,NULL,flushStatsWrapper))
return log("statsdb: Failed to initialize timer callback2.");
m_init = true;
// make the rec cache 0 bytes, cuz we are just using page cache now
if ( ! m_rdb.init ( g_hostdb.m_dir , // working directory
"statsdb" , // dbname
true , // dedup keys
sizeof(StatData) , // fixed record size
200,//g_conf.m_statsdbMinFilesToMerge ,
maxTreeMem ,
maxTreeNodes ,
true , // balance tree?
0 , // m_statsdbMaxCchMem
maxCacheNodes ,
false , // use half keys?
false , // cache from disk?
NULL , // page cache pointer
false , // is titledb?
false ,
sizeof(key96_t) , // key size
false, // bias disk page cache?
true ) ) // is collectionless?
return false;
m_disabled = false;
return true;
}
void processReply ( char *reply , long replyLen ) {
// store our current reply
SafeBuf fb2;
fb2.safeMemcpy(reply,replyLen );
fb2.nullTerm();
// log that we got the reply
log("qa: got reply(len=%li)(errno=%s)=%s",
replyLen,mstrerror(g_errno),reply);
char *content = NULL;
long contentLen = 0;
// get mime
if ( reply ) {
HttpMime mime;
mime.set ( reply, replyLen , NULL );
// only hash content since mime has a timestamp in it
content = mime.getContent();
contentLen = mime.getContentLen();
if ( content && contentLen>0 && content[contentLen] ) {
char *xx=NULL;*xx=0; }
}
if ( ! content ) {
content = "";
contentLen = 0;
}
s_content = content;
// take out <responseTimeMS>
markOut ( content , "<currentTimeUTC>");
markOut ( content , "<responseTimeMS>");
// until i figure this one out, take it out
markOut ( content , "<docsInCollection>");
// until i figure this one out, take it out
markOut ( content , "<hits>");
// for those links in the html pages
markOut ( content, "rand64=");
// for json
markOut ( content , "\"currentTimeUTC\":" );
markOut ( content , "\"responseTimeMS\":");
markOut ( content , "\"docsInCollection\":");
// for xml
markOut ( content , "<currentTimeUTC>" );
markOut ( content , "<responseTimeMS>");
markOut ( content , "<docsInCollection>");
// indexed 1 day ago
markOut ( content,"indexed:");
// modified 1 day ago
markOut ( content,"modified:");
// s_gigabitCount... it is perpetually incrementing static counter
// in PageResults.cpp
markOut(content,"ccc(");
markOut(content,"id=fd");
markOut(content,"id=sd");
// for some reason the term freq seems to change a little in
// the scoring table
markOut(content,"id=tf");
// make checksum. we ignore back to back spaces so this
// hash works for <docsInCollection>10 vs <docsInCollection>9
long contentCRC = 0;
if ( content ) contentCRC = qa_hash32 ( content );
// note it
log("qa: got contentCRC of %lu",contentCRC);
// if what we expected, save to disk if not there yet, then
// call s_callback() to resume the qa pipeline
/*
if ( contentCRC == s_expectedCRC ) {
// save content if good
char fn3[1024];
sprintf(fn3,"%sqa/content.%lu",g_hostdb.m_dir,contentCRC);
File ff; ff.set ( fn3 );
if ( ! ff.doesExist() ) {
// if not there yet then save it
fb2.save(fn3);
}
// . continue on with the qa process
// . which qa function that may be
//s_callback();
return;
}
*/
//
// if crc of content does not match what was expected then do a diff
// so we can see why not
//
// this means caller does not care about the response
if ( ! s_checkCRC ) {
//s_callback();
return;
}
//const char *emsg = "qa: bad contentCRC of %li should be %li "
// "\n";//"phase=%li\n";
//fprintf(stderr,emsg,contentCRC,s_expectedCRC);//,s_phase-1);
// hash url
long urlHash32 = hash32n ( s_url.getUrl() );
// combine test function too since two tests may use the same url
long nameHash = hash32n ( s_qt->m_testName );
// combine together
urlHash32 = hash32h ( nameHash , urlHash32 );
static bool s_init = false;
if ( ! s_init ) {
s_init = true;
s_ht.set(4,4,1024,NULL,0,false,0,"qaht");
// make symlink
//char cmd[512];
//snprintf(cmd,"cd %s/html ;ln -s ../qa ./qa", g_hostdb.m_dir);
//system(cmd);
char dir[1024];
snprintf(dir,1000,"%sqa",g_hostdb.m_dir);
long status = ::mkdir ( dir ,
S_IRUSR | S_IWUSR | S_IXUSR |
S_IRGRP | S_IWGRP | S_IXGRP |
S_IROTH | S_IXOTH );
if ( status == -1 && errno != EEXIST && errno )
log("qa: Failed to make directory %s: %s.",
dir,mstrerror(errno));
// try to load from disk
SafeBuf fn;
fn.safePrintf("%s/qa/",g_hostdb.m_dir);
log("qa: loading crctable.dat");
s_ht.load ( fn.getBufStart() , "crctable.dat" );
}
// break up into lines
char fn2[1024];
sprintf(fn2,"%sqa/content.%lu",g_hostdb.m_dir,contentCRC);
fb2.save ( fn2 );
// look up in hashtable to see what reply crc should be
long *val = (long *)s_ht.getValue ( &urlHash32 );
// just return if the same
if ( val && contentCRC == *val ) {
g_qaOutput.safePrintf("<b style=color:green;>"
"passed test</b><br>%s : "
"<a href=%s>%s</a> (urlhash=%lu "
"crc=<a href=/qa/content.%lu>"
"%lu</a>)<br>"
"<hr>",
s_qt->m_testName,
s_url.getUrl(),
s_url.getUrl(),
urlHash32,
contentCRC,
contentCRC);
return;
}
if ( ! val ) {
// add it so we know
s_ht.addKey ( &urlHash32 , &contentCRC );
g_qaOutput.safePrintf("<b style=color:blue;>"
"first time testing</b><br>%s : "
"<a href=%s>%s</a> "
"(urlhash=%lu "
"crc=<a href=/qa/content.%lu>%lu"
"</a>)<br>"
"<hr>",
s_qt->m_testName,
s_url.getUrl(),
s_url.getUrl(),
urlHash32,
contentCRC,
contentCRC);
return;
}
log("qa: crc changed for url %s from %li to %li",
s_url.getUrl(),*val,contentCRC);
// get response on file
SafeBuf fb1;
char fn1[1024];
sprintf(fn1,"%sqa/content.%lu",g_hostdb.m_dir, *val);
fb1.load(fn1);
fb1.nullTerm();
// do the diff between the two replies so we can see what changed
char cmd[1024];
sprintf(cmd,"diff %s %s > /tmp/diffout",fn1,fn2);
log("qa: %s\n",cmd);
system(cmd);
g_numErrors++;
g_qaOutput.safePrintf("<b style=color:red;>FAILED TEST</b><br>%s : "
"<a href=%s>%s</a> (urlhash=%lu)<br>"
"<input type=checkbox name=urlhash%lu value=1 "
// use ajax to update test crc. if you undo your
// check then it should put the old val back.
// when you first click the checkbox it should
// gray out the diff i guess.
"onclick=submitchanges(%lu,%lu);> "
"Accept changes"
"<br>"
"original on left, new on right. "
"oldcrc = <a href=/qa/content.%lu>%lu</a>"
" != <a href=/qa/content.%lu>%lu</a> = newcrc"
"<br>diff output follows:<br>"
"<pre id=%lu style=background-color:0xffffff;>",
s_qt->m_testName,
s_url.getUrl(),
s_url.getUrl(),
urlHash32,
// input checkbox name field
urlHash32,
// submitchanges() parms
urlHash32,
contentCRC,
// original/old content.%lu
*val,
*val,
// new content.%lu
contentCRC,
contentCRC,
// for the pre tag id:
urlHash32);
// store in output
SafeBuf sb;
sb.load("/tmp/diffout");
g_qaOutput.htmlEncode ( sb.getBufStart() );
g_qaOutput.safePrintf("</pre><br><hr>");
// if this is zero allow it to slide by. it is learning mode i guess.
// so we can learn what crc we need to use.
// otherwise, stop right there for debugging
//if ( s_expectedCRC != 0 ) exit(1);
// keep on going
//s_callback();
}