void BaseApp::updateTime(){
timestamp prev = curr;
curr = getCurrentTime();
frameTime = getTimeDifference(prev, curr);
time = getTimeDifference(start, curr);
}
void Main::copierProgress(Copier* copier) {
struct timeval now;
uint64_t timeSpent;
uint64_t sinceStart;
double throughput;
gettimeofday(&now, NULL);
timeSpent = getTimeDifference(&this->lastTime, &now);
if (timeSpent > 500000) {
sinceStart = getTimeDifference(&this->startTime, &now);
throughput = (copier->getCopied() / (sinceStart / 1000000.0));
fprintf(stderr, "%llu bytes total, %llu seconds, %s/s%c[K\r", copier->getCopied(), sinceStart / 1000000, this->formatter->format("%.1f %s", throughput), 27);
this->lastTime = now;
}
}
int cmc_groupsforuserfromusersystem(int socketha, char *_user, unsigned int us, char ***_groups, char *extra_in) {
int n;
char **groups;
void *all, *p;
#ifdef DEBUG_TIME
struct timeval ts, te;
#endif
int allsize = 512 + 512 + sizeof(us);
if ((all = malloc(allsize)) == NULL) {
perror("cmc_groupsforuserfromusersystem; malloc all buf");
return 0;
}
p = all;
// kopierer all dataen i en pakke
strncpy(p, _user, 512);
p += 512;
memcpy(p, &us, sizeof(us));
p += sizeof(us);
strncpy(p, extra_in, 512);
p += 512;
sendpacked(socketha, cm_groupsforuserfromusersystem, BLDPROTOCOLVERSION, allsize, all, "");
free(all);
if (recv(socketha, &n, sizeof n, MSG_WAITALL) == -1) {
perror("groupsbyuserforcoll recv");
return 0;
}
#ifdef DEBUG_TIME
gettimeofday(&ts, NULL);
#endif
if (n > 0) {
groups = calloc(n, MAX_LDAP_ATTR_LEN);
if (recv(socketha, groups, n * MAX_LDAP_ATTR_LEN, MSG_WAITALL) == -1) {
perror("groupsbyuserforcoll recv");
return 0;
}
*_groups = groups;
} else {
*_groups = NULL;
}
#ifdef DEBUG_TIME
gettimeofday(&te, NULL);
printf("grepme cmc_groupsforuserfromusersystem took: %f\n", getTimeDifference(&ts, &te));
#endif
return n;
}
/*! This method returns the time (in seconds) since the last time the timer
was restarted.
\return double seconds that have passed since last restart of timer */
double Timing::getElapsedTime( int iFactor )
{
#ifdef WIN32
DWORD time2 = GetTickCount();
#else
struct timeval time2;
gettimeofday( &time2, NULL );
#endif
return getTimeDifference( time2, time1 )*iFactor;
}
void WorldDrawer3d::idleCallbackFunction(){
unsigned int diff = getTimeDifference();
// Limit to 50 frames per second
if (diff >= 20)
{
tick += diff;
onIdle();
//redisplay
glutPostRedisplay();
}
}
int cmc_rewrite_url(int socketha, char *collection_in, const char *url_in, enum platform_type ptype,
enum browser_type btype, char *url_out, size_t url_out_len, char *uri_out, size_t uri_out_len,
char *fulluri_out, size_t fulluri_out_len)
{
struct rewriteFormat rewrite;
#ifdef DEBUG_TIME
struct timeval start_time, end_time;
printf("cmc_rewrite_url(collection_in=\"%s\", url_in=\"%s\")\n",collection_in,url_in);
gettimeofday(&start_time, NULL);
#endif
memset(&rewrite, '\0', sizeof(rewrite));
strscpy(rewrite.collection, collection_in, sizeof(rewrite.collection));
strscpy(rewrite.url, url_in, sizeof(rewrite.url));
rewrite.ptype = ptype;
rewrite.btype = btype;
sendpacked(socketha, cm_rewriteurl, BLDPROTOCOLVERSION, sizeof(rewrite), &rewrite, "");
if (recvall(socketha, &rewrite, sizeof(rewrite)) == 0) {
perror("recvall(url)");
return 0;
}
strscpy(url_out, rewrite.url, url_out_len);
strscpy(uri_out, rewrite.uri, uri_out_len);
strscpy(fulluri_out, rewrite.fulluri, fulluri_out_len);
#ifdef DEBUG
printf("~cmc_rewrite_url [uri_out=\"%s\", fulluri=\"%s\"]\n",uri_out,fulluri_out);
#endif
#ifdef DEBUG_TIME
gettimeofday(&end_time, NULL);
printf("~cmc_rewrite_url(2) time %f\n",getTimeDifference(&start_time,&end_time));
#endif
return 1;
}
void
reclose_cache(void)
{
struct reformat *re;
struct hashtable_itr *itr;
char *filesname;
#ifdef DEBUG_TIME
struct timeval start_time, end_time;
gettimeofday(&start_time, NULL);
#endif
//itererer over hash, frigjør alle elementer
if (lots_cache!= NULL && hashtable_count(lots_cache) > 0)
{
itr = hashtable_iterator(lots_cache);
do {
filesname = hashtable_iterator_key(itr);
re = hashtable_iterator_value(itr);
#ifdef DEBUG
printf("reclose_cache: closing \"%s\"\n",filesname);
#endif
if (re != NULL) {
reclose(re);
}
} while (hashtable_iterator_remove(itr));
free(itr);
}
#ifdef DEBUG_TIME
gettimeofday(&end_time, NULL);
printf("Time debug: reclose_cache %f\n",getTimeDifference(&start_time,&end_time));
#endif
return;
}
void brGetPages(int *sockfd,int nrOfServers,struct SiderHederFormat *SiderHeder,struct SiderFormat *Sider,
int *pageNr,int alreadynr) {
int i,n;
int net_status;
#ifdef DEBUG_TIME
struct timeval start_time, end_time;
gettimeofday(&start_time, NULL);
#endif
#ifdef DEBUG
printf("brGetPages: alreadynr %i, *pageNr %i, nrOfServers %i\n",alreadynr,*pageNr,nrOfServers);
#endif
//sejjer om vi har fåt et midlertidig svar på at jobben har begynt
/****************************************************************/
for (i=alreadynr;i<nrOfServers+alreadynr;i++) {
if (sockfd[i] != 0) {
//motter hedderen for svaret
if (bsread (&sockfd[i],sizeof(net_status),(char *)&net_status,maxSocketWait_CanDo)) {
if (net_status != net_CanDo) {
dprintf("net_status wasen net_CanDo but %i\n",net_status);
sockfd[i] = 0;
}
}
}
}
/****************************************************************/
#ifdef DEBUG_TIME
gettimeofday(&end_time, NULL);
dprintf("Time debug: brGetPages.jobstart pages %f\n",getTimeDifference(&start_time,&end_time));
gettimeofday(&start_time, NULL);
#endif
for (i=alreadynr;i<nrOfServers+alreadynr;i++) {
//inaliserer til 0 slik at vi ikke tror at vi har noe data fra servere som ikke svarte
SiderHeder[i].showabal = 0;
//motter hedderen for svaret
if (sockfd[i] != 0) {
if (!bsread (&sockfd[i],sizeof(struct SiderHederFormat),(char *)&SiderHeder[i],maxSocketWait_SiderHeder)) {
dprintf("brGetPages: Failed to bsread SiderHeder");
continue; // ingen vits i å fortsette å lese fra denne hvis vi ikke fik til å gjøre dette read kallet
}
#ifdef ATTRIBUTES
if ((SiderHeder[i].navigation_xml = malloc( sizeof(char) * (SiderHeder[i].navigation_xml_len +1)) ) == NULL) {
perror("Can't malloc data for navigation_xml");
continue; // ingen vits å å fortsette mer
}
if (!bsread (&sockfd[i], SiderHeder[i].navigation_xml_len, SiderHeder[i].navigation_xml, maxSocketWait_SiderHeder)) {
dprintf("brGetPages: Failed to bsread navigation_xml");
continue; // ingen vits å å fortsette mer
}
SiderHeder[i].navigation_xml[SiderHeder[i].navigation_xml_len] = '\0';
#ifdef DEBUG
printf("\n\nnavigation_xml: %s\n\n", SiderHeder[i].navigation_xml);
#endif
#endif
}
}
#ifdef DEBUG_TIME
gettimeofday(&end_time, NULL);
dprintf("Time debug: brGetPages.reading heder pages %f\n",getTimeDifference(&start_time,&end_time));
gettimeofday(&start_time, NULL);
#endif
for (i=alreadynr;i<nrOfServers+alreadynr;i++) {
if (sockfd[i] == 0) {
dprintf("Server nr %i don't have a open socket.",i);
}
else {
int j;
dprintf("aa: i: %i. Server \"%s\" that has %i pages. Soctet %i\n",
i,
SiderHeder[i].servername,
SiderHeder[i].showabal,
sockfd[i]);
#ifdef DEBUG
printf("brGetPages: trying to read %i bytes from server %i\n",sizeof(struct SiderFormat) * SiderHeder[i].showabal,i);
printf("have %i pages\n",SiderHeder[i].showabal);
#endif
for (j = 0; j < SiderHeder[i].showabal; j++) {
if ((n=bsread(&sockfd[i],sizeof(struct SiderFormat), (char *)&Sider[(*pageNr)],maxSocketWait_SiderHeder))) {
size_t len;
int k;
/* Get urls ... */
Sider[*pageNr].urls = calloc(Sider[*pageNr].n_urls, sizeof(*(Sider->urls)));
if (Sider[*pageNr].url == NULL)
err(1, "calloc(urls)");
for (k = 0; k < Sider[*pageNr].n_urls; k++) {
bsread(&sockfd[i], sizeof(len), (char *)&len, maxSocketWait_SiderHeder);
Sider[*pageNr].urls[k].url = malloc(len+1);
bsread(&sockfd[i], len, Sider[*pageNr].urls[k].url, maxSocketWait_SiderHeder);
Sider[*pageNr].urls[k].url[len] = '\0';
bsread(&sockfd[i], sizeof(len), (char *)&len, maxSocketWait_SiderHeder);
Sider[*pageNr].urls[k].uri = malloc(len+1);
bsread(&sockfd[i], len, Sider[*pageNr].urls[k].uri, maxSocketWait_SiderHeder);
Sider[*pageNr].urls[k].uri[len] = '\0';
bsread(&sockfd[i], sizeof(len), (char *)&len, maxSocketWait_SiderHeder);
Sider[*pageNr].urls[k].fulluri = malloc(len+1);
bsread(&sockfd[i], len, Sider[*pageNr].urls[k].fulluri, maxSocketWait_SiderHeder);
Sider[*pageNr].urls[k].fulluri[len] = '\0';
#ifdef DEBUG
printf("n_urls=%i\n", Sider[*pageNr].n_urls);
printf("url=\"%s\", uri=\"%s\", fulluri=\"%s\"\n", Sider[*pageNr].urls[k].url, Sider[*pageNr].urls[k].uri, Sider[*pageNr].urls[k].fulluri);
#endif
}
bsread(&sockfd[i], sizeof(len), (char *)&len, maxSocketWait_SiderHeder);
Sider[*pageNr].attributelen = len;
Sider[*pageNr].attributes = malloc(len+1);
bsread(&sockfd[i], len, Sider[*pageNr].attributes, maxSocketWait_SiderHeder);
Sider[*pageNr].attributes[len] = '\0';
(*pageNr) += 1;
}
}
#ifdef DEBUG
printf("brGetPages: did read %i element\n",n);
#endif
}
}
#ifdef DEBUG_TIME
gettimeofday(&end_time, NULL);
dprintf("Time debug: brGetPages.reading pages %f\n",getTimeDifference(&start_time,&end_time));
#endif
}
void connectHandler(int socket) {
struct packedHedderFormat packedHedder;
int isAuthenticated = 0;
char tkeyForTest[32];
int i,n;
int intrespons;
int count = 0;
container *attrkeys = NULL;
#ifdef DEBUG_TIME
struct timeval start_time, end_time;
struct timeval tot_start_time, tot_end_time;
gettimeofday(&tot_start_time, NULL);
#endif
ionice_benice();
while ((i=recv(socket, &packedHedder, sizeof(struct packedHedderFormat),MSG_WAITALL)) > 0) {
#ifdef DEBUG
printf("size is: %i\nversion: %i\ncommand: %i\n",packedHedder.size,packedHedder.version,packedHedder.command);
#endif
packedHedder.size = packedHedder.size - sizeof(packedHedder);
if (attrkeys == NULL) {
attrkeys = ropen();
}
if (packedHedder.command == bbc_askToAuthenticate) {
if ((i=recv(socket, tkeyForTest, sizeof(tkeyForTest),MSG_WAITALL)) == -1) {
perror("Cant read tkeyForTest");
exit(1);
}
if (1) {
printf("authenticated\n");
intrespons = bbc_authenticate_ok;
bbdocument_init(NULL);
isAuthenticated = 1;
}
else {
printf("authenticate faild\n");
intrespons = bbc_authenticate_feiled;
}
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
}
else {
if (!isAuthenticated) {
printf("user not autentikated\n");
exit(1);
}
if (packedHedder.command == bbc_docadd) {
#ifdef DEBUG
printf("bbc_docadd\n");
#endif
char *subname,*documenturi,*documenttype,*document,*acl_allow,*acl_denied,*title,*doctype;
char *attributes;
int dokument_size;
unsigned int lastmodified;
#ifdef DEBUG_TIME
gettimeofday(&start_time, NULL);
#endif
//subname
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
subname = malloc(intrespons +1);
if ((i=recvall(socket, subname, intrespons)) == 0) {
perror("Cant read subname");
exit(1);
}
//documenturi
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
documenturi = malloc(intrespons +1);
if ((i=recvall(socket, documenturi, intrespons)) == 0) {
perror("Cant read documenturi");
exit(1);
}
//documenttype
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
documenttype = malloc(intrespons +1);
if ((i=recvall(socket, documenttype, intrespons)) == 0) {
perror("Cant read documenttype");
exit(1);
}
//document
//dokument_size
if ((i=recvall(socket, &dokument_size, sizeof(dokument_size))) == 0) {
perror("Cant read dokument_size");
exit(1);
}
document = malloc(dokument_size +1);
if (dokument_size == 0) {
document[0] = '\0';
}
else {
if ((i=recvall(socket, document, dokument_size)) == 0) {
fprintf(stderr,"Can't read document of size %i\n",dokument_size);
perror("recvall");
exit(1);
}
}
//lastmodified
if ((i=recvall(socket, &lastmodified, sizeof(lastmodified))) == 0) {
perror("Cant read lastmodified");
exit(1);
}
//acl_allow
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
acl_allow = malloc(intrespons +1);
if ((i=recvall(socket, acl_allow, intrespons)) == 0) {
perror("Cant read acl_allow");
exit(1);
}
//acl_denied
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
acl_denied = malloc(intrespons +1);
if ((i=recvall(socket, acl_denied, intrespons)) == 0) {
perror("Cant read acl_denied");
exit(1);
}
//title
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
title = malloc(intrespons +1);
if ((i=recvall(socket, title, intrespons)) == 0) {
perror("Cant read title");
exit(1);
}
//doctype
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
doctype = malloc(intrespons +1);
if ((i=recvall(socket, doctype, intrespons)) == 0) {
perror("Cant read doctype");
exit(1);
}
// Attribute list
if ((i = recvall(socket, &intrespons, sizeof(intrespons))) == 0)
err(1, "Can't receive attribute list len");
attributes = malloc(intrespons +1);
if ((i=recvall(socket, attributes, intrespons)) == 0)
err(1, "Can't receive attribute list");
#ifdef DEBUG_TIME
gettimeofday(&end_time, NULL);
printf("Time debug: bbdn_docadd recv data time: %f\n",getTimeDifference(&start_time, &end_time));
#endif
printf("\n");
printf("########################################################\n");
printf("Url: %s\n",documenturi);
printf("got subname \"%s\": title \"%s\". Nr %i, dokument_size %i attrib: %s\n",subname,title,count,dokument_size, attributes);
printf("########################################################\n");
printf("calling bbdocument_add():\n");
#ifdef DEBUG_TIME
gettimeofday(&start_time, NULL);
#endif
intrespons = bbdocument_add(subname,documenturi,documenttype,document,dokument_size,lastmodified,acl_allow,acl_denied,title,doctype, attributes, attrkeys);
printf(":bbdocument_add end\n");
printf("########################################################\n");
#ifdef DEBUG_TIME
gettimeofday(&end_time, NULL);
printf("Time debug: bbdn_docadd runing bbdocument_add() time: %f\n",getTimeDifference(&start_time, &end_time));
#endif
free(subname);
free(documenturi);
free(documenttype);
free(document);
free(acl_allow);
free(acl_denied);
free(title);
free(doctype);
free(attributes);
// send status
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
}
else if (packedHedder.command == bbc_opencollection) {
char *subname;
char path[PATH_MAX];
printf("open collection\n");
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1)
err(1, "Cant read intrespons");
subname = malloc(intrespons +1);
if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1)
err(1, "Cant read subname");
GetFilPathForLot(path, 1, subname);
strcat(path, "fullyCrawled");
unlink(path);
free(subname);
}
else if (packedHedder.command == bbc_closecollection) {
printf("closecollection\n");
char *subname;
//subname
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) {
perror("Cant read intrespons");
exit(1);
}
subname = malloc(intrespons +1);
if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1) {
perror("Cant read subname");
exit(1);
}
bbdocument_close(attrkeys);
attrkeys = NULL;
//toDo må bruke subname, og C ikke perl her
printf("cleanin lots start\n");
char command[PATH_MAX];
snprintf(command,sizeof(command),"perl %s -l -s \"%s\"",bfile("perl/cleanLots.pl"),subname);
printf("running \"%s\"\n",command);
intrespons = system(command);
printf("cleanin lots end\n");
// legger subnamet til listen over ventene subnavn, og huper searchd.
lot_recache_collection(subname);
/* We are done crawling */
{
int fd = lotOpenFileNoCasheByLotNrl(1, "fullyCrawled", ">>", '\0', subname);
if (fd == -1) {
warn("Unable to write fullyCrawled file");
} else {
close(fd);
}
}
free(subname);
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
}
else if (packedHedder.command == bbc_deleteuri) {
printf("deleteuri\n");
char *subname, *uri;
//subname
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) {
perror("Cant read intrespons");
exit(1);
}
subname = malloc(intrespons +1);
if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1) {
perror("Cant read subname");
exit(1);
}
subname[intrespons] = '\0';
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) {
perror("Cant read intrespons");
exit(1);
}
uri = malloc(intrespons +1);
if ((i=recv(socket, uri, intrespons,MSG_WAITALL)) == -1) {
perror("Cant read uri");
exit(1);
}
uri[intrespons] = '\0';
printf("going to delete: %s from %s\n", uri, subname);
/* Add docid to the gced file */
{
FILE *fh;
unsigned int DocID, lastmodified;
unsigned int lotNr;
int err = 0;
if (uriindex_get(uri, &DocID, &lastmodified, subname) == 0) {
fprintf(stderr,"Unable to get uri info. uri=\"%s\",subname=\"%s\".",uri,subname);
perror("Unable to get uri info");
err++;
}
if (!err) {
lotNr = rLotForDOCid(DocID);
if ((fh = lotOpenFileNoCasheByLotNr(lotNr,"gced","a", 'e',subname)) == NULL) {
perror("can't open gced file");
err++;
} else {
fwrite(&DocID, sizeof(DocID), 1, fh);
fclose(fh);
}
}
if (!err) {
struct reformat *re;
if((re = reopen(rLotForDOCid(DocID), sizeof(struct DocumentIndexFormat), "DocumentIndex", subname, RE_HAVE_4_BYTES_VERSION_PREFIX)) == NULL) {
perror("can't reopen()");
err++;
} else {
DIS_delete(RE_DocumentIndex(re, DocID));
reclose(re);
}
}
//markerer at den er skitten
if (!err) {
FILE *dirtfh;
dirtfh = lotOpenFileNoCashe(DocID,"dirty","ab",'e',subname);
fwrite("1",1,1,dirtfh);
fclose(dirtfh);
}
if (err == 0)
bbdocument_delete(uri, subname);
}
free(subname);
intrespons = 1; // Always return ok for now
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
}
else if (packedHedder.command == bbc_deletecollection) {
printf("deletecollection\n");
char *subname, *uri;
//subname
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) {
perror("Cant read intrespons");
exit(1);
}
subname = malloc(intrespons +1);
if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1) {
perror("Cant read subname");
exit(1);
}
subname[intrespons] = '\0';
printf("going to delete collection: %s\n", subname);
intrespons = bbdocument_deletecoll(subname);
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
free(subname);
}
else if (packedHedder.command == bbc_addwhisper) {
whisper_t add;
char *subname;
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1)
err(1, "Cant read intrespons");
subname = malloc(intrespons+1);
if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1) {
perror("Cant read subname");
exit(1);
}
subname[intrespons] = '\0';
if ((i=recv(socket, &add, sizeof(add),MSG_WAITALL)) == -1)
err(1, "Cant read add whisper");
gcwhisper_write(subname, add);
free(subname);
}
else if (packedHedder.command == bbc_HasSufficientSpace) {
char *subname;
//subname
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
subname = malloc(intrespons +1);
if ((i=recvall(socket, subname, intrespons)) == 0) {
perror("Cant read subname");
exit(1);
}
// tester bare i lot 1 her. Må også sjekke andre loter når vi begynner å støtte frlere disker på ES.
intrespons = lotHasSufficientSpace(1, 4096, subname);
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
printf("~Asked for HasSufficientSpace for subname \"%s\". Returnerer %d\n",subname, intrespons);
free(subname);
}
else {
printf("unnown comand. %i\n", packedHedder.command);
}
}
++count;
// #ifdef DEBUG_BREAK_AFTER
// if (count >= DEBUG_BREAK_AFTER) {
// printf("exeting after %i docoments\n",count);
// exit(1);
// }
// #endif
}
#ifdef DEBUG_TIME
gettimeofday(&tot_end_time, NULL);
printf("Time debug: bbdn total time time: %f\n",getTimeDifference(&tot_start_time, &tot_end_time));
#endif
}
/*
This is the routine that is executed from a new thread
*/
void *do_chld(void *arg)
{
// starter loging
bblog(DEBUGINFO, "searchd_child: Starting new thread.");
bblog(DEBUGINFO, "searchd: do_chld()");
// deklarerer variabler
struct searchd_configFORMAT *searchd_config = arg;
int mysocfd = (*searchd_config).newsockfd;
struct timeval globalstart_time, globalend_time;
FILE *LOGFILE;
int i,n;
struct queryNodeHederFormat queryNodeHeder;
struct SiderFormat *Sider;
int net_status;
int ranking;
struct hashtable *crc32hashmap;
int nrOfSubnames;
config_setting_t *cfgstring;
config_setting_t *cfgcollection;
config_setting_t *cfgcollections;
struct subnamesConfigFormat subnamesDefaultsConfig;
char **Data;
int Count;
char groupOrQuery[1024];
groupOrQuery[0] = '\0';
struct SiderHederFormat *SiderHeder;
// Setter signalhånterer for allarm. Hvis allarm skjer i straten av programet, er noe seriøst galt. For eks er vi tom for minne.
//Da fungerer det dårlig å logge, syslog kan kalle malloc og slikt. Vil resette den til en versjon som logger før vi kjører søk.
signal (SIGALRM, catch_alarm_nolog);
/* Set an alarm to go off in a little while. This so we don't run forever if we get en ever loop */
alarm (searchd_config->optAlarm);
dp_priority_locl_start();
// første malloc. Har vi forlite minne vil vi henge her.
if ((SiderHeder = malloc(sizeof(struct SiderHederFormat))) == NULL) {
bblog(ERROR, "malloc()");
bblog(CLEAN, "~do_chld()");
return 0;
}
gettimeofday(&globalstart_time, NULL);
#ifdef WITH_THREAD
bblog(DEBUGINFO, "Child thread [%d]: Socket number = %d", pthread_self(), mysocfd);
#else
bblog(DEBUGINFO, "Socket number = %d",mysocfd);
#endif
#ifdef DEBUG
struct timeval start_time, end_time;
#endif
// Setter signalhånterer for allarm. Hvis allarm skjer i straten av programet, er noe seriøst galt. For eks er vi tom for minne.
//Da fungerer det dårlig å logge, syslog kan kalle malloc og slikt. Nå resetter vi den den til en versjon som logger søk.
signal (SIGALRM, catch_alarm);
/* Disable tcp delay */
int nodelayflag = 1;
setsockopt(mysocfd, IPPROTO_TCP, TCP_NODELAY, &nodelayflag, sizeof(int));
/* read from the given socket */
if ((i=recv(mysocfd, &queryNodeHeder, sizeof(queryNodeHeder),MSG_WAITALL)) == -1) {
bblog_errno(ERROR, "recv()");
}
// Read collections /w cfg.
if ((recv(mysocfd, &nrOfSubnames, sizeof nrOfSubnames, MSG_WAITALL)) == -1)
bblog_errno(ERROR, "recv nrOfSubnames");
bblog(DEBUGINFO, "nrOfSubnames: %d",nrOfSubnames);
struct subnamesFormat subnames[nrOfSubnames];
if (nrOfSubnames > 0) {
if ((recv(mysocfd, &subnames, sizeof subnames, MSG_WAITALL)) == -1)
bblog_errno(ERROR, "recv subnames");
}
//sender svar med en gang at vi kan gjøre dette
net_status = net_CanDo;
//if ((n=sendall(mysocfd,&net_status, sizeof(net_status))) != sizeof(net_status)) {
if ((n=send(mysocfd,&net_status, sizeof(net_status),MSG_NOSIGNAL)) != sizeof(net_status)) {
bblog_errno(ERROR, "searchd_child: Warning! Sent only %i of %i bytes at %s:%d",n,sizeof(net_status),__FILE__,__LINE__);
}
bblog(DEBUGINFO, "MaxsHits %i",queryNodeHeder.MaxsHits);
//Sider = (struct SiderFormat *)malloc(sizeof(struct SiderFormat) * (queryNodeHeder.MaxsHits));
bblog(DEBUGINFO, "Ranking search?");
#ifndef BLACK_BOKS
//ToDo: må ha låsing her
if ((LOGFILE = bfopen("config/query.log","a")) == NULL) {
bblog_errno(ERROR, "logfile");
}
else {
fprintf(LOGFILE,"%s\n",queryNodeHeder.query);
fclose(LOGFILE);
}
#endif
bblog(DEBUGINFO, "searchd_child: Incoming query: %s",queryNodeHeder.query);
strcpy(SiderHeder->servername,servername);
#ifndef WITHOUT_THESAURUS
// TODO: Denne må skalere til flere språk:
char *lang = NULL;
switch (queryNodeHeder.lang)
{
case LANG_NBO: lang = "nbo"; break;
case LANG_ENG: lang = "eng"; break;
}
bblog(INFO, "[searchd] lang_id = %i", queryNodeHeder.lang);
bblog(INFO, "[searchd] lang = %s", lang);
searchd_config->thesaurusp = NULL;
if (lang != NULL && searchd_config->thesaurus_all != NULL)
{
iterator it = map_find(searchd_config->thesaurus_all, lang);
if (it.valid)
{
bblog(INFO, "[searchd] Loading %s thesaurus", lang);
searchd_config->thesaurusp = map_val(it).ptr;
}
else
{
bblog(INFO, "[searchd] No thesaurus for %s", lang);
}
}
#endif
/***************************************/
/****************/
//dekoder subname
bblog(DEBUGINFO, "nrOfSubnames %i",nrOfSubnames);
/*
nrOfSubnames = 1;
subnames = malloc(sizeof(struct subnamesFormat) * nrOfSubnames);
strscpy(subnames[0].subname,"www",sizeof(subnames[0].subname));
*/
#ifdef DEBUG
bblog(DEBUGINFO, "searchd_child: ");
bblog(DEBUGINFO, "##########################################################");
bblog(DEBUGINFO, "searchd_child: subnames:");
bblog(DEBUGINFO, "Total of %i", nrOfSubnames);
for (i=0;i<nrOfSubnames;i++) {
bblog(DEBUGINFO, "searchd_child: subname nr %i: \"%s\"",i,subnames[i].subname);
}
bblog(DEBUGINFO, "searchd_child: ##########################################################");
#endif
SiderHeder->filtypesnrof = MAXFILTYPES;
SiderHeder->errorstrlen=sizeof(SiderHeder->errorstr);
int parsing_failed;
attr_conf *navmenu_cfg = parse_navmenu_cfg(queryNodeHeder.navmenucfg, &parsing_failed, globalOptVerbose);
if (parsing_failed) {
SiderHeder->responstype = searchd_responstype_error;
snprintf(SiderHeder->errorstr, sizeof SiderHeder->errorstr,
"An error occurred while parsing configuration for navigation menu.");
SiderHeder->errorstrlen = strlen(SiderHeder->errorstr); // TODO: is errorstrlen even used?
}
if (!dosearch(queryNodeHeder.query, strlen(queryNodeHeder.query),&Sider,SiderHeder,SiderHeder->hiliteQuery,
servername,subnames,nrOfSubnames,queryNodeHeder.MaxsHits,
queryNodeHeder.start, queryNodeHeder.filterOn,
"",queryNodeHeder.orderby,SiderHeder->dates,queryNodeHeder.search_user,
&SiderHeder->filters,
searchd_config,
SiderHeder->errorstr, &SiderHeder->errorstrlen,
&global_DomainIDs, queryNodeHeder.HTTP_USER_AGENT,
groupOrQuery, queryNodeHeder.anonymous, navmenu_cfg,
spelling
))
{
bblog(WARN, "searchd_child: dosearch did not return success");
SiderHeder->responstype = searchd_responstype_error;
//setter at vi ikke hadde noen svar
SiderHeder->TotaltTreff = 0;
SiderHeder->showabal = 0;
bblog(ERROR, "searchd_child: can't do dosearch: \"%s\"", SiderHeder->errorstr);
}
show_attributes_destroy(navmenu_cfg);
dp_priority_locl_end();
//kopierer inn subnames. Kan bare sende over MAX_COLLECTIONS, men søker i alle
for (i=0;((i<MAX_COLLECTIONS) && (i<nrOfSubnames));i++) {
SiderHeder->subnames[i] = subnames[i];
}
SiderHeder->nrOfSubnames = i--;
if (globalOptVerbose) {
bblog(INFO, "searchd_child: subnames:");
for (i=0;i<SiderHeder->nrOfSubnames;i++) {
bblog(INFO, "searchd_child: \t%s: %i",SiderHeder->subnames[i].subname,SiderHeder->subnames[i].hits);
}
bblog(INFO, "searchd_child:");
}
//finer først tiden vi brukte
gettimeofday(&globalend_time, NULL);
SiderHeder->total_usecs = getTimeDifference(&globalstart_time,&globalend_time);
//printf("query \"%s\", TotaltTreff %i,showabal %i,filtered %i,total_usecs %f\n",queryNodeHeder.query,SiderHeder->TotaltTreff,SiderHeder->showabal,SiderHeder->filtered,SiderHeder->total_usecs);
bblog(CLEAN, "|%-40s | %-11i | %-11i | %-11i | %-11f|",
queryNodeHeder.query,
SiderHeder->TotaltTreff,
SiderHeder->showabal,
SiderHeder->filtered,
SiderHeder->total_usecs);
#ifdef DEBUG
gettimeofday(&start_time, NULL);
#endif
#if 1
#ifdef DEBUG
gettimeofday(&end_time, NULL);
bblog(DEBUGINFO, "searchd_child: Time debug: sending SiderHeder %f",getTimeDifference(&start_time,&end_time));
#endif
#ifdef DEBUG
gettimeofday(&start_time, NULL);
#endif
struct sendarrayFormat{
int size;
void *p;
int copy;
};
void send_to_array (struct sendarrayFormat *sendarray, int *sendarraylength, void *p, int size, int copy) {
#ifdef DEBUG
printf("send_to_array(sendarraylength=%i,size=%i)\n",*sendarraylength,size);
#endif
if (size == 0) {
return;
}
sendarray[*sendarraylength].copy = copy;
if (copy) {
sendarray[*sendarraylength].p = malloc(size);
memcpy(sendarray[*sendarraylength].p,p,size);
sendarray[*sendarraylength].size = size;
}
else {
sendarray[*sendarraylength].p = p;
sendarray[*sendarraylength].size = size;
}
*sendarraylength += 1;
}
void *issueAdd(void *arg) {
int mysocfd = (int) arg;
struct betaler_keywords_visninger_format {
int kid;
int betaler_side_id;
};
struct betaler_keywords_visninger_format betaler_keywords_visninger[10];
char buff[1024];
struct timeval globalstart_time, globalend_time;
unsigned int addid;
char *strpointer;
int siderType_ppctopNr,siderType_ppcsideNr;
struct queryNodeHederFormat queryNodeHeder;
char queryEscaped[MaxQueryLen*2+1];
char ppcprovider[32];
int i,n, y, net_status, showabal;;
//sjekker vårt egent anonsesystem
char mysql_query [2048];
static MYSQL demo_db;
MYSQL_RES *mysqlres; /* To be used to fetch information into */
MYSQL_ROW mysqlrow;
struct SiderHederFormat SiderHeder;
struct ppcPagesFormat ppcPages[10];
struct SiderFormat *Sider;
gettimeofday(&globalstart_time, NULL);
if ((i=recv(mysocfd, &queryNodeHeder, sizeof(queryNodeHeder),MSG_WAITALL)) == -1) {
perror("recv");
}
printf("Query %s\n",queryNodeHeder.query);
Sider = (struct SiderFormat *)malloc(sizeof(struct SiderFormat) * (queryNodeHeder.MaxsHits));
//setter alle sidene som sletett
for (i=0;i<queryNodeHeder.MaxsHits;i++) {
Sider[i].deletet = 1;
}
//sender svar med en gang at vi kan gjøre dette
net_status = net_CanDo;
if ((n=sendall(mysocfd,&net_status, sizeof(net_status))) != sizeof(net_status)) {
printf("send only %i of %i\n",n,sizeof(net_status));
perror("sendall net_status");
}
/********************************************************************************************/
#ifdef DEBUG
printf("sending query to ppc db\n");
#endif
mysql_init(&demo_db);
#ifdef WITH_THREAD
my_thread_init(); // kalt mysql_thread_init() i mysql 5.0
#endif
//if(!mysql_real_connect(&demo_db, "www2.boitho.com", "boitho_remote", "G7J7v5L5Y7", "boitho", 3306, NULL, 0)){
if(!mysql_real_connect(&demo_db, "localhost", "boitho", "G7J7v5L5Y7", "boithoweb", 3306, NULL, 0)){
printf(mysql_error(&demo_db));
//return(1);
pthread_exit((void *)1); /* exit with status */
}
//escaper queryet rikit
mysql_real_escape_string(&demo_db,queryEscaped,queryNodeHeder.query,strlen(queryNodeHeder.query));
sprintf(mysql_query, "select tittel,url,beskrivelse,betaler_sider.bruker_navn,betaler_keywords.betaler,betaler_keywords.kid,betaler_sider.id from betaler_keywords,betaler_sider where betaler_keywords.keyword ='%s' and betaler_keywords.betaler_side_id=betaler_sider.id order by betaler desc",queryEscaped);
if(mysql_real_query(&demo_db, mysql_query, strlen(mysql_query))){ /* Make query */
printf(mysql_error(&demo_db));
//return(1);
pthread_exit((void *)1); /* exit with status */
}
#ifdef DEBUG
printf("sending query to ppc db end\n");
#endif
/********************************************************************************************/
SiderHeder.TotaltTreff = 0;
int nrOfppcPages = 0;
int nrOfBoithoAds = 0;
//printer ut eventuelt ppc ord
mysqlres=mysql_store_result(&demo_db); /* Download result from server */
while ((mysqlrow=mysql_fetch_row(mysqlres)) != NULL) { /* Get a row from the results */
//printf("\t<beskrivelse>%s</beskrivelse>\n",mysqlrow[2]);
//Sider[showabal].type = siderType_ppctop;
strncpy(ppcPages[nrOfppcPages].title,mysqlrow[0],sizeof(ppcPages[nrOfppcPages].title));
strncpy(ppcPages[nrOfppcPages].url,mysqlrow[1],sizeof(ppcPages[nrOfppcPages].url));
strncpy(ppcPages[nrOfppcPages].uri,mysqlrow[1],sizeof(ppcPages[nrOfppcPages].uri));
strncpy(ppcPages[nrOfppcPages].description,mysqlrow[2],sizeof(ppcPages[nrOfppcPages].description));
strncpy(ppcPages[nrOfppcPages].user,mysqlrow[3],sizeof(ppcPages[nrOfppcPages].user));
ppcPages[nrOfppcPages].thumbnail[0] = '\0';
ppcPages[nrOfppcPages].bid = atof(mysqlrow[4]);
ppcPages[nrOfppcPages].keyword_id = atoi(mysqlrow[5]);
ppcPages[nrOfppcPages].DocID = strtoul(mysqlrow[6], (char **)NULL, 10);
ppcPages[nrOfppcPages].allrank = 10000;
#ifdef DEBUG
printf("aa bid %f\n",ppcPages[nrOfppcPages].bid);
printf("\tUrl: %s\n",ppcPages[nrOfppcPages].url);
printf("\tTitle: %s\n",ppcPages[nrOfppcPages].title);
printf("keyword_id -%s-\n",mysqlrow[5]);
#endif
betaler_keywords_visninger[nrOfBoithoAds].kid = ppcPages[nrOfppcPages].keyword_id;
betaler_keywords_visninger[nrOfBoithoAds].betaler_side_id = ppcPages[nrOfppcPages].DocID;
++nrOfppcPages;
++nrOfBoithoAds;
}
mysql_free_result(mysqlres);
/*********************************/
printf("contry: %s\n",queryNodeHeder.GeoIPcontry);
if (strcmp(queryNodeHeder.GeoIPcontry,"NO") == 0) {
strcpy(ppcprovider,"hent");
//strcpy(ppcprovider,"revenuepilot");
}
else {
//alle språk
//strcpy(ppcprovider,"revenuepilot");
//strcpy(ppcprovider,"searchboss");
}
strcpy(ppcprovider,"amazon");
//temp: skrur av 3p xml feeds
//getPpcAds(ppcprovider,ppcPages,&nrOfppcPages,&queryNodeHeder);
//temp: Viser en mindre side da vi får problemer med siste?
//nrOfppcPages--;
showabal = 0;
for (i=0;i<nrOfppcPages;i++) {
/*********************************************/
//Sider[showabal].type = siderType_ppcside;
#ifdef DEBUG
printf("issue add. keyword_id %i\n",ppcPages[i].keyword_id);
#endif
sprintf(mysql_query, "insert into issuedadds values(%s,'%s','%f','%s',%s,'%s','%s','%s','%s','%s','%s','%s','%i','%i')",
"NULL",
queryEscaped,
ppcPages[i].bid,
ppcPages[i].uri,
"NOW()",
0,
ppcPages[i].user,
queryNodeHeder.search_user,
queryNodeHeder.userip,
queryNodeHeder.HTTP_ACCEPT_LANGUAGE,
queryNodeHeder.HTTP_USER_AGENT,
queryNodeHeder.HTTP_REFERER,
ppcPages[i].keyword_id,
ppcPages[i].DocID
);
#ifdef DEBUG
printf("ppc user %s\naffuser %s\n",Sider[i].user,queryNodeHeder.search_user);
#endif
if(mysql_real_query(&demo_db, mysql_query, strlen(mysql_query))){ /* Make query */
printf("Cant insert into issuedadds: %s\nSql query vas %s\n",mysql_error(&demo_db),mysql_query);
//return(1);
pthread_exit((void *)1); /* exit with status */
}
addid = mysql_insert_id(&demo_db);
#ifdef DEBUG
printf("addid %u\n",addid);
#endif
//sprintf(ppcPages[showabal].uri,"http://search.boitho.com/cgi-bin/addout.cgi?addid=%u&addurl=%s",addid,ppcPages[showabal].url);
sprintf(ppcPages[showabal].uri,"http://bbh-001.boitho.com/cgi-bin/addout.cgi?addid=%u&addurl=%s",addid,ppcPages[showabal].url);
//strcpy(Sider[i].uri,buff);
/*********************************************/
if (strlen(ppcPages[i].title) == (sizeof(ppcPages[i].title) -1)) {
//strcpy(Sider[showabal].title,"Title to long.");
strncpy(Sider[showabal].title,ppcPages[i].title,sizeof(Sider[showabal].title) -3);
strcat(Sider[showabal].title,"..");
}
else {
strncpy(Sider[showabal].title,ppcPages[i].title,sizeof(Sider[showabal].title));
}
strncpy(Sider[showabal].description,ppcPages[i].description,sizeof(Sider[showabal].description));
strncpy(Sider[showabal].url,ppcPages[i].url,sizeof(Sider[showabal].url));
strncpy(Sider[showabal].uri,ppcPages[i].uri,sizeof(Sider[showabal].uri));
strncpy(Sider[showabal].user,ppcPages[i].user,sizeof(Sider[showabal].user));
strscpy(Sider[showabal].domain,ppcPages[i].domain,sizeof(Sider[showabal].domain));
strscpy(Sider[showabal].thumbnale,ppcPages[i].thumbnail,sizeof(Sider[showabal].thumbnale));
Sider[showabal].thumbnailwidth = atol(ppcPages[i].thumbnailwidth);
Sider[showabal].thumbnailheight = atol(ppcPages[i].thumbnailheight);
Sider[showabal].bid = ppcPages[i].bid;
Sider[showabal].iindex.allrank = ppcPages[i].allrank;
#ifdef DEBUG
printf("%s\t%s\t%f\n",Sider[showabal].url,Sider[showabal].title,ppcPages[i].bid);
#endif
++showabal;
}
/*********************************/
siderType_ppctopNr = 0;
siderType_ppcsideNr = 0;
for(i=0;i<showabal;i++) {
#ifdef DEBUG
printf("uri %s\n",Sider[i].uri);
#endif
Sider[i].DocumentIndex.crc32 = crc32boitho(Sider[i].description);
Sider[i].deletet = 0;
//lager fin beskrivlse som slutter på .. isteden får bare et kappet ord, hvis beskrivlese er for lang
if (strlen(Sider[i].description) >= 250) {
//søker oss til siste space , eller ; og avslutter der
if ((strpointer = (char *)strrchr(Sider[i].description,' ')) != NULL) {
strpointer[0] = '\0';
}
else if ((strpointer = (char *)strrchr(Sider[i].description,';')) != NULL) {
++strpointer; //pekeren peker på semikolonet. SKal ha det med, så må legge il en
strpointer[0] = '\0';
}
strncat(Sider[i].description,"..",2);
}
//hiliter ordet
sprintf(buff,"<b>%s</b>",queryNodeHeder.query);
strcasesandr(Sider[i].description,sizeof(Sider[i].description),queryNodeHeder.query,buff);
//bestemmer ppc type
//Sider[showabal].type = siderType_ppcside
//Sider[i].type = siderType_ppctop;
if ((siderType_ppctopNr < 2) && (strcasestr(Sider[i].description,queryNodeHeder.query) != 0)) {
Sider[i].type = siderType_ppctop;
++siderType_ppctopNr;
}
else {
Sider[i].type = siderType_ppcside;
++siderType_ppcsideNr;
}
}
//legger datane in i mysql database.
for(i=0;i<showabal;i++) {
}
gettimeofday(&globalend_time, NULL);
SiderHeder.total_usecs = getTimeDifference(&globalstart_time,&globalend_time);
SiderHeder.TotaltTreff = showabal;
SiderHeder.showabal = showabal;
SiderHeder.filtered = 0;
SiderHeder.hiliteQuery[0] = '\0';
sprintf(SiderHeder.servername,"adserver.boitho.com");
//SiderHeder.queryTime = 0;
if ((n=sendall(mysocfd,&SiderHeder, sizeof(SiderHeder))) != sizeof(SiderHeder)) {
printf("send only %i of %i\n",n,sizeof(SiderHeder));
perror("sendall SiderHeder");
}
for(i=0;i<SiderHeder.showabal;i++) {
//for (i=0;i<queryNodeHeder.MaxsHits;i++) {
#ifdef DEBUG
printf("sending %s, deletet %i\n",Sider[i].url,Sider[i].deletet);
printf("bb: -%s-\n",Sider[i].title);
printf("url: -%s-\n",Sider[i].url);
#endif
//if (!Sider[i].deletet) {
if ((n=sendall(mysocfd,&Sider[i], sizeof(struct SiderFormat))) != sizeof(struct SiderFormat)) {
printf("send only %i of %i\n",n,sizeof(struct SiderFormat));
perror("sendall");
}
//}
}
//logger alle visningene vi har hatt på egen ppc ord
for (i=0;i<nrOfBoithoAds;i++) {
sprintf(mysql_query, "insert DELAYED into betaler_keywords_visninger values(NULL,'%i','%i',NOW())",betaler_keywords_visninger[i].kid,betaler_keywords_visninger[i].betaler_side_id);
if(mysql_real_query(&demo_db, mysql_query, strlen(mysql_query))){ /* Make query */
printf(mysql_error(&demo_db));
//return(1);
pthread_exit((void *)0); /* exit with status */
}
}
mysql_close(&demo_db);
//close(mysocfd);
free(Sider);
close(mysocfd);
#ifdef WITH_THREAD
my_thread_end(); // kalt mysql_thread_end() i mysql 5.0
pthread_exit((void *)0); /* exit with status */
#endif
printf("end\n");
//return 0;
}
