int __DPSCALL DpsResAddDocInfoSearchd(DPS_AGENT * query,DPS_DB *cl,DPS_RESULT * Res,size_t clnum){ DPS_SEARCHD_PACKET_HEADER hdr; char * msg=NULL; size_t i; /* num=0,curnum=0;*/ int done = 0; ssize_t nsent,nrecv; char * dinfo=NULL; int rc=DPS_OK; char *textbuf; size_t dlen = 0; TRACE_IN(query, "DpsResAddDocInfoSearchd"); if(!Res->num_rows) { TRACE_OUT(query); return(DPS_OK); } for(i=0;i<Res->num_rows;i++){ size_t ulen; size_t olen; size_t nsec, r; DPS_DOCUMENT *D=&Res->Doc[i]; r = (size_t) 's'; for(nsec = 0; nsec < D->Sections.Root[r].nvars; nsec++) if (strcasecmp(D->Sections.Root[r].Var[nsec].name, "Score") == 0) D->Sections.Root[r].Var[nsec].section = 1; #ifdef WITH_MULTIDBADDR if (D->dbnum != cl->dbnum) continue; #endif textbuf = DpsDocToTextBuf(D, 1, 0); if (textbuf == NULL) {TRACE_OUT(query); return DPS_ERROR;} ulen = dps_strlen(textbuf)+2; olen = dlen; dlen = dlen + ulen; dinfo = (char*)DpsRealloc(dinfo, dlen + 1); if (dinfo == NULL) { DpsFree(textbuf); TRACE_OUT(query); return DPS_ERROR; } dinfo[olen] = '\0'; sprintf(dinfo + olen, "%s\r\n", textbuf); DpsFree(textbuf); } if (dinfo == NULL) { TRACE_OUT(query); return DPS_OK; } hdr.cmd=DPS_SEARCHD_CMD_DOCINFO; hdr.len = dps_strlen(dinfo); nsent = DpsSearchdSendPacket(cl->searchd, &hdr, dinfo); #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Sent DOCINFO size=%d buf=%s\n", hdr.len, dinfo); #endif while(!done){ char * tok, * lt; nrecv = DpsRecvall(cl->searchd, &hdr, sizeof(hdr), 360); if(nrecv!=sizeof(hdr)){ DpsLog(query, DPS_LOG_ERROR, "Received incomplete header from searchd (%d bytes, errno:%d)", (int)nrecv, errno); TRACE_OUT(query); return(DPS_ERROR); }else{ #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Received header cmd=%d len=%d\n",hdr.cmd,hdr.len); #endif } switch(hdr.cmd){ case DPS_SEARCHD_CMD_ERROR: msg=(char*)DpsMalloc(hdr.len+1); if (msg == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, msg, hdr.len, 360); msg[(nrecv >= 0) ? nrecv : 0]='\0'; sprintf(query->Conf->errstr,"Searchd error: '%s'",msg); rc=DPS_ERROR; DPS_FREE(msg); done=1; break; case DPS_SEARCHD_CMD_MESSAGE: msg=(char*)DpsMalloc(hdr.len+1); if (msg == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, msg, hdr.len, 360); msg[(nrecv >= 0) ? nrecv : 0]='\0'; #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Message from searchd: '%s'\n",msg); #endif DPS_FREE(msg); break; case DPS_SEARCHD_CMD_DOCINFO: dinfo = (char*)DpsRealloc(dinfo, hdr.len + 1); if (dinfo == NULL) { done=1; break; } nrecv = DpsRecvall(cl->searchd, dinfo, hdr.len, 360); dinfo[(nrecv > 0) ? nrecv : 0] = '\0'; #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Received DOCINFO size=%d buf=%s\n",hdr.len,dinfo); #endif tok = dps_strtok_r(dinfo, "\r\n", <, NULL); while(tok){ urlid_t Doc_url_id, Res_Doc_url_id; DPS_DOCUMENT Doc; DpsDocInit(&Doc); DpsDocFromTextBuf(&Doc,tok); Doc_url_id = (urlid_t)DpsVarListFindInt(&Doc.Sections, "DP_ID", 0); for(i=0;i<Res->num_rows;i++){ #ifdef WITH_MULTIDBADDR if (Res->Doc[i].dbnum != cl->dbnum) continue; #endif Res_Doc_url_id = (urlid_t)DpsVarListFindInt(&Res->Doc[i].Sections, "DP_ID", 0); if (Res_Doc_url_id == Doc_url_id) { DpsDocFromTextBuf(&Res->Doc[i], tok); break; } } tok = dps_strtok_r(NULL, "\r\n", <, NULL); DpsDocFree(&Doc); } DPS_FREE(dinfo); done=1; break; default: sprintf(query->Conf->errstr,"Unknown searchd response: cmd=%d len=%d",hdr.cmd,hdr.len); rc=DPS_ERROR; done=1; break; } } TRACE_OUT(query); return rc; }
/* Compute one operation and store result */ static int perform(DPS_AGENT *query, DPS_RESULT *Res, DPS_BOOLSTACK *s, int com) { DPS_STACK_ITEM res, *x1, *x2; int rc = DPS_OK, found, flag1; bzero(&res, sizeof(res)); switch(com){ case DPS_STACK_PHRASE_LEFT: x1 = POPARG(s); if (x1 == NULL) { bzero(&res, sizeof(res)); } else { res = *x1; /* FIXME: add checking ? */ if (res.order_from != res.order_to) { DPS_URL_CRD_DB *w; dps_uint4 *pos_real, *order_ideal, *order_real, *gap_ahead/*, *gap_back*/; urlid_t curlid; size_t nwords = res.order_to - res.order_from + 1, nonstop_words; size_t p_cmp, p_ins; res.plast = res.pbegin + res.count; w = res.pcur = res.pchecked = res.pbegin; if ((pos_real = (dps_uint4*)DpsMalloc(5 * nwords * sizeof(dps_uint4) + 1)) == NULL) { DpsLog(query, DPS_LOG_ERROR, "Can't alloc %d bytes %s:%d",(5 * nwords * sizeof(dps_uint4) + 1), __FILE__, __LINE__); return DPS_ERROR; } order_real = pos_real + nwords; gap_ahead = order_real + nwords; order_ideal = gap_ahead + nwords; /* gap_back = order_ideal + nwords;*/ nonstop_words = 0; { register size_t tt; for (tt = res.order_from; tt <= res.order_to; tt++) { #ifdef DEBUG_BOOL #endif if ((Res->items[tt].order_origin & DPS_WORD_ORIGIN_STOP) == 0) { order_ideal[nonstop_words] = tt; gap_ahead[nonstop_words] = (tt == res.order_to) ? 0 : 1; { register size_t zz; for (zz = tt + 1; zz <= res.order_to; zz++) { if (Res->items[zz].order_origin & DPS_WORD_ORIGIN_STOP) gap_ahead[nonstop_words]++; else break; } } nonstop_words++; } } } #ifdef DEBUG_BOOL DpsLog(query, DPS_LOG_EXTRA, "nonstopwords: %d nwords:%d", nonstop_words, nwords); { register size_t tt; for (tt = 0; tt < nonstop_words; tt++) { DpsLog(query, DPS_LOG_EXTRA, "%d:order_ideal:%d gap_ahead:%d", tt, order_ideal[tt], gap_ahead[tt]); } } #endif if (nonstop_words != 0) { while (res.pcur < res.plast) { register size_t tt; curlid = res.pcur->url_id; found = 0; p_ins = 0; p_cmp = nwords - nonstop_words; res.pchecked = res.pcur; /******* ? *****/ for (tt = 0; tt < nwords - nonstop_words; tt++) pos_real[tt] = 0; for (tt = 0; (tt < nonstop_words) && (res.pcur < res.plast) && (res.pcur->url_id == curlid) ; tt++) { order_real[p_ins] = Res->WWList.Word[DPS_WRDNUM(res.pcur->coord)].order; pos_real[p_ins] = DPS_WRDPOS(res.pcur->coord); while((res.pcur < res.plast) && (pos_real[p_ins] == DPS_WRDPOS(res.pcur->coord))) res.pcur++; p_ins++; p_cmp++; p_ins %= nwords; p_cmp %= nwords; } if (tt == nonstop_words) { /* [[[[[[ */ found = 1; for (tt = 0; tt < nonstop_words; tt++) { if (order_real[(p_cmp + tt) % nwords] != order_ideal[tt]) { found = 0; break; } if (gap_ahead[tt] && (tt + gap_ahead[tt] < nwords) && (pos_real[(p_cmp + tt) % nwords] + gap_ahead[tt] != pos_real[(p_cmp + tt + 1) % nwords])) { found = 0; break; } } if (found) { while((res.pchecked < res.pcur) /*&& (res.pchecked->url_id == curlid)*/) { *w = *res.pchecked; w++; res.pchecked++; } res.pcur = res.pchecked; } else { res.pchecked = res.pcur; } /* ]]]]]] */ } while (/*(found == 0) &&*/ (res.pcur < res.plast) && (res.pcur->url_id == curlid)) { order_real[p_ins] = Res->WWList.Word[DPS_WRDNUM(res.pcur->coord)].order; pos_real[p_ins] = DPS_WRDPOS(res.pcur->coord); while((res.pcur < res.plast) && (pos_real[p_ins] == DPS_WRDPOS(res.pcur->coord))) res.pcur++; p_ins++; p_cmp++; p_ins %= nwords; p_cmp %= nwords; /* [[[[[[ */ found = 1; for (tt = 0; tt < nonstop_words; tt++) { if (order_real[(p_cmp + tt) % nwords] != order_ideal[tt]) { found = 0; break; } if (gap_ahead[tt] && (tt + gap_ahead[tt] < nwords) && (pos_real[(p_cmp + tt) % nwords] + gap_ahead[tt] != pos_real[(p_cmp + tt + 1) % nwords])) { found = 0; break; } } if (found) { while((res.pchecked < res.pcur) /*&& (res.pchecked->url_id == curlid)*/) { *w = *res.pchecked; w++; res.pchecked++; } res.pcur = res.pchecked; } else { res.pchecked = res.pcur; } /* ]]]]]] */ } } } res.count = w - res.pbegin; DPS_FREE(pos_real); } } #ifdef DEBUG_BOOL DpsLog(query, DPS_LOG_EXTRA, "Perform <{%d}:%d:%d> ->{%d}", x1 ? x1->count : 0, x1 ? x1->order_from : 0, x1 ? x1->order_to : 0, res.count); #endif rc = PUSHARG(s, &res); break; case DPS_STACK_OR: x1 = POPARG(s); x2 = POPARG(s); if (x2 == NULL || x1 == NULL) { if (x1 != NULL) { res = *x1; x1 = NULL; } if (x2 != NULL) { res = *x2; x2 = NULL; } } else { #ifdef DEBUG_BOOL /* printBoolRes(query, x1);*/ DpsLog(query, DPS_LOG_EXTRA, "^^^"); /* printBoolRes(query, x2);*/ #endif res.order_from = (x1->order_from <= x2->order_from) ? x1->order_from : x2->order_from; res.order_to = (x1->order_to >= x2->order_to) ? x1->order_to : x2->order_to; if (DPS_OK != proceedOR(query, &res, x1, x2)) return DPS_ERROR; DpsStackItemFree(x1); DpsStackItemFree(x2); res.count = res.pcur - res.pbegin; { register size_t tt; int x1origin=0, x2origin=0; for (tt = x1->order_from; tt <= x1->order_to; tt++) { #ifdef DEBUG_BOOL DpsLog(query, DPS_LOG_EXTRA, "\t\t\t\tx1order_origin[%d].%x ", tt, Res->items[tt].origin /*order_origin*/); #endif if (Res->items[tt].origin /*order_origin*/ & DPS_WORD_ORIGIN_STOP) { x1origin = Res->items[tt].origin /*order_origin*/; break; } } for (tt = x2->order_from; tt <= x2->order_to; tt++) { #ifdef DEBUG_BOOL DpsLog(query, DPS_LOG_EXTRA, "\t\t\t\tx2order_origin[%d].%x ", tt, Res->items[tt].origin /*order_origin*/); #endif if (Res->items[tt].origin /*order_origin*/ & DPS_WORD_ORIGIN_STOP) { x2origin = Res->items[tt].origin /*order_origin*/; break; } } x1origin = x1->origin; x2origin = x2->origin; #ifdef DEBUG_BOOL DpsLog(query, DPS_LOG_EXTRA, "\t\t\t\tx1origin.%x x2origin.%x", x1origin, x2origin); #endif if ((x1origin & (DPS_WORD_ORIGIN_STOP | DPS_WORD_ORIGIN_QUERY)) == (DPS_WORD_ORIGIN_STOP|DPS_WORD_ORIGIN_QUERY)) { res.origin = x1origin; } else if ((x2origin & (DPS_WORD_ORIGIN_STOP | DPS_WORD_ORIGIN_QUERY)) == (DPS_WORD_ORIGIN_STOP|DPS_WORD_ORIGIN_QUERY)) { res.origin = x2origin; } else if (((x1origin & DPS_WORD_ORIGIN_STOP) && (x2origin & DPS_WORD_ORIGIN_STOP)) || ( (res.count == 0) && ((x1origin & DPS_WORD_ORIGIN_STOP) || (x2origin & DPS_WORD_ORIGIN_STOP)))) res.origin = DPS_WORD_ORIGIN_STOP; if ((x1origin & DPS_WORD_ORIGIN_ACRONYM) && (x1origin & DPS_WORD_ORIGIN_ACRONYM)) res.origin |= DPS_WORD_ORIGIN_ACRONYM; } } #ifdef DEBUG_BOOL DpsLog(query, DPS_LOG_EXTRA, "Perform {%d}.%x | {%d}.%x -> {%d}.%x", (x1) ? x1->count:-1, (x1)?x1->origin:-1, (x2)?x2->count : -1, (x2) ? x2->origin : -1, res.count, res.origin); /* printBoolRes(query, &res);*/ DpsLog(query, DPS_LOG_EXTRA, "==="); #endif rc = PUSHARG(s, &res); break; case DPS_STACK_NEAR: x1 = POPARG(s); x2 = POPARG(s); if (x2 == NULL || x1 == NULL) { if (x1 != NULL) { res = *x1; x1 = NULL; } if (x2 != NULL) { res = *x2; x2 = NULL; } } else { res.order_from = (x1->order_from <= x2->order_from) ? x1->order_from : x2->order_from; res.order_to = (x1->order_to >= x2->order_to) ? x1->order_to : x2->order_to; if ((x1->origin & DPS_WORD_ORIGIN_STOP) && (x2->origin & DPS_WORD_ORIGIN_STOP) ) { if (DPS_OK != proceedOR(query, &res, x1, x2)) return DPS_ERROR; res.origin = DPS_WORD_ORIGIN_STOP; } else if (x2->origin & DPS_WORD_ORIGIN_STOP) { if (DPS_OK != proceedSTOP(query, &res, x1, x2)) return DPS_ERROR; } else if (x1->origin & DPS_WORD_ORIGIN_STOP ) { if (DPS_OK != proceedSTOP(query, &res, x2, x1)) return DPS_ERROR; } else if (!((x1->cmd & DPS_STACK_WORD_NOT) && (x2->cmd & DPS_STACK_WORD_NOT))) { #ifdef DEBUG_BOOL /* printBoolRes(query, x1);*/ DpsLog(query, DPS_LOG_EXTRA, "^^^"); DpsLog(query, DPS_LOG_DEBUG, "x1.NOT: %d x2.NOT: %d", x1->cmd & DPS_STACK_WORD_NOT, x2->cmd & DPS_STACK_WORD_NOT); /* printBoolRes(query, x2);*/ #endif res.pbegin = res.pcur = (DPS_URL_CRD_DB*)DpsMalloc((x1->count + x2->count + 1) * sizeof(DPS_URL_CRD_DB)); if (res.pbegin == NULL) return DPS_ERROR; x1->pcur = x1->pbegin; x1->plast = x1->pbegin + x1->count; x2->pcur = x2->pbegin; x2->plast = x2->pbegin + x2->count; if (x1->cmd & DPS_STACK_WORD_NOT) { register DPS_STACK_ITEM *t = x1; x1 = x2; x2 = t; } while (x1->pcur < x1->plast && x2->pcur < x2->plast) { while ((x2->pcur < x2->plast) && (x2->pcur->url_id < x1->pcur->url_id)) x2->pcur++; if (x2->pcur >= x2->plast) break; if (x2->pcur->url_id == x1->pcur->url_id) { dps_uint4 pos1 = DPS_WRDPOS(x1->pcur->coord); dps_uint4 pos2 = DPS_WRDPOS(x2->pcur->coord); register urlid_t curlid = x1->pcur->url_id; if (pos1 > pos2) { found = ((pos2 + 16) >= pos1); } else { found = ((pos1 + 16) >= pos2); } x1->pchecked = x1->pcur; x2->pchecked = x2->pcur; while ((!found) && (x1->pchecked < x1->plast) && (x2->pchecked < x2->plast) /*&& (x1->pchecked->url_id == x2->pchecked->url_id)*/ ) { if (x1->pchecked->coord <= x2->pchecked->coord) { x1->pchecked++; pos1 = DPS_WRDPOS(x1->pchecked->coord); if (x1->pchecked->url_id != curlid) break; } else { x2->pchecked++; pos2 = DPS_WRDPOS(x2->pchecked->coord); if (x2->pchecked->url_id != curlid) break; } if (pos1 > pos2) { found = ((pos2 + 16) >= pos1); } else { found = ((pos1 + 16) >= pos2); } } if (x2->cmd & DPS_STACK_WORD_NOT || x1->cmd & DPS_STACK_WORD_NOT) found = !found; if (found) { while ((x1->pcur < x1->plast) && (x2->pcur < x2->plast) /*&& (x1->pcur->url_id == x2->pcur->url_id)*/) { if (x1->pcur->coord <= x2->pcur->coord) { *res.pcur = *x1->pcur; res.pcur++; x1->pcur++; if (x1->pcur->url_id != curlid) break; } else { *res.pcur = *x2->pcur; res.pcur++; x2->pcur++; if (x2->pcur->url_id != curlid) break; } } while ((x1->pcur < x1->plast) && (x1->pcur->url_id == curlid)) { *res.pcur = *x1->pcur; res.pcur++; x1->pcur++; } while ((x2->pcur < x2->plast) && (x2->pcur->url_id == curlid)) { *res.pcur = *x2->pcur; res.pcur++; x2->pcur++; } } else { x1->pcur = x1->pchecked; x2->pcur = x2->pchecked; while ((x1->pcur < x1->plast) && (x1->pcur->url_id == curlid)) x1->pcur++; while ((x2->pcur < x2->plast) && (x2->pcur->url_id == curlid)) x2->pcur++; } } else { register DPS_STACK_ITEM *t = x1; x1 = x2; x2 = t; } } } } DpsStackItemFree(x1); DpsStackItemFree(x2); res.count = res.pcur - res.pbegin; #ifdef DEBUG_BOOL DpsLog(query, DPS_LOG_EXTRA,"Perform {%d}.%x NEAR {%d}.%x - > %d.%d", (x1)?x1->count:-1, (x1)?x1->origin:-1, (x2) ? x2->count : -1, (x2) ? x2->origin: - 1, res.count, res.origin); /* printBoolRes(query, &res);*/ DpsLog(query, DPS_LOG_EXTRA, "==="); #endif rc = PUSHARG(s, &res); break; case DPS_STACK_ANYWORD: x1 = POPARG(s); x2 = POPARG(s); flag1 = 0; if (x2 == NULL || x1 == NULL) { if (x1 != NULL) { res = *x1; x1 = NULL; } if (x2 != NULL) { res = *x2; x2 = NULL; } } else { res.order_from = (x1->order_from <= x2->order_from) ? x1->order_from : x2->order_from; res.order_to = (x1->order_to >= x2->order_to) ? x1->order_to : x2->order_to; if ((x1->origin & DPS_WORD_ORIGIN_STOP) && (x2->origin & DPS_WORD_ORIGIN_STOP) ) { if (DPS_OK != proceedOR(query, &res, x1, x2)) return DPS_ERROR; res.origin = DPS_WORD_ORIGIN_STOP; } else if (x2->origin & DPS_WORD_ORIGIN_STOP) { if (DPS_OK != proceedSTOP(query, &res, x1, x2)) return DPS_ERROR; } else if (x1->origin & DPS_WORD_ORIGIN_STOP ) { if (DPS_OK != proceedSTOP(query, &res, x2, x1)) return DPS_ERROR; } else if (!((x1->cmd & DPS_STACK_WORD_NOT) && (x2->cmd & DPS_STACK_WORD_NOT))) { res.pbegin = res.pcur = (DPS_URL_CRD_DB*)DpsMalloc((x1->count + x2->count + 1) * sizeof(DPS_URL_CRD_DB)); if (res.pbegin == NULL) return DPS_ERROR; x1->pcur = x1->pbegin; x1->plast = x1->pbegin + x1->count; x2->pcur = x2->pbegin; x2->plast = x2->pbegin + x2->count; if (x1->cmd & DPS_STACK_WORD_NOT) { register DPS_STACK_ITEM *t = x1; x1 = x2; x2 = t; flag1 = !flag1; } while (x1->pcur < x1->plast && x2->pcur < x2->plast) { while ((x2->pcur < x2->plast) && (x2->pcur->url_id < x1->pcur->url_id)) x2->pcur++; if (x2->pcur >= x2->plast) break; if (x2->pcur->url_id == x1->pcur->url_id) { dps_int4 pos1 = (dps_int4)DPS_WRDPOS(x1->pcur->coord); dps_int4 pos2 = (dps_int4)DPS_WRDPOS(x2->pcur->coord); register urlid_t curlid = x1->pcur->url_id; found = ((flag1) ? ((pos1 + 2) == pos2) : ((pos2 + 2) == pos1)); x1->pchecked = x1->pcur; x2->pchecked = x2->pcur; while ((!found) && (x1->pchecked < x1->plast) && (x2->pchecked < x2->plast) && (x1->pchecked->url_id == x2->pchecked->url_id)) { if (x1->pchecked->coord <= x2->pchecked->coord) { x1->pchecked++; pos1 = (dps_int4)DPS_WRDPOS(x1->pchecked->coord); } else { x2->pchecked++; pos2 = (dps_int4)DPS_WRDPOS(x1->pchecked->coord); } found = ((flag1) ? ((pos1 + 2) == pos2) : ((pos2 + 2) == pos1)); } if (x2->cmd & DPS_STACK_WORD_NOT || x1->cmd & DPS_STACK_WORD_NOT) found = !found; if (found) { while ((x1->pcur < x1->plast) && (x2->pcur < x2->plast) && (x1->pcur->url_id == x2->pcur->url_id)) { if (x1->pcur->coord <= x2->pcur->coord) { *res.pcur = *x1->pcur; res.pcur++; x1->pcur++; } else { *res.pcur = *x2->pcur; res.pcur++; x2->pcur++; } } while ((x1->pcur < x1->plast) && (x1->pcur->url_id == curlid)) { *res.pcur = *x1->pcur; res.pcur++; x1->pcur++; } while ((x2->pcur < x2->plast) && (x2->pcur->url_id == curlid)) { *res.pcur = *x2->pcur; res.pcur++; x2->pcur++; } } else { x1->pcur = x1->pchecked; x2->pcur = x2->pchecked; while ((x1->pcur < x1->plast) && (x1->pcur->url_id == curlid)) x1->pcur++; while ((x2->pcur < x2->plast) && (x2->pcur->url_id == curlid)) x2->pcur++; } } else { register DPS_STACK_ITEM *t = x1; x1 = x2; x2 = t; flag1 = !flag1; } } } } DpsStackItemFree(x1); DpsStackItemFree(x2); res.count = res.pcur - res.pbegin; #ifdef DEBUG_BOOL DpsLog(query, DPS_LOG_EXTRA, "Perform {%d} ANYWORD {%d} - > %d", (x1) ? x1->count : -1, (x2) ? x2->count : -1, res.count); #endif rc = PUSHARG(s, &res); break; case DPS_STACK_AND: x1 = POPARG(s); x2 = POPARG(s); if (x2 == NULL || x1 == NULL) { if (x1 != NULL) { res = *x1; x1 = NULL; } if (x2 != NULL) { res = *x2; x2 = NULL; } } else { res.order_from = (x1->order_from <= x2->order_from) ? x1->order_from : x2->order_from; res.order_to = (x1->order_to >= x2->order_to) ? x1->order_to : x2->order_to; if ((x1->origin & DPS_WORD_ORIGIN_STOP) && (x2->origin & DPS_WORD_ORIGIN_STOP) ) { if (DPS_OK != proceedOR(query, &res, x1, x2)) return DPS_ERROR; res.origin = DPS_WORD_ORIGIN_STOP; } else if (x2->origin & DPS_WORD_ORIGIN_STOP) { if (DPS_OK != proceedSTOP(query, &res, x1, x2)) return DPS_ERROR; } else if (x1->origin & DPS_WORD_ORIGIN_STOP ) { if (DPS_OK != proceedSTOP(query, &res, x2, x1)) return DPS_ERROR; } else if (!((x1->cmd & DPS_STACK_WORD_NOT) && (x2->cmd & DPS_STACK_WORD_NOT))) { res.pbegin = res.pcur = (DPS_URL_CRD_DB*)DpsMalloc((x1->count + x2->count + 1) * sizeof(DPS_URL_CRD_DB)); if (res.pbegin == NULL) return DPS_ERROR; x1->pcur = x1->pbegin; x1->plast = x1->pbegin + x1->count; x2->pcur = x2->pbegin; x2->plast = x2->pbegin + x2->count; if (x1->cmd & DPS_STACK_WORD_NOT) { register DPS_STACK_ITEM *t = x1; x1 = x2; x2 = t; } if (x2->cmd & DPS_STACK_WORD_NOT) { while (x1->pcur < x1->plast && x2->pcur < x2->plast) { while ((x1->pcur < x1->plast) && (x1->pcur->url_id < x2->pcur->url_id)) { *res.pcur = *x1->pcur; res.pcur++; x1->pcur++; } while ((x2->pcur < x2->plast) && (x2->pcur->url_id < x1->pcur->url_id)) x2->pcur++; if (x2->pcur->url_id == x1->pcur->url_id) { register urlid_t curlid = x1->pcur->url_id; while ((x1->pcur < x1->plast) && (x1->pcur->url_id == curlid)) x1->pcur++; while ((x2->pcur < x2->plast) && (x2->pcur->url_id == curlid)) x2->pcur++; } } while (x1->pcur < x1->plast) { *res.pcur = *x1->pcur; res.pcur++; x1->pcur++; } } else { #if 0 { DPS_URL_CRD_DB *w; for (w = x1->pcur; w < x1->plast; w++) { fprintf(stderr, "x1.url_id:%d .coord:%d\n", w->url_id, w->coord); } for (w = x2->pcur; w < x2->plast; w++) { fprintf(stderr, "x2.url_id:%d .coord:%d\n", w->url_id, w->coord); } } #endif while (x1->pcur < x1->plast && x2->pcur < x2->plast) { while ((x2->pcur < x2->plast) && (x2->pcur->url_id < x1->pcur->url_id)) x2->pcur++; if (x2->pcur >= x2->plast) break; if (x2->pcur->url_id == x1->pcur->url_id) { register urlid_t curlid = x1->pcur->url_id; while ((x1->pcur < x1->plast) && (x2->pcur < x2->plast) && (x1->pcur->url_id == x2->pcur->url_id)) { if (x1->pcur->coord <= x2->pcur->coord) { *res.pcur = *x1->pcur; res.pcur++; x1->pcur++; } else { *res.pcur = *x2->pcur; res.pcur++; x2->pcur++; } } while ((x1->pcur < x1->plast) && (x1->pcur->url_id == curlid)) { *res.pcur = *x1->pcur; res.pcur++; x1->pcur++; } while ((x2->pcur < x2->plast) && (x2->pcur->url_id == curlid)) { *res.pcur = *x2->pcur; res.pcur++; x2->pcur++; } } else { register DPS_STACK_ITEM *t = x1; x1 = x2; x2 = t; } } } } } DpsStackItemFree(x1); DpsStackItemFree(x2); res.count = res.pcur - res.pbegin; #ifdef DEBUG_BOOL #if 0 { DPS_URL_CRD_DB *w = res.pbegin; size_t q; for (q = 0; q < res.count; q++) { fprintf(stderr, "res.url_id:%d .coord:%d\n", w[q].url_id, w[q].coord); } } #endif DpsLog(query, DPS_LOG_EXTRA, "Perform {%d}.%x & {%d}.%x - > {%d}.%x", (x1) ? x1->count : -1, (x1) ? x1->origin : -1, (x2) ? x2->count : -1 , (x2) ? x2->origin : -1, res.count, res.origin); /* printBoolRes(query, &res);*/ DpsLog(query, DPS_LOG_EXTRA, "==="); #endif rc = PUSHARG(s, &res); break; case DPS_STACK_NOT: x1 = POPARG(s); /* res = x1 ? 0 : 1; */ if (x1 != NULL) { x1->cmd ^= DPS_STACK_WORD_NOT; rc = PUSHARG(s, x1); } #ifdef DEBUG_BOOL DpsLog(query, DPS_LOG_EXTRA, "Perform ~ {%d}", (x1) ? x1->count : -1); #endif break; } return rc; }
void DpsParseHTTPResponse(DPS_AGENT *Indexer, DPS_DOCUMENT *Doc) { char *token, *lt, *headers, savec; int oldstatus; DPS_DSTR header; Doc->Buf.content=NULL; oldstatus = DpsVarListFindInt(&Doc->Sections, "Status", 0); DpsVarListReplaceInt(&Doc->Sections, "ResponseSize", (int)Doc->Buf.size); DpsVarListDel(&Doc->Sections, "Content-Length"); DpsVarListDel(&Doc->Sections, "Last-Modified"); if (Doc->Buf.buf == NULL) return; /* Cut HTTP response header first */ for(token=Doc->Buf.buf;*token;token++){ if(!strncmp(token,"\r\n\r\n",4)){ if (token <= Doc->Buf.buf + Doc->Buf.size - 4) { *token='\0'; Doc->Buf.content = token + 4; } break; } else if(!strncmp(token,"\n\n",2)){ if (token <= Doc->Buf.buf + Doc->Buf.size - 2) { *token='\0'; Doc->Buf.content = token + 2; } break; } } /* Bad response, return */ if(!Doc->Buf.content) { if (token <= Doc->Buf.buf + Doc->Buf.size - 4) { if (token[2] == CR_CHAR) Doc->Buf.content = token + 4; else Doc->Buf.content = token + 2; } } /* Copy headers not to break them */ headers = (char*)DpsStrdup(Doc->Buf.buf); /* Now lets parse response header lines */ token = dps_strtok_r(headers, "\r\n", <, &savec); if(!token)return; if(!strncmp(token,"HTTP/",5)){ int status = atoi(token + 8); DpsVarListReplaceStr(&Doc->Sections,"ResponseLine",token); DpsVarListReplaceInt(&Doc->Sections, "Status", (oldstatus > status) ? oldstatus : status ); }else{ DpsFree(headers); return; } token = dps_strtok_r(NULL, "\r\n", <, &savec); DpsDSTRInit(&header, 128); while(token){ if(strchr(token,':')) { if (header.data_size) { DpsParseHTTPHeader(Indexer, Doc, &header); DpsDSTRFree(&header); DpsDSTRInit(&header, 128); } } DpsDSTRAppendStr(&header, token); token = dps_strtok_r(NULL, "\r\n", <, &savec); } if (header.data_size) { DpsParseHTTPHeader(Indexer, Doc, &header); } DpsDSTRFree(&header); DPS_FREE(headers); /* Bad response, return */ if(!Doc->Buf.content) { return; } DpsVarListReplaceInt(&Doc->Sections,"Content-Length", Doc->Buf.buf-Doc->Buf.content+(int)Doc->Buf.size + DpsVarListFindInt(&Doc->Sections,"Content-Length", 0)); }
int DpsSEAMake(DPS_AGENT *Indexer, DPS_DOCUMENT *Doc, DPS_DSTR *excerpt, const char *content_lang, size_t *indexed_size, size_t *indexed_limit, size_t max_word_len, size_t min_word_len, int crossec, int seasec #ifdef HAVE_ASPELL , int have_speller, AspellSpeller *speller #endif ) { DPS_SENTENCELIST List; DPS_MAPSTAT MapStat; DPS_TEXTITEM Item; DPS_VAR *Sec; dpsunicode_t *sentence, *lt, savec; double *links, *lang_cs, w; double delta, pdiv, cur_div; size_t l, sent_len, order; size_t min_len = 10000000, min_pos = 0; int it; register size_t i, j; #ifdef DEBUG char lcstr[4096]; #endif TRACE_IN(Indexer, "DpsSEAMake"); if((Sec = DpsVarListFind(&Doc->Sections, "sea"))) { /* set SEA section to NULL */ DPS_FREE(Sec->val); DPS_FREE(Sec->txt_val); Sec->curlen = 0; } bzero(&List, sizeof(List)); order = 0; sentence = DpsUniStrTok_SEA((dpsunicode_t*)excerpt->data, <); while(sentence) { if (lt != NULL) { savec = *lt; *lt = 0; } #ifdef DEBUG DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)sentence, sizeof(dpsunicode_t) * (DpsUniLen(sentence) + 1)); fprintf(stderr, "Sentence.%d: %s\n", List.nitems, lcstr); #endif if ((sent_len = DpsUniLen(sentence)) >= Indexer->Flags.SEASentenceMinLength) { j = 1; for (i = 0; i < List.nitems; i++) { if (DpsUniStrCmp(sentence, List.Sent[i].sentence) == 0) { j = 0; break; } } if (j) { if ( List.nitems < Indexer->Flags.SEASentences ) { if (List.nitems == List.mitems) { List.mitems += 16; List.Sent = (DPS_SENTENCE*)DpsRealloc(List.Sent, List.mitems * sizeof(DPS_SENTENCE)); if (List.Sent == NULL) { TRACE_OUT(Indexer); return DPS_ERROR;} } List.Sent[List.nitems].sentence = DpsUniDup(sentence); List.Sent[List.nitems].len = sent_len; List.Sent[List.nitems].order = order++; sentence = DpsUniDup(sentence); DpsUniStrToLower(sentence); bzero(&List.Sent[List.nitems].LangMap, sizeof(DPS_LANGMAP)); DpsBuildLangMap(&List.Sent[List.nitems].LangMap, (char*)sentence, sent_len * sizeof(dpsunicode_t), 0, 0); if (sent_len < min_len) { min_len = sent_len; min_pos = List.nitems; } List.nitems++; DPS_FREE(sentence); } else if (sent_len > min_len) { DPS_FREE(List.Sent[min_pos].sentence); List.Sent[min_pos].sentence = DpsUniDup(sentence); List.Sent[min_pos].len = sent_len; List.Sent[min_pos].order = order++; sentence = DpsUniDup(sentence); DpsUniStrToLower(sentence); bzero(&List.Sent[min_pos].LangMap, sizeof(DPS_LANGMAP)); DpsBuildLangMap(&List.Sent[min_pos].LangMap, (char*)sentence, sent_len * sizeof(dpsunicode_t), 0, 0); DPS_FREE(sentence); min_len = List.Sent[0].len; min_pos = 0; for(i = 1; i < List.nitems; i++) if (List.Sent[i].len < min_len) { min_len = List.Sent[i].len; min_pos = i; } } } } #ifdef DEBUG fprintf(stderr, "Sent. len.:%d, Min.allowed: %d\n", sent_len, Indexer->Flags.SEASentenceMinLength); #endif if (lt != NULL) *lt = savec; sentence = DpsUniStrTok_SEA(NULL, <); } DpsLog(Indexer, DPS_LOG_DEBUG, "SEA sentences: %d", List.nitems); if (List.nitems < 4) { for (i = 0; i < List.nitems; i++) DPS_FREE(List.Sent[i].sentence); DPS_FREE(List.Sent); TRACE_OUT(Indexer); return DPS_OK; } links = (double*)DpsMalloc(sizeof(double) * List.nitems * List.nitems); lang_cs = (double*)DpsMalloc(sizeof(double) * List.nitems); /* k ot links[i * List.nitems + j] */ if (links != NULL && lang_cs != NULL) { for (i = 0; i < List.nitems; i++) { DpsPrepareLangMap(&List.Sent[i].LangMap); } for (i = 0; i < List.nitems; i++) { List.Sent[i].Oi = List.Sent[i].di = 0.5; if (Doc->lang_cs_map == NULL) { links[i * List.nitems + i] = 0.0; } else { MapStat.map = &List.Sent[i].LangMap; DpsCheckLangMap6(Doc->lang_cs_map, &List.Sent[i].LangMap, &MapStat, DPS_LM_TOPCNT * DPS_LM_TOPCNT, 2 * DPS_LM_TOPCNT); links[i * List.nitems + i] = (double)MapStat.hits / (2.0 * DPS_LM_TOPCNT) / (List.nitems + 1); } #ifdef DEBUG DpsLog(Indexer, DPS_LOG_INFO, "Link %u->%u: %f [hits:%d miss:%d]", i, i, links[i * List.nitems + i], MapStat.hits, MapStat.miss); #endif for (j = 0; j < List.nitems; j++) { if (j == i) continue; MapStat.map = &List.Sent[j].LangMap; DpsCheckLangMap6(&List.Sent[j].LangMap, &List.Sent[i].LangMap, &MapStat, DPS_LM_TOPCNT * DPS_LM_TOPCNT, 2 * DPS_LM_TOPCNT); links[i * List.nitems + j] = (double)MapStat.hits / (2.0 * DPS_LM_TOPCNT) / (List.nitems + 1); #ifdef DEBUG DpsLog(Indexer, DPS_LOG_INFO, "Link %u->%u: %f [hits:%d miss:%d]", i, j, links[i * List.nitems + j], MapStat.hits, MapStat.miss); #endif } } for (l = 0; l < List.nitems; l++) { w = 0.0; for (i = 0; i < List.nitems; i++) { w += links[l * List.nitems + i] * List.Sent[i].Oi; } w = f(w); if (w < LOW_BORDER_EPS2) w = LOW_BORDER_EPS2; else if (w > HI_BORDER_EPS2) w = HI_BORDER_EPS2; List.Sent[l].di = w; } DpsSort(List.Sent, List.nitems, sizeof(DPS_SENTENCE), (qsort_cmp)SentCmp); #ifdef DEBUG DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)List.Sent[0].sentence, sizeof(dpsunicode_t) * (DpsUniLen(List.Sent[0].sentence) + 1)); fprintf(stderr, "Sent.0: %f %f -- %s\n", List.Sent[0].di, List.Sent[0].Oi, lcstr); DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)List.Sent[1].sentence, sizeof(dpsunicode_t) * (DpsUniLen(List.Sent[1].sentence) + 1)); fprintf(stderr, "Sent.1: %f %f -- %s\n", List.Sent[1].di, List.Sent[1].Oi, lcstr); DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)List.Sent[2].sentence, sizeof(dpsunicode_t) * (DpsUniLen(List.Sent[2].sentence) + 1)); fprintf(stderr, "Sent.2: %f %f -- %s\n", List.Sent[2].di, List.Sent[2].Oi, lcstr); DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)List.Sent[3].sentence, sizeof(dpsunicode_t) * (DpsUniLen(List.Sent[3].sentence) + 1)); fprintf(stderr, "Sent.3: %f %f -- %s\n", List.Sent[3].di, List.Sent[3].Oi, lcstr); DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)List.Sent[4].sentence, sizeof(dpsunicode_t) * (DpsUniLen(List.Sent[4].sentence) + 1)); fprintf(stderr, "Sent.4: %f %f -- %s\n", List.Sent[4].di, List.Sent[4].Oi, lcstr); #endif DpsSort(List.Sent, TOP_SENTENCES, sizeof(DPS_SENTENCE), (qsort_cmp)SentOrderCmp); bzero(&Item, sizeof(Item)); Item.section = seasec; Item.href = NULL; Item.section_name = "sea"; for (i = 0; i < TOP_SENTENCES; i++) { dpsunicode_t *UStr = DpsUniDup(List.Sent[i].sentence); DpsPrepareItem(Indexer, Doc, &Item, List.Sent[i].sentence, UStr, content_lang, indexed_size, indexed_limit, max_word_len, min_word_len, crossec #ifdef HAVE_ASPELL , have_speller, speller, NULL #endif ); DPS_FREE(UStr); } } DPS_FREE(lang_cs); DPS_FREE(links); for (i = 0; i < List.nitems; i++) DPS_FREE(List.Sent[i].sentence); DPS_FREE(List.Sent); TRACE_OUT(Indexer); return DPS_OK; }
void DpsBoolStackFree(DPS_BOOLSTACK *s) { DPS_FREE(s->cstack); DPS_FREE(s->astack); if (s->freeme) DPS_FREE(s); }
int DpsCookiesAdd(DPS_AGENT *Indexer, const char *domain, const char * path, const char *name, const char *value, const char secure, dps_uint4 expires, const char from_config, int insert_flag) { #ifdef HAVE_SQL char buf[3*PATH_MAX]; char path_esc[2*PATH_MAX+1]; DPS_COOKIES *Cookies = &Indexer->Cookies; DPS_COOKIE *Coo; DPS_DB *db; dpshash32_t url_id = DpsStrHash32(domain); size_t i; #ifdef WITH_PARANOIA void *paran = DpsViolationEnter(paran); #endif if (Indexer->flags & DPS_FLAG_UNOCON) { if (Indexer->Conf->dbl.nitems == 0) return DPS_OK; DPS_GETLOCK(Indexer, DPS_LOCK_DB); db = Indexer->Conf->dbl.db[url_id % Indexer->Conf->dbl.nitems]; } else { if (Indexer->dbl.nitems == 0) return DPS_OK; db = Indexer->dbl.db[url_id % Indexer->dbl.nitems]; } (void)DpsDBEscStr(db, path_esc, DPS_NULL2EMPTY(path), dps_min(PATH_MAX,dps_strlen(DPS_NULL2EMPTY(path)))); for (i = 0; i < Cookies->ncookies; i++) { Coo = &Cookies->Cookie[i]; if (!strcasecmp(Coo->domain, domain) && !strcasecmp(Coo->path, DPS_NULL2EMPTY(path)) && !strcasecmp(Coo->name, name) && (Coo->secure == secure)/* && (Coo->from_config == from_config)*/ ) { DPS_FREE(Coo->value); Coo->value = DpsStrdup(value); /* Coo->expires = expires;*/ if (insert_flag) { dps_snprintf(buf, sizeof(buf), "UPDATE cookies SET value='%s',expires=%d WHERE domain='%s' AND path='%s' AND name='%s' AND secure='%c'", value, expires, domain, path_esc, name, secure); DpsSQLAsyncQuery(db, NULL, buf); } if (Indexer->flags & DPS_FLAG_UNOCON) DPS_RELEASELOCK(Indexer, DPS_LOCK_DB); #ifdef WITH_PARANOIA DpsViolationExit(Indexer->handle, paran); #endif return DPS_OK; } } Cookies->Cookie = (DPS_COOKIE*)DpsRealloc(Cookies->Cookie, (Cookies->ncookies + 1) * sizeof(DPS_COOKIE)); if(Cookies->Cookie == NULL) { Cookies->ncookies = 0; if (Indexer->flags & DPS_FLAG_UNOCON) DPS_RELEASELOCK(Indexer, DPS_LOCK_DB); #ifdef WITH_PARANOIA DpsViolationExit(Indexer->handle, paran); #endif return DPS_ERROR; } Coo = &Cookies->Cookie[Cookies->ncookies]; /* Coo->expires = expires;*/ Coo->secure = secure; Coo->from_config = from_config; Coo->domain = DpsStrdup(domain); Coo->path = DpsStrdup(path); Coo->name = DpsStrdup(name); Coo->value = DpsStrdup(value); if (insert_flag) { if (Indexer->Flags.CheckInsertSQL) { dps_snprintf(buf, sizeof(buf), "DELETE FROM cookies WHERE domain='%s' AND path='%s' AND name='%s' AND secure='%c'", domain, path_esc, name, secure); DpsSQLAsyncQuery(db, NULL, buf); } dps_snprintf(buf, sizeof(buf), "INSERT INTO cookies(expires,secure,domain,path,name,value)VALUES(%d,'%c','%s','%s','%s','%s')", expires, secure, domain, path_esc, name, value); DpsSQLAsyncQuery(db, NULL, buf); } Cookies->ncookies++; if (Indexer->flags & DPS_FLAG_UNOCON) DPS_RELEASELOCK(Indexer, DPS_LOCK_DB); #ifdef WITH_PARANOIA DpsViolationExit(Indexer->handle, paran); #endif #endif /*HAVE_SQL*/ return DPS_OK; }
int main(int argc, char ** argv, char **envp) { const char *env, *bcharset, *lcharset, *conf_dir; char template_name[PATH_MAX+6]=""; char *template_filename = NULL; char *query_string = NULL; char self[1024]=""; char *url = NULL; const char *ResultContentType; int res,httpd=0; size_t catcolumns = 0; int page_size,page_number; DPS_ENV *Env; DPS_AGENT *Agent; DPS_VARLIST query_vars; /* Output Content-type if under HTTPD */ /* Some servers do not pass QUERY_STRING */ /* if the query was empty, so check */ /* REQUEST_METHOD too to be safe */ httpd=(getenv("QUERY_STRING")||getenv("REQUEST_METHOD")); if (!(conf_dir=getenv("DPS_ETC_DIR"))) conf_dir=DPS_CONF_DIR; DpsInit(argc, argv, envp); Env=DpsEnvInit(NULL); if (Env == NULL) { if(httpd){ printf("Content-Type: text/plain\r\n\r\n"); } printf("Can't alloc Env\n"); exit(0); } DpsVarListInit(&query_vars); Agent = DpsAgentInit(NULL, Env, 0); if (Agent == NULL) { if(httpd){ printf("Content-Type: text/plain\r\n\r\n"); } printf("Can't alloc Agent\n"); exit(0); } DpsVarListAddEnviron(&Env->Vars,"ENV"); /* Detect self and template name */ if((env = getenv("DPSEARCH_TEMPLATE"))) dps_strncpy(template_name, env, sizeof(template_name) - 1); else if((env = getenv("PATH_INFO")) && env[0]) dps_strncpy(template_name, env + 1, sizeof(template_name) - 1); if((env=getenv("DPSEARCH_SELF"))) dps_strncpy(self,env,sizeof(self)-1); if((env=getenv("QUERY_STRING"))){ query_string = (char*)DpsRealloc(query_string, dps_strlen(env) + 2); if (query_string == NULL) { if(httpd){ printf("Content-Type: text/plain\r\n\r\n"); } printf("Can't alloc query_string\n"); exit(0); } dps_strncpy(query_string, env, dps_strlen(env) + 1); /* Hack for Russian Apache from apache.lexa.ru */ /* QUERY_STRING is already converted to server */ /* character set. We must print original query */ /* string instead however. Under usual apache */ /* we'll use QUERY_STRING. Note that query_vars */ /* list will contain not unescaped values, so */ /* we don't have to escape them when displaying */ env = getenv("CHARSET_SAVED_QUERY_STRING"); DpsParseQStringUnescaped(&query_vars,env?env:query_string); /* Unescape and save variables from QUERY_STRING */ /* Env->Vars will have unescaped values however */ DpsParseQueryString(Agent,&Env->Vars,query_string); template_filename = (char*)DpsStrdup(DpsVarListFindStr(&Env->Vars, "tmplt", "")); if((env=getenv("REDIRECT_STATUS"))){ /* Check Apache internal redirect */ /* via "AddHandler" and "Action" */ if(!self[0]){ dps_strncpy(self,(env=getenv("REDIRECT_URL"))?env:"filler.cgi",sizeof(self)-1); } if(!template_name[0]){ dps_strncpy(template_name,(env=getenv("PATH_TRANSLATED"))?env:"",sizeof(template_name)-1); } if (*template_filename == '\0') { DPS_FREE(template_filename); template_filename = (char*)DpsStrdup("filler.htm"); } }else{ /* CGI executed without Apache internal redirect */ /* Detect $Self variable with OS independant SLASHES */ if(!self[0]){ dps_strncpy(self,(env=getenv("SCRIPT_NAME"))?env:"filler.cgi",sizeof(self)-1); } if(!template_name[0]){ char *s,*e; /*This is with OS specific SLASHES */ env=((env=getenv("SCRIPT_FILENAME"))?env:"filler.cgi"); if(strcmp(conf_dir,".")){ /* Take from the config directory */ dps_snprintf(template_name, sizeof(template_name)-1, "%s/%s", conf_dir,(s=strrchr(env,DPSSLASH))?(s+1):(self)); }else{ /* Take from the current directory */ dps_strncpy(template_name,env,sizeof(template_name)-1); } /* Find right slash if it presents */ s=((s=strrchr(template_name,DPSSLASH))?s:template_name); if (*template_filename == '\0') { /* Find .cgi substring */ if ((e = strstr(s, ".cgi")) != NULL) { /* Replace ".cgi" with ".htm" */ e[1]='h';e[2]='t';e[3]='m'; } else { dps_strcat(s, ".htm"); } e = strrchr(s, '/'); DPS_FREE(template_filename); template_filename = (char*)DpsStrdup(e + 1); } else { dps_strncpy(s + 1, template_filename, sizeof(template_name) - (s - template_name) - 2); } } } }else{ /* Executed from command line */ /* or under server which does not */ /* pass an empty QUERY_STRING var */ if(argv[1]) { query_string = (char*)DpsRealloc(query_string, dps_strlen(argv[1]) + 10); if (query_string == NULL) { if(httpd){ printf("Content-Type: text/plain\r\n\r\n"); } printf("Can't realloc query_string\n"); exit(0); } sprintf(query_string, "q=%s", argv[1]); } else { query_string = (char*)DpsRealloc(query_string, 1024); if (query_string == NULL) { if(httpd){ printf("Content-Type: text/plain\r\n\r\n"); } printf("Can't realloc query_string\n"); exit(0); } sprintf(query_string, "q="); } /* Hack for Russian Apache from apache.lexa.ru */ /* QUERY_STRING is already converted to server */ /* character set. We must print original query */ /* string instead however. Under usual apache */ /* we'll use QUERY_STRING. Note that query_vars */ /* list will contain not unescaped values, so */ /* we don't have to escape them when displaying */ env = getenv("CHARSET_SAVED_QUERY_STRING"); DpsParseQStringUnescaped(&query_vars,env?env:query_string); /* Unescape and save variables from QUERY_STRING */ /* Env->Vars will have unescaped values however */ DpsParseQueryString(Agent,&Env->Vars,query_string); DPS_FREE(template_filename); template_filename = (char*)DpsStrdup(DpsVarListFindStr(&Env->Vars, "tmplt", "")); if (*template_filename == '\0') { DPS_FREE(template_filename); template_filename = (char*)DpsStrdup("filler.htm"); } /*// Get template name from command line variable &tmplt */ if(!template_name[0]) dps_snprintf(template_name,sizeof(template_name),"%s/%s", conf_dir, template_filename); } DpsVarListReplaceStr(&Agent->Conf->Vars, "tmplt", template_filename); DPS_FREE(template_filename); Agent->tmpl.Env_Vars = &Env->Vars; DpsURLNormalizePath(template_name); if (strncmp(template_name, conf_dir, dps_strlen(conf_dir)) || (res = DpsTemplateLoad(Agent, Env, &Agent->tmpl, template_name))) { if (strcmp(template_name, "filler.htm")) { /* trying load default template */ fprintf(stderr, "Can't load template: '%s' %s\n", template_name, Env->errstr); DPS_FREE(template_filename); template_filename = (char*)DpsStrdup("filler.htm"); dps_snprintf(template_name, sizeof(template_name), "%s/%s", conf_dir, template_filename); if ((res = DpsTemplateLoad(Agent, Env, &Agent->tmpl, template_name))) { if(httpd)printf("Content-Type: text/plain\r\n\r\n"); printf("%s\n",Env->errstr); DpsVarListFree(&query_vars); DpsEnvFree(Env); DPS_FREE(query_string); DpsAgentFree(Agent); return(0); } } else { if(httpd)printf("Content-Type: text/plain\r\n\r\n"); printf("%s\n",Env->errstr); DpsVarListFree(&query_vars); DpsEnvFree(Env); DPS_FREE(query_string); DpsAgentFree(Agent); return(0); } } /* set locale if specified */ if ((url = DpsVarListFindStr(&Env->Vars, "Locale", NULL)) != NULL) { setlocale(LC_ALL, url); /*#ifdef HAVE_ASPELL*/ { char *p; if ((p = strchr(url, '.')) != NULL) { *p = '\0'; DpsVarListReplaceStr(&Env->Vars, "g-lc", url); *p = '.'; } } /*#endif*/ url = NULL; } /* Call again to load search Limits if need */ DpsParseQueryString(Agent, &Env->Vars, query_string); Agent->Flags = Env->Flags; Agent->flags |= DPS_FLAG_UNOCON; Env->flags |= DPS_FLAG_UNOCON; DpsSetLogLevel(NULL, DpsVarListFindInt(&Env->Vars, "LogLevel", 0)); DpsOpenLog("filler.cgi", Env, !strcasecmp(DpsVarListFindStr(&Env->Vars, "Log2stderr", (!httpd) ? "yes" : "no"), "yes")); DpsLog(Agent,DPS_LOG_ERROR,"filler.cgi started with '%s'",template_name); DpsLog(Agent, DPS_LOG_DEBUG, "VarDir: '%s'", DpsVarListFindStr(&Agent->Conf->Vars, "VarDir", DPS_VAR_DIR)); DpsLog(Agent, DPS_LOG_DEBUG, "Affixes: %d, Spells: %d, Synonyms: %d, Acronyms: %d, Stopwords: %d", Env->Affixes.naffixes,Env->Spells.nspell, Env->Synonyms.nsynonyms, Env->Acronyms.nacronyms, Env->StopWords.nstopwords); DpsLog(Agent, DPS_LOG_DEBUG, "Chinese dictionary with %d entries", Env->Chi.nwords); DpsLog(Agent, DPS_LOG_DEBUG, "Korean dictionary with %d entries", Env->Korean.nwords); DpsLog(Agent, DPS_LOG_DEBUG, "Thai dictionary with %d entries", Env->Thai.nwords); DpsVarListAddLst(&Agent->Vars, &Env->Vars, NULL, "*"); Agent->tmpl.Env_Vars = &Agent->Vars; /* DpsVarListAddEnviron(&Agent->Vars, "ENV");*/ /****************************************************************************************************************************************/ /* This is for query tracking */ DpsVarListAddStr(&Agent->Vars, "QUERY_STRING", query_string); DpsVarListAddStr(&Agent->Vars, "self", self); env = getenv("HTTP_X_FORWARDER_FOR"); if (env) { DpsVarListAddStr(&Agent->Vars, "IP", env); } else { env = getenv("REMOTE_ADDR"); DpsVarListAddStr(&Agent->Vars, "IP", env ? env : "localhost"); } bcharset = DpsVarListFindStr(&Agent->Vars, "BrowserCharset", "iso-8859-1"); Env->bcs=DpsGetCharSet(bcharset); lcharset = DpsVarListFindStr(&Agent->Vars, "LocalCharset", "iso-8859-1"); Env->lcs=DpsGetCharSet(lcharset); ResultContentType = DpsVarListFindStr(&Agent->Vars, "ResultContentType", "text/html"); if(httpd){ if(!Env->bcs){ printf("Content-Type: text/plain\r\n\r\n"); printf("Unknown BrowserCharset '%s' in template '%s'\n",bcharset,template_name); exit(0); }else if(!Env->lcs){ printf("Content-Type: text/plain\r\n\r\n"); printf("Unknown LocalCharset '%s' in template '%s'\n",lcharset,template_name); exit(0); }else{ printf("Content-type: %s; charset=%s\r\n\r\n", ResultContentType, bcharset); } }else{ if(!Env->bcs){ printf("Unknown BrowserCharset '%s' in template '%s'\n",bcharset,template_name); exit(0); } if(!Env->lcs){ printf("Unknown LocalCharset '%s' in template '%s'\n",lcharset,template_name); exit(0); } } /* These parameters taken from "variable section of template"*/ res = DpsVarListFindInt(&Agent->Vars, "ps", DPS_DEFAULT_PS); page_size = dps_min(res, MAX_PS); page_number = DpsVarListFindInt(&Agent->Vars, "p", 0); if (page_number == 0) { page_number = DpsVarListFindInt(&Agent->Vars, "np", 0); DpsVarListReplaceInt(&Agent->Vars, "p", page_number + 1); } else page_number--; res = DpsVarListFindInt(&Agent->Vars, "np", 0) * page_size; DpsVarListAddInt(&Agent->Vars, "pn", res); catcolumns = (size_t)atoi(DpsVarListFindStr(&Agent->Vars, "CatColumns", "")); DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "top"); DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "restop"); DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "res"); DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "resbot"); DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "bottom"); DpsVarListFree(&query_vars); DpsAgentFree(Agent); DpsEnvFree(Env); DPS_FREE(query_string); DPS_FREE(url); if (httpd) fflush(NULL); else fclose(stdout); #ifdef EFENCE fprintf(stderr, "Memory leaks checking\n"); DpsEfenceCheckLeaks(); #endif #ifdef FILENCE fprintf(stderr, "FD leaks checking\n"); DpsFilenceCheckLeaks(NULL); #endif return DPS_OK; }
extern __C_LINK int __DPSCALL DpsBaseOptimize(DPS_BASE_PARAM *P, int sbase) { struct stat sb; urlid_t base, base_from, base_to; long unsigned ActualSize, OriginalSize, i, nitems; off_t pos, posold, NewItemPos, SSize; dps_uint8 diff, gain; double dr = 0.0, cr = 0.0; ssize_t nread; size_t rsize; ssize_t wr; int OptimizeRatio, res, error_cnt; char buffer[BUFSIZ]; DPS_BASEITEM *hTable; DPS_SORTBASEITEM *si = NULL; OptimizeRatio = DpsVarListFindInt(&P->A->Vars, "OptimizeRatio", 15); P->mode = DPS_WRITE_LOCK; if (sbase == -1) { base_from = 0; base_to = (urlid_t)P->NFiles; } else { base_from = sbase; base_to = sbase + 1; } for (base = base_from; base < base_to; base++) { error_cnt = 0; gain = (dps_uint8)0; P->rec_id = ((base & DPS_BASE_MASK) << DPS_BASE_BITS); if (DpsBaseOpen(P, DPS_WRITE_LOCK) != DPS_OK) { DpsLog(P->A, DPS_LOG_ERROR, "Can't open base %s/%s {%s:%d}", P->subdir, P->basename, __FILE__, __LINE__); DpsBaseClose(P); return DPS_ERROR; } if (lseek(P->Ifd, (off_t)0, SEEK_SET) == (off_t)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seek %s {%s:%d}", P->Ifilename, __FILE__, __LINE__); DpsBaseClose(P); return DPS_ERROR; } if (fstat(P->Sfd, &sb) == 0) { SSize = sb.st_size; } else { if ((SSize = (off_t)lseek(P->Sfd, (off_t)0, SEEK_END)) == (off_t)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seek %s {%s:%d}", P->Sfilename, __FILE__, __LINE__); DpsBaseClose(P); return DPS_ERROR; } } nitems = 0; ActualSize = 0; OriginalSize = 0; while(read(P->Ifd, &P->Item, sizeof(DPS_BASEITEM)) == sizeof(DPS_BASEITEM)) { nitems++; if ((P->Item.rec_id != 0) && ((dps_uint8)P->Item.offset < (dps_uint8)SSize) && (P->Item.size > 0)) { ActualSize += (long unsigned)P->Item.size; OriginalSize += (long unsigned)(P->Item.orig_size ? P->Item.orig_size : P->Item.size); } } if (ftruncate(P->Ifd, (off_t)(nitems * sizeof(DPS_BASEITEM))) != 0) { dps_strerror(P->A, DPS_LOG_EXTRA, "ftruncate error (pos:%ld) [%s:%d]", (off_t)(nitems * sizeof(DPS_BASEITEM)), __FILE__, __LINE__); } dr = (nitems) ? fabs(100.0 * ((long unsigned)SSize - ActualSize) / ((double)SSize + 1.0)) : 0.0; cr = (nitems) ? fabs(100.0 * ActualSize / (OriginalSize + 1)) : 0.0; DpsLog(P->A, DPS_LOG_EXTRA, "Optimize: %s/%s base 0x%X, %ld recs defrag: %.2f%% Ratio: %.2f%% Data: %ld File: %ld", P->subdir, P->basename, P->FileNo, nitems, dr, cr, ActualSize, (long)SSize); if ((dr >= (double)OptimizeRatio) || (ActualSize == 0 && SSize != 0)) { si = (DPS_SORTBASEITEM*)DpsMalloc((nitems + 1) * sizeof(DPS_SORTBASEITEM)); if (si == NULL) { DpsLog(P->A, DPS_LOG_ERROR, "Can't alloc si (%d bytes) at {%s:%d}", (nitems + 1) * sizeof(DPS_SORTBASEITEM), __FILE__, __LINE__); DpsBaseClose(P); return DPS_ERROR; } if (lseek(P->Ifd, (off_t)0, SEEK_SET) == (off_t)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seek %s {%s:%d}", P->Ifilename, __FILE__, __LINE__); DpsBaseClose(P); DPS_FREE(si); return DPS_ERROR; } for (i = 0; (i < nitems) && (read(P->Ifd, &si[i].Item, sizeof(DPS_BASEITEM)) == sizeof(DPS_BASEITEM)); ) { if(si[i].Item.rec_id != 0 && ((dps_uint8)si[i].Item.offset < (dps_uint8)SSize) && (si[i].Item.size > 0) && (si[i].Item.size < ActualSize) ) { i++; } } if (i < nitems) nitems = i; if (nitems > 1) DpsSort((void*)si, (size_t)nitems, sizeof(DPS_SORTBASEITEM), cmpsi); gain = (dps_uint8)0; pos = (off_t)0; posold = (off_t)0; if (nitems > 0) { if ((long unsigned)si[0].Item.offset < (long unsigned)SSize) { posold = (off_t)si[0].Item.offset; } else { si[0].Item.offset = (off_t)0; si[0].Item.size = 0; } } if (nitems > 1) { if (si[0].Item.size > (rsize = (size_t)(si[1].Item.offset - si[0].Item.offset))) { DpsLog(P->A, DPS_LOG_ERROR, "si[0] size adjusted by offset: %ld -> %ld", (long)si[0].Item.size, (long)rsize); si[0].Item.size = rsize; error_cnt++; } } if ((diff = (dps_uint8)posold) > 0) { for( lseek(P->Sfd, posold, SEEK_SET), rsize = 0; (rsize < si[0].Item.size) && ((nread = read(P->Sfd, buffer, (rsize + BUFSIZ < si[0].Item.size) ? BUFSIZ : (si[0].Item.size - rsize) )) > 0); lseek(P->Sfd, posold, SEEK_SET) ) { lseek(P->Sfd, pos, SEEK_SET); (void)write(P->Sfd, buffer, (size_t)nread); rsize += (size_t)nread; posold += (off_t)nread; pos += (off_t)nread; } si[0].Item.offset = 0; if (rsize != si[0].Item.size) { DpsLog(P->A, DPS_LOG_ERROR, "si[0] size adjusted by size: %ld -> %ld", (long)si[0].Item.size, (long)rsize); si[0].Item.size = rsize; error_cnt++; } gain += diff; } if (nitems > 0) for (i = 0; i < nitems - 1; i++) { if ((long unsigned)si[i + 1].Item.offset > (long unsigned)SSize) { DpsLog(P->A, DPS_LOG_ERROR, "si[%ld] too long offset: %ld > %ld, removing", i , (long)si[i + 1].Item.offset, (long)SSize); si[i + 1].Item.size = 0; si[i + 1].Item.offset = si[i].Item.offset + si[i].Item.size; error_cnt++; } else { pos = (off_t)(si[i].Item.offset + si[i].Item.size); posold = (off_t)si[i + 1].Item.offset; if (i < nitems - 2) { if (si[i + 1].Item.size > (rsize = (size_t)(si[i + 2].Item.offset - si[i + 1].Item.offset))) { DpsLog(P->A, DPS_LOG_ERROR, "si[%ld] size adjusted by offset: %ld -> %ld", i + 1, (long)si[i + 1].Item.size, (long)rsize ); si[i + 1].Item.size = rsize; error_cnt++; } } if ((diff = (dps_uint8)posold - (dps_uint8)pos) > 0) { for( lseek(P->Sfd, posold, SEEK_SET), rsize = 0; (rsize < si[i + 1].Item.size) && ((nread = read(P->Sfd, buffer, (rsize + BUFSIZ < si[i + 1].Item.size) ? BUFSIZ : (si[i + 1].Item.size - rsize) )) > 0); lseek(P->Sfd, posold, SEEK_SET) ) { lseek(P->Sfd, pos, SEEK_SET); (void)write(P->Sfd, buffer, (size_t)nread); rsize += (size_t)nread; posold += (off_t)nread; pos += (off_t)nread; } if (rsize != si[i + 1].Item.size) { DpsLog(P->A, DPS_LOG_ERROR, "si[%ld] size adjusted by size: %ld -> %ld", i + 1, (long)si[i + 1].Item.size, (long)rsize); si[i + 1].Item.size = rsize; error_cnt++; } si[i + 1].Item.offset = si[i].Item.offset + si[i].Item.size; gain += diff; } } } posold = SSize; pos = (nitems) ? (off_t)(si[nitems - 1].Item.offset + si[nitems - 1].Item.size) : (off_t)0; if (ftruncate(P->Sfd, (off_t)(pos)) != 0) { dps_strerror(P->A, DPS_LOG_ERROR, "ftruncate error (pos:%ld) [%s:%d]", pos, __FILE__, __LINE__); } SSize = pos; if (posold > pos) { gain += ((dps_uint8)posold - (dps_uint8)pos); } /*if (gain != 0 || OptimizeRatio == 0 || error_cnt > 0)*/ { posold = lseek(P->Ifd, (off_t)0, SEEK_END); (void)ftruncate(P->Ifd, (off_t)0); lseek(P->Ifd, (off_t)0, SEEK_SET); if ((hTable = (DPS_BASEITEM *)DpsXmalloc(sizeof(DPS_BASEITEM) * DPS_HASH_PRIME)) == NULL) { DpsLog(P->A, DPS_LOG_ERROR, "Memory alloc error hTable: %d bytes", sizeof(DPS_BASEITEM) * DPS_HASH_PRIME); DpsBaseClose(P); DPS_FREE(si); return DPS_ERROR; } if ( (wr = write(P->Ifd, hTable, sizeof(DPS_BASEITEM) * DPS_HASH_PRIME)) != sizeof(DPS_BASEITEM) * DPS_HASH_PRIME) { dps_strerror(P->A, DPS_LOG_ERROR, "[%s:%d] Can't set new index for file %s\nwritten %d bytes of %d", __FILE__, __LINE__, P->Ifilename, wr, sizeof(DPS_BASEITEM) * DPS_HASH_PRIME); DPS_FREE(hTable); DpsBaseClose(P); DPS_FREE(si); return DPS_ERROR; } DPS_FREE(hTable); for (i = 0; i < nitems; i++) { if (si[i].Item.rec_id == 0 || si[i].Item.size == 0) continue; if ((long)si[i].Item.offset > (long)SSize) { DpsLog(P->A, DPS_LOG_ERROR, "si[%ld] too long offset: %ld > %ld, removing", i , (long)si[i].Item.offset, (long)SSize); error_cnt++; continue; } P->rec_id = si[i].Item.rec_id; if ((res = DpsBaseSeek(P, DPS_WRITE_LOCK)) != DPS_OK) { DpsBaseClose(P); DPS_FREE(si); return res; } if (P->Item.rec_id != P->rec_id) { if (P->mishash && P->Item.rec_id != 0) { if ((P->Item.next = (dps_uint8)(NewItemPos = lseek(P->Ifd, (off_t)0, SEEK_END))) == (dps_uint8)-1) { DpsBaseClose(P); DPS_FREE(si); return DPS_ERROR; } if (lseek(P->Ifd, (off_t)P->CurrentItemPos, SEEK_SET) == (off_t)-1) { DpsBaseClose(P); DPS_FREE(si); return DPS_ERROR; } if (write(P->Ifd, &P->Item, sizeof(DPS_BASEITEM)) != sizeof(DPS_BASEITEM)) { DpsBaseClose(P); DPS_FREE(si); return DPS_ERROR; } P->CurrentItemPos = (dps_uint8)NewItemPos; } } P->Item = si[i].Item; P->Item.next = (off_t)0; if (lseek(P->Ifd, (off_t)P->CurrentItemPos, SEEK_SET) == (off_t)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seek %s {%s:%d}", P->Ifilename, __FILE__, __LINE__); DpsBaseClose(P); DPS_FREE(si); return DPS_ERROR; } if (write(P->Ifd, &P->Item, sizeof(DPS_BASEITEM)) != sizeof(DPS_BASEITEM)) { DpsLog(P->A, DPS_LOG_ERROR, "Can't write index for file %s {%s:%d}", P->Ifilename, __FILE__, __LINE__); DpsBaseClose(P); DPS_FREE(si); return DPS_ERROR; } } pos = lseek(P->Ifd, (off_t)0, SEEK_END); gain += ((dps_uint8)posold - (dps_uint8)pos); DpsLog(P->A, DPS_LOG_DEBUG, "Optimize: %s/%s base 0x%X cleaned, %ld bytes freed", P->subdir, P->basename, base, gain); } DPS_FREE(si); } if (error_cnt) base--; DpsBaseClose(P); } return DPS_OK; }
int DpsCarryLimitWrite(DPS_AGENT *Indexer, FILE *f, urlid_t *data, size_t num) { unsigned char *bits; urlid_t curr = 0; urlid_t *a; size_t i, n, z; int j; size_t avail, elems; unsigned size, max_bits; unsigned char *table, *base; unsigned global_max, global_sum, global_n; /* allocating mem for bits[i] - minimal bits needed to code a[i] */ if (! (bits = (unsigned char*)malloc(ELEMS_PER_BLOCK * sizeof(unsigned char)))) { DpsLog(Indexer, DPS_LOG_ERROR, "Out of memory [%s:%d]", __FILE__, __LINE__); return DPS_ERROR; } if (! (a = (urlid_t*) malloc(ELEMS_PER_BLOCK * sizeof(urlid_t)))) { DpsLog(Indexer, DPS_LOG_ERROR, "Out of memory [%s:%d]", __FILE__, __LINE__); DPS_FREE(bits); return DPS_ERROR; } CARRY_ENCODE_START(f); size = TRANS_TABLE_STARTER; global_max = global_sum = global_n = 0; for (z = 0; z < num; ) { for (n = 0; (n < ELEMS_PER_BLOCK) && (z < num); n++,z++) { a[n] = data[z] - curr; curr = data[z]; } max_bits = CalcMinBits(a, bits, n, &global_max, &global_sum, &global_n); CARRY_BLOCK_ENCODE_START(n, max_bits); for (i=0; i<n; ) { avail = GET_AVAILABLE_BITS; table = GET_TRANS_TABLE(avail); base= table+(size<<2); /* row in trans table */ /* 1. Modeling: Find j= the first-fit column in base */ for (j=0; j<4; j++) { size = base[j]; if (size >avail) /* must use next word for data */ { avail=32; j=-1; continue; } if ( elems=elems_coded(avail,size,bits,i,n-1) ) break; } /* 2. Coding: Code elements using row "base" & column "j" */ WORD_ENCODE(j+1,2); /* encoding column */ for ( ; elems ; elems--, i++) /* encoding d-gaps */ WORD_ENCODE(a[i],size); } } CARRY_ENCODE_END; DPS_FREE(a); DPS_FREE(bits); return DPS_OK; }
__C_LINK void * __DPSCALL DpsBaseARead(DPS_BASE_PARAM *P, size_t *len) { int res = DPS_OK; char *buf = NULL; #ifdef HAVE_ZLIB Byte *CDoc = NULL; z_stream zstream; #endif if ((res = DpsBaseSeek(P, DPS_READ_LOCK)) != DPS_OK) { *len = 0; return NULL; } if (P->Item.rec_id == P->rec_id) { if (lseek(P->Sfd, (off_t)P->Item.offset, SEEK_SET) == (off_t)-1) { *len = 0; return NULL; } #ifdef HAVE_ZLIB bzero(&zstream, sizeof(zstream)); if ((P->zlib_method == Z_DEFLATED) && (P->Item.orig_size != 0)) { zstream.avail_in = (uInt)P->Item.size; *len = zstream.avail_out = (uInt)(2 * P->Item.size + P->Item.orig_size); CDoc = zstream.next_in = (Byte *) DpsMalloc(P->Item.size + 1); if (CDoc == NULL) { *len = 0; return NULL; } if ((buf = (char*)DpsMalloc(*len + 1)) == NULL) { DPS_FREE(CDoc); *len = 0; return NULL; } /* fprintf(stderr, "BaseARead: Item.size: %d .orig_size: %d len: %d\n", P->Item.size, P->Item.orig_size, *len);*/ zstream.next_out = (Byte *) buf; zstream.zalloc = Z_NULL; zstream.zfree = Z_NULL; zstream.opaque = Z_NULL; if (read(P->Sfd, CDoc, P->Item.size) != (ssize_t)P->Item.size) { DpsLog(P->A, DPS_LOG_ERROR, "[%s/%s] %d read error, rec_id: %x, deleting... -- %d", P->subdir, P->basename, P->Item.size, P->rec_id, __LINE__); DpsBaseDelete(P); DPS_FREE(buf); DPS_FREE(CDoc); return NULL; } inflateInit2(&zstream, P->zlib_windowBits); res = inflate(&zstream, Z_FINISH); /* fprintf(stderr, "inflate exit: %d avail_out: %d total_out: %d avail_in: %d\n", res, zstream.avail_out, zstream.total_out, zstream.avail_in);*/ ; *len = zstream.total_out; inflateEnd(&zstream); DPS_FREE(CDoc); } else #endif { if ((buf = (char*)DpsMalloc((*len = P->Item.size) + 1)) == NULL) { *len = 0; return NULL; } if (read(P->Sfd, buf, P->Item.size) != (ssize_t)P->Item.size) { DpsFree(buf); *len = 0; return NULL; } } } else { DpsLog(P->A, DPS_LOG_DEBUG, "%s:[%s/%s] Not found rec_id: %x", P->vardir, P->subdir, P->basename, P->rec_id); *len = 0; return NULL; } buf[*len] = '0'; #ifdef DEBUG_SEARCH DpsLog(P->A, DPS_LOG_DEBUG, "[%s/%s] ARetrieved rec_id: %x Size: %d->%d", P->subdir, P->basename, P->rec_id, P->Item.size, P->Item.orig_size); #endif return buf; }
__C_LINK int __DPSCALL DpsBaseCheckup(DPS_BASE_PARAM *P, int (*checkrec) (DPS_AGENT *A, const urlid_t rec_id)) { int found; urlid_t i; size_t z; urlid_t *todel = (int*)DpsMalloc(128 * sizeof(urlid_t)); size_t ndel = 0, mdel = 128, totaldel = 0; if (todel == NULL) return DPS_ERROR; for (i = 0; i < (urlid_t)P->NFiles; i++) { if (have_sigterm || have_sigint || have_sigalrm) { DpsLog(P->A, DPS_LOG_EXTRA, "%s signal received. Exiting chackup", (have_sigterm) ? "SIGTERM" : (have_sigint) ? "SIGINT" : "SIGALRM"); DpsBaseClose(P); DPS_FREE(todel); return DPS_OK; } P->rec_id = i << DPS_BASE_BITS; if (DpsBaseOpen(P, DPS_READ_LOCK) != DPS_OK) { DpsBaseClose(P); continue; } if (lseek(P->Ifd, (off_t)0, SEEK_SET) == (off_t)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seeek for file %s", P->Ifilename); DpsBaseClose(P); DPS_FREE(todel); return DPS_ERROR; } while (read(P->Ifd, &P->Item, sizeof(DPS_BASEITEM)) == sizeof(DPS_BASEITEM)) { if (P->Item.rec_id != 0) { found = checkrec(P->A, P->Item.rec_id); if (found == 0) { if (ndel >= mdel) { mdel += 128; todel = (urlid_t*)DpsRealloc(todel, mdel * sizeof(urlid_t)); if (todel == NULL) { DpsBaseClose(P); DpsLog(P->A, DPS_LOG_ERROR, "Can't realloc %d bytes %s:%d", mdel * sizeof(urlid_t), __FILE__, __LINE__); DPS_FREE(todel); return DPS_ERROR; } } todel[ndel++] = P->Item.rec_id; } } } DpsBaseClose(P); for (z = 0; z < ndel; z++) { DpsLog(P->A, DPS_LOG_DEBUG, "Base %s/%s store %03X: deleting url_id: %X", P->subdir, P->basename, i, todel[z]); P->rec_id = todel[z]; DpsBaseDelete(P); } DpsBaseClose(P); DpsLog(P->A, DPS_LOG_INFO, "Base %s/%s store %03X, %d lost records deleted", P->subdir, P->basename, i, ndel); totaldel += ndel; ndel = 0; } DPS_FREE(todel); DpsLog(P->A, DPS_LOG_EXTRA, "Total lost record(s) deleted: %d\n", totaldel); return DPS_OK; }
__C_LINK int __DPSCALL DpsBaseOpen(DPS_BASE_PARAM *P, int mode) { unsigned int hash; size_t filenamelen, z; ssize_t wr; DPS_BASEITEM *hTable; #ifdef DEBUG_SEARCH unsigned long total_ticks, stop_ticks, start_ticks = DpsStartTimer(); #endif TRACE_IN(P->A, "DpsBaseOpen"); if (P->opened) DpsBaseClose(P); if (P->NFiles == 0) P->NFiles = DpsVarListFindUnsigned(&P->A->Vars, "BaseFiles", 0x100); P->FileNo = DPS_FILENO(P->rec_id, P->NFiles); hash = DPS_HASH(P->rec_id); filenamelen = dps_strlen(P->vardir) + dps_strlen(P->subdir) + dps_strlen(P->indname) + dps_strlen(P->basename) + 48; if ( ((P->Ifilename = (char *)DpsMalloc(filenamelen)) == NULL) || ((P->Sfilename = (char *)DpsMalloc(filenamelen)) == NULL) ) { DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); DpsLog(P->A, DPS_LOG_ERROR, "Memory alloc error 2x%d bytes %s:%d", filenamelen, __FILE__, __LINE__); TRACE_OUT(P->A); return DPS_ERROR; } sprintf(P->Sfilename, "%s/%s/%s%04zx.s", P->vardir, P->subdir, P->basename, P->FileNo); sprintf(P->Ifilename, "%s/%s/%s%04zx.i", P->vardir, P->subdir, P->indname, P->FileNo); if ((P->Ifd = DpsOpen2(P->Ifilename, ((mode == DPS_READ_LOCK) ? O_RDONLY : O_RDWR) | DPS_BINARY)) < 0) { if ((mode == DPS_READ_LOCK) || ((P->Ifd = DpsOpen3(P->Ifilename, O_RDWR | O_CREAT | DPS_BINARY /*#ifdef O_DIRECT | O_DIRECT #endif*/ , S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH )) < 0)) { dps_strerror(P->A, (mode == DPS_READ_LOCK && errno == ENOENT) ? DPS_LOG_DEBUG : DPS_LOG_ERROR, "Can't open/create file %s for %s [%s:%d]", P->Ifilename, (mode == DPS_READ_LOCK) ? "read" : "write", __FILE__, __LINE__); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } #if 1 DPS_GETLOCK(P->A, DPS_LOCK_BASE_N(P->FileNo)); #endif DpsWriteLock(P->Ifd); if ((hTable = (DPS_BASEITEM *)DpsXmalloc(sizeof(DPS_BASEITEM) * DPS_HASH_PRIME)) == NULL) { DpsLog(P->A, DPS_LOG_ERROR, "Memory alloc error hTable: %d bytes", sizeof(DPS_BASEITEM) * DPS_HASH_PRIME); DpsUnLock(P->Ifd); #if 1 DPS_RELEASELOCK(P->A, DPS_LOCK_BASE_N(P->FileNo)); #endif DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } if ( (wr = write(P->Ifd, hTable, sizeof(DPS_BASEITEM) * DPS_HASH_PRIME)) != sizeof(DPS_BASEITEM) * DPS_HASH_PRIME) { dps_strerror(P->A, DPS_LOG_ERROR, "Can't set new index for file %s\nwritten %d bytes of %d\nIfd:%d hTable:%x", P->Ifilename, wr, sizeof(DPS_BASEITEM) * DPS_HASH_PRIME, P->Ifd, hTable); DPS_FREE(hTable); DpsUnLock(P->Ifd); #if 1 DPS_RELEASELOCK(P->A, DPS_LOCK_BASE_N(P->FileNo)); #endif DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } DpsUnLock(P->Ifd); #if 1 DPS_RELEASELOCK(P->A, DPS_LOCK_BASE_N(P->FileNo)); #endif DPS_FREE(hTable); if (lseek(P->Ifd, (off_t)0, SEEK_SET) == (off_t)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seek for file %s", P->Ifilename); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } } if (!P->A->Flags.cold_var) { #if 1 DPS_GETLOCK(P->A, DPS_LOCK_BASE_N(P->FileNo)); #endif switch (mode) { case DPS_READ_LOCK: DpsReadLock(P->Ifd); break; case DPS_WRITE_LOCK: DpsWriteLock(P->Ifd); break; } P->locked = 1; } if ((P->Sfd = DpsOpen2(P->Sfilename, ((mode == DPS_READ_LOCK) ? O_RDONLY : O_RDWR) | DPS_BINARY /*#ifdef O_DIRECT | O_DIRECT #endif*/ )) < 0) { if ((mode == DPS_READ_LOCK) || ((P->Sfd = DpsOpen3(P->Sfilename, O_RDWR | O_CREAT | DPS_BINARY /*#ifdef O_DIRECT | O_DIRECT #endif*/ , S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH )) < 0)) { DpsLog(P->A, DPS_LOG_ERROR, "Can't open/create file %s", P->Sfilename); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } } if (!P->A->Flags.cold_var) { switch(mode) { case DPS_READ_LOCK: DpsReadLock(P->Sfd); break; case DPS_WRITE_LOCK: DpsWriteLock(P->Sfd); break; } } #ifdef DEBUG_SEARCH stop_ticks = DpsStartTimer(); total_ticks = stop_ticks - start_ticks; DpsLog(P->A, DPS_LOG_EXTRA, "OpenBase1 %03X in %.5f sec.", P->FileNo, (float)total_ticks / 1000); #endif for (z = 0; z < 3; z++) { /* search rec_id */ if ( (P->CurrentItemPos = (dps_uint8)lseek(P->Ifd, (off_t)(hash * sizeof(DPS_BASEITEM)), SEEK_SET)) == (dps_uint8)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seeek for file %s", P->Ifilename); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } if (read(P->Ifd, &P->Item, sizeof(DPS_BASEITEM)) != sizeof(DPS_BASEITEM)) { DpsLog(P->A, DPS_LOG_ERROR, "{%s:%d} Can't read index for file %s seek:%ld hash: %u (%d)", __FILE__, __LINE__, P->Ifilename, P->CurrentItemPos, hash, hash); bzero(&P->Item, sizeof(P->Item)); /* DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; */ } #ifdef DEBUG_SEARCH stop_ticks = DpsStartTimer(); total_ticks = stop_ticks - start_ticks; DpsLog(P->A, DPS_LOG_EXTRA, "OpenBase2 %03X in %.5f sec.", P->FileNo, (float)total_ticks / 1000); #endif if (P->Item.rec_id == P->rec_id || P->Item.rec_id == 0) P->mishash = 0; else P->mishash = 1; P->PreviousItemPos = P->CurrentItemPos; if (P->mishash) while((P->Item.next != 0) && (P->Item.rec_id != P->rec_id)) { P->PreviousItemPos = P->CurrentItemPos; P->CurrentItemPos = P->Item.next; if (lseek(P->Ifd, (off_t)P->CurrentItemPos, SEEK_SET) == (off_t)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seek for file %s", P->Ifilename); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } if ((wr = read(P->Ifd, &P->Item, sizeof(DPS_BASEITEM))) != sizeof(DPS_BASEITEM)) { if (wr == 0) { DpsLog(P->A, DPS_LOG_ERROR, "Possible corrupted hash chain for file %s, trying to restore (%s:%d)", P->Ifilename, __FILE__, __LINE__); if (lseek(P->Ifd, (off_t)P->PreviousItemPos, SEEK_SET) == (off_t)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seek for file %s (%s:%d)", P->Ifilename, __FILE__, __LINE__); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } if ((wr = read(P->Ifd, &P->Item, sizeof(DPS_BASEITEM))) != sizeof(DPS_BASEITEM)) { DpsLog(P->A, DPS_LOG_ERROR, "Can't read previous pos for file %s (%s:%d)", P->Ifilename, __FILE__, __LINE__); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } P->Item.next = 0; if (lseek(P->Ifd, (off_t)P->PreviousItemPos, SEEK_SET) == (off_t)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seeek for file %s (%s:%d)", P->Ifilename, __FILE__, __LINE__); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } if ((wr = write(P->Ifd, &P->Item, sizeof(DPS_BASEITEM))) != sizeof(DPS_BASEITEM)) { DpsLog(P->A, DPS_LOG_ERROR, "Can't write previous pos for file %s (%s:%d)", P->Ifilename, __FILE__, __LINE__); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } goto search_again; } else { DpsLog(P->A, DPS_LOG_ERROR, "Can't read hash chain for file %s %d of %d bytes (%s:%d)", P->Ifilename, wr, sizeof(DPS_BASEITEM), __FILE__, __LINE__); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } } #ifdef DEBUG_SEARCH stop_ticks = DpsStartTimer(); total_ticks = stop_ticks - start_ticks; DpsLog(P->A, DPS_LOG_EXTRA, "OpenBase3 %03X in %.5f sec.", P->FileNo, (float)total_ticks / 1000); #endif } break; search_again:; } P->opened = 1; P->mode = mode; #ifdef DEBUG_SEARCH stop_ticks = DpsStartTimer(); total_ticks = stop_ticks - start_ticks; DpsLog(P->A, DPS_LOG_EXTRA, "OpenBase4 %03X in %.5f sec.\n", P->FileNo, (float)total_ticks / 1000); #endif /* fprintf(stderr, "Sfd:0x%x - %s\n", P->Sfd, P->Sfilename); fprintf(stderr, "Ifd:0x%x - %s\n", P->Ifd, P->Ifilename);*/ TRACE_OUT(P->A); return DPS_OK; }
__C_LINK int __DPSCALL DpsBaseWrite(DPS_BASE_PARAM *P, void *buffer, size_t len) { dps_uint8 NewItemPos; int res = DPS_OK; size_t size = len; size_t orig_size = 0; void *data = buffer; #ifdef HAVE_ZLIB z_stream zstream; Byte *CData = NULL; bzero(&zstream, sizeof(zstream)); zstream.zalloc = Z_NULL; zstream.zfree = Z_NULL; zstream.opaque = Z_NULL; zstream.next_in = buffer; if ( (P->zlib_method == Z_DEFLATED) && (deflateInit2(&zstream, P->zlib_level, Z_DEFLATED, P->zlib_windowBits, P->zlib_memLevel, P->zlib_strategy) == Z_OK) ) { zstream.avail_in = (uInt)len; zstream.avail_out = (uInt)(/*sizeof(gz_header) +*/ 4096 + 2 * len); CData = zstream.next_out = (Byte *) DpsMalloc(zstream.avail_out); if (zstream.next_out == NULL) { return DPS_ERROR; } deflate(&zstream, Z_FINISH); deflateEnd(&zstream); orig_size = len; size = zstream.total_out; data = CData; } #endif if ((res = DpsBaseSeek(P, DPS_WRITE_LOCK)) != DPS_OK) { goto DpsBaseWrite_exit; } if (P->Item.rec_id == P->rec_id) { if (P->Item.size < size) { if ((P->Item.offset = (dps_uint8)lseek(P->Sfd, (off_t)0, SEEK_END)) == (dps_uint8)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seek file %s {%s:%d}", P->Sfilename, __FILE__, __LINE__); res = DPS_ERROR; goto DpsBaseWrite_exit; } } else { if (lseek(P->Sfd, (off_t)P->Item.offset, SEEK_SET) == (off_t)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seek file %s offset %ld {%s:%d}", P->Sfilename, (long)P->Item.offset, __FILE__, __LINE__); res = DPS_ERROR; goto DpsBaseWrite_exit; } } } else { /* new rec_id added */ if (P->mishash && P->Item.rec_id != 0) { if ((P->Item.next = NewItemPos = (dps_uint8)lseek(P->Ifd, (off_t)0, SEEK_END)) == (dps_uint8)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seek file %s {%s:%d}", P->Ifilename, __FILE__, __LINE__); res = DPS_ERROR; goto DpsBaseWrite_exit; } if (lseek(P->Ifd, (off_t)P->CurrentItemPos, SEEK_SET) == (off_t)-1) { res = DPS_ERROR; goto DpsBaseWrite_exit; } if (write(P->Ifd, &P->Item, sizeof(DPS_BASEITEM)) != sizeof(DPS_BASEITEM)) { res = DPS_ERROR; goto DpsBaseWrite_exit; } P->CurrentItemPos = NewItemPos; P->Item.next = (off_t)0; } P->Item.rec_id = P->rec_id; if ((P->Item.offset = (dps_uint8)lseek(P->Sfd, (off_t)0, SEEK_END)) == (dps_uint8)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seek file %s {%s:%d}", P->Sfilename, __FILE__, __LINE__); res = DPS_ERROR; goto DpsBaseWrite_exit; } } if (write(P->Sfd, data, size) != (ssize_t)size) { dps_strerror(P->A, DPS_LOG_ERROR, "Can't write %ld bytes at %ld of file %s {%s:%d}", (long)size, (long)P->Item.offset, P->Sfilename, __FILE__, __LINE__); res = DPS_ERROR; goto DpsBaseWrite_exit; } if (lseek(P->Ifd, (off_t)P->CurrentItemPos, SEEK_SET) == (off_t)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seek file %s {%s:%d}", P->Ifilename, __FILE__, __LINE__); res = DPS_ERROR; goto DpsBaseWrite_exit; } P->Item.size = size; P->Item.orig_size = orig_size; if (write(P->Ifd, &P->Item, sizeof(DPS_BASEITEM)) != sizeof(DPS_BASEITEM)) { DpsLog(P->A, DPS_LOG_ERROR, "Can't write index for file %s {%s:%d}", P->Ifilename, __FILE__, __LINE__); } /* DpsBaseFsync(P->A, P);*/ #ifdef DEBUG_SEARCH DpsLog(P->A, DPS_LOG_DEBUG, "[%s/%s] Stored rec_id: %x Size: %d", P->subdir, P->basename, P->rec_id, len); #endif DpsBaseWrite_exit: #ifdef HAVE_ZLIB DPS_FREE(CData); #endif return res; }
extern __C_LINK int __DPSCALL DpsBaseRelocate(DPS_AGENT *Agent, int base_type) { DPS_BASE_PARAM O, N; DPS_BASE_PARAM *Old = &O, *New = &N; size_t base, i, ndel, mdel = 128, data_len; urlid_t *todel = (int*)DpsMalloc(128 * sizeof(urlid_t)); void *data; bzero(Old, sizeof(O)); bzero(New, sizeof(N)); switch(base_type) { case 0: /* stored */ Old->subdir = "store"; Old->basename = "doc"; Old->indname = "doc"; Old->mode = DPS_WRITE_LOCK; Old->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "OldStoredFiles", 0x100); Old->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR); Old->A = Agent; New->subdir = "store"; New->basename = "doc"; New->indname = "doc"; New->mode = DPS_WRITE_LOCK; New->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "StoredFiles", 0x100); New->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR); New->A = Agent; DpsLog(Agent, DPS_LOG_INFO, "Relocating stored database"); break; case 1: /* URL data */ Old->subdir = DPS_URLDIR; Old->basename = "info"; Old->indname = "info"; Old->mode = DPS_WRITE_LOCK; Old->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "OldURLDataFiles", 0x300); Old->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR); Old->A = Agent; #ifdef HAVE_ZLIB O.zlib_method = Z_DEFLATED; O.zlib_level = 9; O.zlib_windowBits = DPS_BASE_INFO_WINDOWBITS; O.zlib_memLevel = 9; O.zlib_strategy = DPS_BASE_INFO_STRATEGY; #endif New->subdir = DPS_URLDIR; New->basename = "info"; New->indname = "info"; New->mode = DPS_WRITE_LOCK; New->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "URLDataFiles", 0x300); New->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR); New->A = Agent; #ifdef HAVE_ZLIB N.zlib_method = Z_DEFLATED; N.zlib_level = 9; N.zlib_windowBits = DPS_BASE_INFO_WINDOWBITS; N.zlib_memLevel = 9; N.zlib_strategy = DPS_BASE_INFO_STRATEGY; #endif DpsLog(Agent, DPS_LOG_INFO, "Relocating URLData database"); break; case 2: /* tree wrd */ Old->subdir = DPS_TREEDIR; Old->basename = "wrd"; Old->indname = "wrd"; Old->mode = DPS_WRITE_LOCK; Old->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "OldWrdFiles", 0x300); Old->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR); Old->A = Agent; #ifdef HAVE_ZLIB O.zlib_method = Z_DEFLATED; O.zlib_level = 9; O.zlib_windowBits = DPS_BASE_WRD_WINDOWBITS; O.zlib_memLevel = 9; O.zlib_strategy = DPS_BASE_WRD_STRATEGY; #endif New->subdir = DPS_TREEDIR; New->basename = "wrd"; New->indname = "wrd"; New->mode = DPS_WRITE_LOCK; New->NFiles = (size_t)DpsVarListFindInt(&Agent->Vars, "WrdFiles", 0x300); New->vardir = DpsVarListFindStr(&Agent->Vars, "VarDir", DPS_VAR_DIR); New->A = Agent; #ifdef HAVE_ZLIB N.zlib_method = Z_DEFLATED; N.zlib_level = 9; N.zlib_windowBits = DPS_BASE_WRD_WINDOWBITS; N.zlib_memLevel = 9; N.zlib_strategy = DPS_BASE_WRD_STRATEGY; #endif DpsLog(Agent, DPS_LOG_INFO, "Relocating Wrd database"); break; default: DPS_FREE(todel); return DPS_OK; } for (base = 0; base < O.NFiles; base++) { ndel = 0; if (have_sigterm || have_sigint || have_sigalrm) { DpsLog(Agent, DPS_LOG_EXTRA, "%s signal received. Exiting chackup", (have_sigterm) ? "SIGTERM" : (have_sigint) ? "SIGINT" : "SIGALRM"); DpsBaseClose(Old); DpsBaseClose(New); DPS_FREE(todel); return DPS_OK; } Old->rec_id = (urlid_t)(base << DPS_BASE_BITS); if (DpsBaseOpen(Old, DPS_READ_LOCK) != DPS_OK) { DpsBaseClose(Old); DpsBaseClose(New); continue; } if (lseek(O.Ifd, (off_t)0, SEEK_SET) == (off_t)-1) { DpsLog(Agent, DPS_LOG_ERROR, "Can't seeek for file %s", Old->Ifilename); DpsBaseClose(Old); DpsBaseClose(New); DPS_FREE(todel); return DPS_ERROR; } while (read(Old->Ifd, &Old->Item, sizeof(DPS_BASEITEM)) == sizeof(DPS_BASEITEM)) { if (Old->Item.rec_id != 0) { if (ndel >= mdel) { mdel += 128; todel = (urlid_t*)DpsRealloc(todel, mdel * sizeof(urlid_t)); if (todel == NULL) { DpsBaseClose(Old); DpsBaseClose(New); DpsLog(Agent, DPS_LOG_ERROR, "Can't realloc %d bytes %s:%d", mdel * sizeof(urlid_t), __FILE__, __LINE__); DPS_FREE(todel); return DPS_ERROR; } } todel[ndel++] = Old->Item.rec_id; } } DpsBaseClose(Old); for (i = 0; i < ndel; i++) { Old->rec_id = todel[i]; data = DpsBaseARead(Old, &data_len); if (data == NULL) continue; DpsBaseDelete(Old); DpsBaseClose(Old); New->rec_id = todel[i]; DpsBaseWrite(New, data, data_len); DpsBaseClose(New); DPS_FREE(data); } DpsLog(Agent, DPS_LOG_EXTRA, "\tbase: %d [0x%x], %d records relocated", base, base, ndel); } DPS_FREE(todel); for (base = N.NFiles; base < O.NFiles; base++) { Old->rec_id = (urlid_t)(base << DPS_BASE_BITS); if (DpsBaseOpen(Old, DPS_READ_LOCK) != DPS_OK) { DpsBaseClose(Old); continue; } unlink(O.Ifilename); unlink(O.Sfilename); DpsBaseClose(Old); } return DPS_OK; }
int DpsSearchdGetWordResponse(DPS_AGENT *query,DPS_RESULT *Res,DPS_DB *cl) { DPS_URL_CRD_DB *wrd = NULL; DPS_URLDATA *udt = NULL; #ifdef WITH_REL_TRACK DPS_URLTRACK *trk = NULL; #endif DPS_SEARCHD_PACKET_HEADER hdr; ssize_t nrecv; char *msg; int done=0, rc = DPS_OK; char *wbuf, *p; DPS_WIDEWORDLIST_EX *wwl; DPS_WIDEWORD *ww_ex; DPS_WIDEWORD ww; size_t i; TRACE_IN(query, "DpsSearchdGetWordResponse"); Res->total_found=0; while(!done){ nrecv = DpsRecvall(cl->searchd, &hdr, sizeof(hdr), 360); if(nrecv!=sizeof(hdr)){ sprintf(query->Conf->errstr,"Received incomplete header from searchd (%d bytes,errno:%d)",(int)nrecv, errno); TRACE_OUT(query); return DPS_ERROR;; } #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Received header cmd=%d len=%d\n",hdr.cmd,hdr.len); #endif switch(hdr.cmd){ case DPS_SEARCHD_CMD_ERROR: msg=(char*)DpsMalloc(hdr.len+1); if (msg == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, msg, hdr.len, 360); if (nrecv >= 0) { msg[nrecv]='\0'; sprintf(query->Conf->errstr,"Searchd error: '%s',received:%d", msg, (int)nrecv); } rc = DPS_ERROR; DPS_FREE(msg); done=1; break; case DPS_SEARCHD_CMD_MESSAGE: msg=(char*)DpsMalloc(hdr.len+1); if (msg == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, msg, hdr.len, 360); msg[(nrecv >= 0) ? nrecv : 0] = '\0'; if (strncmp(msg, "Total_found", 11) == 0) { Res->total_found = (size_t)DPS_ATOI(msg + 12); Res->grand_total = (size_t)DPS_ATOI(strchr(msg + 12, (int)' ') + 1); } #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Message from searchd: '%s'\n",msg); #endif DPS_FREE(msg); break; case DPS_SEARCHD_CMD_WORDS: DPS_FREE(wrd); wrd=(DPS_URL_CRD_DB*)DpsMalloc(hdr.len + 1); if (wrd == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, wrd, hdr.len, 360); /*Res->total_found=hdr.len/sizeof(*wrd);*/ Res->num_rows = (nrecv >= 0) ? (size_t)nrecv / sizeof(*wrd) : 0; #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Received words size=%d nwrd=%d\n",hdr.len, Res->num_rows /*Res->total_found*/); #endif done=1; break; case DPS_SEARCHD_CMD_SUGGEST: DPS_FREE(Res->Suggest); Res->Suggest = (char*)DpsMalloc(hdr.len + 1); if (Res->Suggest == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, Res->Suggest, hdr.len, 360); Res->Suggest[(nrecv >=0) ? nrecv : 0] = '\0'; #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Received Suggest size=%d\n", hdr.len); #endif break; case DPS_SEARCHD_CMD_PERSITE: Res->PerSite = (size_t*)DpsMalloc(hdr.len + 1); if (Res->PerSite == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, Res->PerSite, hdr.len, 360); #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Received PerSite size=%d nwrd=%d\n", nrecv, Res->num_rows/*Res->total_found*/); #endif break; case DPS_SEARCHD_CMD_DATA: udt = (DPS_URLDATA*)DpsMalloc(hdr.len + 1); if (udt == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, udt, hdr.len, 360); #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Received URLDATA size=%d nwrd=%d\n", nrecv, Res->num_rows); #endif break; #ifdef WITH_REL_TRACK case DPS_SEARCHD_CMD_TRACKDATA: trk = (DPS_URLTRACK*)DpsMalloc(hdr.len + 1); if (trk == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, trk, hdr.len, 360); #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Received TRACKDATA size=%d nwrd=%d\n", nrecv, Res->num_rows); #endif break; #endif case DPS_SEARCHD_CMD_WITHOFFSET: /* Res->offset = 1;*/ break; case DPS_SEARCHD_CMD_QLC: if ((p = (char *)DpsXmalloc(hdr.len + 1)) != NULL) { if (DpsRecvall(cl->searchd, p, hdr.len, 360)) { DpsVarListReplaceStr(&query->Vars, "q", p); } } DPS_FREE(p); break; case DPS_SEARCHD_CMD_WWL: Res->PerSite = NULL; if ((wbuf = p = (char *)DpsXmalloc(hdr.len + 1)) != NULL) if (DpsRecvall(cl->searchd, wbuf, hdr.len, 360)) { wwl = (DPS_WIDEWORDLIST_EX *)p; p += sizeof(DPS_WIDEWORDLIST_EX); #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "wbuf :%x, wwl: %x, p: %x hdr.len:%d\n", wbuf, wwl, p, hdr.len); DpsLog(query, DPS_LOG_ERROR, "Received WWL nwords=%d nuniq=%d\n", wwl->nwords, wwl->nuniq); #endif /* DpsWideWordListFree(&Res->WWList);*/ for(i = 0; i < wwl->nwords; i++) { /* ww_ex = (DPS_WIDEWORD_EX *)((void*)&p[0]);*/ dps_memcpy((char*)&ww, p, sizeof(DPS_WIDEWORD_EX)); p += sizeof(DPS_WIDEWORD_EX); /* ww.order = ww_ex->order; ww.order_inquery = ww_ex->order_inquery; ww.count = ww_ex->count; ww.len = ww_ex->len; ww.ulen = ww_ex->ulen; ww.origin = ww_ex->origin; ww.crcword = ww_ex->crcword; */ ww.word = p; #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Word {%d}: %s\n", ww.len+1, ww.word); #endif p += ww.len + 1; p += sizeof(dpsunicode_t) - ((SDPALIGN)p % sizeof(dpsunicode_t)); ww.uword = (dpsunicode_t*)p; p += sizeof(dpsunicode_t) * (ww.ulen + 1); DpsWideWordListAdd(&Res->WWList, &ww, DPS_WWL_STRICT); } Res->WWList.nuniq = wwl->nuniq; DPS_FREE(wbuf); } break; default: sprintf(query->Conf->errstr,"Unknown searchd response: cmd=%d len=%d",hdr.cmd,hdr.len); rc = DPS_ERROR; done=1; break; } } Res->CoordList.Coords = wrd; Res->CoordList.Data = udt; #ifdef WITH_REL_TRACK Res->CoordList.Track = trk; #endif TRACE_OUT(query); return rc; }
__C_LINK int __DPSCALL DpsSynonymListLoad(DPS_ENV * Env,const char * filename){ struct stat sb; char *str, *data = NULL, *cur_n = NULL; char lang[64]=""; DPS_CHARSET *cs=NULL; DPS_CHARSET *sys_int=DpsGetCharSet("sys-int"); DPS_CONV file_uni; DPS_WIDEWORD *ww = NULL; size_t key = 1; int flag_th = 0; int fd; char savebyte; if (stat(filename, &sb)) { fprintf(stderr, "Unable to stat synonyms file '%s': %s", filename, strerror(errno)); return DPS_ERROR; } if ((fd = DpsOpen2(filename, O_RDONLY)) <= 0) { dps_snprintf(Env->errstr,sizeof(Env->errstr)-1, "Unable to open synonyms file '%s': %s", filename, strerror(errno)); return DPS_ERROR; } if ((data = (char*)DpsMalloc(sb.st_size + 1)) == NULL) { dps_snprintf(Env->errstr,sizeof(Env->errstr)-1, "Unable to alloc %d bytes", sb.st_size); DpsClose(fd); return DPS_ERROR; } if (read(fd, data, sb.st_size) != (ssize_t)sb.st_size) { dps_snprintf(Env->errstr,sizeof(Env->errstr)-1, "Unable to read synonym file '%s': %s", filename, strerror(errno)); DPS_FREE(data); DpsClose(fd); return DPS_ERROR; } data[sb.st_size] = '\0'; str = data; cur_n = strchr(str, '\n'); if (cur_n != NULL) { cur_n++; savebyte = *cur_n; *cur_n = '\0'; } while(str != NULL) { if(str[0]=='#'||str[0]==' '||str[0]=='\t'||str[0]=='\r'||str[0]=='\n') goto loop_continue; if(!strncasecmp(str,"Charset:",8)){ char * lasttok; char * charset; if((charset = dps_strtok_r(str + 8, " \t\n\r", &lasttok))) { cs=DpsGetCharSet(charset); if(!cs){ dps_snprintf(Env->errstr, sizeof(Env->errstr), "Unknown charset '%s' in synonyms file '%s'", charset, filename); DPS_FREE(data); DpsClose(fd); return DPS_ERROR; } DpsConvInit(&file_uni, cs, sys_int, Env->CharsToEscape, 0); } }else if(!strncasecmp(str,"Language:",9)){ char * lasttok; char * l; if((l = dps_strtok_r(str + 9, " \t\n\r", &lasttok))) { dps_strncpy(lang, l, sizeof(lang)-1); } }else if(!strncasecmp(str, "Thesaurus:", 10)) { char * lasttok; char *tok = dps_strtok_r(str + 10, " \t\n\r", &lasttok); flag_th = (strncasecmp(tok, "yes", 3) == 0) ? 1 : 0; }else{ char *av[255]; size_t ac, i, j; dpsunicode_t *t; if(!cs){ dps_snprintf(Env->errstr,sizeof(Env->errstr)-1,"No Charset command in synonyms file '%s'",filename); DpsClose(fd); DPS_FREE(data); return DPS_ERROR; } if(!lang[0]){ dps_snprintf(Env->errstr,sizeof(Env->errstr)-1,"No Language command in synonyms file '%s'",filename); DpsClose(fd); DPS_FREE(data); return DPS_ERROR; } ac = DpsGetArgs(str, av, 255); if (ac < 2) goto loop_continue; if ((ww = (DPS_WIDEWORD*)DpsRealloc(ww, ac * sizeof(DPS_WIDEWORD))) == NULL) return DPS_ERROR; for (i = 0; i < ac; i++) { ww[i].word = av[i]; ww[i].len = dps_strlen(av[i]); ww[i].uword = t = (dpsunicode_t*)DpsMalloc((3 * ww[i].len + 1) * sizeof(dpsunicode_t)); if (ww[i].uword == NULL) return DPS_ERROR; DpsConv(&file_uni, (char*)ww[i].uword, sizeof(dpsunicode_t) * (3 * ww[i].len + 1), av[i], ww[i].len + 1); DpsUniStrToLower(ww[i].uword); ww[i].uword = DpsUniNormalizeNFC(NULL, ww[i].uword); DPS_FREE(t); } for (i = 0; i < ac - 1; i++) { for (j = i + 1; j < ac; j++) { if((Env->Synonyms.nsynonyms + 1) >= Env->Synonyms.msynonyms){ Env->Synonyms.msynonyms += 64; Env->Synonyms.Synonym = (DPS_SYNONYM*)DpsRealloc(Env->Synonyms.Synonym, sizeof(DPS_SYNONYM)*Env->Synonyms.msynonyms); if (Env->Synonyms.Synonym == NULL) { Env->Synonyms.msynonyms = Env->Synonyms.nsynonyms = 0; return DPS_ERROR; } } bzero((void*)&Env->Synonyms.Synonym[Env->Synonyms.nsynonyms], sizeof(DPS_SYNONYM)); /* Add direct order */ Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].p.uword = DpsUniDup(ww[i].uword); Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].s.uword = DpsUniDup(ww[j].uword); Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].p.count = Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].s.count = (size_t)((flag_th) ? key : 0); Env->Synonyms.nsynonyms++; bzero((void*)&Env->Synonyms.Synonym[Env->Synonyms.nsynonyms], sizeof(DPS_SYNONYM)); /* Add reverse order */ Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].p.uword = DpsUniDup(ww[j].uword); Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].s.uword = DpsUniDup(ww[i].uword); Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].p.count = Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].s.count = (size_t)((flag_th) ? key : 0); Env->Synonyms.nsynonyms++; } } for (i = 0; i < ac; i++) { DPS_FREE(ww[i].uword); } do { key++; } while (key == 0); } loop_continue: str = cur_n; if (str != NULL) { *str = savebyte; cur_n = strchr(str, '\n'); if (cur_n != NULL) { cur_n++; savebyte = *cur_n; *cur_n = '\0'; } } } DPS_FREE(data); DPS_FREE(ww); DpsClose(fd); return DPS_OK; }
int __DPSCALL DpsFindWordsSearchd(DPS_AGENT *query, DPS_RESULT *Res, DPS_DB *searchd) { size_t maxlen = 1024; char *request, *edf = NULL, *e_empty = NULL; const char *df = DpsVarListFindStr(&query->Vars, "DateFormat", NULL); const char *empty = DpsVarListFindStr(&query->Vars, "empty", NULL); const char *qs = DpsVarListFindStr(&query->Vars, "QUERY_STRING", ""); const char *tmplt = DpsVarListFindStr(&query->Vars, "tmplt", ""); int res=DPS_OK; TRACE_IN(query, "DpsFindWordsSearchd"); if (df) { edf = (char*)DpsMalloc(dps_strlen(df) * 10 + 1); if (edf == NULL) { sprintf(query->Conf->errstr,"Can't allocate memory"); TRACE_OUT(query); return DPS_ERROR; } DpsEscapeURL(edf, df); maxlen += dps_strlen(edf); } if (empty) { e_empty = (char*)DpsMalloc(dps_strlen(empty) * 10 + 1); if (e_empty == NULL) { sprintf(query->Conf->errstr, "Can't allocate memory"); TRACE_OUT(query); return DPS_ERROR; } DpsEscapeURL(e_empty, empty); maxlen += dps_strlen(e_empty); } maxlen += dps_strlen(qs) + dps_strlen(tmplt) + 64; if (NULL==(request=(char*)DpsMalloc(maxlen))) { sprintf(query->Conf->errstr,"Can't allocate memory"); DPS_FREE(edf); TRACE_OUT(query); return DPS_ERROR; } dps_snprintf(request, maxlen, "%s&BrowserCharset=%s&IP=%s&g-lc=%s&ExcerptSize=%s&ExcerptPadding=%s&DoExcerpt=%s&tmplt=%s%s%s%s%s%s%s&sp=%s&sy=%s&s=%s", qs, DpsVarListFindStr(&query->Vars, "BrowserCharset", "iso-8859-1"), DpsVarListFindStr(&query->Vars, "IP", "localhost"), DpsVarListFindStr(&query->Vars, "g-lc", "en"), DpsVarListFindStr(&query->Vars, "ExcerptSize", "256"), DpsVarListFindStr(&query->Vars, "ExcerptPadding", "40"), (query->Flags.do_excerpt) ? "yes" : "no", tmplt, (edf) ? "&DateFormat=" : "", (edf) ? edf : "", (e_empty) ? "&empty=" : "", (e_empty) ? e_empty : "", (searchd->label) ? "&label=" : "", (searchd->label) ? searchd->label : "", DpsVarListFindStr(&query->Vars, "sp", "1"), DpsVarListFindStr(&query->Vars, "sy", "1"), DpsVarListFindStr(&query->Vars, "s", "RP") ); DPS_FREE(edf); DPS_FREE(e_empty); request[maxlen-1]='\0'; res = DpsSearchdSendWordRequest(query, searchd, request); DPS_FREE(request); if (DPS_OK != res) { TRACE_OUT(query); return res; } /* res = DpsSearchdGetWordResponse(query, Res, searchd); called later from DpsFind */ TRACE_OUT(query); return res; }
void DpsCookiesFind(DPS_AGENT *Indexer, DPS_SERVER *Server, DPS_DOCUMENT *Doc, const char *hostinfo) { #ifdef HAVE_SQL DPS_DSTR cookie; DPS_COOKIES *Cookies = &Indexer->Cookies; DPS_COOKIE *Coo; size_t i, blen = dps_strlen(hostinfo), slen; int have_no_cookies = DpsVarListFindInt(&Doc->Sections, "have_no_cookies", 1); #ifdef WITH_PARANOIA void *paran = DpsViolationEnter(paran); #endif TRACE_IN(Indexer, "DpsCookiesFind"); DpsDSTRInit(&cookie, 1024); for(i = 0; i < Cookies->ncookies; i++) { Coo = &Cookies->Cookie[i]; slen = dps_strlen(Coo->domain); if (slen > blen) continue; if (Coo->secure == 'y' && strcasecmp(Doc->CurURL.schema, "https")) continue; if (strncasecmp(Coo->path, Doc->CurURL.path, dps_strlen(Coo->path))) continue; if (strcasecmp(Coo->domain, hostinfo + (blen - slen))) continue; if (Coo->from_config != 1) have_no_cookies = 0; if (Coo->name[0] == '\0' && Coo->value[0] == '\0') continue; if (cookie.data_size) DpsDSTRAppend(&cookie, "; ", 2); DpsDSTRAppendStr(&cookie, Coo->name); DpsDSTRAppend(&cookie, "=", 1); DpsDSTRAppendStr(&cookie, Coo->value); } if (have_no_cookies) { char buf[2*PATH_MAX]; dpshash32_t url_id; DPS_DB *db; DPS_SQLRES Res; size_t rows; int rc; if (Server != NULL) { char *PingData = DpsVarListFindStr(&Server->Vars, "AuthPing", NULL); if (PingData != NULL) { char *AuthPing = DpsStrdup(DpsTrim(PingData, " \t\r\n")); int method = DPS_METHOD_GET; dps_base64_decode(AuthPing, PingData, dps_strlen(PingData)); if (!strncasecmp(AuthPing, "GET", 3)) { method = DPS_METHOD_GET; PingData = DpsTrim(AuthPing + 3, " \t\r\n"); } else if (!strncasecmp(AuthPing, "POST", 4)) { method = DPS_METHOD_POST; PingData = DpsTrim(AuthPing + 4, " \t\r\n"); } else { DpsLog(Indexer, DPS_LOG_ERROR, "AuthPing should be GET or POST: %s", AuthPing); PingData = NULL; } if (PingData != NULL) { size_t size = dps_strlen(PingData); { char PingURL[size + 2]; char PingBody[size]; DPS_DOCUMENT *rDoc; int result; rDoc = DpsDocInit(NULL); DpsSpiderParamInit(&rDoc->Spider); DpsVarList2Doc(rDoc, Server); rDoc->Buf.max_size = (size_t)DpsVarListFindInt(&Indexer->Vars, "MaxDocSize", DPS_MAXDOCSIZE); rDoc->Buf.allocated_size = DPS_NET_BUF_SIZE; if ((rDoc->Buf.buf = (char*)DpsMalloc(rDoc->Buf.allocated_size + 1)) == NULL) { DpsDocFree(rDoc); TRACE_OUT(Indexer); return; } rDoc->Buf.buf[0]='\0'; rDoc->subdoc = Indexer->Flags.SubDocLevel + 1; #if 1 dps_snprintf(buf, sizeof(buf), "%s://%s/", DPS_NULL2EMPTY(Doc->CurURL.schema), DPS_NULL2EMPTY(Doc->CurURL.hostinfo)); DpsVarListReplaceStr(&rDoc->Sections, "URL", buf); DpsURLParse(&rDoc->CurURL, buf); DpsLog(Indexer, DPS_LOG_INFO, "HOME: %s", buf); rDoc->method = DPS_METHOD_HEAD; /* DpsVarListFree(&rDoc->RequestHeaders);*/ if (Doc != NULL) { DpsVarListReplaceLst(&rDoc->RequestHeaders, &Doc->RequestHeaders, NULL, "*"); } DpsVarListReplaceStr(&rDoc->Sections, "have_no_cookies", "0"); DpsDocAddDocExtraHeaders(Indexer, Server, rDoc); DpsDocAddConfExtraHeaders(Indexer->Conf, rDoc); DpsVarListReplaceLst(&rDoc->Sections, &Server->Vars, NULL, "*"); DpsDocAddServExtraHeaders(Server, rDoc); DpsVarListLog(Indexer, &rDoc->RequestHeaders, DPS_LOG_DEBUG, "HOME.Request"); if (Doc == NULL || Indexer->Flags.cmd == DPS_IND_FILTER) { DpsDocLookupConn(Indexer, rDoc); } else { DPS_FREE(rDoc->connp.connp); rDoc->connp = Doc->connp; } result = DpsGetURL(Indexer, rDoc, NULL); /* Just get headers from the home as we need only Cookies from it */ DpsDocProcessResponseHeaders(Indexer, rDoc); DpsVarListLog(Indexer, &rDoc->Sections, DPS_LOG_DEBUG, "HOME.Response"); #endif sscanf(PingData, "%s %s", PingURL, PingBody); if (rDoc->method == DPS_METHOD_GET) { dps_strcat(PingURL, "?"); dps_strcat(PingURL, PingBody); } else { DpsVarListReplaceStr(&rDoc->Sections, "body", PingBody); } DpsVarListReplaceStr(&rDoc->Sections, "URL", PingURL); DpsURLParse(&rDoc->CurURL, PingURL); DpsLog(Indexer, DPS_LOG_INFO, "AUTH.PING: %s", PingURL); rDoc->method = method; DpsVarListFree(&rDoc->RequestHeaders); DpsVarListReplaceStr(&rDoc->Sections, "have_no_cookies", "0"); DpsDocAddDocExtraHeaders(Indexer, Server, rDoc); DpsDocAddConfExtraHeaders(Indexer->Conf, rDoc); DpsVarListReplaceLst(&rDoc->Sections, &Server->Vars, NULL, "*"); DpsDocAddServExtraHeaders(Server, rDoc); if (method == DPS_METHOD_POST) { dps_snprintf(buf, sizeof(buf), "application/x-www-form-urlencoded; charset=%s", DpsVarListFindStr(&Indexer->Conf->Vars, "LocalCharset", "iso-8859-1")); DpsVarListReplaceStr(&rDoc->RequestHeaders, "Content-Type", buf); dps_snprintf(buf, sizeof(buf), "%d", dps_strlen(PingBody)); DpsVarListReplaceStr(&rDoc->RequestHeaders, "Content-Length", buf); } DpsVarListLog(Indexer, &rDoc->RequestHeaders, DPS_LOG_DEBUG, "AUTHPING.Request"); #if 0 if (Doc == NULL || Indexer->Flags.cmd == DPS_IND_FILTER) { DpsDocLookupConn(Indexer, rDoc); } else { DPS_FREE(rDoc->connp.connp); rDoc->connp = Doc->connp; } #endif result = DpsGetURL(Indexer, rDoc, NULL); /* Just get it as we need only Cookies from the headers */ DpsDocProcessResponseHeaders(Indexer, rDoc); DpsVarListDel(&rDoc->Sections, "body"); DpsVarListLog(Indexer, &rDoc->Sections, DPS_LOG_DEBUG, "AUTHPING.Response"); if (Doc != NULL) bzero(&rDoc->connp, sizeof(rDoc->connp)); DpsDocFree(rDoc); } } DpsFree(AuthPing); } } while(hostinfo != NULL) { url_id = DpsStrHash32(hostinfo); DpsSQLResInit(&Res); dps_snprintf(buf, sizeof(buf), "SELECT name,value,path,secure FROM cookies WHERE domain='%s'", hostinfo); if (Indexer->flags & DPS_FLAG_UNOCON) { DPS_GETLOCK(Indexer, DPS_LOCK_DB); db = Indexer->Conf->dbl.db[url_id % Indexer->Conf->dbl.nitems]; } else { db = Indexer->dbl.db[url_id % Indexer->dbl.nitems]; } if(DPS_OK == (rc = DpsSQLQuery(db, &Res, buf))) { rows = DpsSQLNumRows(&Res); for(i = 0; i < rows; i++) { DpsCookiesAdd(Indexer, hostinfo, DpsSQLValue(&Res, i, 2), DpsSQLValue(&Res, i, 0), DpsSQLValue(&Res, i, 1), *DpsSQLValue(&Res, i, 3), 0, 0, 0); if (*DpsSQLValue(&Res, i, 3) == 'y' && strcasecmp(Doc->CurURL.schema, "https")) continue; if (strncasecmp(DpsSQLValue(&Res, i, 2), Doc->CurURL.path, dps_strlen(DpsSQLValue(&Res, i, 2)))) continue; if (cookie.data_size) DpsDSTRAppend(&cookie, "; ", 2); DpsDSTRAppendStr(&cookie, DpsSQLValue(&Res, i, 0)); DpsDSTRAppend(&cookie, "=", 1); DpsDSTRAppendStr(&cookie, DpsSQLValue(&Res, i, 1)); } if (rows == 0) { DpsCookiesAdd(Indexer, hostinfo, "/", "", "", 'n', 0, 0, 0); } } DpsSQLFree(&Res); if (Indexer->flags & DPS_FLAG_UNOCON) { DPS_RELEASELOCK(Indexer, DPS_LOCK_DB); } hostinfo = strchr(hostinfo, '.'); if (hostinfo != NULL) hostinfo++; } } if (cookie.data_size) { DpsVarListReplaceStr(&Doc->RequestHeaders, "Cookie", cookie.data); } DpsDSTRFree(&cookie); #endif TRACE_OUT(Indexer); return; }