void DpsWWLBoolItems(DPS_RESULT *Res) { DPS_WIDEWORD Word; DPS_STACK_ITEM *items = Res->items; size_t i; if (Res->WWList.nwords == 0) { for (i = 0; i < Res->nitems; i++) { if (items[i].cmd != DPS_STACK_WORD) continue; Word.order = items[i].order; Word.order_inquery = items[i].order_inquery; Word.count = items[i].count; Word.crcword = items[i].crcword; Word.word = items[i].word; Word.uword = items[i].uword; Word.origin = items[i].origin; items[i].wordnum = DpsWideWordListAdd(&Res->WWList, &Word, DPS_WWL_LOOSE); items[i].count = 0; } } else { for (i = 0; i < Res->nitems; i++) { if (items[i].cmd != DPS_STACK_WORD) continue; Res->WWList.Word[items[i].wordnum].count += items[i].count; items[i].count = 0; } } }
int DpsSearchdGetWordResponse(DPS_AGENT *query,DPS_RESULT *Res,DPS_DB *cl) { DPS_URL_CRD_DB *wrd = NULL; DPS_URLDATA *udt = NULL; #ifdef WITH_REL_TRACK DPS_URLTRACK *trk = NULL; #endif DPS_SEARCHD_PACKET_HEADER hdr; ssize_t nrecv; char *msg; int done=0, rc = DPS_OK; char *wbuf, *p; DPS_WIDEWORDLIST_EX *wwl; DPS_WIDEWORD *ww_ex; DPS_WIDEWORD ww; size_t i; TRACE_IN(query, "DpsSearchdGetWordResponse"); Res->total_found=0; while(!done){ nrecv = DpsRecvall(cl->searchd, &hdr, sizeof(hdr), 360); if(nrecv!=sizeof(hdr)){ sprintf(query->Conf->errstr,"Received incomplete header from searchd (%d bytes,errno:%d)",(int)nrecv, errno); TRACE_OUT(query); return DPS_ERROR;; } #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Received header cmd=%d len=%d\n",hdr.cmd,hdr.len); #endif switch(hdr.cmd){ case DPS_SEARCHD_CMD_ERROR: msg=(char*)DpsMalloc(hdr.len+1); if (msg == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, msg, hdr.len, 360); if (nrecv >= 0) { msg[nrecv]='\0'; sprintf(query->Conf->errstr,"Searchd error: '%s',received:%d", msg, (int)nrecv); } rc = DPS_ERROR; DPS_FREE(msg); done=1; break; case DPS_SEARCHD_CMD_MESSAGE: msg=(char*)DpsMalloc(hdr.len+1); if (msg == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, msg, hdr.len, 360); msg[(nrecv >= 0) ? nrecv : 0] = '\0'; if (strncmp(msg, "Total_found", 11) == 0) { Res->total_found = (size_t)DPS_ATOI(msg + 12); Res->grand_total = (size_t)DPS_ATOI(strchr(msg + 12, (int)' ') + 1); } #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Message from searchd: '%s'\n",msg); #endif DPS_FREE(msg); break; case DPS_SEARCHD_CMD_WORDS: DPS_FREE(wrd); wrd=(DPS_URL_CRD_DB*)DpsMalloc(hdr.len + 1); if (wrd == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, wrd, hdr.len, 360); /*Res->total_found=hdr.len/sizeof(*wrd);*/ Res->num_rows = (nrecv >= 0) ? (size_t)nrecv / sizeof(*wrd) : 0; #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Received words size=%d nwrd=%d\n",hdr.len, Res->num_rows /*Res->total_found*/); #endif done=1; break; case DPS_SEARCHD_CMD_SUGGEST: DPS_FREE(Res->Suggest); Res->Suggest = (char*)DpsMalloc(hdr.len + 1); if (Res->Suggest == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, Res->Suggest, hdr.len, 360); Res->Suggest[(nrecv >=0) ? nrecv : 0] = '\0'; #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Received Suggest size=%d\n", hdr.len); #endif break; case DPS_SEARCHD_CMD_PERSITE: Res->PerSite = (size_t*)DpsMalloc(hdr.len + 1); if (Res->PerSite == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, Res->PerSite, hdr.len, 360); #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Received PerSite size=%d nwrd=%d\n", nrecv, Res->num_rows/*Res->total_found*/); #endif break; case DPS_SEARCHD_CMD_DATA: udt = (DPS_URLDATA*)DpsMalloc(hdr.len + 1); if (udt == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, udt, hdr.len, 360); #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Received URLDATA size=%d nwrd=%d\n", nrecv, Res->num_rows); #endif break; #ifdef WITH_REL_TRACK case DPS_SEARCHD_CMD_TRACKDATA: trk = (DPS_URLTRACK*)DpsMalloc(hdr.len + 1); if (trk == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, trk, hdr.len, 360); #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Received TRACKDATA size=%d nwrd=%d\n", nrecv, Res->num_rows); #endif break; #endif case DPS_SEARCHD_CMD_WITHOFFSET: /* Res->offset = 1;*/ break; case DPS_SEARCHD_CMD_QLC: if ((p = (char *)DpsXmalloc(hdr.len + 1)) != NULL) { if (DpsRecvall(cl->searchd, p, hdr.len, 360)) { DpsVarListReplaceStr(&query->Vars, "q", p); } } DPS_FREE(p); break; case DPS_SEARCHD_CMD_WWL: Res->PerSite = NULL; if ((wbuf = p = (char *)DpsXmalloc(hdr.len + 1)) != NULL) if (DpsRecvall(cl->searchd, wbuf, hdr.len, 360)) { wwl = (DPS_WIDEWORDLIST_EX *)p; p += sizeof(DPS_WIDEWORDLIST_EX); #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "wbuf :%x, wwl: %x, p: %x hdr.len:%d\n", wbuf, wwl, p, hdr.len); DpsLog(query, DPS_LOG_ERROR, "Received WWL nwords=%d nuniq=%d\n", wwl->nwords, wwl->nuniq); #endif /* DpsWideWordListFree(&Res->WWList);*/ for(i = 0; i < wwl->nwords; i++) { /* ww_ex = (DPS_WIDEWORD_EX *)((void*)&p[0]);*/ dps_memcpy((char*)&ww, p, sizeof(DPS_WIDEWORD_EX)); p += sizeof(DPS_WIDEWORD_EX); /* ww.order = ww_ex->order; ww.order_inquery = ww_ex->order_inquery; ww.count = ww_ex->count; ww.len = ww_ex->len; ww.ulen = ww_ex->ulen; ww.origin = ww_ex->origin; ww.crcword = ww_ex->crcword; */ ww.word = p; #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Word {%d}: %s\n", ww.len+1, ww.word); #endif p += ww.len + 1; p += sizeof(dpsunicode_t) - ((SDPALIGN)p % sizeof(dpsunicode_t)); ww.uword = (dpsunicode_t*)p; p += sizeof(dpsunicode_t) * (ww.ulen + 1); DpsWideWordListAdd(&Res->WWList, &ww, DPS_WWL_STRICT); } Res->WWList.nuniq = wwl->nuniq; DPS_FREE(wbuf); } break; default: sprintf(query->Conf->errstr,"Unknown searchd response: cmd=%d len=%d",hdr.cmd,hdr.len); rc = DPS_ERROR; done=1; break; } } Res->CoordList.Coords = wrd; Res->CoordList.Data = udt; #ifdef WITH_REL_TRACK Res->CoordList.Track = trk; #endif TRACE_OUT(query); return rc; }
/* Main function to calculate items sequence */ int DpsCalcBoolItems(DPS_AGENT *query, DPS_RESULT *Res) { DPS_BOOLSTACK *s = DpsBoolStackInit(NULL); DPS_STACK_ITEM *items = Res->items, *res; DPS_WIDEWORD Word; size_t nitems = Res->nitems; size_t i, j; int first_time = (Res->WWList.nwords == 0); if (s == NULL) return DPS_STACK_ERR; if (nitems == 0) { Res->CoordList.Coords = Res->items[0].pbegin; Res->items[0].pbegin = NULL; Res->items[0].count = 0; /* Res->CoordList.ncoords = Res->items[0]->count;*/ DpsBoolStackFree(s); return DPS_OK; } for (i = 0; i < nitems; i++) { if (items[i].cmd != DPS_STACK_WORD) continue; if ((items[i].pbegin == NULL) && ((items[i].origin & DPS_WORD_ORIGIN_STOP) == 0)) { for(j = 0; j < i; j++) if (items[j].crcword == items[i].crcword) break; if (j < i) { items[i].count = items[j].count; items[i].pbegin = (DPS_URL_CRD_DB*)DpsMalloc((items[j].count + 1) * sizeof(DPS_URL_CRD_DB)); if (items[i].pbegin == NULL) { DpsLog(query, DPS_LOG_ERROR, "Can't alloc %d bytes %s:%d", (items[j].count + 1) * sizeof(DPS_URL_CRD_DB), __FILE__, __LINE__); DpsBoolStackFree(s); return DPS_STACK_ERR; } { register size_t z; for (z = 0; z < items[i].count; z++) { items[i].pbegin[z] = items[j].pbegin[z]; items[i].pbegin[z].coord &= 0xFFFFFF00; items[i].pbegin[z].coord += (items[i].wordnum & 0xFF); } } } } if (first_time) { Word.order = items[i].order; Word.count = items[i].count; Word.crcword = items[i].crcword; Word.word = items[i].word; Word.uword = items[i].uword; Word.origin = items[i].origin; DpsWideWordListAdd(&Res->WWList, &Word, DPS_WWL_LOOSE); } } #ifdef DEBUG_BOOL DpsLog(query, DPS_LOG_EXTRA, "--------"); for(i=0;i<nitems;i++){ DpsLog(query, DPS_LOG_EXTRA, "[%d].%d %c : %d -- %s", i, items[i].wordnum, item_type(items[i].cmd), items[i].count, (items[i].word == NULL) ? "<NULL>" : items[i].word); } DpsLog(query, DPS_LOG_EXTRA, "--------"); #endif for(i=0;i<nitems;i++){ int c; #ifdef DEBUG_BOOL DpsLog(query, DPS_LOG_EXTRA, ".[%d].%d %c : %d -- %s, (order_origin:%d)", i, items[i].wordnum, item_type(items[i].cmd), items[i].count, (items[i].word == NULL) ? "<NULL>" : items[i].word, items[i].order_origin); #endif switch(c=items[i].cmd){ case DPS_STACK_RIGHT: /* Perform till LEFT bracket */ while((TOPCMD(s) != DPS_STACK_LEFT) && (TOPCMD(s) != DPS_STACK_BOT)) if (DPS_OK != perform(query, Res, s, POPCMD(s))) { DpsBoolStackFree(s); return DPS_STACK_ERR; } /* Pop LEFT bracket itself */ if(TOPCMD(s) == DPS_STACK_LEFT) POPCMD(s); break; case DPS_STACK_OR: case DPS_STACK_AND: case DPS_STACK_NEAR: case DPS_STACK_ANYWORD: if (s->nastack > 1) while(c <= TOPCMD(s)) { if (DPS_OK != perform(query, Res, s, POPCMD(s))) { DpsBoolStackFree(s); return DPS_STACK_ERR; } } /* IMPORTANT! No break here! That's OK*/ /* Так надо ! */ case DPS_STACK_LEFT: case DPS_STACK_PHRASE_LEFT: case DPS_STACK_NOT: if (PUSHCMD(s,c) != DPS_OK) { DpsBoolStackFree(s); return DPS_STACK_ERR; } break; case DPS_STACK_PHRASE_RIGHT: /* perform till RIGHT phrase quote */ while((TOPCMD(s) != DPS_STACK_PHRASE_LEFT) && (TOPCMD(s) != DPS_STACK_BOT)) if (DPS_OK != perform(query, Res, s, POPCMD(s))) { DpsBoolStackFree(s); return DPS_STACK_ERR; } if (TOPCMD(s) == DPS_STACK_PHRASE_LEFT) perform(query, Res, s, POPCMD(s)); #ifdef DEBUG_BOOL DpsLog(query, DPS_LOG_EXTRA, "[%d] %c", i, item_type(items[i].cmd)); #endif break; case DPS_STACK_WORD: items[i].order_from = items[i].order_to = items[i].order; default: if (DPS_OK != PUSHARG(s, &items[i])) { DpsBoolStackFree(s); return DPS_STACK_ERR; } items[i].pbegin = items[i].plast = NULL; /*items[i].word = NULL; items[i].uword = NULL;*/ /* items[i].count = 0;*/ /* DpsStackItemFree(&items[i]);*/ /* if (DPS_OK != PUSHARG(s, (count[items[i].order]) ? 1UL : 0UL)) { DpsBoolStackFree(s); return DPS_STACK_ERR; }*/ break; } } while(TOPCMD(s) != DPS_STACK_BOT) { if (DPS_OK != perform(query, Res, s, POPCMD(s))) { DpsBoolStackFree(s); return DPS_STACK_ERR; } } res = POPARG(s); if (res != NULL) { Res->CoordList.Coords = res->pbegin; Res->CoordList.ncoords = res->count; res->pbegin = res->plast = NULL; /* res->count = 0;*/ DpsStackItemFree(res); } #ifdef DEBUG_BOOL DpsLog(query, DPS_LOG_EXTRA, "result: %x", res); #endif DpsBoolStackFree(s); return DPS_OK; }
DPS_WIDEWORDLIST * DpsSynonymListFind(const DPS_SYNONYMLIST * List,DPS_WIDEWORD * wword){ DPS_SYNONYM syn,*res,*first,*last; DPS_SYNONYM *psyn, **pres, **pfirst, **plast; DPS_WIDEWORDLIST *Res = NULL; size_t nnorm,i; if(!List->nsynonyms)return NULL; syn.p.uword = wword->uword; res = bsearch(&syn, List->Synonym, List->nsynonyms, sizeof(DPS_SYNONYM), &cmpsyn); if(res){ Res = (DPS_WIDEWORDLIST *)DpsMalloc(sizeof(*Res)); if (Res == NULL) return NULL; DpsWideWordListInit(Res); /* Find first and last synonym */ for(first = res; first >= List->Synonym; first--) { if(DpsUniStrCmp(wword->uword,first->p.uword)){ break; }else{ first->s.order = wword->order; first->s.origin = DPS_WORD_ORIGIN_SYNONYM; DpsWideWordListAdd(Res,&first->s, DPS_WWL_LOOSE); } } for(last=res+1;last<List->Synonym+List->nsynonyms;last++){ if(DpsUniStrCmp(wword->uword,last->p.uword)){ break; }else{ last->s.order=wword->order; last->s.origin = DPS_WORD_ORIGIN_SYNONYM; DpsWideWordListAdd(Res,&last->s, DPS_WWL_LOOSE); } } } syn.s.uword = wword->uword; psyn = &syn; pres = bsearch(&psyn, List->Back, List->nsynonyms, sizeof(DPS_SYNONYM*), &cmpsynback); if(pres) { if (Res == NULL) { Res = (DPS_WIDEWORDLIST *)DpsMalloc(sizeof(*Res)); if (Res == NULL) return NULL; DpsWideWordListInit(Res); } /* Find first and last synonym */ for(pfirst = pres; pfirst >= List->Back; pfirst--) { if(DpsUniStrCmp(wword->uword, (*pfirst)->s.uword)) { break; }else{ (*pfirst)->p.order = wword->order; (*pfirst)->p.origin = DPS_WORD_ORIGIN_SYNONYM; DpsWideWordListAdd(Res, &((*pfirst)->p), DPS_WWL_LOOSE); } } for(plast = pres + 1; plast < List->Back + List->nsynonyms; plast++) { if(DpsUniStrCmp(wword->uword, (*plast)->s.uword)) { break; } else { (*plast)->p.order = wword->order; (*plast)->p.origin = DPS_WORD_ORIGIN_SYNONYM; DpsWideWordListAdd(Res, &((*plast)->p), DPS_WWL_LOOSE); } } } if (Res == NULL) return NULL; /* Now find each of them in reverse order */ if ((nnorm = Res->nwords) > 0) { for(i = 0; i < nnorm; i++) { syn.p.uword = Res->Word[i].uword; res = bsearch(&syn, List->Synonym, List->nsynonyms, sizeof(DPS_SYNONYM), &cmpsyn); if(res){ /* Find first and last synonym */ for(first = res; first >= List->Synonym; first--) { if(DpsUniStrCmp(Res->Word[i].uword, first->p.uword)){ break; }else{ if ((Res->Word[i].count != 0) && (first->p.count != Res->Word[i].count)) continue; first->s.order = wword->order; first->s.origin = DPS_WORD_ORIGIN_SYNONYM; DpsWideWordListAdd(Res, &first->s, DPS_WWL_LOOSE); } } for(last=res+1;last<List->Synonym+List->nsynonyms;last++){ if(DpsUniStrCmp(Res->Word[i].uword, last->p.uword)) { break; }else{ if ((Res->Word[i].count != 0) && (last->p.count != Res->Word[i].count)) continue; last->s.order=wword->order; last->s.origin = DPS_WORD_ORIGIN_SYNONYM; DpsWideWordListAdd(Res, &last->s, DPS_WWL_LOOSE); } } } syn.s.uword = Res->Word[i].uword; pres = bsearch(&psyn, List->Back, List->nsynonyms, sizeof(DPS_SYNONYM*), &cmpsynback); if(pres) { /* Find first and last synonym */ for(pfirst = pres; pfirst >= List->Back; pfirst--) { if(DpsUniStrCmp(syn.s.uword, (*pfirst)->s.uword)) { break; } else { if ((Res->Word[i].count != 0) && ((*pfirst)->s.count != Res->Word[i].count)) continue; (*pfirst)->s.order = wword->order; (*pfirst)->s.origin = DPS_WORD_ORIGIN_SYNONYM; DpsWideWordListAdd(Res, &((*pfirst)->s), DPS_WWL_LOOSE); } } for(plast = pres + 1; plast < List->Back + List->nsynonyms; plast++) { if(DpsUniStrCmp(syn.s.uword, (*plast)->s.uword)) { break; } else { if ((Res->Word[i].count != 0) && ((*plast)->s.count != Res->Word[i].count)) continue; (*plast)->s.order = wword->order; (*plast)->s.origin = DPS_WORD_ORIGIN_SYNONYM; DpsWideWordListAdd(Res, &((*plast)->s), DPS_WWL_LOOSE); } } } } } return(Res); }