/* * allocateMoreSlots is called when there are only enough slot structures * left to support the allocation of a single malloc buffer. */ static void allocateMoreSlots(void) { size_t newSize = allocationListSize + bytesPerPage; void * newAllocation; void * oldAllocation = allocationList; Page_AllowAccess(allocationList, allocationListSize); noAllocationListProtection = 1; internalUse = 1; newAllocation = DpsMalloc(newSize); dps_memcpy(newAllocation, allocationList, allocationListSize); /* was: dps_memmove */ memset(&(((char *)newAllocation)[allocationListSize]), 0, bytesPerPage); allocationList = (Slot *)newAllocation; allocationListSize = newSize; slotCount += slotsPerPage; unUsedSlots += slotsPerPage; /* DpsSort(allocationList, slotCount, sizeof(Slot), (qsort_cmp)cmp_Slot);*/ DpsFree(oldAllocation); /* * Keep access to the allocation list open at this point, because * I am returning to memalign(), which needs that access. */ noAllocationListProtection = 0; internalUse = 0; }
int DpsCookiesAddStr(DPS_AGENT *Indexer, DPS_URL *CurURL, const char *cookie_str, int insert_flag) { char *part, *lpart; char *name = NULL; char *value = NULL; char *domain = NULL, *orig_domain = NULL; char *path = NULL; dps_uint4 expire = 0, need_free_domain = 1, need_free_path = 1; char secure = 'n', savec; for (part = dps_strtok_r(cookie_str, ";" , &lpart, &savec) ; part; part = dps_strtok_r(NULL, ";", &lpart, &savec)) { char *arg; part = DpsTrim(part, " "); if ((arg = strchr(part, '='))) { *arg++ = '\0'; if (!name) { name = part; DpsFree(value); value = DpsStrdup(arg); } else if (!strcasecmp(part, "path")) { DpsFree(path); path = DpsStrdup(arg); } else if (!strcasecmp(part, "domain")) { DpsFree(orig_domain); orig_domain = domain = DpsStrdup(arg); } else if (!strcasecmp(part, "secure")) { secure = 'y'; } else if (!strcasecmp(part, "expires")) { expire = (dps_uint4)DpsHttpDate2Time_t(arg); } } } if (name && value) { if (domain && domain[0] == '.') { domain++; } else { if (domain) DpsFree(orig_domain); domain = (CurURL && CurURL->hostname) ? CurURL->hostname : "localhost"; need_free_domain = 0; } if (!path) { path = (CurURL && CurURL->path) ? CurURL->path : "/"; need_free_path = 0; } DpsCookiesAdd(Indexer, domain, path, name, value, secure, expire, (CurURL==NULL)?1:0, 1); } DpsFree(value); if (need_free_path) DpsFree(path); if (need_free_domain) DpsFree(orig_domain); return DPS_OK; }
extern C_LINKAGE void * _DpsRealloc(void * oldBuffer, size_t newSize, const char *filename, size_t fileline) { void * newBuffer = 0; if ( allocationList == 0 ) initialize(); /* This sets EF_ALIGNMENT */ /* fprintf(stderr, "DpsRealloc: %p at %s:%d\n", oldBuffer, filename, fileline); */ lock(); newBuffer = _DpsMalloc(newSize, filename, fileline); if ( oldBuffer ) { size_t size; Slot * slot; Page_AllowAccess(allocationList, allocationListSize); noAllocationListProtection = 1; slot = slotForUserAddress(oldBuffer); if ( slot == 0 ) EF_Abort("DpsRealloc(%a, %d): address not from DpsMalloc() at %s:%d.", oldBuffer, newSize, filename, fileline); if ( newSize < (size = slot->userSize) ) size = newSize; if ( size > 0 ) dps_memcpy(newBuffer, oldBuffer, size); /* was: dps_memmove */ DpsFree(oldBuffer); noAllocationListProtection = 0; Page_DenyAccess(allocationList, allocationListSize); if ( size < newSize ) memset(&(((char *)newBuffer)[size]), 0, newSize - size); /* Internal memory was re-protected in free() */ } release(); return newBuffer; }
/* * allocateMoreSlots is called when there are only enough slot structures * left to support the allocation of a single malloc buffer. */ static void allocateMoreSlots(void) { size_t newSize = allocationListSize + bytesPerPage; void * newAllocation; void * oldAllocation = allocationList; newAllocation = DpsMalloc(newSize); dps_memmove(newAllocation, allocationList, allocationListSize); memset(&(((char *)newAllocation)[allocationListSize]), 0, bytesPerPage); allocationList = (Slot *)newAllocation; allocationListSize = newSize; slotCount += slotsPerPage; unUsedSlots += slotsPerPage; /* DpsSort(allocationList, slotCount, sizeof(Slot), (qsort_cmp)cmp_Slot);*/ DpsFree(oldAllocation); }
void DpsParseHTTPResponse(DPS_AGENT *Indexer, DPS_DOCUMENT *Doc) { char *token, *lt, *headers; int oldstatus; DPS_DSTR header; Doc->Buf.content=NULL; oldstatus = DpsVarListFindInt(&Doc->Sections, "Status", 0); DpsVarListReplaceInt(&Doc->Sections, "ResponseSize", (int)Doc->Buf.size); DpsVarListDel(&Doc->Sections, "Content-Length"); DpsVarListDel(&Doc->Sections, "Last-Modified"); if (Doc->Buf.buf == NULL) return; /* Cut HTTP response header first */ for(token=Doc->Buf.buf;*token;token++){ if(!strncmp(token,"\r\n\r\n",4)){ *token='\0'; Doc->Buf.content = token + 4; break; }else if(!strncmp(token,"\n\n",2)){ *token='\0'; Doc->Buf.content = token + 2; break; } } /* Bad response, return */ if(!Doc->Buf.content) { if (token < Doc->Buf.buf + Doc->Buf.size - 4) { if (token[2] == '\r') Doc->Buf.content = token + 4; else Doc->Buf.content = token + 2; } else { return; } } /* Copy headers not to break them */ headers = (char*)DpsStrdup(Doc->Buf.buf); /* Now lets parse response header lines */ token = dps_strtok_r(headers,"\r\n",<); if(!token)return; if(!strncmp(token,"HTTP/",5)){ int status = atoi(token + 8); DpsVarListReplaceStr(&Doc->Sections,"ResponseLine",token); DpsVarListReplaceInt(&Doc->Sections, "Status", (oldstatus > status) ? oldstatus : status ); }else{ DpsFree(headers); return; } token = dps_strtok_r(NULL,"\r\n",<); DpsDSTRInit(&header, 128); while(token){ if(strchr(token,':')) { if (header.data_size) { DpsParseHTTPHeader(Indexer, Doc, &header); DpsDSTRFree(&header); DpsDSTRInit(&header, 128); } } DpsDSTRAppendStr(&header, token); token = dps_strtok_r(NULL,"\r\n",<); } if (header.data_size) { DpsParseHTTPHeader(Indexer, Doc, &header); } DpsDSTRFree(&header); DPS_FREE(headers); DpsVarListInsInt(&Doc->Sections,"Content-Length",Doc->Buf.buf-Doc->Buf.content+(int)Doc->Buf.size); }
int __DPSCALL DpsResAddDocInfoSearchd(DPS_AGENT * query,DPS_DB *cl,DPS_RESULT * Res,size_t clnum){ DPS_SEARCHD_PACKET_HEADER hdr; char * msg=NULL; size_t i; /* num=0,curnum=0;*/ int done = 0; ssize_t nsent,nrecv; char * dinfo=NULL; int rc=DPS_OK; char *textbuf; size_t dlen = 0; TRACE_IN(query, "DpsResAddDocInfoSearchd"); if(!Res->num_rows) { TRACE_OUT(query); return(DPS_OK); } for(i=0;i<Res->num_rows;i++){ size_t ulen; size_t olen; size_t nsec, r; DPS_DOCUMENT *D=&Res->Doc[i]; r = (size_t) 's'; for(nsec = 0; nsec < D->Sections.Root[r].nvars; nsec++) if (strcasecmp(D->Sections.Root[r].Var[nsec].name, "Score") == 0) D->Sections.Root[r].Var[nsec].section = 1; #ifdef WITH_MULTIDBADDR if (D->dbnum != cl->dbnum) continue; #endif textbuf = DpsDocToTextBuf(D, 1, 0); if (textbuf == NULL) {TRACE_OUT(query); return DPS_ERROR;} ulen = dps_strlen(textbuf)+2; olen = dlen; dlen = dlen + ulen; dinfo = (char*)DpsRealloc(dinfo, dlen + 1); if (dinfo == NULL) { DpsFree(textbuf); TRACE_OUT(query); return DPS_ERROR; } dinfo[olen] = '\0'; sprintf(dinfo + olen, "%s\r\n", textbuf); DpsFree(textbuf); } if (dinfo == NULL) { TRACE_OUT(query); return DPS_OK; } hdr.cmd=DPS_SEARCHD_CMD_DOCINFO; hdr.len = dps_strlen(dinfo); nsent = DpsSearchdSendPacket(cl->searchd, &hdr, dinfo); #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Sent DOCINFO size=%d buf=%s\n", hdr.len, dinfo); #endif while(!done){ char * tok, * lt; nrecv = DpsRecvall(cl->searchd, &hdr, sizeof(hdr), 360); if(nrecv!=sizeof(hdr)){ DpsLog(query, DPS_LOG_ERROR, "Received incomplete header from searchd (%d bytes, errno:%d)", (int)nrecv, errno); TRACE_OUT(query); return(DPS_ERROR); }else{ #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Received header cmd=%d len=%d\n",hdr.cmd,hdr.len); #endif } switch(hdr.cmd){ case DPS_SEARCHD_CMD_ERROR: msg=(char*)DpsMalloc(hdr.len+1); if (msg == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, msg, hdr.len, 360); msg[(nrecv >= 0) ? nrecv : 0]='\0'; sprintf(query->Conf->errstr,"Searchd error: '%s'",msg); rc=DPS_ERROR; DPS_FREE(msg); done=1; break; case DPS_SEARCHD_CMD_MESSAGE: msg=(char*)DpsMalloc(hdr.len+1); if (msg == NULL) { done = 1; break; } nrecv = DpsRecvall(cl->searchd, msg, hdr.len, 360); msg[(nrecv >= 0) ? nrecv : 0]='\0'; #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Message from searchd: '%s'\n",msg); #endif DPS_FREE(msg); break; case DPS_SEARCHD_CMD_DOCINFO: dinfo = (char*)DpsRealloc(dinfo, hdr.len + 1); if (dinfo == NULL) { done=1; break; } nrecv = DpsRecvall(cl->searchd, dinfo, hdr.len, 360); dinfo[(nrecv > 0) ? nrecv : 0] = '\0'; #ifdef DEBUG_SDP DpsLog(query, DPS_LOG_ERROR, "Received DOCINFO size=%d buf=%s\n",hdr.len,dinfo); #endif tok = dps_strtok_r(dinfo, "\r\n", <, NULL); while(tok){ urlid_t Doc_url_id, Res_Doc_url_id; DPS_DOCUMENT Doc; DpsDocInit(&Doc); DpsDocFromTextBuf(&Doc,tok); Doc_url_id = (urlid_t)DpsVarListFindInt(&Doc.Sections, "DP_ID", 0); for(i=0;i<Res->num_rows;i++){ #ifdef WITH_MULTIDBADDR if (Res->Doc[i].dbnum != cl->dbnum) continue; #endif Res_Doc_url_id = (urlid_t)DpsVarListFindInt(&Res->Doc[i].Sections, "DP_ID", 0); if (Res_Doc_url_id == Doc_url_id) { DpsDocFromTextBuf(&Res->Doc[i], tok); break; } } tok = dps_strtok_r(NULL, "\r\n", <, NULL); DpsDocFree(&Doc); } DPS_FREE(dinfo); done=1; break; default: sprintf(query->Conf->errstr,"Unknown searchd response: cmd=%d len=%d",hdr.cmd,hdr.len); rc=DPS_ERROR; done=1; break; } } TRACE_OUT(query); return rc; }
void DpsParseHTTPResponse(DPS_AGENT *Indexer, DPS_DOCUMENT *Doc) { char *token, *lt, *headers, savec; int status, oldstatus; DPS_DSTR header; time_t now, last_mod_time; Doc->Buf.content=NULL; oldstatus = DpsVarListFindInt(&Doc->Sections, "Status", 0); DpsVarListReplaceInt(&Doc->Sections, "ResponseSize", (int)Doc->Buf.size); DpsVarListDel(&Doc->Sections, "Content-Length"); /* DpsVarListDel(&Doc->Sections, "Last-Modified");*/ /* if it's not deleted Lat-Modified equals to the first appearance in db */ if (Doc->Buf.buf == NULL) return; /* Cut HTTP response header first */ for(token=Doc->Buf.buf;*token;token++){ if(!strncmp(token,"\r\n\r\n",4)){ if (token <= Doc->Buf.buf + Doc->Buf.size - 4) { *token='\0'; Doc->Buf.content = token + 4; } break; } else if(!strncmp(token,"\n\n",2)){ if (token <= Doc->Buf.buf + Doc->Buf.size - 2) { *token='\0'; Doc->Buf.content = token + 2; } break; } } /* Bad response, return */ if(!Doc->Buf.content) { if (token <= Doc->Buf.buf + Doc->Buf.size - 4) { if (token[2] == CR_CHAR) Doc->Buf.content = token + 4; else Doc->Buf.content = token + 2; } } /* Copy headers not to break them */ headers = (char*)DpsStrdup(Doc->Buf.buf); /* Now lets parse response header lines */ token = dps_strtok_r(headers, "\r\n", <, &savec); if(!token) { DpsFree(headers); return; } if(!strncmp(token,"HTTP/",5)){ status = atoi(token + 8); DpsVarListReplaceStr(&Doc->Sections,"ResponseLine",token); DpsVarListReplaceInt(&Doc->Sections, "Status", (oldstatus > status) ? oldstatus : status ); }else{ DpsFree(headers); return; } token = dps_strtok_r(NULL, "\r\n", <, &savec); DpsDSTRInit(&header, 128); while(token){ if(strchr(token,':')) { if (header.data_size) { DpsParseHTTPHeader(Indexer, Doc, &header); DpsDSTRFree(&header); DpsDSTRInit(&header, 128); } } DpsDSTRAppendStr(&header, token); token = dps_strtok_r(NULL, "\r\n", <, &savec); } if (header.data_size) { DpsParseHTTPHeader(Indexer, Doc, &header); } DpsDSTRFree(&header); DPS_FREE(headers); { now = Indexer->now; last_mod_time = DpsHttpDate2Time_t(DpsVarListFindStr(&Doc->Sections, "Last-Modified", "")); if (last_mod_time > now + 3600 * 4) { /* we have a document with Last-Modified time in the future */ DpsLog(Indexer, DPS_LOG_EXTRA, "Last-Modified date is deep in future (%d>%d), dropping it.", last_mod_time, now); DpsVarListDel(&Doc->Sections, "Last-Modified"); } } /* Bad response, return */ if(!Doc->Buf.content) { return; } DpsVarListReplaceInt(&Doc->Sections,"Content-Length", Doc->Buf.buf-Doc->Buf.content+(int)Doc->Buf.size + DpsVarListFindInt(&Doc->Sections,"Content-Length", 0)); }
__C_LINK void * __DPSCALL DpsBaseARead(DPS_BASE_PARAM *P, size_t *len) { int res = DPS_OK; char *buf = NULL; #ifdef HAVE_ZLIB Byte *CDoc = NULL; z_stream zstream; #endif if ((res = DpsBaseSeek(P, DPS_READ_LOCK)) != DPS_OK) { *len = 0; return NULL; } if (P->Item.rec_id == P->rec_id) { if (lseek(P->Sfd, (off_t)P->Item.offset, SEEK_SET) == (off_t)-1) { *len = 0; return NULL; } #ifdef HAVE_ZLIB bzero(&zstream, sizeof(zstream)); if ((P->zlib_method == Z_DEFLATED) && (P->Item.orig_size != 0)) { zstream.avail_in = (uInt)P->Item.size; *len = zstream.avail_out = (uInt)(2 * P->Item.size + P->Item.orig_size); CDoc = zstream.next_in = (Byte *) DpsMalloc(P->Item.size + 1); if (CDoc == NULL) { *len = 0; return NULL; } if ((buf = (char*)DpsMalloc(*len + 1)) == NULL) { DPS_FREE(CDoc); *len = 0; return NULL; } /* fprintf(stderr, "BaseARead: Item.size: %d .orig_size: %d len: %d\n", P->Item.size, P->Item.orig_size, *len);*/ zstream.next_out = (Byte *) buf; zstream.zalloc = Z_NULL; zstream.zfree = Z_NULL; zstream.opaque = Z_NULL; if (read(P->Sfd, CDoc, P->Item.size) != (ssize_t)P->Item.size) { DpsLog(P->A, DPS_LOG_ERROR, "[%s/%s] %d read error, rec_id: %x, deleting... -- %d", P->subdir, P->basename, P->Item.size, P->rec_id, __LINE__); DpsBaseDelete(P); DPS_FREE(buf); DPS_FREE(CDoc); return NULL; } inflateInit2(&zstream, P->zlib_windowBits); res = inflate(&zstream, Z_FINISH); /* fprintf(stderr, "inflate exit: %d avail_out: %d total_out: %d avail_in: %d\n", res, zstream.avail_out, zstream.total_out, zstream.avail_in);*/ ; *len = zstream.total_out; inflateEnd(&zstream); DPS_FREE(CDoc); } else #endif { if ((buf = (char*)DpsMalloc((*len = P->Item.size) + 1)) == NULL) { *len = 0; return NULL; } if (read(P->Sfd, buf, P->Item.size) != (ssize_t)P->Item.size) { DpsFree(buf); *len = 0; return NULL; } } } else { DpsLog(P->A, DPS_LOG_DEBUG, "%s:[%s/%s] Not found rec_id: %x", P->vardir, P->subdir, P->basename, P->rec_id); *len = 0; return NULL; } buf[*len] = '0'; #ifdef DEBUG_SEARCH DpsLog(P->A, DPS_LOG_DEBUG, "[%s/%s] ARetrieved rec_id: %x Size: %d->%d", P->subdir, P->basename, P->rec_id, P->Item.size, P->Item.orig_size); #endif return buf; }
void DpsCookiesFind(DPS_AGENT *Indexer, DPS_SERVER *Server, DPS_DOCUMENT *Doc, const char *hostinfo) { #ifdef HAVE_SQL DPS_DSTR cookie; DPS_COOKIES *Cookies = &Indexer->Cookies; DPS_COOKIE *Coo; size_t i, blen = dps_strlen(hostinfo), slen; int have_no_cookies = DpsVarListFindInt(&Doc->Sections, "have_no_cookies", 1); #ifdef WITH_PARANOIA void *paran = DpsViolationEnter(paran); #endif TRACE_IN(Indexer, "DpsCookiesFind"); DpsDSTRInit(&cookie, 1024); for(i = 0; i < Cookies->ncookies; i++) { Coo = &Cookies->Cookie[i]; slen = dps_strlen(Coo->domain); if (slen > blen) continue; if (Coo->secure == 'y' && strcasecmp(Doc->CurURL.schema, "https")) continue; if (strncasecmp(Coo->path, Doc->CurURL.path, dps_strlen(Coo->path))) continue; if (strcasecmp(Coo->domain, hostinfo + (blen - slen))) continue; if (Coo->from_config != 1) have_no_cookies = 0; if (Coo->name[0] == '\0' && Coo->value[0] == '\0') continue; if (cookie.data_size) DpsDSTRAppend(&cookie, "; ", 2); DpsDSTRAppendStr(&cookie, Coo->name); DpsDSTRAppend(&cookie, "=", 1); DpsDSTRAppendStr(&cookie, Coo->value); } if (have_no_cookies) { char buf[2*PATH_MAX]; dpshash32_t url_id; DPS_DB *db; DPS_SQLRES Res; size_t rows; int rc; if (Server != NULL) { char *PingData = DpsVarListFindStr(&Server->Vars, "AuthPing", NULL); if (PingData != NULL) { char *AuthPing = DpsStrdup(DpsTrim(PingData, " \t\r\n")); int method = DPS_METHOD_GET; dps_base64_decode(AuthPing, PingData, dps_strlen(PingData)); if (!strncasecmp(AuthPing, "GET", 3)) { method = DPS_METHOD_GET; PingData = DpsTrim(AuthPing + 3, " \t\r\n"); } else if (!strncasecmp(AuthPing, "POST", 4)) { method = DPS_METHOD_POST; PingData = DpsTrim(AuthPing + 4, " \t\r\n"); } else { DpsLog(Indexer, DPS_LOG_ERROR, "AuthPing should be GET or POST: %s", AuthPing); PingData = NULL; } if (PingData != NULL) { size_t size = dps_strlen(PingData); { char PingURL[size + 2]; char PingBody[size]; DPS_DOCUMENT *rDoc; int result; rDoc = DpsDocInit(NULL); DpsSpiderParamInit(&rDoc->Spider); DpsVarList2Doc(rDoc, Server); rDoc->Buf.max_size = (size_t)DpsVarListFindInt(&Indexer->Vars, "MaxDocSize", DPS_MAXDOCSIZE); rDoc->Buf.allocated_size = DPS_NET_BUF_SIZE; if ((rDoc->Buf.buf = (char*)DpsMalloc(rDoc->Buf.allocated_size + 1)) == NULL) { DpsDocFree(rDoc); TRACE_OUT(Indexer); return; } rDoc->Buf.buf[0]='\0'; rDoc->subdoc = Indexer->Flags.SubDocLevel + 1; #if 1 dps_snprintf(buf, sizeof(buf), "%s://%s/", DPS_NULL2EMPTY(Doc->CurURL.schema), DPS_NULL2EMPTY(Doc->CurURL.hostinfo)); DpsVarListReplaceStr(&rDoc->Sections, "URL", buf); DpsURLParse(&rDoc->CurURL, buf); DpsLog(Indexer, DPS_LOG_INFO, "HOME: %s", buf); rDoc->method = DPS_METHOD_HEAD; /* DpsVarListFree(&rDoc->RequestHeaders);*/ if (Doc != NULL) { DpsVarListReplaceLst(&rDoc->RequestHeaders, &Doc->RequestHeaders, NULL, "*"); } DpsVarListReplaceStr(&rDoc->Sections, "have_no_cookies", "0"); DpsDocAddDocExtraHeaders(Indexer, Server, rDoc); DpsDocAddConfExtraHeaders(Indexer->Conf, rDoc); DpsVarListReplaceLst(&rDoc->Sections, &Server->Vars, NULL, "*"); DpsDocAddServExtraHeaders(Server, rDoc); DpsVarListLog(Indexer, &rDoc->RequestHeaders, DPS_LOG_DEBUG, "HOME.Request"); if (Doc == NULL || Indexer->Flags.cmd == DPS_IND_FILTER) { DpsDocLookupConn(Indexer, rDoc); } else { DPS_FREE(rDoc->connp.connp); rDoc->connp = Doc->connp; } result = DpsGetURL(Indexer, rDoc, NULL); /* Just get headers from the home as we need only Cookies from it */ DpsDocProcessResponseHeaders(Indexer, rDoc); DpsVarListLog(Indexer, &rDoc->Sections, DPS_LOG_DEBUG, "HOME.Response"); #endif sscanf(PingData, "%s %s", PingURL, PingBody); if (rDoc->method == DPS_METHOD_GET) { dps_strcat(PingURL, "?"); dps_strcat(PingURL, PingBody); } else { DpsVarListReplaceStr(&rDoc->Sections, "body", PingBody); } DpsVarListReplaceStr(&rDoc->Sections, "URL", PingURL); DpsURLParse(&rDoc->CurURL, PingURL); DpsLog(Indexer, DPS_LOG_INFO, "AUTH.PING: %s", PingURL); rDoc->method = method; DpsVarListFree(&rDoc->RequestHeaders); DpsVarListReplaceStr(&rDoc->Sections, "have_no_cookies", "0"); DpsDocAddDocExtraHeaders(Indexer, Server, rDoc); DpsDocAddConfExtraHeaders(Indexer->Conf, rDoc); DpsVarListReplaceLst(&rDoc->Sections, &Server->Vars, NULL, "*"); DpsDocAddServExtraHeaders(Server, rDoc); if (method == DPS_METHOD_POST) { dps_snprintf(buf, sizeof(buf), "application/x-www-form-urlencoded; charset=%s", DpsVarListFindStr(&Indexer->Conf->Vars, "LocalCharset", "iso-8859-1")); DpsVarListReplaceStr(&rDoc->RequestHeaders, "Content-Type", buf); dps_snprintf(buf, sizeof(buf), "%d", dps_strlen(PingBody)); DpsVarListReplaceStr(&rDoc->RequestHeaders, "Content-Length", buf); } DpsVarListLog(Indexer, &rDoc->RequestHeaders, DPS_LOG_DEBUG, "AUTHPING.Request"); #if 0 if (Doc == NULL || Indexer->Flags.cmd == DPS_IND_FILTER) { DpsDocLookupConn(Indexer, rDoc); } else { DPS_FREE(rDoc->connp.connp); rDoc->connp = Doc->connp; } #endif result = DpsGetURL(Indexer, rDoc, NULL); /* Just get it as we need only Cookies from the headers */ DpsDocProcessResponseHeaders(Indexer, rDoc); DpsVarListDel(&rDoc->Sections, "body"); DpsVarListLog(Indexer, &rDoc->Sections, DPS_LOG_DEBUG, "AUTHPING.Response"); if (Doc != NULL) bzero(&rDoc->connp, sizeof(rDoc->connp)); DpsDocFree(rDoc); } } DpsFree(AuthPing); } } while(hostinfo != NULL) { url_id = DpsStrHash32(hostinfo); DpsSQLResInit(&Res); dps_snprintf(buf, sizeof(buf), "SELECT name,value,path,secure FROM cookies WHERE domain='%s'", hostinfo); if (Indexer->flags & DPS_FLAG_UNOCON) { DPS_GETLOCK(Indexer, DPS_LOCK_DB); db = Indexer->Conf->dbl.db[url_id % Indexer->Conf->dbl.nitems]; } else { db = Indexer->dbl.db[url_id % Indexer->dbl.nitems]; } if(DPS_OK == (rc = DpsSQLQuery(db, &Res, buf))) { rows = DpsSQLNumRows(&Res); for(i = 0; i < rows; i++) { DpsCookiesAdd(Indexer, hostinfo, DpsSQLValue(&Res, i, 2), DpsSQLValue(&Res, i, 0), DpsSQLValue(&Res, i, 1), *DpsSQLValue(&Res, i, 3), 0, 0, 0); if (*DpsSQLValue(&Res, i, 3) == 'y' && strcasecmp(Doc->CurURL.schema, "https")) continue; if (strncasecmp(DpsSQLValue(&Res, i, 2), Doc->CurURL.path, dps_strlen(DpsSQLValue(&Res, i, 2)))) continue; if (cookie.data_size) DpsDSTRAppend(&cookie, "; ", 2); DpsDSTRAppendStr(&cookie, DpsSQLValue(&Res, i, 0)); DpsDSTRAppend(&cookie, "=", 1); DpsDSTRAppendStr(&cookie, DpsSQLValue(&Res, i, 1)); } if (rows == 0) { DpsCookiesAdd(Indexer, hostinfo, "/", "", "", 'n', 0, 0, 0); } } DpsSQLFree(&Res); if (Indexer->flags & DPS_FLAG_UNOCON) { DPS_RELEASELOCK(Indexer, DPS_LOCK_DB); } hostinfo = strchr(hostinfo, '.'); if (hostinfo != NULL) hostinfo++; } } if (cookie.data_size) { DpsVarListReplaceStr(&Doc->RequestHeaders, "Cookie", cookie.data); } DpsDSTRFree(&cookie); #endif TRACE_OUT(Indexer); return; }