void DpsCookiesClean(DPS_AGENT *A) { char buf[256]; DPS_DB *db; size_t i, dbfrom = 0, dbto; int res; if (A->Flags.robots_period == 0) return; dps_snprintf(buf, sizeof(buf), "DELETE FROM cookies WHERE expires < %d", A->now); if (A->flags & DPS_FLAG_UNOCON) DPS_GETLOCK(A, DPS_LOCK_CONF); dbto = DPS_DBL_TO(A); if (A->flags & DPS_FLAG_UNOCON) DPS_RELEASELOCK(A, DPS_LOCK_CONF); for (i = dbfrom; i < dbto; i++) { db = DPS_DBL_DB(A, i); if (A->flags & DPS_FLAG_UNOCON) DPS_GETLOCK(A, DPS_LOCK_DB); #ifdef HAVE_SQL res = DpsSQLAsyncQuery(db, NULL, buf); #endif if (res != DPS_OK) { DpsLog(A, DPS_LOG_ERROR, db->errstr); } if (A->flags & DPS_FLAG_UNOCON) DPS_RELEASELOCK(A, DPS_LOCK_DB); if (res != DPS_OK) break; } }
void DpsAppendTarget(DPS_AGENT *Indexer, const char *url, const char *lang, const int hops, int parent) { DPS_DOCUMENT *Doc, *Save; size_t i; TRACE_IN(Indexer, "AppendTarget"); DPS_GETLOCK(Indexer, DPS_LOCK_THREAD); DPS_GETLOCK(Indexer, DPS_LOCK_CONF); if (Indexer->Conf->Targets.num_rows > 0) { for (i = Indexer->Conf->Targets.num_rows - 1; i > 0; i--) { Doc = &Indexer->Conf->Targets.Doc[i]; if ((strcasecmp(DpsVarListFindStr(&Doc->Sections, "URL", ""), url) == 0) && (strcmp(DpsVarListFindStr(&Doc->RequestHeaders, "Accept-Language", ""), lang) == 0)) { DPS_RELEASELOCK(Indexer, DPS_LOCK_CONF); DPS_RELEASELOCK(Indexer, DPS_LOCK_THREAD); TRACE_OUT(Indexer); return; } } } if ((Indexer->Conf->Targets.Doc = DpsRealloc(Save = Indexer->Conf->Targets.Doc, (Indexer->Conf->Targets.num_rows + 1) * sizeof(DPS_DOCUMENT))) == NULL) { Indexer->Conf->Targets.Doc = Save; DPS_RELEASELOCK(Indexer, DPS_LOCK_CONF); DPS_RELEASELOCK(Indexer, DPS_LOCK_THREAD); TRACE_OUT(Indexer); return; } Doc = &Indexer->Conf->Targets.Doc[Indexer->Conf->Targets.num_rows]; DpsDocInit(Doc); DpsVarListAddStr(&Doc->Sections, "URL", url); DpsVarListAddInt(&Doc->Sections, "Hops", hops); DpsVarListDel(&Doc->Sections, "URL_ID"); DpsVarListReplaceInt(&Doc->Sections, "Referrer-ID", parent); if (*lang != '\0') DpsVarListAddStr(&Doc->RequestHeaders, "Accept-Language", lang); if (DPS_OK == DpsURLAction(Indexer, Doc, DPS_URL_ACTION_FINDBYURL)) { urlid_t url_id = DpsVarListFindInt(&Doc->Sections, "DP_ID", 0); if (url_id != 0) Indexer->Conf->Targets.num_rows++; else DpsDocFree(Doc); } /* fprintf(stderr, "-- AppandTarget: url:%s URL_ID:%d\n", url, DpsStrHash32(url));*/ DPS_RELEASELOCK(Indexer, DPS_LOCK_CONF); DpsURLAction(Indexer, Doc, DPS_URL_ACTION_ADD); DPS_RELEASELOCK(Indexer, DPS_LOCK_THREAD); TRACE_OUT(Indexer); return; }
__C_LINK int __DPSCALL DpsBaseOpen(DPS_BASE_PARAM *P, int mode) { unsigned int hash; size_t filenamelen, z; ssize_t wr; DPS_BASEITEM *hTable; #ifdef DEBUG_SEARCH unsigned long total_ticks, stop_ticks, start_ticks = DpsStartTimer(); #endif TRACE_IN(P->A, "DpsBaseOpen"); if (P->opened) DpsBaseClose(P); if (P->NFiles == 0) P->NFiles = DpsVarListFindUnsigned(&P->A->Vars, "BaseFiles", 0x100); P->FileNo = DPS_FILENO(P->rec_id, P->NFiles); hash = DPS_HASH(P->rec_id); filenamelen = dps_strlen(P->vardir) + dps_strlen(P->subdir) + dps_strlen(P->indname) + dps_strlen(P->basename) + 48; if ( ((P->Ifilename = (char *)DpsMalloc(filenamelen)) == NULL) || ((P->Sfilename = (char *)DpsMalloc(filenamelen)) == NULL) ) { DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); DpsLog(P->A, DPS_LOG_ERROR, "Memory alloc error 2x%d bytes %s:%d", filenamelen, __FILE__, __LINE__); TRACE_OUT(P->A); return DPS_ERROR; } sprintf(P->Sfilename, "%s/%s/%s%04zx.s", P->vardir, P->subdir, P->basename, P->FileNo); sprintf(P->Ifilename, "%s/%s/%s%04zx.i", P->vardir, P->subdir, P->indname, P->FileNo); if ((P->Ifd = DpsOpen2(P->Ifilename, ((mode == DPS_READ_LOCK) ? O_RDONLY : O_RDWR) | DPS_BINARY)) < 0) { if ((mode == DPS_READ_LOCK) || ((P->Ifd = DpsOpen3(P->Ifilename, O_RDWR | O_CREAT | DPS_BINARY /*#ifdef O_DIRECT | O_DIRECT #endif*/ , S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH )) < 0)) { dps_strerror(P->A, (mode == DPS_READ_LOCK && errno == ENOENT) ? DPS_LOG_DEBUG : DPS_LOG_ERROR, "Can't open/create file %s for %s [%s:%d]", P->Ifilename, (mode == DPS_READ_LOCK) ? "read" : "write", __FILE__, __LINE__); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } #if 1 DPS_GETLOCK(P->A, DPS_LOCK_BASE_N(P->FileNo)); #endif DpsWriteLock(P->Ifd); if ((hTable = (DPS_BASEITEM *)DpsXmalloc(sizeof(DPS_BASEITEM) * DPS_HASH_PRIME)) == NULL) { DpsLog(P->A, DPS_LOG_ERROR, "Memory alloc error hTable: %d bytes", sizeof(DPS_BASEITEM) * DPS_HASH_PRIME); DpsUnLock(P->Ifd); #if 1 DPS_RELEASELOCK(P->A, DPS_LOCK_BASE_N(P->FileNo)); #endif DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } if ( (wr = write(P->Ifd, hTable, sizeof(DPS_BASEITEM) * DPS_HASH_PRIME)) != sizeof(DPS_BASEITEM) * DPS_HASH_PRIME) { dps_strerror(P->A, DPS_LOG_ERROR, "Can't set new index for file %s\nwritten %d bytes of %d\nIfd:%d hTable:%x", P->Ifilename, wr, sizeof(DPS_BASEITEM) * DPS_HASH_PRIME, P->Ifd, hTable); DPS_FREE(hTable); DpsUnLock(P->Ifd); #if 1 DPS_RELEASELOCK(P->A, DPS_LOCK_BASE_N(P->FileNo)); #endif DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } DpsUnLock(P->Ifd); #if 1 DPS_RELEASELOCK(P->A, DPS_LOCK_BASE_N(P->FileNo)); #endif DPS_FREE(hTable); if (lseek(P->Ifd, (off_t)0, SEEK_SET) == (off_t)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seek for file %s", P->Ifilename); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } } if (!P->A->Flags.cold_var) { #if 1 DPS_GETLOCK(P->A, DPS_LOCK_BASE_N(P->FileNo)); #endif switch (mode) { case DPS_READ_LOCK: DpsReadLock(P->Ifd); break; case DPS_WRITE_LOCK: DpsWriteLock(P->Ifd); break; } P->locked = 1; } if ((P->Sfd = DpsOpen2(P->Sfilename, ((mode == DPS_READ_LOCK) ? O_RDONLY : O_RDWR) | DPS_BINARY /*#ifdef O_DIRECT | O_DIRECT #endif*/ )) < 0) { if ((mode == DPS_READ_LOCK) || ((P->Sfd = DpsOpen3(P->Sfilename, O_RDWR | O_CREAT | DPS_BINARY /*#ifdef O_DIRECT | O_DIRECT #endif*/ , S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH )) < 0)) { DpsLog(P->A, DPS_LOG_ERROR, "Can't open/create file %s", P->Sfilename); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } } if (!P->A->Flags.cold_var) { switch(mode) { case DPS_READ_LOCK: DpsReadLock(P->Sfd); break; case DPS_WRITE_LOCK: DpsWriteLock(P->Sfd); break; } } #ifdef DEBUG_SEARCH stop_ticks = DpsStartTimer(); total_ticks = stop_ticks - start_ticks; DpsLog(P->A, DPS_LOG_EXTRA, "OpenBase1 %03X in %.5f sec.", P->FileNo, (float)total_ticks / 1000); #endif for (z = 0; z < 3; z++) { /* search rec_id */ if ( (P->CurrentItemPos = (dps_uint8)lseek(P->Ifd, (off_t)(hash * sizeof(DPS_BASEITEM)), SEEK_SET)) == (dps_uint8)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seeek for file %s", P->Ifilename); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } if (read(P->Ifd, &P->Item, sizeof(DPS_BASEITEM)) != sizeof(DPS_BASEITEM)) { DpsLog(P->A, DPS_LOG_ERROR, "{%s:%d} Can't read index for file %s seek:%ld hash: %u (%d)", __FILE__, __LINE__, P->Ifilename, P->CurrentItemPos, hash, hash); bzero(&P->Item, sizeof(P->Item)); /* DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; */ } #ifdef DEBUG_SEARCH stop_ticks = DpsStartTimer(); total_ticks = stop_ticks - start_ticks; DpsLog(P->A, DPS_LOG_EXTRA, "OpenBase2 %03X in %.5f sec.", P->FileNo, (float)total_ticks / 1000); #endif if (P->Item.rec_id == P->rec_id || P->Item.rec_id == 0) P->mishash = 0; else P->mishash = 1; P->PreviousItemPos = P->CurrentItemPos; if (P->mishash) while((P->Item.next != 0) && (P->Item.rec_id != P->rec_id)) { P->PreviousItemPos = P->CurrentItemPos; P->CurrentItemPos = P->Item.next; if (lseek(P->Ifd, (off_t)P->CurrentItemPos, SEEK_SET) == (off_t)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seek for file %s", P->Ifilename); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } if ((wr = read(P->Ifd, &P->Item, sizeof(DPS_BASEITEM))) != sizeof(DPS_BASEITEM)) { if (wr == 0) { DpsLog(P->A, DPS_LOG_ERROR, "Possible corrupted hash chain for file %s, trying to restore (%s:%d)", P->Ifilename, __FILE__, __LINE__); if (lseek(P->Ifd, (off_t)P->PreviousItemPos, SEEK_SET) == (off_t)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seek for file %s (%s:%d)", P->Ifilename, __FILE__, __LINE__); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } if ((wr = read(P->Ifd, &P->Item, sizeof(DPS_BASEITEM))) != sizeof(DPS_BASEITEM)) { DpsLog(P->A, DPS_LOG_ERROR, "Can't read previous pos for file %s (%s:%d)", P->Ifilename, __FILE__, __LINE__); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } P->Item.next = 0; if (lseek(P->Ifd, (off_t)P->PreviousItemPos, SEEK_SET) == (off_t)-1) { DpsLog(P->A, DPS_LOG_ERROR, "Can't seeek for file %s (%s:%d)", P->Ifilename, __FILE__, __LINE__); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } if ((wr = write(P->Ifd, &P->Item, sizeof(DPS_BASEITEM))) != sizeof(DPS_BASEITEM)) { DpsLog(P->A, DPS_LOG_ERROR, "Can't write previous pos for file %s (%s:%d)", P->Ifilename, __FILE__, __LINE__); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } goto search_again; } else { DpsLog(P->A, DPS_LOG_ERROR, "Can't read hash chain for file %s %d of %d bytes (%s:%d)", P->Ifilename, wr, sizeof(DPS_BASEITEM), __FILE__, __LINE__); DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename); TRACE_OUT(P->A); return DPS_ERROR; } } #ifdef DEBUG_SEARCH stop_ticks = DpsStartTimer(); total_ticks = stop_ticks - start_ticks; DpsLog(P->A, DPS_LOG_EXTRA, "OpenBase3 %03X in %.5f sec.", P->FileNo, (float)total_ticks / 1000); #endif } break; search_again:; } P->opened = 1; P->mode = mode; #ifdef DEBUG_SEARCH stop_ticks = DpsStartTimer(); total_ticks = stop_ticks - start_ticks; DpsLog(P->A, DPS_LOG_EXTRA, "OpenBase4 %03X in %.5f sec.\n", P->FileNo, (float)total_ticks / 1000); #endif /* fprintf(stderr, "Sfd:0x%x - %s\n", P->Sfd, P->Sfilename); fprintf(stderr, "Ifd:0x%x - %s\n", P->Ifd, P->Ifilename);*/ TRACE_OUT(P->A); return DPS_OK; }
int DpsCookiesAdd(DPS_AGENT *Indexer, const char *domain, const char * path, const char *name, const char *value, const char secure, dps_uint4 expires, const char from_config, int insert_flag) { #ifdef HAVE_SQL char buf[3*PATH_MAX]; char path_esc[2*PATH_MAX+1]; DPS_COOKIES *Cookies = &Indexer->Cookies; DPS_COOKIE *Coo; DPS_DB *db; dpshash32_t url_id = DpsStrHash32(domain); size_t i; #ifdef WITH_PARANOIA void *paran = DpsViolationEnter(paran); #endif if (Indexer->flags & DPS_FLAG_UNOCON) { if (Indexer->Conf->dbl.nitems == 0) return DPS_OK; DPS_GETLOCK(Indexer, DPS_LOCK_DB); db = Indexer->Conf->dbl.db[url_id % Indexer->Conf->dbl.nitems]; } else { if (Indexer->dbl.nitems == 0) return DPS_OK; db = Indexer->dbl.db[url_id % Indexer->dbl.nitems]; } (void)DpsDBEscStr(db, path_esc, DPS_NULL2EMPTY(path), dps_min(PATH_MAX,dps_strlen(DPS_NULL2EMPTY(path)))); for (i = 0; i < Cookies->ncookies; i++) { Coo = &Cookies->Cookie[i]; if (!strcasecmp(Coo->domain, domain) && !strcasecmp(Coo->path, DPS_NULL2EMPTY(path)) && !strcasecmp(Coo->name, name) && (Coo->secure == secure)/* && (Coo->from_config == from_config)*/ ) { DPS_FREE(Coo->value); Coo->value = DpsStrdup(value); /* Coo->expires = expires;*/ if (insert_flag) { dps_snprintf(buf, sizeof(buf), "UPDATE cookies SET value='%s',expires=%d WHERE domain='%s' AND path='%s' AND name='%s' AND secure='%c'", value, expires, domain, path_esc, name, secure); DpsSQLAsyncQuery(db, NULL, buf); } if (Indexer->flags & DPS_FLAG_UNOCON) DPS_RELEASELOCK(Indexer, DPS_LOCK_DB); #ifdef WITH_PARANOIA DpsViolationExit(Indexer->handle, paran); #endif return DPS_OK; } } Cookies->Cookie = (DPS_COOKIE*)DpsRealloc(Cookies->Cookie, (Cookies->ncookies + 1) * sizeof(DPS_COOKIE)); if(Cookies->Cookie == NULL) { Cookies->ncookies = 0; if (Indexer->flags & DPS_FLAG_UNOCON) DPS_RELEASELOCK(Indexer, DPS_LOCK_DB); #ifdef WITH_PARANOIA DpsViolationExit(Indexer->handle, paran); #endif return DPS_ERROR; } Coo = &Cookies->Cookie[Cookies->ncookies]; /* Coo->expires = expires;*/ Coo->secure = secure; Coo->from_config = from_config; Coo->domain = DpsStrdup(domain); Coo->path = DpsStrdup(path); Coo->name = DpsStrdup(name); Coo->value = DpsStrdup(value); if (insert_flag) { if (Indexer->Flags.CheckInsertSQL) { dps_snprintf(buf, sizeof(buf), "DELETE FROM cookies WHERE domain='%s' AND path='%s' AND name='%s' AND secure='%c'", domain, path_esc, name, secure); DpsSQLAsyncQuery(db, NULL, buf); } dps_snprintf(buf, sizeof(buf), "INSERT INTO cookies(expires,secure,domain,path,name,value)VALUES(%d,'%c','%s','%s','%s','%s')", expires, secure, domain, path_esc, name, value); DpsSQLAsyncQuery(db, NULL, buf); } Cookies->ncookies++; if (Indexer->flags & DPS_FLAG_UNOCON) DPS_RELEASELOCK(Indexer, DPS_LOCK_DB); #ifdef WITH_PARANOIA DpsViolationExit(Indexer->handle, paran); #endif #endif /*HAVE_SQL*/ return DPS_OK; }
void DpsCookiesFind(DPS_AGENT *Indexer, DPS_SERVER *Server, DPS_DOCUMENT *Doc, const char *hostinfo) { #ifdef HAVE_SQL DPS_DSTR cookie; DPS_COOKIES *Cookies = &Indexer->Cookies; DPS_COOKIE *Coo; size_t i, blen = dps_strlen(hostinfo), slen; int have_no_cookies = DpsVarListFindInt(&Doc->Sections, "have_no_cookies", 1); #ifdef WITH_PARANOIA void *paran = DpsViolationEnter(paran); #endif TRACE_IN(Indexer, "DpsCookiesFind"); DpsDSTRInit(&cookie, 1024); for(i = 0; i < Cookies->ncookies; i++) { Coo = &Cookies->Cookie[i]; slen = dps_strlen(Coo->domain); if (slen > blen) continue; if (Coo->secure == 'y' && strcasecmp(Doc->CurURL.schema, "https")) continue; if (strncasecmp(Coo->path, Doc->CurURL.path, dps_strlen(Coo->path))) continue; if (strcasecmp(Coo->domain, hostinfo + (blen - slen))) continue; if (Coo->from_config != 1) have_no_cookies = 0; if (Coo->name[0] == '\0' && Coo->value[0] == '\0') continue; if (cookie.data_size) DpsDSTRAppend(&cookie, "; ", 2); DpsDSTRAppendStr(&cookie, Coo->name); DpsDSTRAppend(&cookie, "=", 1); DpsDSTRAppendStr(&cookie, Coo->value); } if (have_no_cookies) { char buf[2*PATH_MAX]; dpshash32_t url_id; DPS_DB *db; DPS_SQLRES Res; size_t rows; int rc; if (Server != NULL) { char *PingData = DpsVarListFindStr(&Server->Vars, "AuthPing", NULL); if (PingData != NULL) { char *AuthPing = DpsStrdup(DpsTrim(PingData, " \t\r\n")); int method = DPS_METHOD_GET; dps_base64_decode(AuthPing, PingData, dps_strlen(PingData)); if (!strncasecmp(AuthPing, "GET", 3)) { method = DPS_METHOD_GET; PingData = DpsTrim(AuthPing + 3, " \t\r\n"); } else if (!strncasecmp(AuthPing, "POST", 4)) { method = DPS_METHOD_POST; PingData = DpsTrim(AuthPing + 4, " \t\r\n"); } else { DpsLog(Indexer, DPS_LOG_ERROR, "AuthPing should be GET or POST: %s", AuthPing); PingData = NULL; } if (PingData != NULL) { size_t size = dps_strlen(PingData); { char PingURL[size + 2]; char PingBody[size]; DPS_DOCUMENT *rDoc; int result; rDoc = DpsDocInit(NULL); DpsSpiderParamInit(&rDoc->Spider); DpsVarList2Doc(rDoc, Server); rDoc->Buf.max_size = (size_t)DpsVarListFindInt(&Indexer->Vars, "MaxDocSize", DPS_MAXDOCSIZE); rDoc->Buf.allocated_size = DPS_NET_BUF_SIZE; if ((rDoc->Buf.buf = (char*)DpsMalloc(rDoc->Buf.allocated_size + 1)) == NULL) { DpsDocFree(rDoc); TRACE_OUT(Indexer); return; } rDoc->Buf.buf[0]='\0'; rDoc->subdoc = Indexer->Flags.SubDocLevel + 1; #if 1 dps_snprintf(buf, sizeof(buf), "%s://%s/", DPS_NULL2EMPTY(Doc->CurURL.schema), DPS_NULL2EMPTY(Doc->CurURL.hostinfo)); DpsVarListReplaceStr(&rDoc->Sections, "URL", buf); DpsURLParse(&rDoc->CurURL, buf); DpsLog(Indexer, DPS_LOG_INFO, "HOME: %s", buf); rDoc->method = DPS_METHOD_HEAD; /* DpsVarListFree(&rDoc->RequestHeaders);*/ if (Doc != NULL) { DpsVarListReplaceLst(&rDoc->RequestHeaders, &Doc->RequestHeaders, NULL, "*"); } DpsVarListReplaceStr(&rDoc->Sections, "have_no_cookies", "0"); DpsDocAddDocExtraHeaders(Indexer, Server, rDoc); DpsDocAddConfExtraHeaders(Indexer->Conf, rDoc); DpsVarListReplaceLst(&rDoc->Sections, &Server->Vars, NULL, "*"); DpsDocAddServExtraHeaders(Server, rDoc); DpsVarListLog(Indexer, &rDoc->RequestHeaders, DPS_LOG_DEBUG, "HOME.Request"); if (Doc == NULL || Indexer->Flags.cmd == DPS_IND_FILTER) { DpsDocLookupConn(Indexer, rDoc); } else { DPS_FREE(rDoc->connp.connp); rDoc->connp = Doc->connp; } result = DpsGetURL(Indexer, rDoc, NULL); /* Just get headers from the home as we need only Cookies from it */ DpsDocProcessResponseHeaders(Indexer, rDoc); DpsVarListLog(Indexer, &rDoc->Sections, DPS_LOG_DEBUG, "HOME.Response"); #endif sscanf(PingData, "%s %s", PingURL, PingBody); if (rDoc->method == DPS_METHOD_GET) { dps_strcat(PingURL, "?"); dps_strcat(PingURL, PingBody); } else { DpsVarListReplaceStr(&rDoc->Sections, "body", PingBody); } DpsVarListReplaceStr(&rDoc->Sections, "URL", PingURL); DpsURLParse(&rDoc->CurURL, PingURL); DpsLog(Indexer, DPS_LOG_INFO, "AUTH.PING: %s", PingURL); rDoc->method = method; DpsVarListFree(&rDoc->RequestHeaders); DpsVarListReplaceStr(&rDoc->Sections, "have_no_cookies", "0"); DpsDocAddDocExtraHeaders(Indexer, Server, rDoc); DpsDocAddConfExtraHeaders(Indexer->Conf, rDoc); DpsVarListReplaceLst(&rDoc->Sections, &Server->Vars, NULL, "*"); DpsDocAddServExtraHeaders(Server, rDoc); if (method == DPS_METHOD_POST) { dps_snprintf(buf, sizeof(buf), "application/x-www-form-urlencoded; charset=%s", DpsVarListFindStr(&Indexer->Conf->Vars, "LocalCharset", "iso-8859-1")); DpsVarListReplaceStr(&rDoc->RequestHeaders, "Content-Type", buf); dps_snprintf(buf, sizeof(buf), "%d", dps_strlen(PingBody)); DpsVarListReplaceStr(&rDoc->RequestHeaders, "Content-Length", buf); } DpsVarListLog(Indexer, &rDoc->RequestHeaders, DPS_LOG_DEBUG, "AUTHPING.Request"); #if 0 if (Doc == NULL || Indexer->Flags.cmd == DPS_IND_FILTER) { DpsDocLookupConn(Indexer, rDoc); } else { DPS_FREE(rDoc->connp.connp); rDoc->connp = Doc->connp; } #endif result = DpsGetURL(Indexer, rDoc, NULL); /* Just get it as we need only Cookies from the headers */ DpsDocProcessResponseHeaders(Indexer, rDoc); DpsVarListDel(&rDoc->Sections, "body"); DpsVarListLog(Indexer, &rDoc->Sections, DPS_LOG_DEBUG, "AUTHPING.Response"); if (Doc != NULL) bzero(&rDoc->connp, sizeof(rDoc->connp)); DpsDocFree(rDoc); } } DpsFree(AuthPing); } } while(hostinfo != NULL) { url_id = DpsStrHash32(hostinfo); DpsSQLResInit(&Res); dps_snprintf(buf, sizeof(buf), "SELECT name,value,path,secure FROM cookies WHERE domain='%s'", hostinfo); if (Indexer->flags & DPS_FLAG_UNOCON) { DPS_GETLOCK(Indexer, DPS_LOCK_DB); db = Indexer->Conf->dbl.db[url_id % Indexer->Conf->dbl.nitems]; } else { db = Indexer->dbl.db[url_id % Indexer->dbl.nitems]; } if(DPS_OK == (rc = DpsSQLQuery(db, &Res, buf))) { rows = DpsSQLNumRows(&Res); for(i = 0; i < rows; i++) { DpsCookiesAdd(Indexer, hostinfo, DpsSQLValue(&Res, i, 2), DpsSQLValue(&Res, i, 0), DpsSQLValue(&Res, i, 1), *DpsSQLValue(&Res, i, 3), 0, 0, 0); if (*DpsSQLValue(&Res, i, 3) == 'y' && strcasecmp(Doc->CurURL.schema, "https")) continue; if (strncasecmp(DpsSQLValue(&Res, i, 2), Doc->CurURL.path, dps_strlen(DpsSQLValue(&Res, i, 2)))) continue; if (cookie.data_size) DpsDSTRAppend(&cookie, "; ", 2); DpsDSTRAppendStr(&cookie, DpsSQLValue(&Res, i, 0)); DpsDSTRAppend(&cookie, "=", 1); DpsDSTRAppendStr(&cookie, DpsSQLValue(&Res, i, 1)); } if (rows == 0) { DpsCookiesAdd(Indexer, hostinfo, "/", "", "", 'n', 0, 0, 0); } } DpsSQLFree(&Res); if (Indexer->flags & DPS_FLAG_UNOCON) { DPS_RELEASELOCK(Indexer, DPS_LOCK_DB); } hostinfo = strchr(hostinfo, '.'); if (hostinfo != NULL) hostinfo++; } } if (cookie.data_size) { DpsVarListReplaceStr(&Doc->RequestHeaders, "Cookie", cookie.data); } DpsDSTRFree(&cookie); #endif TRACE_OUT(Indexer); return; }