int DpsCookiesAddStr(DPS_AGENT *Indexer, DPS_URL *CurURL, const char *cookie_str, int insert_flag) { char *part, *lpart; char *name = NULL; char *value = NULL; char *domain = NULL, *orig_domain = NULL; char *path = NULL; dps_uint4 expire = 0, need_free_domain = 1, need_free_path = 1; char secure = 'n', savec; for (part = dps_strtok_r(cookie_str, ";" , &lpart, &savec) ; part; part = dps_strtok_r(NULL, ";", &lpart, &savec)) { char *arg; part = DpsTrim(part, " "); if ((arg = strchr(part, '='))) { *arg++ = '\0'; if (!name) { name = part; DpsFree(value); value = DpsStrdup(arg); } else if (!strcasecmp(part, "path")) { DpsFree(path); path = DpsStrdup(arg); } else if (!strcasecmp(part, "domain")) { DpsFree(orig_domain); orig_domain = domain = DpsStrdup(arg); } else if (!strcasecmp(part, "secure")) { secure = 'y'; } else if (!strcasecmp(part, "expires")) { expire = (dps_uint4)DpsHttpDate2Time_t(arg); } } } if (name && value) { if (domain && domain[0] == '.') { domain++; } else { if (domain) DpsFree(orig_domain); domain = (CurURL && CurURL->hostname) ? CurURL->hostname : "localhost"; need_free_domain = 0; } if (!path) { path = (CurURL && CurURL->path) ? CurURL->path : "/"; need_free_path = 0; } DpsCookiesAdd(Indexer, domain, path, name, value, secure, expire, (CurURL==NULL)?1:0, 1); } DpsFree(value); if (need_free_path) DpsFree(path); if (need_free_domain) DpsFree(orig_domain); return DPS_OK; }
__C_LINK void __DPSCALL DpsTextListAdd(DPS_TEXTLIST * tlist,const DPS_TEXTITEM *item){ #ifdef WITH_PARANOIA void * paran = DpsViolationEnter(paran); #endif if(!item->str) { #ifdef WITH_PARANOIA DpsViolationExit(-1, paran); #endif return; } #ifdef DEBUG_MEM if (tlist->mitems) { mprotect(tlist->Items, tlist->nitems * sizeof(DPS_TEXTITEM), PROT_READ | PROT_WRITE); /* fprintf(stderr, "addr: %x, len: %d till %x -- PROT_BOTH\n", tlist->Items, tlist->nitems * sizeof(DPS_TEXTITEM), tlist->Items + tlist->nitems );*/ } #endif if (tlist->nitems + 1 > tlist->mitems) { tlist->mitems += DPS_TEXTLIST_PAS; tlist->Items = (DPS_TEXTITEM*)DpsRealloc(tlist->Items, (tlist->mitems) * sizeof(DPS_TEXTITEM) + 4096); if (tlist->Items == NULL) { tlist->nitems = tlist->mitems = 0; return; } } tlist->Items[tlist->nitems].str = (char*)DpsStrdup(item->str); tlist->Items[tlist->nitems].href = (item->href != NULL) ? (char*)DpsStrdup(item->href) : NULL; tlist->Items[tlist->nitems].section_name = (item->section_name != NULL) ? (char*)DpsStrdup(item->section_name) : NULL; tlist->Items[tlist->nitems].section = item->section; tlist->Items[tlist->nitems].strict = item->strict; tlist->Items[tlist->nitems].len = item->len; tlist->Items[tlist->nitems].marked = 0; tlist->nitems++; #ifdef DEBUG_MEM if (tlist->mitems) { mprotect(tlist->Items, tlist->nitems * sizeof(DPS_TEXTITEM), PROT_READ); /* fprintf(stderr, "addr: %x, len: %d till %x -- PROT_READ\n", tlist->Items, tlist->nitems * sizeof(DPS_TEXTITEM), tlist->Items + tlist->nitems );*/ } #endif #ifdef WITH_PARANOIA DpsViolationExit(-1, paran); #endif return; }
int Dps_ftp_login(DPS_CONN *connp, char *user, char *passwd){ char *buf; char user_tmp[32], passwd_tmp[64]; int code; size_t len; DPS_FREE(connp->user); DPS_FREE(connp->pass); if (!user) dps_snprintf(user_tmp, 32, "anonymous"); else { dps_snprintf(user_tmp, 32, "%s", user); connp->user = (char*)DpsStrdup(user); } if (!passwd) dps_snprintf(passwd_tmp, 64, "*****@*****.**", PACKAGE, VERSION); else { dps_snprintf(passwd_tmp, 32, "%s", passwd); connp->pass = (char*)DpsStrdup(passwd); } len = dps_strlen(user_tmp) + 5 /*dps_strlen("USER ")*/; buf = (char *) DpsXmalloc(len+1); if (buf == NULL) return -1; dps_snprintf(buf, len+1, "USER %s", user_tmp); code = Dps_ftp_send_cmd(connp, buf); DPS_FREE(buf); if (code == -1) return -1; else if (code == 2) /* Don't need password */ return 0; len = dps_strlen(passwd_tmp) + 5 /*dps_strlen("PASS ")*/; buf = (char *) DpsXmalloc(len+1); if (buf == NULL) return -1; dps_snprintf(buf, len+1, "PASS %s", passwd_tmp); code = Dps_ftp_send_cmd( connp, buf); DPS_FREE(buf); if (code > 3) return -1; return 0; }
char * DpsEnvErrMsg(DPS_ENV * Conf) { size_t i; DPS_DB *db; for(i = 0; i<Conf->dbl.nitems; i++){ db = &Conf->dbl.db[i]; if (db->errcode) { char *oe = (char*)DpsStrdup(Conf->errstr); dps_snprintf(Conf->errstr, 2048, "DB err: %s - %s", db->errstr, oe); DPS_FREE(oe); } } return(Conf->errstr); }
DPS_ENV *DpsEnvInit(DPS_ENV *Conf){ #ifdef MECAB const char *mecab_argv[8] = {"mecab", "-F", "%m ", "-B", " ", "-E", " ", NULL}; #endif if(!Conf){ Conf=(DPS_ENV *)DpsMalloc(sizeof(DPS_ENV)); if (Conf == NULL) return NULL; bzero((void*)Conf, sizeof(*Conf)); Conf->freeme=1; }else{ bzero((void*)Conf, sizeof(*Conf)); } Conf->Flags.OptimizeAtUpdate = 1; Conf->Flags.do_excerpt = 1; Conf->Flags.PopRankNeoIterations = 3; Conf->Flags.GuesserBytes = 512; Conf->Flags.robots_period = 604800; /* one week */ Conf->Flags.URLInfoSQL = 1; Conf->Flags.SRVInfoSQL = 1; Conf->Flags.CheckInsertSQL = 1; Conf->Flags.mark_for_index = 1; Conf->Flags.MaxSiteLevel = 2; Conf->Flags.SEASentences = 32; Conf->Flags.SEASentenceMinLength = 64; Conf->Flags.PagesInGroup = 1; Conf->Flags.SubDocCnt = 5; Conf->Flags.MaxCrawlDelay = 300; Conf->Flags.rel_nofollow = 1; Conf->Flags.bind_addr.sin_family = AF_INET; Conf->Flags.use_meta = 1; Conf->WordParam.min_word_len = 1; Conf->WordParam.max_word_len = 32; Conf->WordParam.correct_factor = 1; Conf->WordParam.incorrect_factor = 1; Conf->url_number = 0x7FFFFFFF; Conf->lcs=DpsGetCharSet("latin1"); Conf->bcs=DpsGetCharSet("latin1"); Conf->CharsToEscape = DpsStrdup("\"&<>"); #ifdef MECAB /* Conf->mecab = mecab_new2 ("mecab -F \"%m \" -B \" \" -E \" \"");*/ Conf->mecab = mecab_new(7, (char**)mecab_argv); #endif return(Conf); }
int _DpsOpen3(const char *path, int flags, int mode, char *filename, size_t fileline) { register Slot * slot; register size_t count; Slot * fullSlot = NULL; int address; if ( allocationList == NULL ) initialize(); lock(); if ( unUsedSlots < 2 ) { allocateMoreSlots(); } for ( slot = allocationList, count = slotCount ; count > 0; count-- ) { if ( slot->mode == FREE || slot->mode == NOT_IN_USE) { fullSlot = slot; break; } slot++; } if ( !fullSlot ) FE_InternalError("No empty file slot."); fullSlot->fileline = fileline; dps_strncpy(fullSlot->filename, filename, DPS_FILENAMELEN); fullSlot->path = DpsStrdup(path); fullSlot->fd = address = open(path, flags, mode); if (address > 0) { fullSlot->mode = ALLOCATED; unUsedSlots--; } release(); return address; }
void DpsParseHTTPResponse(DPS_AGENT *Indexer, DPS_DOCUMENT *Doc) { char *token, *lt, *headers; int oldstatus; DPS_DSTR header; Doc->Buf.content=NULL; oldstatus = DpsVarListFindInt(&Doc->Sections, "Status", 0); DpsVarListReplaceInt(&Doc->Sections, "ResponseSize", (int)Doc->Buf.size); DpsVarListDel(&Doc->Sections, "Content-Length"); DpsVarListDel(&Doc->Sections, "Last-Modified"); if (Doc->Buf.buf == NULL) return; /* Cut HTTP response header first */ for(token=Doc->Buf.buf;*token;token++){ if(!strncmp(token,"\r\n\r\n",4)){ *token='\0'; Doc->Buf.content = token + 4; break; }else if(!strncmp(token,"\n\n",2)){ *token='\0'; Doc->Buf.content = token + 2; break; } } /* Bad response, return */ if(!Doc->Buf.content) { if (token < Doc->Buf.buf + Doc->Buf.size - 4) { if (token[2] == '\r') Doc->Buf.content = token + 4; else Doc->Buf.content = token + 2; } else { return; } } /* Copy headers not to break them */ headers = (char*)DpsStrdup(Doc->Buf.buf); /* Now lets parse response header lines */ token = dps_strtok_r(headers,"\r\n",<); if(!token)return; if(!strncmp(token,"HTTP/",5)){ int status = atoi(token + 8); DpsVarListReplaceStr(&Doc->Sections,"ResponseLine",token); DpsVarListReplaceInt(&Doc->Sections, "Status", (oldstatus > status) ? oldstatus : status ); }else{ DpsFree(headers); return; } token = dps_strtok_r(NULL,"\r\n",<); DpsDSTRInit(&header, 128); while(token){ if(strchr(token,':')) { if (header.data_size) { DpsParseHTTPHeader(Indexer, Doc, &header); DpsDSTRFree(&header); DpsDSTRInit(&header, 128); } } DpsDSTRAppendStr(&header, token); token = dps_strtok_r(NULL,"\r\n",<); } if (header.data_size) { DpsParseHTTPHeader(Indexer, Doc, &header); } DpsDSTRFree(&header); DPS_FREE(headers); DpsVarListInsInt(&Doc->Sections,"Content-Length",Doc->Buf.buf-Doc->Buf.content+(int)Doc->Buf.size); }
int main(int argc,char **argv, char **envp) { int ch, sleeps = 1, optimize = 0, obi = 0; unsigned int from = 0, to = 0xFFF, p_to = 0; DPS_ENV * Env; const char * config_name = DPS_CONF_DIR "/cached.conf"; DpsInit(argc, argv, envp); /* Initialize library */ DpsInitMutexes(); Env=DpsEnvInit(NULL); if (Env == NULL) exit(1); DpsSetLockProc(Env, DpsLockProc); /*#ifndef HAVE_SETPROCTITLE*/ ARGV = argv; ARGC = argc; /*#endif*/ while ((ch = getopt(argc, argv, "blt:f:op:w:v:h?")) != -1){ switch (ch) { case 'f': sscanf(optarg, "%x", &from); break; case 't': sscanf(optarg, "%x", &p_to); break; case 'w': DpsVarListReplaceStr(&Env->Vars, "VarDir", optarg); break; case 'v': DpsSetLogLevel(NULL, atoi(optarg)); break; case 'b': obi++; break; case 'o': optimize++; break; case 'p': sleeps = atoi(optarg); break; case 'h': case '?': default: usage(); DpsEnvFree(Env); DpsDeInit(); DpsDestroyMutexes(); return 1; break; } } argc -= optind; argv += optind; if(argc > 1) { usage(); DpsEnvFree(Env); DpsDeInit(); DpsDestroyMutexes(); return 1; } else if (argc == 1) { config_name = argv[0]; } { DPS_LOGDEL *del_buf=NULL; size_t del_count = 0, log, bytes, n = 0; int dd, log_fd; struct stat sb; char dname[PATH_MAX] = ""; DPS_BASE_PARAM P; DPS_LOGWORD *log_buf = NULL; DPS_AGENT *Indexer = DpsAgentInit(NULL, Env, 0); log2stderr = 1; if (Indexer == NULL) { fprintf(stderr, "Can't alloc Agent at %s:%d\n", __FILE__, __LINE__); exit(DPS_ERROR); } if(DPS_OK != DpsEnvLoad(Indexer, config_name, (dps_uint8)0)){ fprintf(stderr, "%s\n", DpsEnvErrMsg(Env)); DpsEnvFree(Env); DpsDeInit(); DpsDestroyMutexes(); return DPS_ERROR; } DpsOpenLog("splitter", Env, log2stderr); Indexer->flags = Env->flags = DPS_FLAG_UNOCON; DpsVarListAddLst(&Indexer->Vars, &Env->Vars, NULL, "*"); bzero(&P, sizeof(P)); P.subdir = DPS_TREEDIR; P.basename = "wrd"; P.indname = "wrd"; P.mode = DPS_WRITE_LOCK; P.NFiles = DpsVarListFindInt(&Indexer->Conf->Vars, "WrdFiles", 0x300); P.vardir = DpsStrdup(DpsVarListFindStr(&Indexer->Conf->Vars, "VarDir", DPS_VAR_DIR)); P.A = Indexer; if (p_to != 0) to = p_to; else to = P.NFiles - 1; #ifdef HAVE_ZLIB P.zlib_method = Z_DEFLATED; P.zlib_level = 9; P.zlib_windowBits = DPS_BASE_WRD_WINDOWBITS; P.zlib_memLevel = 9; P.zlib_strategy = DPS_BASE_WRD_STRATEGY; #endif /* Open del log file */ dps_snprintf(dname,sizeof(dname),"%s%c%s%cdel-split.log", P.vardir, DPSSLASH, DPS_SPLDIR, DPSSLASH); if((dd = DpsOpen2(dname, O_RDONLY | DPS_BINARY)) < 0) { dps_strerror(NULL, 0, "Can't open del log '%s'", dname); exit(DPS_ERROR); } DpsLog(Indexer, DPS_LOG_DEBUG, "VarDir: %s, WrdFiles: %d [%x]", P.vardir, P.NFiles, P.NFiles); /* Allocate del buffer */ fstat(dd, &sb); if (sb.st_size != 0) { del_buf=(DPS_LOGDEL*)DpsMalloc((size_t)sb.st_size + 1); if (del_buf == NULL) { fprintf(stderr, "Can't alloc %d bytes at %s:%d\n", (int)sb.st_size, __FILE__, __LINE__); exit(0); } del_count=read(dd,del_buf,(size_t)sb.st_size)/sizeof(DPS_LOGDEL); } DpsClose(dd); /* Remove duplicates URLs in DEL log */ /* Keep only oldest records for each URL */ if (del_count > 0) { DpsLog(Indexer, DPS_LOG_DEBUG, "Sorting del_buf: %d items", del_count); if (del_count > 1) DpsSort(del_buf, (size_t)del_count, sizeof(DPS_LOGDEL), DpsCmpurldellog); DpsLog(Indexer, DPS_LOG_DEBUG, "Removing DelLogDups"); del_count = DpsRemoveDelLogDups(del_buf, del_count); } DpsLog(Indexer, DPS_LOG_DEBUG, "Processing Bufs from %d [%x] to %d [%x]", from, from, to, to); for(log = from; log <= to; log++) { /* Open log file */ dps_snprintf(dname, sizeof(dname), "%s%c%s%c%03X-split.log", P.vardir, DPSSLASH, DPS_SPLDIR, DPSSLASH, log); if((log_fd = DpsOpen2(dname, O_RDWR|DPS_BINARY)) < 0){ if (errno == ENOENT) { dps_strerror(Indexer, DPS_LOG_DEBUG, "Can't open '%s'", dname); n = 0; /* continue;*/ } else { dps_strerror(Indexer, DPS_LOG_ERROR, "Can't open '%s'", dname); continue; } } else { DpsWriteLock(log_fd); DpsLog(Indexer, DPS_LOG_DEBUG, "Processing Log: %x", log); fstat(log_fd, &sb); log_buf = (sb.st_size > 0) ? (DPS_LOGWORD*)DpsMalloc((size_t)sb.st_size + 1) : NULL; if (log_buf != NULL) { unlink(dname); bytes = read(log_fd,log_buf,(size_t)sb.st_size); (void)ftruncate(log_fd, (off_t)0); DpsUnLock(log_fd); DpsClose(log_fd); n = bytes / sizeof(DPS_LOGWORD); DpsLog(Indexer, DPS_LOG_DEBUG, "Sorting log_buf: %d items", n); if (n > 1) DpsSort(log_buf, n, sizeof(DPS_LOGWORD), (qsort_cmp)DpsCmplog); DpsLog(Indexer, DPS_LOG_DEBUG, "Removing OldWords"); n = DpsRemoveOldWords(log_buf, n, del_buf, del_count); if (n > 1) DpsSort(log_buf, n, sizeof(DPS_LOGWORD), (qsort_cmp)DpsCmplog_wrd); } else { n = 0; DpsUnLock(log_fd); DpsClose(log_fd); } } DpsLog(Indexer, DPS_LOG_DEBUG, "Processing Buf, optimize: %d", optimize); if (obi) DpsBaseOptimize(&P, log); DpsProcessBuf(Indexer, &P, log, log_buf, n, del_buf, del_count); if (optimize) DpsBaseOptimize(&P, log); DpsBaseClose(&P); DPS_FREE(log_buf); DpsLog(Indexer, DPS_LOG_DEBUG, "pas done: %d from %d to %d", log, from, to); DPSSLEEP(sleeps); } DPS_FREE(del_buf); DpsAgentFree(Indexer); DPS_FREE(P.vardir); } fprintf(stderr, "Splitting done.\n"); DpsEnvFree(Env); DpsDeInit(); DpsDestroyMutexes(); #ifdef EFENCE fprintf(stderr, "Memory leaks checking\n"); DpsEfenceCheckLeaks(); #endif #ifdef FILENCE fprintf(stderr, "FD leaks checking\n"); DpsFilenceCheckLeaks(NULL); #endif return 0; }
int _DpsURLParse(DPS_URL *url, const char *str, const char *filename, int line) { #else int DpsURLParse(DPS_URL *url, const char *str) { #endif char *schema,*anchor,*file,*query; char *s; /* size_t len = dps_strlen(str);*/ #ifdef WITH_PARANOIA void * paran = DpsViolationEnter(paran); #endif #ifdef DEBUG_URL fprintf(stderr, " -- %s:%d Parser url: %s\n", filename, line, str); #endif DPS_FREE(url->schema); DPS_FREE(url->specific); DPS_FREE(url->hostinfo); DPS_FREE(url->hostname); DPS_FREE(url->anchor); DPS_FREE(url->auth); url->port=0; url->default_port=0; DPS_FREE(url->path); DPS_FREE(url->directory); DPS_FREE(url->filename); DPS_FREE(url->query_string); /* if(len >= DPS_URLSIZE)return(DPS_URL_LONG); FIXME: Chage this cheking for configured parameter, not DPS_URLSIZE */ s = (char*)DpsStrdup(str); if (s == NULL) { #ifdef WITH_PARANOIA DpsViolationExit(-1, paran); #endif return DPS_ERROR; } url->len = dps_strlen(str); /* Find possible schema end than */ /* Check that it is really schema */ /* It must consist of alphas only */ /* We will take in account digits */ /* also for oracle8:// for example */ /* We must check it because */ /* It might be anchor also */ /* For example: */ /* "mod/index.html#a:1" */ if((schema=strchr(s,':'))){ const char * ch; for(ch=s;ch<schema;ch++){ if(!isalnum(*ch)){ /* Bad character */ /* so it is not schema */ schema=0;break; } } } if(schema){ /* Have scheme - absolute path */ *schema=0; url->schema = (char*)DpsStrdup(s); url->specific = (char*)DpsStrdup(schema + 1); *schema=':'; if(!strcasecmp(url->schema,"http"))url->default_port=80; else if(!strcasecmp(url->schema,"https"))url->default_port=443; else if(!strcasecmp(url->schema,"nntp"))url->default_port=119; else if(!strcasecmp(url->schema,"news"))url->default_port=119; else if(!strcasecmp(url->schema,"ftp"))url->default_port=21; if(!strncmp(url->specific,"//",2)){ char *ss,*hostname; /* Have hostinfo */ if((ss=strchr(url->specific+2,'/'))){ /* Have hostname with path */ *ss=0; url->hostinfo = (char*)DpsStrdup(url->specific + 2); *ss='/'; url->path = (char*)DpsStrdup(ss); }else{ /* Hostname without path */ if ((ss = strchr(url->specific + 2, '?'))) { /* Have hostname with parameters */ *ss = 0; url->hostinfo = (char*)DpsStrdup(url->specific + 2); *ss='?'; url->path = (char*)DpsStrdup("/"); }else { url->hostinfo = (char*)DpsStrdup(url->specific + 2); url->path = (char*)DpsStrdup("/"); } } if((hostname=strchr(url->hostinfo,'@'))){ /* Username and password is given */ /* Store auth string user:password */ *hostname=0; url->auth = (char*)DpsStrdup(url->hostinfo); *hostname='@'; hostname++; }else{ hostname = url->hostinfo; } /* FIXME: for(h=hostname;*h;h++){ if( *h>='A' && *h<='Z') *h=(*h)-'A'+'a'; } */ if((ss=strchr(hostname,':'))){ *ss=0; url->hostname = (char*)DpsStrdup(hostname); *ss=':'; url->port=atoi(ss+1); }else{ url->hostname = (char*)DpsStrdup(hostname); url->port=0; } }else{ /* Have not host but have schema */ /* This is possible for: */ /* file: mailto: htdb: news: */ /* As far as we do not need mailto: just ignore it */ if(!strcasecmp(url->schema,"mailto") || !strcasecmp(url->schema,"javascript") || !strcasecmp(url->schema,"feed") ) { DPS_FREE(s); #ifdef WITH_PARANOIA DpsViolationExit(-1, paran); #endif return(DPS_URL_BAD); } else if(!strcasecmp(url->schema,"file")) url->path = (char*)DpsStrdup(url->specific); else if(!strcasecmp(url->schema,"exec")) url->path = (char*)DpsStrdup(url->specific); else if(!strcasecmp(url->schema,"cgi")) url->path = (char*)DpsStrdup(url->specific); else if(!strcasecmp(url->schema,"htdb")) url->path = (char*)DpsStrdup(url->specific); else if(!strcasecmp(url->schema,"news")){ /* Now we will use localhost as NNTP */ /* server as it is not indicated in URL */ url->hostname = (char*)DpsStrdup("localhost"); url->path = (char*)DpsMalloc(dps_strlen(url->specific) + 2); if (url->path == NULL) { DPS_FREE(s); #ifdef WITH_PARANOIA DpsViolationExit(-1, paran); #endif return DPS_ERROR; } sprintf(url->path,"/%s",url->specific); url->default_port=119; }else{ /* Unknown strange schema */ DPS_FREE(s); #ifdef WITH_PARANOIA DpsViolationExit(-1, paran); #endif return(DPS_URL_BAD); } } }else{ url->path = (char*)DpsStrdup(s); } /* Cat an anchor if exist */ if((anchor=strstr(url->path,"#")))*anchor=0; /* If path is not full just copy it to filename */ /* i.e. neither /usr/local/ nor c:/windows/temp/ */ if((url->path != NULL) && (url->path[0]!='/') && (url->path[0]!='?') && (url->path[1]!=':')) { /* Relative path */ if(!strncmp(url->path,"./",2)) url->filename = (char*)DpsStrdup(url->path + 2); else url->filename = (char*)DpsStrdup(url->path); url->path[0] = 0; } /* truncate path to query_string */ /* and store query_string */ if((query=strrchr(url->path,'?'))){ url->query_string = (char*)DpsStrdup(query); *(query) = 0; } DpsURLNormalizePath(url->path); /* Now find right '/' sign and copy the rest to filename */ if((file=strrchr(url->path,'/'))&&(strcmp(file,"/"))){ url->filename = (char*)DpsStrdup(file + 1); *(file+1)=0; } /* Now find right '/' sign and copy the rest to directory */ if ((file = strrchr(url->path,'/'))) { char *p_save = file; for(file--; (file > url->path) && (*file != '/'); file--); file++; if (*file) { *p_save = '\0'; url->directory = (char*)DpsStrdup(file); *p_save = '/'; } } DPS_FREE(s); if (url->hostname != NULL) { DpsRTrim(url->hostname, "."); url->domain_level = 1; for (s = url->hostname; *s; s++) { *s = dps_tolower(*s); if (*s == '.') url->domain_level++; if (strchr(",'\";", (int)*s)) { #ifdef WITH_PARANOIA DpsViolationExit(-1, paran); #endif return DPS_URL_BAD; } } } if (url->hostinfo != NULL) { DpsRTrim(url->hostinfo, "."); s = strchr(url->hostinfo, '@'); for (s = (s == NULL) ? url->hostinfo : s + 1; *s; s++) *s = dps_tolower(*s); } if (url->schema != NULL) for (s = url->schema; *s; s++) *s = dps_tolower(*s); /* fprintf(stderr, "url: .path: %s port:%d\n", url->path, url->port);*/ #ifdef WITH_PARANOIA DpsViolationExit(-1, paran); #endif return DPS_OK; }
void RelLink(DPS_AGENT *Indexer, DPS_URL *curURL, DPS_URL *newURL, char **str, int ReverseAliasFlag) { const char *schema = newURL->schema ? newURL->schema : curURL->schema; const char *hostname = newURL->hostname ? newURL->hostname : curURL->hostname; const char *auth = newURL->auth ? newURL->auth : curURL->auth; const char *path = (newURL->path && newURL->path[0]) ? newURL->path : curURL->path; const char *fname = ((newURL->filename && newURL->filename[0]) || (newURL->path && newURL->path[0])) ? newURL->filename : curURL->filename; const char *query_string = newURL->query_string; char *pathfile = (char*)DpsMalloc(dps_strlen(DPS_NULL2EMPTY(path)) + dps_strlen(DPS_NULL2EMPTY(fname)) + dps_strlen(DPS_NULL2EMPTY(query_string)) + 5); int cascade; DPS_MATCH *Alias; char *alias = NULL; size_t aliassize, nparts = 10; DPS_MATCH_PART Parts[10]; if (newURL->hostinfo == NULL) newURL->charset_id = curURL->charset_id; if (pathfile == NULL) return; /* sprintf(pathfile, "/%s%s%s", DPS_NULL2EMPTY(path), DPS_NULL2EMPTY(fname), DPS_NULL2EMPTY(query_string));*/ pathfile[0] = '/'; dps_strcpy(pathfile + 1, DPS_NULL2EMPTY(path)); dps_strcat(pathfile, DPS_NULL2EMPTY(fname)); dps_strcat(pathfile, DPS_NULL2EMPTY(query_string)); DpsURLNormalizePath(pathfile); if (!strcasecmp(DPS_NULL2EMPTY(schema), "mailto") || !strcasecmp(DPS_NULL2EMPTY(schema), "javascript") || !strcasecmp(DPS_NULL2EMPTY(schema), "feed") ) { *str = (char*)DpsMalloc(dps_strlen(DPS_NULL2EMPTY(schema)) + dps_strlen(DPS_NULL2EMPTY(newURL->specific)) + 4); if (*str == NULL) return; /* sprintf(*str, "%s:%s", DPS_NULL2EMPTY(schema), DPS_NULL2EMPTY(newURL->specific));*/ dps_strcpy(*str, DPS_NULL2EMPTY(schema)); dps_strcat(*str, ":"); dps_strcat(*str, DPS_NULL2EMPTY(newURL->specific)); } else if(/*!strcasecmp(DPS_NULL2EMPTY(schema), "file") ||*/ !strcasecmp(DPS_NULL2EMPTY(schema), "htdb")) { *str = (char*)DpsMalloc(dps_strlen(DPS_NULL2EMPTY(schema)) + dps_strlen(pathfile) + 4); if (*str == NULL) return; /* sprintf(*str, "%s:%s", DPS_NULL2EMPTY(schema), pathfile);*/ dps_strcpy(*str, DPS_NULL2EMPTY(schema)); dps_strcat(*str, ":"); dps_strcat(*str, pathfile); }else{ *str = (char*)DpsMalloc(dps_strlen(DPS_NULL2EMPTY(schema)) + dps_strlen(pathfile) + dps_strlen(DPS_NULL2EMPTY(hostname)) + dps_strlen(DPS_NULL2EMPTY(auth)) + 8); if (*str == NULL) return; /* sprintf(*str, "%s://%s%s", DPS_NULL2EMPTY(schema), DPS_NULL2EMPTY(hostinfo), pathfile);*/ dps_strcpy(*str, DPS_NULL2EMPTY(schema)); dps_strcat(*str, "://"); if (auth) { dps_strcat(*str, auth); dps_strcat(*str,"@"); } dps_strcat(*str, DPS_NULL2EMPTY(hostname)); dps_strcat(*str, pathfile); } if(!strncmp(*str, "ftp://", 6) && (strstr(*str, ";type="))) *(strstr(*str, ";type")) = '\0'; DPS_FREE(pathfile); if (ReverseAliasFlag) { const char *alias_prog = DpsVarListFindStr(&Indexer->Vars, "ReverseAliasProg", NULL); if (alias_prog) { int result; aliassize = 256 + 2 * dps_strlen(*str); alias = (char*)DpsRealloc(alias, aliassize); if (alias == NULL) { DpsLog(Indexer, DPS_LOG_ERROR, "No memory (%d bytes). %s line %d", aliassize, __FILE__, __LINE__); goto ret; } alias[0] = '\0'; result = DpsAliasProg(Indexer, alias_prog, *str, alias, aliassize - 1); DpsLog(Indexer, DPS_LOG_EXTRA, "ReverseAliasProg result: '%s'", alias); if(result != DPS_OK) goto ret; DPS_FREE(*str); *str = (char*)DpsStrdup(alias); } for(cascade = 0; ((Alias=DpsMatchListFind(&Indexer->Conf->ReverseAliases,*str,nparts,Parts))) && (cascade < 1024); cascade++) { aliassize = dps_strlen(Alias->arg) + dps_strlen(Alias->pattern) + dps_strlen(*str) + 128; alias = (char*)DpsRealloc(alias, aliassize); if (alias == NULL) { DpsLog(Indexer, DPS_LOG_ERROR, "No memory (%d bytes). %s line %d", aliassize, __FILE__, __LINE__); goto ret; } DpsMatchApply(alias,aliassize,*str,Alias->arg,Alias,nparts,Parts); if(alias[0]){ DpsLog(Indexer,DPS_LOG_DEBUG,"ReverseAlias%d: pattern:%s, arg:%s -> '%s'", cascade, Alias->pattern, Alias->arg, alias); DPS_FREE(*str); *str = (char*)DpsStrdup(alias); } else break; if (Alias->last) break; } } ret: DPS_FREE(alias); }
void DpsParseHTTPResponse(DPS_AGENT *Indexer, DPS_DOCUMENT *Doc) { char *token, *lt, *headers, savec; int status, oldstatus; DPS_DSTR header; time_t now, last_mod_time; Doc->Buf.content=NULL; oldstatus = DpsVarListFindInt(&Doc->Sections, "Status", 0); DpsVarListReplaceInt(&Doc->Sections, "ResponseSize", (int)Doc->Buf.size); DpsVarListDel(&Doc->Sections, "Content-Length"); /* DpsVarListDel(&Doc->Sections, "Last-Modified");*/ /* if it's not deleted Lat-Modified equals to the first appearance in db */ if (Doc->Buf.buf == NULL) return; /* Cut HTTP response header first */ for(token=Doc->Buf.buf;*token;token++){ if(!strncmp(token,"\r\n\r\n",4)){ if (token <= Doc->Buf.buf + Doc->Buf.size - 4) { *token='\0'; Doc->Buf.content = token + 4; } break; } else if(!strncmp(token,"\n\n",2)){ if (token <= Doc->Buf.buf + Doc->Buf.size - 2) { *token='\0'; Doc->Buf.content = token + 2; } break; } } /* Bad response, return */ if(!Doc->Buf.content) { if (token <= Doc->Buf.buf + Doc->Buf.size - 4) { if (token[2] == CR_CHAR) Doc->Buf.content = token + 4; else Doc->Buf.content = token + 2; } } /* Copy headers not to break them */ headers = (char*)DpsStrdup(Doc->Buf.buf); /* Now lets parse response header lines */ token = dps_strtok_r(headers, "\r\n", <, &savec); if(!token) { DpsFree(headers); return; } if(!strncmp(token,"HTTP/",5)){ status = atoi(token + 8); DpsVarListReplaceStr(&Doc->Sections,"ResponseLine",token); DpsVarListReplaceInt(&Doc->Sections, "Status", (oldstatus > status) ? oldstatus : status ); }else{ DpsFree(headers); return; } token = dps_strtok_r(NULL, "\r\n", <, &savec); DpsDSTRInit(&header, 128); while(token){ if(strchr(token,':')) { if (header.data_size) { DpsParseHTTPHeader(Indexer, Doc, &header); DpsDSTRFree(&header); DpsDSTRInit(&header, 128); } } DpsDSTRAppendStr(&header, token); token = dps_strtok_r(NULL, "\r\n", <, &savec); } if (header.data_size) { DpsParseHTTPHeader(Indexer, Doc, &header); } DpsDSTRFree(&header); DPS_FREE(headers); { now = Indexer->now; last_mod_time = DpsHttpDate2Time_t(DpsVarListFindStr(&Doc->Sections, "Last-Modified", "")); if (last_mod_time > now + 3600 * 4) { /* we have a document with Last-Modified time in the future */ DpsLog(Indexer, DPS_LOG_EXTRA, "Last-Modified date is deep in future (%d>%d), dropping it.", last_mod_time, now); DpsVarListDel(&Doc->Sections, "Last-Modified"); } } /* Bad response, return */ if(!Doc->Buf.content) { return; } DpsVarListReplaceInt(&Doc->Sections,"Content-Length", Doc->Buf.buf-Doc->Buf.content+(int)Doc->Buf.size + DpsVarListFindInt(&Doc->Sections,"Content-Length", 0)); }
int DpsAddStackItem(DPS_AGENT *query, DPS_RESULT *Res, DPS_PREPARE_STATE *state, char *word, dpsunicode_t *uword) { int origin; size_t i; size_t wlen = (uword == NULL) ? 0 : DpsUniLen(uword); dpshash32_t crcword = (word == NULL) ? 0 : DpsStrHash32(word); #ifdef DEBUG_BOOL DpsLog(query, DPS_LOG_EXTRA, "0[%d].%x %c -- %s [%x] .secno:%d\n", state->order, state->origin, item_type(state->cmd), (word == NULL) ? "<NULL>" : word, crcword, state->secno[state->p_secno]); #endif if((uword != NULL) && ( DpsStopListFind(&query->Conf->StopWords, uword, state->qlang) || (query->WordParam.min_word_len > wlen) || (query->WordParam.max_word_len < wlen)) ) { origin = state->origin | DPS_WORD_ORIGIN_STOP; } else { origin = state->origin; } if (state->cmd == DPS_STACK_WORD && !(origin & DPS_WORD_ORIGIN_QUERY)) { for (i = 0; i < Res->nitems; i++) { if ((Res->items[i].order == state->order) && (Res->items[i].crcword == crcword)) return DPS_OK; } } if (Res->nitems >= Res->mitems - 2) { Res->mitems += DPS_MAXSTACK; Res->items = (DPS_STACK_ITEM*)DpsRealloc(Res->items, Res->mitems * sizeof(DPS_STACK_ITEM)); if (Res->items == NULL) { DpsLog(query, DPS_LOG_ERROR, "Can't alloc %d bytes for %d mitems", Res->mitems * sizeof(DPS_STACK_ITEM), Res->mitems); return DPS_ERROR; } } if (Res->nitems > 0) { if (state->cmd == DPS_STACK_OR || state->cmd == DPS_STACK_AND || state->cmd == DPS_STACK_NEAR || state->cmd == DPS_STACK_ANYWORD) { if (Res->items[Res->nitems-1].cmd == DPS_STACK_AND || Res->items[Res->nitems-1].cmd == DPS_STACK_OR || Res->items[Res->nitems-1].cmd == DPS_STACK_NEAR || Res->items[Res->nitems-1].cmd == DPS_STACK_ANYWORD) { return DPS_OK; } } if ((Res->nitems > 0) && (state->cmd == DPS_STACK_WORD) && ( (Res->items[Res->nitems-1].cmd == DPS_STACK_WORD) || (Res->items[Res->nitems-1].cmd == DPS_STACK_RIGHT) || (Res->items[Res->nitems-1].cmd == DPS_STACK_PHRASE_RIGHT) )) { Res->items[Res->nitems].cmd = DPS_STACK_OR; Res->items[Res->nitems].order = 0; Res->items[Res->nitems].origin = 0; Res->items[Res->nitems].count = 0; Res->items[Res->nitems].len = 0; Res->items[Res->nitems].crcword = 0; Res->items[Res->nitems].word = NULL; Res->items[Res->nitems].ulen = 0; Res->items[Res->nitems].uword = NULL; Res->items[Res->nitems].pbegin = NULL; Res->items[Res->nitems].order_origin = 0; Res->items[Res->nitems].secno = state->secno[state->p_secno]; Res->nitems++; Res->ncmds++; #ifdef DEBUG_BOOL DpsLog(query, DPS_LOG_EXTRA, "1[%d].%x %c -- %s", 0, 0, item_type(DPS_STACK_OR), "<NULL>"); #endif } if ((Res->nitems > 0) && (state->cmd == DPS_STACK_LEFT) && ( (Res->items[Res->nitems-1].cmd == DPS_STACK_RIGHT) || (Res->items[Res->nitems-1].cmd == DPS_STACK_PHRASE_RIGHT) )) { Res->items[Res->nitems].cmd = state->add_cmd; Res->items[Res->nitems].order = 0; Res->items[Res->nitems].origin = 0; Res->items[Res->nitems].count = 0; Res->items[Res->nitems].len = 0; Res->items[Res->nitems].crcword = 0; Res->items[Res->nitems].word = NULL; Res->items[Res->nitems].ulen = 0; Res->items[Res->nitems].uword = NULL; Res->items[Res->nitems].pbegin = NULL; Res->items[Res->nitems].order_origin = 0; Res->items[Res->nitems].secno = state->secno[state->p_secno]; Res->nitems++; Res->ncmds++; #ifdef DEBUG_BOOL DpsLog(query, DPS_LOG_EXTRA, "1[%d].%x %c -- %s", 0, 0, item_type(state->add_cmd), "<NULL>"); #endif } } Res->items[Res->nitems].cmd = state->cmd; Res->items[Res->nitems].order = state->order; Res->items[Res->nitems].order_inquery = state->order_inquery; Res->items[Res->nitems].origin = origin; Res->items[Res->nitems].count = 0; Res->items[Res->nitems].len = (word == NULL) ? 0 : dps_strlen(word); Res->items[Res->nitems].crcword = crcword; Res->items[Res->nitems].word = (word == NULL) ? NULL : DpsStrdup(word); Res->items[Res->nitems].ulen = wlen; Res->items[Res->nitems].uword = (uword == NULL) ? NULL : DpsUniDup(uword); Res->items[Res->nitems].pbegin = NULL; Res->items[Res->nitems].order_origin = 0; Res->items[Res->nitems].wordnum = Res->nitems; Res->items[Res->nitems].secno = state->secno[state->p_secno]; Res->nitems++; if (state->cmd != DPS_STACK_WORD) { Res->ncmds++; } else { Res->items[state->order].order_origin |= origin; if (state->order > Res->max_order) Res->max_order = state->order; if (state->order_inquery > Res->max_order_inquery) Res->max_order_inquery = state->order; } /* if ((state->cmd == DPS_STACK_WORD) && state->order > Res->max_order) Res->max_order = state->order;*/ #ifdef DEBUG_BOOL DpsLog(query, DPS_LOG_EXTRA, "1[%d,%d].%x %c -- %s", state->order, state->order_inquery, state->origin, item_type(state->cmd), (word == NULL) ? "<NULL>" : word); #endif return DPS_OK; }
int main(int argc, char ** argv, char **envp) { const char *env, *bcharset, *lcharset, *conf_dir; char template_name[PATH_MAX+6]=""; char *template_filename = NULL; char *query_string = NULL; char self[1024]=""; char *url = NULL; const char *ResultContentType; int res,httpd=0; size_t catcolumns = 0; int page_size,page_number; DPS_ENV *Env; DPS_AGENT *Agent; DPS_VARLIST query_vars; /* Output Content-type if under HTTPD */ /* Some servers do not pass QUERY_STRING */ /* if the query was empty, so check */ /* REQUEST_METHOD too to be safe */ httpd=(getenv("QUERY_STRING")||getenv("REQUEST_METHOD")); if (!(conf_dir=getenv("DPS_ETC_DIR"))) conf_dir=DPS_CONF_DIR; DpsInit(argc, argv, envp); Env=DpsEnvInit(NULL); if (Env == NULL) { if(httpd){ printf("Content-Type: text/plain\r\n\r\n"); } printf("Can't alloc Env\n"); exit(0); } DpsVarListInit(&query_vars); Agent = DpsAgentInit(NULL, Env, 0); if (Agent == NULL) { if(httpd){ printf("Content-Type: text/plain\r\n\r\n"); } printf("Can't alloc Agent\n"); exit(0); } DpsVarListAddEnviron(&Env->Vars,"ENV"); /* Detect self and template name */ if((env = getenv("DPSEARCH_TEMPLATE"))) dps_strncpy(template_name, env, sizeof(template_name) - 1); else if((env = getenv("PATH_INFO")) && env[0]) dps_strncpy(template_name, env + 1, sizeof(template_name) - 1); if((env=getenv("DPSEARCH_SELF"))) dps_strncpy(self,env,sizeof(self)-1); if((env=getenv("QUERY_STRING"))){ query_string = (char*)DpsRealloc(query_string, dps_strlen(env) + 2); if (query_string == NULL) { if(httpd){ printf("Content-Type: text/plain\r\n\r\n"); } printf("Can't alloc query_string\n"); exit(0); } dps_strncpy(query_string, env, dps_strlen(env) + 1); /* Hack for Russian Apache from apache.lexa.ru */ /* QUERY_STRING is already converted to server */ /* character set. We must print original query */ /* string instead however. Under usual apache */ /* we'll use QUERY_STRING. Note that query_vars */ /* list will contain not unescaped values, so */ /* we don't have to escape them when displaying */ env = getenv("CHARSET_SAVED_QUERY_STRING"); DpsParseQStringUnescaped(&query_vars,env?env:query_string); /* Unescape and save variables from QUERY_STRING */ /* Env->Vars will have unescaped values however */ DpsParseQueryString(Agent,&Env->Vars,query_string); template_filename = (char*)DpsStrdup(DpsVarListFindStr(&Env->Vars, "tmplt", "")); if((env=getenv("REDIRECT_STATUS"))){ /* Check Apache internal redirect */ /* via "AddHandler" and "Action" */ if(!self[0]){ dps_strncpy(self,(env=getenv("REDIRECT_URL"))?env:"filler.cgi",sizeof(self)-1); } if(!template_name[0]){ dps_strncpy(template_name,(env=getenv("PATH_TRANSLATED"))?env:"",sizeof(template_name)-1); } if (*template_filename == '\0') { DPS_FREE(template_filename); template_filename = (char*)DpsStrdup("filler.htm"); } }else{ /* CGI executed without Apache internal redirect */ /* Detect $Self variable with OS independant SLASHES */ if(!self[0]){ dps_strncpy(self,(env=getenv("SCRIPT_NAME"))?env:"filler.cgi",sizeof(self)-1); } if(!template_name[0]){ char *s,*e; /*This is with OS specific SLASHES */ env=((env=getenv("SCRIPT_FILENAME"))?env:"filler.cgi"); if(strcmp(conf_dir,".")){ /* Take from the config directory */ dps_snprintf(template_name, sizeof(template_name)-1, "%s/%s", conf_dir,(s=strrchr(env,DPSSLASH))?(s+1):(self)); }else{ /* Take from the current directory */ dps_strncpy(template_name,env,sizeof(template_name)-1); } /* Find right slash if it presents */ s=((s=strrchr(template_name,DPSSLASH))?s:template_name); if (*template_filename == '\0') { /* Find .cgi substring */ if ((e = strstr(s, ".cgi")) != NULL) { /* Replace ".cgi" with ".htm" */ e[1]='h';e[2]='t';e[3]='m'; } else { dps_strcat(s, ".htm"); } e = strrchr(s, '/'); DPS_FREE(template_filename); template_filename = (char*)DpsStrdup(e + 1); } else { dps_strncpy(s + 1, template_filename, sizeof(template_name) - (s - template_name) - 2); } } } }else{ /* Executed from command line */ /* or under server which does not */ /* pass an empty QUERY_STRING var */ if(argv[1]) { query_string = (char*)DpsRealloc(query_string, dps_strlen(argv[1]) + 10); if (query_string == NULL) { if(httpd){ printf("Content-Type: text/plain\r\n\r\n"); } printf("Can't realloc query_string\n"); exit(0); } sprintf(query_string, "q=%s", argv[1]); } else { query_string = (char*)DpsRealloc(query_string, 1024); if (query_string == NULL) { if(httpd){ printf("Content-Type: text/plain\r\n\r\n"); } printf("Can't realloc query_string\n"); exit(0); } sprintf(query_string, "q="); } /* Hack for Russian Apache from apache.lexa.ru */ /* QUERY_STRING is already converted to server */ /* character set. We must print original query */ /* string instead however. Under usual apache */ /* we'll use QUERY_STRING. Note that query_vars */ /* list will contain not unescaped values, so */ /* we don't have to escape them when displaying */ env = getenv("CHARSET_SAVED_QUERY_STRING"); DpsParseQStringUnescaped(&query_vars,env?env:query_string); /* Unescape and save variables from QUERY_STRING */ /* Env->Vars will have unescaped values however */ DpsParseQueryString(Agent,&Env->Vars,query_string); DPS_FREE(template_filename); template_filename = (char*)DpsStrdup(DpsVarListFindStr(&Env->Vars, "tmplt", "")); if (*template_filename == '\0') { DPS_FREE(template_filename); template_filename = (char*)DpsStrdup("filler.htm"); } /*// Get template name from command line variable &tmplt */ if(!template_name[0]) dps_snprintf(template_name,sizeof(template_name),"%s/%s", conf_dir, template_filename); } DpsVarListReplaceStr(&Agent->Conf->Vars, "tmplt", template_filename); DPS_FREE(template_filename); Agent->tmpl.Env_Vars = &Env->Vars; DpsURLNormalizePath(template_name); if (strncmp(template_name, conf_dir, dps_strlen(conf_dir)) || (res = DpsTemplateLoad(Agent, Env, &Agent->tmpl, template_name))) { if (strcmp(template_name, "filler.htm")) { /* trying load default template */ fprintf(stderr, "Can't load template: '%s' %s\n", template_name, Env->errstr); DPS_FREE(template_filename); template_filename = (char*)DpsStrdup("filler.htm"); dps_snprintf(template_name, sizeof(template_name), "%s/%s", conf_dir, template_filename); if ((res = DpsTemplateLoad(Agent, Env, &Agent->tmpl, template_name))) { if(httpd)printf("Content-Type: text/plain\r\n\r\n"); printf("%s\n",Env->errstr); DpsVarListFree(&query_vars); DpsEnvFree(Env); DPS_FREE(query_string); DpsAgentFree(Agent); return(0); } } else { if(httpd)printf("Content-Type: text/plain\r\n\r\n"); printf("%s\n",Env->errstr); DpsVarListFree(&query_vars); DpsEnvFree(Env); DPS_FREE(query_string); DpsAgentFree(Agent); return(0); } } /* set locale if specified */ if ((url = DpsVarListFindStr(&Env->Vars, "Locale", NULL)) != NULL) { setlocale(LC_ALL, url); /*#ifdef HAVE_ASPELL*/ { char *p; if ((p = strchr(url, '.')) != NULL) { *p = '\0'; DpsVarListReplaceStr(&Env->Vars, "g-lc", url); *p = '.'; } } /*#endif*/ url = NULL; } /* Call again to load search Limits if need */ DpsParseQueryString(Agent, &Env->Vars, query_string); Agent->Flags = Env->Flags; Agent->flags |= DPS_FLAG_UNOCON; Env->flags |= DPS_FLAG_UNOCON; DpsSetLogLevel(NULL, DpsVarListFindInt(&Env->Vars, "LogLevel", 0)); DpsOpenLog("filler.cgi", Env, !strcasecmp(DpsVarListFindStr(&Env->Vars, "Log2stderr", (!httpd) ? "yes" : "no"), "yes")); DpsLog(Agent,DPS_LOG_ERROR,"filler.cgi started with '%s'",template_name); DpsLog(Agent, DPS_LOG_DEBUG, "VarDir: '%s'", DpsVarListFindStr(&Agent->Conf->Vars, "VarDir", DPS_VAR_DIR)); DpsLog(Agent, DPS_LOG_DEBUG, "Affixes: %d, Spells: %d, Synonyms: %d, Acronyms: %d, Stopwords: %d", Env->Affixes.naffixes,Env->Spells.nspell, Env->Synonyms.nsynonyms, Env->Acronyms.nacronyms, Env->StopWords.nstopwords); DpsLog(Agent, DPS_LOG_DEBUG, "Chinese dictionary with %d entries", Env->Chi.nwords); DpsLog(Agent, DPS_LOG_DEBUG, "Korean dictionary with %d entries", Env->Korean.nwords); DpsLog(Agent, DPS_LOG_DEBUG, "Thai dictionary with %d entries", Env->Thai.nwords); DpsVarListAddLst(&Agent->Vars, &Env->Vars, NULL, "*"); Agent->tmpl.Env_Vars = &Agent->Vars; /* DpsVarListAddEnviron(&Agent->Vars, "ENV");*/ /****************************************************************************************************************************************/ /* This is for query tracking */ DpsVarListAddStr(&Agent->Vars, "QUERY_STRING", query_string); DpsVarListAddStr(&Agent->Vars, "self", self); env = getenv("HTTP_X_FORWARDER_FOR"); if (env) { DpsVarListAddStr(&Agent->Vars, "IP", env); } else { env = getenv("REMOTE_ADDR"); DpsVarListAddStr(&Agent->Vars, "IP", env ? env : "localhost"); } bcharset = DpsVarListFindStr(&Agent->Vars, "BrowserCharset", "iso-8859-1"); Env->bcs=DpsGetCharSet(bcharset); lcharset = DpsVarListFindStr(&Agent->Vars, "LocalCharset", "iso-8859-1"); Env->lcs=DpsGetCharSet(lcharset); ResultContentType = DpsVarListFindStr(&Agent->Vars, "ResultContentType", "text/html"); if(httpd){ if(!Env->bcs){ printf("Content-Type: text/plain\r\n\r\n"); printf("Unknown BrowserCharset '%s' in template '%s'\n",bcharset,template_name); exit(0); }else if(!Env->lcs){ printf("Content-Type: text/plain\r\n\r\n"); printf("Unknown LocalCharset '%s' in template '%s'\n",lcharset,template_name); exit(0); }else{ printf("Content-type: %s; charset=%s\r\n\r\n", ResultContentType, bcharset); } }else{ if(!Env->bcs){ printf("Unknown BrowserCharset '%s' in template '%s'\n",bcharset,template_name); exit(0); } if(!Env->lcs){ printf("Unknown LocalCharset '%s' in template '%s'\n",lcharset,template_name); exit(0); } } /* These parameters taken from "variable section of template"*/ res = DpsVarListFindInt(&Agent->Vars, "ps", DPS_DEFAULT_PS); page_size = dps_min(res, MAX_PS); page_number = DpsVarListFindInt(&Agent->Vars, "p", 0); if (page_number == 0) { page_number = DpsVarListFindInt(&Agent->Vars, "np", 0); DpsVarListReplaceInt(&Agent->Vars, "p", page_number + 1); } else page_number--; res = DpsVarListFindInt(&Agent->Vars, "np", 0) * page_size; DpsVarListAddInt(&Agent->Vars, "pn", res); catcolumns = (size_t)atoi(DpsVarListFindStr(&Agent->Vars, "CatColumns", "")); DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "top"); DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "restop"); DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "res"); DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "resbot"); DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "bottom"); DpsVarListFree(&query_vars); DpsAgentFree(Agent); DpsEnvFree(Env); DPS_FREE(query_string); DPS_FREE(url); if (httpd) fflush(NULL); else fclose(stdout); #ifdef EFENCE fprintf(stderr, "Memory leaks checking\n"); DpsEfenceCheckLeaks(); #endif #ifdef FILENCE fprintf(stderr, "FD leaks checking\n"); DpsFilenceCheckLeaks(NULL); #endif return DPS_OK; }
int DpsCookiesAdd(DPS_AGENT *Indexer, const char *domain, const char * path, const char *name, const char *value, const char secure, dps_uint4 expires, const char from_config, int insert_flag) { #ifdef HAVE_SQL char buf[3*PATH_MAX]; char path_esc[2*PATH_MAX+1]; DPS_COOKIES *Cookies = &Indexer->Cookies; DPS_COOKIE *Coo; DPS_DB *db; dpshash32_t url_id = DpsStrHash32(domain); size_t i; #ifdef WITH_PARANOIA void *paran = DpsViolationEnter(paran); #endif if (Indexer->flags & DPS_FLAG_UNOCON) { if (Indexer->Conf->dbl.nitems == 0) return DPS_OK; DPS_GETLOCK(Indexer, DPS_LOCK_DB); db = Indexer->Conf->dbl.db[url_id % Indexer->Conf->dbl.nitems]; } else { if (Indexer->dbl.nitems == 0) return DPS_OK; db = Indexer->dbl.db[url_id % Indexer->dbl.nitems]; } (void)DpsDBEscStr(db, path_esc, DPS_NULL2EMPTY(path), dps_min(PATH_MAX,dps_strlen(DPS_NULL2EMPTY(path)))); for (i = 0; i < Cookies->ncookies; i++) { Coo = &Cookies->Cookie[i]; if (!strcasecmp(Coo->domain, domain) && !strcasecmp(Coo->path, DPS_NULL2EMPTY(path)) && !strcasecmp(Coo->name, name) && (Coo->secure == secure)/* && (Coo->from_config == from_config)*/ ) { DPS_FREE(Coo->value); Coo->value = DpsStrdup(value); /* Coo->expires = expires;*/ if (insert_flag) { dps_snprintf(buf, sizeof(buf), "UPDATE cookies SET value='%s',expires=%d WHERE domain='%s' AND path='%s' AND name='%s' AND secure='%c'", value, expires, domain, path_esc, name, secure); DpsSQLAsyncQuery(db, NULL, buf); } if (Indexer->flags & DPS_FLAG_UNOCON) DPS_RELEASELOCK(Indexer, DPS_LOCK_DB); #ifdef WITH_PARANOIA DpsViolationExit(Indexer->handle, paran); #endif return DPS_OK; } } Cookies->Cookie = (DPS_COOKIE*)DpsRealloc(Cookies->Cookie, (Cookies->ncookies + 1) * sizeof(DPS_COOKIE)); if(Cookies->Cookie == NULL) { Cookies->ncookies = 0; if (Indexer->flags & DPS_FLAG_UNOCON) DPS_RELEASELOCK(Indexer, DPS_LOCK_DB); #ifdef WITH_PARANOIA DpsViolationExit(Indexer->handle, paran); #endif return DPS_ERROR; } Coo = &Cookies->Cookie[Cookies->ncookies]; /* Coo->expires = expires;*/ Coo->secure = secure; Coo->from_config = from_config; Coo->domain = DpsStrdup(domain); Coo->path = DpsStrdup(path); Coo->name = DpsStrdup(name); Coo->value = DpsStrdup(value); if (insert_flag) { if (Indexer->Flags.CheckInsertSQL) { dps_snprintf(buf, sizeof(buf), "DELETE FROM cookies WHERE domain='%s' AND path='%s' AND name='%s' AND secure='%c'", domain, path_esc, name, secure); DpsSQLAsyncQuery(db, NULL, buf); } dps_snprintf(buf, sizeof(buf), "INSERT INTO cookies(expires,secure,domain,path,name,value)VALUES(%d,'%c','%s','%s','%s','%s')", expires, secure, domain, path_esc, name, value); DpsSQLAsyncQuery(db, NULL, buf); } Cookies->ncookies++; if (Indexer->flags & DPS_FLAG_UNOCON) DPS_RELEASELOCK(Indexer, DPS_LOCK_DB); #ifdef WITH_PARANOIA DpsViolationExit(Indexer->handle, paran); #endif #endif /*HAVE_SQL*/ return DPS_OK; }
void DpsCookiesFind(DPS_AGENT *Indexer, DPS_SERVER *Server, DPS_DOCUMENT *Doc, const char *hostinfo) { #ifdef HAVE_SQL DPS_DSTR cookie; DPS_COOKIES *Cookies = &Indexer->Cookies; DPS_COOKIE *Coo; size_t i, blen = dps_strlen(hostinfo), slen; int have_no_cookies = DpsVarListFindInt(&Doc->Sections, "have_no_cookies", 1); #ifdef WITH_PARANOIA void *paran = DpsViolationEnter(paran); #endif TRACE_IN(Indexer, "DpsCookiesFind"); DpsDSTRInit(&cookie, 1024); for(i = 0; i < Cookies->ncookies; i++) { Coo = &Cookies->Cookie[i]; slen = dps_strlen(Coo->domain); if (slen > blen) continue; if (Coo->secure == 'y' && strcasecmp(Doc->CurURL.schema, "https")) continue; if (strncasecmp(Coo->path, Doc->CurURL.path, dps_strlen(Coo->path))) continue; if (strcasecmp(Coo->domain, hostinfo + (blen - slen))) continue; if (Coo->from_config != 1) have_no_cookies = 0; if (Coo->name[0] == '\0' && Coo->value[0] == '\0') continue; if (cookie.data_size) DpsDSTRAppend(&cookie, "; ", 2); DpsDSTRAppendStr(&cookie, Coo->name); DpsDSTRAppend(&cookie, "=", 1); DpsDSTRAppendStr(&cookie, Coo->value); } if (have_no_cookies) { char buf[2*PATH_MAX]; dpshash32_t url_id; DPS_DB *db; DPS_SQLRES Res; size_t rows; int rc; if (Server != NULL) { char *PingData = DpsVarListFindStr(&Server->Vars, "AuthPing", NULL); if (PingData != NULL) { char *AuthPing = DpsStrdup(DpsTrim(PingData, " \t\r\n")); int method = DPS_METHOD_GET; dps_base64_decode(AuthPing, PingData, dps_strlen(PingData)); if (!strncasecmp(AuthPing, "GET", 3)) { method = DPS_METHOD_GET; PingData = DpsTrim(AuthPing + 3, " \t\r\n"); } else if (!strncasecmp(AuthPing, "POST", 4)) { method = DPS_METHOD_POST; PingData = DpsTrim(AuthPing + 4, " \t\r\n"); } else { DpsLog(Indexer, DPS_LOG_ERROR, "AuthPing should be GET or POST: %s", AuthPing); PingData = NULL; } if (PingData != NULL) { size_t size = dps_strlen(PingData); { char PingURL[size + 2]; char PingBody[size]; DPS_DOCUMENT *rDoc; int result; rDoc = DpsDocInit(NULL); DpsSpiderParamInit(&rDoc->Spider); DpsVarList2Doc(rDoc, Server); rDoc->Buf.max_size = (size_t)DpsVarListFindInt(&Indexer->Vars, "MaxDocSize", DPS_MAXDOCSIZE); rDoc->Buf.allocated_size = DPS_NET_BUF_SIZE; if ((rDoc->Buf.buf = (char*)DpsMalloc(rDoc->Buf.allocated_size + 1)) == NULL) { DpsDocFree(rDoc); TRACE_OUT(Indexer); return; } rDoc->Buf.buf[0]='\0'; rDoc->subdoc = Indexer->Flags.SubDocLevel + 1; #if 1 dps_snprintf(buf, sizeof(buf), "%s://%s/", DPS_NULL2EMPTY(Doc->CurURL.schema), DPS_NULL2EMPTY(Doc->CurURL.hostinfo)); DpsVarListReplaceStr(&rDoc->Sections, "URL", buf); DpsURLParse(&rDoc->CurURL, buf); DpsLog(Indexer, DPS_LOG_INFO, "HOME: %s", buf); rDoc->method = DPS_METHOD_HEAD; /* DpsVarListFree(&rDoc->RequestHeaders);*/ if (Doc != NULL) { DpsVarListReplaceLst(&rDoc->RequestHeaders, &Doc->RequestHeaders, NULL, "*"); } DpsVarListReplaceStr(&rDoc->Sections, "have_no_cookies", "0"); DpsDocAddDocExtraHeaders(Indexer, Server, rDoc); DpsDocAddConfExtraHeaders(Indexer->Conf, rDoc); DpsVarListReplaceLst(&rDoc->Sections, &Server->Vars, NULL, "*"); DpsDocAddServExtraHeaders(Server, rDoc); DpsVarListLog(Indexer, &rDoc->RequestHeaders, DPS_LOG_DEBUG, "HOME.Request"); if (Doc == NULL || Indexer->Flags.cmd == DPS_IND_FILTER) { DpsDocLookupConn(Indexer, rDoc); } else { DPS_FREE(rDoc->connp.connp); rDoc->connp = Doc->connp; } result = DpsGetURL(Indexer, rDoc, NULL); /* Just get headers from the home as we need only Cookies from it */ DpsDocProcessResponseHeaders(Indexer, rDoc); DpsVarListLog(Indexer, &rDoc->Sections, DPS_LOG_DEBUG, "HOME.Response"); #endif sscanf(PingData, "%s %s", PingURL, PingBody); if (rDoc->method == DPS_METHOD_GET) { dps_strcat(PingURL, "?"); dps_strcat(PingURL, PingBody); } else { DpsVarListReplaceStr(&rDoc->Sections, "body", PingBody); } DpsVarListReplaceStr(&rDoc->Sections, "URL", PingURL); DpsURLParse(&rDoc->CurURL, PingURL); DpsLog(Indexer, DPS_LOG_INFO, "AUTH.PING: %s", PingURL); rDoc->method = method; DpsVarListFree(&rDoc->RequestHeaders); DpsVarListReplaceStr(&rDoc->Sections, "have_no_cookies", "0"); DpsDocAddDocExtraHeaders(Indexer, Server, rDoc); DpsDocAddConfExtraHeaders(Indexer->Conf, rDoc); DpsVarListReplaceLst(&rDoc->Sections, &Server->Vars, NULL, "*"); DpsDocAddServExtraHeaders(Server, rDoc); if (method == DPS_METHOD_POST) { dps_snprintf(buf, sizeof(buf), "application/x-www-form-urlencoded; charset=%s", DpsVarListFindStr(&Indexer->Conf->Vars, "LocalCharset", "iso-8859-1")); DpsVarListReplaceStr(&rDoc->RequestHeaders, "Content-Type", buf); dps_snprintf(buf, sizeof(buf), "%d", dps_strlen(PingBody)); DpsVarListReplaceStr(&rDoc->RequestHeaders, "Content-Length", buf); } DpsVarListLog(Indexer, &rDoc->RequestHeaders, DPS_LOG_DEBUG, "AUTHPING.Request"); #if 0 if (Doc == NULL || Indexer->Flags.cmd == DPS_IND_FILTER) { DpsDocLookupConn(Indexer, rDoc); } else { DPS_FREE(rDoc->connp.connp); rDoc->connp = Doc->connp; } #endif result = DpsGetURL(Indexer, rDoc, NULL); /* Just get it as we need only Cookies from the headers */ DpsDocProcessResponseHeaders(Indexer, rDoc); DpsVarListDel(&rDoc->Sections, "body"); DpsVarListLog(Indexer, &rDoc->Sections, DPS_LOG_DEBUG, "AUTHPING.Response"); if (Doc != NULL) bzero(&rDoc->connp, sizeof(rDoc->connp)); DpsDocFree(rDoc); } } DpsFree(AuthPing); } } while(hostinfo != NULL) { url_id = DpsStrHash32(hostinfo); DpsSQLResInit(&Res); dps_snprintf(buf, sizeof(buf), "SELECT name,value,path,secure FROM cookies WHERE domain='%s'", hostinfo); if (Indexer->flags & DPS_FLAG_UNOCON) { DPS_GETLOCK(Indexer, DPS_LOCK_DB); db = Indexer->Conf->dbl.db[url_id % Indexer->Conf->dbl.nitems]; } else { db = Indexer->dbl.db[url_id % Indexer->dbl.nitems]; } if(DPS_OK == (rc = DpsSQLQuery(db, &Res, buf))) { rows = DpsSQLNumRows(&Res); for(i = 0; i < rows; i++) { DpsCookiesAdd(Indexer, hostinfo, DpsSQLValue(&Res, i, 2), DpsSQLValue(&Res, i, 0), DpsSQLValue(&Res, i, 1), *DpsSQLValue(&Res, i, 3), 0, 0, 0); if (*DpsSQLValue(&Res, i, 3) == 'y' && strcasecmp(Doc->CurURL.schema, "https")) continue; if (strncasecmp(DpsSQLValue(&Res, i, 2), Doc->CurURL.path, dps_strlen(DpsSQLValue(&Res, i, 2)))) continue; if (cookie.data_size) DpsDSTRAppend(&cookie, "; ", 2); DpsDSTRAppendStr(&cookie, DpsSQLValue(&Res, i, 0)); DpsDSTRAppend(&cookie, "=", 1); DpsDSTRAppendStr(&cookie, DpsSQLValue(&Res, i, 1)); } if (rows == 0) { DpsCookiesAdd(Indexer, hostinfo, "/", "", "", 'n', 0, 0, 0); } } DpsSQLFree(&Res); if (Indexer->flags & DPS_FLAG_UNOCON) { DPS_RELEASELOCK(Indexer, DPS_LOCK_DB); } hostinfo = strchr(hostinfo, '.'); if (hostinfo != NULL) hostinfo++; } } if (cookie.data_size) { DpsVarListReplaceStr(&Doc->RequestHeaders, "Cookie", cookie.data); } DpsDSTRFree(&cookie); #endif TRACE_OUT(Indexer); return; }