int DpsCookiesAddStr(DPS_AGENT *Indexer, DPS_URL *CurURL, const char *cookie_str, int insert_flag) {
char *part, *lpart;
char *name = NULL;
char *value = NULL;
char *domain = NULL, *orig_domain = NULL;
char *path = NULL;
dps_uint4 expire = 0, need_free_domain = 1, need_free_path = 1;
char secure = 'n', savec;
for (part = dps_strtok_r(cookie_str, ";" , &lpart, &savec) ; part;
part = dps_strtok_r(NULL, ";", &lpart, &savec)) {
char *arg;
part = DpsTrim(part, " ");
if ((arg = strchr(part, '='))) {
*arg++ = '\0';
if (!name) {
name = part;
DpsFree(value);
value = DpsStrdup(arg);
} else
if (!strcasecmp(part, "path")) {
DpsFree(path);
path = DpsStrdup(arg);
} else
if (!strcasecmp(part, "domain")) {
DpsFree(orig_domain);
orig_domain = domain = DpsStrdup(arg);
} else
if (!strcasecmp(part, "secure")) {
secure = 'y';
} else
if (!strcasecmp(part, "expires")) {
expire = (dps_uint4)DpsHttpDate2Time_t(arg);
}
}
}
if (name && value) {
if (domain && domain[0] == '.') {
domain++;
} else {
if (domain) DpsFree(orig_domain);
domain = (CurURL && CurURL->hostname) ? CurURL->hostname : "localhost";
need_free_domain = 0;
}
if (!path) {
path = (CurURL && CurURL->path) ? CurURL->path : "/";
need_free_path = 0;
}
DpsCookiesAdd(Indexer, domain, path, name, value, secure, expire, (CurURL==NULL)?1:0, 1);
}
DpsFree(value);
if (need_free_path) DpsFree(path);
if (need_free_domain) DpsFree(orig_domain);
return DPS_OK;
}
__C_LINK void __DPSCALL DpsTextListAdd(DPS_TEXTLIST * tlist,const DPS_TEXTITEM *item){
#ifdef WITH_PARANOIA
void * paran = DpsViolationEnter(paran);
#endif
if(!item->str) {
#ifdef WITH_PARANOIA
DpsViolationExit(-1, paran);
#endif
return;
}
#ifdef DEBUG_MEM
if (tlist->mitems) {
mprotect(tlist->Items, tlist->nitems * sizeof(DPS_TEXTITEM), PROT_READ | PROT_WRITE);
/* fprintf(stderr, "addr: %x, len: %d till %x -- PROT_BOTH\n", tlist->Items, tlist->nitems * sizeof(DPS_TEXTITEM),
tlist->Items + tlist->nitems );*/
}
#endif
if (tlist->nitems + 1 > tlist->mitems) {
tlist->mitems += DPS_TEXTLIST_PAS;
tlist->Items = (DPS_TEXTITEM*)DpsRealloc(tlist->Items, (tlist->mitems) * sizeof(DPS_TEXTITEM) + 4096);
if (tlist->Items == NULL) {
tlist->nitems = tlist->mitems = 0;
return;
}
}
tlist->Items[tlist->nitems].str = (char*)DpsStrdup(item->str);
tlist->Items[tlist->nitems].href = (item->href != NULL) ? (char*)DpsStrdup(item->href) : NULL;
tlist->Items[tlist->nitems].section_name = (item->section_name != NULL) ? (char*)DpsStrdup(item->section_name) : NULL;
tlist->Items[tlist->nitems].section = item->section;
tlist->Items[tlist->nitems].strict = item->strict;
tlist->Items[tlist->nitems].len = item->len;
tlist->Items[tlist->nitems].marked = 0;
tlist->nitems++;
#ifdef DEBUG_MEM
if (tlist->mitems) {
mprotect(tlist->Items, tlist->nitems * sizeof(DPS_TEXTITEM), PROT_READ);
/* fprintf(stderr, "addr: %x, len: %d till %x -- PROT_READ\n", tlist->Items, tlist->nitems * sizeof(DPS_TEXTITEM),
tlist->Items + tlist->nitems );*/
}
#endif
#ifdef WITH_PARANOIA
DpsViolationExit(-1, paran);
#endif
return;
}
int Dps_ftp_login(DPS_CONN *connp, char *user, char *passwd){
char *buf;
char user_tmp[32], passwd_tmp[64];
int code;
size_t len;
DPS_FREE(connp->user);
DPS_FREE(connp->pass);
if (!user)
dps_snprintf(user_tmp, 32, "anonymous");
else {
dps_snprintf(user_tmp, 32, "%s", user);
connp->user = (char*)DpsStrdup(user);
}
if (!passwd)
dps_snprintf(passwd_tmp, 64, "*****@*****.**", PACKAGE, VERSION);
else {
dps_snprintf(passwd_tmp, 32, "%s", passwd);
connp->pass = (char*)DpsStrdup(passwd);
}
len = dps_strlen(user_tmp) + 5 /*dps_strlen("USER ")*/;
buf = (char *) DpsXmalloc(len+1);
if (buf == NULL) return -1;
dps_snprintf(buf, len+1, "USER %s", user_tmp);
code = Dps_ftp_send_cmd(connp, buf);
DPS_FREE(buf);
if (code == -1)
return -1;
else if (code == 2) /* Don't need password */
return 0;
len = dps_strlen(passwd_tmp) + 5 /*dps_strlen("PASS ")*/;
buf = (char *) DpsXmalloc(len+1);
if (buf == NULL) return -1;
dps_snprintf(buf, len+1, "PASS %s", passwd_tmp);
code = Dps_ftp_send_cmd( connp, buf);
DPS_FREE(buf);
if (code > 3)
return -1;
return 0;
}
char * DpsEnvErrMsg(DPS_ENV * Conf) {
size_t i;
DPS_DB *db;
for(i = 0; i<Conf->dbl.nitems; i++){
db = &Conf->dbl.db[i];
if (db->errcode) {
char *oe = (char*)DpsStrdup(Conf->errstr);
dps_snprintf(Conf->errstr, 2048, "DB err: %s - %s", db->errstr, oe);
DPS_FREE(oe);
}
}
return(Conf->errstr);
}
DPS_ENV *DpsEnvInit(DPS_ENV *Conf){
#ifdef MECAB
const char *mecab_argv[8] = {"mecab", "-F", "%m ", "-B", " ", "-E", " ", NULL};
#endif
if(!Conf){
Conf=(DPS_ENV *)DpsMalloc(sizeof(DPS_ENV));
if (Conf == NULL) return NULL;
bzero((void*)Conf, sizeof(*Conf));
Conf->freeme=1;
}else{
bzero((void*)Conf, sizeof(*Conf));
}
Conf->Flags.OptimizeAtUpdate = 1;
Conf->Flags.do_excerpt = 1;
Conf->Flags.PopRankNeoIterations = 3;
Conf->Flags.GuesserBytes = 512;
Conf->Flags.robots_period = 604800; /* one week */
Conf->Flags.URLInfoSQL = 1;
Conf->Flags.SRVInfoSQL = 1;
Conf->Flags.CheckInsertSQL = 1;
Conf->Flags.mark_for_index = 1;
Conf->Flags.MaxSiteLevel = 2;
Conf->Flags.SEASentences = 32;
Conf->Flags.SEASentenceMinLength = 64;
Conf->Flags.PagesInGroup = 1;
Conf->Flags.SubDocCnt = 5;
Conf->Flags.MaxCrawlDelay = 300;
Conf->Flags.rel_nofollow = 1;
Conf->Flags.bind_addr.sin_family = AF_INET;
Conf->Flags.use_meta = 1;
Conf->WordParam.min_word_len = 1;
Conf->WordParam.max_word_len = 32;
Conf->WordParam.correct_factor = 1;
Conf->WordParam.incorrect_factor = 1;
Conf->url_number = 0x7FFFFFFF;
Conf->lcs=DpsGetCharSet("latin1");
Conf->bcs=DpsGetCharSet("latin1");
Conf->CharsToEscape = DpsStrdup("\"&<>");
#ifdef MECAB
/* Conf->mecab = mecab_new2 ("mecab -F \"%m \" -B \" \" -E \" \"");*/
Conf->mecab = mecab_new(7, (char**)mecab_argv);
#endif
return(Conf);
}
int _DpsOpen3(const char *path, int flags, int mode, char *filename, size_t fileline) {
register Slot * slot;
register size_t count;
Slot * fullSlot = NULL;
int address;
if ( allocationList == NULL )
initialize();
lock();
if ( unUsedSlots < 2 ) {
allocateMoreSlots();
}
for ( slot = allocationList, count = slotCount ; count > 0; count-- ) {
if ( slot->mode == FREE || slot->mode == NOT_IN_USE) {
fullSlot = slot;
break;
}
slot++;
}
if ( !fullSlot )
FE_InternalError("No empty file slot.");
fullSlot->fileline = fileline;
dps_strncpy(fullSlot->filename, filename, DPS_FILENAMELEN);
fullSlot->path = DpsStrdup(path);
fullSlot->fd = address = open(path, flags, mode);
if (address > 0) {
fullSlot->mode = ALLOCATED;
unUsedSlots--;
}
release();
return address;
}
void DpsParseHTTPResponse(DPS_AGENT *Indexer, DPS_DOCUMENT *Doc) {
char *token, *lt, *headers;
int oldstatus;
DPS_DSTR header;
Doc->Buf.content=NULL;
oldstatus = DpsVarListFindInt(&Doc->Sections, "Status", 0);
DpsVarListReplaceInt(&Doc->Sections, "ResponseSize", (int)Doc->Buf.size);
DpsVarListDel(&Doc->Sections, "Content-Length");
DpsVarListDel(&Doc->Sections, "Last-Modified");
if (Doc->Buf.buf == NULL) return;
/* Cut HTTP response header first */
for(token=Doc->Buf.buf;*token;token++){
if(!strncmp(token,"\r\n\r\n",4)){
*token='\0';
Doc->Buf.content = token + 4;
break;
}else
if(!strncmp(token,"\n\n",2)){
*token='\0';
Doc->Buf.content = token + 2;
break;
}
}
/* Bad response, return */
if(!Doc->Buf.content) {
if (token < Doc->Buf.buf + Doc->Buf.size - 4) {
if (token[2] == '\r') Doc->Buf.content = token + 4;
else Doc->Buf.content = token + 2;
} else {
return;
}
}
/* Copy headers not to break them */
headers = (char*)DpsStrdup(Doc->Buf.buf);
/* Now lets parse response header lines */
token = dps_strtok_r(headers,"\r\n",<);
if(!token)return;
if(!strncmp(token,"HTTP/",5)){
int status = atoi(token + 8);
DpsVarListReplaceStr(&Doc->Sections,"ResponseLine",token);
DpsVarListReplaceInt(&Doc->Sections, "Status", (oldstatus > status) ? oldstatus : status );
}else{
DpsFree(headers);
return;
}
token = dps_strtok_r(NULL,"\r\n",<);
DpsDSTRInit(&header, 128);
while(token){
if(strchr(token,':')) {
if (header.data_size) {
DpsParseHTTPHeader(Indexer, Doc, &header);
DpsDSTRFree(&header);
DpsDSTRInit(&header, 128);
}
}
DpsDSTRAppendStr(&header, token);
token = dps_strtok_r(NULL,"\r\n",<);
}
if (header.data_size) {
DpsParseHTTPHeader(Indexer, Doc, &header);
}
DpsDSTRFree(&header);
DPS_FREE(headers);
DpsVarListInsInt(&Doc->Sections,"Content-Length",Doc->Buf.buf-Doc->Buf.content+(int)Doc->Buf.size);
}
int main(int argc,char **argv, char **envp) {
int ch, sleeps = 1, optimize = 0, obi = 0;
unsigned int from = 0, to = 0xFFF, p_to = 0;
DPS_ENV * Env;
const char * config_name = DPS_CONF_DIR "/cached.conf";
DpsInit(argc, argv, envp); /* Initialize library */
DpsInitMutexes();
Env=DpsEnvInit(NULL);
if (Env == NULL) exit(1);
DpsSetLockProc(Env, DpsLockProc);
/*#ifndef HAVE_SETPROCTITLE*/
ARGV = argv;
ARGC = argc;
/*#endif*/
while ((ch = getopt(argc, argv, "blt:f:op:w:v:h?")) != -1){
switch (ch) {
case 'f':
sscanf(optarg, "%x", &from);
break;
case 't':
sscanf(optarg, "%x", &p_to);
break;
case 'w':
DpsVarListReplaceStr(&Env->Vars, "VarDir", optarg);
break;
case 'v': DpsSetLogLevel(NULL, atoi(optarg)); break;
case 'b': obi++; break;
case 'o': optimize++; break;
case 'p': sleeps = atoi(optarg); break;
case 'h':
case '?':
default:
usage();
DpsEnvFree(Env);
DpsDeInit();
DpsDestroyMutexes();
return 1;
break;
}
}
argc -= optind;
argv += optind;
if(argc > 1) {
usage();
DpsEnvFree(Env);
DpsDeInit();
DpsDestroyMutexes();
return 1;
} else if (argc == 1) {
config_name = argv[0];
}
{
DPS_LOGDEL *del_buf=NULL;
size_t del_count = 0, log, bytes, n = 0;
int dd, log_fd;
struct stat sb;
char dname[PATH_MAX] = "";
DPS_BASE_PARAM P;
DPS_LOGWORD *log_buf = NULL;
DPS_AGENT *Indexer = DpsAgentInit(NULL, Env, 0);
log2stderr = 1;
if (Indexer == NULL) {
fprintf(stderr, "Can't alloc Agent at %s:%d\n", __FILE__, __LINE__);
exit(DPS_ERROR);
}
if(DPS_OK != DpsEnvLoad(Indexer, config_name, (dps_uint8)0)){
fprintf(stderr, "%s\n", DpsEnvErrMsg(Env));
DpsEnvFree(Env);
DpsDeInit();
DpsDestroyMutexes();
return DPS_ERROR;
}
DpsOpenLog("splitter", Env, log2stderr);
Indexer->flags = Env->flags = DPS_FLAG_UNOCON;
DpsVarListAddLst(&Indexer->Vars, &Env->Vars, NULL, "*");
bzero(&P, sizeof(P));
P.subdir = DPS_TREEDIR;
P.basename = "wrd";
P.indname = "wrd";
P.mode = DPS_WRITE_LOCK;
P.NFiles = DpsVarListFindInt(&Indexer->Conf->Vars, "WrdFiles", 0x300);
P.vardir = DpsStrdup(DpsVarListFindStr(&Indexer->Conf->Vars, "VarDir", DPS_VAR_DIR));
P.A = Indexer;
if (p_to != 0) to = p_to;
else to = P.NFiles - 1;
#ifdef HAVE_ZLIB
P.zlib_method = Z_DEFLATED;
P.zlib_level = 9;
P.zlib_windowBits = DPS_BASE_WRD_WINDOWBITS;
P.zlib_memLevel = 9;
P.zlib_strategy = DPS_BASE_WRD_STRATEGY;
#endif
/* Open del log file */
dps_snprintf(dname,sizeof(dname),"%s%c%s%cdel-split.log", P.vardir, DPSSLASH, DPS_SPLDIR, DPSSLASH);
if((dd = DpsOpen2(dname, O_RDONLY | DPS_BINARY)) < 0) {
dps_strerror(NULL, 0, "Can't open del log '%s'", dname);
exit(DPS_ERROR);
}
DpsLog(Indexer, DPS_LOG_DEBUG, "VarDir: %s, WrdFiles: %d [%x]", P.vardir, P.NFiles, P.NFiles);
/* Allocate del buffer */
fstat(dd, &sb);
if (sb.st_size != 0) {
del_buf=(DPS_LOGDEL*)DpsMalloc((size_t)sb.st_size + 1);
if (del_buf == NULL) {
fprintf(stderr, "Can't alloc %d bytes at %s:%d\n", (int)sb.st_size, __FILE__, __LINE__);
exit(0);
}
del_count=read(dd,del_buf,(size_t)sb.st_size)/sizeof(DPS_LOGDEL);
}
DpsClose(dd);
/* Remove duplicates URLs in DEL log */
/* Keep only oldest records for each URL */
if (del_count > 0) {
DpsLog(Indexer, DPS_LOG_DEBUG, "Sorting del_buf: %d items", del_count);
if (del_count > 1) DpsSort(del_buf, (size_t)del_count, sizeof(DPS_LOGDEL), DpsCmpurldellog);
DpsLog(Indexer, DPS_LOG_DEBUG, "Removing DelLogDups");
del_count = DpsRemoveDelLogDups(del_buf, del_count);
}
DpsLog(Indexer, DPS_LOG_DEBUG, "Processing Bufs from %d [%x] to %d [%x]", from, from, to, to);
for(log = from; log <= to; log++) {
/* Open log file */
dps_snprintf(dname, sizeof(dname), "%s%c%s%c%03X-split.log", P.vardir, DPSSLASH, DPS_SPLDIR, DPSSLASH, log);
if((log_fd = DpsOpen2(dname, O_RDWR|DPS_BINARY)) < 0){
if (errno == ENOENT) {
dps_strerror(Indexer, DPS_LOG_DEBUG, "Can't open '%s'", dname);
n = 0;
/* continue;*/
} else {
dps_strerror(Indexer, DPS_LOG_ERROR, "Can't open '%s'", dname);
continue;
}
} else {
DpsWriteLock(log_fd);
DpsLog(Indexer, DPS_LOG_DEBUG, "Processing Log: %x", log);
fstat(log_fd, &sb);
log_buf = (sb.st_size > 0) ? (DPS_LOGWORD*)DpsMalloc((size_t)sb.st_size + 1) : NULL;
if (log_buf != NULL) {
unlink(dname);
bytes = read(log_fd,log_buf,(size_t)sb.st_size);
(void)ftruncate(log_fd, (off_t)0);
DpsUnLock(log_fd);
DpsClose(log_fd);
n = bytes / sizeof(DPS_LOGWORD);
DpsLog(Indexer, DPS_LOG_DEBUG, "Sorting log_buf: %d items", n);
if (n > 1) DpsSort(log_buf, n, sizeof(DPS_LOGWORD), (qsort_cmp)DpsCmplog);
DpsLog(Indexer, DPS_LOG_DEBUG, "Removing OldWords");
n = DpsRemoveOldWords(log_buf, n, del_buf, del_count);
if (n > 1) DpsSort(log_buf, n, sizeof(DPS_LOGWORD), (qsort_cmp)DpsCmplog_wrd);
} else {
n = 0;
DpsUnLock(log_fd);
DpsClose(log_fd);
}
}
DpsLog(Indexer, DPS_LOG_DEBUG, "Processing Buf, optimize: %d", optimize);
if (obi) DpsBaseOptimize(&P, log);
DpsProcessBuf(Indexer, &P, log, log_buf, n, del_buf, del_count);
if (optimize) DpsBaseOptimize(&P, log);
DpsBaseClose(&P);
DPS_FREE(log_buf);
DpsLog(Indexer, DPS_LOG_DEBUG, "pas done: %d from %d to %d", log, from, to);
DPSSLEEP(sleeps);
}
DPS_FREE(del_buf);
DpsAgentFree(Indexer);
DPS_FREE(P.vardir);
}
fprintf(stderr, "Splitting done.\n");
DpsEnvFree(Env);
DpsDeInit();
DpsDestroyMutexes();
#ifdef EFENCE
fprintf(stderr, "Memory leaks checking\n");
DpsEfenceCheckLeaks();
#endif
#ifdef FILENCE
fprintf(stderr, "FD leaks checking\n");
DpsFilenceCheckLeaks(NULL);
#endif
return 0;
}
int _DpsURLParse(DPS_URL *url, const char *str, const char *filename, int line) {
#else
int DpsURLParse(DPS_URL *url, const char *str) {
#endif
char *schema,*anchor,*file,*query;
char *s;
/* size_t len = dps_strlen(str);*/
#ifdef WITH_PARANOIA
void * paran = DpsViolationEnter(paran);
#endif
#ifdef DEBUG_URL
fprintf(stderr, " -- %s:%d Parser url: %s\n", filename, line, str);
#endif
DPS_FREE(url->schema);
DPS_FREE(url->specific);
DPS_FREE(url->hostinfo);
DPS_FREE(url->hostname);
DPS_FREE(url->anchor);
DPS_FREE(url->auth);
url->port=0;
url->default_port=0;
DPS_FREE(url->path);
DPS_FREE(url->directory);
DPS_FREE(url->filename);
DPS_FREE(url->query_string);
/* if(len >= DPS_URLSIZE)return(DPS_URL_LONG); FIXME: Chage this cheking for configured parameter, not DPS_URLSIZE */
s = (char*)DpsStrdup(str);
if (s == NULL) {
#ifdef WITH_PARANOIA
DpsViolationExit(-1, paran);
#endif
return DPS_ERROR;
}
url->len = dps_strlen(str);
/* Find possible schema end than */
/* Check that it is really schema */
/* It must consist of alphas only */
/* We will take in account digits */
/* also for oracle8:// for example */
/* We must check it because */
/* It might be anchor also */
/* For example: */
/* "mod/index.html#a:1" */
if((schema=strchr(s,':'))){
const char * ch;
for(ch=s;ch<schema;ch++){
if(!isalnum(*ch)){
/* Bad character */
/* so it is not schema */
schema=0;break;
}
}
}
if(schema){
/* Have scheme - absolute path */
*schema=0;
url->schema = (char*)DpsStrdup(s);
url->specific = (char*)DpsStrdup(schema + 1);
*schema=':';
if(!strcasecmp(url->schema,"http"))url->default_port=80;
else
if(!strcasecmp(url->schema,"https"))url->default_port=443;
else
if(!strcasecmp(url->schema,"nntp"))url->default_port=119;
else
if(!strcasecmp(url->schema,"news"))url->default_port=119;
else
if(!strcasecmp(url->schema,"ftp"))url->default_port=21;
if(!strncmp(url->specific,"//",2)){
char *ss,*hostname;
/* Have hostinfo */
if((ss=strchr(url->specific+2,'/'))){
/* Have hostname with path */
*ss=0;
url->hostinfo = (char*)DpsStrdup(url->specific + 2);
*ss='/';
url->path = (char*)DpsStrdup(ss);
}else{
/* Hostname without path */
if ((ss = strchr(url->specific + 2, '?'))) {
/* Have hostname with parameters */
*ss = 0;
url->hostinfo = (char*)DpsStrdup(url->specific + 2);
*ss='?';
url->path = (char*)DpsStrdup("/");
}else {
url->hostinfo = (char*)DpsStrdup(url->specific + 2);
url->path = (char*)DpsStrdup("/");
}
}
if((hostname=strchr(url->hostinfo,'@'))){
/* Username and password is given */
/* Store auth string user:password */
*hostname=0;
url->auth = (char*)DpsStrdup(url->hostinfo);
*hostname='@';
hostname++;
}else{
hostname = url->hostinfo;
}
/*
FIXME:
for(h=hostname;*h;h++){
if( *h>='A' && *h<='Z')
*h=(*h)-'A'+'a';
}
*/
if((ss=strchr(hostname,':'))){
*ss=0;
url->hostname = (char*)DpsStrdup(hostname);
*ss=':';
url->port=atoi(ss+1);
}else{
url->hostname = (char*)DpsStrdup(hostname);
url->port=0;
}
}else{
/* Have not host but have schema */
/* This is possible for: */
/* file: mailto: htdb: news: */
/* As far as we do not need mailto: just ignore it */
if(!strcasecmp(url->schema,"mailto")
|| !strcasecmp(url->schema,"javascript")
|| !strcasecmp(url->schema,"feed")
) {
DPS_FREE(s);
#ifdef WITH_PARANOIA
DpsViolationExit(-1, paran);
#endif
return(DPS_URL_BAD);
} else
if(!strcasecmp(url->schema,"file"))
url->path = (char*)DpsStrdup(url->specific);
else
if(!strcasecmp(url->schema,"exec"))
url->path = (char*)DpsStrdup(url->specific);
else
if(!strcasecmp(url->schema,"cgi"))
url->path = (char*)DpsStrdup(url->specific);
else
if(!strcasecmp(url->schema,"htdb"))
url->path = (char*)DpsStrdup(url->specific);
else
if(!strcasecmp(url->schema,"news")){
/* Now we will use localhost as NNTP */
/* server as it is not indicated in URL */
url->hostname = (char*)DpsStrdup("localhost");
url->path = (char*)DpsMalloc(dps_strlen(url->specific) + 2);
if (url->path == NULL) {
DPS_FREE(s);
#ifdef WITH_PARANOIA
DpsViolationExit(-1, paran);
#endif
return DPS_ERROR;
}
sprintf(url->path,"/%s",url->specific);
url->default_port=119;
}else{
/* Unknown strange schema */
DPS_FREE(s);
#ifdef WITH_PARANOIA
DpsViolationExit(-1, paran);
#endif
return(DPS_URL_BAD);
}
}
}else{
url->path = (char*)DpsStrdup(s);
}
/* Cat an anchor if exist */
if((anchor=strstr(url->path,"#")))*anchor=0;
/* If path is not full just copy it to filename */
/* i.e. neither /usr/local/ nor c:/windows/temp/ */
if((url->path != NULL) && (url->path[0]!='/') && (url->path[0]!='?') && (url->path[1]!=':')) {
/* Relative path */
if(!strncmp(url->path,"./",2))
url->filename = (char*)DpsStrdup(url->path + 2);
else
url->filename = (char*)DpsStrdup(url->path);
url->path[0] = 0;
}
/* truncate path to query_string */
/* and store query_string */
if((query=strrchr(url->path,'?'))){
url->query_string = (char*)DpsStrdup(query);
*(query) = 0;
}
DpsURLNormalizePath(url->path);
/* Now find right '/' sign and copy the rest to filename */
if((file=strrchr(url->path,'/'))&&(strcmp(file,"/"))){
url->filename = (char*)DpsStrdup(file + 1);
*(file+1)=0;
}
/* Now find right '/' sign and copy the rest to directory */
if ((file = strrchr(url->path,'/'))) {
char *p_save = file;
for(file--; (file > url->path) && (*file != '/'); file--);
file++;
if (*file) {
*p_save = '\0';
url->directory = (char*)DpsStrdup(file);
*p_save = '/';
}
}
DPS_FREE(s);
if (url->hostname != NULL) {
DpsRTrim(url->hostname, ".");
url->domain_level = 1;
for (s = url->hostname; *s; s++) {
*s = dps_tolower(*s);
if (*s == '.') url->domain_level++;
if (strchr(",'\";", (int)*s)) {
#ifdef WITH_PARANOIA
DpsViolationExit(-1, paran);
#endif
return DPS_URL_BAD;
}
}
}
if (url->hostinfo != NULL) {
DpsRTrim(url->hostinfo, ".");
s = strchr(url->hostinfo, '@');
for (s = (s == NULL) ? url->hostinfo : s + 1; *s; s++) *s = dps_tolower(*s);
}
if (url->schema != NULL) for (s = url->schema; *s; s++) *s = dps_tolower(*s);
/* fprintf(stderr, "url: .path: %s port:%d\n", url->path, url->port);*/
#ifdef WITH_PARANOIA
DpsViolationExit(-1, paran);
#endif
return DPS_OK;
}
void RelLink(DPS_AGENT *Indexer, DPS_URL *curURL, DPS_URL *newURL, char **str, int ReverseAliasFlag) {
const char *schema = newURL->schema ? newURL->schema : curURL->schema;
const char *hostname = newURL->hostname ? newURL->hostname : curURL->hostname;
const char *auth = newURL->auth ? newURL->auth : curURL->auth;
const char *path = (newURL->path && newURL->path[0]) ? newURL->path : curURL->path;
const char *fname = ((newURL->filename && newURL->filename[0]) || (newURL->path && newURL->path[0])) ?
newURL->filename : curURL->filename;
const char *query_string = newURL->query_string;
char *pathfile = (char*)DpsMalloc(dps_strlen(DPS_NULL2EMPTY(path)) + dps_strlen(DPS_NULL2EMPTY(fname)) +
dps_strlen(DPS_NULL2EMPTY(query_string)) + 5);
int cascade;
DPS_MATCH *Alias;
char *alias = NULL;
size_t aliassize, nparts = 10;
DPS_MATCH_PART Parts[10];
if (newURL->hostinfo == NULL) newURL->charset_id = curURL->charset_id;
if (pathfile == NULL) return;
/* sprintf(pathfile, "/%s%s%s", DPS_NULL2EMPTY(path), DPS_NULL2EMPTY(fname), DPS_NULL2EMPTY(query_string));*/
pathfile[0] = '/';
dps_strcpy(pathfile + 1, DPS_NULL2EMPTY(path)); dps_strcat(pathfile, DPS_NULL2EMPTY(fname)); dps_strcat(pathfile, DPS_NULL2EMPTY(query_string));
DpsURLNormalizePath(pathfile);
if (!strcasecmp(DPS_NULL2EMPTY(schema), "mailto")
|| !strcasecmp(DPS_NULL2EMPTY(schema), "javascript")
|| !strcasecmp(DPS_NULL2EMPTY(schema), "feed")
) {
*str = (char*)DpsMalloc(dps_strlen(DPS_NULL2EMPTY(schema)) + dps_strlen(DPS_NULL2EMPTY(newURL->specific)) + 4);
if (*str == NULL) return;
/* sprintf(*str, "%s:%s", DPS_NULL2EMPTY(schema), DPS_NULL2EMPTY(newURL->specific));*/
dps_strcpy(*str, DPS_NULL2EMPTY(schema)); dps_strcat(*str, ":"); dps_strcat(*str, DPS_NULL2EMPTY(newURL->specific));
} else if(/*!strcasecmp(DPS_NULL2EMPTY(schema), "file") ||*/ !strcasecmp(DPS_NULL2EMPTY(schema), "htdb")) {
*str = (char*)DpsMalloc(dps_strlen(DPS_NULL2EMPTY(schema)) + dps_strlen(pathfile) + 4);
if (*str == NULL) return;
/* sprintf(*str, "%s:%s", DPS_NULL2EMPTY(schema), pathfile);*/
dps_strcpy(*str, DPS_NULL2EMPTY(schema)); dps_strcat(*str, ":"); dps_strcat(*str, pathfile);
}else{
*str = (char*)DpsMalloc(dps_strlen(DPS_NULL2EMPTY(schema)) + dps_strlen(pathfile) + dps_strlen(DPS_NULL2EMPTY(hostname)) + dps_strlen(DPS_NULL2EMPTY(auth)) + 8);
if (*str == NULL) return;
/* sprintf(*str, "%s://%s%s", DPS_NULL2EMPTY(schema), DPS_NULL2EMPTY(hostinfo), pathfile);*/
dps_strcpy(*str, DPS_NULL2EMPTY(schema)); dps_strcat(*str, "://");
if (auth) {
dps_strcat(*str, auth); dps_strcat(*str,"@");
}
dps_strcat(*str, DPS_NULL2EMPTY(hostname)); dps_strcat(*str, pathfile);
}
if(!strncmp(*str, "ftp://", 6) && (strstr(*str, ";type=")))
*(strstr(*str, ";type")) = '\0';
DPS_FREE(pathfile);
if (ReverseAliasFlag) {
const char *alias_prog = DpsVarListFindStr(&Indexer->Vars, "ReverseAliasProg", NULL);
if (alias_prog) {
int result;
aliassize = 256 + 2 * dps_strlen(*str);
alias = (char*)DpsRealloc(alias, aliassize);
if (alias == NULL) {
DpsLog(Indexer, DPS_LOG_ERROR, "No memory (%d bytes). %s line %d", aliassize, __FILE__, __LINE__);
goto ret;
}
alias[0] = '\0';
result = DpsAliasProg(Indexer, alias_prog, *str, alias, aliassize - 1);
DpsLog(Indexer, DPS_LOG_EXTRA, "ReverseAliasProg result: '%s'", alias);
if(result != DPS_OK) goto ret;
DPS_FREE(*str);
*str = (char*)DpsStrdup(alias);
}
for(cascade = 0; ((Alias=DpsMatchListFind(&Indexer->Conf->ReverseAliases,*str,nparts,Parts))) && (cascade < 1024); cascade++) {
aliassize = dps_strlen(Alias->arg) + dps_strlen(Alias->pattern) + dps_strlen(*str) + 128;
alias = (char*)DpsRealloc(alias, aliassize);
if (alias == NULL) {
DpsLog(Indexer, DPS_LOG_ERROR, "No memory (%d bytes). %s line %d", aliassize, __FILE__, __LINE__);
goto ret;
}
DpsMatchApply(alias,aliassize,*str,Alias->arg,Alias,nparts,Parts);
if(alias[0]){
DpsLog(Indexer,DPS_LOG_DEBUG,"ReverseAlias%d: pattern:%s, arg:%s -> '%s'", cascade, Alias->pattern, Alias->arg, alias);
DPS_FREE(*str);
*str = (char*)DpsStrdup(alias);
} else break;
if (Alias->last) break;
}
}
ret:
DPS_FREE(alias);
}
void DpsParseHTTPResponse(DPS_AGENT *Indexer, DPS_DOCUMENT *Doc) {
char *token, *lt, *headers, savec;
int status, oldstatus;
DPS_DSTR header;
time_t now, last_mod_time;
Doc->Buf.content=NULL;
oldstatus = DpsVarListFindInt(&Doc->Sections, "Status", 0);
DpsVarListReplaceInt(&Doc->Sections, "ResponseSize", (int)Doc->Buf.size);
DpsVarListDel(&Doc->Sections, "Content-Length");
/* DpsVarListDel(&Doc->Sections, "Last-Modified");*/ /* if it's not deleted Lat-Modified equals to the first appearance in db */
if (Doc->Buf.buf == NULL) return;
/* Cut HTTP response header first */
for(token=Doc->Buf.buf;*token;token++){
if(!strncmp(token,"\r\n\r\n",4)){
if (token <= Doc->Buf.buf + Doc->Buf.size - 4) {
*token='\0';
Doc->Buf.content = token + 4;
}
break;
} else if(!strncmp(token,"\n\n",2)){
if (token <= Doc->Buf.buf + Doc->Buf.size - 2) {
*token='\0';
Doc->Buf.content = token + 2;
}
break;
}
}
/* Bad response, return */
if(!Doc->Buf.content) {
if (token <= Doc->Buf.buf + Doc->Buf.size - 4) {
if (token[2] == CR_CHAR) Doc->Buf.content = token + 4;
else Doc->Buf.content = token + 2;
}
}
/* Copy headers not to break them */
headers = (char*)DpsStrdup(Doc->Buf.buf);
/* Now lets parse response header lines */
token = dps_strtok_r(headers, "\r\n", <, &savec);
if(!token) {
DpsFree(headers);
return;
}
if(!strncmp(token,"HTTP/",5)){
status = atoi(token + 8);
DpsVarListReplaceStr(&Doc->Sections,"ResponseLine",token);
DpsVarListReplaceInt(&Doc->Sections, "Status", (oldstatus > status) ? oldstatus : status );
}else{
DpsFree(headers);
return;
}
token = dps_strtok_r(NULL, "\r\n", <, &savec);
DpsDSTRInit(&header, 128);
while(token){
if(strchr(token,':')) {
if (header.data_size) {
DpsParseHTTPHeader(Indexer, Doc, &header);
DpsDSTRFree(&header);
DpsDSTRInit(&header, 128);
}
}
DpsDSTRAppendStr(&header, token);
token = dps_strtok_r(NULL, "\r\n", <, &savec);
}
if (header.data_size) {
DpsParseHTTPHeader(Indexer, Doc, &header);
}
DpsDSTRFree(&header);
DPS_FREE(headers);
{
now = Indexer->now;
last_mod_time = DpsHttpDate2Time_t(DpsVarListFindStr(&Doc->Sections, "Last-Modified", ""));
if (last_mod_time > now + 3600 * 4) { /* we have a document with Last-Modified time in the future */
DpsLog(Indexer, DPS_LOG_EXTRA, "Last-Modified date is deep in future (%d>%d), dropping it.", last_mod_time, now);
DpsVarListDel(&Doc->Sections, "Last-Modified");
}
}
/* Bad response, return */
if(!Doc->Buf.content) {
return;
}
DpsVarListReplaceInt(&Doc->Sections,"Content-Length", Doc->Buf.buf-Doc->Buf.content+(int)Doc->Buf.size + DpsVarListFindInt(&Doc->Sections,"Content-Length", 0));
}
int DpsAddStackItem(DPS_AGENT *query, DPS_RESULT *Res, DPS_PREPARE_STATE *state, char *word, dpsunicode_t *uword) {
int origin;
size_t i;
size_t wlen = (uword == NULL) ? 0 : DpsUniLen(uword);
dpshash32_t crcword = (word == NULL) ? 0 : DpsStrHash32(word);
#ifdef DEBUG_BOOL
DpsLog(query, DPS_LOG_EXTRA, "0[%d].%x %c -- %s [%x] .secno:%d\n", state->order, state->origin, item_type(state->cmd),
(word == NULL) ? "<NULL>" : word, crcword, state->secno[state->p_secno]);
#endif
if((uword != NULL) && ( DpsStopListFind(&query->Conf->StopWords, uword, state->qlang) ||
(query->WordParam.min_word_len > wlen) ||
(query->WordParam.max_word_len < wlen)) ) {
origin = state->origin | DPS_WORD_ORIGIN_STOP;
} else {
origin = state->origin;
}
if (state->cmd == DPS_STACK_WORD && !(origin & DPS_WORD_ORIGIN_QUERY)) {
for (i = 0; i < Res->nitems; i++) {
if ((Res->items[i].order == state->order) && (Res->items[i].crcword == crcword)) return DPS_OK;
}
}
if (Res->nitems >= Res->mitems - 2) {
Res->mitems += DPS_MAXSTACK;
Res->items = (DPS_STACK_ITEM*)DpsRealloc(Res->items, Res->mitems * sizeof(DPS_STACK_ITEM));
if (Res->items == NULL) {
DpsLog(query, DPS_LOG_ERROR, "Can't alloc %d bytes for %d mitems", Res->mitems * sizeof(DPS_STACK_ITEM), Res->mitems);
return DPS_ERROR;
}
}
if (Res->nitems > 0) {
if (state->cmd == DPS_STACK_OR || state->cmd == DPS_STACK_AND || state->cmd == DPS_STACK_NEAR || state->cmd == DPS_STACK_ANYWORD) {
if (Res->items[Res->nitems-1].cmd == DPS_STACK_AND || Res->items[Res->nitems-1].cmd == DPS_STACK_OR
|| Res->items[Res->nitems-1].cmd == DPS_STACK_NEAR || Res->items[Res->nitems-1].cmd == DPS_STACK_ANYWORD) {
return DPS_OK;
}
}
if ((Res->nitems > 0) && (state->cmd == DPS_STACK_WORD)
&& (
(Res->items[Res->nitems-1].cmd == DPS_STACK_WORD)
|| (Res->items[Res->nitems-1].cmd == DPS_STACK_RIGHT)
|| (Res->items[Res->nitems-1].cmd == DPS_STACK_PHRASE_RIGHT)
)) {
Res->items[Res->nitems].cmd = DPS_STACK_OR;
Res->items[Res->nitems].order = 0;
Res->items[Res->nitems].origin = 0;
Res->items[Res->nitems].count = 0;
Res->items[Res->nitems].len = 0;
Res->items[Res->nitems].crcword = 0;
Res->items[Res->nitems].word = NULL;
Res->items[Res->nitems].ulen = 0;
Res->items[Res->nitems].uword = NULL;
Res->items[Res->nitems].pbegin = NULL;
Res->items[Res->nitems].order_origin = 0;
Res->items[Res->nitems].secno = state->secno[state->p_secno];
Res->nitems++;
Res->ncmds++;
#ifdef DEBUG_BOOL
DpsLog(query, DPS_LOG_EXTRA, "1[%d].%x %c -- %s", 0, 0, item_type(DPS_STACK_OR), "<NULL>");
#endif
}
if ((Res->nitems > 0) && (state->cmd == DPS_STACK_LEFT)
&& (
(Res->items[Res->nitems-1].cmd == DPS_STACK_RIGHT)
|| (Res->items[Res->nitems-1].cmd == DPS_STACK_PHRASE_RIGHT)
)) {
Res->items[Res->nitems].cmd = state->add_cmd;
Res->items[Res->nitems].order = 0;
Res->items[Res->nitems].origin = 0;
Res->items[Res->nitems].count = 0;
Res->items[Res->nitems].len = 0;
Res->items[Res->nitems].crcword = 0;
Res->items[Res->nitems].word = NULL;
Res->items[Res->nitems].ulen = 0;
Res->items[Res->nitems].uword = NULL;
Res->items[Res->nitems].pbegin = NULL;
Res->items[Res->nitems].order_origin = 0;
Res->items[Res->nitems].secno = state->secno[state->p_secno];
Res->nitems++;
Res->ncmds++;
#ifdef DEBUG_BOOL
DpsLog(query, DPS_LOG_EXTRA, "1[%d].%x %c -- %s", 0, 0, item_type(state->add_cmd), "<NULL>");
#endif
}
}
Res->items[Res->nitems].cmd = state->cmd;
Res->items[Res->nitems].order = state->order;
Res->items[Res->nitems].order_inquery = state->order_inquery;
Res->items[Res->nitems].origin = origin;
Res->items[Res->nitems].count = 0;
Res->items[Res->nitems].len = (word == NULL) ? 0 : dps_strlen(word);
Res->items[Res->nitems].crcword = crcword;
Res->items[Res->nitems].word = (word == NULL) ? NULL : DpsStrdup(word);
Res->items[Res->nitems].ulen = wlen;
Res->items[Res->nitems].uword = (uword == NULL) ? NULL : DpsUniDup(uword);
Res->items[Res->nitems].pbegin = NULL;
Res->items[Res->nitems].order_origin = 0;
Res->items[Res->nitems].wordnum = Res->nitems;
Res->items[Res->nitems].secno = state->secno[state->p_secno];
Res->nitems++;
if (state->cmd != DPS_STACK_WORD) {
Res->ncmds++;
} else {
Res->items[state->order].order_origin |= origin;
if (state->order > Res->max_order) Res->max_order = state->order;
if (state->order_inquery > Res->max_order_inquery) Res->max_order_inquery = state->order;
}
/* if ((state->cmd == DPS_STACK_WORD) && state->order > Res->max_order) Res->max_order = state->order;*/
#ifdef DEBUG_BOOL
DpsLog(query, DPS_LOG_EXTRA, "1[%d,%d].%x %c -- %s", state->order, state->order_inquery, state->origin, item_type(state->cmd),
(word == NULL) ? "<NULL>" : word);
#endif
return DPS_OK;
}
int main(int argc, char ** argv, char **envp) {
const char *env, *bcharset, *lcharset, *conf_dir;
char template_name[PATH_MAX+6]="";
char *template_filename = NULL;
char *query_string = NULL;
char self[1024]="";
char *url = NULL;
const char *ResultContentType;
int res,httpd=0;
size_t catcolumns = 0;
int page_size,page_number;
DPS_ENV *Env;
DPS_AGENT *Agent;
DPS_VARLIST query_vars;
/* Output Content-type if under HTTPD */
/* Some servers do not pass QUERY_STRING */
/* if the query was empty, so check */
/* REQUEST_METHOD too to be safe */
httpd=(getenv("QUERY_STRING")||getenv("REQUEST_METHOD"));
if (!(conf_dir=getenv("DPS_ETC_DIR")))
conf_dir=DPS_CONF_DIR;
DpsInit(argc, argv, envp);
Env=DpsEnvInit(NULL);
if (Env == NULL) {
if(httpd){
printf("Content-Type: text/plain\r\n\r\n");
}
printf("Can't alloc Env\n");
exit(0);
}
DpsVarListInit(&query_vars);
Agent = DpsAgentInit(NULL, Env, 0);
if (Agent == NULL) {
if(httpd){
printf("Content-Type: text/plain\r\n\r\n");
}
printf("Can't alloc Agent\n");
exit(0);
}
DpsVarListAddEnviron(&Env->Vars,"ENV");
/* Detect self and template name */
if((env = getenv("DPSEARCH_TEMPLATE")))
dps_strncpy(template_name, env, sizeof(template_name) - 1);
else if((env = getenv("PATH_INFO")) && env[0])
dps_strncpy(template_name, env + 1, sizeof(template_name) - 1);
if((env=getenv("DPSEARCH_SELF")))
dps_strncpy(self,env,sizeof(self)-1);
if((env=getenv("QUERY_STRING"))){
query_string = (char*)DpsRealloc(query_string, dps_strlen(env) + 2);
if (query_string == NULL) {
if(httpd){
printf("Content-Type: text/plain\r\n\r\n");
}
printf("Can't alloc query_string\n");
exit(0);
}
dps_strncpy(query_string, env, dps_strlen(env) + 1);
/* Hack for Russian Apache from apache.lexa.ru */
/* QUERY_STRING is already converted to server */
/* character set. We must print original query */
/* string instead however. Under usual apache */
/* we'll use QUERY_STRING. Note that query_vars */
/* list will contain not unescaped values, so */
/* we don't have to escape them when displaying */
env = getenv("CHARSET_SAVED_QUERY_STRING");
DpsParseQStringUnescaped(&query_vars,env?env:query_string);
/* Unescape and save variables from QUERY_STRING */
/* Env->Vars will have unescaped values however */
DpsParseQueryString(Agent,&Env->Vars,query_string);
template_filename = (char*)DpsStrdup(DpsVarListFindStr(&Env->Vars, "tmplt", ""));
if((env=getenv("REDIRECT_STATUS"))){
/* Check Apache internal redirect */
/* via "AddHandler" and "Action" */
if(!self[0]){
dps_strncpy(self,(env=getenv("REDIRECT_URL"))?env:"filler.cgi",sizeof(self)-1);
}
if(!template_name[0]){
dps_strncpy(template_name,(env=getenv("PATH_TRANSLATED"))?env:"",sizeof(template_name)-1);
}
if (*template_filename == '\0') {
DPS_FREE(template_filename);
template_filename = (char*)DpsStrdup("filler.htm");
}
}else{
/* CGI executed without Apache internal redirect */
/* Detect $Self variable with OS independant SLASHES */
if(!self[0]){
dps_strncpy(self,(env=getenv("SCRIPT_NAME"))?env:"filler.cgi",sizeof(self)-1);
}
if(!template_name[0]){
char *s,*e;
/*This is with OS specific SLASHES */
env=((env=getenv("SCRIPT_FILENAME"))?env:"filler.cgi");
if(strcmp(conf_dir,".")){
/* Take from the config directory */
dps_snprintf(template_name, sizeof(template_name)-1, "%s/%s",
conf_dir,(s=strrchr(env,DPSSLASH))?(s+1):(self));
}else{
/* Take from the current directory */
dps_strncpy(template_name,env,sizeof(template_name)-1);
}
/* Find right slash if it presents */
s=((s=strrchr(template_name,DPSSLASH))?s:template_name);
if (*template_filename == '\0') {
/* Find .cgi substring */
if ((e = strstr(s, ".cgi")) != NULL) {
/* Replace ".cgi" with ".htm" */
e[1]='h';e[2]='t';e[3]='m';
} else {
dps_strcat(s, ".htm");
}
e = strrchr(s, '/');
DPS_FREE(template_filename);
template_filename = (char*)DpsStrdup(e + 1);
} else {
dps_strncpy(s + 1, template_filename, sizeof(template_name) - (s - template_name) - 2);
}
}
}
}else{
/* Executed from command line */
/* or under server which does not */
/* pass an empty QUERY_STRING var */
if(argv[1]) {
query_string = (char*)DpsRealloc(query_string, dps_strlen(argv[1]) + 10);
if (query_string == NULL) {
if(httpd){
printf("Content-Type: text/plain\r\n\r\n");
}
printf("Can't realloc query_string\n");
exit(0);
}
sprintf(query_string, "q=%s", argv[1]);
} else {
query_string = (char*)DpsRealloc(query_string, 1024);
if (query_string == NULL) {
if(httpd){
printf("Content-Type: text/plain\r\n\r\n");
}
printf("Can't realloc query_string\n");
exit(0);
}
sprintf(query_string, "q=");
}
/* Hack for Russian Apache from apache.lexa.ru */
/* QUERY_STRING is already converted to server */
/* character set. We must print original query */
/* string instead however. Under usual apache */
/* we'll use QUERY_STRING. Note that query_vars */
/* list will contain not unescaped values, so */
/* we don't have to escape them when displaying */
env = getenv("CHARSET_SAVED_QUERY_STRING");
DpsParseQStringUnescaped(&query_vars,env?env:query_string);
/* Unescape and save variables from QUERY_STRING */
/* Env->Vars will have unescaped values however */
DpsParseQueryString(Agent,&Env->Vars,query_string);
DPS_FREE(template_filename);
template_filename = (char*)DpsStrdup(DpsVarListFindStr(&Env->Vars, "tmplt", ""));
if (*template_filename == '\0') {
DPS_FREE(template_filename); template_filename = (char*)DpsStrdup("filler.htm");
}
/*// Get template name from command line variable &tmplt */
if(!template_name[0])
dps_snprintf(template_name,sizeof(template_name),"%s/%s", conf_dir, template_filename);
}
DpsVarListReplaceStr(&Agent->Conf->Vars, "tmplt", template_filename);
DPS_FREE(template_filename);
Agent->tmpl.Env_Vars = &Env->Vars;
DpsURLNormalizePath(template_name);
if (strncmp(template_name, conf_dir, dps_strlen(conf_dir))
|| (res = DpsTemplateLoad(Agent, Env, &Agent->tmpl, template_name))) {
if (strcmp(template_name, "filler.htm")) { /* trying load default template */
fprintf(stderr, "Can't load template: '%s' %s\n", template_name, Env->errstr);
DPS_FREE(template_filename);
template_filename = (char*)DpsStrdup("filler.htm");
dps_snprintf(template_name, sizeof(template_name), "%s/%s", conf_dir, template_filename);
if ((res = DpsTemplateLoad(Agent, Env, &Agent->tmpl, template_name))) {
if(httpd)printf("Content-Type: text/plain\r\n\r\n");
printf("%s\n",Env->errstr);
DpsVarListFree(&query_vars);
DpsEnvFree(Env);
DPS_FREE(query_string);
DpsAgentFree(Agent);
return(0);
}
} else {
if(httpd)printf("Content-Type: text/plain\r\n\r\n");
printf("%s\n",Env->errstr);
DpsVarListFree(&query_vars);
DpsEnvFree(Env);
DPS_FREE(query_string);
DpsAgentFree(Agent);
return(0);
}
}
/* set locale if specified */
if ((url = DpsVarListFindStr(&Env->Vars, "Locale", NULL)) != NULL) {
setlocale(LC_ALL, url);
/*#ifdef HAVE_ASPELL*/
{ char *p;
if ((p = strchr(url, '.')) != NULL) {
*p = '\0';
DpsVarListReplaceStr(&Env->Vars, "g-lc", url);
*p = '.';
}
}
/*#endif*/
url = NULL;
}
/* Call again to load search Limits if need */
DpsParseQueryString(Agent, &Env->Vars, query_string);
Agent->Flags = Env->Flags;
Agent->flags |= DPS_FLAG_UNOCON;
Env->flags |= DPS_FLAG_UNOCON;
DpsSetLogLevel(NULL, DpsVarListFindInt(&Env->Vars, "LogLevel", 0));
DpsOpenLog("filler.cgi", Env, !strcasecmp(DpsVarListFindStr(&Env->Vars, "Log2stderr", (!httpd) ? "yes" : "no"), "yes"));
DpsLog(Agent,DPS_LOG_ERROR,"filler.cgi started with '%s'",template_name);
DpsLog(Agent, DPS_LOG_DEBUG, "VarDir: '%s'", DpsVarListFindStr(&Agent->Conf->Vars, "VarDir", DPS_VAR_DIR));
DpsLog(Agent, DPS_LOG_DEBUG, "Affixes: %d, Spells: %d, Synonyms: %d, Acronyms: %d, Stopwords: %d",
Env->Affixes.naffixes,Env->Spells.nspell,
Env->Synonyms.nsynonyms,
Env->Acronyms.nacronyms,
Env->StopWords.nstopwords);
DpsLog(Agent, DPS_LOG_DEBUG, "Chinese dictionary with %d entries", Env->Chi.nwords);
DpsLog(Agent, DPS_LOG_DEBUG, "Korean dictionary with %d entries", Env->Korean.nwords);
DpsLog(Agent, DPS_LOG_DEBUG, "Thai dictionary with %d entries", Env->Thai.nwords);
DpsVarListAddLst(&Agent->Vars, &Env->Vars, NULL, "*");
Agent->tmpl.Env_Vars = &Agent->Vars;
/* DpsVarListAddEnviron(&Agent->Vars, "ENV");*/
/****************************************************************************************************************************************/
/* This is for query tracking */
DpsVarListAddStr(&Agent->Vars, "QUERY_STRING", query_string);
DpsVarListAddStr(&Agent->Vars, "self", self);
env = getenv("HTTP_X_FORWARDER_FOR");
if (env) {
DpsVarListAddStr(&Agent->Vars, "IP", env);
} else {
env = getenv("REMOTE_ADDR");
DpsVarListAddStr(&Agent->Vars, "IP", env ? env : "localhost");
}
bcharset = DpsVarListFindStr(&Agent->Vars, "BrowserCharset", "iso-8859-1");
Env->bcs=DpsGetCharSet(bcharset);
lcharset = DpsVarListFindStr(&Agent->Vars, "LocalCharset", "iso-8859-1");
Env->lcs=DpsGetCharSet(lcharset);
ResultContentType = DpsVarListFindStr(&Agent->Vars, "ResultContentType", "text/html");
if(httpd){
if(!Env->bcs){
printf("Content-Type: text/plain\r\n\r\n");
printf("Unknown BrowserCharset '%s' in template '%s'\n",bcharset,template_name);
exit(0);
}else if(!Env->lcs){
printf("Content-Type: text/plain\r\n\r\n");
printf("Unknown LocalCharset '%s' in template '%s'\n",lcharset,template_name);
exit(0);
}else{
printf("Content-type: %s; charset=%s\r\n\r\n", ResultContentType, bcharset);
}
}else{
if(!Env->bcs){
printf("Unknown BrowserCharset '%s' in template '%s'\n",bcharset,template_name);
exit(0);
}
if(!Env->lcs){
printf("Unknown LocalCharset '%s' in template '%s'\n",lcharset,template_name);
exit(0);
}
}
/* These parameters taken from "variable section of template"*/
res = DpsVarListFindInt(&Agent->Vars, "ps", DPS_DEFAULT_PS);
page_size = dps_min(res, MAX_PS);
page_number = DpsVarListFindInt(&Agent->Vars, "p", 0);
if (page_number == 0) {
page_number = DpsVarListFindInt(&Agent->Vars, "np", 0);
DpsVarListReplaceInt(&Agent->Vars, "p", page_number + 1);
} else page_number--;
res = DpsVarListFindInt(&Agent->Vars, "np", 0) * page_size;
DpsVarListAddInt(&Agent->Vars, "pn", res);
catcolumns = (size_t)atoi(DpsVarListFindStr(&Agent->Vars, "CatColumns", ""));
DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "top");
DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "restop");
DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "res");
DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "resbot");
DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "bottom");
DpsVarListFree(&query_vars);
DpsAgentFree(Agent);
DpsEnvFree(Env);
DPS_FREE(query_string);
DPS_FREE(url);
if (httpd) fflush(NULL); else fclose(stdout);
#ifdef EFENCE
fprintf(stderr, "Memory leaks checking\n");
DpsEfenceCheckLeaks();
#endif
#ifdef FILENCE
fprintf(stderr, "FD leaks checking\n");
DpsFilenceCheckLeaks(NULL);
#endif
return DPS_OK;
}
int DpsCookiesAdd(DPS_AGENT *Indexer, const char *domain, const char * path, const char *name, const char *value, const char secure,
dps_uint4 expires, const char from_config, int insert_flag) {
#ifdef HAVE_SQL
char buf[3*PATH_MAX];
char path_esc[2*PATH_MAX+1];
DPS_COOKIES *Cookies = &Indexer->Cookies;
DPS_COOKIE *Coo;
DPS_DB *db;
dpshash32_t url_id = DpsStrHash32(domain);
size_t i;
#ifdef WITH_PARANOIA
void *paran = DpsViolationEnter(paran);
#endif
if (Indexer->flags & DPS_FLAG_UNOCON) {
if (Indexer->Conf->dbl.nitems == 0) return DPS_OK;
DPS_GETLOCK(Indexer, DPS_LOCK_DB);
db = Indexer->Conf->dbl.db[url_id % Indexer->Conf->dbl.nitems];
} else {
if (Indexer->dbl.nitems == 0) return DPS_OK;
db = Indexer->dbl.db[url_id % Indexer->dbl.nitems];
}
(void)DpsDBEscStr(db, path_esc, DPS_NULL2EMPTY(path), dps_min(PATH_MAX,dps_strlen(DPS_NULL2EMPTY(path))));
for (i = 0; i < Cookies->ncookies; i++) {
Coo = &Cookies->Cookie[i];
if (!strcasecmp(Coo->domain, domain) && !strcasecmp(Coo->path, DPS_NULL2EMPTY(path)) && !strcasecmp(Coo->name, name) && (Coo->secure == secure)/* && (Coo->from_config == from_config)*/ ) {
DPS_FREE(Coo->value);
Coo->value = DpsStrdup(value);
/* Coo->expires = expires;*/
if (insert_flag) {
dps_snprintf(buf, sizeof(buf), "UPDATE cookies SET value='%s',expires=%d WHERE domain='%s' AND path='%s' AND name='%s' AND secure='%c'",
value, expires, domain, path_esc, name, secure);
DpsSQLAsyncQuery(db, NULL, buf);
}
if (Indexer->flags & DPS_FLAG_UNOCON) DPS_RELEASELOCK(Indexer, DPS_LOCK_DB);
#ifdef WITH_PARANOIA
DpsViolationExit(Indexer->handle, paran);
#endif
return DPS_OK;
}
}
Cookies->Cookie = (DPS_COOKIE*)DpsRealloc(Cookies->Cookie, (Cookies->ncookies + 1) * sizeof(DPS_COOKIE));
if(Cookies->Cookie == NULL) {
Cookies->ncookies = 0;
if (Indexer->flags & DPS_FLAG_UNOCON) DPS_RELEASELOCK(Indexer, DPS_LOCK_DB);
#ifdef WITH_PARANOIA
DpsViolationExit(Indexer->handle, paran);
#endif
return DPS_ERROR;
}
Coo = &Cookies->Cookie[Cookies->ncookies];
/* Coo->expires = expires;*/
Coo->secure = secure;
Coo->from_config = from_config;
Coo->domain = DpsStrdup(domain);
Coo->path = DpsStrdup(path);
Coo->name = DpsStrdup(name);
Coo->value = DpsStrdup(value);
if (insert_flag) {
if (Indexer->Flags.CheckInsertSQL) {
dps_snprintf(buf, sizeof(buf), "DELETE FROM cookies WHERE domain='%s' AND path='%s' AND name='%s' AND secure='%c'",
domain, path_esc, name, secure);
DpsSQLAsyncQuery(db, NULL, buf);
}
dps_snprintf(buf, sizeof(buf), "INSERT INTO cookies(expires,secure,domain,path,name,value)VALUES(%d,'%c','%s','%s','%s','%s')",
expires, secure, domain, path_esc, name, value);
DpsSQLAsyncQuery(db, NULL, buf);
}
Cookies->ncookies++;
if (Indexer->flags & DPS_FLAG_UNOCON) DPS_RELEASELOCK(Indexer, DPS_LOCK_DB);
#ifdef WITH_PARANOIA
DpsViolationExit(Indexer->handle, paran);
#endif
#endif /*HAVE_SQL*/
return DPS_OK;
}
void DpsCookiesFind(DPS_AGENT *Indexer, DPS_SERVER *Server, DPS_DOCUMENT *Doc, const char *hostinfo) {
#ifdef HAVE_SQL
DPS_DSTR cookie;
DPS_COOKIES *Cookies = &Indexer->Cookies;
DPS_COOKIE *Coo;
size_t i, blen = dps_strlen(hostinfo), slen;
int have_no_cookies = DpsVarListFindInt(&Doc->Sections, "have_no_cookies", 1);
#ifdef WITH_PARANOIA
void *paran = DpsViolationEnter(paran);
#endif
TRACE_IN(Indexer, "DpsCookiesFind");
DpsDSTRInit(&cookie, 1024);
for(i = 0; i < Cookies->ncookies; i++) {
Coo = &Cookies->Cookie[i];
slen = dps_strlen(Coo->domain);
if (slen > blen) continue;
if (Coo->secure == 'y' && strcasecmp(Doc->CurURL.schema, "https")) continue;
if (strncasecmp(Coo->path, Doc->CurURL.path, dps_strlen(Coo->path))) continue;
if (strcasecmp(Coo->domain, hostinfo + (blen - slen))) continue;
if (Coo->from_config != 1) have_no_cookies = 0;
if (Coo->name[0] == '\0' && Coo->value[0] == '\0') continue;
if (cookie.data_size)
DpsDSTRAppend(&cookie, "; ", 2);
DpsDSTRAppendStr(&cookie, Coo->name);
DpsDSTRAppend(&cookie, "=", 1);
DpsDSTRAppendStr(&cookie, Coo->value);
}
if (have_no_cookies) {
char buf[2*PATH_MAX];
dpshash32_t url_id;
DPS_DB *db;
DPS_SQLRES Res;
size_t rows;
int rc;
if (Server != NULL) {
char *PingData = DpsVarListFindStr(&Server->Vars, "AuthPing", NULL);
if (PingData != NULL) {
char *AuthPing = DpsStrdup(DpsTrim(PingData, " \t\r\n"));
int method = DPS_METHOD_GET;
dps_base64_decode(AuthPing, PingData, dps_strlen(PingData));
if (!strncasecmp(AuthPing, "GET", 3)) {
method = DPS_METHOD_GET;
PingData = DpsTrim(AuthPing + 3, " \t\r\n");
} else if (!strncasecmp(AuthPing, "POST", 4)) {
method = DPS_METHOD_POST;
PingData = DpsTrim(AuthPing + 4, " \t\r\n");
} else {
DpsLog(Indexer, DPS_LOG_ERROR, "AuthPing should be GET or POST: %s", AuthPing);
PingData = NULL;
}
if (PingData != NULL) {
size_t size = dps_strlen(PingData);
{
char PingURL[size + 2];
char PingBody[size];
DPS_DOCUMENT *rDoc;
int result;
rDoc = DpsDocInit(NULL);
DpsSpiderParamInit(&rDoc->Spider);
DpsVarList2Doc(rDoc, Server);
rDoc->Buf.max_size = (size_t)DpsVarListFindInt(&Indexer->Vars, "MaxDocSize", DPS_MAXDOCSIZE);
rDoc->Buf.allocated_size = DPS_NET_BUF_SIZE;
if ((rDoc->Buf.buf = (char*)DpsMalloc(rDoc->Buf.allocated_size + 1)) == NULL) {
DpsDocFree(rDoc);
TRACE_OUT(Indexer);
return;
}
rDoc->Buf.buf[0]='\0';
rDoc->subdoc = Indexer->Flags.SubDocLevel + 1;
#if 1
dps_snprintf(buf, sizeof(buf), "%s://%s/", DPS_NULL2EMPTY(Doc->CurURL.schema), DPS_NULL2EMPTY(Doc->CurURL.hostinfo));
DpsVarListReplaceStr(&rDoc->Sections, "URL", buf);
DpsURLParse(&rDoc->CurURL, buf);
DpsLog(Indexer, DPS_LOG_INFO, "HOME: %s", buf);
rDoc->method = DPS_METHOD_HEAD;
/* DpsVarListFree(&rDoc->RequestHeaders);*/
if (Doc != NULL) {
DpsVarListReplaceLst(&rDoc->RequestHeaders, &Doc->RequestHeaders, NULL, "*");
}
DpsVarListReplaceStr(&rDoc->Sections, "have_no_cookies", "0");
DpsDocAddDocExtraHeaders(Indexer, Server, rDoc);
DpsDocAddConfExtraHeaders(Indexer->Conf, rDoc);
DpsVarListReplaceLst(&rDoc->Sections, &Server->Vars, NULL, "*");
DpsDocAddServExtraHeaders(Server, rDoc);
DpsVarListLog(Indexer, &rDoc->RequestHeaders, DPS_LOG_DEBUG, "HOME.Request");
if (Doc == NULL || Indexer->Flags.cmd == DPS_IND_FILTER) {
DpsDocLookupConn(Indexer, rDoc);
} else {
DPS_FREE(rDoc->connp.connp);
rDoc->connp = Doc->connp;
}
result = DpsGetURL(Indexer, rDoc, NULL); /* Just get headers from the home as we need only Cookies from it */
DpsDocProcessResponseHeaders(Indexer, rDoc);
DpsVarListLog(Indexer, &rDoc->Sections, DPS_LOG_DEBUG, "HOME.Response");
#endif
sscanf(PingData, "%s %s", PingURL, PingBody);
if (rDoc->method == DPS_METHOD_GET) {
dps_strcat(PingURL, "?");
dps_strcat(PingURL, PingBody);
} else {
DpsVarListReplaceStr(&rDoc->Sections, "body", PingBody);
}
DpsVarListReplaceStr(&rDoc->Sections, "URL", PingURL);
DpsURLParse(&rDoc->CurURL, PingURL);
DpsLog(Indexer, DPS_LOG_INFO, "AUTH.PING: %s", PingURL);
rDoc->method = method;
DpsVarListFree(&rDoc->RequestHeaders);
DpsVarListReplaceStr(&rDoc->Sections, "have_no_cookies", "0");
DpsDocAddDocExtraHeaders(Indexer, Server, rDoc);
DpsDocAddConfExtraHeaders(Indexer->Conf, rDoc);
DpsVarListReplaceLst(&rDoc->Sections, &Server->Vars, NULL, "*");
DpsDocAddServExtraHeaders(Server, rDoc);
if (method == DPS_METHOD_POST) {
dps_snprintf(buf, sizeof(buf), "application/x-www-form-urlencoded; charset=%s", DpsVarListFindStr(&Indexer->Conf->Vars, "LocalCharset", "iso-8859-1"));
DpsVarListReplaceStr(&rDoc->RequestHeaders, "Content-Type", buf);
dps_snprintf(buf, sizeof(buf), "%d", dps_strlen(PingBody));
DpsVarListReplaceStr(&rDoc->RequestHeaders, "Content-Length", buf);
}
DpsVarListLog(Indexer, &rDoc->RequestHeaders, DPS_LOG_DEBUG, "AUTHPING.Request");
#if 0
if (Doc == NULL || Indexer->Flags.cmd == DPS_IND_FILTER) {
DpsDocLookupConn(Indexer, rDoc);
} else {
DPS_FREE(rDoc->connp.connp);
rDoc->connp = Doc->connp;
}
#endif
result = DpsGetURL(Indexer, rDoc, NULL); /* Just get it as we need only Cookies from the headers */
DpsDocProcessResponseHeaders(Indexer, rDoc);
DpsVarListDel(&rDoc->Sections, "body");
DpsVarListLog(Indexer, &rDoc->Sections, DPS_LOG_DEBUG, "AUTHPING.Response");
if (Doc != NULL) bzero(&rDoc->connp, sizeof(rDoc->connp));
DpsDocFree(rDoc);
}
}
DpsFree(AuthPing);
}
}
while(hostinfo != NULL) {
url_id = DpsStrHash32(hostinfo);
DpsSQLResInit(&Res);
dps_snprintf(buf, sizeof(buf), "SELECT name,value,path,secure FROM cookies WHERE domain='%s'", hostinfo);
if (Indexer->flags & DPS_FLAG_UNOCON) {
DPS_GETLOCK(Indexer, DPS_LOCK_DB);
db = Indexer->Conf->dbl.db[url_id % Indexer->Conf->dbl.nitems];
} else {
db = Indexer->dbl.db[url_id % Indexer->dbl.nitems];
}
if(DPS_OK == (rc = DpsSQLQuery(db, &Res, buf))) {
rows = DpsSQLNumRows(&Res);
for(i = 0; i < rows; i++) {
DpsCookiesAdd(Indexer, hostinfo, DpsSQLValue(&Res, i, 2), DpsSQLValue(&Res, i, 0), DpsSQLValue(&Res, i, 1),
*DpsSQLValue(&Res, i, 3), 0, 0, 0);
if (*DpsSQLValue(&Res, i, 3) == 'y' && strcasecmp(Doc->CurURL.schema, "https")) continue;
if (strncasecmp(DpsSQLValue(&Res, i, 2), Doc->CurURL.path, dps_strlen(DpsSQLValue(&Res, i, 2)))) continue;
if (cookie.data_size)
DpsDSTRAppend(&cookie, "; ", 2);
DpsDSTRAppendStr(&cookie, DpsSQLValue(&Res, i, 0));
DpsDSTRAppend(&cookie, "=", 1);
DpsDSTRAppendStr(&cookie, DpsSQLValue(&Res, i, 1));
}
if (rows == 0) {
DpsCookiesAdd(Indexer, hostinfo, "/", "", "", 'n', 0, 0, 0);
}
}
DpsSQLFree(&Res);
if (Indexer->flags & DPS_FLAG_UNOCON) {
DPS_RELEASELOCK(Indexer, DPS_LOCK_DB);
}
hostinfo = strchr(hostinfo, '.');
if (hostinfo != NULL) hostinfo++;
}
}
if (cookie.data_size) {
DpsVarListReplaceStr(&Doc->RequestHeaders, "Cookie", cookie.data);
}
DpsDSTRFree(&cookie);
#endif
TRACE_OUT(Indexer);
return;
}
