__C_LINK char * __DPSCALL DpsSGMLUnescape(char * str){
char *s = str,*e = str, c, z;
/*****************/
while(*s){
if(*s=='&'){
if(*(s+1)=='#'){
for(e=s+2;(e-s<DPS_MAX_SGML_LEN)&&(*e<='9')&&(*e>='0');e++);
if(*e==';'){
int v=atoi(s+2);
if(v>=0&&v<=255) {
*s=(char)v;
} else {
*s = ' ';
}
dps_memmove(s+1, e+1, dps_strlen(e + 1) + 1);
}
}else{
for(e=s+1;(e-s<DPS_MAX_SGML_LEN)&&(((*e<='z')&&(*e>='a'))||((*e<='Z')&&(*e>='A')));e++);
z = *e;
*e = '\0';
if( (z == ';') && (c=(char)DpsSgmlToUni(s+1))){
*s=c;
dps_memmove(s+1, e+1, dps_strlen(e + 1) + 1);
}
if (z != ';') *e = z;
else s++;
}
}
s++;
}
return(str);
}
int Dps_ftp_connect(DPS_AGENT *Agent, DPS_CONN *connp, char *hostname, int port, char *user, char *passwd, int timeout) {
size_t len;
if (!connp)
return -1;
if (connp->connected == DPS_NET_CONNECTED)
Dps_ftp_close(connp);
connp->connected = DPS_NET_NOTCONNECTED;
if (!port) {
connp->port = 21;
connp->connp->port = 20;
} else {
connp->port = port;
connp->connp->port = port - 1;
}
connp->timeout = timeout;
if (!hostname)
return -1;
len = dps_strlen(hostname);
connp->hostname = DpsXrealloc(connp->hostname, len+1);
if (connp->hostname == NULL) return -1;
dps_snprintf(connp->hostname, len+1, "%s", hostname);
if (Dps_ftp_open_control_port(Agent, connp))
return -1;
if (Dps_ftp_login(connp, user, passwd))
return -1;
Dps_ftp_set_binary(connp);
connp->connected = DPS_NET_CONNECTED;
return 0;
}
static int open_socket(DPS_AGENT *A, char *unix_socket) {
char unix_path[128];
struct sockaddr_un unix_addr;
int sockfd, saddrlen;
if (DpsRelVarName(A->Conf, unix_path, sizeof(unix_path), unix_socket) < 105) {
} else {
DpsLog(A, DPS_LOG_ERROR, "Unix socket name '%s' is too large", unix_path);
return(DPS_NET_CANT_CONNECT);
}
if ((sockfd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) {
DpsLog(A, DPS_LOG_ERROR, "unix socket() error %d", errno);
return(DPS_NET_CANT_CONNECT);
}
DpsSockOpt(A, sockfd);
bzero((void*)&unix_addr, sizeof(unix_addr));
unix_addr.sun_family = AF_UNIX;
dps_strncpy(unix_addr.sun_path, unix_path, sizeof(unix_addr.sun_path));
saddrlen = sizeof(unix_addr.sun_family) + dps_strlen(unix_addr.sun_path);
if(connect(sockfd, (struct sockaddr *)&unix_addr, sizeof (unix_addr))) {
dps_strerror(A, DPS_LOG_ERROR, "unix socket '%s' connect() error", unix_path);
return(DPS_NET_CANT_CONNECT);
}
return sockfd;
}
int socket_read_line( DPS_CONN *connp ){
size_t num_read_total, buf_size;
buf_size = DPS_NET_BUF_SIZE;
num_read_total = 0;
if (connp->buf)
DPS_FREE(connp->buf);
connp->buf_len_total = 0;
connp->buf_len = 0;
while(1){
if (connp->buf_len_total <= num_read_total+DPS_NET_BUF_SIZE) {
connp->buf_len_total += DPS_NET_BUF_SIZE;
connp->buf = DpsXrealloc(connp->buf, (size_t)(connp->buf_len_total +1));
if (connp->buf == NULL) return -1;
}
/* if(!recv(connp->conn_fd,&(connp->buf[num_read_total]),1,0))
return -1;*/
if(read(connp->conn_fd, &(connp->buf[num_read_total]), 1) <= 0)
return -1;
if (connp->buf[num_read_total] == NL_CHAR)
break;
else if (connp->buf[num_read_total] == 0)
break;
num_read_total++;
}
connp->buf_len = dps_strlen(connp->buf);
return num_read_total;
}
int Dps_ftp_send_cmd(DPS_CONN *connp, const char *cmd) {
char *buf;
size_t len;
connp->err = 0;
len = dps_strlen(cmd)+2;
buf = DpsXmalloc(len+1);
if (buf == NULL) return -1;
dps_snprintf(buf, len+1, "%s\r\n", cmd);
socket_buf_clear(connp);
if (socket_write(connp, buf)){
DPS_FREE( buf);
return -1;
}
#ifdef DEBUG_FTP
fprintf(stderr, "ftp://%s (cmd) : %s", connp->hostname, buf);
/*DpsLog(connp->indexer, DPS_LOG_DEBUG, "ftp://%s (cmd) : %s", connp->hostname, buf);*/
#endif
DPS_FREE(buf);
if (Dps_ftp_read_line(connp))
return -1;
#ifdef DEBUG_FTP
fprintf(stderr, "ftp://%s (reply): %s", connp->hostname, connp->buf);
/*DpsLog(connp->indexer, DPS_LOG_DEBUG, "ftp://%s (reply): %s", connp->hostname, connp->buf);*/
#endif
return(Dps_ftp_get_reply(connp));
}
int Dps_ftp_login(DPS_CONN *connp, char *user, char *passwd){
char *buf;
char user_tmp[32], passwd_tmp[64];
int code;
size_t len;
DPS_FREE(connp->user);
DPS_FREE(connp->pass);
if (!user)
dps_snprintf(user_tmp, 32, "anonymous");
else {
dps_snprintf(user_tmp, 32, "%s", user);
connp->user = (char*)DpsStrdup(user);
}
if (!passwd)
dps_snprintf(passwd_tmp, 64, "*****@*****.**", PACKAGE, VERSION);
else {
dps_snprintf(passwd_tmp, 32, "%s", passwd);
connp->pass = (char*)DpsStrdup(passwd);
}
len = dps_strlen(user_tmp) + 5 /*dps_strlen("USER ")*/;
buf = (char *) DpsXmalloc(len+1);
if (buf == NULL) return -1;
dps_snprintf(buf, len+1, "USER %s", user_tmp);
code = Dps_ftp_send_cmd(connp, buf);
DPS_FREE(buf);
if (code == -1)
return -1;
else if (code == 2) /* Don't need password */
return 0;
len = dps_strlen(passwd_tmp) + 5 /*dps_strlen("PASS ")*/;
buf = (char *) DpsXmalloc(len+1);
if (buf == NULL) return -1;
dps_snprintf(buf, len+1, "PASS %s", passwd_tmp);
code = Dps_ftp_send_cmd( connp, buf);
DPS_FREE(buf);
if (code > 3)
return -1;
return 0;
}
extern C_LINKAGE char * _DpsStrdup(const char *str, const char *filename, size_t fileline) {
size_t len;
char *copy;
len = dps_strlen(str) + 1;
if ((copy = _DpsMalloc(len, filename, fileline)) == NULL)
return (NULL);
dps_memcpy(copy, str, len); /* was: dps_memmove */
return (copy);
}
int socket_write(DPS_CONN *connp, const char *buf){
if (socket_select(connp, DPS_NET_READ_TIMEOUT, 'w') == -1)
return -1;
if (DpsSend(connp->conn_fd, buf, dps_strlen(buf), 0) == -1){
connp->err = DPS_NET_ERROR;
return -1;
}
return 0;
}
static int DpsSearchdSendWordRequest(DPS_AGENT *query, const DPS_DB *cl, const char *q) {
DPS_SEARCHD_PACKET_HEADER hdr;
ssize_t nsent;
size_t nitems = (query->flags & DPS_FLAG_UNOCON) ? query->Conf->dbl.nitems : query->dbl.nitems;
TRACE_IN(query, "DpsSearchdSendWordRequest");
hdr.cmd = (nitems > 1) ? DPS_SEARCHD_CMD_WORDS_ALL : DPS_SEARCHD_CMD_WORDS;
hdr.len = dps_strlen(q);
nsent = DpsSearchdSendPacket(cl->searchd, &hdr, q);
TRACE_OUT(query);
return DPS_OK;
}
int Dps_ftp_get(DPS_CONN *c, DPS_CONN *d, char *path, size_t max_doc_size){
char *cmd;
size_t len;
if (!path)
return -1;
len = dps_strlen(path) + 16;
cmd = DpsXmalloc(len+1);
if (cmd == NULL) return -1;
dps_snprintf(cmd, len+1, "RETR %s", path);
if (Dps_ftp_send_data_cmd(c, d, cmd, max_doc_size) == -1 && d->err != DPS_NET_FILE_TL) {
DPS_FREE(cmd);
return -1;
}
DPS_FREE(cmd);
return 0;
}
int Dps_ftp_mdtm(DPS_CONN *c, char *path){
char *cmd;
int code;
size_t len;
if (!path)
return -1;
len = dps_strlen(path) + 16;
cmd = DpsXmalloc(len+1);
if (cmd == NULL) return -1;
dps_snprintf(cmd, len+1, "MDTM %s", path);
code = Dps_ftp_send_cmd(c, cmd);
DPS_FREE(cmd);
if (code == -1){
return -1;
}else if ( code >3 ){
c->err = code;
return -1;
}
return (DpsFTPDate2Time_t(c->buf));
}
int Dps_ftp_list(DPS_CONN *c, DPS_CONN *d, char *path, char *filename, size_t max_doc_size){
char *cmd;
size_t len;
if (!filename){
cmd = DpsXmalloc(16);
if (cmd == NULL) return -1;
sprintf(cmd, "LIST");
}else{
len = dps_strlen(filename) + 16;
cmd = DpsXmalloc(len+1);
if (cmd == NULL) return -1;
dps_snprintf(cmd, len+1, "LIST %s", filename);
}
if (Dps_ftp_send_data_cmd(c, d, cmd, max_doc_size)== -1) {
DPS_FREE(cmd);
/*DpsLog(c->indexer, DPS_LOG_DEBUG, "(ftp_list-err)->%s", d->buf);*/
return -1;
}
DPS_FREE(cmd);
return ftp_parse_list(d, path);
}
ssize_t Dps_ftp_size(DPS_CONN *c, char *path) {
char *cmd;
int code;
size_t len;
if (!path)
return -1;
len = dps_strlen(path) + 16;
cmd = DpsXmalloc(len + 1);
if (cmd == NULL) return -1;
dps_snprintf(cmd, len + 1, "SIZE %s", path);
code = Dps_ftp_send_cmd(c, cmd);
DPS_FREE(cmd);
if (code == -1){
return -1;
}else if ( code >3 ){
c->err = code;
return -1;
}
sscanf(c->buf, "213 %u", &len);
return len;
}
int Dps_ftp_cwd(DPS_CONN *c, char *path){
char *cmd;
int code;
size_t len;
if (!path)
return -1;
if (*path == '\0') return 0;
len = dps_strlen(path) + 16;
cmd = DpsXmalloc(len+1);
if (cmd == NULL) return -1;
dps_snprintf(cmd, len+1, "CWD %s", path);
code = Dps_ftp_send_cmd(c, cmd);
DPS_FREE(cmd);
if (code == -1){
return -1;
}else if ( code >3 ){
c->err = code;
return -1;
}
return 0;
}
__C_LINK int __DPSCALL DpsSynonymListLoad(DPS_ENV * Env,const char * filename){
struct stat sb;
char *str, *data = NULL, *cur_n = NULL;
char lang[64]="";
DPS_CHARSET *cs=NULL;
DPS_CHARSET *sys_int=DpsGetCharSet("sys-int");
DPS_CONV file_uni;
DPS_WIDEWORD *ww = NULL;
size_t key = 1;
int flag_th = 0;
int fd;
char savebyte;
if (stat(filename, &sb)) {
fprintf(stderr, "Unable to stat synonyms file '%s': %s", filename, strerror(errno));
return DPS_ERROR;
}
if ((fd = DpsOpen2(filename, O_RDONLY)) <= 0) {
dps_snprintf(Env->errstr,sizeof(Env->errstr)-1, "Unable to open synonyms file '%s': %s", filename, strerror(errno));
return DPS_ERROR;
}
if ((data = (char*)DpsMalloc(sb.st_size + 1)) == NULL) {
dps_snprintf(Env->errstr,sizeof(Env->errstr)-1, "Unable to alloc %d bytes", sb.st_size);
DpsClose(fd);
return DPS_ERROR;
}
if (read(fd, data, sb.st_size) != (ssize_t)sb.st_size) {
dps_snprintf(Env->errstr,sizeof(Env->errstr)-1, "Unable to read synonym file '%s': %s", filename, strerror(errno));
DPS_FREE(data);
DpsClose(fd);
return DPS_ERROR;
}
data[sb.st_size] = '\0';
str = data;
cur_n = strchr(str, '\n');
if (cur_n != NULL) {
cur_n++;
savebyte = *cur_n;
*cur_n = '\0';
}
while(str != NULL) {
if(str[0]=='#'||str[0]==' '||str[0]=='\t'||str[0]=='\r'||str[0]=='\n') goto loop_continue;
if(!strncasecmp(str,"Charset:",8)){
char * lasttok;
char * charset;
if((charset = dps_strtok_r(str + 8, " \t\n\r", &lasttok))) {
cs=DpsGetCharSet(charset);
if(!cs){
dps_snprintf(Env->errstr, sizeof(Env->errstr), "Unknown charset '%s' in synonyms file '%s'",
charset, filename);
DPS_FREE(data);
DpsClose(fd);
return DPS_ERROR;
}
DpsConvInit(&file_uni, cs, sys_int, Env->CharsToEscape, 0);
}
}else
if(!strncasecmp(str,"Language:",9)){
char * lasttok;
char * l;
if((l = dps_strtok_r(str + 9, " \t\n\r", &lasttok))) {
dps_strncpy(lang, l, sizeof(lang)-1);
}
}else
if(!strncasecmp(str, "Thesaurus:", 10)) {
char * lasttok;
char *tok = dps_strtok_r(str + 10, " \t\n\r", &lasttok);
flag_th = (strncasecmp(tok, "yes", 3) == 0) ? 1 : 0;
}else{
char *av[255];
size_t ac, i, j;
dpsunicode_t *t;
if(!cs){
dps_snprintf(Env->errstr,sizeof(Env->errstr)-1,"No Charset command in synonyms file '%s'",filename);
DpsClose(fd); DPS_FREE(data);
return DPS_ERROR;
}
if(!lang[0]){
dps_snprintf(Env->errstr,sizeof(Env->errstr)-1,"No Language command in synonyms file '%s'",filename);
DpsClose(fd); DPS_FREE(data);
return DPS_ERROR;
}
ac = DpsGetArgs(str, av, 255);
if (ac < 2) goto loop_continue;
if ((ww = (DPS_WIDEWORD*)DpsRealloc(ww, ac * sizeof(DPS_WIDEWORD))) == NULL) return DPS_ERROR;
for (i = 0; i < ac; i++) {
ww[i].word = av[i];
ww[i].len = dps_strlen(av[i]);
ww[i].uword = t = (dpsunicode_t*)DpsMalloc((3 * ww[i].len + 1) * sizeof(dpsunicode_t));
if (ww[i].uword == NULL) return DPS_ERROR;
DpsConv(&file_uni, (char*)ww[i].uword, sizeof(dpsunicode_t) * (3 * ww[i].len + 1), av[i], ww[i].len + 1);
DpsUniStrToLower(ww[i].uword);
ww[i].uword = DpsUniNormalizeNFC(NULL, ww[i].uword);
DPS_FREE(t);
}
for (i = 0; i < ac - 1; i++) {
for (j = i + 1; j < ac; j++) {
if((Env->Synonyms.nsynonyms + 1) >= Env->Synonyms.msynonyms){
Env->Synonyms.msynonyms += 64;
Env->Synonyms.Synonym = (DPS_SYNONYM*)DpsRealloc(Env->Synonyms.Synonym,
sizeof(DPS_SYNONYM)*Env->Synonyms.msynonyms);
if (Env->Synonyms.Synonym == NULL) {
Env->Synonyms.msynonyms = Env->Synonyms.nsynonyms = 0;
return DPS_ERROR;
}
}
bzero((void*)&Env->Synonyms.Synonym[Env->Synonyms.nsynonyms], sizeof(DPS_SYNONYM));
/* Add direct order */
Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].p.uword = DpsUniDup(ww[i].uword);
Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].s.uword = DpsUniDup(ww[j].uword);
Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].p.count =
Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].s.count = (size_t)((flag_th) ? key : 0);
Env->Synonyms.nsynonyms++;
bzero((void*)&Env->Synonyms.Synonym[Env->Synonyms.nsynonyms], sizeof(DPS_SYNONYM));
/* Add reverse order */
Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].p.uword = DpsUniDup(ww[j].uword);
Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].s.uword = DpsUniDup(ww[i].uword);
Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].p.count =
Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].s.count = (size_t)((flag_th) ? key : 0);
Env->Synonyms.nsynonyms++;
}
}
for (i = 0; i < ac; i++) {
DPS_FREE(ww[i].uword);
}
do { key++; } while (key == 0);
}
loop_continue:
str = cur_n;
if (str != NULL) {
*str = savebyte;
cur_n = strchr(str, '\n');
if (cur_n != NULL) {
cur_n++;
savebyte = *cur_n;
*cur_n = '\0';
}
}
}
DPS_FREE(data);
DPS_FREE(ww);
DpsClose(fd);
return DPS_OK;
}
static int ftp_parse_list(DPS_CONN *connp, char *path){
char *line, *buf_in, *ch, *buf_out, *tok, *fname;
int len_h, len_f,len_p, i;
char *dir, savec;
size_t len,buf_len,cur_len;
if (!connp->buf || !connp->buf_len)
return 0;
buf_in = connp->buf;
/* 22 = dps_strlen(<a href=\"ftp://%s%s%s/\"></a>)*/
len_h = dps_strlen(connp->hostname) + ((connp->user) ? dps_strlen(connp->user) : 0) + ((connp->pass) ? dps_strlen(connp->pass) : 0) + 2 + 22;
len_p = dps_strlen(path);
cur_len = 0;
buf_len = connp->buf_len;
buf_out = DpsXmalloc(buf_len + 1);
if (buf_out == NULL) return -1;
buf_out[0] = '\0';
line = dps_strtok_r(buf_in, "\r\n", &tok, &savec);
do{
if (!(fname = strtok(line, " ")))
continue;
/* drwxrwxrwx x user group size month date time file_name */
for(i=0; i<7; i++)
if (!(fname = strtok(NULL, " ")))
break;
if (!(fname = strtok(NULL, "")))
continue;
len = 0 ;
len_f = len_h + len_p + dps_strlen(fname);
if ((cur_len+len_f) >= buf_len){
buf_len += DPS_NET_BUF_SIZE;
buf_out = DpsXrealloc(buf_out, buf_len + 1);
}
switch (line[0]){
case 'd':
if (!fname || !strcmp(fname, ".") || !strcmp(fname, ".."))
break;
len = len_f;
dps_snprintf(DPS_STREND(buf_out) /*buf_out+cur_len*/, len+1, "<a href=\"ftp://%s%s%s%s%s/%s%s/\"></a>\n",
(connp->user) ? connp->user : "", (connp->user) ? ":" : "",
(connp->pass) ? connp->pass : "", (connp->user || connp->pass) ? "@" : "",
connp->hostname, path, fname);
break;
case 'l':
ch = strstr (fname, " -> ");
if (!ch)
break;
len = ch - fname;
dir = DpsMalloc(len+1);
if (dir == NULL) return -1;
dps_snprintf(dir, len+1, "%s", fname);
len = len_h + len_p + dps_strlen(dir);
dps_snprintf(DPS_STREND(buf_out)/*buf_out+cur_len*/, len+1, "<a href=\"ftp://%s%s%s%s%s/%s%s/\"></a>\n",
(connp->user) ? connp->user : "", (connp->user) ? ":" : "",
(connp->pass) ? connp->pass : "", (connp->user || connp->pass) ? "@" : "",
connp->hostname, path, dir);
DPS_FREE(dir);
/*ch +=4;*/
/* Check if it is absolute link */
/* if ((ch[0] == '/') || (ch[0] == '\\') ||
((isalpha(ch[0]) && (ch[1]==':')))){
len = len_h+dps_strlen(ch);
dps_snprintf(buf_out+cur_len, len+1, "<a href=\"ftp://%s%s/\"></a>",
connp->hostname, ch);
}else{
len = len_h+len_p+dps_strlen(ch);
dps_snprintf(buf_out+cur_len, len+1, "<a href=\"ftp://%s%s%s/\"></a>",
connp->hostname, path, ch);
}
*/
break;
case '-':
len = len_f;
dps_snprintf(DPS_STREND(buf_out)/*buf_out+cur_len*/, len+1, "<a href=\"ftp://%s%s%s%s%s/%s%s\"></a>\n",
(connp->user) ? connp->user : "", (connp->user) ? ":" : "",
(connp->pass) ? connp->pass : "", (connp->user || connp->pass) ? "@" : "",
connp->hostname, path, fname);
break;
}
cur_len += len;
}while( (line = dps_strtok_r(NULL, "\r\n", &tok, &savec)));
if (cur_len+1 > connp->buf_len_total){
connp->buf_len_total = cur_len;
connp->buf = DpsXrealloc(connp->buf, (size_t)connp->buf_len_total+1);
if (connp->buf == NULL) return -1;
}
bzero(connp->buf, ((size_t)connp->buf_len_total+1));
dps_memmove(connp->buf, buf_out, cur_len);
DPS_FREE(buf_out);
return 0;
}
void
FE_Printv(const char * pattern, va_list args)
{
static const char bad_pattern[] =
"\nBad pattern specifier %%%c in FE_Print().\n";
const char * s = pattern;
char c;
while ( (c = *s++) != '\0' ) {
if ( c == '%' ) {
c = *s++;
switch ( c ) {
case '%':
(void) write(2, &c, 1);
break;
case 'a':
/*
* Print an address passed as a void pointer.
* The type of fe_number must be set so that
* it is large enough to contain all of the
* bits of a void pointer.
*/
printNumber(
(fe_number)va_arg(args, void *)
,0x10);
break;
case 's':
{
const char * string;
size_t length;
string = va_arg(args, char *);
length = dps_strlen(string);
(void) write(2, string, length);
}
break;
case 'd':
{
int n = va_arg(args, int);
if ( n < 0 ) {
char cc = '-';
write(2, &cc, 1);
n = -n;
}
printNumber(n, 10);
}
break;
case 'x':
printNumber(va_arg(args, u_int), 0x10);
break;
case 'c':
{
char c = va_arg(args, int); /* Note: char is promoted to int. */
(void) write(2, &c, 1);
}
break;
default:
{
FE_Print(bad_pattern, c);
}
}
}
else
int __DPSCALL DpsSearchdCatAction(DPS_AGENT *A, DPS_CATEGORY *C, int cmd, void *db) {
DPS_DB *searchd = db;
DPS_SEARCHD_PACKET_HEADER hdr;
char *buf;
ssize_t nsent, nrecv;
int done = 0;
int rc=DPS_OK;
char *msg = NULL;
char *dinfo = NULL;
TRACE_IN(A, "DpsSearchdCatAction");
hdr.cmd = DPS_SEARCHD_CMD_CATINFO;
hdr.len = sizeof(int) + dps_strlen(C->addr) + 1;
if ((buf = (char*)DpsMalloc(hdr.len + 1)) == NULL) {
DpsLog(A, DPS_LOG_ERROR, "Out of memory");
TRACE_OUT(A);
return DPS_ERROR;
}
*((int*)buf) = cmd;
dps_strcpy(buf + sizeof(int), C->addr);
nsent = DpsSearchdSendPacket(searchd->searchd, &hdr, buf);
DPS_FREE(buf);
while(!done) {
char * tok, * lt;
nrecv = DpsRecvall(searchd->searchd, &hdr, sizeof(hdr), 360);
if(nrecv != sizeof(hdr)){
DpsLog(A, DPS_LOG_ERROR, "Received incomplete header from searchd (%d bytes)", (int)nrecv);
TRACE_OUT(A);
return(DPS_ERROR);
}else{
#ifdef DEBUG_SDP
DpsLog(A, DPS_LOG_ERROR, "Received header cmd=%d len=%d\n", hdr.cmd, hdr.len);
#endif
}
switch(hdr.cmd){
case DPS_SEARCHD_CMD_ERROR:
msg = (char*)DpsMalloc(hdr.len + 1);
if (msg == NULL) {
done=1;
break;
}
nrecv = DpsRecvall(searchd->searchd, msg, hdr.len, 360);
msg[(nrecv >= 0) ? nrecv : 0 ] = '\0';
sprintf(A->Conf->errstr, "Searchd error: '%s'", msg);
rc=DPS_ERROR;
DPS_FREE(msg);
done=1;
break;
case DPS_SEARCHD_CMD_MESSAGE:
msg=(char*)DpsMalloc(hdr.len+1);
if (msg == NULL) {
done=1;
break;
}
nrecv = DpsRecvall(searchd->searchd, msg, hdr.len, 360);
msg[(nrecv >= 0) ? nrecv : 0] = '\0';
#ifdef DEBUG_SDP
DpsLog(A, DPS_LOG_ERROR, "Message from searchd: '%s'\n",msg);
#endif
DPS_FREE(msg);
break;
case DPS_SEARCHD_CMD_CATINFO:
dinfo=(char*)DpsMalloc(hdr.len+1);
if (dinfo == NULL) {
done=1;
break;
}
nrecv = DpsRecvall(searchd->searchd, dinfo, hdr.len, 360);
dinfo[(nrecv >= 0) ? nrecv : 0] = '\0';
#ifdef DEBUG_SDP
DpsLog(A, DPS_LOG_ERROR, "Received CATINFO size=%d buf=%s\n",hdr.len,dinfo);
#endif
C->ncategories = 0;
tok = dps_strtok_r(dinfo, "\r\n", <, NULL);
while(tok){
DpsCatFromTextBuf(C, tok);
tok = dps_strtok_r(NULL, "\r\n", <, NULL);
}
DPS_FREE(dinfo);
done=1;
break;
default:
sprintf(A->Conf->errstr, "Unknown searchd response: cmd=%d len=%d", hdr.cmd, hdr.len);
rc=DPS_ERROR;
done = 1;
break;
}
}
TRACE_OUT(A);
return rc;
}
int DpsCloneListSearchd(DPS_AGENT *Indexer, DPS_DOCUMENT *Doc, DPS_RESULT *Res, DPS_DB *db) {
DPS_SEARCHD_PACKET_HEADER hdr;
ssize_t nsent,nrecv;
char *msg = NULL, *dinfo = NULL;
char *tok, *lt;
char buf[128];
int done = 0;
int rc = DPS_OK;
TRACE_IN(Indexer, "DpsCloneListSearchd");
dps_snprintf(buf, 128, "%s", DpsVarListFindStr(&Doc->Sections, "DP_ID", "0"));
hdr.cmd = DPS_SEARCHD_CMD_CLONES;
hdr.len = dps_strlen(buf);
nsent = DpsSearchdSendPacket(db->searchd, &hdr, buf);
while(!done){
nrecv = DpsRecvall(db->searchd, &hdr, sizeof(hdr), 360);
if(nrecv != sizeof(hdr)){
DpsLog(Indexer, DPS_LOG_ERROR, "Received incomplete header from searchd (%d bytes)", (int)nrecv);
TRACE_OUT(Indexer);
return(DPS_ERROR);
}else{
#ifdef DEBUG_SDP
DpsLog(Indexer, DPS_LOG_DEBUG, "Received header cmd=%d len=%d\n", hdr.cmd, hdr.len);
#endif
}
switch(hdr.cmd){
case DPS_SEARCHD_CMD_ERROR:
msg = (char*)DpsMalloc(hdr.len + 1);
if (msg == NULL) {
done=1;
break;
}
nrecv = DpsRecvall(db->searchd, msg, hdr.len, 360);
msg[(nrecv >= 0) ? nrecv : 0] = '\0';
sprintf(Indexer->Conf->errstr, "Searchd error: '%s'", msg);
rc = DPS_ERROR;
DPS_FREE(msg);
done = 1;
break;
case DPS_SEARCHD_CMD_DOCINFO:
dinfo = (char*)DpsMalloc(hdr.len + 1);
if (dinfo == NULL) {
done=1;
break;
}
nrecv = DpsRecvall(db->searchd, dinfo, hdr.len, 360);
dinfo[(nrecv >= 0) ? nrecv : 0] = '\0';
#ifdef DEBUG_SDP
DpsLog(Indexer, DPS_LOG_DEBUG, "Received DOCINFO size=%d buf=%s\n", hdr.len, dinfo);
#endif
if (strcasecmp(dinfo, "nocloneinfo") != 0) {
tok = dps_strtok_r(dinfo, "\r\n", <, NULL);
while(tok){
DPS_DOCUMENT *D;
size_t nd = Res->num_rows++;
Res->Doc = (DPS_DOCUMENT*)DpsRealloc(Res->Doc, (Res->num_rows + 1) * sizeof(DPS_DOCUMENT));
if (Res->Doc == NULL) {
sprintf(Indexer->Conf->errstr, "Realloc error");
rc = DPS_ERROR;
break;
}
D = &Res->Doc[nd];
DpsDocInit(D);
DpsDocFromTextBuf(D, tok);
tok = dps_strtok_r(NULL, "\r\n", <, NULL);
}
}
DPS_FREE(dinfo);
done = 1;
break;
default:
sprintf(Indexer->Conf->errstr, "Unknown searchd response: cmd=%d len=%d", hdr.cmd, hdr.len);
rc = DPS_ERROR;
done = 1;
break;
}
}
TRACE_OUT(Indexer);
return rc;
}
int __DPSCALL DpsFindWordsSearchd(DPS_AGENT *query, DPS_RESULT *Res, DPS_DB *searchd) {
size_t maxlen = 1024;
char *request, *edf = NULL, *e_empty = NULL;
const char *df = DpsVarListFindStr(&query->Vars, "DateFormat", NULL);
const char *empty = DpsVarListFindStr(&query->Vars, "empty", NULL);
const char *qs = DpsVarListFindStr(&query->Vars, "QUERY_STRING", "");
const char *tmplt = DpsVarListFindStr(&query->Vars, "tmplt", "");
int res=DPS_OK;
TRACE_IN(query, "DpsFindWordsSearchd");
if (df) {
edf = (char*)DpsMalloc(dps_strlen(df) * 10 + 1);
if (edf == NULL) {
sprintf(query->Conf->errstr,"Can't allocate memory");
TRACE_OUT(query);
return DPS_ERROR;
}
DpsEscapeURL(edf, df);
maxlen += dps_strlen(edf);
}
if (empty) {
e_empty = (char*)DpsMalloc(dps_strlen(empty) * 10 + 1);
if (e_empty == NULL) {
sprintf(query->Conf->errstr, "Can't allocate memory");
TRACE_OUT(query);
return DPS_ERROR;
}
DpsEscapeURL(e_empty, empty);
maxlen += dps_strlen(e_empty);
}
maxlen += dps_strlen(qs) + dps_strlen(tmplt) + 64;
if (NULL==(request=(char*)DpsMalloc(maxlen))) {
sprintf(query->Conf->errstr,"Can't allocate memory");
DPS_FREE(edf);
TRACE_OUT(query);
return DPS_ERROR;
}
dps_snprintf(request, maxlen, "%s&BrowserCharset=%s&IP=%s&g-lc=%s&ExcerptSize=%s&ExcerptPadding=%s&DoExcerpt=%s&tmplt=%s%s%s%s%s%s%s&sp=%s&sy=%s&s=%s",
qs,
DpsVarListFindStr(&query->Vars, "BrowserCharset", "iso-8859-1"),
DpsVarListFindStr(&query->Vars, "IP", "localhost"),
DpsVarListFindStr(&query->Vars, "g-lc", "en"),
DpsVarListFindStr(&query->Vars, "ExcerptSize", "256"),
DpsVarListFindStr(&query->Vars, "ExcerptPadding", "40"),
(query->Flags.do_excerpt) ? "yes" : "no",
tmplt,
(edf) ? "&DateFormat=" : "", (edf) ? edf : "",
(e_empty) ? "&empty=" : "", (e_empty) ? e_empty : "",
(searchd->label) ? "&label=" : "", (searchd->label) ? searchd->label : "",
DpsVarListFindStr(&query->Vars, "sp", "1"),
DpsVarListFindStr(&query->Vars, "sy", "1"),
DpsVarListFindStr(&query->Vars, "s", "RP")
);
DPS_FREE(edf);
DPS_FREE(e_empty);
request[maxlen-1]='\0';
res = DpsSearchdSendWordRequest(query, searchd, request);
DPS_FREE(request);
if (DPS_OK != res) {
TRACE_OUT(query);
return res;
}
/* res = DpsSearchdGetWordResponse(query, Res, searchd); called later from DpsFind */
TRACE_OUT(query);
return res;
}
char * DpsURLNormalizePath(char * str){
char * s=str;
char * q;
char * d;
/* Hide query string */
if((q=strchr(s,'?'))){
*q++='\0';
if(!*q)q=NULL;
}
/* Remove all "/../" entries */
while((d=strstr(str,"/../"))){
char * p;
if(d>str){
/* Skip non-slashes */
for(p=d-1;(*p!='/')&&(p>str);p--);
/* Skip slashes */
while((p>(str+1))&&(*(p-1)=='/'))p--;
}else{
/* We are at the top level and have ../ */
/* Remove it too to avoid loops */
p=str;
}
dps_memmove(p,d+3,dps_strlen(d)-2);
}
/* Remove remove trailig "/.." */
d=str+dps_strlen(str);
if((d-str>2)&&(!strcmp(d-3,"/.."))){
d-=4;
while((d>str)&&(*d!='/'))d--;
if(*d=='/')d[1]='\0';
else dps_strcpy(str,"/");
}
/* Remove all "/./" entries */
while((d=strstr(str,"/./"))){
dps_memmove(d,d+2,dps_strlen(d)-1);
}
/* Remove the trailing "/." */
if((d=str+dps_strlen(str))>(str+2))
if(!strcmp(d-2,"/."))
*(d-1)='\0';
/* Remove all "//" entries */
while((d=strstr(str,"//"))){
dps_memmove(d,d+1,dps_strlen(d));
}
/* Replace "%7E" with "~" */
/* Actually it is to be done */
/* for all valid characters */
/* which do not require escaping */
/* However I'm lazy, do it for 7E */
/* as the most often "abused" */
while((d=strstr(str,"%7E"))){
*d='~';
dps_memmove(d+1,d+3,dps_strlen(d+3)+1);
}
/* Restore query string */
if(q){
char * e=str+dps_strlen(str);
*e='?';
dps_memmove(e+1,q,dps_strlen(q)+1);
}
return str;
}
int __DPSCALL DpsResAddDocInfoSearchd(DPS_AGENT * query,DPS_DB *cl,DPS_RESULT * Res,size_t clnum){
DPS_SEARCHD_PACKET_HEADER hdr;
char * msg=NULL;
size_t i; /* num=0,curnum=0;*/
int done = 0;
ssize_t nsent,nrecv;
char * dinfo=NULL;
int rc=DPS_OK;
char *textbuf;
size_t dlen = 0;
TRACE_IN(query, "DpsResAddDocInfoSearchd");
if(!Res->num_rows) { TRACE_OUT(query); return(DPS_OK); }
for(i=0;i<Res->num_rows;i++){
size_t ulen;
size_t olen;
size_t nsec, r;
DPS_DOCUMENT *D=&Res->Doc[i];
r = (size_t) 's';
for(nsec = 0; nsec < D->Sections.Root[r].nvars; nsec++)
if (strcasecmp(D->Sections.Root[r].Var[nsec].name, "Score") == 0) D->Sections.Root[r].Var[nsec].section = 1;
#ifdef WITH_MULTIDBADDR
if (D->dbnum != cl->dbnum) continue;
#endif
textbuf = DpsDocToTextBuf(D, 1, 0);
if (textbuf == NULL) {TRACE_OUT(query); return DPS_ERROR;}
ulen = dps_strlen(textbuf)+2;
olen = dlen;
dlen = dlen + ulen;
dinfo = (char*)DpsRealloc(dinfo, dlen + 1);
if (dinfo == NULL) {
DpsFree(textbuf);
TRACE_OUT(query);
return DPS_ERROR;
}
dinfo[olen] = '\0';
sprintf(dinfo + olen, "%s\r\n", textbuf);
DpsFree(textbuf);
}
if (dinfo == NULL) {
TRACE_OUT(query);
return DPS_OK;
}
hdr.cmd=DPS_SEARCHD_CMD_DOCINFO;
hdr.len = dps_strlen(dinfo);
nsent = DpsSearchdSendPacket(cl->searchd, &hdr, dinfo);
#ifdef DEBUG_SDP
DpsLog(query, DPS_LOG_ERROR, "Sent DOCINFO size=%d buf=%s\n", hdr.len, dinfo);
#endif
while(!done){
char * tok, * lt;
nrecv = DpsRecvall(cl->searchd, &hdr, sizeof(hdr), 360);
if(nrecv!=sizeof(hdr)){
DpsLog(query, DPS_LOG_ERROR, "Received incomplete header from searchd (%d bytes, errno:%d)", (int)nrecv, errno);
TRACE_OUT(query);
return(DPS_ERROR);
}else{
#ifdef DEBUG_SDP
DpsLog(query, DPS_LOG_ERROR, "Received header cmd=%d len=%d\n",hdr.cmd,hdr.len);
#endif
}
switch(hdr.cmd){
case DPS_SEARCHD_CMD_ERROR:
msg=(char*)DpsMalloc(hdr.len+1);
if (msg == NULL) {
done = 1;
break;
}
nrecv = DpsRecvall(cl->searchd, msg, hdr.len, 360);
msg[(nrecv >= 0) ? nrecv : 0]='\0';
sprintf(query->Conf->errstr,"Searchd error: '%s'",msg);
rc=DPS_ERROR;
DPS_FREE(msg);
done=1;
break;
case DPS_SEARCHD_CMD_MESSAGE:
msg=(char*)DpsMalloc(hdr.len+1);
if (msg == NULL) {
done = 1;
break;
}
nrecv = DpsRecvall(cl->searchd, msg, hdr.len, 360);
msg[(nrecv >= 0) ? nrecv : 0]='\0';
#ifdef DEBUG_SDP
DpsLog(query, DPS_LOG_ERROR, "Message from searchd: '%s'\n",msg);
#endif
DPS_FREE(msg);
break;
case DPS_SEARCHD_CMD_DOCINFO:
dinfo = (char*)DpsRealloc(dinfo, hdr.len + 1);
if (dinfo == NULL) {
done=1;
break;
}
nrecv = DpsRecvall(cl->searchd, dinfo, hdr.len, 360);
dinfo[(nrecv > 0) ? nrecv : 0] = '\0';
#ifdef DEBUG_SDP
DpsLog(query, DPS_LOG_ERROR, "Received DOCINFO size=%d buf=%s\n",hdr.len,dinfo);
#endif
tok = dps_strtok_r(dinfo, "\r\n", <, NULL);
while(tok){
urlid_t Doc_url_id, Res_Doc_url_id;
DPS_DOCUMENT Doc;
DpsDocInit(&Doc);
DpsDocFromTextBuf(&Doc,tok);
Doc_url_id = (urlid_t)DpsVarListFindInt(&Doc.Sections, "DP_ID", 0);
for(i=0;i<Res->num_rows;i++){
#ifdef WITH_MULTIDBADDR
if (Res->Doc[i].dbnum != cl->dbnum) continue;
#endif
Res_Doc_url_id = (urlid_t)DpsVarListFindInt(&Res->Doc[i].Sections, "DP_ID", 0);
if (Res_Doc_url_id == Doc_url_id) {
DpsDocFromTextBuf(&Res->Doc[i], tok);
break;
}
}
tok = dps_strtok_r(NULL, "\r\n", <, NULL);
DpsDocFree(&Doc);
}
DPS_FREE(dinfo);
done=1;
break;
default:
sprintf(query->Conf->errstr,"Unknown searchd response: cmd=%d len=%d",hdr.cmd,hdr.len);
rc=DPS_ERROR;
done=1;
break;
}
}
TRACE_OUT(query);
return rc;
}
int DpsAddStackItem(DPS_AGENT *query, DPS_RESULT *Res, DPS_PREPARE_STATE *state, char *word, dpsunicode_t *uword) {
int origin;
size_t i;
size_t wlen = (uword == NULL) ? 0 : DpsUniLen(uword);
dpshash32_t crcword = (word == NULL) ? 0 : DpsStrHash32(word);
#ifdef DEBUG_BOOL
DpsLog(query, DPS_LOG_EXTRA, "0[%d].%x %c -- %s [%x] .secno:%d\n", state->order, state->origin, item_type(state->cmd),
(word == NULL) ? "<NULL>" : word, crcword, state->secno[state->p_secno]);
#endif
if((uword != NULL) && ( DpsStopListFind(&query->Conf->StopWords, uword, state->qlang) ||
(query->WordParam.min_word_len > wlen) ||
(query->WordParam.max_word_len < wlen)) ) {
origin = state->origin | DPS_WORD_ORIGIN_STOP;
} else {
origin = state->origin;
}
if (state->cmd == DPS_STACK_WORD && !(origin & DPS_WORD_ORIGIN_QUERY)) {
for (i = 0; i < Res->nitems; i++) {
if ((Res->items[i].order == state->order) && (Res->items[i].crcword == crcword)) return DPS_OK;
}
}
if (Res->nitems >= Res->mitems - 2) {
Res->mitems += DPS_MAXSTACK;
Res->items = (DPS_STACK_ITEM*)DpsRealloc(Res->items, Res->mitems * sizeof(DPS_STACK_ITEM));
if (Res->items == NULL) {
DpsLog(query, DPS_LOG_ERROR, "Can't alloc %d bytes for %d mitems", Res->mitems * sizeof(DPS_STACK_ITEM), Res->mitems);
return DPS_ERROR;
}
}
if (Res->nitems > 0) {
if (state->cmd == DPS_STACK_OR || state->cmd == DPS_STACK_AND || state->cmd == DPS_STACK_NEAR || state->cmd == DPS_STACK_ANYWORD) {
if (Res->items[Res->nitems-1].cmd == DPS_STACK_AND || Res->items[Res->nitems-1].cmd == DPS_STACK_OR
|| Res->items[Res->nitems-1].cmd == DPS_STACK_NEAR || Res->items[Res->nitems-1].cmd == DPS_STACK_ANYWORD) {
return DPS_OK;
}
}
if ((Res->nitems > 0) && (state->cmd == DPS_STACK_WORD)
&& (
(Res->items[Res->nitems-1].cmd == DPS_STACK_WORD)
|| (Res->items[Res->nitems-1].cmd == DPS_STACK_RIGHT)
|| (Res->items[Res->nitems-1].cmd == DPS_STACK_PHRASE_RIGHT)
)) {
Res->items[Res->nitems].cmd = DPS_STACK_OR;
Res->items[Res->nitems].order = 0;
Res->items[Res->nitems].origin = 0;
Res->items[Res->nitems].count = 0;
Res->items[Res->nitems].len = 0;
Res->items[Res->nitems].crcword = 0;
Res->items[Res->nitems].word = NULL;
Res->items[Res->nitems].ulen = 0;
Res->items[Res->nitems].uword = NULL;
Res->items[Res->nitems].pbegin = NULL;
Res->items[Res->nitems].order_origin = 0;
Res->items[Res->nitems].secno = state->secno[state->p_secno];
Res->nitems++;
Res->ncmds++;
#ifdef DEBUG_BOOL
DpsLog(query, DPS_LOG_EXTRA, "1[%d].%x %c -- %s", 0, 0, item_type(DPS_STACK_OR), "<NULL>");
#endif
}
if ((Res->nitems > 0) && (state->cmd == DPS_STACK_LEFT)
&& (
(Res->items[Res->nitems-1].cmd == DPS_STACK_RIGHT)
|| (Res->items[Res->nitems-1].cmd == DPS_STACK_PHRASE_RIGHT)
)) {
Res->items[Res->nitems].cmd = state->add_cmd;
Res->items[Res->nitems].order = 0;
Res->items[Res->nitems].origin = 0;
Res->items[Res->nitems].count = 0;
Res->items[Res->nitems].len = 0;
Res->items[Res->nitems].crcword = 0;
Res->items[Res->nitems].word = NULL;
Res->items[Res->nitems].ulen = 0;
Res->items[Res->nitems].uword = NULL;
Res->items[Res->nitems].pbegin = NULL;
Res->items[Res->nitems].order_origin = 0;
Res->items[Res->nitems].secno = state->secno[state->p_secno];
Res->nitems++;
Res->ncmds++;
#ifdef DEBUG_BOOL
DpsLog(query, DPS_LOG_EXTRA, "1[%d].%x %c -- %s", 0, 0, item_type(state->add_cmd), "<NULL>");
#endif
}
}
Res->items[Res->nitems].cmd = state->cmd;
Res->items[Res->nitems].order = state->order;
Res->items[Res->nitems].order_inquery = state->order_inquery;
Res->items[Res->nitems].origin = origin;
Res->items[Res->nitems].count = 0;
Res->items[Res->nitems].len = (word == NULL) ? 0 : dps_strlen(word);
Res->items[Res->nitems].crcword = crcword;
Res->items[Res->nitems].word = (word == NULL) ? NULL : DpsStrdup(word);
Res->items[Res->nitems].ulen = wlen;
Res->items[Res->nitems].uword = (uword == NULL) ? NULL : DpsUniDup(uword);
Res->items[Res->nitems].pbegin = NULL;
Res->items[Res->nitems].order_origin = 0;
Res->items[Res->nitems].wordnum = Res->nitems;
Res->items[Res->nitems].secno = state->secno[state->p_secno];
Res->nitems++;
if (state->cmd != DPS_STACK_WORD) {
Res->ncmds++;
} else {
Res->items[state->order].order_origin |= origin;
if (state->order > Res->max_order) Res->max_order = state->order;
if (state->order_inquery > Res->max_order_inquery) Res->max_order_inquery = state->order;
}
/* if ((state->cmd == DPS_STACK_WORD) && state->order > Res->max_order) Res->max_order = state->order;*/
#ifdef DEBUG_BOOL
DpsLog(query, DPS_LOG_EXTRA, "1[%d,%d].%x %c -- %s", state->order, state->order_inquery, state->origin, item_type(state->cmd),
(word == NULL) ? "<NULL>" : word);
#endif
return DPS_OK;
}
int _DpsURLParse(DPS_URL *url, const char *str, const char *filename, int line) {
#else
int DpsURLParse(DPS_URL *url, const char *str) {
#endif
char *schema,*anchor,*file,*query;
char *s;
/* size_t len = dps_strlen(str);*/
#ifdef WITH_PARANOIA
void * paran = DpsViolationEnter(paran);
#endif
#ifdef DEBUG_URL
fprintf(stderr, " -- %s:%d Parser url: %s\n", filename, line, str);
#endif
DPS_FREE(url->schema);
DPS_FREE(url->specific);
DPS_FREE(url->hostinfo);
DPS_FREE(url->hostname);
DPS_FREE(url->anchor);
DPS_FREE(url->auth);
url->port=0;
url->default_port=0;
DPS_FREE(url->path);
DPS_FREE(url->directory);
DPS_FREE(url->filename);
DPS_FREE(url->query_string);
/* if(len >= DPS_URLSIZE)return(DPS_URL_LONG); FIXME: Chage this cheking for configured parameter, not DPS_URLSIZE */
s = (char*)DpsStrdup(str);
if (s == NULL) {
#ifdef WITH_PARANOIA
DpsViolationExit(-1, paran);
#endif
return DPS_ERROR;
}
url->len = dps_strlen(str);
/* Find possible schema end than */
/* Check that it is really schema */
/* It must consist of alphas only */
/* We will take in account digits */
/* also for oracle8:// for example */
/* We must check it because */
/* It might be anchor also */
/* For example: */
/* "mod/index.html#a:1" */
if((schema=strchr(s,':'))){
const char * ch;
for(ch=s;ch<schema;ch++){
if(!isalnum(*ch)){
/* Bad character */
/* so it is not schema */
schema=0;break;
}
}
}
if(schema){
/* Have scheme - absolute path */
*schema=0;
url->schema = (char*)DpsStrdup(s);
url->specific = (char*)DpsStrdup(schema + 1);
*schema=':';
if(!strcasecmp(url->schema,"http"))url->default_port=80;
else
if(!strcasecmp(url->schema,"https"))url->default_port=443;
else
if(!strcasecmp(url->schema,"nntp"))url->default_port=119;
else
if(!strcasecmp(url->schema,"news"))url->default_port=119;
else
if(!strcasecmp(url->schema,"ftp"))url->default_port=21;
if(!strncmp(url->specific,"//",2)){
char *ss,*hostname;
/* Have hostinfo */
if((ss=strchr(url->specific+2,'/'))){
/* Have hostname with path */
*ss=0;
url->hostinfo = (char*)DpsStrdup(url->specific + 2);
*ss='/';
url->path = (char*)DpsStrdup(ss);
}else{
/* Hostname without path */
if ((ss = strchr(url->specific + 2, '?'))) {
/* Have hostname with parameters */
*ss = 0;
url->hostinfo = (char*)DpsStrdup(url->specific + 2);
*ss='?';
url->path = (char*)DpsStrdup("/");
}else {
url->hostinfo = (char*)DpsStrdup(url->specific + 2);
url->path = (char*)DpsStrdup("/");
}
}
if((hostname=strchr(url->hostinfo,'@'))){
/* Username and password is given */
/* Store auth string user:password */
*hostname=0;
url->auth = (char*)DpsStrdup(url->hostinfo);
*hostname='@';
hostname++;
}else{
hostname = url->hostinfo;
}
/*
FIXME:
for(h=hostname;*h;h++){
if( *h>='A' && *h<='Z')
*h=(*h)-'A'+'a';
}
*/
if((ss=strchr(hostname,':'))){
*ss=0;
url->hostname = (char*)DpsStrdup(hostname);
*ss=':';
url->port=atoi(ss+1);
}else{
url->hostname = (char*)DpsStrdup(hostname);
url->port=0;
}
}else{
/* Have not host but have schema */
/* This is possible for: */
/* file: mailto: htdb: news: */
/* As far as we do not need mailto: just ignore it */
if(!strcasecmp(url->schema,"mailto")
|| !strcasecmp(url->schema,"javascript")
|| !strcasecmp(url->schema,"feed")
) {
DPS_FREE(s);
#ifdef WITH_PARANOIA
DpsViolationExit(-1, paran);
#endif
return(DPS_URL_BAD);
} else
if(!strcasecmp(url->schema,"file"))
url->path = (char*)DpsStrdup(url->specific);
else
if(!strcasecmp(url->schema,"exec"))
url->path = (char*)DpsStrdup(url->specific);
else
if(!strcasecmp(url->schema,"cgi"))
url->path = (char*)DpsStrdup(url->specific);
else
if(!strcasecmp(url->schema,"htdb"))
url->path = (char*)DpsStrdup(url->specific);
else
if(!strcasecmp(url->schema,"news")){
/* Now we will use localhost as NNTP */
/* server as it is not indicated in URL */
url->hostname = (char*)DpsStrdup("localhost");
url->path = (char*)DpsMalloc(dps_strlen(url->specific) + 2);
if (url->path == NULL) {
DPS_FREE(s);
#ifdef WITH_PARANOIA
DpsViolationExit(-1, paran);
#endif
return DPS_ERROR;
}
sprintf(url->path,"/%s",url->specific);
url->default_port=119;
}else{
/* Unknown strange schema */
DPS_FREE(s);
#ifdef WITH_PARANOIA
DpsViolationExit(-1, paran);
#endif
return(DPS_URL_BAD);
}
}
}else{
url->path = (char*)DpsStrdup(s);
}
/* Cat an anchor if exist */
if((anchor=strstr(url->path,"#")))*anchor=0;
/* If path is not full just copy it to filename */
/* i.e. neither /usr/local/ nor c:/windows/temp/ */
if((url->path != NULL) && (url->path[0]!='/') && (url->path[0]!='?') && (url->path[1]!=':')) {
/* Relative path */
if(!strncmp(url->path,"./",2))
url->filename = (char*)DpsStrdup(url->path + 2);
else
url->filename = (char*)DpsStrdup(url->path);
url->path[0] = 0;
}
/* truncate path to query_string */
/* and store query_string */
if((query=strrchr(url->path,'?'))){
url->query_string = (char*)DpsStrdup(query);
*(query) = 0;
}
DpsURLNormalizePath(url->path);
/* Now find right '/' sign and copy the rest to filename */
if((file=strrchr(url->path,'/'))&&(strcmp(file,"/"))){
url->filename = (char*)DpsStrdup(file + 1);
*(file+1)=0;
}
/* Now find right '/' sign and copy the rest to directory */
if ((file = strrchr(url->path,'/'))) {
char *p_save = file;
for(file--; (file > url->path) && (*file != '/'); file--);
file++;
if (*file) {
*p_save = '\0';
url->directory = (char*)DpsStrdup(file);
*p_save = '/';
}
}
DPS_FREE(s);
if (url->hostname != NULL) {
DpsRTrim(url->hostname, ".");
url->domain_level = 1;
for (s = url->hostname; *s; s++) {
*s = dps_tolower(*s);
if (*s == '.') url->domain_level++;
if (strchr(",'\";", (int)*s)) {
#ifdef WITH_PARANOIA
DpsViolationExit(-1, paran);
#endif
return DPS_URL_BAD;
}
}
}
if (url->hostinfo != NULL) {
DpsRTrim(url->hostinfo, ".");
s = strchr(url->hostinfo, '@');
for (s = (s == NULL) ? url->hostinfo : s + 1; *s; s++) *s = dps_tolower(*s);
}
if (url->schema != NULL) for (s = url->schema; *s; s++) *s = dps_tolower(*s);
/* fprintf(stderr, "url: .path: %s port:%d\n", url->path, url->port);*/
#ifdef WITH_PARANOIA
DpsViolationExit(-1, paran);
#endif
return DPS_OK;
}
void RelLink(DPS_AGENT *Indexer, DPS_URL *curURL, DPS_URL *newURL, char **str, int ReverseAliasFlag) {
const char *schema = newURL->schema ? newURL->schema : curURL->schema;
const char *hostname = newURL->hostname ? newURL->hostname : curURL->hostname;
const char *auth = newURL->auth ? newURL->auth : curURL->auth;
const char *path = (newURL->path && newURL->path[0]) ? newURL->path : curURL->path;
const char *fname = ((newURL->filename && newURL->filename[0]) || (newURL->path && newURL->path[0])) ?
newURL->filename : curURL->filename;
const char *query_string = newURL->query_string;
char *pathfile = (char*)DpsMalloc(dps_strlen(DPS_NULL2EMPTY(path)) + dps_strlen(DPS_NULL2EMPTY(fname)) +
dps_strlen(DPS_NULL2EMPTY(query_string)) + 5);
int cascade;
DPS_MATCH *Alias;
char *alias = NULL;
size_t aliassize, nparts = 10;
DPS_MATCH_PART Parts[10];
if (newURL->hostinfo == NULL) newURL->charset_id = curURL->charset_id;
if (pathfile == NULL) return;
/* sprintf(pathfile, "/%s%s%s", DPS_NULL2EMPTY(path), DPS_NULL2EMPTY(fname), DPS_NULL2EMPTY(query_string));*/
pathfile[0] = '/';
dps_strcpy(pathfile + 1, DPS_NULL2EMPTY(path)); dps_strcat(pathfile, DPS_NULL2EMPTY(fname)); dps_strcat(pathfile, DPS_NULL2EMPTY(query_string));
DpsURLNormalizePath(pathfile);
if (!strcasecmp(DPS_NULL2EMPTY(schema), "mailto")
|| !strcasecmp(DPS_NULL2EMPTY(schema), "javascript")
|| !strcasecmp(DPS_NULL2EMPTY(schema), "feed")
) {
*str = (char*)DpsMalloc(dps_strlen(DPS_NULL2EMPTY(schema)) + dps_strlen(DPS_NULL2EMPTY(newURL->specific)) + 4);
if (*str == NULL) return;
/* sprintf(*str, "%s:%s", DPS_NULL2EMPTY(schema), DPS_NULL2EMPTY(newURL->specific));*/
dps_strcpy(*str, DPS_NULL2EMPTY(schema)); dps_strcat(*str, ":"); dps_strcat(*str, DPS_NULL2EMPTY(newURL->specific));
} else if(/*!strcasecmp(DPS_NULL2EMPTY(schema), "file") ||*/ !strcasecmp(DPS_NULL2EMPTY(schema), "htdb")) {
*str = (char*)DpsMalloc(dps_strlen(DPS_NULL2EMPTY(schema)) + dps_strlen(pathfile) + 4);
if (*str == NULL) return;
/* sprintf(*str, "%s:%s", DPS_NULL2EMPTY(schema), pathfile);*/
dps_strcpy(*str, DPS_NULL2EMPTY(schema)); dps_strcat(*str, ":"); dps_strcat(*str, pathfile);
}else{
*str = (char*)DpsMalloc(dps_strlen(DPS_NULL2EMPTY(schema)) + dps_strlen(pathfile) + dps_strlen(DPS_NULL2EMPTY(hostname)) + dps_strlen(DPS_NULL2EMPTY(auth)) + 8);
if (*str == NULL) return;
/* sprintf(*str, "%s://%s%s", DPS_NULL2EMPTY(schema), DPS_NULL2EMPTY(hostinfo), pathfile);*/
dps_strcpy(*str, DPS_NULL2EMPTY(schema)); dps_strcat(*str, "://");
if (auth) {
dps_strcat(*str, auth); dps_strcat(*str,"@");
}
dps_strcat(*str, DPS_NULL2EMPTY(hostname)); dps_strcat(*str, pathfile);
}
if(!strncmp(*str, "ftp://", 6) && (strstr(*str, ";type=")))
*(strstr(*str, ";type")) = '\0';
DPS_FREE(pathfile);
if (ReverseAliasFlag) {
const char *alias_prog = DpsVarListFindStr(&Indexer->Vars, "ReverseAliasProg", NULL);
if (alias_prog) {
int result;
aliassize = 256 + 2 * dps_strlen(*str);
alias = (char*)DpsRealloc(alias, aliassize);
if (alias == NULL) {
DpsLog(Indexer, DPS_LOG_ERROR, "No memory (%d bytes). %s line %d", aliassize, __FILE__, __LINE__);
goto ret;
}
alias[0] = '\0';
result = DpsAliasProg(Indexer, alias_prog, *str, alias, aliassize - 1);
DpsLog(Indexer, DPS_LOG_EXTRA, "ReverseAliasProg result: '%s'", alias);
if(result != DPS_OK) goto ret;
DPS_FREE(*str);
*str = (char*)DpsStrdup(alias);
}
for(cascade = 0; ((Alias=DpsMatchListFind(&Indexer->Conf->ReverseAliases,*str,nparts,Parts))) && (cascade < 1024); cascade++) {
aliassize = dps_strlen(Alias->arg) + dps_strlen(Alias->pattern) + dps_strlen(*str) + 128;
alias = (char*)DpsRealloc(alias, aliassize);
if (alias == NULL) {
DpsLog(Indexer, DPS_LOG_ERROR, "No memory (%d bytes). %s line %d", aliassize, __FILE__, __LINE__);
goto ret;
}
DpsMatchApply(alias,aliassize,*str,Alias->arg,Alias,nparts,Parts);
if(alias[0]){
DpsLog(Indexer,DPS_LOG_DEBUG,"ReverseAlias%d: pattern:%s, arg:%s -> '%s'", cascade, Alias->pattern, Alias->arg, alias);
DPS_FREE(*str);
*str = (char*)DpsStrdup(alias);
} else break;
if (Alias->last) break;
}
}
ret:
DPS_FREE(alias);
}
/*
* HP-UX 8/9.01 strcat reads a word past source when doing unaligned copies!
* Work around it here. The bug report has been filed with HP.
*/
char *strcat(char *d, const char *s)
{
dps_strcpy(d + dps_strlen(d), s);
return d;
}
__C_LINK int __DPSCALL DpsBaseOpen(DPS_BASE_PARAM *P, int mode) {
unsigned int hash;
size_t filenamelen, z;
ssize_t wr;
DPS_BASEITEM *hTable;
#ifdef DEBUG_SEARCH
unsigned long total_ticks, stop_ticks, start_ticks = DpsStartTimer();
#endif
TRACE_IN(P->A, "DpsBaseOpen");
if (P->opened) DpsBaseClose(P);
if (P->NFiles == 0) P->NFiles = DpsVarListFindUnsigned(&P->A->Vars, "BaseFiles", 0x100);
P->FileNo = DPS_FILENO(P->rec_id, P->NFiles);
hash = DPS_HASH(P->rec_id);
filenamelen = dps_strlen(P->vardir) + dps_strlen(P->subdir) + dps_strlen(P->indname) + dps_strlen(P->basename) + 48;
if (
((P->Ifilename = (char *)DpsMalloc(filenamelen)) == NULL) ||
((P->Sfilename = (char *)DpsMalloc(filenamelen)) == NULL) ) {
DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename);
DpsLog(P->A, DPS_LOG_ERROR, "Memory alloc error 2x%d bytes %s:%d", filenamelen, __FILE__, __LINE__);
TRACE_OUT(P->A);
return DPS_ERROR;
}
sprintf(P->Sfilename, "%s/%s/%s%04zx.s", P->vardir, P->subdir, P->basename, P->FileNo);
sprintf(P->Ifilename, "%s/%s/%s%04zx.i", P->vardir, P->subdir, P->indname, P->FileNo);
if ((P->Ifd = DpsOpen2(P->Ifilename, ((mode == DPS_READ_LOCK) ? O_RDONLY : O_RDWR) | DPS_BINARY)) < 0) {
if ((mode == DPS_READ_LOCK) || ((P->Ifd = DpsOpen3(P->Ifilename, O_RDWR | O_CREAT | DPS_BINARY
/*#ifdef O_DIRECT
| O_DIRECT
#endif*/
,
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH
)) < 0)) {
dps_strerror(P->A, (mode == DPS_READ_LOCK && errno == ENOENT) ? DPS_LOG_DEBUG : DPS_LOG_ERROR, "Can't open/create file %s for %s [%s:%d]",
P->Ifilename, (mode == DPS_READ_LOCK) ? "read" : "write", __FILE__, __LINE__);
DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename);
TRACE_OUT(P->A);
return DPS_ERROR;
}
#if 1
DPS_GETLOCK(P->A, DPS_LOCK_BASE_N(P->FileNo));
#endif
DpsWriteLock(P->Ifd);
if ((hTable = (DPS_BASEITEM *)DpsXmalloc(sizeof(DPS_BASEITEM) * DPS_HASH_PRIME)) == NULL) {
DpsLog(P->A, DPS_LOG_ERROR, "Memory alloc error hTable: %d bytes", sizeof(DPS_BASEITEM) * DPS_HASH_PRIME);
DpsUnLock(P->Ifd);
#if 1
DPS_RELEASELOCK(P->A, DPS_LOCK_BASE_N(P->FileNo));
#endif
DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename);
TRACE_OUT(P->A);
return DPS_ERROR;
}
if ( (wr = write(P->Ifd, hTable, sizeof(DPS_BASEITEM) * DPS_HASH_PRIME)) != sizeof(DPS_BASEITEM) * DPS_HASH_PRIME) {
dps_strerror(P->A, DPS_LOG_ERROR, "Can't set new index for file %s\nwritten %d bytes of %d\nIfd:%d hTable:%x",
P->Ifilename, wr, sizeof(DPS_BASEITEM) * DPS_HASH_PRIME, P->Ifd, hTable);
DPS_FREE(hTable);
DpsUnLock(P->Ifd);
#if 1
DPS_RELEASELOCK(P->A, DPS_LOCK_BASE_N(P->FileNo));
#endif
DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename);
TRACE_OUT(P->A);
return DPS_ERROR;
}
DpsUnLock(P->Ifd);
#if 1
DPS_RELEASELOCK(P->A, DPS_LOCK_BASE_N(P->FileNo));
#endif
DPS_FREE(hTable);
if (lseek(P->Ifd, (off_t)0, SEEK_SET) == (off_t)-1) {
DpsLog(P->A, DPS_LOG_ERROR, "Can't seek for file %s", P->Ifilename);
DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename);
TRACE_OUT(P->A);
return DPS_ERROR;
}
}
if (!P->A->Flags.cold_var) {
#if 1
DPS_GETLOCK(P->A, DPS_LOCK_BASE_N(P->FileNo));
#endif
switch (mode) {
case DPS_READ_LOCK:
DpsReadLock(P->Ifd);
break;
case DPS_WRITE_LOCK:
DpsWriteLock(P->Ifd);
break;
}
P->locked = 1;
}
if ((P->Sfd = DpsOpen2(P->Sfilename, ((mode == DPS_READ_LOCK) ? O_RDONLY : O_RDWR) | DPS_BINARY
/*#ifdef O_DIRECT
| O_DIRECT
#endif*/
)) < 0) {
if ((mode == DPS_READ_LOCK) || ((P->Sfd = DpsOpen3(P->Sfilename, O_RDWR | O_CREAT | DPS_BINARY
/*#ifdef O_DIRECT
| O_DIRECT
#endif*/
,
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH
)) < 0)) {
DpsLog(P->A, DPS_LOG_ERROR, "Can't open/create file %s", P->Sfilename);
DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename);
TRACE_OUT(P->A);
return DPS_ERROR;
}
}
if (!P->A->Flags.cold_var) {
switch(mode) {
case DPS_READ_LOCK:
DpsReadLock(P->Sfd);
break;
case DPS_WRITE_LOCK:
DpsWriteLock(P->Sfd);
break;
}
}
#ifdef DEBUG_SEARCH
stop_ticks = DpsStartTimer();
total_ticks = stop_ticks - start_ticks;
DpsLog(P->A, DPS_LOG_EXTRA, "OpenBase1 %03X in %.5f sec.", P->FileNo, (float)total_ticks / 1000);
#endif
for (z = 0; z < 3; z++) {
/* search rec_id */
if ( (P->CurrentItemPos = (dps_uint8)lseek(P->Ifd, (off_t)(hash * sizeof(DPS_BASEITEM)), SEEK_SET)) == (dps_uint8)-1) {
DpsLog(P->A, DPS_LOG_ERROR, "Can't seeek for file %s", P->Ifilename);
DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename);
TRACE_OUT(P->A);
return DPS_ERROR;
}
if (read(P->Ifd, &P->Item, sizeof(DPS_BASEITEM)) != sizeof(DPS_BASEITEM)) {
DpsLog(P->A, DPS_LOG_ERROR, "{%s:%d} Can't read index for file %s seek:%ld hash: %u (%d)",
__FILE__, __LINE__, P->Ifilename, P->CurrentItemPos, hash, hash);
bzero(&P->Item, sizeof(P->Item));
/* DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename);
TRACE_OUT(P->A);
return DPS_ERROR;
*/
}
#ifdef DEBUG_SEARCH
stop_ticks = DpsStartTimer();
total_ticks = stop_ticks - start_ticks;
DpsLog(P->A, DPS_LOG_EXTRA, "OpenBase2 %03X in %.5f sec.", P->FileNo, (float)total_ticks / 1000);
#endif
if (P->Item.rec_id == P->rec_id || P->Item.rec_id == 0) P->mishash = 0;
else P->mishash = 1;
P->PreviousItemPos = P->CurrentItemPos;
if (P->mishash)
while((P->Item.next != 0) && (P->Item.rec_id != P->rec_id)) {
P->PreviousItemPos = P->CurrentItemPos;
P->CurrentItemPos = P->Item.next;
if (lseek(P->Ifd, (off_t)P->CurrentItemPos, SEEK_SET) == (off_t)-1) {
DpsLog(P->A, DPS_LOG_ERROR, "Can't seek for file %s", P->Ifilename);
DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename);
TRACE_OUT(P->A);
return DPS_ERROR;
}
if ((wr = read(P->Ifd, &P->Item, sizeof(DPS_BASEITEM))) != sizeof(DPS_BASEITEM)) {
if (wr == 0) {
DpsLog(P->A, DPS_LOG_ERROR, "Possible corrupted hash chain for file %s, trying to restore (%s:%d)",
P->Ifilename, __FILE__, __LINE__);
if (lseek(P->Ifd, (off_t)P->PreviousItemPos, SEEK_SET) == (off_t)-1) {
DpsLog(P->A, DPS_LOG_ERROR, "Can't seek for file %s (%s:%d)", P->Ifilename, __FILE__, __LINE__);
DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename);
TRACE_OUT(P->A);
return DPS_ERROR;
}
if ((wr = read(P->Ifd, &P->Item, sizeof(DPS_BASEITEM))) != sizeof(DPS_BASEITEM)) {
DpsLog(P->A, DPS_LOG_ERROR, "Can't read previous pos for file %s (%s:%d)", P->Ifilename, __FILE__, __LINE__);
DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename);
TRACE_OUT(P->A);
return DPS_ERROR;
}
P->Item.next = 0;
if (lseek(P->Ifd, (off_t)P->PreviousItemPos, SEEK_SET) == (off_t)-1) {
DpsLog(P->A, DPS_LOG_ERROR, "Can't seeek for file %s (%s:%d)", P->Ifilename, __FILE__, __LINE__);
DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename);
TRACE_OUT(P->A);
return DPS_ERROR;
}
if ((wr = write(P->Ifd, &P->Item, sizeof(DPS_BASEITEM))) != sizeof(DPS_BASEITEM)) {
DpsLog(P->A, DPS_LOG_ERROR, "Can't write previous pos for file %s (%s:%d)", P->Ifilename, __FILE__, __LINE__);
DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename);
TRACE_OUT(P->A);
return DPS_ERROR;
}
goto search_again;
} else {
DpsLog(P->A, DPS_LOG_ERROR, "Can't read hash chain for file %s %d of %d bytes (%s:%d)",
P->Ifilename, wr, sizeof(DPS_BASEITEM), __FILE__, __LINE__);
DPS_FREE(P->Ifilename); DPS_FREE(P->Sfilename);
TRACE_OUT(P->A);
return DPS_ERROR;
}
}
#ifdef DEBUG_SEARCH
stop_ticks = DpsStartTimer();
total_ticks = stop_ticks - start_ticks;
DpsLog(P->A, DPS_LOG_EXTRA, "OpenBase3 %03X in %.5f sec.", P->FileNo, (float)total_ticks / 1000);
#endif
}
break;
search_again:;
}
P->opened = 1;
P->mode = mode;
#ifdef DEBUG_SEARCH
stop_ticks = DpsStartTimer();
total_ticks = stop_ticks - start_ticks;
DpsLog(P->A, DPS_LOG_EXTRA, "OpenBase4 %03X in %.5f sec.\n", P->FileNo, (float)total_ticks / 1000);
#endif
/* fprintf(stderr, "Sfd:0x%x - %s\n", P->Sfd, P->Sfilename);
fprintf(stderr, "Ifd:0x%x - %s\n", P->Ifd, P->Ifilename);*/
TRACE_OUT(P->A);
return DPS_OK;
}
int main(int argc, char ** argv, char **envp) {
const char *env, *bcharset, *lcharset, *conf_dir;
char template_name[PATH_MAX+6]="";
char *template_filename = NULL;
char *query_string = NULL;
char self[1024]="";
char *url = NULL;
const char *ResultContentType;
int res,httpd=0;
size_t catcolumns = 0;
int page_size,page_number;
DPS_ENV *Env;
DPS_AGENT *Agent;
DPS_VARLIST query_vars;
/* Output Content-type if under HTTPD */
/* Some servers do not pass QUERY_STRING */
/* if the query was empty, so check */
/* REQUEST_METHOD too to be safe */
httpd=(getenv("QUERY_STRING")||getenv("REQUEST_METHOD"));
if (!(conf_dir=getenv("DPS_ETC_DIR")))
conf_dir=DPS_CONF_DIR;
DpsInit(argc, argv, envp);
Env=DpsEnvInit(NULL);
if (Env == NULL) {
if(httpd){
printf("Content-Type: text/plain\r\n\r\n");
}
printf("Can't alloc Env\n");
exit(0);
}
DpsVarListInit(&query_vars);
Agent = DpsAgentInit(NULL, Env, 0);
if (Agent == NULL) {
if(httpd){
printf("Content-Type: text/plain\r\n\r\n");
}
printf("Can't alloc Agent\n");
exit(0);
}
DpsVarListAddEnviron(&Env->Vars,"ENV");
/* Detect self and template name */
if((env = getenv("DPSEARCH_TEMPLATE")))
dps_strncpy(template_name, env, sizeof(template_name) - 1);
else if((env = getenv("PATH_INFO")) && env[0])
dps_strncpy(template_name, env + 1, sizeof(template_name) - 1);
if((env=getenv("DPSEARCH_SELF")))
dps_strncpy(self,env,sizeof(self)-1);
if((env=getenv("QUERY_STRING"))){
query_string = (char*)DpsRealloc(query_string, dps_strlen(env) + 2);
if (query_string == NULL) {
if(httpd){
printf("Content-Type: text/plain\r\n\r\n");
}
printf("Can't alloc query_string\n");
exit(0);
}
dps_strncpy(query_string, env, dps_strlen(env) + 1);
/* Hack for Russian Apache from apache.lexa.ru */
/* QUERY_STRING is already converted to server */
/* character set. We must print original query */
/* string instead however. Under usual apache */
/* we'll use QUERY_STRING. Note that query_vars */
/* list will contain not unescaped values, so */
/* we don't have to escape them when displaying */
env = getenv("CHARSET_SAVED_QUERY_STRING");
DpsParseQStringUnescaped(&query_vars,env?env:query_string);
/* Unescape and save variables from QUERY_STRING */
/* Env->Vars will have unescaped values however */
DpsParseQueryString(Agent,&Env->Vars,query_string);
template_filename = (char*)DpsStrdup(DpsVarListFindStr(&Env->Vars, "tmplt", ""));
if((env=getenv("REDIRECT_STATUS"))){
/* Check Apache internal redirect */
/* via "AddHandler" and "Action" */
if(!self[0]){
dps_strncpy(self,(env=getenv("REDIRECT_URL"))?env:"filler.cgi",sizeof(self)-1);
}
if(!template_name[0]){
dps_strncpy(template_name,(env=getenv("PATH_TRANSLATED"))?env:"",sizeof(template_name)-1);
}
if (*template_filename == '\0') {
DPS_FREE(template_filename);
template_filename = (char*)DpsStrdup("filler.htm");
}
}else{
/* CGI executed without Apache internal redirect */
/* Detect $Self variable with OS independant SLASHES */
if(!self[0]){
dps_strncpy(self,(env=getenv("SCRIPT_NAME"))?env:"filler.cgi",sizeof(self)-1);
}
if(!template_name[0]){
char *s,*e;
/*This is with OS specific SLASHES */
env=((env=getenv("SCRIPT_FILENAME"))?env:"filler.cgi");
if(strcmp(conf_dir,".")){
/* Take from the config directory */
dps_snprintf(template_name, sizeof(template_name)-1, "%s/%s",
conf_dir,(s=strrchr(env,DPSSLASH))?(s+1):(self));
}else{
/* Take from the current directory */
dps_strncpy(template_name,env,sizeof(template_name)-1);
}
/* Find right slash if it presents */
s=((s=strrchr(template_name,DPSSLASH))?s:template_name);
if (*template_filename == '\0') {
/* Find .cgi substring */
if ((e = strstr(s, ".cgi")) != NULL) {
/* Replace ".cgi" with ".htm" */
e[1]='h';e[2]='t';e[3]='m';
} else {
dps_strcat(s, ".htm");
}
e = strrchr(s, '/');
DPS_FREE(template_filename);
template_filename = (char*)DpsStrdup(e + 1);
} else {
dps_strncpy(s + 1, template_filename, sizeof(template_name) - (s - template_name) - 2);
}
}
}
}else{
/* Executed from command line */
/* or under server which does not */
/* pass an empty QUERY_STRING var */
if(argv[1]) {
query_string = (char*)DpsRealloc(query_string, dps_strlen(argv[1]) + 10);
if (query_string == NULL) {
if(httpd){
printf("Content-Type: text/plain\r\n\r\n");
}
printf("Can't realloc query_string\n");
exit(0);
}
sprintf(query_string, "q=%s", argv[1]);
} else {
query_string = (char*)DpsRealloc(query_string, 1024);
if (query_string == NULL) {
if(httpd){
printf("Content-Type: text/plain\r\n\r\n");
}
printf("Can't realloc query_string\n");
exit(0);
}
sprintf(query_string, "q=");
}
/* Hack for Russian Apache from apache.lexa.ru */
/* QUERY_STRING is already converted to server */
/* character set. We must print original query */
/* string instead however. Under usual apache */
/* we'll use QUERY_STRING. Note that query_vars */
/* list will contain not unescaped values, so */
/* we don't have to escape them when displaying */
env = getenv("CHARSET_SAVED_QUERY_STRING");
DpsParseQStringUnescaped(&query_vars,env?env:query_string);
/* Unescape and save variables from QUERY_STRING */
/* Env->Vars will have unescaped values however */
DpsParseQueryString(Agent,&Env->Vars,query_string);
DPS_FREE(template_filename);
template_filename = (char*)DpsStrdup(DpsVarListFindStr(&Env->Vars, "tmplt", ""));
if (*template_filename == '\0') {
DPS_FREE(template_filename); template_filename = (char*)DpsStrdup("filler.htm");
}
/*// Get template name from command line variable &tmplt */
if(!template_name[0])
dps_snprintf(template_name,sizeof(template_name),"%s/%s", conf_dir, template_filename);
}
DpsVarListReplaceStr(&Agent->Conf->Vars, "tmplt", template_filename);
DPS_FREE(template_filename);
Agent->tmpl.Env_Vars = &Env->Vars;
DpsURLNormalizePath(template_name);
if (strncmp(template_name, conf_dir, dps_strlen(conf_dir))
|| (res = DpsTemplateLoad(Agent, Env, &Agent->tmpl, template_name))) {
if (strcmp(template_name, "filler.htm")) { /* trying load default template */
fprintf(stderr, "Can't load template: '%s' %s\n", template_name, Env->errstr);
DPS_FREE(template_filename);
template_filename = (char*)DpsStrdup("filler.htm");
dps_snprintf(template_name, sizeof(template_name), "%s/%s", conf_dir, template_filename);
if ((res = DpsTemplateLoad(Agent, Env, &Agent->tmpl, template_name))) {
if(httpd)printf("Content-Type: text/plain\r\n\r\n");
printf("%s\n",Env->errstr);
DpsVarListFree(&query_vars);
DpsEnvFree(Env);
DPS_FREE(query_string);
DpsAgentFree(Agent);
return(0);
}
} else {
if(httpd)printf("Content-Type: text/plain\r\n\r\n");
printf("%s\n",Env->errstr);
DpsVarListFree(&query_vars);
DpsEnvFree(Env);
DPS_FREE(query_string);
DpsAgentFree(Agent);
return(0);
}
}
/* set locale if specified */
if ((url = DpsVarListFindStr(&Env->Vars, "Locale", NULL)) != NULL) {
setlocale(LC_ALL, url);
/*#ifdef HAVE_ASPELL*/
{ char *p;
if ((p = strchr(url, '.')) != NULL) {
*p = '\0';
DpsVarListReplaceStr(&Env->Vars, "g-lc", url);
*p = '.';
}
}
/*#endif*/
url = NULL;
}
/* Call again to load search Limits if need */
DpsParseQueryString(Agent, &Env->Vars, query_string);
Agent->Flags = Env->Flags;
Agent->flags |= DPS_FLAG_UNOCON;
Env->flags |= DPS_FLAG_UNOCON;
DpsSetLogLevel(NULL, DpsVarListFindInt(&Env->Vars, "LogLevel", 0));
DpsOpenLog("filler.cgi", Env, !strcasecmp(DpsVarListFindStr(&Env->Vars, "Log2stderr", (!httpd) ? "yes" : "no"), "yes"));
DpsLog(Agent,DPS_LOG_ERROR,"filler.cgi started with '%s'",template_name);
DpsLog(Agent, DPS_LOG_DEBUG, "VarDir: '%s'", DpsVarListFindStr(&Agent->Conf->Vars, "VarDir", DPS_VAR_DIR));
DpsLog(Agent, DPS_LOG_DEBUG, "Affixes: %d, Spells: %d, Synonyms: %d, Acronyms: %d, Stopwords: %d",
Env->Affixes.naffixes,Env->Spells.nspell,
Env->Synonyms.nsynonyms,
Env->Acronyms.nacronyms,
Env->StopWords.nstopwords);
DpsLog(Agent, DPS_LOG_DEBUG, "Chinese dictionary with %d entries", Env->Chi.nwords);
DpsLog(Agent, DPS_LOG_DEBUG, "Korean dictionary with %d entries", Env->Korean.nwords);
DpsLog(Agent, DPS_LOG_DEBUG, "Thai dictionary with %d entries", Env->Thai.nwords);
DpsVarListAddLst(&Agent->Vars, &Env->Vars, NULL, "*");
Agent->tmpl.Env_Vars = &Agent->Vars;
/* DpsVarListAddEnviron(&Agent->Vars, "ENV");*/
/****************************************************************************************************************************************/
/* This is for query tracking */
DpsVarListAddStr(&Agent->Vars, "QUERY_STRING", query_string);
DpsVarListAddStr(&Agent->Vars, "self", self);
env = getenv("HTTP_X_FORWARDER_FOR");
if (env) {
DpsVarListAddStr(&Agent->Vars, "IP", env);
} else {
env = getenv("REMOTE_ADDR");
DpsVarListAddStr(&Agent->Vars, "IP", env ? env : "localhost");
}
bcharset = DpsVarListFindStr(&Agent->Vars, "BrowserCharset", "iso-8859-1");
Env->bcs=DpsGetCharSet(bcharset);
lcharset = DpsVarListFindStr(&Agent->Vars, "LocalCharset", "iso-8859-1");
Env->lcs=DpsGetCharSet(lcharset);
ResultContentType = DpsVarListFindStr(&Agent->Vars, "ResultContentType", "text/html");
if(httpd){
if(!Env->bcs){
printf("Content-Type: text/plain\r\n\r\n");
printf("Unknown BrowserCharset '%s' in template '%s'\n",bcharset,template_name);
exit(0);
}else if(!Env->lcs){
printf("Content-Type: text/plain\r\n\r\n");
printf("Unknown LocalCharset '%s' in template '%s'\n",lcharset,template_name);
exit(0);
}else{
printf("Content-type: %s; charset=%s\r\n\r\n", ResultContentType, bcharset);
}
}else{
if(!Env->bcs){
printf("Unknown BrowserCharset '%s' in template '%s'\n",bcharset,template_name);
exit(0);
}
if(!Env->lcs){
printf("Unknown LocalCharset '%s' in template '%s'\n",lcharset,template_name);
exit(0);
}
}
/* These parameters taken from "variable section of template"*/
res = DpsVarListFindInt(&Agent->Vars, "ps", DPS_DEFAULT_PS);
page_size = dps_min(res, MAX_PS);
page_number = DpsVarListFindInt(&Agent->Vars, "p", 0);
if (page_number == 0) {
page_number = DpsVarListFindInt(&Agent->Vars, "np", 0);
DpsVarListReplaceInt(&Agent->Vars, "p", page_number + 1);
} else page_number--;
res = DpsVarListFindInt(&Agent->Vars, "np", 0) * page_size;
DpsVarListAddInt(&Agent->Vars, "pn", res);
catcolumns = (size_t)atoi(DpsVarListFindStr(&Agent->Vars, "CatColumns", ""));
DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "top");
DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "restop");
DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "res");
DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "resbot");
DpsTemplatePrint(Agent, (DPS_OUTPUTFUNCTION)&fprintf, stdout, NULL, 0, &Agent->tmpl, "bottom");
DpsVarListFree(&query_vars);
DpsAgentFree(Agent);
DpsEnvFree(Env);
DPS_FREE(query_string);
DPS_FREE(url);
if (httpd) fflush(NULL); else fclose(stdout);
#ifdef EFENCE
fprintf(stderr, "Memory leaks checking\n");
DpsEfenceCheckLeaks();
#endif
#ifdef FILENCE
fprintf(stderr, "FD leaks checking\n");
DpsFilenceCheckLeaks(NULL);
#endif
return DPS_OK;
}
__C_LINK int __DPSCALL DpsCacheMakeIndexes(DPS_AGENT *Indexer, DPS_DB *db) {
DPS_UINT8URLIDLIST L8;
DPS_UINT4URLIDLIST L4;
DPS_VARLIST *v = &Indexer->Conf->Vars;
size_t i, r;
char *ind, *nm, *lfname;
bzero(&L4, sizeof(DPS_UINT4URLIDLIST));
bzero(&L8, sizeof(DPS_UINT8URLIDLIST));
r = (size_t) 'l';
for (i = 0; i < v->Root[r].nvars; i++) {
if (!strncasecmp("Limit-", v->Root[r].Var[i].name, 6)) {
ind = v->Root[r].Var[i].val;
lfname = v->Root[r].Var[i].name;
nm = lfname + 6;
if (!strcasecmp(ind, "category")) {
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Category index creation", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating category index");
if (DPS_OK == DpsLimit8(Indexer, &L8, "Category", DPS_IFIELD_TYPE_HEX8STR, db)) {
MakeNestedIndex(Indexer, &L8, DPS_LIMFNAME_CAT, db);
}
} else if (!strcasecmp(ind, "tag")) {
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Tag index creation", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating tag index");
if (DPS_OK == DpsLimit4(Indexer, &L4, "Tag", DPS_IFIELD_TYPE_STRCRC32, db)) {
MakeLinearIndex(Indexer, &L4, DPS_LIMFNAME_TAG, db);
}
} else if (!strcasecmp(ind, "link")) {
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Link index creation", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating link index");
if (DPS_OK == DpsLimit4(Indexer, &L4, "link", DPS_IFIELD_TYPE_INT, db)) {
MakeLinearIndex(Indexer, &L4, DPS_LIMFNAME_LINK, db);
}
} else if (!strcasecmp(ind, "time")) {
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Time index creation", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating time index");
if (DPS_OK == DpsLimit4(Indexer, &L4, "last_mod_time", DPS_IFIELD_TYPE_HOUR, db)) {
MakeLinearIndex(Indexer, &L4, DPS_LIMFNAME_TIME, db);
}
} else if (!strcasecmp(ind, "hostname")) {
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Hostname index creation", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating hostname index");
if (DPS_OK == DpsLimit4(Indexer, &L4, "url", DPS_IFIELD_TYPE_HOSTNAME, db)) {
MakeLinearIndex(Indexer, &L4, DPS_LIMFNAME_HOST, db);
}
} else if (!strcasecmp(ind, "language")) {
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Language index creation", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating language index");
if (DPS_OK == DpsLimit4(Indexer, &L4, "Content-Language", DPS_IFIELD_TYPE_STR2CRC32, db)) {
MakeLinearIndex(Indexer, &L4, DPS_LIMFNAME_LANG, db);
}
} else if (!strcasecmp(ind, "content")) {
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Content-Type index creation", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating Content-Type index");
if (DPS_OK == DpsLimit4(Indexer, &L4, "Content-Type", DPS_IFIELD_TYPE_STRCRC32, db)) {
MakeLinearIndex(Indexer, &L4, DPS_LIMFNAME_CTYPE, db);
}
} else if (!strcasecmp(ind, "siteid")) {
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Site_id index creation", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating Site_id index");
if (DPS_OK == DpsLimit4(Indexer, &L4, "site_id", DPS_IFIELD_TYPE_INT, db)) {
MakeLinearIndex(Indexer, &L4, DPS_LIMFNAME_SITE, db);
}
} else {
char *buf, *req, *dbaddr;
DPS_DB ldb, *pdb = &ldb;
size_t buf_len = dps_strlen(nm) + 16;
if ((buf = (char*) DpsMalloc(buf_len * sizeof(char))) == NULL) {
DpsLog(Indexer, DPS_LOG_ERROR, "Can't alloc %d chars at %s:%d", buf_len, __FILE__, __LINE__);
return DPS_ERROR;
}
dps_setproctitle("[%d] %s index creation", Indexer->handle, nm);
DpsLog(Indexer, DPS_LOG_EXTRA, "Creating %s index", nm);
dps_snprintf(buf, buf_len, "Req-%s", nm);
req = DpsVarListFindStr(&Indexer->Conf->Vars, buf, NULL);
if (req != NULL) {
dps_snprintf(buf, buf_len, "dbaddr-%s", nm);
dbaddr = DpsVarListFindStr(&Indexer->Conf->Vars, buf, NULL);
if (dbaddr != NULL) {
DpsDBSetAddr(pdb, dbaddr, DPS_OPEN_MODE_READ);
} else {
pdb = db;
}
if (!strcasecmp(ind, "nex8str")) {
if (DPS_OK == DpsSQLLimit8(Indexer, &L8, req, DPS_IFIELD_TYPE_HEX8STR, pdb)) {
MakeNestedIndex(Indexer, &L8, lfname, pdb);
}
} else {
int field_type = DPS_IFIELD_TYPE_INT;
if (!strcasecmp(ind, "strcrc32")) field_type = DPS_IFIELD_TYPE_STRCRC32;
else if (!strcasecmp(ind, "hour")) field_type = DPS_IFIELD_TYPE_HOUR;
else if (!strcasecmp(ind, "hostname")) field_type = DPS_IFIELD_TYPE_HOSTNAME;
else if (!strcasecmp(ind, "char2")) field_type = DPS_IFIELD_TYPE_STR2CRC32;
else if (!strcasecmp(ind, "int")) field_type = DPS_IFIELD_TYPE_INT;
if (DPS_OK == DpsSQLLimit4(Indexer, &L4, req, field_type, db)) {
MakeLinearIndex(Indexer, &L4, lfname, db);
}
}
}
}
/* To see the URL being indexed in "ps" output on xBSD */
dps_setproctitle("[%d] Indexes done.", Indexer->handle);
DpsLog(Indexer, DPS_LOG_EXTRA, "Done");
}
}
return DPS_OK;
}
int DpsCookiesAdd(DPS_AGENT *Indexer, const char *domain, const char * path, const char *name, const char *value, const char secure,
dps_uint4 expires, const char from_config, int insert_flag) {
#ifdef HAVE_SQL
char buf[3*PATH_MAX];
char path_esc[2*PATH_MAX+1];
DPS_COOKIES *Cookies = &Indexer->Cookies;
DPS_COOKIE *Coo;
DPS_DB *db;
dpshash32_t url_id = DpsStrHash32(domain);
size_t i;
#ifdef WITH_PARANOIA
void *paran = DpsViolationEnter(paran);
#endif
if (Indexer->flags & DPS_FLAG_UNOCON) {
if (Indexer->Conf->dbl.nitems == 0) return DPS_OK;
DPS_GETLOCK(Indexer, DPS_LOCK_DB);
db = Indexer->Conf->dbl.db[url_id % Indexer->Conf->dbl.nitems];
} else {
if (Indexer->dbl.nitems == 0) return DPS_OK;
db = Indexer->dbl.db[url_id % Indexer->dbl.nitems];
}
(void)DpsDBEscStr(db, path_esc, DPS_NULL2EMPTY(path), dps_min(PATH_MAX,dps_strlen(DPS_NULL2EMPTY(path))));
for (i = 0; i < Cookies->ncookies; i++) {
Coo = &Cookies->Cookie[i];
if (!strcasecmp(Coo->domain, domain) && !strcasecmp(Coo->path, DPS_NULL2EMPTY(path)) && !strcasecmp(Coo->name, name) && (Coo->secure == secure)/* && (Coo->from_config == from_config)*/ ) {
DPS_FREE(Coo->value);
Coo->value = DpsStrdup(value);
/* Coo->expires = expires;*/
if (insert_flag) {
dps_snprintf(buf, sizeof(buf), "UPDATE cookies SET value='%s',expires=%d WHERE domain='%s' AND path='%s' AND name='%s' AND secure='%c'",
value, expires, domain, path_esc, name, secure);
DpsSQLAsyncQuery(db, NULL, buf);
}
if (Indexer->flags & DPS_FLAG_UNOCON) DPS_RELEASELOCK(Indexer, DPS_LOCK_DB);
#ifdef WITH_PARANOIA
DpsViolationExit(Indexer->handle, paran);
#endif
return DPS_OK;
}
}
Cookies->Cookie = (DPS_COOKIE*)DpsRealloc(Cookies->Cookie, (Cookies->ncookies + 1) * sizeof(DPS_COOKIE));
if(Cookies->Cookie == NULL) {
Cookies->ncookies = 0;
if (Indexer->flags & DPS_FLAG_UNOCON) DPS_RELEASELOCK(Indexer, DPS_LOCK_DB);
#ifdef WITH_PARANOIA
DpsViolationExit(Indexer->handle, paran);
#endif
return DPS_ERROR;
}
Coo = &Cookies->Cookie[Cookies->ncookies];
/* Coo->expires = expires;*/
Coo->secure = secure;
Coo->from_config = from_config;
Coo->domain = DpsStrdup(domain);
Coo->path = DpsStrdup(path);
Coo->name = DpsStrdup(name);
Coo->value = DpsStrdup(value);
if (insert_flag) {
if (Indexer->Flags.CheckInsertSQL) {
dps_snprintf(buf, sizeof(buf), "DELETE FROM cookies WHERE domain='%s' AND path='%s' AND name='%s' AND secure='%c'",
domain, path_esc, name, secure);
DpsSQLAsyncQuery(db, NULL, buf);
}
dps_snprintf(buf, sizeof(buf), "INSERT INTO cookies(expires,secure,domain,path,name,value)VALUES(%d,'%c','%s','%s','%s','%s')",
expires, secure, domain, path_esc, name, value);
DpsSQLAsyncQuery(db, NULL, buf);
}
Cookies->ncookies++;
if (Indexer->flags & DPS_FLAG_UNOCON) DPS_RELEASELOCK(Indexer, DPS_LOCK_DB);
#ifdef WITH_PARANOIA
DpsViolationExit(Indexer->handle, paran);
#endif
#endif /*HAVE_SQL*/
return DPS_OK;
}
