static void DpsParseHTTPHeader(DPS_AGENT *Indexer, DPS_DOCUMENT *Doc, DPS_DSTR *header) {
char *val, *header_name;
char secname[128], savec;
DPS_VAR *Sec;
DPS_TEXTITEM Item;
if ((val = strchr(header_name = header->data, ':'))) {
/*
fprintf(stderr, "HEADER: %s\n", header_name);
*/
*val++='\0';
val = DpsTrim(val," \t:");
if (!strcasecmp(header_name, "Content-Type") || !strcasecmp(header_name, "Content-Encoding")) {
register char *v;
for(v=val ; *v ; v++)
*v = (char)dps_tolower((int)*v);
} else if (Doc->Spider.use_robots && !strcasecmp(header_name, "X-Robots-Tag")) {
char * lt;
char * rtok;
rtok = dps_strtok_r(val, " ,\r\n\t", <, &savec);
while(rtok){
if(!strcasecmp(rtok, "ALL")){
/* Left Server parameters unchanged */
}else if(!strcasecmp(rtok, "NONE")){
Doc->Spider.follow = DPS_FOLLOW_NO;
Doc->Spider.index = 0;
if (DpsNeedLog(DPS_LOG_DEBUG)) {
DpsVarListReplaceInt(&Doc->Sections, "Index", 0);
DpsVarListReplaceInt(&Doc->Sections, "Follow", DPS_FOLLOW_NO);
}
}else if(!strcasecmp(rtok, "NOINDEX")) {
Doc->Spider.index = 0;
/* Doc->method = DPS_METHOD_DISALLOW;*/
if (DpsNeedLog(DPS_LOG_DEBUG)) DpsVarListReplaceInt(&Doc->Sections, "Index", 0);
}else if(!strcasecmp(rtok, "NOFOLLOW")) {
Doc->Spider.follow = DPS_FOLLOW_NO;
if (DpsNeedLog(DPS_LOG_DEBUG)) DpsVarListReplaceInt(&Doc->Sections, "Follow", DPS_FOLLOW_NO);
}else if(!strcasecmp(rtok, "NOARCHIVE")) {
DpsVarListReplaceStr(&Doc->Sections, "Z", "");
}else if(!strcasecmp(rtok, "INDEX")) {
/* left server value unchanged */
if (DpsNeedLog(DPS_LOG_DEBUG)) DpsVarListReplaceInt(&Doc->Sections, "Index", Doc->Spider.index);
}else if(!strcasecmp(rtok, "FOLLOW")) {
/* left server value unchanged */
if (DpsNeedLog(DPS_LOG_DEBUG)) DpsVarListReplaceInt(&Doc->Sections, "Follow", Doc->Spider.follow);
}
rtok = dps_strtok_r(NULL, " \r\n\t", <, &savec);
}
} else if (Doc->Spider.use_cookies && !strcasecmp(header_name, "Set-Cookie")) {
DpsCookiesAddStr(Indexer, &Doc->CurURL, val, 1);
return;
}
}
DpsVarListReplaceStr(&Doc->Sections, header_name, val ? val : "<NULL>");
dps_snprintf(secname,sizeof(secname),"header.%s", header_name);
secname[sizeof(secname)-1]='\0';
if((Sec = DpsVarListFind(&Doc->Sections, secname)) && val ) {
bzero((void*)&Item, sizeof(Item));
Item.href = NULL;
Item.str = val;
Item.section = Sec->section;
Item.section_name = secname;
Item.strict = Sec->strict;
Item.len = 0;
(void)DpsTextListAdd(&Doc->TextList, &Item);
}
}
static void DpsParseHTTPHeader(DPS_AGENT *Indexer, DPS_DOCUMENT *Doc, DPS_DSTR *header) {
char *val, *header_name;
char secname[128];
DPS_VAR *Sec;
DPS_TEXTITEM Item;
if ((val = strchr(header_name = header->data, ':'))) {
/*
fprintf(stderr, "HEADER: %s\n", header_name);
*/
*val++='\0';
val = DpsTrim(val," \t:");
if (!strcasecmp(header_name, "Content-Type") || !strcasecmp(header_name, "Content-Encoding")) {
char *v;
for(v=val ; *v ; v++)
*v = dps_tolower(*v);
} else if (Doc->Spider.use_cookies && !strcasecmp(header_name, "Set-Cookie")) {
char *part, *lpart;
char *name = NULL;
char *value = NULL;
const char *domain = NULL;
const char *path = NULL;
dps_uint4 expire = 0;
char secure = 'n';
for (part = dps_strtok_r(val, ";" , &lpart) ; part;
part = dps_strtok_r(NULL, ";", &lpart)) {
char *arg;
part = DpsTrim(part, " ");
if ((arg = strchr(part, '='))) {
*arg++ = '\0';
if (!name) {
name = part;
value = arg;
} else
if (!strcasecmp(part, "path")) {
path = arg;
} else
if (!strcasecmp(part, "domain")) {
domain = arg;
} else
if (!strcasecmp(part, "secure")) {
secure = 'y';
} else
if (!strcasecmp(part, "expires")) {
expire = (dps_uint4)DpsHttpDate2Time_t(arg);
}
}
}
if (name && value) {
if (domain && domain[0] == '.') {
domain++;
} else {
domain = Doc->CurURL.hostname ? Doc->CurURL.hostname : "localhost";
}
if (!path) {
path = Doc->CurURL.path ? Doc->CurURL.path : "/";
}
DpsCookiesAdd(Indexer, domain, path, name, value, secure, expire, 1);
}
/* token = dps_strtok_r(NULL,"\r\n",<);
continue;*/
return;
}
}
DpsVarListReplaceStr(&Doc->Sections, header_name, val ? val : "<NULL>");
dps_snprintf(secname,sizeof(secname),"header.%s", header_name);
secname[sizeof(secname)-1]='\0';
if((Sec = DpsVarListFind(&Doc->Sections, secname)) && val ) {
Item.href = NULL;
Item.str = val;
Item.section = Sec->section;
Item.section_name = secname;
Item.len = 0;
DpsTextListAdd(&Doc->TextList, &Item);
}
}
int DpsSEAMake(DPS_AGENT *Indexer, DPS_DOCUMENT *Doc, DPS_DSTR *excerpt,
const char *content_lang, size_t *indexed_size, size_t *indexed_limit,
size_t max_word_len, size_t min_word_len, int crossec, int seasec
#ifdef HAVE_ASPELL
, int have_speller, AspellSpeller *speller
#endif
) {
DPS_SENTENCELIST List;
DPS_MAPSTAT MapStat;
DPS_TEXTITEM Item;
DPS_VAR *Sec;
dpsunicode_t *sentence, *lt, savec;
double *links, *lang_cs, w;
double delta, pdiv, cur_div;
size_t l, sent_len, order;
size_t min_len = 10000000, min_pos = 0;
int it;
register size_t i, j;
#ifdef DEBUG
char lcstr[4096];
#endif
TRACE_IN(Indexer, "DpsSEAMake");
if((Sec = DpsVarListFind(&Doc->Sections, "sea"))) { /* set SEA section to NULL */
DPS_FREE(Sec->val);
DPS_FREE(Sec->txt_val);
Sec->curlen = 0;
}
bzero(&List, sizeof(List));
order = 0;
sentence = DpsUniStrTok_SEA((dpsunicode_t*)excerpt->data, <);
while(sentence) {
if (lt != NULL) { savec = *lt; *lt = 0; }
#ifdef DEBUG
DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)sentence, sizeof(dpsunicode_t) * (DpsUniLen(sentence) + 1));
fprintf(stderr, "Sentence.%d: %s\n", List.nitems, lcstr);
#endif
if ((sent_len = DpsUniLen(sentence)) >= Indexer->Flags.SEASentenceMinLength) {
j = 1;
for (i = 0; i < List.nitems; i++) {
if (DpsUniStrCmp(sentence, List.Sent[i].sentence) == 0) {
j = 0; break;
}
}
if (j) {
if ( List.nitems < Indexer->Flags.SEASentences ) {
if (List.nitems == List.mitems) {
List.mitems += 16;
List.Sent = (DPS_SENTENCE*)DpsRealloc(List.Sent, List.mitems * sizeof(DPS_SENTENCE));
if (List.Sent == NULL) { TRACE_OUT(Indexer); return DPS_ERROR;}
}
List.Sent[List.nitems].sentence = DpsUniDup(sentence);
List.Sent[List.nitems].len = sent_len;
List.Sent[List.nitems].order = order++;
sentence = DpsUniDup(sentence);
DpsUniStrToLower(sentence);
bzero(&List.Sent[List.nitems].LangMap, sizeof(DPS_LANGMAP));
DpsBuildLangMap(&List.Sent[List.nitems].LangMap, (char*)sentence, sent_len * sizeof(dpsunicode_t), 0, 0);
if (sent_len < min_len) { min_len = sent_len; min_pos = List.nitems; }
List.nitems++;
DPS_FREE(sentence);
} else if (sent_len > min_len) {
DPS_FREE(List.Sent[min_pos].sentence);
List.Sent[min_pos].sentence = DpsUniDup(sentence);
List.Sent[min_pos].len = sent_len;
List.Sent[min_pos].order = order++;
sentence = DpsUniDup(sentence);
DpsUniStrToLower(sentence);
bzero(&List.Sent[min_pos].LangMap, sizeof(DPS_LANGMAP));
DpsBuildLangMap(&List.Sent[min_pos].LangMap, (char*)sentence, sent_len * sizeof(dpsunicode_t), 0, 0);
DPS_FREE(sentence);
min_len = List.Sent[0].len; min_pos = 0;
for(i = 1; i < List.nitems; i++) if (List.Sent[i].len < min_len) { min_len = List.Sent[i].len; min_pos = i; }
}
}
}
#ifdef DEBUG
fprintf(stderr, "Sent. len.:%d, Min.allowed: %d\n", sent_len, Indexer->Flags.SEASentenceMinLength);
#endif
if (lt != NULL) *lt = savec;
sentence = DpsUniStrTok_SEA(NULL, <);
}
DpsLog(Indexer, DPS_LOG_DEBUG, "SEA sentences: %d", List.nitems);
if (List.nitems < 4) {
for (i = 0; i < List.nitems; i++) DPS_FREE(List.Sent[i].sentence);
DPS_FREE(List.Sent);
TRACE_OUT(Indexer);
return DPS_OK;
}
links = (double*)DpsMalloc(sizeof(double) * List.nitems * List.nitems);
lang_cs = (double*)DpsMalloc(sizeof(double) * List.nitems);
/*
k ot
links[i * List.nitems + j]
*/
if (links != NULL && lang_cs != NULL) {
for (i = 0; i < List.nitems; i++) {
DpsPrepareLangMap(&List.Sent[i].LangMap);
}
for (i = 0; i < List.nitems; i++) {
List.Sent[i].Oi = List.Sent[i].di = 0.5;
if (Doc->lang_cs_map == NULL) {
links[i * List.nitems + i] = 0.0;
} else {
MapStat.map = &List.Sent[i].LangMap;
DpsCheckLangMap6(Doc->lang_cs_map, &List.Sent[i].LangMap, &MapStat, DPS_LM_TOPCNT * DPS_LM_TOPCNT, 2 * DPS_LM_TOPCNT);
links[i * List.nitems + i] = (double)MapStat.hits / (2.0 * DPS_LM_TOPCNT) / (List.nitems + 1);
}
#ifdef DEBUG
DpsLog(Indexer, DPS_LOG_INFO, "Link %u->%u: %f [hits:%d miss:%d]", i, i, links[i * List.nitems + i], MapStat.hits, MapStat.miss);
#endif
for (j = 0; j < List.nitems; j++) {
if (j == i) continue;
MapStat.map = &List.Sent[j].LangMap;
DpsCheckLangMap6(&List.Sent[j].LangMap, &List.Sent[i].LangMap, &MapStat, DPS_LM_TOPCNT * DPS_LM_TOPCNT, 2 * DPS_LM_TOPCNT);
links[i * List.nitems + j] = (double)MapStat.hits / (2.0 * DPS_LM_TOPCNT) / (List.nitems + 1);
#ifdef DEBUG
DpsLog(Indexer, DPS_LOG_INFO, "Link %u->%u: %f [hits:%d miss:%d]", i, j, links[i * List.nitems + j], MapStat.hits, MapStat.miss);
#endif
}
}
for (l = 0; l < List.nitems; l++) {
w = 0.0;
for (i = 0; i < List.nitems; i++) {
w += links[l * List.nitems + i] * List.Sent[i].Oi;
}
w = f(w);
if (w < LOW_BORDER_EPS2) w = LOW_BORDER_EPS2;
else if (w > HI_BORDER_EPS2) w = HI_BORDER_EPS2;
List.Sent[l].di = w;
}
DpsSort(List.Sent, List.nitems, sizeof(DPS_SENTENCE), (qsort_cmp)SentCmp);
#ifdef DEBUG
DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)List.Sent[0].sentence, sizeof(dpsunicode_t) * (DpsUniLen(List.Sent[0].sentence) + 1));
fprintf(stderr, "Sent.0: %f %f -- %s\n", List.Sent[0].di, List.Sent[0].Oi, lcstr);
DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)List.Sent[1].sentence, sizeof(dpsunicode_t) * (DpsUniLen(List.Sent[1].sentence) + 1));
fprintf(stderr, "Sent.1: %f %f -- %s\n", List.Sent[1].di, List.Sent[1].Oi, lcstr);
DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)List.Sent[2].sentence, sizeof(dpsunicode_t) * (DpsUniLen(List.Sent[2].sentence) + 1));
fprintf(stderr, "Sent.2: %f %f -- %s\n", List.Sent[2].di, List.Sent[2].Oi, lcstr);
DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)List.Sent[3].sentence, sizeof(dpsunicode_t) * (DpsUniLen(List.Sent[3].sentence) + 1));
fprintf(stderr, "Sent.3: %f %f -- %s\n", List.Sent[3].di, List.Sent[3].Oi, lcstr);
DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)List.Sent[4].sentence, sizeof(dpsunicode_t) * (DpsUniLen(List.Sent[4].sentence) + 1));
fprintf(stderr, "Sent.4: %f %f -- %s\n", List.Sent[4].di, List.Sent[4].Oi, lcstr);
#endif
DpsSort(List.Sent, TOP_SENTENCES, sizeof(DPS_SENTENCE), (qsort_cmp)SentOrderCmp);
bzero(&Item, sizeof(Item));
Item.section = seasec;
Item.href = NULL;
Item.section_name = "sea";
for (i = 0; i < TOP_SENTENCES; i++) {
dpsunicode_t *UStr = DpsUniDup(List.Sent[i].sentence);
DpsPrepareItem(Indexer, Doc, &Item, List.Sent[i].sentence, UStr, content_lang, indexed_size, indexed_limit,
max_word_len, min_word_len, crossec
#ifdef HAVE_ASPELL
, have_speller, speller, NULL
#endif
);
DPS_FREE(UStr);
}
}
DPS_FREE(lang_cs);
DPS_FREE(links);
for (i = 0; i < List.nitems; i++) DPS_FREE(List.Sent[i].sentence);
DPS_FREE(List.Sent);
TRACE_OUT(Indexer);
return DPS_OK;
}
