int DpsSEAMake(DPS_AGENT *Indexer, DPS_DOCUMENT *Doc, DPS_DSTR *excerpt, const char *content_lang, size_t *indexed_size, size_t *indexed_limit, size_t max_word_len, size_t min_word_len, int crossec, int seasec #ifdef HAVE_ASPELL , int have_speller, AspellSpeller *speller #endif ) { DPS_SENTENCELIST List; DPS_MAPSTAT MapStat; DPS_TEXTITEM Item; DPS_VAR *Sec; dpsunicode_t *sentence, *lt, savec; double *links, *lang_cs, w; double delta, pdiv, cur_div; size_t l, sent_len, order; size_t min_len = 10000000, min_pos = 0; int it; register size_t i, j; #ifdef DEBUG char lcstr[4096]; #endif TRACE_IN(Indexer, "DpsSEAMake"); if((Sec = DpsVarListFind(&Doc->Sections, "sea"))) { /* set SEA section to NULL */ DPS_FREE(Sec->val); DPS_FREE(Sec->txt_val); Sec->curlen = 0; } bzero(&List, sizeof(List)); order = 0; sentence = DpsUniStrTok_SEA((dpsunicode_t*)excerpt->data, <); while(sentence) { if (lt != NULL) { savec = *lt; *lt = 0; } #ifdef DEBUG DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)sentence, sizeof(dpsunicode_t) * (DpsUniLen(sentence) + 1)); fprintf(stderr, "Sentence.%d: %s\n", List.nitems, lcstr); #endif if ((sent_len = DpsUniLen(sentence)) >= Indexer->Flags.SEASentenceMinLength) { j = 1; for (i = 0; i < List.nitems; i++) { if (DpsUniStrCmp(sentence, List.Sent[i].sentence) == 0) { j = 0; break; } } if (j) { if ( List.nitems < Indexer->Flags.SEASentences ) { if (List.nitems == List.mitems) { List.mitems += 16; List.Sent = (DPS_SENTENCE*)DpsRealloc(List.Sent, List.mitems * sizeof(DPS_SENTENCE)); if (List.Sent == NULL) { TRACE_OUT(Indexer); return DPS_ERROR;} } List.Sent[List.nitems].sentence = DpsUniDup(sentence); List.Sent[List.nitems].len = sent_len; List.Sent[List.nitems].order = order++; sentence = DpsUniDup(sentence); DpsUniStrToLower(sentence); bzero(&List.Sent[List.nitems].LangMap, sizeof(DPS_LANGMAP)); DpsBuildLangMap(&List.Sent[List.nitems].LangMap, (char*)sentence, sent_len * sizeof(dpsunicode_t), 0, 0); if (sent_len < min_len) { min_len = sent_len; min_pos = List.nitems; } List.nitems++; DPS_FREE(sentence); } else if (sent_len > min_len) { DPS_FREE(List.Sent[min_pos].sentence); List.Sent[min_pos].sentence = DpsUniDup(sentence); List.Sent[min_pos].len = sent_len; List.Sent[min_pos].order = order++; sentence = DpsUniDup(sentence); DpsUniStrToLower(sentence); bzero(&List.Sent[min_pos].LangMap, sizeof(DPS_LANGMAP)); DpsBuildLangMap(&List.Sent[min_pos].LangMap, (char*)sentence, sent_len * sizeof(dpsunicode_t), 0, 0); DPS_FREE(sentence); min_len = List.Sent[0].len; min_pos = 0; for(i = 1; i < List.nitems; i++) if (List.Sent[i].len < min_len) { min_len = List.Sent[i].len; min_pos = i; } } } } #ifdef DEBUG fprintf(stderr, "Sent. len.:%d, Min.allowed: %d\n", sent_len, Indexer->Flags.SEASentenceMinLength); #endif if (lt != NULL) *lt = savec; sentence = DpsUniStrTok_SEA(NULL, <); } DpsLog(Indexer, DPS_LOG_DEBUG, "SEA sentences: %d", List.nitems); if (List.nitems < 4) { for (i = 0; i < List.nitems; i++) DPS_FREE(List.Sent[i].sentence); DPS_FREE(List.Sent); TRACE_OUT(Indexer); return DPS_OK; } links = (double*)DpsMalloc(sizeof(double) * List.nitems * List.nitems); lang_cs = (double*)DpsMalloc(sizeof(double) * List.nitems); /* k ot links[i * List.nitems + j] */ if (links != NULL && lang_cs != NULL) { for (i = 0; i < List.nitems; i++) { DpsPrepareLangMap(&List.Sent[i].LangMap); } for (i = 0; i < List.nitems; i++) { List.Sent[i].Oi = List.Sent[i].di = 0.5; if (Doc->lang_cs_map == NULL) { links[i * List.nitems + i] = 0.0; } else { MapStat.map = &List.Sent[i].LangMap; DpsCheckLangMap6(Doc->lang_cs_map, &List.Sent[i].LangMap, &MapStat, DPS_LM_TOPCNT * DPS_LM_TOPCNT, 2 * DPS_LM_TOPCNT); links[i * List.nitems + i] = (double)MapStat.hits / (2.0 * DPS_LM_TOPCNT) / (List.nitems + 1); } #ifdef DEBUG DpsLog(Indexer, DPS_LOG_INFO, "Link %u->%u: %f [hits:%d miss:%d]", i, i, links[i * List.nitems + i], MapStat.hits, MapStat.miss); #endif for (j = 0; j < List.nitems; j++) { if (j == i) continue; MapStat.map = &List.Sent[j].LangMap; DpsCheckLangMap6(&List.Sent[j].LangMap, &List.Sent[i].LangMap, &MapStat, DPS_LM_TOPCNT * DPS_LM_TOPCNT, 2 * DPS_LM_TOPCNT); links[i * List.nitems + j] = (double)MapStat.hits / (2.0 * DPS_LM_TOPCNT) / (List.nitems + 1); #ifdef DEBUG DpsLog(Indexer, DPS_LOG_INFO, "Link %u->%u: %f [hits:%d miss:%d]", i, j, links[i * List.nitems + j], MapStat.hits, MapStat.miss); #endif } } for (l = 0; l < List.nitems; l++) { w = 0.0; for (i = 0; i < List.nitems; i++) { w += links[l * List.nitems + i] * List.Sent[i].Oi; } w = f(w); if (w < LOW_BORDER_EPS2) w = LOW_BORDER_EPS2; else if (w > HI_BORDER_EPS2) w = HI_BORDER_EPS2; List.Sent[l].di = w; } DpsSort(List.Sent, List.nitems, sizeof(DPS_SENTENCE), (qsort_cmp)SentCmp); #ifdef DEBUG DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)List.Sent[0].sentence, sizeof(dpsunicode_t) * (DpsUniLen(List.Sent[0].sentence) + 1)); fprintf(stderr, "Sent.0: %f %f -- %s\n", List.Sent[0].di, List.Sent[0].Oi, lcstr); DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)List.Sent[1].sentence, sizeof(dpsunicode_t) * (DpsUniLen(List.Sent[1].sentence) + 1)); fprintf(stderr, "Sent.1: %f %f -- %s\n", List.Sent[1].di, List.Sent[1].Oi, lcstr); DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)List.Sent[2].sentence, sizeof(dpsunicode_t) * (DpsUniLen(List.Sent[2].sentence) + 1)); fprintf(stderr, "Sent.2: %f %f -- %s\n", List.Sent[2].di, List.Sent[2].Oi, lcstr); DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)List.Sent[3].sentence, sizeof(dpsunicode_t) * (DpsUniLen(List.Sent[3].sentence) + 1)); fprintf(stderr, "Sent.3: %f %f -- %s\n", List.Sent[3].di, List.Sent[3].Oi, lcstr); DpsConv(&Indexer->uni_lc, lcstr, sizeof(lcstr), (char*)List.Sent[4].sentence, sizeof(dpsunicode_t) * (DpsUniLen(List.Sent[4].sentence) + 1)); fprintf(stderr, "Sent.4: %f %f -- %s\n", List.Sent[4].di, List.Sent[4].Oi, lcstr); #endif DpsSort(List.Sent, TOP_SENTENCES, sizeof(DPS_SENTENCE), (qsort_cmp)SentOrderCmp); bzero(&Item, sizeof(Item)); Item.section = seasec; Item.href = NULL; Item.section_name = "sea"; for (i = 0; i < TOP_SENTENCES; i++) { dpsunicode_t *UStr = DpsUniDup(List.Sent[i].sentence); DpsPrepareItem(Indexer, Doc, &Item, List.Sent[i].sentence, UStr, content_lang, indexed_size, indexed_limit, max_word_len, min_word_len, crossec #ifdef HAVE_ASPELL , have_speller, speller, NULL #endif ); DPS_FREE(UStr); } } DPS_FREE(lang_cs); DPS_FREE(links); for (i = 0; i < List.nitems; i++) DPS_FREE(List.Sent[i].sentence); DPS_FREE(List.Sent); TRACE_OUT(Indexer); return DPS_OK; }
__C_LINK int __DPSCALL DpsImportDictionary(DPS_ENV * Conf, const char *lang, const char *charset, const char *filename, int skip_noflag, const char *first_letters){ struct stat sb; char *str, *data = NULL, *cur_n = NULL; char *lstr; dpsunicode_t *ustr; DPS_CHARSET *sys_int; DPS_CHARSET *dict_charset; DPS_CONV touni; DPS_CONV fromuni; int fd; char savebyte; if ((lstr = (char*) DpsMalloc(2048)) == NULL) { DPS_FREE(str); return DPS_ERROR; } if ((ustr = (dpsunicode_t*) DpsMalloc(8192)) == NULL) { DPS_FREE(lstr); return DPS_ERROR; } dict_charset = DpsGetCharSet(charset); sys_int = DpsGetCharSet("sys-int"); if ((dict_charset == NULL) || (sys_int == NULL)) { DPS_FREE(lstr); DPS_FREE(ustr); return DPS_ERROR; } DpsConvInit(&touni, dict_charset, sys_int, Conf->CharsToEscape, 0); DpsConvInit(&fromuni, sys_int, dict_charset, Conf->CharsToEscape, 0); if (stat(filename, &sb)) { fprintf(stderr, "Unable to stat synonyms file '%s': %s", filename, strerror(errno)); DPS_FREE(lstr); DPS_FREE(ustr); return DPS_ERROR; } if ((fd = DpsOpen2(filename, O_RDONLY)) <= 0) { fprintf(stderr, "Unable to open synonyms file '%s': %s", filename, strerror(errno)); return DPS_ERROR; } if ((data = (char*)DpsMalloc(sb.st_size + 1)) == NULL) { fprintf(stderr, "Unable to alloc %ld bytes", (long)sb.st_size); DpsClose(fd); DPS_FREE(lstr); DPS_FREE(ustr); return DPS_ERROR; } if (read(fd, data, sb.st_size) != (ssize_t)sb.st_size) { fprintf(stderr, "Unable to read synonym file '%s': %s", filename, strerror(errno)); DPS_FREE(data); DpsClose(fd); DPS_FREE(lstr); DPS_FREE(ustr); return DPS_ERROR; } data[sb.st_size] = '\0'; str = data; cur_n = strchr(str, '\n'); if (cur_n != NULL) { cur_n++; savebyte = *cur_n; *cur_n = '\0'; } DpsClose(fd); while(str != NULL) { char *s; const char *flag; int res; flag = NULL; s = str; while(*s){ if(*s == '\r') *s = '\0'; if(*s == '\n') *s = '\0'; s++; } if((s=strchr(str,'/'))){ *s=0; s++;flag=s; while(*s){ if(((*s>='A')&&(*s<='Z'))||((*s>='a')&&(*s<='z')))s++; else{ *s=0; break; } } }else{ if(skip_noflag) goto loop_continue; flag=""; } res = DpsConv(&touni, (char*)ustr, 8192, str, 1024); DpsUniStrToLower(ustr); /* Dont load words if first letter is not required */ /* It allows to optimize loading at search time */ if(*first_letters) { DpsConv(&fromuni, lstr, 2048, ((const char*)ustr),(size_t)res); if(!strchr(first_letters,lstr[0])) goto loop_continue; } res = DpsSpellAdd(&Conf->Spells,ustr,flag,lang); if (res != DPS_OK) { DPS_FREE(lstr); DPS_FREE(ustr); DPS_FREE(data); return res; } if (Conf->Flags.use_accentext) { dpsunicode_t *af_uwrd = DpsUniAccentStrip(ustr); if (DpsUniStrCmp(af_uwrd, ustr) != 0) { res = DpsSpellAdd(&Conf->Spells, af_uwrd, flag, lang); if (res != DPS_OK) { DPS_FREE(lstr); DPS_FREE(ustr); DPS_FREE(data); DPS_FREE(af_uwrd); return res; } } DPS_FREE(af_uwrd); if (strncasecmp(lang, "de", 2) == 0) { dpsunicode_t *de_uwrd = DpsUniGermanReplace(ustr); if (DpsUniStrCmp(de_uwrd, ustr) != 0) { res = DpsSpellAdd(&Conf->Spells, de_uwrd, flag, lang); if (res != DPS_OK) { DPS_FREE(lstr); DPS_FREE(ustr); DPS_FREE(data); DPS_FREE(de_uwrd); return res; } } DPS_FREE(de_uwrd); } } loop_continue: str = cur_n; if (str != NULL) { *str = savebyte; cur_n = strchr(str, '\n'); if (cur_n != NULL) { cur_n++; savebyte = *cur_n; *cur_n = '\0'; } } } DPS_FREE(data); DPS_FREE(lstr); DPS_FREE(ustr); return DPS_OK; }
__C_LINK int __DPSCALL DpsSynonymListLoad(DPS_ENV * Env,const char * filename){ struct stat sb; char *str, *data = NULL, *cur_n = NULL; char lang[64]=""; DPS_CHARSET *cs=NULL; DPS_CHARSET *sys_int=DpsGetCharSet("sys-int"); DPS_CONV file_uni; DPS_WIDEWORD *ww = NULL; size_t key = 1; int flag_th = 0; int fd; char savebyte; if (stat(filename, &sb)) { fprintf(stderr, "Unable to stat synonyms file '%s': %s", filename, strerror(errno)); return DPS_ERROR; } if ((fd = DpsOpen2(filename, O_RDONLY)) <= 0) { dps_snprintf(Env->errstr,sizeof(Env->errstr)-1, "Unable to open synonyms file '%s': %s", filename, strerror(errno)); return DPS_ERROR; } if ((data = (char*)DpsMalloc(sb.st_size + 1)) == NULL) { dps_snprintf(Env->errstr,sizeof(Env->errstr)-1, "Unable to alloc %d bytes", sb.st_size); DpsClose(fd); return DPS_ERROR; } if (read(fd, data, sb.st_size) != (ssize_t)sb.st_size) { dps_snprintf(Env->errstr,sizeof(Env->errstr)-1, "Unable to read synonym file '%s': %s", filename, strerror(errno)); DPS_FREE(data); DpsClose(fd); return DPS_ERROR; } data[sb.st_size] = '\0'; str = data; cur_n = strchr(str, '\n'); if (cur_n != NULL) { cur_n++; savebyte = *cur_n; *cur_n = '\0'; } while(str != NULL) { if(str[0]=='#'||str[0]==' '||str[0]=='\t'||str[0]=='\r'||str[0]=='\n') goto loop_continue; if(!strncasecmp(str,"Charset:",8)){ char * lasttok; char * charset; if((charset = dps_strtok_r(str + 8, " \t\n\r", &lasttok))) { cs=DpsGetCharSet(charset); if(!cs){ dps_snprintf(Env->errstr, sizeof(Env->errstr), "Unknown charset '%s' in synonyms file '%s'", charset, filename); DPS_FREE(data); DpsClose(fd); return DPS_ERROR; } DpsConvInit(&file_uni, cs, sys_int, Env->CharsToEscape, 0); } }else if(!strncasecmp(str,"Language:",9)){ char * lasttok; char * l; if((l = dps_strtok_r(str + 9, " \t\n\r", &lasttok))) { dps_strncpy(lang, l, sizeof(lang)-1); } }else if(!strncasecmp(str, "Thesaurus:", 10)) { char * lasttok; char *tok = dps_strtok_r(str + 10, " \t\n\r", &lasttok); flag_th = (strncasecmp(tok, "yes", 3) == 0) ? 1 : 0; }else{ char *av[255]; size_t ac, i, j; dpsunicode_t *t; if(!cs){ dps_snprintf(Env->errstr,sizeof(Env->errstr)-1,"No Charset command in synonyms file '%s'",filename); DpsClose(fd); DPS_FREE(data); return DPS_ERROR; } if(!lang[0]){ dps_snprintf(Env->errstr,sizeof(Env->errstr)-1,"No Language command in synonyms file '%s'",filename); DpsClose(fd); DPS_FREE(data); return DPS_ERROR; } ac = DpsGetArgs(str, av, 255); if (ac < 2) goto loop_continue; if ((ww = (DPS_WIDEWORD*)DpsRealloc(ww, ac * sizeof(DPS_WIDEWORD))) == NULL) return DPS_ERROR; for (i = 0; i < ac; i++) { ww[i].word = av[i]; ww[i].len = dps_strlen(av[i]); ww[i].uword = t = (dpsunicode_t*)DpsMalloc((3 * ww[i].len + 1) * sizeof(dpsunicode_t)); if (ww[i].uword == NULL) return DPS_ERROR; DpsConv(&file_uni, (char*)ww[i].uword, sizeof(dpsunicode_t) * (3 * ww[i].len + 1), av[i], ww[i].len + 1); DpsUniStrToLower(ww[i].uword); ww[i].uword = DpsUniNormalizeNFC(NULL, ww[i].uword); DPS_FREE(t); } for (i = 0; i < ac - 1; i++) { for (j = i + 1; j < ac; j++) { if((Env->Synonyms.nsynonyms + 1) >= Env->Synonyms.msynonyms){ Env->Synonyms.msynonyms += 64; Env->Synonyms.Synonym = (DPS_SYNONYM*)DpsRealloc(Env->Synonyms.Synonym, sizeof(DPS_SYNONYM)*Env->Synonyms.msynonyms); if (Env->Synonyms.Synonym == NULL) { Env->Synonyms.msynonyms = Env->Synonyms.nsynonyms = 0; return DPS_ERROR; } } bzero((void*)&Env->Synonyms.Synonym[Env->Synonyms.nsynonyms], sizeof(DPS_SYNONYM)); /* Add direct order */ Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].p.uword = DpsUniDup(ww[i].uword); Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].s.uword = DpsUniDup(ww[j].uword); Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].p.count = Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].s.count = (size_t)((flag_th) ? key : 0); Env->Synonyms.nsynonyms++; bzero((void*)&Env->Synonyms.Synonym[Env->Synonyms.nsynonyms], sizeof(DPS_SYNONYM)); /* Add reverse order */ Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].p.uword = DpsUniDup(ww[j].uword); Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].s.uword = DpsUniDup(ww[i].uword); Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].p.count = Env->Synonyms.Synonym[Env->Synonyms.nsynonyms].s.count = (size_t)((flag_th) ? key : 0); Env->Synonyms.nsynonyms++; } } for (i = 0; i < ac; i++) { DPS_FREE(ww[i].uword); } do { key++; } while (key == 0); } loop_continue: str = cur_n; if (str != NULL) { *str = savebyte; cur_n = strchr(str, '\n'); if (cur_n != NULL) { cur_n++; savebyte = *cur_n; *cur_n = '\0'; } } } DPS_FREE(data); DPS_FREE(ww); DpsClose(fd); return DPS_OK; }