void query_read(char *qname) { FILE *fp; unsigned win = 0, nw = 0, nq=0, qin; int i; uint32_t *wlist = malloc(sizeof(wlist[0])*QMAX); uint32_t *qlist = malloc(sizeof(qlist[0])*QMAX); int16_t *map = malloc(sizeof(map[0])*ddN.W); if ( !map || !wlist || !qlist) yap_quit("Cannot allocate memory in query_read()\n"); fp = fopen(qname,"r"); if ( !fp ) yap_sysquit("Cannot open query bag file '%s'\n", qname); for (i=0; i<ddN.W; i++) map[i] = -1; nw = 0; qin = 0; while ( fscanf(fp," %u", &nw) == 1 ) { for (i=0; i<nw; i++) { if ( fscanf(fp," %u", &win) != 1 || win>=ddN.W ) yap_sysquit("Cannot read %d-th entry from '%s'\n", i, qname); if ( map[win]<0 ) { qlist[nq] = qin; wlist[nq] = win; map[win] = nq++; if ( nq>=QMAX ) yap_quit("Predefined query length maximum (%d) too small\n", QMAX); } else { /* * word appears already: same query, drop, other query, copy */ if ( qlist[map[win]]!=qin ) { qlist[nq] = qin; wlist[nq] = win; nq++; if ( nq>=QMAX ) yap_quit("Predefined query length maximum (%d) too small\n", QMAX); } } } qin++; nw = 0; } if ( ferror(fp) ) yap_sysquit("Cannot read data line from '%s'\n", qname); fclose(fp); ddP.query = map; ddP.qid = realloc(qlist, nq*sizeof(qlist[0])); ddP.qword = realloc(wlist, nq*sizeof(wlist[0])); ddP.qposn = malloc(sizeof(ddP.qposn[0])*(qin+1)); if ( !ddP.qword || !ddP.qid || !ddP.qposn ) yap_quit("Cannot allocate memory in query_read()\n"); ddP.n_words = nq; ddP.n_query = qin; ddP.qposn[0] = 0; for (i=1; i<ddP.n_words; i++) { if ( ddP.qid[i] != ddP.qid[i-1] ) ddP.qposn[ddP.qid[i]] = i; } ddP.qposn[ddP.n_query] = ddP.n_words; }
void read_u32sparsetri(int ns, int nr, int nc, uint32_t ***x, char *fname) { FILE *fp = fopen(fname,"r"); int i, u, v, w; int Nin; int sparse=0; if ( !fp ) yap_sysquit("Cannot open file '%s' for read\n", fname); if ( fscanf(fp, "%d", &Nin)!=1 || Nin!=ns ) yap_quit("Number blocks wrong for sparse matrix in '%s', should be %d\n", fname, ns); if ( fscanf(fp, "%d", &Nin)!=1 || Nin!=nr ) yap_quit("Number rows wrong for sparse matrix in '%s', should be %d\n", fname, nr); if ( fscanf(fp, "%d", &Nin)!=1 || Nin!=nc ) yap_quit("Number columns wrong for sparse matrix in '%s', should be %d\n", fname, nc); if ( fscanf(fp, "%d", &sparse)!=1 ) yap_quit("Cannot read sparsity in matrix in '%s'\n", fname); for (i = 0; i < sparse; i++) { if ( fscanf(fp, "%d %d %d %u", &u, &v, &w, &Nin) !=4 ) yap_quit("Cannot read line %d in matrix in '%s'\n", i, fname); x[u][v][w] = Nin; } if ( ferror(fp) ) yap_sysquit("Error on reading file '%s' ", fname); fclose(fp); }
void write_u16vec (int n, uint16_t *x, char *fname) // { FILE *fp = fopen(fname,"w"); int i; if ( !fp ) yap_sysquit("Cannot open file '%s' for write\n", fname); for (i = 0; i < n; i++) fprintf(fp, "%d\n", x[i] ); if ( ferror(fp) ) yap_sysquit("Error on writing file '%s' ", fname); fclose(fp); }
/* * pick up line from file "stem.ext" starting with "par" and an "=", * and return stuff after "=" but only first len chars */ static char *readextpar(char *stem, char *ext, char *par, char *buf, int len) { char *file = yap_makename(stem,ext); FILE *fp = fopen(file,"r"); int parlen = strlen(par); int buflen = len+parlen+50; char *linebuf = malloc(buflen+2); if ( !fp ) yap_sysquit("Cannot open parameter file '%s' ", file); if ( !linebuf ) yap_quit("Out of memory in readpar(%s)\n", par); buf[0] = 0; linebuf[0] = 0; while ( fgets(&linebuf[0],buflen,fp) ) { if ( strstr(&linebuf[0],par)==&linebuf[0] && (linebuf[parlen]==' ' || linebuf[parlen]=='=') ) { char *ret = strstr(&linebuf[0],"="); if ( !ret ) { yap_quit("Badly formatted parameter file '%s': %s\n", file, &linebuf[0]); } ret++; // yap_message("ret = '%s', length %d\n", ret, strlen(ret)); { int i; i = strlen(ret); if ( i>=len ) { i = len-1; ret[i] = 0; i--; } for ( ; i>=0; i--) buf[i] = ret[i]; /* for some weird reason, this fails occasionally!! */ // strncpy(buf, ret, len); } strncpy(buf, ret, len); break; } } if ( ferror(fp) ) yap_sysquit("Error on parameter file '%s' ", file); fclose(fp); free(file); free(linebuf); if ( buf[0] ) return &buf[0]; else return NULL; }
void write_dvec(char *dfile, int N, double *d) // { int i; FILE *fp = fopen(dfile ,"w"); if ( !fp ) yap_sysquit( "Cannot open file '%s' for write\n", dfile); for (i = 0; i < N; i++) { if ( !fprintf(fp,"%lg\n",d[i]) ) { yap_sysquit( "Cannot write to '%s' position %d\n", dfile, i); } } if ( ferror(fp) ) yap_sysquit("Error on writing file '%s' ", dfile); fclose(fp); }
void read_dvec(char *dfile, int N, double *d) // { int i; FILE *fp = fopen(dfile ,"r"); if ( !fp ) yap_sysquit( "Cannot open file '%s' for read\n", dfile); for (i = 0; i < N; i++) { if ( !fscanf(fp," %lg",&d[i]) ) { yap_sysquit( "Cannot read from '%s' position %d\n", dfile, i); } } if ( ferror(fp) ) yap_sysquit("Error on reading file '%s' ", dfile); fclose(fp); }
void write_dmat(char *dfile, int N, int C, double **d) // { int i, j; FILE *fp = fopen(dfile ,"w"); if ( !fp ) yap_sysquit( "Cannot open file '%s' for write\n", dfile); for (i = 0; i < N; i++) { for (j = 0; j < C; j++) { if ( !fprintf(fp,"%d %d %lg\n",i, j, d[i][j]) ) { yap_sysquit( "Cannot write to '%s' position %d,\n", dfile, i, j); } } } if ( ferror(fp) ) yap_sysquit("Error on writing file '%s' ", dfile); fclose(fp); }
void read_u16vec(char *dfile, int N, uint16_t *d) // { int i; FILE *fp = fopen(dfile ,"r"); if ( !fp ) yap_sysquit( "Cannot open file '%s' for read\n", dfile); for (i = 0; i < N; i++) { unsigned u; if ( !fscanf(fp," %u", &u) ) { yap_sysquit( "Cannot read from '%s' position %d\n", dfile, i); } d[i] = u; } if ( ferror(fp) ) yap_sysquit("Error on reading file '%s' ", dfile); fclose(fp); }
void write_u32sparseco(int nr, int nc, uint32_t **x, char *fname, uint32_t cutoff ) // { FILE *fp = fopen(fname,"w"); int i, j; if ( !fp ) yap_sysquit("Cannot open file '%s' for write\n", fname); fprintf(fp, "%d\n", nr); fprintf(fp, "%d\n", nc); fprintf(fp, "%d\n", countnnz32(nr,nc,x,cutoff)); for (i = 0; i < nr; i++) for (j = 0; j < nc; j++) if ( x[i][j]>cutoff ) fprintf(fp, "%d %d %u\n", i, j, (unsigned)x[i][j]); if ( ferror(fp) ) yap_sysquit("Error on writing file '%s' ", fname); fclose(fp); }
void read_didwid(char *wfile, char *dfile, int N, uint32_t *d, uint32_t *w) // { int i,wt,dt; FILE *fp1 = fopen(wfile ,"r"); FILE *fp2 = fopen(dfile ,"r"); if ( !fp1 ) yap_sysquit( "Cannot open file '%s' for read\n", wfile); if ( !fp2 ) yap_sysquit( "Cannot open file '%s' for read\n", dfile); for (i = 0; i < N; i++) { if (fscanf(fp1,"%d",&wt)) w[i]=wt-1; if (fscanf(fp2,"%d",&dt)) d[i]=dt-1; } if ( ferror(fp1) ) yap_sysquit("Error on reading file '%s' ", wfile); if ( ferror(fp2) ) yap_sysquit("Error on reading file '%s' ", dfile); fclose(fp1); fclose(fp2); }
unsigned countntot(char *fname) // { unsigned ntot=0; char buf[BUFSIZ]; FILE *fp = fopen(fname ,"r"); if ( !fp ) yap_sysquit( "Cannot open file '%s' for read\n", fname); while (fgets(buf, BUFSIZ, fp)) ntot++; fclose(fp); assert(ntot>0); return ntot; }
/* * print out the topic topk=10 words. report the PMI score. */ double report_pmi(char *topfile, /* name of topics file */ char *pmifile, /* name of PMI file */ int T, /* total topics */ int W, /* total words */ int E, /* number of epochs */ int topk, double *tp) { int lineno = 0; int i,k, thee; /* * mapping from local index to actual word index */ uint32_t *wind = u32vec(topk*T*E); int n_wind = 0; /* * boolean vector ... is word used */ uint32_t *wuse = u32vec(W/32+1); /* * PMI's by local index */ uint32_t *topic = u32vec(topk); float *coherency = fvec(E); double **pmi; float ave = 0; char *line; size_t n_line; FILE *fr; if ( !wind || !wuse ) yap_quit("Out of memory in report_pmi()\n"); /* * read in file of top word indices in topic */ fr = fopen(topfile,"r"); if ( !fr ) yap_sysquit("Topic file '%s' not read\n", topfile); line = NULL; n_line = 0; lineno = 0; while ( getline(&line, &n_line, fr)>0 ) { char *buf = line; unsigned j; int e = 0; lineno ++; buf += strspn(buf," \t\n"); // skip space if ( (E==1 && sscanf(buf, "%d: ", &k)<1) || (E>1 && sscanf(buf, "%d,%d: ", &e, &k)<2) ) yap_quit("Cannot read topic in topic line %d from file '%s'\n", lineno, topfile); if ( k<0 || k>=T ) continue; if ( e<0 || e>=E ) continue; for (i = 0; i<topk && *buf; i++) { buf = strpbrk(buf," \t\n"); // skip to next space if ( sscanf(buf, " %u", &j) <1 ) { if ( verbose>2 ) yap_message("Cannot read word %d in topic line %d from file '%s'\n", i+1, lineno, topfile); break; } if ( j>=W) { yap_quit("Bad word %d in topic line %d from file '%s'\n", i+1, lineno, topfile); } buf += strspn(buf," \t\n"); // skip space /* * check if word exists, and set up its index */ if ( wuse[j/32U] & (1U<<(j%32U)) ) { // yes, so search for it int ii; for (ii=0; ii<n_wind; ii++) if ( wind[ii]==j ) break; if ( ii>=n_wind ) yap_quit("Lookup of word %d failed at line %d in report_pmi()\n", (int)j, lineno); } else { // no, so add it wuse[j/32U] |= (1U<<(j%32U)); wind[n_wind] = j; n_wind++; } } free(line); line = NULL; n_line = 0; } fclose(fr); pmi = dmat(n_wind,n_wind); /* * build hash table now since we know size */ hashsize = n_wind*2; hashtab = malloc(sizeof(*hashtab)*hashsize); if ( !pmi || !hashtab ) yap_quit("Out of memory in report_pmi()\n"); for (i=0; i<hashsize; i++) hashtab[i] = 0; for (i=0; i<n_wind; i++) addw(wind[i],i); /* * load up PMI file, only keeping words mentioned in hash table */ { unsigned t1, t2; double value; int zcat = 0; fr = fopen(pmifile,"r"); if ( !fr ) { /* * try to zcat it */ char *cmd = malloc(strlen(pmifile)+20); sprintf(cmd,"%s.gz", pmifile); fr = fopen(cmd,"r"); if ( !fr ) yap_sysquit("Cannot open pmifile '%s' in report_pmi()\n", pmifile); fclose(fr); sprintf(cmd,"gunzip -c %s", pmifile); fr = popen(cmd,"r"); if ( !fr ) yap_sysquit("Cannot open or zcat pmifile '%s' in report_pmi()\n", pmifile); zcat = 1; free(cmd); } while (fscanf(fr, "%u %u %lg", &t1, &t2, &value)==3 ) { if ( t1>=W || t2>= W ) yap_quit("Illegal word index in report_pmi()\n"); if ( t1!= t2 && ( wuse[t1/32U] & (1U<<(t1%32U)) ) && ( wuse[t2/32U] & (1U<<(t2%32U))) ) { int i1, i2; i1 = findw(t1,wind); i2 = findw(t2,wind); if ( i1==UINT32_MAX || i2==UINT32_MAX ) yap_quit("Could not locate word index in report_pmi()\n"); pmi[i1][i2]=value; pmi[i2][i1]=value; } } if ( zcat ) pclose(fr); else fclose(fr); } /* * compute PMI score for each topic */ fr = fopen(topfile,"r"); if ( !fr ) yap_sysquit("Topic file '%s' not read\n", topfile); line = NULL; n_line = 0; thee = 0; lineno = 0; if ( E>1 ) yap_message("PMI %d:: ", 0); else yap_message("PMI :: "); while ( getline(&line, &n_line, fr)>0 ) { /* * repeat logic above to read topic file again */ char *buf = line; unsigned j; int cnt = 0; int e = 0; double coh = 0; buf += strspn(buf," \t\n"); // skip space if ( (E==1 && sscanf(buf, "%d: ", &k)<1) || (E>1 && sscanf(buf, "%d,%d: ", &e, &k)<2) ) yap_quit("Cannot read topic in topic line %d from file '%s'\n", lineno, topfile); if ( k<0 || k>=T ) continue; if ( e<0 || e>=E ) continue; if ( e!=thee ) { thee = e; yap_message("\nPMI %d:: ", e); } for (i = 0; i<topk && *buf; i++) { buf = strpbrk(buf," \t\n"); // skip to next space if ( sscanf(buf, " %u", &j) <1 ) { yap_message("Cannot read word %d in topic line %d from file '%s'\n", i+1, lineno, topfile); break; } if ( j>=W) { yap_quit("Bad word %d in topic line %d from file '%s'\n", i+1, lineno, topfile); } buf += strspn(buf," \t\n"); // skip space topic[i] = findw(j,wind); } if ( i<topk ) topic[i] = W; /* * topics now read */ for (i=0; i<topk && topic[i]<W; i++) { for (j=i+1; j<topk && topic[j]<W; j++) { coh += pmi[topic[i]][topic[j]]; cnt ++; } } if ( cnt>0 ) coh /= cnt; coherency[e] += coh * tp[k]; yap_message(" %d:%.3lf", k, coh); } fclose(fr); yap_message("\nPMI ="); if ( E==1 ) { yap_message(" %.3lf\n", coherency[0]); ave = coherency[0]; } else { int e; for (e=0; e<E; e++) { ave += coherency[e]; yap_message(" %.3lf", coherency[e]); } ave /= E; yap_message(" -> %.3lf\n", ave); } free(wind); free(coherency); free(wuse); free(topic); free(pmi[0]); free(pmi); free(hashtab); hashtab = NULL; hashsize = 0; return ave; }
void hca_displaytopics(char *stem, char *resstem, int topword, enum ScoreType scoretype, int pmicount, int fullreport) { int w,k; uint32_t *termindk = NULL; uint32_t *indk = NULL; int Nk_tot = 0; double (*termtscore)(int) = NULL; double (*tscore)(int) = NULL; double sparsityword = 0; double sparsitydoc = 0; double underused = 0; uint32_t *top1cnt = NULL; FILE *fp; float *tpmi = NULL; char *topfile; char *repfile; uint32_t *psort; FILE *rp = NULL; float *gtvec = globalprop(); //#define XTRA // prints model topic probs after observed #ifdef XTRA double *gtavec = calloc(ddN.T,sizeof(gtavec[0])); #endif float *gpvec = calloc(ddN.W,sizeof(gpvec[0])); float *pvec = calloc(ddN.W,sizeof(pvec[0])); #ifdef KL float *dfvec = calloc(ddN.W,sizeof(dfvec[0])); #endif double *ngalpha = NULL; T_stats_t *termstats; #ifdef XTRA get_probs(gtavec); #endif if ( pmicount>topword ) pmicount = topword; if ( scoretype == ST_idf ) { tscore = idfscore; } else if ( scoretype == ST_phirat ) { tscore = phiratioscore; } else if ( scoretype == ST_phi ) { tscore = phiscore; } else if ( scoretype == ST_count ) { tscore = countscore; } else if ( scoretype == ST_cost ) { tscore = costscore; } else if ( scoretype == ST_Q ) { tscore = Qscore; lowerQ = 1.0/ddN.T; } if ( ddS.TwT==NULL && ddP.phi==NULL && scoretype == ST_phirat ) yap_quit("Cannot use '-orat' option with this model/settings.\n"); if ( ddP.PYalpha==H_NG ) { /* * provide an estimate of alpha */ ngalpha = dvec(ddN.T); get_probs(ngalpha); for (k=0; k<ddN.T; k++) { ddP.alphapr[k] = ngalpha[k]; } } /* * returns null if no relevant data file */ termstats = tstats_init(ddS.z, ddD.NdTcum, ddN.T, ddN.DT, stem); if ( termstats ) { if ( scoretype == ST_idf ) { termtscore = termidfscore; } else termtscore = termcountscore; } /* * first collect counts of each word/term, * and build gpvec (mean word probs) */ build_NwK(); if ( termstats ) build_termNwK(termstats); { /* * gpvec[] is normalised NwK[] */ double tot = 0; for (w=0; w<ddN.W; w++) tot += gpvec[w] = NwK[w]+0.1; for (w=0; w<ddN.W; w++) gpvec[w] /= tot; } if ( ddS.Nwt ) { for (k=0; k<ddN.T; k++) { Nk_tot += ddS.NWt[k]; } } psort = sorttops(gtvec, ddN.T); top1cnt = hca_top1cnt(); if ( !top1cnt ) yap_quit("Cannot allocate top1cnt in hca_displaytopics()\n"); if ( pmicount ) { tpmi = malloc(sizeof(*tpmi)*(ddN.T+1)); if ( !tpmi ) yap_quit("Cannot allocate tpmi in hca_displaytopics()\n"); } indk = malloc(sizeof(*indk)*ddN.W); if ( !indk ) yap_quit("Cannot allocate indk in hca_displaytopics()\n"); if ( termstats ) { termindk = malloc(sizeof(*indk)*termstats->K); if ( !termindk ) yap_quit("Cannot allocate termindk in hca_displaytopics()\n"); } data_df(stem); #ifdef KL for (w=0; w<ddN.W; w++) dfvec[w] = ddD.df[w]; #endif /* * two passes through, * first to build the top words and dump to file */ repfile = yap_makename(resstem,".topset"); topfile = yap_makename(resstem,".toplst"); fp = fopen(topfile,"w"); if ( !fp ) yap_sysquit("Cannot open file '%s' for write\n", topfile); yap_message("\n"); for (k=0; k<ddN.T; k++) { int cnt, termcnt = 0; tscorek = k; /* * build sorted word list */ cnt = buildindk(k, indk); topk(topword, cnt, indk, tscore); if ( cnt==0 ) continue; if ( termstats ) { termcnt = buildtermindk(k, termindk, termstats); topk(topword, termcnt, termindk, termtscore); } /* * dump words to file */ fprintf(fp,"%d: ", k); for (w=0; w<topword && w<cnt; w++) { fprintf(fp," %d", (int)indk[w]); } if ( termstats ) { for (w=0; w<topword && w<termcnt; w++) { fprintf(fp," %d", (int)termstats->Kmin+termindk[w]); } } fprintf(fp, "\n"); } if ( ddP.PYbeta && (ddP.phi==NULL || ddP.betapr) ) { int cnt; /* * dump root words */ tscorek = -1; cnt = buildindk(-1, indk); topk(topword, cnt, indk, (ddP.phi==NULL)?countscore:phiscore); fprintf(fp,"-1:"); for (w=0; w<topword && w<cnt; w++) { fprintf(fp," %d", (int)indk[w]); } fprintf(fp, "\n"); } fclose(fp); if ( verbose>1 ) yap_message("\n"); if ( pmicount ) { /* * compute PMI */ char *toppmifile; char *pmifile; double *tp; tp = dvec(ddN.T); pmifile=yap_makename(stem,".pmi"); toppmifile=yap_makename(resstem,".toppmi"); get_probs(tp); report_pmi(topfile, pmifile, toppmifile, ddN.T, ddN.W, 1, pmicount, tp, tpmi); free(toppmifile); free(pmifile); free(tp); } /* * now report words and diagnostics */ //ttop_open(topfile); if ( fullreport ) { rp = fopen(repfile,"w"); if ( !rp ) yap_sysquit("Cannot open file '%s' for write\n", repfile); fprintf(rp, "#topic index rank prop word-sparse doc-sparse eff-words eff-docs docs-bound top-one " "dist-unif dist-unigrm"); if ( PCTL_BURSTY() ) fprintf(rp, " burst-concent"); if ( ddN.tokens ) fprintf(rp, " ave-length"); fprintf(rp, " coher"); if ( pmicount ) fprintf(rp, " pmi"); fprintf(rp, "\n#word topic index rank"); if ( ddS.Nwt ) fprintf(rp, " count"); fprintf(rp, " prop cumm df coher\n"); } for (k=0; k<ddN.T; k++) { int cnt, termcnt = 0; int kk = psort[k]; uint32_t **dfmtx; if ( ddP.phi==NULL && ddS.NWt[kk]==0 ) continue; /* * grab word prob vec for later use */ if ( ddS.Nwt ) { int w; for (w=0; w<ddN.W; w++) pvec[w] = wordprob(w,kk); } else if ( ddP.phi ) fv_copy(pvec, ddP.phi[kk], ddN.W); else if ( ddS.phi ) fv_copy(pvec, ddS.phi[kk], ddN.W); /* * rebuild word list */ tscorek = kk; cnt = buildindk(kk, indk); topk(topword, cnt, indk, tscore); if ( topword<cnt ) cnt = topword; assert(cnt>0); if ( termstats ) { termcnt = buildtermindk(kk, termindk, termstats); topk(topword, termcnt, termindk, termtscore); if ( topword<termcnt ) termcnt = topword; } /* * df stats for topic returned as matrix */ dfmtx = hca_dfmtx(indk, cnt, kk); if ( ddS.Nwt && (ddS.NWt[kk]*ddN.T*100<Nk_tot || ddS.NWt[kk]<5 )) underused++; /* * print stats for topic * Mallet: tokens, doc_ent, ave-word-len, coher., * uni-dist, corp-dist, eff-no-words */ yap_message("Topic %d/%d", kk, k); { /* * compute diagnostics */ double prop = gtvec[kk]; float *dprop = docprop(kk); double spw = 0; double spd = ((double)nonzero_Ndt(kk))/((double)ddN.DT); #ifdef KL double ew = fv_kl(dfvec,pvec,ddN.W); #else double ew = exp(fv_entropy(pvec,ddN.W)); #endif double ud = fv_helldistunif(pvec,ddN.W); double pd = fv_helldist(pvec,gpvec,ddN.W); double sl = fv_avestrlen(pvec,ddN.tokens,ddN.W); double co = coherence(dfmtx, cnt); double ed = dprop?exp(fv_entropy(dprop,ddN.DT)):ddN.DT; #define MALLET_EW #ifdef MALLET_EW double ewp = dprop?(1.0/fv_expprob(pvec,ddN.W)):ddN.W; #endif double da = dprop?fv_bound(dprop,ddN.DT,1.0/sqrt((double)ddN.T)):0; sparsitydoc += spd; yap_message((ddN.T>200)?" p=%.3lf%%":" p=%.2lf%%",100*prop); #ifdef XTRA yap_message((ddN.T>200)?"/%.3lf%%":"/%.2lf%%",100*gtavec[kk]); #endif if ( ddS.Nwt ) { spw = ((double)nonzero_Nwt(kk))/((double)ddN.W); sparsityword += spw; yap_message(" ws=%.1lf%%", 100*(1-spw)); } yap_message(" ds=%.1lf%%", 100*(1-spd) ); #ifdef KL yap_message(" ew=%lf", ew); #else yap_message(" ew=%.0lf", ew); #endif #ifdef MALLET_EW yap_message(" ewp=%.1lf", ewp); #endif yap_message(" ed=%.1lf", ed); yap_message(" da=%.0lf", da+0.1); yap_message(" t1=%u", top1cnt[kk]); yap_message(" ud=%.3lf", ud); yap_message(" pd=%.3lf", pd); if ( PCTL_BURSTY() ) yap_message(" bd=%.3lf", ddP.bdk[kk]); if ( ddP.NGbeta ) { /* * approx. as sqrt(var(lambda_k)/lambda-normaliser */ double ngvar = sqrt(ddP.NGalpha[kk]) * (ngalpha[kk]/ddP.NGalpha[kk]); yap_message(" ng=%.4lf,%.4lf", ngalpha[kk], ngvar/ngalpha[kk]); if ( ddS.sparse ) yap_message(",%.4f", 1-((float)ddS.sparseD[kk])/ddN.DTused); if ( verbose>2 ) yap_message(" ngl=%.4lf,%.4lf, nga=%.4lf,%.4lf", ddP.NGalpha[kk]/ddP.NGbeta[kk], sqrt(ddP.NGalpha[kk]/ddP.NGbeta[kk]/ddP.NGbeta[kk]), ddP.NGalpha[kk], ddP.NGbeta[kk]); } if ( ddN.tokens ) yap_message(" sl=%.2lf", sl); yap_message(" co=%.3lf%%", co); if ( pmicount ) yap_message(" pmi=%.3f", tpmi[kk]); if ( fullreport ) { fprintf(rp,"topic %d %d", kk, k); fprintf(rp," %.6lf", prop); if ( ddS.Nwt ) { fprintf(rp," %.6lf", (1-spw)); } else { fprintf(rp," 0"); } fprintf(rp," %.6lf", (1-spd) ); #ifdef KL yap_message(" %lf", ew); #else fprintf(rp," %.2lf", ew); #endif #ifdef MALLET_EW fprintf(rp," %.2lf", ewp); #endif fprintf(rp," %.2lf", ed); fprintf(rp," %.0lf", da+0.1); fprintf(rp," %u", top1cnt[kk]); fprintf(rp," %.6lf", ud); fprintf(rp," %.6lf", pd); if ( PCTL_BURSTY() ) fprintf(rp," %.3lf", ddP.bdk[kk]); fprintf(rp," %.4lf", (ddN.tokens)?sl:0); fprintf(rp," %.6lf", co); if ( pmicount ) fprintf(rp," %.4f", tpmi[kk]); fprintf(rp,"\n"); } if ( dprop) free(dprop); } if ( verbose>1 ) { double pcumm = 0; /* * print top words: * Mallet: rank, count, prob, cumm, docs, coh */ yap_message("\ntopic %d/%d", kk, k); yap_message(" words="); for (w=0; w<cnt; w++) { if ( w>0 ) yap_message(","); if ( ddN.tokens ) yap_message("%s", ddN.tokens[indk[w]]); else yap_message("%d", indk[w]); if ( verbose>2 ) { if ( scoretype == ST_count ) yap_message("(%d)", (int)(tscore(indk[w])+0.2)); else yap_message("(%6lf)", tscore(indk[w])); } if ( fullreport ) { fprintf(rp, "word %d %d %d", kk, indk[w], w); if ( ddS.Nwt ) fprintf(rp, " %d", ddS.Nwt[indk[w]][kk]); pcumm += pvec[indk[w]]; fprintf(rp, " %.6f %.6f", pvec[indk[w]], pcumm); fprintf(rp, " %d", dfmtx[w][w]); fprintf(rp, " %.6f", coherence_word(dfmtx, cnt, w)); if ( ddN.tokens ) fprintf(rp, " %s", ddN.tokens[indk[w]]); fprintf(rp, "\n"); } } if ( termstats ) { yap_message(" terms="); for (w=0; w<termcnt; w++) { if ( w>0 ) yap_message(","); if ( ddN.tokens ) yap_message("%s", termstats->tokens[termindk[w]]); else yap_message("%d", termstats->Kmin+termindk[w]); if ( verbose>2 ) { if ( scoretype == ST_count ) yap_message("(%d)", (int)(termtscore(termindk[w])+0.2)); else yap_message("(%6lf)", termtscore(termindk[w])); } if ( fullreport ) { fprintf(rp, "term %d %d %d", kk, termindk[w], w); fprintf(rp, " %d", termstats->Nkt[termindk[w]][kk]); fprintf(rp, " %s", termstats->tokens[termindk[w]]); fprintf(rp, "\n"); } } } } yap_message("\n"); free(dfmtx[0]); free(dfmtx); } if ( verbose>1 && ddP.PYbeta ) { int cnt; double pcumm = 0; /* * print root words */ tscorek = -1; cnt = buildindk(-1,indk); /* this case gives bad results */ // if ( scoretype == ST_phirat ) topk(topword, cnt, indk, phiratioscore); topk(topword, cnt, indk, (ddP.phi==NULL)?countscore:phiscore); /* * cannot build df mtx for root because * it is latent w.r.t. topics */ yap_message("Topic root words="); if ( fullreport ) { int w; if ( ddP.phi && ddP.PYbeta!=H_PDP ) { for (w=0; w<ddN.W; w++) pvec[w] = ddS.phi[ddN.T][w]; } else { for (w=0; w<ddN.W; w++) pvec[w] = betabasewordprob(w); } #ifdef KL double ew = fv_kl(dfvec,pvec,ddN.W); #else double ew = exp(fv_entropy(pvec,ddN.W)); #endif double ud = fv_helldistunif(pvec,ddN.W); double pd = fv_helldist(pvec,gpvec,ddN.W); fprintf(rp,"topic -1 -1 0 0"); fprintf(rp," %.4lf", ew); fprintf(rp," %.6lf", ud); fprintf(rp," %.6lf", pd); fprintf(rp,"\n"); } for (w=0; w<topword && w<cnt; w++) { if ( w>0 ) yap_message(","); if ( ddN.tokens ) yap_message("%s", ddN.tokens[indk[w]]); else yap_message("%d", indk[w]); if ( verbose>2 && !ddP.phi ) yap_message("(%6lf)", countscore(indk[w])); if ( fullreport ) { fprintf(rp, "word %d %d %d", -1, indk[w], w); if ( ddS.TwT ) fprintf(rp, " %d", ddS.TwT[w]); pcumm += pvec[indk[w]]; fprintf(rp, " %.6f %.6f", pvec[indk[w]], pcumm); fprintf(rp, " 0 0"); if ( ddN.tokens ) fprintf(rp, " %s", ddN.tokens[indk[w]]); fprintf(rp, "\n"); } } yap_message("\nTopical words="); topk(topword, cnt, indk, phiinvratioscore); for (w=0; w<topword && w<cnt; w++) { if ( w>0 ) yap_message(","); if ( ddN.tokens ) yap_message("%s", ddN.tokens[indk[w]]); else yap_message("%d", indk[w]); } yap_message("\n"); } yap_message("\n"); if ( rp ) fclose(rp); if ( ddS.Nwt ) yap_message("Average topicXword sparsity = %.2lf%%\n", 100*(1-sparsityword/ddN.T) ); yap_message("Average docXtopic sparsity = %.2lf%%\n" "Underused topics = %.1lf%%\n", 100*(1-sparsitydoc/ddN.T), 100.0*underused/(double)ddN.T); if ( ddS.sparse && ddP.PYalpha==H_NG ) { double avesp = 0; // correct_docsp(); for (k=0; k<ddN.T; k++) { avesp += gtvec[k]; } // check gtvec[] sums to 1 assert(fabs(avesp-1.0)<0.00001); avesp = 0; for (k=0; k<ddN.T; k++) { avesp += gtvec[k]*((float)ddS.sparseD[k])/ddN.DTused; assert(ddS.sparseD[k]<=ddN.DTused); } assert(avesp<=1.0); assert(avesp>0.0); yap_message("IBP sparsity = %.2lf%%\n", 100*(1-avesp)); } if ( pmicount ) yap_message("Average PMI = %.3f\n", tpmi[ddN.T]); /* * print */ if ( 1 ) { float **cmtx = hca_topmtx(); int t1, t2; int m1, m2; float mval; char *corfile = yap_makename(resstem,".topcor"); fp = fopen(corfile,"w"); if ( !fp ) yap_sysquit("Cannot open file '%s' for write\n", corfile); /* * print file */ for (t1=0; t1<ddN.T; t1++) { for (t2=0; t2<t1; t2++) if ( cmtx[t1][t2]>1.0e-7 ) fprintf(fp, "%d %d %0.6f\n", t1, t2, cmtx[t1][t2]); } fclose(fp); free(corfile); /* * display maximum */ m1 = 1; m2 = 0; mval = cmtx[1][0]; for (t1=0; t1<ddN.T; t1++) { for (t2=0; t2<t1; t2++) { if ( mval<cmtx[t1][t2] ) { mval = cmtx[t1][t2]; m1 = t1; m2 = t2; } } } yap_message("Maximum correlated topics (%d,%d) = %f\n", m1, m2, mval); free(cmtx[0]); free(cmtx); } /* * print burstiness report */ if ( PCTL_BURSTY() ) { int tottbl = 0; int totmlttbl = 0; int totmlt = 0; int i; for (i=0; i<ddN.NT; i++) { if ( Z_issetr(ddS.z[i]) ) { if ( M_multi(i) ) totmlttbl++; tottbl++; } if ( M_multi(i) ) totmlt++; } yap_message("Burst report: multis=%.2lf%%, tables=%.2lf%%, tbls-in-multis=%.2lf%%\n", 100.0*((double)ddM.dim_multiind)/ddN.N, 100.0*((double)tottbl)/ddN.NT, 100.0*((double)totmlttbl)/totmlt); } yap_message("\n"); free(topfile); if ( repfile ) free(repfile); if ( top1cnt ) free(top1cnt); free(indk); free(psort); if ( ngalpha ) free(ngalpha); if ( pmicount ) free(tpmi); if ( NwK ) { free(NwK); NwK = NULL; } #ifdef KL free(dfvec); #endif free(pvec); free(gtvec); free(gpvec); tstats_free(termstats); }
void hca_displaytopics(char *resstem, int topword, enum ScoreType scoretype) { int w,k; int *indk = NULL; int Nk_tot = 0; double (*tscore)(int) = NULL; double sparsityword = 0; double sparsitydoc = 0; double underused = 0; char *fname = yap_makename(resstem,".top"); int nophi = (ddP.phi==NULL) && (ddS.phi==NULL); FILE *fp; if ( scoretype == ST_idf ) { tscore = idfscore; } else if ( scoretype == ST_phi ) { tscore = phiscore; } else if ( scoretype == ST_count ) { tscore = countscore; } else if ( scoretype == ST_cost ) { tscore = costscore; } else if ( scoretype == ST_Q ) { tscore = Qscore; lowerQ = 1.0/ddN.T; } fp = fopen(fname,"w"); if ( !fp ) yap_sysquit("Cannot open file '%s' for write\n", fname); /* * first collect counts of each word/term */ if ( scoretype != ST_count && scoretype != ST_phi ) { NwK = u32vec(ddN.W); if ( !NwK ) yap_quit("Out of memory in hca_displaytopics()\n"); for (w=0; w<ddN.W; w++) { NwK[w] = 0; } NWK = 0; for (w=0; w<ddN.W; w++) { for (k=0; k<ddN.T; k++) { NwK[w] += ddS.Nwt[w][k]; // should use CCT_ReadN() } NWK += NwK[w]; } } assert(ddN.tokens); for (k=0; k<ddN.T; k++) { Nk_tot += ddS.NWt[k]; } indk = malloc(sizeof(*indk)*ddN.W); if ( !indk ) yap_quit("Cannot allocate indk\n"); for (k=0; k<ddN.T; k++) { int cnt; double spw; double spd; tscorek = k; /* * print top words */ cnt=0; if ( ddP.phi==NULL ) { for (w=0; w<ddN.W; w++) { if ( ddS.Nwt[w][k]>0 ) indk[cnt++] = w; } } else { float **phi; if ( ddP.phi ) phi = ddP.phi; else phi = ddS.phi; for (w=0; w<ddN.W; w++) { if ( phi[k][w]>0.5/ddN.W ) indk[cnt++] = w; } } topk(topword, cnt, indk, tscore); spd = ((double)nonzero_Ndt(k))/((double)ddN.DT); sparsitydoc += spd; if ( nophi ) { spw = ((double)nonzero_Nwt(k))/((double)ddN.W); sparsityword += spw; } if ( ddS.NWt[k]*ddN.T*100<Nk_tot ) underused++; yap_message("\nTopic %d (", k); if ( ddP.phi==NULL ) yap_message((ddN.T>200)?"p=%.3lf%%,":"p=%.2lf%%,", 100*((double)ddS.NWt[k])/(double)Nk_tot); if ( nophi ) yap_message("ws=%.1lf%%,", 100*(1-spw)); else yap_message("#=%.0lf,", exp(phi_entropy(k))); yap_message("ds=%.1lf%%", 100*(1-spd) ); fprintf(fp,"%d: ", k); yap_message(") words ="); for (w=0; w<topword && w<cnt; w++) { fprintf(fp," %d", (int)indk[w]); if ( verbose>2 ) { double score = tscore(indk[w]); yap_message(",%s(%6lf)", ddN.tokens[indk[w]], score); } else yap_message(",%s", ddN.tokens[indk[w]]); } yap_message("\n"); fprintf(fp, "\n"); } if ( ddP.PYbeta && nophi ) { int cnt; /* * print root words */ tscorek = -1; cnt=0; for (w=0; w<ddN.W; w++) { if ( ddS.TwT[w]>0 ) indk[cnt++] = w; } topk(topword, cnt, indk, tscore); yap_message("\nTopic root words ="); fprintf(fp,"-1:"); for (w=0; w<topword && w<cnt; w++) { fprintf(fp," %d", (int)indk[w]); if ( verbose>2 ) { double score = tscore(indk[w]); yap_message(",%s(%6lf)", ddN.tokens[indk[w]], score); } else yap_message(",%s", ddN.tokens[indk[w]]); } yap_message("\n"); fprintf(fp, "\n"); } if ( nophi ) yap_message("Average topicXword sparsity = %.2lf%%, ", 100*(1-sparsityword/ddN.T) ); yap_message("Average docXtopic sparsity = %.2lf%%, " "underused topics = %.1lf%%\n", 100*(1-sparsitydoc/ddN.T), 100.0*underused/(double)ddN.T); if ( ddP.bdk!=NULL) { int tottbl = 0; int totmlttbl = 0; int totmlt = 0; int i; for (i=0; i<ddN.NT; i++) { if ( Z_issetr(ddS.z[i]) ) { if ( M_multi(i) ) totmlttbl++; tottbl++; } if ( M_multi(i) ) totmlt++; } yap_message("doc PYP report: multis=%.2lf%%, tables=%.2lf%%, tbls-in-multis=%.2lf%%\n", 100.0*((double)ddM.dim_multiind)/ddN.N, 100.0*((double)tottbl)/ddN.NT, 100.0*((double)totmlttbl)/totmlt); } fclose(fp); free(fname); free(indk); if ( scoretype != ST_count ) { free(NwK); NwK = NULL; } }
void hca_displaytopics(char *stem, char *resstem, int topword, enum ScoreType scoretype, int pmicount, int fullreport) { int w,k; uint32_t *indk = NULL; int Nk_tot = 0; double (*tscore)(int) = NULL; double sparsityword = 0; double sparsitydoc = 0; double underused = 0; uint32_t *top1cnt = NULL; FILE *fp; float *tpmi = NULL; char *topfile; char *repfile; uint32_t *psort; FILE *rp = NULL; float *gtvec = globalprop(); float *gpvec = calloc(ddN.W,sizeof(gpvec[0])); float *pvec = calloc(ddN.W,sizeof(pvec[0])); if ( pmicount>topword ) pmicount = topword; if ( scoretype == ST_idf ) { tscore = idfscore; } else if ( scoretype == ST_phi ) { tscore = phiscore; } else if ( scoretype == ST_count ) { tscore = countscore; } else if ( scoretype == ST_cost ) { tscore = costscore; } else if ( scoretype == ST_Q ) { tscore = Qscore; lowerQ = 1.0/ddN.T; } /* * first collect counts of each word/term, * and build gpvec (mean word probs) */ build_NwK(); { /* * gpvec[] is normalised NwK[] */ double tot = 0; for (w=0; w<ddN.W; w++) tot += gpvec[w] = NwK[w]+0.1; for (w=0; w<ddN.W; w++) gpvec[w] /= tot; } if ( ddS.Nwt ) { for (k=0; k<ddN.T; k++) { Nk_tot += ddS.NWt[k]; } } psort = sorttops(gtvec, ddN.T); top1cnt = hca_top1cnt(); if ( !top1cnt ) yap_quit("Cannot allocate top1cnt in hca_displaytopics()\n"); if ( pmicount ) { tpmi = malloc(sizeof(*tpmi)*(ddN.T+1)); if ( !tpmi ) yap_quit("Cannot allocate tpmi in hca_displaytopics()\n"); } indk = malloc(sizeof(*indk)*ddN.W); if ( !indk ) yap_quit("Cannot allocate indk in hca_displaytopics()\n"); /* * two passes through, * first to build the top words and dump to file */ repfile = yap_makename(resstem,".topset"); topfile = yap_makename(resstem,".toplst"); fp = fopen(topfile,"w"); if ( !fp ) yap_sysquit("Cannot open file '%s' for write\n", topfile); yap_message("\n"); for (k=0; k<ddN.T; k++) { int cnt; tscorek = k; /* * build sorted word list */ cnt = buildindk(k, indk); topk(topword, cnt, indk, tscore); if ( cnt==0 ) continue; /* * dump words to file */ fprintf(fp,"%d: ", k); for (w=0; w<topword && w<cnt; w++) { fprintf(fp," %d", (int)indk[w]); } fprintf(fp, "\n"); } if ( ddP.PYbeta && (ddP.phi==NULL || ddP.betapr) ) { int cnt; /* * dump root words */ tscorek = -1; cnt = buildindk(-1, indk); topk(topword, cnt, indk, (ddP.phi==NULL)?countscore:phiscore); fprintf(fp,"-1:"); for (w=0; w<topword && w<cnt; w++) { fprintf(fp," %d", (int)indk[w]); } fprintf(fp, "\n"); } fclose(fp); if ( verbose>1 ) yap_message("\n"); if ( pmicount ) { /* * compute PMI */ char *toppmifile; char *pmifile; double *tp; tp = dvec(ddN.T); pmifile=yap_makename(stem,".pmi"); toppmifile=yap_makename(resstem,".toppmi"); get_probs(tp); report_pmi(topfile, pmifile, toppmifile, ddN.T, ddN.W, 1, pmicount, tp, tpmi); free(toppmifile); free(pmifile); free(tp); } /* * now report words and diagnostics */ //ttop_open(topfile); if ( fullreport ) { rp = fopen(repfile,"w"); if ( !rp ) yap_sysquit("Cannot open file '%s' for write\n", repfile); fprintf(rp, "#topic index rank prop word-sparse doc-sparse eff-words eff-docs docs-bound top-one " "dist-unif dist-unigrm"); if ( PCTL_BURSTY() ) fprintf(rp, " burst-concent"); if ( ddN.tokens ) fprintf(rp, " ave-length"); fprintf(rp, " coher"); if ( pmicount ) fprintf(rp, " pmi"); fprintf(rp, "\n#word topic index rank"); if ( ddS.Nwt ) fprintf(rp, " count"); fprintf(rp, " prop cumm df coher\n"); } for (k=0; k<ddN.T; k++) { int cnt; int kk = psort[k]; uint32_t **dfmtx; if ( ddP.phi==NULL && ddS.NWt[kk]==0 ) continue; /* * grab word prob vec for later use */ if ( ddS.Nwt ) { int w; for (w=0; w<ddN.W; w++) pvec[w] = wordprob(w,kk); } else if ( ddP.phi ) fv_copy(pvec, ddP.phi[kk], ddN.W); else if ( ddS.phi ) fv_copy(pvec, ddS.phi[kk], ddN.W); /* * rebuild word list */ tscorek = kk; cnt = buildindk(kk, indk); topk(topword, cnt, indk, tscore); if ( topword<cnt ) cnt = topword; assert(cnt>0); /* * df stats for topic returned as matrix */ dfmtx = hca_dfmtx(indk, cnt, kk); if ( ddS.Nwt && (ddS.NWt[kk]*ddN.T*100<Nk_tot || ddS.NWt[kk]<5 )) underused++; /* * print stats for topic * Mallet: tokens, doc_ent, ave-word-len, coher., * uni-dist, corp-dist, eff-no-words */ yap_message("Topic %d/%d", kk, k); { /* * compute diagnostics */ double prop = gtvec[kk]; float *dprop = docprop(kk); double spw = 0; double spd = ((double)nonzero_Ndt(kk))/((double)ddN.DT); double ew = exp(fv_entropy(pvec,ddN.W)); double ud = fv_helldistunif(pvec,ddN.W); double pd = fv_helldist(pvec,gpvec,ddN.W); double sl = fv_avestrlen(pvec,ddN.tokens,ddN.W); double co = coherence(dfmtx, cnt); double ed = dprop?exp(fv_entropy(dprop,ddN.DT)):ddN.DT; double da = dprop?fv_bound(dprop,ddN.DT,1.0/sqrt((double)ddN.T)):0; sparsitydoc += spd; yap_message((ddN.T>200)?" p=%.3lf%%":" p=%.2lf%%",100*prop); if ( ddS.Nwt ) { spw = ((double)nonzero_Nwt(kk))/((double)ddN.W); sparsityword += spw; yap_message(" ws=%.1lf%%", 100*(1-spw)); } yap_message(" ds=%.1lf%%", 100*(1-spd) ); yap_message(" ew=%.0lf", ew); yap_message(" ed=%.1lf", ed); yap_message(" da=%.0lf", da+0.1); yap_message(" t1=%u", top1cnt[kk]); yap_message(" ud=%.3lf", ud); yap_message(" pd=%.3lf", pd); if ( PCTL_BURSTY() ) yap_message(" bd=%.3lf", ddP.bdk[kk]); if ( ddN.tokens ) yap_message(" sl=%.2lf", sl); yap_message(" co=%.3lf%%", co); if ( pmicount ) yap_message(" pmi=%.3f", tpmi[kk]); if ( fullreport ) { fprintf(rp,"topic %d %d", kk, k); fprintf(rp," %.6lf", prop); if ( ddS.Nwt ) { fprintf(rp," %.6lf", (1-spw)); } else { fprintf(rp," 0"); } fprintf(rp," %.6lf", (1-spd) ); fprintf(rp," %.2lf", ew); fprintf(rp," %.2lf", ed); fprintf(rp," %.0lf", da+0.1); fprintf(rp," %u", top1cnt[kk]); fprintf(rp," %.6lf", ud); fprintf(rp," %.6lf", pd); if ( PCTL_BURSTY() ) fprintf(rp," %.3lf", ddP.bdk[kk]); fprintf(rp," %.4lf", (ddN.tokens)?sl:0); fprintf(rp," %.6lf", co); if ( pmicount ) fprintf(rp," %.4f", tpmi[kk]); fprintf(rp,"\n"); } if ( dprop) free(dprop); } if ( verbose>1 ) { double pcumm = 0; /* * print top words: * Mallet: rank, count, prob, cumm, docs, coh */ yap_message("\ntopic %d/%d", kk, k); yap_message(" words="); for (w=0; w<cnt; w++) { if ( w>0 ) yap_message(","); if ( ddN.tokens ) yap_message("%s", ddN.tokens[indk[w]]); else yap_message("%d", indk[w]); if ( verbose>2 ) yap_message("(%6lf)", tscore(indk[w])); if ( fullreport ) { fprintf(rp, "word %d %d %d", kk, indk[w], w); if ( ddS.Nwt ) fprintf(rp, " %d", ddS.Nwt[indk[w]][kk]); pcumm += pvec[indk[w]]; fprintf(rp, " %.6f %.6f", pvec[indk[w]], pcumm); fprintf(rp, " %d", dfmtx[w][w]); fprintf(rp, " %.6f", coherence_word(dfmtx, cnt, w)); if ( ddN.tokens ) fprintf(rp, " %s", ddN.tokens[indk[w]]); fprintf(rp, "\n"); } } } yap_message("\n"); free(dfmtx[0]); free(dfmtx); } if ( verbose>1 && ddP.PYbeta && (ddP.phi==NULL || ddP.betapr) ) { int cnt; double pcumm = 0; /* * print root words */ tscorek = -1; cnt = buildindk(-1,indk); topk(topword, cnt, indk, (ddP.phi==NULL)?countscore:phiscore); /* * cannot build df mtx for root because * it is latent w.r.t. topics */ yap_message("Topic root words="); if ( fullreport ) { int w; for (w=0; w<ddN.W; w++) pvec[w] = betabasewordprob(w); double ew = exp(fv_entropy(pvec,ddN.W)); double ud = fv_helldistunif(pvec,ddN.W); double pd = fv_helldist(pvec,gpvec,ddN.W); fprintf(rp,"topic -1 -1 0 0"); fprintf(rp," %.4lf", ew); fprintf(rp," %.6lf", ud); fprintf(rp," %.6lf", pd); fprintf(rp,"\n"); } for (w=0; w<topword && w<cnt; w++) { if ( w>0 ) yap_message(","); if ( ddN.tokens ) yap_message("%s", ddN.tokens[indk[w]]); else yap_message("%d", indk[w]); if ( verbose>2 ) yap_message("(%6lf)", countscore(indk[w])); if ( fullreport ) { fprintf(rp, "word %d %d %d", -1, indk[w], w); if ( ddS.TwT ) fprintf(rp, " %d", ddS.TwT[w]); pcumm += pvec[indk[w]]; fprintf(rp, " %.6f %.6f", pvec[indk[w]], pcumm); fprintf(rp, " 0 0"); if ( ddN.tokens ) fprintf(rp, " %s", ddN.tokens[indk[w]]); fprintf(rp, "\n"); } } yap_message("\n"); } yap_message("\n"); if ( rp ) fclose(rp); if ( ddS.Nwt ) yap_message("Average topicXword sparsity = %.2lf%%\n", 100*(1-sparsityword/ddN.T) ); yap_message("Average docXtopic sparsity = %.2lf%%\n" "Underused topics = %.1lf%%\n", 100*(1-sparsitydoc/ddN.T), 100.0*underused/(double)ddN.T); if ( pmicount ) yap_message("Average PMI = %.3f\n", tpmi[ddN.T]); /* * print */ if ( 1 ) { float **cmtx = hca_topmtx(); int t1, t2; int m1, m2; float mval; char *corfile = yap_makename(resstem,".topcor"); fp = fopen(corfile,"w"); if ( !fp ) yap_sysquit("Cannot open file '%s' for write\n", corfile); /* * print file */ for (t1=0; t1<ddN.T; t1++) { for (t2=0; t2<t1; t2++) if ( cmtx[t1][t2]>1.0e-3 ) fprintf(fp, "%d %d %0.6f\n", t1, t2, cmtx[t1][t2]); } fclose(fp); free(corfile); /* * display maximum */ m1 = 1; m2 = 0; mval = cmtx[1][0]; for (t1=0; t1<ddN.T; t1++) { for (t2=0; t2<t1; t2++) { if ( mval<cmtx[t1][t2] ) { mval = cmtx[t1][t2]; m1 = t1; m2 = t2; } } } yap_message("Maximum correlated topics (%d,%d) = %f\n", m1, m2, mval); free(cmtx[0]); free(cmtx); } /* * print burstiness report */ if ( PCTL_BURSTY() ) { int tottbl = 0; int totmlttbl = 0; int totmlt = 0; int i; for (i=0; i<ddN.NT; i++) { if ( Z_issetr(ddS.z[i]) ) { if ( M_multi(i) ) totmlttbl++; tottbl++; } if ( M_multi(i) ) totmlt++; } yap_message("Burst report: multis=%.2lf%%, tables=%.2lf%%, tbls-in-multis=%.2lf%%\n", 100.0*((double)ddM.dim_multiind)/ddN.N, 100.0*((double)tottbl)/ddN.NT, 100.0*((double)totmlttbl)/totmlt); } yap_message("\n"); free(topfile); if ( repfile ) free(repfile); if ( top1cnt ) free(top1cnt); free(indk); free(psort); if ( pmicount ) free(tpmi); if ( NwK ) { free(NwK); NwK = NULL; } free(pvec); free(gtvec); free(gpvec); }
/* * run regular gibbs cycles on the data with phi used; * the evaluation on each doc, and sample word probs * * if qparts>0, split collection into parts and only search this * * K = number of top results to retain */ void gibbs_query(int K, char *qname, int dots, int this_qpart, int qparts) { /* * mapping from query word posn. to its mi in current doc * >ddN.N = not in current doc * -ve = has no mi since occurs just once, found at * posn (-map[]-1) * non -ve = mi value */ int *mimap = NULL; /* * usual stuff for Gibbs loop over docs */ int i, j; float *fact = fvec(ddN.T*4); D_MiSi_t dD; /* * an index into topk[] which maintains ordering */ int *topind; /* * these store statistics of the results, for printing * these are unordered, ordered by topind[] */ /* document score */ float *topscore; /* document number */ int *topk; /* flags if ord is irrelevant, thus not scored */ char *wordunused; /* * per word stats for top results saved */ int *found; float *topcnt; float *topwordscore; /* * temporary versions for when gibbs running */ int *found_buf; float *topcnt_buf; float *topwordscore_buf; double *logprob; /* * search here */ int startdoc = 0; int enddoc = ddN.DT; /* * setup */ topcnt = malloc(sizeof(topcnt[0])*K*ddP.n_words); topwordscore = malloc(sizeof(topwordscore[0])*K*ddP.n_words); found = malloc(sizeof(found)*ddP.n_words*K); wordunused = malloc(sizeof(wordunused[0])*ddP.n_words); topcnt_buf = malloc(sizeof(topcnt[0])*ddP.n_words); topwordscore_buf = malloc(sizeof(topwordscore[0])*ddP.n_words); found_buf = malloc(sizeof(found)*ddP.n_words); if ( !topcnt || !topwordscore || !found || !topcnt_buf || !topwordscore_buf || !found_buf ) yap_quit("Cannot allocate memory in gibbs_query()\n"); logprob = malloc(sizeof(logprob[0])*ddP.n_query); topscore = malloc(sizeof(topscore[0])*K*ddP.n_query); topind = malloc(sizeof(topind[0])*K*ddP.n_query); topk = malloc(sizeof(topk[0])*K*ddP.n_query); if ( ddP.bdk!=NULL ) mimap = malloc(sizeof(mimap[0])*ddP.n_words); if ( !topk || !topscore || !logprob || !topind ) yap_quit("Cannot allocate memory in gibbs_query()\n"); for (i=0; i<ddP.n_words; i++) { wordunused[i] = 0; } for (i=0; i<K*ddP.n_query; i++) { topind[i] = i%K; topk[i] = -1; topscore[i] = INFINITY; } /* * check words to exclude using topics */ if ( ddP.n_excludetopic>0 ) { double *tprob = malloc(sizeof(tprob[0])*ddN.T); get_probs(tprob); yap_probs(); if ( verbose>1 ) yap_message("Excluding words: "); for (i=0; i<ddP.n_words; i++) { int t = besttopic(ddP.qword[i],tprob); if ( Q_excludetopic(t) ) { wordunused[i] = 1; if ( verbose>1 ) yap_message(" %d/%d", (int)ddP.qword[i], t); } } if ( verbose>1 ) yap_message("\n"); free(tprob); } if ( ddP.bdk!=NULL ) misi_init(&ddM,&dD); if ( qparts>0 ) { startdoc = ((double)this_qpart)/qparts * ddN.DT; enddoc = ((double)this_qpart+1.0)/qparts * ddN.DT; } for(i=startdoc; i<enddoc; i++) { int thisw = add_doc(i, GibbsNone); int r; if ( thisw<=1 ) { remove_doc(i, GibbsNone); continue; } if ( ddP.bdk!=NULL ) misi_build(&dD, i, 0); map_query(i, mimap, found_buf); for (j=0; j<ddP.n_words; j++) { topcnt_buf[j] = 0; topwordscore_buf[j] = 0; } for (r=0; r<ddP.queryiter; r++) { gibbs_lda(GibbsNone, ddN.T, i, ddD.NdT[i], fact, &dD, 0, 0); query_docprob(i, mimap, fact, &dD, topcnt_buf, topwordscore_buf); } /* * now adjust stats */ for (j=0; j<ddP.n_query; j++) logprob[j] = 0; for (j=0; j<ddP.n_words; j++) { if ( wordunused[j]>0 ) continue; if ( ddP.query[ddP.qword[j]]==j ) { topcnt_buf[j] /= ddP.queryiter; topwordscore_buf[j] /= ddP.queryiter; } else { /* word in previous query so copy */ int jj = ddP.query[ddP.qword[j]]; topcnt_buf[j] = topcnt_buf[jj]; topwordscore_buf[j] = topwordscore_buf[jj]; found_buf[j] = found_buf[jj]; } if ( wordunused[j]==0 ) logprob[ddP.qid[j]] += topwordscore_buf[j]; } if ( dots>0 && i>0 && (i%dots==0) ) yap_message("."); if ( ddP.bdk!=NULL ) misi_unbuild(&dD,i,0); remove_doc(i, GibbsNone); /* * enter into the arrays */ for (j=0; j<ddP.n_query; j++) { if ( i<K || logprob[j] < topscore[j*K+topind[j*K+K-1]] ) { int newind, l; /* * better than current lowest */ newind = bubble((i<K)?(i+1):K, &topind[j*K], &topscore[j*K], logprob[j]); /* * save the current details */ topscore[j*K+newind] = logprob[j]; topk[j*K+newind] = i; for (l=ddP.qposn[j]; l<ddP.qposn[j+1]; l++) { topcnt[newind*ddP.n_words+l] = topcnt_buf[l]; topwordscore[newind*ddP.n_words+l] = topwordscore_buf[l]; found[newind*ddP.n_words+l] = found_buf[l]; } } } } if ( dots>0 ) yap_message("\n"); /* * write result */ { float *ws = fvec(ddP.n_words); FILE *fp = fopen(qname,"w"); int q; if ( !fp ) yap_sysquit("Cannot write query results to '%s'\n", qname); for (q=0; q<ddP.n_query; q++) { int nw = ddP.qposn[q+1]-ddP.qposn[q]; for (i=0; i<K && i<ddN.DT && topk[topind[q*K+i]]>=0; i++) { int l, ind = topind[q*K+i]; double tfidf; tfidf = bm25(topk[q*K+ind],&found[ind*ddP.n_words+ddP.qposn[q]], &ddP.qword[ddP.qposn[q]], nw, ws); assert(ind>=0 && ind<K); fprintf(fp, "%d %d ", q, topk[q*K+ind]); fprintf(fp, "%.4f %.4lf ", topscore[q*K+ind]/nw, tfidf); if ( verbose>1 ) { for (l=ddP.qposn[q]; l<ddP.qposn[q+1]; l++) fprintf(fp, "%d ", found[ind*ddP.n_words+l]); for (l=ddP.qposn[q]; l<ddP.qposn[q+1]; l++) fprintf(fp, "%f ", topcnt[ind*ddP.n_words+l]); for (l=ddP.qposn[q]; l<ddP.qposn[q+1]; l++) fprintf(fp, "%f ", topwordscore[ind*ddP.n_words+l]); for (l=0; l<nw; l++) fprintf(fp, "%lf ", ws[l]); } fprintf(fp, "\n"); } } fclose(fp); free(ws); } /* * clean up */ free(fact); if ( ddP.bdk!=NULL ) misi_free(&dD); if ( mimap ) free(mimap); free(found); free(topwordscore); free(topcnt); free(found_buf); free(topwordscore_buf); free(topcnt_buf); free(topscore); free(topind); free(topk); free(logprob); }