void query_read(char *qname) {
FILE *fp;
unsigned win = 0, nw = 0, nq=0, qin;
int i;
uint32_t *wlist = malloc(sizeof(wlist[0])*QMAX);
uint32_t *qlist = malloc(sizeof(qlist[0])*QMAX);
int16_t *map = malloc(sizeof(map[0])*ddN.W);
if ( !map || !wlist || !qlist)
yap_quit("Cannot allocate memory in query_read()\n");
fp = fopen(qname,"r");
if ( !fp )
yap_sysquit("Cannot open query bag file '%s'\n", qname);
for (i=0; i<ddN.W; i++)
map[i] = -1;
nw = 0; qin = 0;
while ( fscanf(fp," %u", &nw) == 1 ) {
for (i=0; i<nw; i++) {
if ( fscanf(fp," %u", &win) != 1 || win>=ddN.W )
yap_sysquit("Cannot read %d-th entry from '%s'\n",
i, qname);
if ( map[win]<0 ) {
qlist[nq] = qin;
wlist[nq] = win;
map[win] = nq++;
if ( nq>=QMAX )
yap_quit("Predefined query length maximum (%d) too small\n", QMAX);
} else {
/*
* word appears already: same query, drop, other query, copy
*/
if ( qlist[map[win]]!=qin ) {
qlist[nq] = qin;
wlist[nq] = win;
nq++;
if ( nq>=QMAX )
yap_quit("Predefined query length maximum (%d) too small\n", QMAX);
}
}
}
qin++;
nw = 0;
}
if ( ferror(fp) )
yap_sysquit("Cannot read data line from '%s'\n", qname);
fclose(fp);
ddP.query = map;
ddP.qid = realloc(qlist, nq*sizeof(qlist[0]));
ddP.qword = realloc(wlist, nq*sizeof(wlist[0]));
ddP.qposn = malloc(sizeof(ddP.qposn[0])*(qin+1));
if ( !ddP.qword || !ddP.qid || !ddP.qposn )
yap_quit("Cannot allocate memory in query_read()\n");
ddP.n_words = nq;
ddP.n_query = qin;
ddP.qposn[0] = 0;
for (i=1; i<ddP.n_words; i++) {
if ( ddP.qid[i] != ddP.qid[i-1] )
ddP.qposn[ddP.qid[i]] = i;
}
ddP.qposn[ddP.n_query] = ddP.n_words;
}
void read_u32sparsetri(int ns, int nr, int nc, uint32_t ***x, char *fname)
{
FILE *fp = fopen(fname,"r");
int i, u, v, w;
int Nin;
int sparse=0;
if ( !fp )
yap_sysquit("Cannot open file '%s' for read\n", fname);
if ( fscanf(fp, "%d", &Nin)!=1 || Nin!=ns )
yap_quit("Number blocks wrong for sparse matrix in '%s', should be %d\n",
fname, ns);
if ( fscanf(fp, "%d", &Nin)!=1 || Nin!=nr )
yap_quit("Number rows wrong for sparse matrix in '%s', should be %d\n",
fname, nr);
if ( fscanf(fp, "%d", &Nin)!=1 || Nin!=nc )
yap_quit("Number columns wrong for sparse matrix in '%s', should be %d\n",
fname, nc);
if ( fscanf(fp, "%d", &sparse)!=1 )
yap_quit("Cannot read sparsity in matrix in '%s'\n", fname);
for (i = 0; i < sparse; i++) {
if ( fscanf(fp, "%d %d %d %u", &u, &v, &w, &Nin) !=4 )
yap_quit("Cannot read line %d in matrix in '%s'\n", i, fname);
x[u][v][w] = Nin;
}
if ( ferror(fp) )
yap_sysquit("Error on reading file '%s' ", fname);
fclose(fp);
}
void write_u16vec (int n, uint16_t *x, char *fname) //
{
FILE *fp = fopen(fname,"w");
int i;
if ( !fp )
yap_sysquit("Cannot open file '%s' for write\n", fname);
for (i = 0; i < n; i++)
fprintf(fp, "%d\n", x[i] );
if ( ferror(fp) )
yap_sysquit("Error on writing file '%s' ", fname);
fclose(fp);
}
/*
* pick up line from file "stem.ext" starting with "par" and an "=",
* and return stuff after "=" but only first len chars
*/
static char *readextpar(char *stem, char *ext, char *par, char *buf, int len) {
char *file = yap_makename(stem,ext);
FILE *fp = fopen(file,"r");
int parlen = strlen(par);
int buflen = len+parlen+50;
char *linebuf = malloc(buflen+2);
if ( !fp )
yap_sysquit("Cannot open parameter file '%s' ", file);
if ( !linebuf )
yap_quit("Out of memory in readpar(%s)\n", par);
buf[0] = 0;
linebuf[0] = 0;
while ( fgets(&linebuf[0],buflen,fp) ) {
if ( strstr(&linebuf[0],par)==&linebuf[0] &&
(linebuf[parlen]==' ' || linebuf[parlen]=='=') ) {
char *ret = strstr(&linebuf[0],"=");
if ( !ret ) {
yap_quit("Badly formatted parameter file '%s': %s\n",
file, &linebuf[0]);
}
ret++;
// yap_message("ret = '%s', length %d\n", ret, strlen(ret));
{
int i;
i = strlen(ret);
if ( i>=len ) {
i = len-1;
ret[i] = 0;
i--;
}
for ( ; i>=0; i--)
buf[i] = ret[i];
/* for some weird reason, this fails occasionally!! */
// strncpy(buf, ret, len);
}
strncpy(buf, ret, len);
break;
}
}
if ( ferror(fp) )
yap_sysquit("Error on parameter file '%s' ", file);
fclose(fp);
free(file);
free(linebuf);
if ( buf[0] )
return &buf[0];
else
return NULL;
}
void write_dvec(char *dfile, int N, double *d) //
{
int i;
FILE *fp = fopen(dfile ,"w");
if ( !fp )
yap_sysquit( "Cannot open file '%s' for write\n", dfile);
for (i = 0; i < N; i++) {
if ( !fprintf(fp,"%lg\n",d[i]) ) {
yap_sysquit( "Cannot write to '%s' position %d\n", dfile, i);
}
}
if ( ferror(fp) )
yap_sysquit("Error on writing file '%s' ", dfile);
fclose(fp);
}
void read_dvec(char *dfile, int N, double *d) //
{
int i;
FILE *fp = fopen(dfile ,"r");
if ( !fp )
yap_sysquit( "Cannot open file '%s' for read\n", dfile);
for (i = 0; i < N; i++) {
if ( !fscanf(fp," %lg",&d[i]) ) {
yap_sysquit( "Cannot read from '%s' position %d\n", dfile, i);
}
}
if ( ferror(fp) )
yap_sysquit("Error on reading file '%s' ", dfile);
fclose(fp);
}
void write_dmat(char *dfile, int N, int C, double **d) //
{
int i, j;
FILE *fp = fopen(dfile ,"w");
if ( !fp )
yap_sysquit( "Cannot open file '%s' for write\n", dfile);
for (i = 0; i < N; i++) {
for (j = 0; j < C; j++) {
if ( !fprintf(fp,"%d %d %lg\n",i, j, d[i][j]) ) {
yap_sysquit( "Cannot write to '%s' position %d,\n", dfile, i, j);
}
}
}
if ( ferror(fp) )
yap_sysquit("Error on writing file '%s' ", dfile);
fclose(fp);
}
void read_u16vec(char *dfile, int N, uint16_t *d) //
{
int i;
FILE *fp = fopen(dfile ,"r");
if ( !fp )
yap_sysquit( "Cannot open file '%s' for read\n", dfile);
for (i = 0; i < N; i++) {
unsigned u;
if ( !fscanf(fp," %u", &u) ) {
yap_sysquit( "Cannot read from '%s' position %d\n", dfile, i);
}
d[i] = u;
}
if ( ferror(fp) )
yap_sysquit("Error on reading file '%s' ", dfile);
fclose(fp);
}
void write_u32sparseco(int nr, int nc, uint32_t **x, char *fname,
uint32_t cutoff ) //
{
FILE *fp = fopen(fname,"w");
int i, j;
if ( !fp )
yap_sysquit("Cannot open file '%s' for write\n", fname);
fprintf(fp, "%d\n", nr);
fprintf(fp, "%d\n", nc);
fprintf(fp, "%d\n", countnnz32(nr,nc,x,cutoff));
for (i = 0; i < nr; i++)
for (j = 0; j < nc; j++)
if ( x[i][j]>cutoff )
fprintf(fp, "%d %d %u\n", i, j, (unsigned)x[i][j]);
if ( ferror(fp) )
yap_sysquit("Error on writing file '%s' ", fname);
fclose(fp);
}
void read_didwid(char *wfile, char *dfile, int N, uint32_t *d, uint32_t *w) //
{
int i,wt,dt;
FILE *fp1 = fopen(wfile ,"r");
FILE *fp2 = fopen(dfile ,"r");
if ( !fp1 )
yap_sysquit( "Cannot open file '%s' for read\n", wfile);
if ( !fp2 )
yap_sysquit( "Cannot open file '%s' for read\n", dfile);
for (i = 0; i < N; i++) {
if (fscanf(fp1,"%d",&wt)) w[i]=wt-1;
if (fscanf(fp2,"%d",&dt)) d[i]=dt-1;
}
if ( ferror(fp1) )
yap_sysquit("Error on reading file '%s' ", wfile);
if ( ferror(fp2) )
yap_sysquit("Error on reading file '%s' ", dfile);
fclose(fp1);
fclose(fp2);
}
unsigned countntot(char *fname) //
{
unsigned ntot=0;
char buf[BUFSIZ];
FILE *fp = fopen(fname ,"r");
if ( !fp )
yap_sysquit( "Cannot open file '%s' for read\n", fname);
while (fgets(buf, BUFSIZ, fp)) ntot++;
fclose(fp);
assert(ntot>0);
return ntot;
}
/*
* print out the topic topk=10 words. report the PMI score.
*/
double report_pmi(char *topfile, /* name of topics file */
char *pmifile, /* name of PMI file */
int T, /* total topics */
int W, /* total words */
int E, /* number of epochs */
int topk,
double *tp)
{
int lineno = 0;
int i,k, thee;
/*
* mapping from local index to actual word index
*/
uint32_t *wind = u32vec(topk*T*E);
int n_wind = 0;
/*
* boolean vector ... is word used
*/
uint32_t *wuse = u32vec(W/32+1);
/*
* PMI's by local index
*/
uint32_t *topic = u32vec(topk);
float *coherency = fvec(E);
double **pmi;
float ave = 0;
char *line;
size_t n_line;
FILE *fr;
if ( !wind || !wuse )
yap_quit("Out of memory in report_pmi()\n");
/*
* read in file of top word indices in topic
*/
fr = fopen(topfile,"r");
if ( !fr )
yap_sysquit("Topic file '%s' not read\n", topfile);
line = NULL;
n_line = 0;
lineno = 0;
while ( getline(&line, &n_line, fr)>0 ) {
char *buf = line;
unsigned j;
int e = 0;
lineno ++;
buf += strspn(buf," \t\n"); // skip space
if ( (E==1 && sscanf(buf, "%d: ", &k)<1) ||
(E>1 && sscanf(buf, "%d,%d: ", &e, &k)<2) )
yap_quit("Cannot read topic in topic line %d from file '%s'\n",
lineno, topfile);
if ( k<0 || k>=T )
continue;
if ( e<0 || e>=E )
continue;
for (i = 0; i<topk && *buf; i++) {
buf = strpbrk(buf," \t\n"); // skip to next space
if ( sscanf(buf, " %u", &j) <1 ) {
if ( verbose>2 )
yap_message("Cannot read word %d in topic line %d from file '%s'\n",
i+1, lineno, topfile);
break;
}
if ( j>=W) {
yap_quit("Bad word %d in topic line %d from file '%s'\n",
i+1, lineno, topfile);
}
buf += strspn(buf," \t\n"); // skip space
/*
* check if word exists, and set up its index
*/
if ( wuse[j/32U] & (1U<<(j%32U)) ) {
// yes, so search for it
int ii;
for (ii=0; ii<n_wind; ii++)
if ( wind[ii]==j )
break;
if ( ii>=n_wind )
yap_quit("Lookup of word %d failed at line %d in report_pmi()\n",
(int)j, lineno);
} else {
// no, so add it
wuse[j/32U] |= (1U<<(j%32U));
wind[n_wind] = j;
n_wind++;
}
}
free(line);
line = NULL;
n_line = 0;
}
fclose(fr);
pmi = dmat(n_wind,n_wind);
/*
* build hash table now since we know size
*/
hashsize = n_wind*2;
hashtab = malloc(sizeof(*hashtab)*hashsize);
if ( !pmi || !hashtab )
yap_quit("Out of memory in report_pmi()\n");
for (i=0; i<hashsize; i++)
hashtab[i] = 0;
for (i=0; i<n_wind; i++)
addw(wind[i],i);
/*
* load up PMI file, only keeping words mentioned in hash table
*/
{
unsigned t1, t2;
double value;
int zcat = 0;
fr = fopen(pmifile,"r");
if ( !fr ) {
/*
* try to zcat it
*/
char *cmd = malloc(strlen(pmifile)+20);
sprintf(cmd,"%s.gz", pmifile);
fr = fopen(cmd,"r");
if ( !fr )
yap_sysquit("Cannot open pmifile '%s' in report_pmi()\n",
pmifile);
fclose(fr);
sprintf(cmd,"gunzip -c %s", pmifile);
fr = popen(cmd,"r");
if ( !fr )
yap_sysquit("Cannot open or zcat pmifile '%s' in report_pmi()\n",
pmifile);
zcat = 1;
free(cmd);
}
while (fscanf(fr, "%u %u %lg", &t1, &t2, &value)==3 ) {
if ( t1>=W || t2>= W )
yap_quit("Illegal word index in report_pmi()\n");
if ( t1!= t2 && ( wuse[t1/32U] & (1U<<(t1%32U)) )
&& ( wuse[t2/32U] & (1U<<(t2%32U))) ) {
int i1, i2;
i1 = findw(t1,wind);
i2 = findw(t2,wind);
if ( i1==UINT32_MAX || i2==UINT32_MAX )
yap_quit("Could not locate word index in report_pmi()\n");
pmi[i1][i2]=value;
pmi[i2][i1]=value;
}
}
if ( zcat )
pclose(fr);
else
fclose(fr);
}
/*
* compute PMI score for each topic
*/
fr = fopen(topfile,"r");
if ( !fr )
yap_sysquit("Topic file '%s' not read\n", topfile);
line = NULL;
n_line = 0;
thee = 0;
lineno = 0;
if ( E>1 )
yap_message("PMI %d:: ", 0);
else
yap_message("PMI :: ");
while ( getline(&line, &n_line, fr)>0 ) {
/*
* repeat logic above to read topic file again
*/
char *buf = line;
unsigned j;
int cnt = 0;
int e = 0;
double coh = 0;
buf += strspn(buf," \t\n"); // skip space
if ( (E==1 && sscanf(buf, "%d: ", &k)<1) ||
(E>1 && sscanf(buf, "%d,%d: ", &e, &k)<2) )
yap_quit("Cannot read topic in topic line %d from file '%s'\n",
lineno, topfile);
if ( k<0 || k>=T )
continue;
if ( e<0 || e>=E )
continue;
if ( e!=thee ) {
thee = e;
yap_message("\nPMI %d:: ", e);
}
for (i = 0; i<topk && *buf; i++) {
buf = strpbrk(buf," \t\n"); // skip to next space
if ( sscanf(buf, " %u", &j) <1 ) {
yap_message("Cannot read word %d in topic line %d from file '%s'\n", i+1, lineno, topfile);
break;
}
if ( j>=W) {
yap_quit("Bad word %d in topic line %d from file '%s'\n", i+1, lineno, topfile);
}
buf += strspn(buf," \t\n"); // skip space
topic[i] = findw(j,wind);
}
if ( i<topk )
topic[i] = W;
/*
* topics now read
*/
for (i=0; i<topk && topic[i]<W; i++) {
for (j=i+1; j<topk && topic[j]<W; j++) {
coh += pmi[topic[i]][topic[j]];
cnt ++;
}
}
if ( cnt>0 ) coh /= cnt;
coherency[e] += coh * tp[k];
yap_message(" %d:%.3lf", k, coh);
}
fclose(fr);
yap_message("\nPMI =");
if ( E==1 ) {
yap_message(" %.3lf\n", coherency[0]);
ave = coherency[0];
} else {
int e;
for (e=0; e<E; e++) {
ave += coherency[e];
yap_message(" %.3lf", coherency[e]);
}
ave /= E;
yap_message(" -> %.3lf\n", ave);
}
free(wind);
free(coherency);
free(wuse);
free(topic);
free(pmi[0]); free(pmi);
free(hashtab);
hashtab = NULL;
hashsize = 0;
return ave;
}
void hca_displaytopics(char *stem, char *resstem, int topword,
enum ScoreType scoretype, int pmicount, int fullreport) {
int w,k;
uint32_t *termindk = NULL;
uint32_t *indk = NULL;
int Nk_tot = 0;
double (*termtscore)(int) = NULL;
double (*tscore)(int) = NULL;
double sparsityword = 0;
double sparsitydoc = 0;
double underused = 0;
uint32_t *top1cnt = NULL;
FILE *fp;
float *tpmi = NULL;
char *topfile;
char *repfile;
uint32_t *psort;
FILE *rp = NULL;
float *gtvec = globalprop();
//#define XTRA // prints model topic probs after observed
#ifdef XTRA
double *gtavec = calloc(ddN.T,sizeof(gtavec[0]));
#endif
float *gpvec = calloc(ddN.W,sizeof(gpvec[0]));
float *pvec = calloc(ddN.W,sizeof(pvec[0]));
#ifdef KL
float *dfvec = calloc(ddN.W,sizeof(dfvec[0]));
#endif
double *ngalpha = NULL;
T_stats_t *termstats;
#ifdef XTRA
get_probs(gtavec);
#endif
if ( pmicount>topword )
pmicount = topword;
if ( scoretype == ST_idf ) {
tscore = idfscore;
} else if ( scoretype == ST_phirat ) {
tscore = phiratioscore;
} else if ( scoretype == ST_phi ) {
tscore = phiscore;
} else if ( scoretype == ST_count ) {
tscore = countscore;
} else if ( scoretype == ST_cost ) {
tscore = costscore;
} else if ( scoretype == ST_Q ) {
tscore = Qscore;
lowerQ = 1.0/ddN.T;
}
if ( ddS.TwT==NULL && ddP.phi==NULL && scoretype == ST_phirat )
yap_quit("Cannot use '-orat' option with this model/settings.\n");
if ( ddP.PYalpha==H_NG ) {
/*
* provide an estimate of alpha
*/
ngalpha = dvec(ddN.T);
get_probs(ngalpha);
for (k=0; k<ddN.T; k++) {
ddP.alphapr[k] = ngalpha[k];
}
}
/*
* returns null if no relevant data file
*/
termstats = tstats_init(ddS.z, ddD.NdTcum, ddN.T, ddN.DT, stem);
if ( termstats ) {
if ( scoretype == ST_idf ) {
termtscore = termidfscore;
} else
termtscore = termcountscore;
}
/*
* first collect counts of each word/term,
* and build gpvec (mean word probs)
*/
build_NwK();
if ( termstats )
build_termNwK(termstats);
{
/*
* gpvec[] is normalised NwK[]
*/
double tot = 0;
for (w=0; w<ddN.W; w++)
tot += gpvec[w] = NwK[w]+0.1;
for (w=0; w<ddN.W; w++)
gpvec[w] /= tot;
}
if ( ddS.Nwt ) {
for (k=0; k<ddN.T; k++) {
Nk_tot += ddS.NWt[k];
}
}
psort = sorttops(gtvec, ddN.T);
top1cnt = hca_top1cnt();
if ( !top1cnt )
yap_quit("Cannot allocate top1cnt in hca_displaytopics()\n");
if ( pmicount ) {
tpmi = malloc(sizeof(*tpmi)*(ddN.T+1));
if ( !tpmi )
yap_quit("Cannot allocate tpmi in hca_displaytopics()\n");
}
indk = malloc(sizeof(*indk)*ddN.W);
if ( !indk )
yap_quit("Cannot allocate indk in hca_displaytopics()\n");
if ( termstats ) {
termindk = malloc(sizeof(*indk)*termstats->K);
if ( !termindk )
yap_quit("Cannot allocate termindk in hca_displaytopics()\n");
}
data_df(stem);
#ifdef KL
for (w=0; w<ddN.W; w++)
dfvec[w] = ddD.df[w];
#endif
/*
* two passes through,
* first to build the top words and dump to file
*/
repfile = yap_makename(resstem,".topset");
topfile = yap_makename(resstem,".toplst");
fp = fopen(topfile,"w");
if ( !fp )
yap_sysquit("Cannot open file '%s' for write\n", topfile);
yap_message("\n");
for (k=0; k<ddN.T; k++) {
int cnt, termcnt = 0;
tscorek = k;
/*
* build sorted word list
*/
cnt = buildindk(k, indk);
topk(topword, cnt, indk, tscore);
if ( cnt==0 )
continue;
if ( termstats ) {
termcnt = buildtermindk(k, termindk, termstats);
topk(topword, termcnt, termindk, termtscore);
}
/*
* dump words to file
*/
fprintf(fp,"%d: ", k);
for (w=0; w<topword && w<cnt; w++) {
fprintf(fp," %d", (int)indk[w]);
}
if ( termstats ) {
for (w=0; w<topword && w<termcnt; w++) {
fprintf(fp," %d", (int)termstats->Kmin+termindk[w]);
}
}
fprintf(fp, "\n");
}
if ( ddP.PYbeta && (ddP.phi==NULL || ddP.betapr) ) {
int cnt;
/*
* dump root words
*/
tscorek = -1;
cnt = buildindk(-1, indk);
topk(topword, cnt, indk, (ddP.phi==NULL)?countscore:phiscore);
fprintf(fp,"-1:");
for (w=0; w<topword && w<cnt; w++) {
fprintf(fp," %d", (int)indk[w]);
}
fprintf(fp, "\n");
}
fclose(fp);
if ( verbose>1 ) yap_message("\n");
if ( pmicount ) {
/*
* compute PMI
*/
char *toppmifile;
char *pmifile;
double *tp;
tp = dvec(ddN.T);
pmifile=yap_makename(stem,".pmi");
toppmifile=yap_makename(resstem,".toppmi");
get_probs(tp);
report_pmi(topfile, pmifile, toppmifile, ddN.T, ddN.W, 1,
pmicount, tp, tpmi);
free(toppmifile);
free(pmifile);
free(tp);
}
/*
* now report words and diagnostics
*/
//ttop_open(topfile);
if ( fullreport ) {
rp = fopen(repfile,"w");
if ( !rp )
yap_sysquit("Cannot open file '%s' for write\n", repfile);
fprintf(rp, "#topic index rank prop word-sparse doc-sparse eff-words eff-docs docs-bound top-one "
"dist-unif dist-unigrm");
if ( PCTL_BURSTY() )
fprintf(rp, " burst-concent");
if ( ddN.tokens )
fprintf(rp, " ave-length");
fprintf(rp, " coher");
if ( pmicount )
fprintf(rp, " pmi");
fprintf(rp, "\n#word topic index rank");
if ( ddS.Nwt )
fprintf(rp, " count");
fprintf(rp, " prop cumm df coher\n");
}
for (k=0; k<ddN.T; k++) {
int cnt, termcnt = 0;
int kk = psort[k];
uint32_t **dfmtx;
if ( ddP.phi==NULL && ddS.NWt[kk]==0 )
continue;
/*
* grab word prob vec for later use
*/
if ( ddS.Nwt ) {
int w;
for (w=0; w<ddN.W; w++)
pvec[w] = wordprob(w,kk);
} else if ( ddP.phi )
fv_copy(pvec, ddP.phi[kk], ddN.W);
else if ( ddS.phi )
fv_copy(pvec, ddS.phi[kk], ddN.W);
/*
* rebuild word list
*/
tscorek = kk;
cnt = buildindk(kk, indk);
topk(topword, cnt, indk, tscore);
if ( topword<cnt )
cnt = topword;
assert(cnt>0);
if ( termstats ) {
termcnt = buildtermindk(kk, termindk, termstats);
topk(topword, termcnt, termindk, termtscore);
if ( topword<termcnt )
termcnt = topword;
}
/*
* df stats for topic returned as matrix
*/
dfmtx = hca_dfmtx(indk, cnt, kk);
if ( ddS.Nwt && (ddS.NWt[kk]*ddN.T*100<Nk_tot || ddS.NWt[kk]<5 ))
underused++;
/*
* print stats for topic
* Mallet: tokens, doc_ent, ave-word-len, coher.,
* uni-dist, corp-dist, eff-no-words
*/
yap_message("Topic %d/%d", kk, k);
{
/*
* compute diagnostics
*/
double prop = gtvec[kk];
float *dprop = docprop(kk);
double spw = 0;
double spd = ((double)nonzero_Ndt(kk))/((double)ddN.DT);
#ifdef KL
double ew = fv_kl(dfvec,pvec,ddN.W);
#else
double ew = exp(fv_entropy(pvec,ddN.W));
#endif
double ud = fv_helldistunif(pvec,ddN.W);
double pd = fv_helldist(pvec,gpvec,ddN.W);
double sl = fv_avestrlen(pvec,ddN.tokens,ddN.W);
double co = coherence(dfmtx, cnt);
double ed = dprop?exp(fv_entropy(dprop,ddN.DT)):ddN.DT;
#define MALLET_EW
#ifdef MALLET_EW
double ewp = dprop?(1.0/fv_expprob(pvec,ddN.W)):ddN.W;
#endif
double da = dprop?fv_bound(dprop,ddN.DT,1.0/sqrt((double)ddN.T)):0;
sparsitydoc += spd;
yap_message((ddN.T>200)?" p=%.3lf%%":" p=%.2lf%%",100*prop);
#ifdef XTRA
yap_message((ddN.T>200)?"/%.3lf%%":"/%.2lf%%",100*gtavec[kk]);
#endif
if ( ddS.Nwt ) {
spw = ((double)nonzero_Nwt(kk))/((double)ddN.W);
sparsityword += spw;
yap_message(" ws=%.1lf%%", 100*(1-spw));
}
yap_message(" ds=%.1lf%%", 100*(1-spd) );
#ifdef KL
yap_message(" ew=%lf", ew);
#else
yap_message(" ew=%.0lf", ew);
#endif
#ifdef MALLET_EW
yap_message(" ewp=%.1lf", ewp);
#endif
yap_message(" ed=%.1lf", ed);
yap_message(" da=%.0lf", da+0.1);
yap_message(" t1=%u", top1cnt[kk]);
yap_message(" ud=%.3lf", ud);
yap_message(" pd=%.3lf", pd);
if ( PCTL_BURSTY() )
yap_message(" bd=%.3lf", ddP.bdk[kk]);
if ( ddP.NGbeta ) {
/*
* approx. as sqrt(var(lambda_k)/lambda-normaliser
*/
double ngvar = sqrt(ddP.NGalpha[kk])
* (ngalpha[kk]/ddP.NGalpha[kk]);
yap_message(" ng=%.4lf,%.4lf",
ngalpha[kk], ngvar/ngalpha[kk]);
if ( ddS.sparse )
yap_message(",%.4f", 1-((float)ddS.sparseD[kk])/ddN.DTused);
if ( verbose>2 )
yap_message(" ngl=%.4lf,%.4lf, nga=%.4lf,%.4lf",
ddP.NGalpha[kk]/ddP.NGbeta[kk],
sqrt(ddP.NGalpha[kk]/ddP.NGbeta[kk]/ddP.NGbeta[kk]),
ddP.NGalpha[kk], ddP.NGbeta[kk]);
}
if ( ddN.tokens )
yap_message(" sl=%.2lf", sl);
yap_message(" co=%.3lf%%", co);
if ( pmicount )
yap_message(" pmi=%.3f", tpmi[kk]);
if ( fullreport ) {
fprintf(rp,"topic %d %d", kk, k);
fprintf(rp," %.6lf", prop);
if ( ddS.Nwt ) {
fprintf(rp," %.6lf", (1-spw));
} else {
fprintf(rp," 0");
}
fprintf(rp," %.6lf", (1-spd) );
#ifdef KL
yap_message(" %lf", ew);
#else
fprintf(rp," %.2lf", ew);
#endif
#ifdef MALLET_EW
fprintf(rp," %.2lf", ewp);
#endif
fprintf(rp," %.2lf", ed);
fprintf(rp," %.0lf", da+0.1);
fprintf(rp," %u", top1cnt[kk]);
fprintf(rp," %.6lf", ud);
fprintf(rp," %.6lf", pd);
if ( PCTL_BURSTY() )
fprintf(rp," %.3lf", ddP.bdk[kk]);
fprintf(rp," %.4lf", (ddN.tokens)?sl:0);
fprintf(rp," %.6lf", co);
if ( pmicount )
fprintf(rp," %.4f", tpmi[kk]);
fprintf(rp,"\n");
}
if ( dprop) free(dprop);
}
if ( verbose>1 ) {
double pcumm = 0;
/*
* print top words:
* Mallet: rank, count, prob, cumm, docs, coh
*/
yap_message("\ntopic %d/%d", kk, k);
yap_message(" words=");
for (w=0; w<cnt; w++) {
if ( w>0 ) yap_message(",");
if ( ddN.tokens )
yap_message("%s", ddN.tokens[indk[w]]);
else
yap_message("%d", indk[w]);
if ( verbose>2 ) {
if ( scoretype == ST_count )
yap_message("(%d)", (int)(tscore(indk[w])+0.2));
else
yap_message("(%6lf)", tscore(indk[w]));
}
if ( fullreport ) {
fprintf(rp, "word %d %d %d", kk, indk[w], w);
if ( ddS.Nwt )
fprintf(rp, " %d", ddS.Nwt[indk[w]][kk]);
pcumm += pvec[indk[w]];
fprintf(rp, " %.6f %.6f", pvec[indk[w]], pcumm);
fprintf(rp, " %d", dfmtx[w][w]);
fprintf(rp, " %.6f", coherence_word(dfmtx, cnt, w));
if ( ddN.tokens )
fprintf(rp, " %s", ddN.tokens[indk[w]]);
fprintf(rp, "\n");
}
}
if ( termstats ) {
yap_message(" terms=");
for (w=0; w<termcnt; w++) {
if ( w>0 ) yap_message(",");
if ( ddN.tokens )
yap_message("%s", termstats->tokens[termindk[w]]);
else
yap_message("%d", termstats->Kmin+termindk[w]);
if ( verbose>2 ) {
if ( scoretype == ST_count )
yap_message("(%d)", (int)(termtscore(termindk[w])+0.2));
else
yap_message("(%6lf)", termtscore(termindk[w]));
}
if ( fullreport ) {
fprintf(rp, "term %d %d %d", kk, termindk[w], w);
fprintf(rp, " %d", termstats->Nkt[termindk[w]][kk]);
fprintf(rp, " %s", termstats->tokens[termindk[w]]);
fprintf(rp, "\n");
}
}
}
}
yap_message("\n");
free(dfmtx[0]); free(dfmtx);
}
if ( verbose>1 && ddP.PYbeta ) {
int cnt;
double pcumm = 0;
/*
* print root words
*/
tscorek = -1;
cnt = buildindk(-1,indk);
/* this case gives bad results */
// if ( scoretype == ST_phirat ) topk(topword, cnt, indk, phiratioscore);
topk(topword, cnt, indk, (ddP.phi==NULL)?countscore:phiscore);
/*
* cannot build df mtx for root because
* it is latent w.r.t. topics
*/
yap_message("Topic root words=");
if ( fullreport ) {
int w;
if ( ddP.phi && ddP.PYbeta!=H_PDP ) {
for (w=0; w<ddN.W; w++)
pvec[w] = ddS.phi[ddN.T][w];
} else {
for (w=0; w<ddN.W; w++)
pvec[w] = betabasewordprob(w);
}
#ifdef KL
double ew = fv_kl(dfvec,pvec,ddN.W);
#else
double ew = exp(fv_entropy(pvec,ddN.W));
#endif
double ud = fv_helldistunif(pvec,ddN.W);
double pd = fv_helldist(pvec,gpvec,ddN.W);
fprintf(rp,"topic -1 -1 0 0");
fprintf(rp," %.4lf", ew);
fprintf(rp," %.6lf", ud);
fprintf(rp," %.6lf", pd);
fprintf(rp,"\n");
}
for (w=0; w<topword && w<cnt; w++) {
if ( w>0 ) yap_message(",");
if ( ddN.tokens )
yap_message("%s", ddN.tokens[indk[w]]);
else
yap_message("%d", indk[w]);
if ( verbose>2 && !ddP.phi )
yap_message("(%6lf)", countscore(indk[w]));
if ( fullreport ) {
fprintf(rp, "word %d %d %d", -1, indk[w], w);
if ( ddS.TwT )
fprintf(rp, " %d", ddS.TwT[w]);
pcumm += pvec[indk[w]];
fprintf(rp, " %.6f %.6f", pvec[indk[w]], pcumm);
fprintf(rp, " 0 0");
if ( ddN.tokens )
fprintf(rp, " %s", ddN.tokens[indk[w]]);
fprintf(rp, "\n");
}
}
yap_message("\nTopical words=");
topk(topword, cnt, indk, phiinvratioscore);
for (w=0; w<topword && w<cnt; w++) {
if ( w>0 ) yap_message(",");
if ( ddN.tokens )
yap_message("%s", ddN.tokens[indk[w]]);
else
yap_message("%d", indk[w]);
}
yap_message("\n");
}
yap_message("\n");
if ( rp )
fclose(rp);
if ( ddS.Nwt )
yap_message("Average topicXword sparsity = %.2lf%%\n",
100*(1-sparsityword/ddN.T) );
yap_message("Average docXtopic sparsity = %.2lf%%\n"
"Underused topics = %.1lf%%\n",
100*(1-sparsitydoc/ddN.T),
100.0*underused/(double)ddN.T);
if ( ddS.sparse && ddP.PYalpha==H_NG ) {
double avesp = 0;
// correct_docsp();
for (k=0; k<ddN.T; k++) {
avesp += gtvec[k];
}
// check gtvec[] sums to 1
assert(fabs(avesp-1.0)<0.00001);
avesp = 0;
for (k=0; k<ddN.T; k++) {
avesp += gtvec[k]*((float)ddS.sparseD[k])/ddN.DTused;
assert(ddS.sparseD[k]<=ddN.DTused);
}
assert(avesp<=1.0);
assert(avesp>0.0);
yap_message("IBP sparsity = %.2lf%%\n", 100*(1-avesp));
}
if ( pmicount )
yap_message("Average PMI = %.3f\n", tpmi[ddN.T]);
/*
* print
*/
if ( 1 ) {
float **cmtx = hca_topmtx();
int t1, t2;
int m1, m2;
float mval;
char *corfile = yap_makename(resstem,".topcor");
fp = fopen(corfile,"w");
if ( !fp )
yap_sysquit("Cannot open file '%s' for write\n", corfile);
/*
* print file
*/
for (t1=0; t1<ddN.T; t1++) {
for (t2=0; t2<t1; t2++)
if ( cmtx[t1][t2]>1.0e-7 )
fprintf(fp, "%d %d %0.6f\n", t1, t2, cmtx[t1][t2]);
}
fclose(fp);
free(corfile);
/*
* display maximum
*/
m1 = 1; m2 = 0;
mval = cmtx[1][0];
for (t1=0; t1<ddN.T; t1++) {
for (t2=0; t2<t1; t2++) {
if ( mval<cmtx[t1][t2] ) {
mval = cmtx[t1][t2];
m1 = t1;
m2 = t2;
}
}
}
yap_message("Maximum correlated topics (%d,%d) = %f\n", m1, m2, mval);
free(cmtx[0]); free(cmtx);
}
/*
* print burstiness report
*/
if ( PCTL_BURSTY() ) {
int tottbl = 0;
int totmlttbl = 0;
int totmlt = 0;
int i;
for (i=0; i<ddN.NT; i++) {
if ( Z_issetr(ddS.z[i]) ) {
if ( M_multi(i) )
totmlttbl++;
tottbl++;
}
if ( M_multi(i) )
totmlt++;
}
yap_message("Burst report: multis=%.2lf%%, tables=%.2lf%%, tbls-in-multis=%.2lf%%\n",
100.0*((double)ddM.dim_multiind)/ddN.N,
100.0*((double)tottbl)/ddN.NT,
100.0*((double)totmlttbl)/totmlt);
}
yap_message("\n");
free(topfile);
if ( repfile ) free(repfile);
if ( top1cnt ) free(top1cnt);
free(indk);
free(psort);
if ( ngalpha )
free(ngalpha);
if ( pmicount )
free(tpmi);
if ( NwK ) {
free(NwK);
NwK = NULL;
}
#ifdef KL
free(dfvec);
#endif
free(pvec);
free(gtvec);
free(gpvec);
tstats_free(termstats);
}
void hca_displaytopics(char *resstem, int topword, enum ScoreType scoretype) {
int w,k;
int *indk = NULL;
int Nk_tot = 0;
double (*tscore)(int) = NULL;
double sparsityword = 0;
double sparsitydoc = 0;
double underused = 0;
char *fname = yap_makename(resstem,".top");
int nophi = (ddP.phi==NULL) && (ddS.phi==NULL);
FILE *fp;
if ( scoretype == ST_idf ) {
tscore = idfscore;
} else if ( scoretype == ST_phi ) {
tscore = phiscore;
} else if ( scoretype == ST_count ) {
tscore = countscore;
} else if ( scoretype == ST_cost ) {
tscore = costscore;
} else if ( scoretype == ST_Q ) {
tscore = Qscore;
lowerQ = 1.0/ddN.T;
}
fp = fopen(fname,"w");
if ( !fp )
yap_sysquit("Cannot open file '%s' for write\n", fname);
/*
* first collect counts of each word/term
*/
if ( scoretype != ST_count && scoretype != ST_phi ) {
NwK = u32vec(ddN.W);
if ( !NwK )
yap_quit("Out of memory in hca_displaytopics()\n");
for (w=0; w<ddN.W; w++) {
NwK[w] = 0;
}
NWK = 0;
for (w=0; w<ddN.W; w++) {
for (k=0; k<ddN.T; k++) {
NwK[w] += ddS.Nwt[w][k]; // should use CCT_ReadN()
}
NWK += NwK[w];
}
}
assert(ddN.tokens);
for (k=0; k<ddN.T; k++) {
Nk_tot += ddS.NWt[k];
}
indk = malloc(sizeof(*indk)*ddN.W);
if ( !indk )
yap_quit("Cannot allocate indk\n");
for (k=0; k<ddN.T; k++) {
int cnt;
double spw;
double spd;
tscorek = k;
/*
* print top words
*/
cnt=0;
if ( ddP.phi==NULL ) {
for (w=0; w<ddN.W; w++) {
if ( ddS.Nwt[w][k]>0 ) indk[cnt++] = w;
}
} else {
float **phi;
if ( ddP.phi )
phi = ddP.phi;
else
phi = ddS.phi;
for (w=0; w<ddN.W; w++) {
if ( phi[k][w]>0.5/ddN.W ) indk[cnt++] = w;
}
}
topk(topword, cnt, indk, tscore);
spd = ((double)nonzero_Ndt(k))/((double)ddN.DT);
sparsitydoc += spd;
if ( nophi ) {
spw = ((double)nonzero_Nwt(k))/((double)ddN.W);
sparsityword += spw;
}
if ( ddS.NWt[k]*ddN.T*100<Nk_tot )
underused++;
yap_message("\nTopic %d (", k);
if ( ddP.phi==NULL )
yap_message((ddN.T>200)?"p=%.3lf%%,":"p=%.2lf%%,",
100*((double)ddS.NWt[k])/(double)Nk_tot);
if ( nophi )
yap_message("ws=%.1lf%%,", 100*(1-spw));
else
yap_message("#=%.0lf,", exp(phi_entropy(k)));
yap_message("ds=%.1lf%%", 100*(1-spd) );
fprintf(fp,"%d: ", k);
yap_message(") words =");
for (w=0; w<topword && w<cnt; w++) {
fprintf(fp," %d", (int)indk[w]);
if ( verbose>2 ) {
double score = tscore(indk[w]);
yap_message(",%s(%6lf)", ddN.tokens[indk[w]], score);
} else
yap_message(",%s", ddN.tokens[indk[w]]);
}
yap_message("\n");
fprintf(fp, "\n");
}
if ( ddP.PYbeta && nophi ) {
int cnt;
/*
* print root words
*/
tscorek = -1;
cnt=0;
for (w=0; w<ddN.W; w++) {
if ( ddS.TwT[w]>0 ) indk[cnt++] = w;
}
topk(topword, cnt, indk, tscore);
yap_message("\nTopic root words =");
fprintf(fp,"-1:");
for (w=0; w<topword && w<cnt; w++) {
fprintf(fp," %d", (int)indk[w]);
if ( verbose>2 ) {
double score = tscore(indk[w]);
yap_message(",%s(%6lf)", ddN.tokens[indk[w]], score);
} else
yap_message(",%s", ddN.tokens[indk[w]]);
}
yap_message("\n");
fprintf(fp, "\n");
}
if ( nophi )
yap_message("Average topicXword sparsity = %.2lf%%, ",
100*(1-sparsityword/ddN.T) );
yap_message("Average docXtopic sparsity = %.2lf%%, "
"underused topics = %.1lf%%\n",
100*(1-sparsitydoc/ddN.T),
100.0*underused/(double)ddN.T);
if ( ddP.bdk!=NULL) {
int tottbl = 0;
int totmlttbl = 0;
int totmlt = 0;
int i;
for (i=0; i<ddN.NT; i++) {
if ( Z_issetr(ddS.z[i]) ) {
if ( M_multi(i) )
totmlttbl++;
tottbl++;
}
if ( M_multi(i) )
totmlt++;
}
yap_message("doc PYP report: multis=%.2lf%%, tables=%.2lf%%, tbls-in-multis=%.2lf%%\n",
100.0*((double)ddM.dim_multiind)/ddN.N,
100.0*((double)tottbl)/ddN.NT,
100.0*((double)totmlttbl)/totmlt);
}
fclose(fp);
free(fname);
free(indk);
if ( scoretype != ST_count ) {
free(NwK);
NwK = NULL;
}
}
void hca_displaytopics(char *stem, char *resstem, int topword,
enum ScoreType scoretype, int pmicount, int fullreport) {
int w,k;
uint32_t *indk = NULL;
int Nk_tot = 0;
double (*tscore)(int) = NULL;
double sparsityword = 0;
double sparsitydoc = 0;
double underused = 0;
uint32_t *top1cnt = NULL;
FILE *fp;
float *tpmi = NULL;
char *topfile;
char *repfile;
uint32_t *psort;
FILE *rp = NULL;
float *gtvec = globalprop();
float *gpvec = calloc(ddN.W,sizeof(gpvec[0]));
float *pvec = calloc(ddN.W,sizeof(pvec[0]));
if ( pmicount>topword )
pmicount = topword;
if ( scoretype == ST_idf ) {
tscore = idfscore;
} else if ( scoretype == ST_phi ) {
tscore = phiscore;
} else if ( scoretype == ST_count ) {
tscore = countscore;
} else if ( scoretype == ST_cost ) {
tscore = costscore;
} else if ( scoretype == ST_Q ) {
tscore = Qscore;
lowerQ = 1.0/ddN.T;
}
/*
* first collect counts of each word/term,
* and build gpvec (mean word probs)
*/
build_NwK();
{
/*
* gpvec[] is normalised NwK[]
*/
double tot = 0;
for (w=0; w<ddN.W; w++)
tot += gpvec[w] = NwK[w]+0.1;
for (w=0; w<ddN.W; w++)
gpvec[w] /= tot;
}
if ( ddS.Nwt ) {
for (k=0; k<ddN.T; k++) {
Nk_tot += ddS.NWt[k];
}
}
psort = sorttops(gtvec, ddN.T);
top1cnt = hca_top1cnt();
if ( !top1cnt )
yap_quit("Cannot allocate top1cnt in hca_displaytopics()\n");
if ( pmicount ) {
tpmi = malloc(sizeof(*tpmi)*(ddN.T+1));
if ( !tpmi )
yap_quit("Cannot allocate tpmi in hca_displaytopics()\n");
}
indk = malloc(sizeof(*indk)*ddN.W);
if ( !indk )
yap_quit("Cannot allocate indk in hca_displaytopics()\n");
/*
* two passes through,
* first to build the top words and dump to file
*/
repfile = yap_makename(resstem,".topset");
topfile = yap_makename(resstem,".toplst");
fp = fopen(topfile,"w");
if ( !fp )
yap_sysquit("Cannot open file '%s' for write\n", topfile);
yap_message("\n");
for (k=0; k<ddN.T; k++) {
int cnt;
tscorek = k;
/*
* build sorted word list
*/
cnt = buildindk(k, indk);
topk(topword, cnt, indk, tscore);
if ( cnt==0 )
continue;
/*
* dump words to file
*/
fprintf(fp,"%d: ", k);
for (w=0; w<topword && w<cnt; w++) {
fprintf(fp," %d", (int)indk[w]);
}
fprintf(fp, "\n");
}
if ( ddP.PYbeta && (ddP.phi==NULL || ddP.betapr) ) {
int cnt;
/*
* dump root words
*/
tscorek = -1;
cnt = buildindk(-1, indk);
topk(topword, cnt, indk, (ddP.phi==NULL)?countscore:phiscore);
fprintf(fp,"-1:");
for (w=0; w<topword && w<cnt; w++) {
fprintf(fp," %d", (int)indk[w]);
}
fprintf(fp, "\n");
}
fclose(fp);
if ( verbose>1 ) yap_message("\n");
if ( pmicount ) {
/*
* compute PMI
*/
char *toppmifile;
char *pmifile;
double *tp;
tp = dvec(ddN.T);
pmifile=yap_makename(stem,".pmi");
toppmifile=yap_makename(resstem,".toppmi");
get_probs(tp);
report_pmi(topfile, pmifile, toppmifile, ddN.T, ddN.W, 1,
pmicount, tp, tpmi);
free(toppmifile);
free(pmifile);
free(tp);
}
/*
* now report words and diagnostics
*/
//ttop_open(topfile);
if ( fullreport ) {
rp = fopen(repfile,"w");
if ( !rp )
yap_sysquit("Cannot open file '%s' for write\n", repfile);
fprintf(rp, "#topic index rank prop word-sparse doc-sparse eff-words eff-docs docs-bound top-one "
"dist-unif dist-unigrm");
if ( PCTL_BURSTY() )
fprintf(rp, " burst-concent");
if ( ddN.tokens )
fprintf(rp, " ave-length");
fprintf(rp, " coher");
if ( pmicount )
fprintf(rp, " pmi");
fprintf(rp, "\n#word topic index rank");
if ( ddS.Nwt )
fprintf(rp, " count");
fprintf(rp, " prop cumm df coher\n");
}
for (k=0; k<ddN.T; k++) {
int cnt;
int kk = psort[k];
uint32_t **dfmtx;
if ( ddP.phi==NULL && ddS.NWt[kk]==0 )
continue;
/*
* grab word prob vec for later use
*/
if ( ddS.Nwt ) {
int w;
for (w=0; w<ddN.W; w++)
pvec[w] = wordprob(w,kk);
} else if ( ddP.phi )
fv_copy(pvec, ddP.phi[kk], ddN.W);
else if ( ddS.phi )
fv_copy(pvec, ddS.phi[kk], ddN.W);
/*
* rebuild word list
*/
tscorek = kk;
cnt = buildindk(kk, indk);
topk(topword, cnt, indk, tscore);
if ( topword<cnt )
cnt = topword;
assert(cnt>0);
/*
* df stats for topic returned as matrix
*/
dfmtx = hca_dfmtx(indk, cnt, kk);
if ( ddS.Nwt && (ddS.NWt[kk]*ddN.T*100<Nk_tot || ddS.NWt[kk]<5 ))
underused++;
/*
* print stats for topic
* Mallet: tokens, doc_ent, ave-word-len, coher.,
* uni-dist, corp-dist, eff-no-words
*/
yap_message("Topic %d/%d", kk, k);
{
/*
* compute diagnostics
*/
double prop = gtvec[kk];
float *dprop = docprop(kk);
double spw = 0;
double spd = ((double)nonzero_Ndt(kk))/((double)ddN.DT);
double ew = exp(fv_entropy(pvec,ddN.W));
double ud = fv_helldistunif(pvec,ddN.W);
double pd = fv_helldist(pvec,gpvec,ddN.W);
double sl = fv_avestrlen(pvec,ddN.tokens,ddN.W);
double co = coherence(dfmtx, cnt);
double ed = dprop?exp(fv_entropy(dprop,ddN.DT)):ddN.DT;
double da = dprop?fv_bound(dprop,ddN.DT,1.0/sqrt((double)ddN.T)):0;
sparsitydoc += spd;
yap_message((ddN.T>200)?" p=%.3lf%%":" p=%.2lf%%",100*prop);
if ( ddS.Nwt ) {
spw = ((double)nonzero_Nwt(kk))/((double)ddN.W);
sparsityword += spw;
yap_message(" ws=%.1lf%%", 100*(1-spw));
}
yap_message(" ds=%.1lf%%", 100*(1-spd) );
yap_message(" ew=%.0lf", ew);
yap_message(" ed=%.1lf", ed);
yap_message(" da=%.0lf", da+0.1);
yap_message(" t1=%u", top1cnt[kk]);
yap_message(" ud=%.3lf", ud);
yap_message(" pd=%.3lf", pd);
if ( PCTL_BURSTY() )
yap_message(" bd=%.3lf", ddP.bdk[kk]);
if ( ddN.tokens )
yap_message(" sl=%.2lf", sl);
yap_message(" co=%.3lf%%", co);
if ( pmicount )
yap_message(" pmi=%.3f", tpmi[kk]);
if ( fullreport ) {
fprintf(rp,"topic %d %d", kk, k);
fprintf(rp," %.6lf", prop);
if ( ddS.Nwt ) {
fprintf(rp," %.6lf", (1-spw));
} else {
fprintf(rp," 0");
}
fprintf(rp," %.6lf", (1-spd) );
fprintf(rp," %.2lf", ew);
fprintf(rp," %.2lf", ed);
fprintf(rp," %.0lf", da+0.1);
fprintf(rp," %u", top1cnt[kk]);
fprintf(rp," %.6lf", ud);
fprintf(rp," %.6lf", pd);
if ( PCTL_BURSTY() )
fprintf(rp," %.3lf", ddP.bdk[kk]);
fprintf(rp," %.4lf", (ddN.tokens)?sl:0);
fprintf(rp," %.6lf", co);
if ( pmicount )
fprintf(rp," %.4f", tpmi[kk]);
fprintf(rp,"\n");
}
if ( dprop) free(dprop);
}
if ( verbose>1 ) {
double pcumm = 0;
/*
* print top words:
* Mallet: rank, count, prob, cumm, docs, coh
*/
yap_message("\ntopic %d/%d", kk, k);
yap_message(" words=");
for (w=0; w<cnt; w++) {
if ( w>0 ) yap_message(",");
if ( ddN.tokens )
yap_message("%s", ddN.tokens[indk[w]]);
else
yap_message("%d", indk[w]);
if ( verbose>2 )
yap_message("(%6lf)", tscore(indk[w]));
if ( fullreport ) {
fprintf(rp, "word %d %d %d", kk, indk[w], w);
if ( ddS.Nwt )
fprintf(rp, " %d", ddS.Nwt[indk[w]][kk]);
pcumm += pvec[indk[w]];
fprintf(rp, " %.6f %.6f", pvec[indk[w]], pcumm);
fprintf(rp, " %d", dfmtx[w][w]);
fprintf(rp, " %.6f", coherence_word(dfmtx, cnt, w));
if ( ddN.tokens )
fprintf(rp, " %s", ddN.tokens[indk[w]]);
fprintf(rp, "\n");
}
}
}
yap_message("\n");
free(dfmtx[0]); free(dfmtx);
}
if ( verbose>1 && ddP.PYbeta && (ddP.phi==NULL || ddP.betapr) ) {
int cnt;
double pcumm = 0;
/*
* print root words
*/
tscorek = -1;
cnt = buildindk(-1,indk);
topk(topword, cnt, indk, (ddP.phi==NULL)?countscore:phiscore);
/*
* cannot build df mtx for root because
* it is latent w.r.t. topics
*/
yap_message("Topic root words=");
if ( fullreport ) {
int w;
for (w=0; w<ddN.W; w++)
pvec[w] = betabasewordprob(w);
double ew = exp(fv_entropy(pvec,ddN.W));
double ud = fv_helldistunif(pvec,ddN.W);
double pd = fv_helldist(pvec,gpvec,ddN.W);
fprintf(rp,"topic -1 -1 0 0");
fprintf(rp," %.4lf", ew);
fprintf(rp," %.6lf", ud);
fprintf(rp," %.6lf", pd);
fprintf(rp,"\n");
}
for (w=0; w<topword && w<cnt; w++) {
if ( w>0 ) yap_message(",");
if ( ddN.tokens )
yap_message("%s", ddN.tokens[indk[w]]);
else
yap_message("%d", indk[w]);
if ( verbose>2 )
yap_message("(%6lf)", countscore(indk[w]));
if ( fullreport ) {
fprintf(rp, "word %d %d %d", -1, indk[w], w);
if ( ddS.TwT )
fprintf(rp, " %d", ddS.TwT[w]);
pcumm += pvec[indk[w]];
fprintf(rp, " %.6f %.6f", pvec[indk[w]], pcumm);
fprintf(rp, " 0 0");
if ( ddN.tokens )
fprintf(rp, " %s", ddN.tokens[indk[w]]);
fprintf(rp, "\n");
}
}
yap_message("\n");
}
yap_message("\n");
if ( rp )
fclose(rp);
if ( ddS.Nwt )
yap_message("Average topicXword sparsity = %.2lf%%\n",
100*(1-sparsityword/ddN.T) );
yap_message("Average docXtopic sparsity = %.2lf%%\n"
"Underused topics = %.1lf%%\n",
100*(1-sparsitydoc/ddN.T),
100.0*underused/(double)ddN.T);
if ( pmicount )
yap_message("Average PMI = %.3f\n", tpmi[ddN.T]);
/*
* print
*/
if ( 1 ) {
float **cmtx = hca_topmtx();
int t1, t2;
int m1, m2;
float mval;
char *corfile = yap_makename(resstem,".topcor");
fp = fopen(corfile,"w");
if ( !fp )
yap_sysquit("Cannot open file '%s' for write\n", corfile);
/*
* print file
*/
for (t1=0; t1<ddN.T; t1++) {
for (t2=0; t2<t1; t2++)
if ( cmtx[t1][t2]>1.0e-3 )
fprintf(fp, "%d %d %0.6f\n", t1, t2, cmtx[t1][t2]);
}
fclose(fp);
free(corfile);
/*
* display maximum
*/
m1 = 1; m2 = 0;
mval = cmtx[1][0];
for (t1=0; t1<ddN.T; t1++) {
for (t2=0; t2<t1; t2++) {
if ( mval<cmtx[t1][t2] ) {
mval = cmtx[t1][t2];
m1 = t1;
m2 = t2;
}
}
}
yap_message("Maximum correlated topics (%d,%d) = %f\n", m1, m2, mval);
free(cmtx[0]); free(cmtx);
}
/*
* print burstiness report
*/
if ( PCTL_BURSTY() ) {
int tottbl = 0;
int totmlttbl = 0;
int totmlt = 0;
int i;
for (i=0; i<ddN.NT; i++) {
if ( Z_issetr(ddS.z[i]) ) {
if ( M_multi(i) )
totmlttbl++;
tottbl++;
}
if ( M_multi(i) )
totmlt++;
}
yap_message("Burst report: multis=%.2lf%%, tables=%.2lf%%, tbls-in-multis=%.2lf%%\n",
100.0*((double)ddM.dim_multiind)/ddN.N,
100.0*((double)tottbl)/ddN.NT,
100.0*((double)totmlttbl)/totmlt);
}
yap_message("\n");
free(topfile);
if ( repfile ) free(repfile);
if ( top1cnt ) free(top1cnt);
free(indk);
free(psort);
if ( pmicount )
free(tpmi);
if ( NwK ) {
free(NwK);
NwK = NULL;
}
free(pvec);
free(gtvec);
free(gpvec);
}
/*
* run regular gibbs cycles on the data with phi used;
* the evaluation on each doc, and sample word probs
*
* if qparts>0, split collection into parts and only search this
*
* K = number of top results to retain
*/
void gibbs_query(int K, char *qname, int dots, int this_qpart, int qparts) {
/*
* mapping from query word posn. to its mi in current doc
* >ddN.N = not in current doc
* -ve = has no mi since occurs just once, found at
* posn (-map[]-1)
* non -ve = mi value
*/
int *mimap = NULL;
/*
* usual stuff for Gibbs loop over docs
*/
int i, j;
float *fact = fvec(ddN.T*4);
D_MiSi_t dD;
/*
* an index into topk[] which maintains ordering
*/
int *topind;
/*
* these store statistics of the results, for printing
* these are unordered, ordered by topind[]
*/
/* document score */
float *topscore;
/* document number */
int *topk;
/* flags if ord is irrelevant, thus not scored */
char *wordunused;
/*
* per word stats for top results saved
*/
int *found;
float *topcnt;
float *topwordscore;
/*
* temporary versions for when gibbs running
*/
int *found_buf;
float *topcnt_buf;
float *topwordscore_buf;
double *logprob;
/*
* search here
*/
int startdoc = 0;
int enddoc = ddN.DT;
/*
* setup
*/
topcnt = malloc(sizeof(topcnt[0])*K*ddP.n_words);
topwordscore = malloc(sizeof(topwordscore[0])*K*ddP.n_words);
found = malloc(sizeof(found)*ddP.n_words*K);
wordunused = malloc(sizeof(wordunused[0])*ddP.n_words);
topcnt_buf = malloc(sizeof(topcnt[0])*ddP.n_words);
topwordscore_buf = malloc(sizeof(topwordscore[0])*ddP.n_words);
found_buf = malloc(sizeof(found)*ddP.n_words);
if ( !topcnt || !topwordscore || !found ||
!topcnt_buf || !topwordscore_buf || !found_buf )
yap_quit("Cannot allocate memory in gibbs_query()\n");
logprob = malloc(sizeof(logprob[0])*ddP.n_query);
topscore = malloc(sizeof(topscore[0])*K*ddP.n_query);
topind = malloc(sizeof(topind[0])*K*ddP.n_query);
topk = malloc(sizeof(topk[0])*K*ddP.n_query);
if ( ddP.bdk!=NULL )
mimap = malloc(sizeof(mimap[0])*ddP.n_words);
if ( !topk || !topscore || !logprob || !topind )
yap_quit("Cannot allocate memory in gibbs_query()\n");
for (i=0; i<ddP.n_words; i++) {
wordunused[i] = 0;
}
for (i=0; i<K*ddP.n_query; i++) {
topind[i] = i%K;
topk[i] = -1;
topscore[i] = INFINITY;
}
/*
* check words to exclude using topics
*/
if ( ddP.n_excludetopic>0 ) {
double *tprob = malloc(sizeof(tprob[0])*ddN.T);
get_probs(tprob);
yap_probs();
if ( verbose>1 )
yap_message("Excluding words: ");
for (i=0; i<ddP.n_words; i++) {
int t = besttopic(ddP.qword[i],tprob);
if ( Q_excludetopic(t) ) {
wordunused[i] = 1;
if ( verbose>1 )
yap_message(" %d/%d", (int)ddP.qword[i], t);
}
}
if ( verbose>1 )
yap_message("\n");
free(tprob);
}
if ( ddP.bdk!=NULL ) misi_init(&ddM,&dD);
if ( qparts>0 ) {
startdoc = ((double)this_qpart)/qparts * ddN.DT;
enddoc = ((double)this_qpart+1.0)/qparts * ddN.DT;
}
for(i=startdoc; i<enddoc; i++) {
int thisw = add_doc(i, GibbsNone);
int r;
if ( thisw<=1 ) {
remove_doc(i, GibbsNone);
continue;
}
if ( ddP.bdk!=NULL )
misi_build(&dD, i, 0);
map_query(i, mimap, found_buf);
for (j=0; j<ddP.n_words; j++) {
topcnt_buf[j] = 0;
topwordscore_buf[j] = 0;
}
for (r=0; r<ddP.queryiter; r++) {
gibbs_lda(GibbsNone, ddN.T, i, ddD.NdT[i], fact, &dD, 0, 0);
query_docprob(i, mimap, fact, &dD, topcnt_buf, topwordscore_buf);
}
/*
* now adjust stats
*/
for (j=0; j<ddP.n_query; j++)
logprob[j] = 0;
for (j=0; j<ddP.n_words; j++) {
if ( wordunused[j]>0 )
continue;
if ( ddP.query[ddP.qword[j]]==j ) {
topcnt_buf[j] /= ddP.queryiter;
topwordscore_buf[j] /= ddP.queryiter;
} else {
/* word in previous query so copy */
int jj = ddP.query[ddP.qword[j]];
topcnt_buf[j] = topcnt_buf[jj];
topwordscore_buf[j] = topwordscore_buf[jj];
found_buf[j] = found_buf[jj];
}
if ( wordunused[j]==0 )
logprob[ddP.qid[j]] += topwordscore_buf[j];
}
if ( dots>0 && i>0 && (i%dots==0) )
yap_message(".");
if ( ddP.bdk!=NULL ) misi_unbuild(&dD,i,0);
remove_doc(i, GibbsNone);
/*
* enter into the arrays
*/
for (j=0; j<ddP.n_query; j++) {
if ( i<K || logprob[j] < topscore[j*K+topind[j*K+K-1]] ) {
int newind, l;
/*
* better than current lowest
*/
newind = bubble((i<K)?(i+1):K,
&topind[j*K], &topscore[j*K], logprob[j]);
/*
* save the current details
*/
topscore[j*K+newind] = logprob[j];
topk[j*K+newind] = i;
for (l=ddP.qposn[j]; l<ddP.qposn[j+1]; l++) {
topcnt[newind*ddP.n_words+l] = topcnt_buf[l];
topwordscore[newind*ddP.n_words+l] = topwordscore_buf[l];
found[newind*ddP.n_words+l] = found_buf[l];
}
}
}
}
if ( dots>0 ) yap_message("\n");
/*
* write result
*/
{
float *ws = fvec(ddP.n_words);
FILE *fp = fopen(qname,"w");
int q;
if ( !fp )
yap_sysquit("Cannot write query results to '%s'\n", qname);
for (q=0; q<ddP.n_query; q++) {
int nw = ddP.qposn[q+1]-ddP.qposn[q];
for (i=0; i<K && i<ddN.DT && topk[topind[q*K+i]]>=0; i++) {
int l, ind = topind[q*K+i];
double tfidf;
tfidf = bm25(topk[q*K+ind],&found[ind*ddP.n_words+ddP.qposn[q]],
&ddP.qword[ddP.qposn[q]], nw, ws);
assert(ind>=0 && ind<K);
fprintf(fp, "%d %d ", q, topk[q*K+ind]);
fprintf(fp, "%.4f %.4lf ", topscore[q*K+ind]/nw, tfidf);
if ( verbose>1 ) {
for (l=ddP.qposn[q]; l<ddP.qposn[q+1]; l++)
fprintf(fp, "%d ", found[ind*ddP.n_words+l]);
for (l=ddP.qposn[q]; l<ddP.qposn[q+1]; l++)
fprintf(fp, "%f ", topcnt[ind*ddP.n_words+l]);
for (l=ddP.qposn[q]; l<ddP.qposn[q+1]; l++)
fprintf(fp, "%f ", topwordscore[ind*ddP.n_words+l]);
for (l=0; l<nw; l++)
fprintf(fp, "%lf ", ws[l]);
}
fprintf(fp, "\n");
}
}
fclose(fp);
free(ws);
}
/*
* clean up
*/
free(fact);
if ( ddP.bdk!=NULL ) misi_free(&dD);
if ( mimap ) free(mimap);
free(found);
free(topwordscore);
free(topcnt);
free(found_buf);
free(topwordscore_buf);
free(topcnt_buf);
free(topscore);
free(topind);
free(topk);
free(logprob);
}
