void query_read(char *qname) {
FILE *fp;
unsigned win = 0, nw = 0, nq=0, qin;
int i;
uint32_t *wlist = malloc(sizeof(wlist[0])*QMAX);
uint32_t *qlist = malloc(sizeof(qlist[0])*QMAX);
int16_t *map = malloc(sizeof(map[0])*ddN.W);
if ( !map || !wlist || !qlist)
yap_quit("Cannot allocate memory in query_read()\n");
fp = fopen(qname,"r");
if ( !fp )
yap_sysquit("Cannot open query bag file '%s'\n", qname);
for (i=0; i<ddN.W; i++)
map[i] = -1;
nw = 0; qin = 0;
while ( fscanf(fp," %u", &nw) == 1 ) {
for (i=0; i<nw; i++) {
if ( fscanf(fp," %u", &win) != 1 || win>=ddN.W )
yap_sysquit("Cannot read %d-th entry from '%s'\n",
i, qname);
if ( map[win]<0 ) {
qlist[nq] = qin;
wlist[nq] = win;
map[win] = nq++;
if ( nq>=QMAX )
yap_quit("Predefined query length maximum (%d) too small\n", QMAX);
} else {
/*
* word appears already: same query, drop, other query, copy
*/
if ( qlist[map[win]]!=qin ) {
qlist[nq] = qin;
wlist[nq] = win;
nq++;
if ( nq>=QMAX )
yap_quit("Predefined query length maximum (%d) too small\n", QMAX);
}
}
}
qin++;
nw = 0;
}
if ( ferror(fp) )
yap_sysquit("Cannot read data line from '%s'\n", qname);
fclose(fp);
ddP.query = map;
ddP.qid = realloc(qlist, nq*sizeof(qlist[0]));
ddP.qword = realloc(wlist, nq*sizeof(wlist[0]));
ddP.qposn = malloc(sizeof(ddP.qposn[0])*(qin+1));
if ( !ddP.qword || !ddP.qid || !ddP.qposn )
yap_quit("Cannot allocate memory in query_read()\n");
ddP.n_words = nq;
ddP.n_query = qin;
ddP.qposn[0] = 0;
for (i=1; i<ddP.n_words; i++) {
if ( ddP.qid[i] != ddP.qid[i-1] )
ddP.qposn[ddP.qid[i]] = i;
}
ddP.qposn[ddP.n_query] = ddP.n_words;
}
void read_u32sparsetri(int ns, int nr, int nc, uint32_t ***x, char *fname)
{
FILE *fp = fopen(fname,"r");
int i, u, v, w;
int Nin;
int sparse=0;
if ( !fp )
yap_sysquit("Cannot open file '%s' for read\n", fname);
if ( fscanf(fp, "%d", &Nin)!=1 || Nin!=ns )
yap_quit("Number blocks wrong for sparse matrix in '%s', should be %d\n",
fname, ns);
if ( fscanf(fp, "%d", &Nin)!=1 || Nin!=nr )
yap_quit("Number rows wrong for sparse matrix in '%s', should be %d\n",
fname, nr);
if ( fscanf(fp, "%d", &Nin)!=1 || Nin!=nc )
yap_quit("Number columns wrong for sparse matrix in '%s', should be %d\n",
fname, nc);
if ( fscanf(fp, "%d", &sparse)!=1 )
yap_quit("Cannot read sparsity in matrix in '%s'\n", fname);
for (i = 0; i < sparse; i++) {
if ( fscanf(fp, "%d %d %d %u", &u, &v, &w, &Nin) !=4 )
yap_quit("Cannot read line %d in matrix in '%s'\n", i, fname);
x[u][v][w] = Nin;
}
if ( ferror(fp) )
yap_sysquit("Error on reading file '%s' ", fname);
fclose(fp);
}
static void build_NwK() {
int w, k;
NwK = u32vec(ddN.W);
if ( !NwK )
yap_quit("Out of memory in hca_displaytopics()\n");
for (w=0; w<ddN.W; w++) {
NwK[w] = 0;
}
NWK = 0;
if ( !ddS.Nwt ) {
/*
* recompute from scratch
*/
int i;
for (i=0; i<ddN.NT; i++)
NwK[ddD.w[i]]++;
for (w=0; w<ddN.W; w++)
NWK += NwK[w];
} else {
for (w=0; w<ddN.W; w++) {
for (k=0; k<ddN.T; k++) {
NwK[w] += ddS.Nwt[w][k]; // should use CCT_ReadN()
}
NWK += NwK[w];
}
}
if ( NWK==0 )
yap_quit("empty NWK in build_NwK()\n");
}
/*
* pick up line from file "stem.ext" starting with "par" and an "=",
* and return stuff after "=" but only first len chars
*/
static char *readextpar(char *stem, char *ext, char *par, char *buf, int len) {
char *file = yap_makename(stem,ext);
FILE *fp = fopen(file,"r");
int parlen = strlen(par);
int buflen = len+parlen+50;
char *linebuf = malloc(buflen+2);
if ( !fp )
yap_sysquit("Cannot open parameter file '%s' ", file);
if ( !linebuf )
yap_quit("Out of memory in readpar(%s)\n", par);
buf[0] = 0;
linebuf[0] = 0;
while ( fgets(&linebuf[0],buflen,fp) ) {
if ( strstr(&linebuf[0],par)==&linebuf[0] &&
(linebuf[parlen]==' ' || linebuf[parlen]=='=') ) {
char *ret = strstr(&linebuf[0],"=");
if ( !ret ) {
yap_quit("Badly formatted parameter file '%s': %s\n",
file, &linebuf[0]);
}
ret++;
// yap_message("ret = '%s', length %d\n", ret, strlen(ret));
{
int i;
i = strlen(ret);
if ( i>=len ) {
i = len-1;
ret[i] = 0;
i--;
}
for ( ; i>=0; i--)
buf[i] = ret[i];
/* for some weird reason, this fails occasionally!! */
// strncpy(buf, ret, len);
}
strncpy(buf, ret, len);
break;
}
}
if ( ferror(fp) )
yap_sysquit("Error on parameter file '%s' ", file);
fclose(fp);
free(file);
free(linebuf);
if ( buf[0] )
return &buf[0];
else
return NULL;
}
static void merge_init_Twt(int k1, int k2, merge_beta_t *M) {
int w;
/*
* build local store
*/
M->Twt = u16vec(ddN.W);
M->TwT = u32vec(ddN.W);
M->Nwt = u32vec(ddN.W);
if ( !M->Nwt || !M->TwT || !M->Twt )
yap_quit("Out of memory in likemerge()\n");
/*
* initialise all data entries to simple sum
*/
M->NWt = ddS.NWt[k1] + ddS.NWt[k2];
M->TWt = ddS.TWt[k1] + ddS.TWt[k2];
for (w=0; w<ddN.W; w++) {
M->Twt[w] = ddS.Twt[w][k1] + ddS.Twt[w][k2];
M->Nwt[w] = ddS.Nwt[w][k1] + ddS.Nwt[w][k2];
M->TwT[w] = ddS.TwT[w];
}
M->TWTm = ddS.TWT - M->TWt;
#ifndef NDEBUG
for (w=0; w<ddN.W; w++) {
assert(M->Twt[w]<=M->Nwt[w]);
}
#endif
}
static void merge_init_Tdt(int k1, int k2, merge_alpha_t *M) {
int d, k;
/*
* build local store
*/
M->Tdt = u16vec(ddN.DT);
M->TdT = u16vec(ddN.DT);
M->Ndt = u16vec(ddN.DT);
if ( !M->Ndt || !M->TdT || !M->Tdt )
yap_quit("Out of memory in likemerge()\n");
/*
* initialise all data entries to simple sum
*/
M->TDt = ddS.TDt[k1] + ddS.TDt[k2];
for (d=0; d<ddN.DT; d++) {
M->Tdt[d] = ddS.Tdt[d][k1] + ddS.Tdt[d][k2];
M->Ndt[d] = ddS.Ndt[d][k1] + ddS.Ndt[d][k2];
M->TdT[d] = 0;
for (k=0; k<ddN.T; k++)
M->TdT[d] += ddS.Tdt[d][k];
assert(M->Tdt[d]<=M->Ndt[d]);
}
M->TDTm = ddS.TDT - M->TDt;
#ifndef NDEBUG
for (d=0; d<ddN.DT; d++) {
assert(M->Tdt[d]<=M->Ndt[d]);
}
#endif
}
uint32_t *u32vec(int n)
{
uint32_t *x = (uint32_t*)calloc(n,sizeof(uint32_t));
memallocd += n*sizeof(uint32_t);
if ( !x)
yap_quit("cannot allocate uint32_t space of %ld\n", n*sizeof(uint32_t));
return x;
}
sparse_vec *smat(int nr) //
{
sparse_vec *x = (sparse_vec *)calloc(nr,sizeof(sparse_vec));
memallocd += nr*sizeof(sparse_vec);
if ( !x)
yap_quit("cannot allocate sparse_vec space of %ld\n", nr *sizeof(sparse_vec));
return x;
}
double *dvec(int n) //
{
double *x = (double*)calloc(n,sizeof(double));
memallocd += n*sizeof(double);
if ( !x)
yap_quit("cannot allocate double space of %ld\n", n*sizeof(double));
return x;
}
float *fvec(int n) //
{
float *x = (float*)calloc(n,sizeof(float));
memallocd += n*sizeof(float);
if ( !x)
yap_quit("cannot allocate float space of %ld\n", n*sizeof(float));
return x;
}
static void build_termNwK(T_stats_t *ptr) {
int w, k;
termNwK = u32vec(ptr->K);
if ( !termNwK )
yap_quit("Out of memory in hca_displaytopics()\n");
for (w=0; w<ptr->K; w++) {
termNwK[w] = 0;
}
termNWK = 0;
for (w=0; w<ptr->K; w++) {
for (k=0; k<ddN.T; k++) {
termNwK[w] += ptr->Nkt[w][k];
}
termNWK += termNwK[w];
}
if ( termNWK==0 )
yap_quit("empty termNWK in build_termNwK(), collocations empty!\n");
termNwk = ptr->Nkt;
}
/*
* place all topic k2 data in topic k1
*/
double likemerge(int k1, int k2) {
int t, d, k;
/*
* storing/saving
*/
uint32_t TDt;
uint16_t *Tdt;
/*
* cache
*/
uint16_t *TdT;
uint16_t *Ndt;
/*
* sorting on moves
*/
uint16_t *Tdt_up;
uint16_t *Tdt_down;
float *score_up;
float *score_down;
float base_up;
float base_down;
double likelihood = 0;
if ( k1<=0 || k2<=0 )
return 0.0;
/*
* we save all the old values relevant to topic k1,
* and overwrite temporarily so we can use standard
* routines/scoring functions
*/
Tdt = u16vec(ddN.DT);
TdT = u16vec(ddN.DT);
Ndt = u16vec(ddN.DT);
Tdt_up = u16vec(ddN.DT);
Tdt_down = u16vec(ddN.DT);
score_up = u16vec(ddN.DT);
score_down = u16vec(ddN.DT);
if ( !score_down || !score_up || !Ndt || !TdT ||
!Tdt || !Tdt_up || !Tdt_down )
yap_quit("Out of memory in likemerge()\n");
/*
* initialise
*/
TDt = 0; ???????????????
for (d=0; d<ddN.DT; d++) {
void hca_displaytopics(char *stem, char *resstem, int topword,
enum ScoreType scoretype, int pmicount, int fullreport) {
int w,k;
uint32_t *termindk = NULL;
uint32_t *indk = NULL;
int Nk_tot = 0;
double (*termtscore)(int) = NULL;
double (*tscore)(int) = NULL;
double sparsityword = 0;
double sparsitydoc = 0;
double underused = 0;
uint32_t *top1cnt = NULL;
FILE *fp;
float *tpmi = NULL;
char *topfile;
char *repfile;
uint32_t *psort;
FILE *rp = NULL;
float *gtvec = globalprop();
//#define XTRA // prints model topic probs after observed
#ifdef XTRA
double *gtavec = calloc(ddN.T,sizeof(gtavec[0]));
#endif
float *gpvec = calloc(ddN.W,sizeof(gpvec[0]));
float *pvec = calloc(ddN.W,sizeof(pvec[0]));
#ifdef KL
float *dfvec = calloc(ddN.W,sizeof(dfvec[0]));
#endif
double *ngalpha = NULL;
T_stats_t *termstats;
#ifdef XTRA
get_probs(gtavec);
#endif
if ( pmicount>topword )
pmicount = topword;
if ( scoretype == ST_idf ) {
tscore = idfscore;
} else if ( scoretype == ST_phirat ) {
tscore = phiratioscore;
} else if ( scoretype == ST_phi ) {
tscore = phiscore;
} else if ( scoretype == ST_count ) {
tscore = countscore;
} else if ( scoretype == ST_cost ) {
tscore = costscore;
} else if ( scoretype == ST_Q ) {
tscore = Qscore;
lowerQ = 1.0/ddN.T;
}
if ( ddS.TwT==NULL && ddP.phi==NULL && scoretype == ST_phirat )
yap_quit("Cannot use '-orat' option with this model/settings.\n");
if ( ddP.PYalpha==H_NG ) {
/*
* provide an estimate of alpha
*/
ngalpha = dvec(ddN.T);
get_probs(ngalpha);
for (k=0; k<ddN.T; k++) {
ddP.alphapr[k] = ngalpha[k];
}
}
/*
* returns null if no relevant data file
*/
termstats = tstats_init(ddS.z, ddD.NdTcum, ddN.T, ddN.DT, stem);
if ( termstats ) {
if ( scoretype == ST_idf ) {
termtscore = termidfscore;
} else
termtscore = termcountscore;
}
/*
* first collect counts of each word/term,
* and build gpvec (mean word probs)
*/
build_NwK();
if ( termstats )
build_termNwK(termstats);
{
/*
* gpvec[] is normalised NwK[]
*/
double tot = 0;
for (w=0; w<ddN.W; w++)
tot += gpvec[w] = NwK[w]+0.1;
for (w=0; w<ddN.W; w++)
gpvec[w] /= tot;
}
if ( ddS.Nwt ) {
for (k=0; k<ddN.T; k++) {
Nk_tot += ddS.NWt[k];
}
}
psort = sorttops(gtvec, ddN.T);
top1cnt = hca_top1cnt();
if ( !top1cnt )
yap_quit("Cannot allocate top1cnt in hca_displaytopics()\n");
if ( pmicount ) {
tpmi = malloc(sizeof(*tpmi)*(ddN.T+1));
if ( !tpmi )
yap_quit("Cannot allocate tpmi in hca_displaytopics()\n");
}
indk = malloc(sizeof(*indk)*ddN.W);
if ( !indk )
yap_quit("Cannot allocate indk in hca_displaytopics()\n");
if ( termstats ) {
termindk = malloc(sizeof(*indk)*termstats->K);
if ( !termindk )
yap_quit("Cannot allocate termindk in hca_displaytopics()\n");
}
data_df(stem);
#ifdef KL
for (w=0; w<ddN.W; w++)
dfvec[w] = ddD.df[w];
#endif
/*
* two passes through,
* first to build the top words and dump to file
*/
repfile = yap_makename(resstem,".topset");
topfile = yap_makename(resstem,".toplst");
fp = fopen(topfile,"w");
if ( !fp )
yap_sysquit("Cannot open file '%s' for write\n", topfile);
yap_message("\n");
for (k=0; k<ddN.T; k++) {
int cnt, termcnt = 0;
tscorek = k;
/*
* build sorted word list
*/
cnt = buildindk(k, indk);
topk(topword, cnt, indk, tscore);
if ( cnt==0 )
continue;
if ( termstats ) {
termcnt = buildtermindk(k, termindk, termstats);
topk(topword, termcnt, termindk, termtscore);
}
/*
* dump words to file
*/
fprintf(fp,"%d: ", k);
for (w=0; w<topword && w<cnt; w++) {
fprintf(fp," %d", (int)indk[w]);
}
if ( termstats ) {
for (w=0; w<topword && w<termcnt; w++) {
fprintf(fp," %d", (int)termstats->Kmin+termindk[w]);
}
}
fprintf(fp, "\n");
}
if ( ddP.PYbeta && (ddP.phi==NULL || ddP.betapr) ) {
int cnt;
/*
* dump root words
*/
tscorek = -1;
cnt = buildindk(-1, indk);
topk(topword, cnt, indk, (ddP.phi==NULL)?countscore:phiscore);
fprintf(fp,"-1:");
for (w=0; w<topword && w<cnt; w++) {
fprintf(fp," %d", (int)indk[w]);
}
fprintf(fp, "\n");
}
fclose(fp);
if ( verbose>1 ) yap_message("\n");
if ( pmicount ) {
/*
* compute PMI
*/
char *toppmifile;
char *pmifile;
double *tp;
tp = dvec(ddN.T);
pmifile=yap_makename(stem,".pmi");
toppmifile=yap_makename(resstem,".toppmi");
get_probs(tp);
report_pmi(topfile, pmifile, toppmifile, ddN.T, ddN.W, 1,
pmicount, tp, tpmi);
free(toppmifile);
free(pmifile);
free(tp);
}
/*
* now report words and diagnostics
*/
//ttop_open(topfile);
if ( fullreport ) {
rp = fopen(repfile,"w");
if ( !rp )
yap_sysquit("Cannot open file '%s' for write\n", repfile);
fprintf(rp, "#topic index rank prop word-sparse doc-sparse eff-words eff-docs docs-bound top-one "
"dist-unif dist-unigrm");
if ( PCTL_BURSTY() )
fprintf(rp, " burst-concent");
if ( ddN.tokens )
fprintf(rp, " ave-length");
fprintf(rp, " coher");
if ( pmicount )
fprintf(rp, " pmi");
fprintf(rp, "\n#word topic index rank");
if ( ddS.Nwt )
fprintf(rp, " count");
fprintf(rp, " prop cumm df coher\n");
}
for (k=0; k<ddN.T; k++) {
int cnt, termcnt = 0;
int kk = psort[k];
uint32_t **dfmtx;
if ( ddP.phi==NULL && ddS.NWt[kk]==0 )
continue;
/*
* grab word prob vec for later use
*/
if ( ddS.Nwt ) {
int w;
for (w=0; w<ddN.W; w++)
pvec[w] = wordprob(w,kk);
} else if ( ddP.phi )
fv_copy(pvec, ddP.phi[kk], ddN.W);
else if ( ddS.phi )
fv_copy(pvec, ddS.phi[kk], ddN.W);
/*
* rebuild word list
*/
tscorek = kk;
cnt = buildindk(kk, indk);
topk(topword, cnt, indk, tscore);
if ( topword<cnt )
cnt = topword;
assert(cnt>0);
if ( termstats ) {
termcnt = buildtermindk(kk, termindk, termstats);
topk(topword, termcnt, termindk, termtscore);
if ( topword<termcnt )
termcnt = topword;
}
/*
* df stats for topic returned as matrix
*/
dfmtx = hca_dfmtx(indk, cnt, kk);
if ( ddS.Nwt && (ddS.NWt[kk]*ddN.T*100<Nk_tot || ddS.NWt[kk]<5 ))
underused++;
/*
* print stats for topic
* Mallet: tokens, doc_ent, ave-word-len, coher.,
* uni-dist, corp-dist, eff-no-words
*/
yap_message("Topic %d/%d", kk, k);
{
/*
* compute diagnostics
*/
double prop = gtvec[kk];
float *dprop = docprop(kk);
double spw = 0;
double spd = ((double)nonzero_Ndt(kk))/((double)ddN.DT);
#ifdef KL
double ew = fv_kl(dfvec,pvec,ddN.W);
#else
double ew = exp(fv_entropy(pvec,ddN.W));
#endif
double ud = fv_helldistunif(pvec,ddN.W);
double pd = fv_helldist(pvec,gpvec,ddN.W);
double sl = fv_avestrlen(pvec,ddN.tokens,ddN.W);
double co = coherence(dfmtx, cnt);
double ed = dprop?exp(fv_entropy(dprop,ddN.DT)):ddN.DT;
#define MALLET_EW
#ifdef MALLET_EW
double ewp = dprop?(1.0/fv_expprob(pvec,ddN.W)):ddN.W;
#endif
double da = dprop?fv_bound(dprop,ddN.DT,1.0/sqrt((double)ddN.T)):0;
sparsitydoc += spd;
yap_message((ddN.T>200)?" p=%.3lf%%":" p=%.2lf%%",100*prop);
#ifdef XTRA
yap_message((ddN.T>200)?"/%.3lf%%":"/%.2lf%%",100*gtavec[kk]);
#endif
if ( ddS.Nwt ) {
spw = ((double)nonzero_Nwt(kk))/((double)ddN.W);
sparsityword += spw;
yap_message(" ws=%.1lf%%", 100*(1-spw));
}
yap_message(" ds=%.1lf%%", 100*(1-spd) );
#ifdef KL
yap_message(" ew=%lf", ew);
#else
yap_message(" ew=%.0lf", ew);
#endif
#ifdef MALLET_EW
yap_message(" ewp=%.1lf", ewp);
#endif
yap_message(" ed=%.1lf", ed);
yap_message(" da=%.0lf", da+0.1);
yap_message(" t1=%u", top1cnt[kk]);
yap_message(" ud=%.3lf", ud);
yap_message(" pd=%.3lf", pd);
if ( PCTL_BURSTY() )
yap_message(" bd=%.3lf", ddP.bdk[kk]);
if ( ddP.NGbeta ) {
/*
* approx. as sqrt(var(lambda_k)/lambda-normaliser
*/
double ngvar = sqrt(ddP.NGalpha[kk])
* (ngalpha[kk]/ddP.NGalpha[kk]);
yap_message(" ng=%.4lf,%.4lf",
ngalpha[kk], ngvar/ngalpha[kk]);
if ( ddS.sparse )
yap_message(",%.4f", 1-((float)ddS.sparseD[kk])/ddN.DTused);
if ( verbose>2 )
yap_message(" ngl=%.4lf,%.4lf, nga=%.4lf,%.4lf",
ddP.NGalpha[kk]/ddP.NGbeta[kk],
sqrt(ddP.NGalpha[kk]/ddP.NGbeta[kk]/ddP.NGbeta[kk]),
ddP.NGalpha[kk], ddP.NGbeta[kk]);
}
if ( ddN.tokens )
yap_message(" sl=%.2lf", sl);
yap_message(" co=%.3lf%%", co);
if ( pmicount )
yap_message(" pmi=%.3f", tpmi[kk]);
if ( fullreport ) {
fprintf(rp,"topic %d %d", kk, k);
fprintf(rp," %.6lf", prop);
if ( ddS.Nwt ) {
fprintf(rp," %.6lf", (1-spw));
} else {
fprintf(rp," 0");
}
fprintf(rp," %.6lf", (1-spd) );
#ifdef KL
yap_message(" %lf", ew);
#else
fprintf(rp," %.2lf", ew);
#endif
#ifdef MALLET_EW
fprintf(rp," %.2lf", ewp);
#endif
fprintf(rp," %.2lf", ed);
fprintf(rp," %.0lf", da+0.1);
fprintf(rp," %u", top1cnt[kk]);
fprintf(rp," %.6lf", ud);
fprintf(rp," %.6lf", pd);
if ( PCTL_BURSTY() )
fprintf(rp," %.3lf", ddP.bdk[kk]);
fprintf(rp," %.4lf", (ddN.tokens)?sl:0);
fprintf(rp," %.6lf", co);
if ( pmicount )
fprintf(rp," %.4f", tpmi[kk]);
fprintf(rp,"\n");
}
if ( dprop) free(dprop);
}
if ( verbose>1 ) {
double pcumm = 0;
/*
* print top words:
* Mallet: rank, count, prob, cumm, docs, coh
*/
yap_message("\ntopic %d/%d", kk, k);
yap_message(" words=");
for (w=0; w<cnt; w++) {
if ( w>0 ) yap_message(",");
if ( ddN.tokens )
yap_message("%s", ddN.tokens[indk[w]]);
else
yap_message("%d", indk[w]);
if ( verbose>2 ) {
if ( scoretype == ST_count )
yap_message("(%d)", (int)(tscore(indk[w])+0.2));
else
yap_message("(%6lf)", tscore(indk[w]));
}
if ( fullreport ) {
fprintf(rp, "word %d %d %d", kk, indk[w], w);
if ( ddS.Nwt )
fprintf(rp, " %d", ddS.Nwt[indk[w]][kk]);
pcumm += pvec[indk[w]];
fprintf(rp, " %.6f %.6f", pvec[indk[w]], pcumm);
fprintf(rp, " %d", dfmtx[w][w]);
fprintf(rp, " %.6f", coherence_word(dfmtx, cnt, w));
if ( ddN.tokens )
fprintf(rp, " %s", ddN.tokens[indk[w]]);
fprintf(rp, "\n");
}
}
if ( termstats ) {
yap_message(" terms=");
for (w=0; w<termcnt; w++) {
if ( w>0 ) yap_message(",");
if ( ddN.tokens )
yap_message("%s", termstats->tokens[termindk[w]]);
else
yap_message("%d", termstats->Kmin+termindk[w]);
if ( verbose>2 ) {
if ( scoretype == ST_count )
yap_message("(%d)", (int)(termtscore(termindk[w])+0.2));
else
yap_message("(%6lf)", termtscore(termindk[w]));
}
if ( fullreport ) {
fprintf(rp, "term %d %d %d", kk, termindk[w], w);
fprintf(rp, " %d", termstats->Nkt[termindk[w]][kk]);
fprintf(rp, " %s", termstats->tokens[termindk[w]]);
fprintf(rp, "\n");
}
}
}
}
yap_message("\n");
free(dfmtx[0]); free(dfmtx);
}
if ( verbose>1 && ddP.PYbeta ) {
int cnt;
double pcumm = 0;
/*
* print root words
*/
tscorek = -1;
cnt = buildindk(-1,indk);
/* this case gives bad results */
// if ( scoretype == ST_phirat ) topk(topword, cnt, indk, phiratioscore);
topk(topword, cnt, indk, (ddP.phi==NULL)?countscore:phiscore);
/*
* cannot build df mtx for root because
* it is latent w.r.t. topics
*/
yap_message("Topic root words=");
if ( fullreport ) {
int w;
if ( ddP.phi && ddP.PYbeta!=H_PDP ) {
for (w=0; w<ddN.W; w++)
pvec[w] = ddS.phi[ddN.T][w];
} else {
for (w=0; w<ddN.W; w++)
pvec[w] = betabasewordprob(w);
}
#ifdef KL
double ew = fv_kl(dfvec,pvec,ddN.W);
#else
double ew = exp(fv_entropy(pvec,ddN.W));
#endif
double ud = fv_helldistunif(pvec,ddN.W);
double pd = fv_helldist(pvec,gpvec,ddN.W);
fprintf(rp,"topic -1 -1 0 0");
fprintf(rp," %.4lf", ew);
fprintf(rp," %.6lf", ud);
fprintf(rp," %.6lf", pd);
fprintf(rp,"\n");
}
for (w=0; w<topword && w<cnt; w++) {
if ( w>0 ) yap_message(",");
if ( ddN.tokens )
yap_message("%s", ddN.tokens[indk[w]]);
else
yap_message("%d", indk[w]);
if ( verbose>2 && !ddP.phi )
yap_message("(%6lf)", countscore(indk[w]));
if ( fullreport ) {
fprintf(rp, "word %d %d %d", -1, indk[w], w);
if ( ddS.TwT )
fprintf(rp, " %d", ddS.TwT[w]);
pcumm += pvec[indk[w]];
fprintf(rp, " %.6f %.6f", pvec[indk[w]], pcumm);
fprintf(rp, " 0 0");
if ( ddN.tokens )
fprintf(rp, " %s", ddN.tokens[indk[w]]);
fprintf(rp, "\n");
}
}
yap_message("\nTopical words=");
topk(topword, cnt, indk, phiinvratioscore);
for (w=0; w<topword && w<cnt; w++) {
if ( w>0 ) yap_message(",");
if ( ddN.tokens )
yap_message("%s", ddN.tokens[indk[w]]);
else
yap_message("%d", indk[w]);
}
yap_message("\n");
}
yap_message("\n");
if ( rp )
fclose(rp);
if ( ddS.Nwt )
yap_message("Average topicXword sparsity = %.2lf%%\n",
100*(1-sparsityword/ddN.T) );
yap_message("Average docXtopic sparsity = %.2lf%%\n"
"Underused topics = %.1lf%%\n",
100*(1-sparsitydoc/ddN.T),
100.0*underused/(double)ddN.T);
if ( ddS.sparse && ddP.PYalpha==H_NG ) {
double avesp = 0;
// correct_docsp();
for (k=0; k<ddN.T; k++) {
avesp += gtvec[k];
}
// check gtvec[] sums to 1
assert(fabs(avesp-1.0)<0.00001);
avesp = 0;
for (k=0; k<ddN.T; k++) {
avesp += gtvec[k]*((float)ddS.sparseD[k])/ddN.DTused;
assert(ddS.sparseD[k]<=ddN.DTused);
}
assert(avesp<=1.0);
assert(avesp>0.0);
yap_message("IBP sparsity = %.2lf%%\n", 100*(1-avesp));
}
if ( pmicount )
yap_message("Average PMI = %.3f\n", tpmi[ddN.T]);
/*
* print
*/
if ( 1 ) {
float **cmtx = hca_topmtx();
int t1, t2;
int m1, m2;
float mval;
char *corfile = yap_makename(resstem,".topcor");
fp = fopen(corfile,"w");
if ( !fp )
yap_sysquit("Cannot open file '%s' for write\n", corfile);
/*
* print file
*/
for (t1=0; t1<ddN.T; t1++) {
for (t2=0; t2<t1; t2++)
if ( cmtx[t1][t2]>1.0e-7 )
fprintf(fp, "%d %d %0.6f\n", t1, t2, cmtx[t1][t2]);
}
fclose(fp);
free(corfile);
/*
* display maximum
*/
m1 = 1; m2 = 0;
mval = cmtx[1][0];
for (t1=0; t1<ddN.T; t1++) {
for (t2=0; t2<t1; t2++) {
if ( mval<cmtx[t1][t2] ) {
mval = cmtx[t1][t2];
m1 = t1;
m2 = t2;
}
}
}
yap_message("Maximum correlated topics (%d,%d) = %f\n", m1, m2, mval);
free(cmtx[0]); free(cmtx);
}
/*
* print burstiness report
*/
if ( PCTL_BURSTY() ) {
int tottbl = 0;
int totmlttbl = 0;
int totmlt = 0;
int i;
for (i=0; i<ddN.NT; i++) {
if ( Z_issetr(ddS.z[i]) ) {
if ( M_multi(i) )
totmlttbl++;
tottbl++;
}
if ( M_multi(i) )
totmlt++;
}
yap_message("Burst report: multis=%.2lf%%, tables=%.2lf%%, tbls-in-multis=%.2lf%%\n",
100.0*((double)ddM.dim_multiind)/ddN.N,
100.0*((double)tottbl)/ddN.NT,
100.0*((double)totmlttbl)/totmlt);
}
yap_message("\n");
free(topfile);
if ( repfile ) free(repfile);
if ( top1cnt ) free(top1cnt);
free(indk);
free(psort);
if ( ngalpha )
free(ngalpha);
if ( pmicount )
free(tpmi);
if ( NwK ) {
free(NwK);
NwK = NULL;
}
#ifdef KL
free(dfvec);
#endif
free(pvec);
free(gtvec);
free(gpvec);
tstats_free(termstats);
}
static void merge_opt_Tdt(int k1, int k2, merge_alpha_t *M) {
int d;
struct heap_s up;
struct heap_s down;
/*
* sorting on moves,
* stores doc index for
*/
uint32_t *Tdt_up;
uint32_t *Tdt_down;
/*
* change from incr/decr this docs Tdt
*/
float *score_up;
float *score_down;
Tdt_up = u32vec(ddN.DT);
Tdt_down = u32vec(ddN.DT);
score_up = fvec(ddN.DT);
score_down = fvec(ddN.DT);
if ( !score_down || !score_up || !Tdt_up || !Tdt_down )
yap_quit("Out of memory in likemerge()\n");
/*
* initialise sort
*/
for (d=0; d<ddN.DT; d++) {
assert(M->Tdt[d]<=M->Ndt[d]);
/* don't change for some docs */
Tdt_up[d] = d;
Tdt_down[d] = d;
if ( M->Tdt[d]<M->Ndt[d] )
score_up[d] = (ddP.bpar + ddP.apar*M->TdT[d])
* S_V(ddC.SX,M->Ndt[d],M->Tdt[d]+1);
else
score_up[d] = 0;
if ( M->Tdt[d]>1 )
score_down[d] = 1.0 / S_V(ddC.SX,M->Ndt[d],M->Tdt[d])
/(ddP.bpar + ddP.apar*(M->TdT[d]-1));
else
score_down[d] = 0;
assert((M->Tdt[d]>1)||score_down[d]==0);
assert((M->Tdt[d]<M->Ndt[d])||score_up[d]==0);
assert(M->Tdt[d]<=M->Ndt[d]);
}
assert(M->TDt>0);
/*
* use a heap, so only top of heap is least
*/
heap_init(&up, Tdt_up, ddN.DT, fveccmp, (void *)score_up);
heap_init(&down, Tdt_down, ddN.DT, fveccmp, (void *)score_down);
while ( 1 ) {
float upv;
float downv;
upv = merge_alphabasetopicprob(M->TDTm+M->TDt, M->TDt, k1)
*score_up[heap_front(&up)];
if ( M->TDt>1 )
downv = score_down[heap_front(&down)]
/ merge_alphabasetopicprob(M->TDTm+M->TDt-1, M->TDt-1, k1);
else
downv = 0.0;
if ( downv>upv && downv>1.0 ){
// decrement this
d = heap_front(&down);
M->TdT[d]--;
M->Tdt[d]--;
assert(M->Tdt[d]>0);
M->TDt--;
heap_pop(&down);
heap_remove(&up,d);
} else if ( downv<upv && upv>1.0 ){
// increment this
d = heap_front(&up);
M->TdT[d]++;
M->Tdt[d]++;
assert(M->Tdt[d]<=M->Ndt[d]);
M->TDt++;
heap_pop(&up);
heap_remove(&down,d);
} else {
// none are better so quit
break;
}
if ( M->Tdt[d]<M->Ndt[d] )
score_up[d] = (ddP.bpar + ddP.apar*M->TdT[d])
* S_V(ddC.SX,M->Ndt[d],M->Tdt[d]+1);
else
score_up[d] = 0;
if ( M->Tdt[d]>1 )
score_down[d] = 1.0 / S_V(ddC.SX,M->Ndt[d],M->Tdt[d])
/(ddP.bpar + ddP.apar*(M->TdT[d]-1));
else
score_down[d] = 0;
assert(M->Tdt[d]>1||score_down[d]==0);
assert(M->Tdt[d]<M->Ndt[d] ||score_up[d]==0);
assert(M->Tdt[d]<=M->Ndt[d]);
/*
* now adjust the two heaps for new vals for [d]
*/
heap_push(&down,d);
heap_push(&up,d);
}
free(score_up);
free(score_down);
heap_free(&up);
heap_free(&down);
}
int myarmsMH(double xl, double xr,
double (*myfunc)(double x, void *mydata),
void *mydata, double *xval, char *label,
int doMH) {
double result = *xval;
double *resvec = NULL;
double startval = *xval;
int errcode;
if ( fabs(*xval-xr)/(xr)<0.00001 ) {
*xval = xr * 0.999 + xl * 0.001;
}
if ( fabs(*xval-xl)/xl<0.00001 ) {
*xval = xl * 0.999 + xr * 0.001;
}
myarms_evals = 0;
if ( doMH ) {
resvec = malloc(sizeof(resvec[0])*NSAMP);
errcode = myarms_simple(6, &xl, &xr, myfunc, mydata, doMH, xval,
resvec, NSAMP);
result = resvec[NSAMP-1];
free(resvec);
} else
errcode = myarms_simple(6, &xl, &xr, myfunc, mydata, 0, xval, &result, 1);
/*
* 1007, 1003 is out of bounds
*/
if ( errcode && errcode!=1007 && errcode!=1003 && errcode!=2001
&& (errcode!=2000 || startval!=result ) ) {
yap_quit(" myarmsMH(%lf,%s)->%d = %lf,%lf%s->%lf, w %d calls, quitting\n",
startval, label, errcode,
myarms_last, result, (!ISFINITE(result))?"(inf)":"",
*xval, myarms_evals);
}
if ( errcode==1007 || errcode==1003 ) {
/*
* hit bounds, for safety use start val
*/
yap_message(" error myarmsMH(%lf,%s)->%d = %lf,%lf%s->%lf, w %d calls, quitting\n",
startval, label, errcode,
myarms_last, result, (!ISFINITE(result))?"(inf)":"",
*xval, myarms_evals);
result = startval;
}
if ( errcode==2001 ) {
/*
* too many calls in Metrop-Hastings
*/
yap_message(" error myarmsMH(%lf,%s)->%d = %lf,%lf%s->%lf, w %d calls, quitting\n",
startval, label, errcode,
myarms_last, result, (!ISFINITE(result))?"(inf)":"",
*xval, myarms_evals);
}
/*
* note, sometimes the value is returned
* unchanged .... seems to be when the
* posterior is really focussed
*/
if ( verbose>1 || !ISFINITE(result) || startval==result )
yap_message(" myarmsMH(%s) = %lf%s<-%lf, w %d calls %s\n",
label,
result, (!ISFINITE(result))?"(inf)":"",
*xval, myarms_evals,
(startval==result)?"UNCHANGED":"");
if ( ISFINITE(result) ) {
if ( result<xl )
result = xl;
else if ( result>xr )
result = xr;
*xval = result;
}
if ( !ISFINITE(result) ) {
return 1;
}
return 0;
}
void hca_displaytopics(char *resstem, int topword, enum ScoreType scoretype) {
int w,k;
int *indk = NULL;
int Nk_tot = 0;
double (*tscore)(int) = NULL;
double sparsityword = 0;
double sparsitydoc = 0;
double underused = 0;
char *fname = yap_makename(resstem,".top");
int nophi = (ddP.phi==NULL) && (ddS.phi==NULL);
FILE *fp;
if ( scoretype == ST_idf ) {
tscore = idfscore;
} else if ( scoretype == ST_phi ) {
tscore = phiscore;
} else if ( scoretype == ST_count ) {
tscore = countscore;
} else if ( scoretype == ST_cost ) {
tscore = costscore;
} else if ( scoretype == ST_Q ) {
tscore = Qscore;
lowerQ = 1.0/ddN.T;
}
fp = fopen(fname,"w");
if ( !fp )
yap_sysquit("Cannot open file '%s' for write\n", fname);
/*
* first collect counts of each word/term
*/
if ( scoretype != ST_count && scoretype != ST_phi ) {
NwK = u32vec(ddN.W);
if ( !NwK )
yap_quit("Out of memory in hca_displaytopics()\n");
for (w=0; w<ddN.W; w++) {
NwK[w] = 0;
}
NWK = 0;
for (w=0; w<ddN.W; w++) {
for (k=0; k<ddN.T; k++) {
NwK[w] += ddS.Nwt[w][k]; // should use CCT_ReadN()
}
NWK += NwK[w];
}
}
assert(ddN.tokens);
for (k=0; k<ddN.T; k++) {
Nk_tot += ddS.NWt[k];
}
indk = malloc(sizeof(*indk)*ddN.W);
if ( !indk )
yap_quit("Cannot allocate indk\n");
for (k=0; k<ddN.T; k++) {
int cnt;
double spw;
double spd;
tscorek = k;
/*
* print top words
*/
cnt=0;
if ( ddP.phi==NULL ) {
for (w=0; w<ddN.W; w++) {
if ( ddS.Nwt[w][k]>0 ) indk[cnt++] = w;
}
} else {
float **phi;
if ( ddP.phi )
phi = ddP.phi;
else
phi = ddS.phi;
for (w=0; w<ddN.W; w++) {
if ( phi[k][w]>0.5/ddN.W ) indk[cnt++] = w;
}
}
topk(topword, cnt, indk, tscore);
spd = ((double)nonzero_Ndt(k))/((double)ddN.DT);
sparsitydoc += spd;
if ( nophi ) {
spw = ((double)nonzero_Nwt(k))/((double)ddN.W);
sparsityword += spw;
}
if ( ddS.NWt[k]*ddN.T*100<Nk_tot )
underused++;
yap_message("\nTopic %d (", k);
if ( ddP.phi==NULL )
yap_message((ddN.T>200)?"p=%.3lf%%,":"p=%.2lf%%,",
100*((double)ddS.NWt[k])/(double)Nk_tot);
if ( nophi )
yap_message("ws=%.1lf%%,", 100*(1-spw));
else
yap_message("#=%.0lf,", exp(phi_entropy(k)));
yap_message("ds=%.1lf%%", 100*(1-spd) );
fprintf(fp,"%d: ", k);
yap_message(") words =");
for (w=0; w<topword && w<cnt; w++) {
fprintf(fp," %d", (int)indk[w]);
if ( verbose>2 ) {
double score = tscore(indk[w]);
yap_message(",%s(%6lf)", ddN.tokens[indk[w]], score);
} else
yap_message(",%s", ddN.tokens[indk[w]]);
}
yap_message("\n");
fprintf(fp, "\n");
}
if ( ddP.PYbeta && nophi ) {
int cnt;
/*
* print root words
*/
tscorek = -1;
cnt=0;
for (w=0; w<ddN.W; w++) {
if ( ddS.TwT[w]>0 ) indk[cnt++] = w;
}
topk(topword, cnt, indk, tscore);
yap_message("\nTopic root words =");
fprintf(fp,"-1:");
for (w=0; w<topword && w<cnt; w++) {
fprintf(fp," %d", (int)indk[w]);
if ( verbose>2 ) {
double score = tscore(indk[w]);
yap_message(",%s(%6lf)", ddN.tokens[indk[w]], score);
} else
yap_message(",%s", ddN.tokens[indk[w]]);
}
yap_message("\n");
fprintf(fp, "\n");
}
if ( nophi )
yap_message("Average topicXword sparsity = %.2lf%%, ",
100*(1-sparsityword/ddN.T) );
yap_message("Average docXtopic sparsity = %.2lf%%, "
"underused topics = %.1lf%%\n",
100*(1-sparsitydoc/ddN.T),
100.0*underused/(double)ddN.T);
if ( ddP.bdk!=NULL) {
int tottbl = 0;
int totmlttbl = 0;
int totmlt = 0;
int i;
for (i=0; i<ddN.NT; i++) {
if ( Z_issetr(ddS.z[i]) ) {
if ( M_multi(i) )
totmlttbl++;
tottbl++;
}
if ( M_multi(i) )
totmlt++;
}
yap_message("doc PYP report: multis=%.2lf%%, tables=%.2lf%%, tbls-in-multis=%.2lf%%\n",
100.0*((double)ddM.dim_multiind)/ddN.N,
100.0*((double)tottbl)/ddN.NT,
100.0*((double)totmlttbl)/totmlt);
}
fclose(fp);
free(fname);
free(indk);
if ( scoretype != ST_count ) {
free(NwK);
NwK = NULL;
}
}
/*==========================================
* main
*========================================== */
int main(int argc, char* argv[])
{
int c, iter, ITER=0, seed=0;
enum dataType data = LdaC;
enum dataType testdata = LdaC;
int dots = 0;
enum GibbsType fix_hold = GibbsNone;
char *stem;
char *resstem;
int topwords = 20;
int noerrorlog = 0;
int displayed = 0;
int load_vocab = 0;
int checkpoint = 0;
int restart = 0;
int dopmi = 0;
int restart_hca = 0;
int load_phi = 0;
int load_mu = 0;
int procs = 1;
int maxW = 0;
enum ScoreType score=ST_idf;
double BM0val=0, BM1val =0, BP0val=0, BP1val=0;
clock_t t1=0, t2=0, t3=0;
double tot_time = 0;
double psample_time = 0;
enum ParType par;
/*
* default values
*/
ddN.T = 10;
ITER = 100;
ddN.TEST = 0;
pctl_init();
while ( (c=getopt(argc, argv,"b:c:C:d:ef:F:g:G:h:K:l:L:N:o:pq:vr:s:S:t:T:vVW:"))>=0 ) {
switch ( c ) {
case 'b':
if ( !optarg || sscanf(optarg,"%d",&ddP.back)!=1 )
yap_quit("Need a valid 'b' argument\n");
break;
case 'c':
if ( !optarg || sscanf(optarg,"%d",&checkpoint)!=1 )
yap_quit("Need a valid 'c' argument\n");
break;
case 'C':
if ( !optarg || sscanf(optarg,"%d",&ITER)!=1 )
yap_quit("Need a valid 'C' argument\n");
break;
case 'd':
if ( !optarg || sscanf(optarg,"%d",&dots)!=1 )
yap_quit("Need a valid 'd' argument\n");
break;
case 'e':
noerrorlog++;
break;
case 'f':
if ( strcmp(optarg,"witdit")==0 )
data = WitDit;
else if ( strcmp(optarg,"docword")==0 )
data = Docword;
else if ( strcmp(optarg,"ldac")==0 )
data = LdaC;
else if ( strcmp(optarg,"bag")==0 )
data = TxtBag;
else if ( strcmp(optarg,"lst")==0 )
data = SeqTxtBag;
else
yap_quit("Illegal data type for -f\n");
break;
case 'F':
if ( strcmp(optarg,"all")==0 ) {
for (par=ParAM; par<=ParBB; par++)
ddT[par].fix = 1;
} else {
par = findpar(optarg);
if ( par==ParNone )
yap_quit("Illegal arg for -F\n");
ddT[par].fix = 1;
}
break;
case 'g':
{
char var[100];
int st=0;
if ( !optarg || sscanf(optarg,"%[^, ],%d", &var[0], &st)<1 )
yap_quit("Need a valid 'g' argument\n");
par = findpar(var);
if ( par==ParBP1 )
ddP.kbatch = st;
else
yap_quit("Illegal var for -g\n");
}
break;
case 'G':
{
char var[100];
int st=0, cy=0;
if ( !optarg || sscanf(optarg,"%[^, ],%d,%d",
&var[0], &cy, &st)<2 || st<0 || cy<0 )
yap_quit("Need a valid 'G' argument\n");
par = findpar(var);
if ( par==ParNone || par==ParB0P || par==ParB0M )
yap_quit("Illegal var for -G\n");
ddT[par].fix = 0;
ddT[par].start = st;
ddT[par].cycles = cy;
}
break;
case 'h':
{
fix_hold = GibbsHold;
if ( !optarg )
yap_quit("Need a valid 'h' argument\n");
if ( strncmp(optarg,"dict,",5)==0 ) {
if ( sscanf(&optarg[5],"%d",&ddP.hold_dict)<1 || ddP.hold_dict<2 )
yap_quit("Need a valid 'hdict' argument\n");
} else if ( strncmp(optarg,"fract,",6)==0 ) {
if ( sscanf(&optarg[6],"%lf",&ddP.hold_fraction)<1
|| ddP.hold_fraction<=0 || ddP.hold_fraction>=1 )
yap_quit("Need a valid 'hfract' argument\n");
} else if ( strncmp(optarg,"doc,",4)==0 ) {
if ( sscanf(&optarg[4],"%d",&ddP.hold_every)<1 || ddP.hold_every<2 )
yap_quit("Need a valid 'hdoc' argument\n");
} else
yap_quit("Need a valid 'h' argument\n");
}
break;
case 'K':
if ( !optarg || sscanf(optarg,"%d",&ddN.T)!=1 )
yap_quit("Need a valid 'K' argument\n");
break;
case 'l':
if ( !optarg )
yap_quit("Need a valid 'l ' argument\n");
if ( strncmp(optarg,"phi,",4)==0 ) {
if ( sscanf(&optarg[4],"%d,%d",&ddP.phiiter, &ddP.phiburn)<2 )
yap_quit("Need a valid 'l word,' argument\n");
} else if ( strncmp(optarg,"theta,",6)==0 ) {
if ( sscanf(&optarg[6],"%d,%d",&ddP.thetaiter, &ddP.thetaburn)<2 )
yap_quit("Need a valid 'l word,' argument\n");
} else if ( strncmp(optarg,"mu,",3)==0 ) {
if ( sscanf(&optarg[3],"%d,%d",&ddP.muiter, &ddP.muburn)<2 )
yap_quit("Need a valid 'l word,' argument\n");
} else if ( strncmp(optarg,"prog,",5)==0 ) {
if ( sscanf(&optarg[5],"%d,%d",&ddP.progiter, &ddP.progburn)<2 )
yap_quit("Need a valid 'l prog,' argument\n");
} else
yap_quit("Need a valid DIAG code in 'l' argument\n");
break;
case 'L':
if ( !optarg )
yap_quit("Need a valid 'L ' argument\n");
if ( strncmp(optarg,"like,",5)==0 ) {
if ( sscanf(&optarg[5],"%d,%d",&ddP.mltiter, &ddP.mltburn)<1 )
yap_quit("Need a valid 'L like' argument\n");
} else
yap_quit("Need a valid DIAG code in 'L' argument\n");
break;
case 'N':
if ( !optarg || sscanf(optarg,"%d,%d", &ddP.maxN, &ddP.maxM)<1 )
yap_quit("Need a valid 'N' argument\n");
break;
case 'o':
{
char *ptr = strchr(optarg, ',');
int len = strlen(optarg);
if ( ptr )
len = ptr - optarg;
if ( strncmp(optarg,"idf",len)==0 )
score = ST_idf;
else if ( strncmp(optarg,"count",len)==0 )
score = ST_count;
else if ( strncmp(optarg,"Q",len)==0 )
score = ST_Q;
else if ( strncmp(optarg,"cost",len)==0 )
score = ST_cost;
else
yap_quit("Need a valid parameter for 'o' argument\n");
if ( ptr ) {
/* there was a second arg */
if ( sscanf(ptr+1, "%d", &topwords) != 1)
yap_quit("Need a valid second 'o' argument\n");
}
break;
}
break;
case 'p':
dopmi++;
break;
case 'q':
if(!optarg || sscanf(optarg, "%d", &procs) != 1)
yap_quit("Need a valid 'q' argument\n");
break;
case 'r':
if(!optarg )
yap_quit("Need a valid 'r' argument\n");
if ( strcmp(optarg,"tca")==0 )
restart++;
else if ( strcmp(optarg,"hca")==0 )
restart_hca++;
else if ( strcmp(optarg,"phi")==0 )
load_phi++;
else if ( strcmp(optarg,"mu")==0 )
load_mu++;
else
yap_quit("Need a valid 'r' argument\n");
break;
case 's':
if ( !optarg || sscanf(optarg,"%d",&seed)!=1 )
yap_quit("Need a valid 's' argument\n");
break;
case 'S':
{
char var[100];
double vin=0;
if ( !optarg || sscanf(optarg,"%[^=, ]=%lf",
&var[0], &vin)<2 )
yap_quit("Need a valid 'S' argument\n");
par = findpar(var);
if ( par==ParNone )
yap_quit("Illegal var for -S\n");
else if ( par==ParBM0 )
BM0val = vin;
else if ( par==ParBM1 )
BM1val = vin;
else if ( par==ParBP0 )
BP0val = vin;
else if ( par==ParBP1 )
BP1val = vin;
else
*(ddT[par].ptr) = vin;
}
break;
case 't':
if ( !optarg || sscanf(optarg,"%d",&ddP.training)!=1 )
yap_quit("Need a valid 't' argument\n");
break;
case 'T':
if ( !optarg )
yap_quit("Need a valid 'T' argument\n");
{
char *tname = data_name(optarg,data);
FILE *fp = fopen(tname,"r");
if ( fp==NULL ) {
free(tname);
tname = data_name(optarg,testdata);
fp = fopen(tname,"r");
} else {
testdata = data;
}
free(tname);
if ( fp!=NULL ) {
/* its a valid test filename */
ddP.teststem = optarg;
fclose(fp);
} else if ( sscanf(optarg,"%d",&ddN.TEST)!=1 )
yap_quit("Need a valid 'T' argument\n");
}
break;
case 'v':
verbose++;
break;
case 'V':
load_vocab = 1;
break;
case 'W':
if ( !optarg || sscanf(optarg,"%d",&maxW)<1 )
yap_quit("Need a valid 'W' argument\n");
break;
default:
yap_quit("Unknown option '%c'\n", c);
}
}
if (argc-optind != 2) {
usage();
exit(-1);
}
if ( optind>=argc ) {
yap_quit("No arguments given\n");
}
stem = strdup(argv[optind++]);
resstem = strdup(argv[optind++]);
if ( dopmi )
load_vocab = 1;
if ( dopmi && verbose !=2 ) {
/*
* due to the use of the ".top" file
* its really multi-purpose
*/
yap_quit("When computing PMI verbose must be exactly 2\n");
}
if ( noerrorlog==0 ) {
char *wname = yap_makename(resstem, ".log");
yap_file(wname);
free(wname);
}
yap_commandline(argc, argv);
#ifdef H_THREADS
yap_message(" Threads,");
#endif
if ( restart || restart_hca ) {
char *fname = yap_makename(resstem,".par");
FILE *fp = fopen(fname,"r");
char *buf;
if ( !fp )
yap_quit("Parameter file '%s' doesn't exist\n", fname);
fclose(fp);
free(fname);
buf = readpar(resstem,"T",50);
if ( !buf )
yap_quit("Parameter file '%s' has no T\n", fname);
ddN.T = atoi(buf);
free(buf);
if ( restart ) {
buf = readpar(resstem,"E",50);
if ( !buf )
yap_quit("Parameter file '%s' has no E\n", fname);
ddN.E = atoi(buf);
free(buf);
pctl_read(resstem);
}
if ( maxW==0 ) {
buf = readpar(resstem,"W",50);
if ( buf ) {
maxW = atoi(buf);
free(buf);
}
}
if ( ddP.training==0 ) {
buf = readpar(resstem,"TRAIN",50);
if ( buf ) {
ddP.training = atoi(buf);
free(buf);
}
}
if ( ddN.TEST==0 ) {
buf = readpar(resstem,"TEST",50);
if ( buf ) {
ddN.TEST = atoi(buf);
free(buf);
}
}
}
assert(ddN.T>0);
assert(ddN.TEST>=0);
assert(restart || restart_hca || ITER>0);
if ( load_phi && ddP.phiiter>0 )
yap_quit("Options '-l phi,...' and '-r phi' incompatible\n");
if ( load_mu && ddP.muiter>0 )
yap_quit("Options '-l mu,...' and '-r mu' incompatible\n");
/*
* set random number generator
*/
if ( seed ) {
rng_seed(rngp,seed);
} else {
rng_time(rngp,&seed);
}
yap_message("Setting seed = %lu\n", seed);
/*
* read data and get dimensions
*/
{
D_bag_t *dbp = data_read(stem, data);
int training = pctl_training(dbp->D);
if ( ddP.teststem ) {
D_bag_t *dbpt = data_read(ddP.teststem, testdata);
/* need to load a separate test set, strip to bare training */
data_shrink(dbp, training);
ddN.TEST = dbpt->D;
data_append(dbp, dbpt);
free(dbpt->w); free(dbpt->d); free(dbpt);
}
if ( maxW>0 ) {
if ( dbp->W <= maxW )
dbp->W = maxW;
if ( dbp->W > maxW )
data_vocabshrink(dbp, maxW);
}
/*
* transfer into system
*/
ddN.D = dbp->D;
ddN.W = dbp->W;
ddN.N = dbp->N;
ddN.NT = dbp->N;
ddN.DT = training;
ddD.w = dbp->w;
ddD.d = dbp->d;
free(dbp);
if ( ddN.DT<ddN.D ) {
/* recompute NT */
int i;
for (i=0; i<ddN.N; i++)
if ( ddD.d[i]>=ddN.DT )
break;
ddN.NT = i;
}
}
data_read_epoch(stem);
/*
* at this point, dimensions are fixed, so load phi and mu if needed
*/
if ( load_phi )
pctl_loadphi(resstem);
if ( load_mu )
pctl_loadmu(resstem);
/*
* correct parameters after command line
*/
pctl_fix(ITER);
if ( BM0val>0 ) {
ddP.b_mu[0] = BM0val;
}
if ( BM1val>0 ) {
int i;
for (i=1; i<ddN.E; i++)
ddP.b_mu[i] = BM1val;
}
if ( BP0val>0 ) {
int i;
for (i=0; i<ddN.T; i++)
ddP.b_phi[0][i] = BP0val;
}
if ( BP1val>0 ) {
int i;
if ( ddN.E==1 )
yap_quit("b_phi[1] invalid when epochs==1\n");
for (i=0; i<ddN.T; i++)
ddP.b_phi[1][i] = BP1val;
}
pctl_samplereport();
/*
* all data structures
*/
data_alloc();
if ( ddP.phiiter>0 )
phi_init(resstem);
else
ddS.phi = NULL;
if ( ddP.muiter>0 )
mu_init(resstem);
else
ddS.mu = NULL;
if ( ddP.thetaiter>0 )
theta_init(resstem);
else
ddS.theta = NULL;
tca_alloc();
if ( PCTL_BURSTY() )
dmi_init(&ddM, ddS.z, ddD.w, ddD.N_dTcum,
ddN.T, ddN.N, ddN.W, ddN.D, ddN.DT,
(fix_hold==GibbsHold)?pctl_hold:NULL);
if ( load_vocab ) {
data_vocab(stem);
}
cache_init();
/*
* yap some details
*/
data_report(ITER, seed);
pctl_report();
/*
* load/init topic assignments and prepare statistics
*/
if ( restart || restart_hca) {
tca_read_z(resstem, 0, ddN.DT);
tca_rand_z(ddN.T, ddN.DT, ddN.D);
} else {
tca_rand_z(ddN.T, 0, ddN.D);
}
tca_reset_stats(resstem, restart, 0);
if ( (restart || restart_hca ) && ITER )
yap_message("Initial log_2(perp)=%lf\n", -M_LOG2E * likelihood()/ddN.NT);
if ( ITER )
yap_report("cycles: ");
for (iter=0; iter<ITER; iter++) {
int pro;
double thislp = 0;
int thisNd = 0;
int doc;
#ifdef H_THREADS
pthread_t thread[procs];
#endif
D_pargs_p parg[procs];
#ifdef MU_CACHE
mu_side_fact_reinit();
#endif
#ifdef PHI_CACHE
phi_cache_reinit();
#endif
t1 = clock();
/*
* sampling
*/
#ifdef IND_STATS
ddP.doc_ind_stats = u32tri(ddN.T,ddN.E,ddN.E);
ddP.word_ind_stats = u32tri(ddN.T,ddN.E,ddN.E);
#endif
/* a bit complex if no threads! */
doc = 0;
for (pro = 0 ; pro < procs ; pro++){
parg[pro].dots=dots;
parg[pro].procs=procs;
parg[pro].doc = &doc;
#ifndef H_THREADS
sampling_p(&parg[pro]);
#else
if ( procs==1 )
sampling_p(&parg[pro]);
else if( pthread_create(&thread[pro],NULL,sampling_p,(void*) &parg[pro]) != 0){
yap_message("thread failed %d\n",pro+1 );
}
#endif
}
#ifdef H_THREADS
if ( procs>1 ) {
//waiting for threads to finish
for (pro = 0; pro < procs; pro++){
pthread_join(thread[pro], NULL);
}
}
#endif
// getting lp, Nd and clock
for(pro = 0; pro < procs; pro++){
thislp += parg[pro].thislp;
thisNd += parg[pro].thisNd;
tot_time += parg[pro].tot_time;
}
#ifdef H_THREADS
if ( procs>1 )
tca_reset_stats(NULL,1,1);
#endif
/*
* full check
*/
#ifndef NDEBUG
{
int e, d;
check_cp_et();
for (e=0; e<ddN.E; e++)
check_m_vte(e);
for (d=0; d<ddN.DT; d++)
check_n_dt(d);
}
#endif
#ifdef IND_STATS
{
char *fname = yap_makename(resstem,".istats");
FILE *ifp = fopen(fname,"a");
int e1, e2, kk;
fprintf(ifp,"Iteration %d\n", iter);
for (kk=0; kk<ddN.T; kk++) {
fprintf(ifp," Topic %d\n", kk);
for (e1=0; e1<ddN.E; e1++) {
fprintf(ifp," Epoch %d\n ", e1);
for (e2=0; e2<ddN.E; e2++)
fprintf(ifp," %u", (unsigned)ddP.doc_ind_stats[kk][e1][e2]);
fprintf(ifp,"\n ");
for (e2=0; e2<ddN.E; e2++)
fprintf(ifp," %u", (unsigned)ddP.word_ind_stats[kk][e1][e2]);
fprintf(ifp,"\n");
}
}
fclose(ifp);
free(ddP.doc_ind_stats[0][0]); free(ddP.doc_ind_stats[0]);
free(ddP.doc_ind_stats);
free(ddP.word_ind_stats[0][0]); free(ddP.word_ind_stats[0]);
free(ddP.word_ind_stats);
free(fname);
}
#endif
/*
* sample hyperparameters
*/
t3 = clock();
pctl_sample(iter, procs);
/*
* do time calcs here to remove diagnostics+reporting
*/
t2 = clock();
tot_time += (double)(t2 - t1) / CLOCKS_PER_SEC;
psample_time += (double)(t2 - t3) / CLOCKS_PER_SEC;
/*
* progress reports
*/
if ( ( iter>ddP.progburn && (iter%ddP.progiter)==0 ) || iter+1>=ITER ) {
yap_message(" %d\nlog_2(perp)=%lf,%lf", iter,
-M_LOG2E * likelihood()/ddN.NT, -M_LOG2E * thislp/thisNd);
pctl_update(iter);
if ( verbose && iter%10==0 )
yap_probs();
if ( iter>0 && verbose>1 ) {
if ( ddN.tokens ) {
tca_displaytopics(resstem,topwords,score);
displayed++;
}
}
if ( iter+1<ITER ) {
// yap_message("\n");
yap_report("cycles: ");
}
} else {
yap_message(" %d", iter);
if ( verbose>1) yap_message("\n");
}
if ( checkpoint>0 && iter>0 && iter%checkpoint==0 ) {
data_checkpoint(resstem, stem, iter+1);
yap_message(" checkpointed\n");
tca_report(resstem, stem, ITER, procs, fix_hold,
(dopmi&&displayed>0)?1:0);
}
if ( ddP.phiiter>0 && iter>ddP.phiburn && (iter%ddP.phiiter)==0 )
phi_update();
if ( ddP.thetaiter>0 && iter>ddP.thetaburn && (iter%ddP.thetaiter)==0 )
theta_update();
if ( ddP.muiter>0 && iter>ddP.muburn && (iter%ddP.muiter)==0 )
mu_update();
} // over iter
if ( ITER )
yap_report("Finished after %d cycles on average of %lf+%lf(s) per cycle\n",
iter, (tot_time-psample_time)/iter, psample_time/iter);
if ( ( verbose==1 || ((iter+1)%5!=0 && verbose>1) ) ) {
if ( ddN.tokens ) {
tca_displaytopics(resstem,topwords,score);
displayed++;
}
}
yap_probs();
if ( ITER>0 )
data_checkpoint(resstem, stem, ITER);
tca_report(resstem, stem, ITER, procs, fix_hold, (dopmi&&displayed>0)?1:0);
if ( ddP.phiiter>0 )
phi_save(resstem);
if ( ddP.thetaiter>0 )
theta_save(resstem);
if ( ddP.muiter>0 )
mu_save(resstem);
/*
* free
*/
phi_free();
theta_free();
mu_free();
cache_free();
pctl_free();
data_free();
dmi_free(&ddM);
tca_free();
free(stem);
free(resstem);
rng_free(rngp);
return 0;
}
void hca_displaytopics(char *stem, char *resstem, int topword,
enum ScoreType scoretype, int pmicount, int fullreport) {
int w,k;
uint32_t *indk = NULL;
int Nk_tot = 0;
double (*tscore)(int) = NULL;
double sparsityword = 0;
double sparsitydoc = 0;
double underused = 0;
uint32_t *top1cnt = NULL;
FILE *fp;
float *tpmi = NULL;
char *topfile;
char *repfile;
uint32_t *psort;
FILE *rp = NULL;
float *gtvec = globalprop();
float *gpvec = calloc(ddN.W,sizeof(gpvec[0]));
float *pvec = calloc(ddN.W,sizeof(pvec[0]));
if ( pmicount>topword )
pmicount = topword;
if ( scoretype == ST_idf ) {
tscore = idfscore;
} else if ( scoretype == ST_phi ) {
tscore = phiscore;
} else if ( scoretype == ST_count ) {
tscore = countscore;
} else if ( scoretype == ST_cost ) {
tscore = costscore;
} else if ( scoretype == ST_Q ) {
tscore = Qscore;
lowerQ = 1.0/ddN.T;
}
/*
* first collect counts of each word/term,
* and build gpvec (mean word probs)
*/
build_NwK();
{
/*
* gpvec[] is normalised NwK[]
*/
double tot = 0;
for (w=0; w<ddN.W; w++)
tot += gpvec[w] = NwK[w]+0.1;
for (w=0; w<ddN.W; w++)
gpvec[w] /= tot;
}
if ( ddS.Nwt ) {
for (k=0; k<ddN.T; k++) {
Nk_tot += ddS.NWt[k];
}
}
psort = sorttops(gtvec, ddN.T);
top1cnt = hca_top1cnt();
if ( !top1cnt )
yap_quit("Cannot allocate top1cnt in hca_displaytopics()\n");
if ( pmicount ) {
tpmi = malloc(sizeof(*tpmi)*(ddN.T+1));
if ( !tpmi )
yap_quit("Cannot allocate tpmi in hca_displaytopics()\n");
}
indk = malloc(sizeof(*indk)*ddN.W);
if ( !indk )
yap_quit("Cannot allocate indk in hca_displaytopics()\n");
/*
* two passes through,
* first to build the top words and dump to file
*/
repfile = yap_makename(resstem,".topset");
topfile = yap_makename(resstem,".toplst");
fp = fopen(topfile,"w");
if ( !fp )
yap_sysquit("Cannot open file '%s' for write\n", topfile);
yap_message("\n");
for (k=0; k<ddN.T; k++) {
int cnt;
tscorek = k;
/*
* build sorted word list
*/
cnt = buildindk(k, indk);
topk(topword, cnt, indk, tscore);
if ( cnt==0 )
continue;
/*
* dump words to file
*/
fprintf(fp,"%d: ", k);
for (w=0; w<topword && w<cnt; w++) {
fprintf(fp," %d", (int)indk[w]);
}
fprintf(fp, "\n");
}
if ( ddP.PYbeta && (ddP.phi==NULL || ddP.betapr) ) {
int cnt;
/*
* dump root words
*/
tscorek = -1;
cnt = buildindk(-1, indk);
topk(topword, cnt, indk, (ddP.phi==NULL)?countscore:phiscore);
fprintf(fp,"-1:");
for (w=0; w<topword && w<cnt; w++) {
fprintf(fp," %d", (int)indk[w]);
}
fprintf(fp, "\n");
}
fclose(fp);
if ( verbose>1 ) yap_message("\n");
if ( pmicount ) {
/*
* compute PMI
*/
char *toppmifile;
char *pmifile;
double *tp;
tp = dvec(ddN.T);
pmifile=yap_makename(stem,".pmi");
toppmifile=yap_makename(resstem,".toppmi");
get_probs(tp);
report_pmi(topfile, pmifile, toppmifile, ddN.T, ddN.W, 1,
pmicount, tp, tpmi);
free(toppmifile);
free(pmifile);
free(tp);
}
/*
* now report words and diagnostics
*/
//ttop_open(topfile);
if ( fullreport ) {
rp = fopen(repfile,"w");
if ( !rp )
yap_sysquit("Cannot open file '%s' for write\n", repfile);
fprintf(rp, "#topic index rank prop word-sparse doc-sparse eff-words eff-docs docs-bound top-one "
"dist-unif dist-unigrm");
if ( PCTL_BURSTY() )
fprintf(rp, " burst-concent");
if ( ddN.tokens )
fprintf(rp, " ave-length");
fprintf(rp, " coher");
if ( pmicount )
fprintf(rp, " pmi");
fprintf(rp, "\n#word topic index rank");
if ( ddS.Nwt )
fprintf(rp, " count");
fprintf(rp, " prop cumm df coher\n");
}
for (k=0; k<ddN.T; k++) {
int cnt;
int kk = psort[k];
uint32_t **dfmtx;
if ( ddP.phi==NULL && ddS.NWt[kk]==0 )
continue;
/*
* grab word prob vec for later use
*/
if ( ddS.Nwt ) {
int w;
for (w=0; w<ddN.W; w++)
pvec[w] = wordprob(w,kk);
} else if ( ddP.phi )
fv_copy(pvec, ddP.phi[kk], ddN.W);
else if ( ddS.phi )
fv_copy(pvec, ddS.phi[kk], ddN.W);
/*
* rebuild word list
*/
tscorek = kk;
cnt = buildindk(kk, indk);
topk(topword, cnt, indk, tscore);
if ( topword<cnt )
cnt = topword;
assert(cnt>0);
/*
* df stats for topic returned as matrix
*/
dfmtx = hca_dfmtx(indk, cnt, kk);
if ( ddS.Nwt && (ddS.NWt[kk]*ddN.T*100<Nk_tot || ddS.NWt[kk]<5 ))
underused++;
/*
* print stats for topic
* Mallet: tokens, doc_ent, ave-word-len, coher.,
* uni-dist, corp-dist, eff-no-words
*/
yap_message("Topic %d/%d", kk, k);
{
/*
* compute diagnostics
*/
double prop = gtvec[kk];
float *dprop = docprop(kk);
double spw = 0;
double spd = ((double)nonzero_Ndt(kk))/((double)ddN.DT);
double ew = exp(fv_entropy(pvec,ddN.W));
double ud = fv_helldistunif(pvec,ddN.W);
double pd = fv_helldist(pvec,gpvec,ddN.W);
double sl = fv_avestrlen(pvec,ddN.tokens,ddN.W);
double co = coherence(dfmtx, cnt);
double ed = dprop?exp(fv_entropy(dprop,ddN.DT)):ddN.DT;
double da = dprop?fv_bound(dprop,ddN.DT,1.0/sqrt((double)ddN.T)):0;
sparsitydoc += spd;
yap_message((ddN.T>200)?" p=%.3lf%%":" p=%.2lf%%",100*prop);
if ( ddS.Nwt ) {
spw = ((double)nonzero_Nwt(kk))/((double)ddN.W);
sparsityword += spw;
yap_message(" ws=%.1lf%%", 100*(1-spw));
}
yap_message(" ds=%.1lf%%", 100*(1-spd) );
yap_message(" ew=%.0lf", ew);
yap_message(" ed=%.1lf", ed);
yap_message(" da=%.0lf", da+0.1);
yap_message(" t1=%u", top1cnt[kk]);
yap_message(" ud=%.3lf", ud);
yap_message(" pd=%.3lf", pd);
if ( PCTL_BURSTY() )
yap_message(" bd=%.3lf", ddP.bdk[kk]);
if ( ddN.tokens )
yap_message(" sl=%.2lf", sl);
yap_message(" co=%.3lf%%", co);
if ( pmicount )
yap_message(" pmi=%.3f", tpmi[kk]);
if ( fullreport ) {
fprintf(rp,"topic %d %d", kk, k);
fprintf(rp," %.6lf", prop);
if ( ddS.Nwt ) {
fprintf(rp," %.6lf", (1-spw));
} else {
fprintf(rp," 0");
}
fprintf(rp," %.6lf", (1-spd) );
fprintf(rp," %.2lf", ew);
fprintf(rp," %.2lf", ed);
fprintf(rp," %.0lf", da+0.1);
fprintf(rp," %u", top1cnt[kk]);
fprintf(rp," %.6lf", ud);
fprintf(rp," %.6lf", pd);
if ( PCTL_BURSTY() )
fprintf(rp," %.3lf", ddP.bdk[kk]);
fprintf(rp," %.4lf", (ddN.tokens)?sl:0);
fprintf(rp," %.6lf", co);
if ( pmicount )
fprintf(rp," %.4f", tpmi[kk]);
fprintf(rp,"\n");
}
if ( dprop) free(dprop);
}
if ( verbose>1 ) {
double pcumm = 0;
/*
* print top words:
* Mallet: rank, count, prob, cumm, docs, coh
*/
yap_message("\ntopic %d/%d", kk, k);
yap_message(" words=");
for (w=0; w<cnt; w++) {
if ( w>0 ) yap_message(",");
if ( ddN.tokens )
yap_message("%s", ddN.tokens[indk[w]]);
else
yap_message("%d", indk[w]);
if ( verbose>2 )
yap_message("(%6lf)", tscore(indk[w]));
if ( fullreport ) {
fprintf(rp, "word %d %d %d", kk, indk[w], w);
if ( ddS.Nwt )
fprintf(rp, " %d", ddS.Nwt[indk[w]][kk]);
pcumm += pvec[indk[w]];
fprintf(rp, " %.6f %.6f", pvec[indk[w]], pcumm);
fprintf(rp, " %d", dfmtx[w][w]);
fprintf(rp, " %.6f", coherence_word(dfmtx, cnt, w));
if ( ddN.tokens )
fprintf(rp, " %s", ddN.tokens[indk[w]]);
fprintf(rp, "\n");
}
}
}
yap_message("\n");
free(dfmtx[0]); free(dfmtx);
}
if ( verbose>1 && ddP.PYbeta && (ddP.phi==NULL || ddP.betapr) ) {
int cnt;
double pcumm = 0;
/*
* print root words
*/
tscorek = -1;
cnt = buildindk(-1,indk);
topk(topword, cnt, indk, (ddP.phi==NULL)?countscore:phiscore);
/*
* cannot build df mtx for root because
* it is latent w.r.t. topics
*/
yap_message("Topic root words=");
if ( fullreport ) {
int w;
for (w=0; w<ddN.W; w++)
pvec[w] = betabasewordprob(w);
double ew = exp(fv_entropy(pvec,ddN.W));
double ud = fv_helldistunif(pvec,ddN.W);
double pd = fv_helldist(pvec,gpvec,ddN.W);
fprintf(rp,"topic -1 -1 0 0");
fprintf(rp," %.4lf", ew);
fprintf(rp," %.6lf", ud);
fprintf(rp," %.6lf", pd);
fprintf(rp,"\n");
}
for (w=0; w<topword && w<cnt; w++) {
if ( w>0 ) yap_message(",");
if ( ddN.tokens )
yap_message("%s", ddN.tokens[indk[w]]);
else
yap_message("%d", indk[w]);
if ( verbose>2 )
yap_message("(%6lf)", countscore(indk[w]));
if ( fullreport ) {
fprintf(rp, "word %d %d %d", -1, indk[w], w);
if ( ddS.TwT )
fprintf(rp, " %d", ddS.TwT[w]);
pcumm += pvec[indk[w]];
fprintf(rp, " %.6f %.6f", pvec[indk[w]], pcumm);
fprintf(rp, " 0 0");
if ( ddN.tokens )
fprintf(rp, " %s", ddN.tokens[indk[w]]);
fprintf(rp, "\n");
}
}
yap_message("\n");
}
yap_message("\n");
if ( rp )
fclose(rp);
if ( ddS.Nwt )
yap_message("Average topicXword sparsity = %.2lf%%\n",
100*(1-sparsityword/ddN.T) );
yap_message("Average docXtopic sparsity = %.2lf%%\n"
"Underused topics = %.1lf%%\n",
100*(1-sparsitydoc/ddN.T),
100.0*underused/(double)ddN.T);
if ( pmicount )
yap_message("Average PMI = %.3f\n", tpmi[ddN.T]);
/*
* print
*/
if ( 1 ) {
float **cmtx = hca_topmtx();
int t1, t2;
int m1, m2;
float mval;
char *corfile = yap_makename(resstem,".topcor");
fp = fopen(corfile,"w");
if ( !fp )
yap_sysquit("Cannot open file '%s' for write\n", corfile);
/*
* print file
*/
for (t1=0; t1<ddN.T; t1++) {
for (t2=0; t2<t1; t2++)
if ( cmtx[t1][t2]>1.0e-3 )
fprintf(fp, "%d %d %0.6f\n", t1, t2, cmtx[t1][t2]);
}
fclose(fp);
free(corfile);
/*
* display maximum
*/
m1 = 1; m2 = 0;
mval = cmtx[1][0];
for (t1=0; t1<ddN.T; t1++) {
for (t2=0; t2<t1; t2++) {
if ( mval<cmtx[t1][t2] ) {
mval = cmtx[t1][t2];
m1 = t1;
m2 = t2;
}
}
}
yap_message("Maximum correlated topics (%d,%d) = %f\n", m1, m2, mval);
free(cmtx[0]); free(cmtx);
}
/*
* print burstiness report
*/
if ( PCTL_BURSTY() ) {
int tottbl = 0;
int totmlttbl = 0;
int totmlt = 0;
int i;
for (i=0; i<ddN.NT; i++) {
if ( Z_issetr(ddS.z[i]) ) {
if ( M_multi(i) )
totmlttbl++;
tottbl++;
}
if ( M_multi(i) )
totmlt++;
}
yap_message("Burst report: multis=%.2lf%%, tables=%.2lf%%, tbls-in-multis=%.2lf%%\n",
100.0*((double)ddM.dim_multiind)/ddN.N,
100.0*((double)tottbl)/ddN.NT,
100.0*((double)totmlttbl)/totmlt);
}
yap_message("\n");
free(topfile);
if ( repfile ) free(repfile);
if ( top1cnt ) free(top1cnt);
free(indk);
free(psort);
if ( pmicount )
free(tpmi);
if ( NwK ) {
free(NwK);
NwK = NULL;
}
free(pvec);
free(gtvec);
free(gpvec);
}
/*
* compute likelihood ratio difference based on *M
*/
static double merge_like_Twt(int k1, int k2, merge_beta_t *M) {
int i, w;
double likelihood = 0;
#ifndef BWPAR0
double lbw = log(ddP.bwpar);
#endif
double law = log(ddP.awpar);
double TW_diff = 0;
#ifdef BWPAR0
yap_quit("BWPAR0 unimpleented in merge\n");
#endif
for (i=0; i<ddN.W; i++) {
likelihood -= S_S(ddC.SY,ddS.Nwt[i][k1],ddS.Twt[i][k2]);
likelihood -= S_S(ddC.SY,ddS.Nwt[i][k1],ddS.Twt[i][k2]);
likelihood += S_S(ddC.SY,M->Nwt[i],M->Twt[i]);
}
if ( ddP.awpar==0 ) {
#ifdef BWPAR0
likelihood += M->TWt-ddS.TWt[k1]*log(ddP_bwpar(k1))
-ddS.TWt[k2]*log(ddP_bwpar(k2));
#else
likelihood += (M->TWt-ddS.TWt[k1]-ddS.TWt[k2])*lbw;
#endif
} else {
likelihood += (M->TWt-ddS.TWt[k1]-ddS.TWt[k2])*law
+ gammadiff((int)M->TWt, ddP.bwpar/ddP.awpar, 0.0)
- gammadiff((int)ddS.TWt[k1], ddP_bwpar(k1)/ddP.awpar, 0.0)
- gammadiff((int)ddS.TWt[k2], ddP_bwpar(k2)/ddP.awpar, 0.0);
}
likelihood += gammadiff((int)ddS.NWt[k1], ddP_bwpar(k1), 0.0);
likelihood += gammadiff((int)ddS.NWt[k2], ddP_bwpar(k2), 0.0);
likelihood -= gammadiff((int)M->NWt, ddP.bwpar, 0.0);
yap_infinite(likelihood);
if ( ddP.PYbeta==H_PDP ) {
for (w=0; w<ddN.W; w++) {
if ( ddS.TwT[w]>0 ) {
// ???????????????
likelihood += ddS.TwT[w]*log(ddP.betapr[w]);
}
}
} else if ( ddP.PYbeta==H_HDP ) {
yap_quit("merge with PYbeta unimplemented\n");
likelihood += lgamma(M->TWTm+M->TWt-TW_diff+ddP.bw0)
- lgamma(M->TWTm+M->TWt+ddP.bw0);
for (w=0; w<ddN.W; w++) {
// ???????????
likelihood -= gammadiff(ddS.TWt[k1], ddP.bw0*ddP.betapr[k1], 0.0);
likelihood -= gammadiff(ddS.TWt[k2], ddP.bw0*ddP.betapr[k2], 0.0);
likelihood += gammadiff(M->TWt, ddP.bw0*ddP.betapr[k1], 0.0);
}
} else {
double lgaw0 = lgamma(1-ddP.aw0);
likelihood += lgamma(M->TWTm+M->TWt-TW_diff+ddP.bw0)
- lgamma(M->TWTm+M->TWt+ddP.bw0);
/* because k2 gone to zero, so one less topic */
likelihood -= log(ddP.bw0+ddP.aw0*(ddS.TWTnz-1));
if ( ddS.TWt[k2]>1 )
likelihood -= lgamma(ddS.TWt[k2]-ddP.aw0) - lgaw0;
if ( ddS.TWt[k1]>1 )
likelihood -= lgamma(ddS.TWt[k1]-ddP.aw0) - lgaw0;
likelihood += lgamma(M->TWt-ddP.aw0) - lgaw0;
}
yap_infinite(likelihood);
return likelihood;
}
void like_merge(float minprop, double scale, int best) {
int k1, k2;
double realdiff = 0;
double likediff;
int got=0;
/* only use this for reporting ; should disable in production */
float **cmtx;
int title = 0;
int mincount = minprop * ddN.NT;
bestmerge_t B[ddN.T];
if ( mincount<5 )
mincount = 5;
assert(ddP.phi==NULL);
assert(ddP.theta==NULL);
for (k1=0; k1<ddN.T; k1++)
B[k1].ml = 0;
cmtx = hca_topmtx();
if ( !cmtx )
yap_quit("Out of memory in like_merge()\n");
if ( ddP.PYbeta!=H_None )
yap_quit("Non-parametric beta unimplemented with merge\n");
for (k1=1; k1<ddN.T; k1++) {
if ( ddS.NWt[k1]<=mincount )
continue;
for (k2=0; k2<k1; k2++) {
if ( ddS.NWt[k2]<=mincount )
continue;
/* now have a pair to check (k1,k2) with OK counts */
if ( ddP.PYalpha==H_None )
likediff = likemerge_DIRalpha(k1,k2);
else
likediff = likemerge_alpha(k1, k2);
if ( ddP.PYbeta==H_None )
likediff += likemerge_DIRbeta(k1,k2);
else
likediff += likemerge_beta(k1, k2);
if ( likediff>0 ) {
got++;
if ( title==0 && verbose ) {
double like = scale * likelihood();
yap_message("\nPre merge log_2(perp)=%.4lf", like);
realdiff = like;
}
if ( verbose>1 ) {
if ( title==0 )
yap_message(", merge report:\n");
yap_message(" %d+%d cor=%0.6f like+=%0.6g", k1, k2, cmtx[k1][k2],
scale * likediff);
}
title = 1;
if ( likediff>B[k1].ml ) {
B[k1].ml = likediff;
B[k1].k2 = k2;
if ( verbose>1 ) yap_message(" stored");
}
if ( likediff>B[k2].ml ) {
B[k2].ml = likediff;
B[k2].k2 = k1;
if ( verbose>1 ) yap_message(" stored");
}
if ( verbose>1 ) yap_message("\n");
} else if ( verbose>2 ) {
yap_message(" %d+%d cor=%0.6f like+=%0.6g\n", k1, k2, cmtx[k1][k2],
scale * likediff);
}
}
}
while ( got && best-->0 && (k1=next_best(&B[0]))>=0 ) {
/*
* have a good merge at position k1;
*/
merge_alpha_t Ma;
merge_beta_t Mb;
k2 = B[k1].k2;
yap_message(" best merge is %d+%d giving diff=%lf\n", k1, k2,
scale* B[k1].ml);
Ma.Tdt = NULL;
Mb.Twt = NULL;
if ( ddP.PYalpha )
merge_init_Tdt(k1, k2, &Ma);
if ( ddP.PYbeta )
merge_init_Twt(k1, k2, &Mb);
// WRAY: need to checkk what this does, it it why change?
hca_merge_stats(k1, k2, Ma.Tdt, Mb.Twt);
// hca_correct_tdt(0);
if ( ddP.PYalpha )
merge_free_Tdt(&Ma);
if ( ddP.PYbeta )
merge_free_Twt(&Mb);
/* block them from getting picked again */
B[k1].ml = 0;
B[k2].ml = 0;
{
int k;
for (k=0; k<ddN.T; k++) {
if ( B[k].k2==k1 || B[k].k2==k2 )
B[k].ml = 0;
}
}
}
if ( got && verbose ) {
double like = scale * likelihood();
realdiff -= like;
yap_message("\nPost merge log_2(perp)=%.4lf (%.6lf)", like, realdiff);
}
if ( got==0 && verbose ) {
yap_message("Merge found no candidates\n");
}
free(cmtx[0]); free(cmtx);
}
/*
* print out the topic topk=10 words. report the PMI score.
*/
double report_pmi(char *topfile, /* name of topics file */
char *pmifile, /* name of PMI file */
int T, /* total topics */
int W, /* total words */
int E, /* number of epochs */
int topk,
double *tp)
{
int lineno = 0;
int i,k, thee;
/*
* mapping from local index to actual word index
*/
uint32_t *wind = u32vec(topk*T*E);
int n_wind = 0;
/*
* boolean vector ... is word used
*/
uint32_t *wuse = u32vec(W/32+1);
/*
* PMI's by local index
*/
uint32_t *topic = u32vec(topk);
float *coherency = fvec(E);
double **pmi;
float ave = 0;
char *line;
size_t n_line;
FILE *fr;
if ( !wind || !wuse )
yap_quit("Out of memory in report_pmi()\n");
/*
* read in file of top word indices in topic
*/
fr = fopen(topfile,"r");
if ( !fr )
yap_sysquit("Topic file '%s' not read\n", topfile);
line = NULL;
n_line = 0;
lineno = 0;
while ( getline(&line, &n_line, fr)>0 ) {
char *buf = line;
unsigned j;
int e = 0;
lineno ++;
buf += strspn(buf," \t\n"); // skip space
if ( (E==1 && sscanf(buf, "%d: ", &k)<1) ||
(E>1 && sscanf(buf, "%d,%d: ", &e, &k)<2) )
yap_quit("Cannot read topic in topic line %d from file '%s'\n",
lineno, topfile);
if ( k<0 || k>=T )
continue;
if ( e<0 || e>=E )
continue;
for (i = 0; i<topk && *buf; i++) {
buf = strpbrk(buf," \t\n"); // skip to next space
if ( sscanf(buf, " %u", &j) <1 ) {
if ( verbose>2 )
yap_message("Cannot read word %d in topic line %d from file '%s'\n",
i+1, lineno, topfile);
break;
}
if ( j>=W) {
yap_quit("Bad word %d in topic line %d from file '%s'\n",
i+1, lineno, topfile);
}
buf += strspn(buf," \t\n"); // skip space
/*
* check if word exists, and set up its index
*/
if ( wuse[j/32U] & (1U<<(j%32U)) ) {
// yes, so search for it
int ii;
for (ii=0; ii<n_wind; ii++)
if ( wind[ii]==j )
break;
if ( ii>=n_wind )
yap_quit("Lookup of word %d failed at line %d in report_pmi()\n",
(int)j, lineno);
} else {
// no, so add it
wuse[j/32U] |= (1U<<(j%32U));
wind[n_wind] = j;
n_wind++;
}
}
free(line);
line = NULL;
n_line = 0;
}
fclose(fr);
pmi = dmat(n_wind,n_wind);
/*
* build hash table now since we know size
*/
hashsize = n_wind*2;
hashtab = malloc(sizeof(*hashtab)*hashsize);
if ( !pmi || !hashtab )
yap_quit("Out of memory in report_pmi()\n");
for (i=0; i<hashsize; i++)
hashtab[i] = 0;
for (i=0; i<n_wind; i++)
addw(wind[i],i);
/*
* load up PMI file, only keeping words mentioned in hash table
*/
{
unsigned t1, t2;
double value;
int zcat = 0;
fr = fopen(pmifile,"r");
if ( !fr ) {
/*
* try to zcat it
*/
char *cmd = malloc(strlen(pmifile)+20);
sprintf(cmd,"%s.gz", pmifile);
fr = fopen(cmd,"r");
if ( !fr )
yap_sysquit("Cannot open pmifile '%s' in report_pmi()\n",
pmifile);
fclose(fr);
sprintf(cmd,"gunzip -c %s", pmifile);
fr = popen(cmd,"r");
if ( !fr )
yap_sysquit("Cannot open or zcat pmifile '%s' in report_pmi()\n",
pmifile);
zcat = 1;
free(cmd);
}
while (fscanf(fr, "%u %u %lg", &t1, &t2, &value)==3 ) {
if ( t1>=W || t2>= W )
yap_quit("Illegal word index in report_pmi()\n");
if ( t1!= t2 && ( wuse[t1/32U] & (1U<<(t1%32U)) )
&& ( wuse[t2/32U] & (1U<<(t2%32U))) ) {
int i1, i2;
i1 = findw(t1,wind);
i2 = findw(t2,wind);
if ( i1==UINT32_MAX || i2==UINT32_MAX )
yap_quit("Could not locate word index in report_pmi()\n");
pmi[i1][i2]=value;
pmi[i2][i1]=value;
}
}
if ( zcat )
pclose(fr);
else
fclose(fr);
}
/*
* compute PMI score for each topic
*/
fr = fopen(topfile,"r");
if ( !fr )
yap_sysquit("Topic file '%s' not read\n", topfile);
line = NULL;
n_line = 0;
thee = 0;
lineno = 0;
if ( E>1 )
yap_message("PMI %d:: ", 0);
else
yap_message("PMI :: ");
while ( getline(&line, &n_line, fr)>0 ) {
/*
* repeat logic above to read topic file again
*/
char *buf = line;
unsigned j;
int cnt = 0;
int e = 0;
double coh = 0;
buf += strspn(buf," \t\n"); // skip space
if ( (E==1 && sscanf(buf, "%d: ", &k)<1) ||
(E>1 && sscanf(buf, "%d,%d: ", &e, &k)<2) )
yap_quit("Cannot read topic in topic line %d from file '%s'\n",
lineno, topfile);
if ( k<0 || k>=T )
continue;
if ( e<0 || e>=E )
continue;
if ( e!=thee ) {
thee = e;
yap_message("\nPMI %d:: ", e);
}
for (i = 0; i<topk && *buf; i++) {
buf = strpbrk(buf," \t\n"); // skip to next space
if ( sscanf(buf, " %u", &j) <1 ) {
yap_message("Cannot read word %d in topic line %d from file '%s'\n", i+1, lineno, topfile);
break;
}
if ( j>=W) {
yap_quit("Bad word %d in topic line %d from file '%s'\n", i+1, lineno, topfile);
}
buf += strspn(buf," \t\n"); // skip space
topic[i] = findw(j,wind);
}
if ( i<topk )
topic[i] = W;
/*
* topics now read
*/
for (i=0; i<topk && topic[i]<W; i++) {
for (j=i+1; j<topk && topic[j]<W; j++) {
coh += pmi[topic[i]][topic[j]];
cnt ++;
}
}
if ( cnt>0 ) coh /= cnt;
coherency[e] += coh * tp[k];
yap_message(" %d:%.3lf", k, coh);
}
fclose(fr);
yap_message("\nPMI =");
if ( E==1 ) {
yap_message(" %.3lf\n", coherency[0]);
ave = coherency[0];
} else {
int e;
for (e=0; e<E; e++) {
ave += coherency[e];
yap_message(" %.3lf", coherency[e]);
}
ave /= E;
yap_message(" -> %.3lf\n", ave);
}
free(wind);
free(coherency);
free(wuse);
free(topic);
free(pmi[0]); free(pmi);
free(hashtab);
hashtab = NULL;
hashsize = 0;
return ave;
}
double likelihood_PYbeta() {
int i,t;
double likelihood = 0;
double lbw = log(ddP.bwpar);
double law = log(ddP.awpar);
likelihood += pctl_gammaprior(ddP.bwpar);
/*
* term for k-th node
*/
#ifdef BWPAR0
for (t=1; t<ddN.T; t++) {
#else
for (t=0; t<ddN.T; t++) {
#endif
uint32_t Tw_ = 0;
for (i=0; i<ddN.W; i++) {
int tt = ddS.Twt[i][t];
int nn = ddS.Nwt[i][t];
if ( nn>0 ) {
Tw_ += tt;
likelihood += S_S(ddC.SY,nn,tt);
#if 1
if ( !finite(likelihood) || isinf(likelihood) || isnan(likelihood) )
yap_quit("Like=%lf: Nwt[%d][%d]=%d Twt[i][t]=%d S.M=%d S.N=%d\n",
likelihood,
i, t, (int)ddS.Nwt[i][t],(int)ddS.Twt[i][t],ddC.SY->usedM, ddC.SY->usedN);
#endif
}
}
yap_infinite(likelihood);
if ( ddP.awpar==0 ) {
likelihood += Tw_*lbw;
} else {
#ifdef L_CACHE
likelihood += Tw_*law + gcache_value(&ddC.lgbaw, (int)Tw_);
#else
likelihood += Tw_*law + gammadiff((int)Tw_, ddP.bwpar/ddP.awpar, 0.0);
#endif
}
#ifdef L_CACHE
likelihood -= gcache_value(&ddC.lgbw, (int)ddS.NWt[t]);
#else
likelihood -= gammadiff((int)ddS.NWt[t], ddP.bwpar, 0.0);
#endif
yap_infinite(likelihood);
}
yap_infinite(likelihood);
return likelihood;
}
double likelihood_PYbeta_PDP() {
/*
* the constant prior
*/
int j;
double likelihood = 0;
for (j=0; j<ddN.W; j++) {
if ( ddS.TwT[j]>0 ) {
likelihood += ddS.TwT[j]*log(ddP.betapr[j]);
}
}
//yap_infinite(likelihood);
return likelihood;
}
/*
* run regular gibbs cycles on the data with phi used;
* the evaluation on each doc, and sample word probs
*
* if qparts>0, split collection into parts and only search this
*
* K = number of top results to retain
*/
void gibbs_query(int K, char *qname, int dots, int this_qpart, int qparts) {
/*
* mapping from query word posn. to its mi in current doc
* >ddN.N = not in current doc
* -ve = has no mi since occurs just once, found at
* posn (-map[]-1)
* non -ve = mi value
*/
int *mimap = NULL;
/*
* usual stuff for Gibbs loop over docs
*/
int i, j;
float *fact = fvec(ddN.T*4);
D_MiSi_t dD;
/*
* an index into topk[] which maintains ordering
*/
int *topind;
/*
* these store statistics of the results, for printing
* these are unordered, ordered by topind[]
*/
/* document score */
float *topscore;
/* document number */
int *topk;
/* flags if ord is irrelevant, thus not scored */
char *wordunused;
/*
* per word stats for top results saved
*/
int *found;
float *topcnt;
float *topwordscore;
/*
* temporary versions for when gibbs running
*/
int *found_buf;
float *topcnt_buf;
float *topwordscore_buf;
double *logprob;
/*
* search here
*/
int startdoc = 0;
int enddoc = ddN.DT;
/*
* setup
*/
topcnt = malloc(sizeof(topcnt[0])*K*ddP.n_words);
topwordscore = malloc(sizeof(topwordscore[0])*K*ddP.n_words);
found = malloc(sizeof(found)*ddP.n_words*K);
wordunused = malloc(sizeof(wordunused[0])*ddP.n_words);
topcnt_buf = malloc(sizeof(topcnt[0])*ddP.n_words);
topwordscore_buf = malloc(sizeof(topwordscore[0])*ddP.n_words);
found_buf = malloc(sizeof(found)*ddP.n_words);
if ( !topcnt || !topwordscore || !found ||
!topcnt_buf || !topwordscore_buf || !found_buf )
yap_quit("Cannot allocate memory in gibbs_query()\n");
logprob = malloc(sizeof(logprob[0])*ddP.n_query);
topscore = malloc(sizeof(topscore[0])*K*ddP.n_query);
topind = malloc(sizeof(topind[0])*K*ddP.n_query);
topk = malloc(sizeof(topk[0])*K*ddP.n_query);
if ( ddP.bdk!=NULL )
mimap = malloc(sizeof(mimap[0])*ddP.n_words);
if ( !topk || !topscore || !logprob || !topind )
yap_quit("Cannot allocate memory in gibbs_query()\n");
for (i=0; i<ddP.n_words; i++) {
wordunused[i] = 0;
}
for (i=0; i<K*ddP.n_query; i++) {
topind[i] = i%K;
topk[i] = -1;
topscore[i] = INFINITY;
}
/*
* check words to exclude using topics
*/
if ( ddP.n_excludetopic>0 ) {
double *tprob = malloc(sizeof(tprob[0])*ddN.T);
get_probs(tprob);
yap_probs();
if ( verbose>1 )
yap_message("Excluding words: ");
for (i=0; i<ddP.n_words; i++) {
int t = besttopic(ddP.qword[i],tprob);
if ( Q_excludetopic(t) ) {
wordunused[i] = 1;
if ( verbose>1 )
yap_message(" %d/%d", (int)ddP.qword[i], t);
}
}
if ( verbose>1 )
yap_message("\n");
free(tprob);
}
if ( ddP.bdk!=NULL ) misi_init(&ddM,&dD);
if ( qparts>0 ) {
startdoc = ((double)this_qpart)/qparts * ddN.DT;
enddoc = ((double)this_qpart+1.0)/qparts * ddN.DT;
}
for(i=startdoc; i<enddoc; i++) {
int thisw = add_doc(i, GibbsNone);
int r;
if ( thisw<=1 ) {
remove_doc(i, GibbsNone);
continue;
}
if ( ddP.bdk!=NULL )
misi_build(&dD, i, 0);
map_query(i, mimap, found_buf);
for (j=0; j<ddP.n_words; j++) {
topcnt_buf[j] = 0;
topwordscore_buf[j] = 0;
}
for (r=0; r<ddP.queryiter; r++) {
gibbs_lda(GibbsNone, ddN.T, i, ddD.NdT[i], fact, &dD, 0, 0);
query_docprob(i, mimap, fact, &dD, topcnt_buf, topwordscore_buf);
}
/*
* now adjust stats
*/
for (j=0; j<ddP.n_query; j++)
logprob[j] = 0;
for (j=0; j<ddP.n_words; j++) {
if ( wordunused[j]>0 )
continue;
if ( ddP.query[ddP.qword[j]]==j ) {
topcnt_buf[j] /= ddP.queryiter;
topwordscore_buf[j] /= ddP.queryiter;
} else {
/* word in previous query so copy */
int jj = ddP.query[ddP.qword[j]];
topcnt_buf[j] = topcnt_buf[jj];
topwordscore_buf[j] = topwordscore_buf[jj];
found_buf[j] = found_buf[jj];
}
if ( wordunused[j]==0 )
logprob[ddP.qid[j]] += topwordscore_buf[j];
}
if ( dots>0 && i>0 && (i%dots==0) )
yap_message(".");
if ( ddP.bdk!=NULL ) misi_unbuild(&dD,i,0);
remove_doc(i, GibbsNone);
/*
* enter into the arrays
*/
for (j=0; j<ddP.n_query; j++) {
if ( i<K || logprob[j] < topscore[j*K+topind[j*K+K-1]] ) {
int newind, l;
/*
* better than current lowest
*/
newind = bubble((i<K)?(i+1):K,
&topind[j*K], &topscore[j*K], logprob[j]);
/*
* save the current details
*/
topscore[j*K+newind] = logprob[j];
topk[j*K+newind] = i;
for (l=ddP.qposn[j]; l<ddP.qposn[j+1]; l++) {
topcnt[newind*ddP.n_words+l] = topcnt_buf[l];
topwordscore[newind*ddP.n_words+l] = topwordscore_buf[l];
found[newind*ddP.n_words+l] = found_buf[l];
}
}
}
}
if ( dots>0 ) yap_message("\n");
/*
* write result
*/
{
float *ws = fvec(ddP.n_words);
FILE *fp = fopen(qname,"w");
int q;
if ( !fp )
yap_sysquit("Cannot write query results to '%s'\n", qname);
for (q=0; q<ddP.n_query; q++) {
int nw = ddP.qposn[q+1]-ddP.qposn[q];
for (i=0; i<K && i<ddN.DT && topk[topind[q*K+i]]>=0; i++) {
int l, ind = topind[q*K+i];
double tfidf;
tfidf = bm25(topk[q*K+ind],&found[ind*ddP.n_words+ddP.qposn[q]],
&ddP.qword[ddP.qposn[q]], nw, ws);
assert(ind>=0 && ind<K);
fprintf(fp, "%d %d ", q, topk[q*K+ind]);
fprintf(fp, "%.4f %.4lf ", topscore[q*K+ind]/nw, tfidf);
if ( verbose>1 ) {
for (l=ddP.qposn[q]; l<ddP.qposn[q+1]; l++)
fprintf(fp, "%d ", found[ind*ddP.n_words+l]);
for (l=ddP.qposn[q]; l<ddP.qposn[q+1]; l++)
fprintf(fp, "%f ", topcnt[ind*ddP.n_words+l]);
for (l=ddP.qposn[q]; l<ddP.qposn[q+1]; l++)
fprintf(fp, "%f ", topwordscore[ind*ddP.n_words+l]);
for (l=0; l<nw; l++)
fprintf(fp, "%lf ", ws[l]);
}
fprintf(fp, "\n");
}
}
fclose(fp);
free(ws);
}
/*
* clean up
*/
free(fact);
if ( ddP.bdk!=NULL ) misi_free(&dD);
if ( mimap ) free(mimap);
free(found);
free(topwordscore);
free(topcnt);
free(found_buf);
free(topwordscore_buf);
free(topcnt_buf);
free(topscore);
free(topind);
free(topk);
free(logprob);
}
