template<typename T, typename AT> void
accW_( const Mat& _src, Mat& _dst, double _alpha )
{
AT alpha = (AT)_alpha, beta = (AT)(1 - _alpha);
Size size = _src.size();
size.width *= _src.channels();
if( _src.isContinuous() && _dst.isContinuous() )
{
size.width *= size.height;
size.height = 1;
}
int i, j;
for( i = 0; i < size.height; i++ )
{
const T* src = (const T*)(_src.data + _src.step*i);
AT* dst = (AT*)(_dst.data + _dst.step*i);
for( j = 0; j <= size.width - 4; j += 4 )
{
AT t0, t1;
t0 = addw(src[j], alpha, dst[j], beta);
t1 = addw(src[j+1], alpha, dst[j+1], beta);
dst[j] = t0; dst[j+1] = t1;
t0 = addw(src[j+2], alpha, dst[j+2], beta);
t1 = addw(src[j+3], alpha, dst[j+3], beta);
dst[j+2] = t0; dst[j+3] = t1;
}
for( ; j < size.width; j++ )
dst[j] = addw(src[j], alpha, dst[j], beta);
}
}
template<typename T, typename AT> void
accWMask_( const Mat& _src, Mat& _dst, double _alpha, const Mat& _mask )
{
typedef typename DataType<AT>::channel_type AT1;
AT1 alpha = (AT1)_alpha, beta = (AT1)(1 - _alpha);
Size size = _src.size();
if( _src.isContinuous() && _dst.isContinuous() && _mask.isContinuous() )
{
size.width *= size.height;
size.height = 1;
}
int i, j;
for( i = 0; i < size.height; i++ )
{
const T* src = (const T*)(_src.data + _src.step*i);
AT* dst = (AT*)(_dst.data + _dst.step*i);
const uchar* mask = _mask.data + _mask.step*i;
for( j = 0; j < size.width; j++ )
if( mask[j] )
dst[j] = addw(src[j], alpha, dst[j], beta);
}
}
/*
* print out the topic topk=10 words. report the PMI score.
*/
double report_pmi(char *topfile, /* name of topics file */
char *pmifile, /* name of PMI file */
int T, /* total topics */
int W, /* total words */
int E, /* number of epochs */
int topk,
double *tp)
{
int lineno = 0;
int i,k, thee;
/*
* mapping from local index to actual word index
*/
uint32_t *wind = u32vec(topk*T*E);
int n_wind = 0;
/*
* boolean vector ... is word used
*/
uint32_t *wuse = u32vec(W/32+1);
/*
* PMI's by local index
*/
uint32_t *topic = u32vec(topk);
float *coherency = fvec(E);
double **pmi;
float ave = 0;
char *line;
size_t n_line;
FILE *fr;
if ( !wind || !wuse )
yap_quit("Out of memory in report_pmi()\n");
/*
* read in file of top word indices in topic
*/
fr = fopen(topfile,"r");
if ( !fr )
yap_sysquit("Topic file '%s' not read\n", topfile);
line = NULL;
n_line = 0;
lineno = 0;
while ( getline(&line, &n_line, fr)>0 ) {
char *buf = line;
unsigned j;
int e = 0;
lineno ++;
buf += strspn(buf," \t\n"); // skip space
if ( (E==1 && sscanf(buf, "%d: ", &k)<1) ||
(E>1 && sscanf(buf, "%d,%d: ", &e, &k)<2) )
yap_quit("Cannot read topic in topic line %d from file '%s'\n",
lineno, topfile);
if ( k<0 || k>=T )
continue;
if ( e<0 || e>=E )
continue;
for (i = 0; i<topk && *buf; i++) {
buf = strpbrk(buf," \t\n"); // skip to next space
if ( sscanf(buf, " %u", &j) <1 ) {
if ( verbose>2 )
yap_message("Cannot read word %d in topic line %d from file '%s'\n",
i+1, lineno, topfile);
break;
}
if ( j>=W) {
yap_quit("Bad word %d in topic line %d from file '%s'\n",
i+1, lineno, topfile);
}
buf += strspn(buf," \t\n"); // skip space
/*
* check if word exists, and set up its index
*/
if ( wuse[j/32U] & (1U<<(j%32U)) ) {
// yes, so search for it
int ii;
for (ii=0; ii<n_wind; ii++)
if ( wind[ii]==j )
break;
if ( ii>=n_wind )
yap_quit("Lookup of word %d failed at line %d in report_pmi()\n",
(int)j, lineno);
} else {
// no, so add it
wuse[j/32U] |= (1U<<(j%32U));
wind[n_wind] = j;
n_wind++;
}
}
free(line);
line = NULL;
n_line = 0;
}
fclose(fr);
pmi = dmat(n_wind,n_wind);
/*
* build hash table now since we know size
*/
hashsize = n_wind*2;
hashtab = malloc(sizeof(*hashtab)*hashsize);
if ( !pmi || !hashtab )
yap_quit("Out of memory in report_pmi()\n");
for (i=0; i<hashsize; i++)
hashtab[i] = 0;
for (i=0; i<n_wind; i++)
addw(wind[i],i);
/*
* load up PMI file, only keeping words mentioned in hash table
*/
{
unsigned t1, t2;
double value;
int zcat = 0;
fr = fopen(pmifile,"r");
if ( !fr ) {
/*
* try to zcat it
*/
char *cmd = malloc(strlen(pmifile)+20);
sprintf(cmd,"%s.gz", pmifile);
fr = fopen(cmd,"r");
if ( !fr )
yap_sysquit("Cannot open pmifile '%s' in report_pmi()\n",
pmifile);
fclose(fr);
sprintf(cmd,"gunzip -c %s", pmifile);
fr = popen(cmd,"r");
if ( !fr )
yap_sysquit("Cannot open or zcat pmifile '%s' in report_pmi()\n",
pmifile);
zcat = 1;
free(cmd);
}
while (fscanf(fr, "%u %u %lg", &t1, &t2, &value)==3 ) {
if ( t1>=W || t2>= W )
yap_quit("Illegal word index in report_pmi()\n");
if ( t1!= t2 && ( wuse[t1/32U] & (1U<<(t1%32U)) )
&& ( wuse[t2/32U] & (1U<<(t2%32U))) ) {
int i1, i2;
i1 = findw(t1,wind);
i2 = findw(t2,wind);
if ( i1==UINT32_MAX || i2==UINT32_MAX )
yap_quit("Could not locate word index in report_pmi()\n");
pmi[i1][i2]=value;
pmi[i2][i1]=value;
}
}
if ( zcat )
pclose(fr);
else
fclose(fr);
}
/*
* compute PMI score for each topic
*/
fr = fopen(topfile,"r");
if ( !fr )
yap_sysquit("Topic file '%s' not read\n", topfile);
line = NULL;
n_line = 0;
thee = 0;
lineno = 0;
if ( E>1 )
yap_message("PMI %d:: ", 0);
else
yap_message("PMI :: ");
while ( getline(&line, &n_line, fr)>0 ) {
/*
* repeat logic above to read topic file again
*/
char *buf = line;
unsigned j;
int cnt = 0;
int e = 0;
double coh = 0;
buf += strspn(buf," \t\n"); // skip space
if ( (E==1 && sscanf(buf, "%d: ", &k)<1) ||
(E>1 && sscanf(buf, "%d,%d: ", &e, &k)<2) )
yap_quit("Cannot read topic in topic line %d from file '%s'\n",
lineno, topfile);
if ( k<0 || k>=T )
continue;
if ( e<0 || e>=E )
continue;
if ( e!=thee ) {
thee = e;
yap_message("\nPMI %d:: ", e);
}
for (i = 0; i<topk && *buf; i++) {
buf = strpbrk(buf," \t\n"); // skip to next space
if ( sscanf(buf, " %u", &j) <1 ) {
yap_message("Cannot read word %d in topic line %d from file '%s'\n", i+1, lineno, topfile);
break;
}
if ( j>=W) {
yap_quit("Bad word %d in topic line %d from file '%s'\n", i+1, lineno, topfile);
}
buf += strspn(buf," \t\n"); // skip space
topic[i] = findw(j,wind);
}
if ( i<topk )
topic[i] = W;
/*
* topics now read
*/
for (i=0; i<topk && topic[i]<W; i++) {
for (j=i+1; j<topk && topic[j]<W; j++) {
coh += pmi[topic[i]][topic[j]];
cnt ++;
}
}
if ( cnt>0 ) coh /= cnt;
coherency[e] += coh * tp[k];
yap_message(" %d:%.3lf", k, coh);
}
fclose(fr);
yap_message("\nPMI =");
if ( E==1 ) {
yap_message(" %.3lf\n", coherency[0]);
ave = coherency[0];
} else {
int e;
for (e=0; e<E; e++) {
ave += coherency[e];
yap_message(" %.3lf", coherency[e]);
}
ave /= E;
yap_message(" -> %.3lf\n", ave);
}
free(wind);
free(coherency);
free(wuse);
free(topic);
free(pmi[0]); free(pmi);
free(hashtab);
hashtab = NULL;
hashsize = 0;
return ave;
}
int
dorecipe(Node *node)
{
int did = 0;
char buf[BIGBLOCK], cwd[256];
Arc *a, *aa;
Node *n;
Rule *r = 0;
Symtab *s;
Word head, ahead, lp, ln, *w, *ww, *aw;
aa = 0;
/*
pick up the rule
*/
for(a = node->prereqs; a; a = a->next)
if(*a->r->recipe)
r = (aa = a)->r;
/*
no recipe? go to buggery!
*/
if(r == 0){
if(!(node->flags&VIRTUAL) && !(node->flags&NORECIPE)){
if(getwd(cwd, sizeof cwd))
fprint(2, "mk: no recipe to make '%s' in directory %s\n", node->name, cwd);
else
fprint(2, "mk: no recipe to make '%s'\n", node->name);
Exit();
}
if(strchr(node->name, '(') && node->time == 0)
MADESET(node, MADE);
else
update(0, node);
if(tflag){
if(!(node->flags&VIRTUAL))
touch(node->name);
else if(explain)
Bprint(&bout, "no touch of virtual '%s'\n", node->name);
}
return(did);
}
/*
build the node list
*/
node->next = 0;
head.next = 0;
ww = &head;
ahead.next = 0;
aw = &ahead;
if(r->attr®EXP){
ww->next = newword(node->name);
aw->next = newword(node->name);
} else {
for(w = r->alltargets; w; w = w->next){
if(r->attr&META)
subst(aa->stem, w->s, buf, sizeof(buf));
else
strecpy(buf, buf + sizeof buf - 1, w->s);
aw->next = newword(buf);
aw = aw->next;
if((s = symlook(buf, S_NODE, 0)) == 0)
continue; /* not a node we are interested in */
n = s->u.ptr;
if(aflag == 0 && n->time) {
for(a = n->prereqs; a; a = a->next)
if(a->n && outofdate(n, a, 0))
break;
if(a == 0)
continue;
}
ww->next = newword(buf);
ww = ww->next;
if(n == node) continue;
n->next = node->next;
node->next = n;
}
}
for(n = node; n; n = n->next)
if((n->flags&READY) == 0)
return(did);
/*
gather the params for the job
*/
lp.next = ln.next = 0;
for(n = node; n; n = n->next){
for(a = n->prereqs; a; a = a->next){
if(a->n){
addw(&lp, a->n->name);
if(outofdate(n, a, 0)){
addw(&ln, a->n->name);
if(explain)
fprint(1, "%s(%ld) < %s(%ld)\n",
n->name, n->time, a->n->name, a->n->time);
}
} else {
if(explain)
fprint(1, "%s has no prerequisites\n",
n->name);
}
}
MADESET(n, BEINGMADE);
}
/* print("lt=%s ln=%s lp=%s\n",wtos(head.next, ' '),wtos(ln.next, ' '),wtos(lp.next, ' '));/**/
run(newjob(r, node, aa->stem, aa->match, lp.next, ln.next, head.next, ahead.next));
return(1);
}
