/*
* Initialize the TTY structures.
*/
void
tty_init()
{
int i;
TScreen *screen = &term->screen;
dbprintf(("tty_init.."));
if (tty_inited) {
return;
}
tty_inited = 1;
mu_init(&read_mutex);
mu_init(&write_mutex);
uniqport(&ttyport);
priv2pub(&ttyport, &ttycap.cap_port);
tios.c_iflag = ICRNL | IXON;
tios.c_oflag = OPOST;
tios.c_cflag = CREAD | CS8 | B9600;
tios.c_lflag = ECHO | ECHOE | ECHOK | ICANON | ISIG;
tios.c_cc[VEOF] = CTRL('D');
tios.c_cc[VEOL] = _POSIX_VDISABLE;
tios.c_cc[VERASE] = CTRL('H');
tios.c_cc[VINTR] = CTRL('C');
tios.c_cc[VKILL] = CTRL('U');
tios.c_cc[VMIN] = 1;
tios.c_cc[VQUIT] = CTRL('\\');
tios.c_cc[VSUSP] = _POSIX_VDISABLE;
tios.c_cc[VTIME] = 0;
tios.c_cc[VSTART] = CTRL('Q');
tios.c_cc[VSTOP] = CTRL('S');
screen->tty_inq = cb_alloc(TTY_INQSIZE);
screen->tty_outq = cb_alloc(TTY_OUTQSIZE);
screen->max_col = def_cols;
screen->max_row = def_rows;
/*
* Start tty threads. Ordinarily two should suffice, one for standard
* input and one for standard (error) output.
*/
for (i = 0; i < TTY_NTHREADS; i++) {
if (!thread_newthread(ttythread, TTY_THREAD_STACKSIZE, (char*) 0, 0)) {
FatalError("unable to start tty thread.");
}
}
}
void tcpipinit()
{
prog_name= "kernel TCP/IP";
mu_init(&mu_generic);
mu_lock(&mu_generic);
tcpip_dirinit();
sr_init_cap_names();
sr_set_cap_name(ETH_DEV0, "eth");
sr_set_cap_name(IP_DEV0, "ip");
sr_set_cap_name(TCP_DEV0, "tcp");
sr_set_cap_name(UDP_DEV0, "udp");
#if 0
sr_enable_linger_right();
#endif
bf_init();
clck_init();
sr_init();
eth_init();
arp_init();
ip_init();
tcp_init();
udp_init();
add_default_gws();
mu_unlock(&mu_generic);
tcpip_chmod();
}
void
gs_init_ttl()
{
register Inodenum i;
#ifndef AGE_IN_INODE
Res_size numbytes;
numbytes = Superblock.s_numinodes;
CEILING_BLKSZ(numbytes);
Inode_local_ttl = (unsigned char *) a_alloc(A_CACHE_MEM, numbytes);
Inode_global_ttl = (unsigned char *) a_alloc(A_CACHE_MEM, numbytes);
if (Inode_local_ttl == 0 || Inode_global_ttl == 0)
bpanic("Insufficient memory for ttl structures.");
#endif
if ( init_done ) return ;
dbmessage(28,("Init_ttl starts..."));
p_ttl_updates = ttl_updates ;
mu_init(&update_mutex) ;
sema_init(&can_age,0) ;
sema_init(&awaiting_accept,0);
refuse_age=1 ; refusing_member= -1 ; delay_threshold= HARD_THRESHOLD ;
/* Initially set all lifetimes to unknown */
for (i = 1; i < Superblock.s_numinodes; i++)
{
INIT_LOCAL_TTL(i) ; INIT_GLOBAL_TTL(i) ;
}
init_done= 1;
dbmessage(28,("Init_ttl finishes"));
}
int main(int argc, char *argv[])
{
mu_init(argc, argv);
mu_run_test(MU_PASS, test_convert_str);
mu_run_test(MU_PASS, test_compress_escapes);
mu_run_test(MU_PASS, test_str);
mu_run_test(MU_PASS, test_getline);
mu_fini();
}
static void
prof_init()
{
mu_init(&Freelist_mu);
init_func_stack();
init_func_call();
init_func_info();
Prof_initialized = 1;
}
static struct thd_info *
enter()
{
/* Flag to avoid recursion while getting the thread id: */
static int Prof_getting_id = 0;
static int ident;
register int *thread_id;
register struct thd_info *thdp;
if (!Prof_initialized) {
prof_init();
}
if (Prof_getting_id) {
return NULL;
}
Prof_getting_id = 1;
thread_id = (int *) thread_alloc(&ident, sizeof(int));
Prof_getting_id = 0;
/* When initialized, the glocal variable referred to by `ident'
* contains the thread number plus one.
*/
if (*thread_id == 0) {
/* first time called within this thread */
if (Prof_nthreads < MAXTHREADS) {
*thread_id = ++Prof_nthreads;
/* initialize thread info structure */
thdp = &thread_info[*thread_id - 1];
thdp->thd_here = 0;
thdp->thd_funcs = NULL;
thdp->thd_stack = NULL;
mu_init(&thdp->thd_mu);
} else {
thdp = NULL;
}
} else {
thdp = &thread_info[*thread_id - 1];
}
if (thdp != NULL) {
if (!thdp->thd_here) {
thdp->thd_here = 1;
mu_lock(&thdp->thd_mu);
} else {
thdp = NULL;
}
}
return thdp;
}
int main(int argc, char *argv[])
{
mu_init(argc, argv);
mu_run_test(MU_PASS, test_init);
mu_fini();
}
int
bullet_init()
{
vir_bytes size;
peer_bits test_var ;
/* Initialize to logging */
bs_log_msgs=1 ;
#ifndef USER_LEVEL
bs_print_msgs = 0;
#endif
bs_debug_level = DEBUG_LEVEL ;
/* Check that inode size is sensible - otherwise the compiler is odd */
if (sizeof (Inode) != INODE_SIZE)
{
bwarn("Inode size is %d, should be %d\nServer is inactive!",
sizeof(Inode), INODE_SIZE);
return 0;
}
/* Check that the peer bitmap can contain all members */
test_var=1 ;
test_var <<= (S_MAXMEMBER-1) ;
if ( test_var == 0 || test_var!= (1<<(S_MAXMEMBER-1)) ) {
bwarn("type \"peer_bits\" is too small\nServer is inactive!");
return 0;
}
/* Figure out which vdisk to use and read the superblock */
if (get_disk_server_cap() == 0)
return 0;
/* Now that we know there is a disk we can announce ourselves */
message("BULLET SERVER INITIALIZATION");
/*
* Set up pointers to beginning and end of buffer cache.
* Make sure that our buffer cache is block aligned! The inode table
* goes at the start of the cache memory and relies on blocksize
* alignment.
* NB: We must leave some memory free for other kernel threads and
* possible user processes (such as soap). It leaves
* BS_MEM_RESERVE bytes if no fixed cache size is defined.
*/
#ifdef BULLET_MEMSIZE
size = BULLET_MEMSIZE;
#else
size = seg_maxfree();
if (size < BS_MEM_RESERVE)
{
bpanic(
"bullet_init: memory reserve (0x%x) exceeds free memory (0x%x)\n",
BS_MEM_RESERVE, size);
}
size -= BS_MEM_RESERVE;
#endif /* BULLET_MEMSIZE */
/*
* The following allocate may take a long time, especially when lots
* of memory (32Mb :-) is involved. However, since kernel threads are
* not preempted the enqueued interrupt routines never get a chance to
* run, and eventually the interrupt queue will overflow causing a
* panic.
*/
#ifndef USER_LEVEL
disable();
#endif /* USER_LEVEL */
Bc_begin = (bufptr) aalloc(size, (int) Blksizemin1 + 1);
#ifndef USER_LEVEL
enable();
#endif /* USER_LEVEL */
Bc_end = Bc_begin + size;
if (Bc_begin >= Bc_end)
bpanic("bullet_init: no buffer cache");
/* Initialise resource allocation. NB: cache_mem is initially free! */
a_init(BYTES_TO_MEG(Bc_end - Bc_begin) +
BLOCKS_TO_MEG(Superblock.s_numblocks, Blksizemin1 + 1));
if (a_begin(A_CACHE_MEM, (Res_addr) Bc_begin, (Res_addr) Bc_end) < 0)
bpanic("Cannot begin resource allocator for buffer cache.\n");
/* Initialise buffer cache management */
cache_init();
Largest_file_size = (b_fsize) a_notused(A_CACHE_MEM);
/* Print some cheery statistics about the current state of the server */
message("Buffer cache size = 0x%lx bytes", Bc_end - Bc_begin);
message("Largest possible file size = 0x%lx bytes",Largest_file_size);
/* Calculate the size of the super struct in local blocks */
/* It is the of disk blocks into a D_PHYSBLK */
if ( D_PHYS_SHIFT<=Superblock.s_blksize ) {
/* Super info takes one block or less */
bs_super_size= 1 ;
} else {
/* To be honest, bs_supersize will always be one, because
* D_PHYS_SHIFT seems to be the minimum.
*/
bwarn("Super block takes more then one block") ;
bs_super_size= 1 << (D_PHYS_SHIFT-Superblock.s_blksize) ;
}
/* Super Block Lock */
mu_init(&SuperLock) ;
/* Init group structures */
bs_grp_init() ;
return 1 ;
}
void
inode_init()
{
register Inode * ip;
register Inode * lim;
Res_size numbytes;
Inodenum i;
disk_addr inodeblks;
/*
* We allocate space at the beginning of the cache memory for the in-core
* inode table.
* NOTE WELL: The cache memory must be properly aligned so that
* write_inode works properly. Therefore we round
* up to blocksize.
*/
numbytes = Superblock.s_numinodes * sizeof (Inode);
/* Round up to blocksize */
numbytes += Blksizemin1;
numbytes &= ~Blksizemin1;
inodeblks = numbytes >> Superblock.s_blksize;
Inode_table = (Inode *) a_alloc(A_CACHE_MEM, numbytes);
numbytes = Superblock.s_numinodes * sizeof (mutex);
CEILING_BLKSZ(numbytes);
Inode_lock = (mutex *) a_alloc(A_CACHE_MEM, numbytes);
if (Inode_table == 0 || Inode_lock == 0)
bpanic("Insufficient memory for inode table cache.");
/* Initialize the locks to free */
for (i = 0; i < Superblock.s_numinodes; i++)
mu_init(Inode_lock + i);
#if !defined(USER_LEVEL) && !defined(NDEBUG)
numbytes = sizeof (struct thread *) * Superblock.s_numinodes;
CEILING_BLKSZ(numbytes);
ILKCHK = (struct thread **) a_alloc(A_CACHE_MEM, numbytes);
if (ILKCHK == 0)
bpanic("Insufficient memory for inode cross-check table.");
(void) memset((_VOIDSTAR) ILKCHK, 0, (size_t) numbytes);
#endif
read_inodetab(inodeblks);
lim = Limit = Inode_table + Superblock.s_numinodes;
Last_free = 0;
/*
* Go through the inodes and mark those that are allocated (in use).
* Remember the first free inode.
*/
for (ip = Inode_table + 1; ip < lim; ip++)
{
/*
* Files with the intent flag still on were never written to disk
* so mark them as empty.
*/
if (ip->i_flags & I_INTENT)
ip->i_lifetime = 0;
if (ip->i_lifetime != 0)
ip->i_flags = I_ALLOCED;
else
ip->i_flags = 0;
if (Last_free == 0 && ip->i_flags == 0)
Last_free = ip;
}
/*
* See if there are any inodes.
* If not then we still need to have last_free pointing at the inode table!
*/
if (Last_free == 0)
{
Last_free = Inode_table + 1;
bwarn("no free inodes!\n");
return;
}
}
/*==========================================
* main
*========================================== */
int main(int argc, char* argv[])
{
int c, iter, ITER=0, seed=0;
enum dataType data = LdaC;
enum dataType testdata = LdaC;
int dots = 0;
enum GibbsType fix_hold = GibbsNone;
char *stem;
char *resstem;
int topwords = 20;
int noerrorlog = 0;
int displayed = 0;
int load_vocab = 0;
int checkpoint = 0;
int restart = 0;
int dopmi = 0;
int restart_hca = 0;
int load_phi = 0;
int load_mu = 0;
int procs = 1;
int maxW = 0;
enum ScoreType score=ST_idf;
double BM0val=0, BM1val =0, BP0val=0, BP1val=0;
clock_t t1=0, t2=0, t3=0;
double tot_time = 0;
double psample_time = 0;
enum ParType par;
/*
* default values
*/
ddN.T = 10;
ITER = 100;
ddN.TEST = 0;
pctl_init();
while ( (c=getopt(argc, argv,"b:c:C:d:ef:F:g:G:h:K:l:L:N:o:pq:vr:s:S:t:T:vVW:"))>=0 ) {
switch ( c ) {
case 'b':
if ( !optarg || sscanf(optarg,"%d",&ddP.back)!=1 )
yap_quit("Need a valid 'b' argument\n");
break;
case 'c':
if ( !optarg || sscanf(optarg,"%d",&checkpoint)!=1 )
yap_quit("Need a valid 'c' argument\n");
break;
case 'C':
if ( !optarg || sscanf(optarg,"%d",&ITER)!=1 )
yap_quit("Need a valid 'C' argument\n");
break;
case 'd':
if ( !optarg || sscanf(optarg,"%d",&dots)!=1 )
yap_quit("Need a valid 'd' argument\n");
break;
case 'e':
noerrorlog++;
break;
case 'f':
if ( strcmp(optarg,"witdit")==0 )
data = WitDit;
else if ( strcmp(optarg,"docword")==0 )
data = Docword;
else if ( strcmp(optarg,"ldac")==0 )
data = LdaC;
else if ( strcmp(optarg,"bag")==0 )
data = TxtBag;
else if ( strcmp(optarg,"lst")==0 )
data = SeqTxtBag;
else
yap_quit("Illegal data type for -f\n");
break;
case 'F':
if ( strcmp(optarg,"all")==0 ) {
for (par=ParAM; par<=ParBB; par++)
ddT[par].fix = 1;
} else {
par = findpar(optarg);
if ( par==ParNone )
yap_quit("Illegal arg for -F\n");
ddT[par].fix = 1;
}
break;
case 'g':
{
char var[100];
int st=0;
if ( !optarg || sscanf(optarg,"%[^, ],%d", &var[0], &st)<1 )
yap_quit("Need a valid 'g' argument\n");
par = findpar(var);
if ( par==ParBP1 )
ddP.kbatch = st;
else
yap_quit("Illegal var for -g\n");
}
break;
case 'G':
{
char var[100];
int st=0, cy=0;
if ( !optarg || sscanf(optarg,"%[^, ],%d,%d",
&var[0], &cy, &st)<2 || st<0 || cy<0 )
yap_quit("Need a valid 'G' argument\n");
par = findpar(var);
if ( par==ParNone || par==ParB0P || par==ParB0M )
yap_quit("Illegal var for -G\n");
ddT[par].fix = 0;
ddT[par].start = st;
ddT[par].cycles = cy;
}
break;
case 'h':
{
fix_hold = GibbsHold;
if ( !optarg )
yap_quit("Need a valid 'h' argument\n");
if ( strncmp(optarg,"dict,",5)==0 ) {
if ( sscanf(&optarg[5],"%d",&ddP.hold_dict)<1 || ddP.hold_dict<2 )
yap_quit("Need a valid 'hdict' argument\n");
} else if ( strncmp(optarg,"fract,",6)==0 ) {
if ( sscanf(&optarg[6],"%lf",&ddP.hold_fraction)<1
|| ddP.hold_fraction<=0 || ddP.hold_fraction>=1 )
yap_quit("Need a valid 'hfract' argument\n");
} else if ( strncmp(optarg,"doc,",4)==0 ) {
if ( sscanf(&optarg[4],"%d",&ddP.hold_every)<1 || ddP.hold_every<2 )
yap_quit("Need a valid 'hdoc' argument\n");
} else
yap_quit("Need a valid 'h' argument\n");
}
break;
case 'K':
if ( !optarg || sscanf(optarg,"%d",&ddN.T)!=1 )
yap_quit("Need a valid 'K' argument\n");
break;
case 'l':
if ( !optarg )
yap_quit("Need a valid 'l ' argument\n");
if ( strncmp(optarg,"phi,",4)==0 ) {
if ( sscanf(&optarg[4],"%d,%d",&ddP.phiiter, &ddP.phiburn)<2 )
yap_quit("Need a valid 'l word,' argument\n");
} else if ( strncmp(optarg,"theta,",6)==0 ) {
if ( sscanf(&optarg[6],"%d,%d",&ddP.thetaiter, &ddP.thetaburn)<2 )
yap_quit("Need a valid 'l word,' argument\n");
} else if ( strncmp(optarg,"mu,",3)==0 ) {
if ( sscanf(&optarg[3],"%d,%d",&ddP.muiter, &ddP.muburn)<2 )
yap_quit("Need a valid 'l word,' argument\n");
} else if ( strncmp(optarg,"prog,",5)==0 ) {
if ( sscanf(&optarg[5],"%d,%d",&ddP.progiter, &ddP.progburn)<2 )
yap_quit("Need a valid 'l prog,' argument\n");
} else
yap_quit("Need a valid DIAG code in 'l' argument\n");
break;
case 'L':
if ( !optarg )
yap_quit("Need a valid 'L ' argument\n");
if ( strncmp(optarg,"like,",5)==0 ) {
if ( sscanf(&optarg[5],"%d,%d",&ddP.mltiter, &ddP.mltburn)<1 )
yap_quit("Need a valid 'L like' argument\n");
} else
yap_quit("Need a valid DIAG code in 'L' argument\n");
break;
case 'N':
if ( !optarg || sscanf(optarg,"%d,%d", &ddP.maxN, &ddP.maxM)<1 )
yap_quit("Need a valid 'N' argument\n");
break;
case 'o':
{
char *ptr = strchr(optarg, ',');
int len = strlen(optarg);
if ( ptr )
len = ptr - optarg;
if ( strncmp(optarg,"idf",len)==0 )
score = ST_idf;
else if ( strncmp(optarg,"count",len)==0 )
score = ST_count;
else if ( strncmp(optarg,"Q",len)==0 )
score = ST_Q;
else if ( strncmp(optarg,"cost",len)==0 )
score = ST_cost;
else
yap_quit("Need a valid parameter for 'o' argument\n");
if ( ptr ) {
/* there was a second arg */
if ( sscanf(ptr+1, "%d", &topwords) != 1)
yap_quit("Need a valid second 'o' argument\n");
}
break;
}
break;
case 'p':
dopmi++;
break;
case 'q':
if(!optarg || sscanf(optarg, "%d", &procs) != 1)
yap_quit("Need a valid 'q' argument\n");
break;
case 'r':
if(!optarg )
yap_quit("Need a valid 'r' argument\n");
if ( strcmp(optarg,"tca")==0 )
restart++;
else if ( strcmp(optarg,"hca")==0 )
restart_hca++;
else if ( strcmp(optarg,"phi")==0 )
load_phi++;
else if ( strcmp(optarg,"mu")==0 )
load_mu++;
else
yap_quit("Need a valid 'r' argument\n");
break;
case 's':
if ( !optarg || sscanf(optarg,"%d",&seed)!=1 )
yap_quit("Need a valid 's' argument\n");
break;
case 'S':
{
char var[100];
double vin=0;
if ( !optarg || sscanf(optarg,"%[^=, ]=%lf",
&var[0], &vin)<2 )
yap_quit("Need a valid 'S' argument\n");
par = findpar(var);
if ( par==ParNone )
yap_quit("Illegal var for -S\n");
else if ( par==ParBM0 )
BM0val = vin;
else if ( par==ParBM1 )
BM1val = vin;
else if ( par==ParBP0 )
BP0val = vin;
else if ( par==ParBP1 )
BP1val = vin;
else
*(ddT[par].ptr) = vin;
}
break;
case 't':
if ( !optarg || sscanf(optarg,"%d",&ddP.training)!=1 )
yap_quit("Need a valid 't' argument\n");
break;
case 'T':
if ( !optarg )
yap_quit("Need a valid 'T' argument\n");
{
char *tname = data_name(optarg,data);
FILE *fp = fopen(tname,"r");
if ( fp==NULL ) {
free(tname);
tname = data_name(optarg,testdata);
fp = fopen(tname,"r");
} else {
testdata = data;
}
free(tname);
if ( fp!=NULL ) {
/* its a valid test filename */
ddP.teststem = optarg;
fclose(fp);
} else if ( sscanf(optarg,"%d",&ddN.TEST)!=1 )
yap_quit("Need a valid 'T' argument\n");
}
break;
case 'v':
verbose++;
break;
case 'V':
load_vocab = 1;
break;
case 'W':
if ( !optarg || sscanf(optarg,"%d",&maxW)<1 )
yap_quit("Need a valid 'W' argument\n");
break;
default:
yap_quit("Unknown option '%c'\n", c);
}
}
if (argc-optind != 2) {
usage();
exit(-1);
}
if ( optind>=argc ) {
yap_quit("No arguments given\n");
}
stem = strdup(argv[optind++]);
resstem = strdup(argv[optind++]);
if ( dopmi )
load_vocab = 1;
if ( dopmi && verbose !=2 ) {
/*
* due to the use of the ".top" file
* its really multi-purpose
*/
yap_quit("When computing PMI verbose must be exactly 2\n");
}
if ( noerrorlog==0 ) {
char *wname = yap_makename(resstem, ".log");
yap_file(wname);
free(wname);
}
yap_commandline(argc, argv);
#ifdef H_THREADS
yap_message(" Threads,");
#endif
if ( restart || restart_hca ) {
char *fname = yap_makename(resstem,".par");
FILE *fp = fopen(fname,"r");
char *buf;
if ( !fp )
yap_quit("Parameter file '%s' doesn't exist\n", fname);
fclose(fp);
free(fname);
buf = readpar(resstem,"T",50);
if ( !buf )
yap_quit("Parameter file '%s' has no T\n", fname);
ddN.T = atoi(buf);
free(buf);
if ( restart ) {
buf = readpar(resstem,"E",50);
if ( !buf )
yap_quit("Parameter file '%s' has no E\n", fname);
ddN.E = atoi(buf);
free(buf);
pctl_read(resstem);
}
if ( maxW==0 ) {
buf = readpar(resstem,"W",50);
if ( buf ) {
maxW = atoi(buf);
free(buf);
}
}
if ( ddP.training==0 ) {
buf = readpar(resstem,"TRAIN",50);
if ( buf ) {
ddP.training = atoi(buf);
free(buf);
}
}
if ( ddN.TEST==0 ) {
buf = readpar(resstem,"TEST",50);
if ( buf ) {
ddN.TEST = atoi(buf);
free(buf);
}
}
}
assert(ddN.T>0);
assert(ddN.TEST>=0);
assert(restart || restart_hca || ITER>0);
if ( load_phi && ddP.phiiter>0 )
yap_quit("Options '-l phi,...' and '-r phi' incompatible\n");
if ( load_mu && ddP.muiter>0 )
yap_quit("Options '-l mu,...' and '-r mu' incompatible\n");
/*
* set random number generator
*/
if ( seed ) {
rng_seed(rngp,seed);
} else {
rng_time(rngp,&seed);
}
yap_message("Setting seed = %lu\n", seed);
/*
* read data and get dimensions
*/
{
D_bag_t *dbp = data_read(stem, data);
int training = pctl_training(dbp->D);
if ( ddP.teststem ) {
D_bag_t *dbpt = data_read(ddP.teststem, testdata);
/* need to load a separate test set, strip to bare training */
data_shrink(dbp, training);
ddN.TEST = dbpt->D;
data_append(dbp, dbpt);
free(dbpt->w); free(dbpt->d); free(dbpt);
}
if ( maxW>0 ) {
if ( dbp->W <= maxW )
dbp->W = maxW;
if ( dbp->W > maxW )
data_vocabshrink(dbp, maxW);
}
/*
* transfer into system
*/
ddN.D = dbp->D;
ddN.W = dbp->W;
ddN.N = dbp->N;
ddN.NT = dbp->N;
ddN.DT = training;
ddD.w = dbp->w;
ddD.d = dbp->d;
free(dbp);
if ( ddN.DT<ddN.D ) {
/* recompute NT */
int i;
for (i=0; i<ddN.N; i++)
if ( ddD.d[i]>=ddN.DT )
break;
ddN.NT = i;
}
}
data_read_epoch(stem);
/*
* at this point, dimensions are fixed, so load phi and mu if needed
*/
if ( load_phi )
pctl_loadphi(resstem);
if ( load_mu )
pctl_loadmu(resstem);
/*
* correct parameters after command line
*/
pctl_fix(ITER);
if ( BM0val>0 ) {
ddP.b_mu[0] = BM0val;
}
if ( BM1val>0 ) {
int i;
for (i=1; i<ddN.E; i++)
ddP.b_mu[i] = BM1val;
}
if ( BP0val>0 ) {
int i;
for (i=0; i<ddN.T; i++)
ddP.b_phi[0][i] = BP0val;
}
if ( BP1val>0 ) {
int i;
if ( ddN.E==1 )
yap_quit("b_phi[1] invalid when epochs==1\n");
for (i=0; i<ddN.T; i++)
ddP.b_phi[1][i] = BP1val;
}
pctl_samplereport();
/*
* all data structures
*/
data_alloc();
if ( ddP.phiiter>0 )
phi_init(resstem);
else
ddS.phi = NULL;
if ( ddP.muiter>0 )
mu_init(resstem);
else
ddS.mu = NULL;
if ( ddP.thetaiter>0 )
theta_init(resstem);
else
ddS.theta = NULL;
tca_alloc();
if ( PCTL_BURSTY() )
dmi_init(&ddM, ddS.z, ddD.w, ddD.N_dTcum,
ddN.T, ddN.N, ddN.W, ddN.D, ddN.DT,
(fix_hold==GibbsHold)?pctl_hold:NULL);
if ( load_vocab ) {
data_vocab(stem);
}
cache_init();
/*
* yap some details
*/
data_report(ITER, seed);
pctl_report();
/*
* load/init topic assignments and prepare statistics
*/
if ( restart || restart_hca) {
tca_read_z(resstem, 0, ddN.DT);
tca_rand_z(ddN.T, ddN.DT, ddN.D);
} else {
tca_rand_z(ddN.T, 0, ddN.D);
}
tca_reset_stats(resstem, restart, 0);
if ( (restart || restart_hca ) && ITER )
yap_message("Initial log_2(perp)=%lf\n", -M_LOG2E * likelihood()/ddN.NT);
if ( ITER )
yap_report("cycles: ");
for (iter=0; iter<ITER; iter++) {
int pro;
double thislp = 0;
int thisNd = 0;
int doc;
#ifdef H_THREADS
pthread_t thread[procs];
#endif
D_pargs_p parg[procs];
#ifdef MU_CACHE
mu_side_fact_reinit();
#endif
#ifdef PHI_CACHE
phi_cache_reinit();
#endif
t1 = clock();
/*
* sampling
*/
#ifdef IND_STATS
ddP.doc_ind_stats = u32tri(ddN.T,ddN.E,ddN.E);
ddP.word_ind_stats = u32tri(ddN.T,ddN.E,ddN.E);
#endif
/* a bit complex if no threads! */
doc = 0;
for (pro = 0 ; pro < procs ; pro++){
parg[pro].dots=dots;
parg[pro].procs=procs;
parg[pro].doc = &doc;
#ifndef H_THREADS
sampling_p(&parg[pro]);
#else
if ( procs==1 )
sampling_p(&parg[pro]);
else if( pthread_create(&thread[pro],NULL,sampling_p,(void*) &parg[pro]) != 0){
yap_message("thread failed %d\n",pro+1 );
}
#endif
}
#ifdef H_THREADS
if ( procs>1 ) {
//waiting for threads to finish
for (pro = 0; pro < procs; pro++){
pthread_join(thread[pro], NULL);
}
}
#endif
// getting lp, Nd and clock
for(pro = 0; pro < procs; pro++){
thislp += parg[pro].thislp;
thisNd += parg[pro].thisNd;
tot_time += parg[pro].tot_time;
}
#ifdef H_THREADS
if ( procs>1 )
tca_reset_stats(NULL,1,1);
#endif
/*
* full check
*/
#ifndef NDEBUG
{
int e, d;
check_cp_et();
for (e=0; e<ddN.E; e++)
check_m_vte(e);
for (d=0; d<ddN.DT; d++)
check_n_dt(d);
}
#endif
#ifdef IND_STATS
{
char *fname = yap_makename(resstem,".istats");
FILE *ifp = fopen(fname,"a");
int e1, e2, kk;
fprintf(ifp,"Iteration %d\n", iter);
for (kk=0; kk<ddN.T; kk++) {
fprintf(ifp," Topic %d\n", kk);
for (e1=0; e1<ddN.E; e1++) {
fprintf(ifp," Epoch %d\n ", e1);
for (e2=0; e2<ddN.E; e2++)
fprintf(ifp," %u", (unsigned)ddP.doc_ind_stats[kk][e1][e2]);
fprintf(ifp,"\n ");
for (e2=0; e2<ddN.E; e2++)
fprintf(ifp," %u", (unsigned)ddP.word_ind_stats[kk][e1][e2]);
fprintf(ifp,"\n");
}
}
fclose(ifp);
free(ddP.doc_ind_stats[0][0]); free(ddP.doc_ind_stats[0]);
free(ddP.doc_ind_stats);
free(ddP.word_ind_stats[0][0]); free(ddP.word_ind_stats[0]);
free(ddP.word_ind_stats);
free(fname);
}
#endif
/*
* sample hyperparameters
*/
t3 = clock();
pctl_sample(iter, procs);
/*
* do time calcs here to remove diagnostics+reporting
*/
t2 = clock();
tot_time += (double)(t2 - t1) / CLOCKS_PER_SEC;
psample_time += (double)(t2 - t3) / CLOCKS_PER_SEC;
/*
* progress reports
*/
if ( ( iter>ddP.progburn && (iter%ddP.progiter)==0 ) || iter+1>=ITER ) {
yap_message(" %d\nlog_2(perp)=%lf,%lf", iter,
-M_LOG2E * likelihood()/ddN.NT, -M_LOG2E * thislp/thisNd);
pctl_update(iter);
if ( verbose && iter%10==0 )
yap_probs();
if ( iter>0 && verbose>1 ) {
if ( ddN.tokens ) {
tca_displaytopics(resstem,topwords,score);
displayed++;
}
}
if ( iter+1<ITER ) {
// yap_message("\n");
yap_report("cycles: ");
}
} else {
yap_message(" %d", iter);
if ( verbose>1) yap_message("\n");
}
if ( checkpoint>0 && iter>0 && iter%checkpoint==0 ) {
data_checkpoint(resstem, stem, iter+1);
yap_message(" checkpointed\n");
tca_report(resstem, stem, ITER, procs, fix_hold,
(dopmi&&displayed>0)?1:0);
}
if ( ddP.phiiter>0 && iter>ddP.phiburn && (iter%ddP.phiiter)==0 )
phi_update();
if ( ddP.thetaiter>0 && iter>ddP.thetaburn && (iter%ddP.thetaiter)==0 )
theta_update();
if ( ddP.muiter>0 && iter>ddP.muburn && (iter%ddP.muiter)==0 )
mu_update();
} // over iter
if ( ITER )
yap_report("Finished after %d cycles on average of %lf+%lf(s) per cycle\n",
iter, (tot_time-psample_time)/iter, psample_time/iter);
if ( ( verbose==1 || ((iter+1)%5!=0 && verbose>1) ) ) {
if ( ddN.tokens ) {
tca_displaytopics(resstem,topwords,score);
displayed++;
}
}
yap_probs();
if ( ITER>0 )
data_checkpoint(resstem, stem, ITER);
tca_report(resstem, stem, ITER, procs, fix_hold, (dopmi&&displayed>0)?1:0);
if ( ddP.phiiter>0 )
phi_save(resstem);
if ( ddP.thetaiter>0 )
theta_save(resstem);
if ( ddP.muiter>0 )
mu_save(resstem);
/*
* free
*/
phi_free();
theta_free();
mu_free();
cache_free();
pctl_free();
data_free();
dmi_free(&ddM);
tca_free();
free(stem);
free(resstem);
rng_free(rngp);
return 0;
}
void init_pool_sems ()
{
mu_init (&sleep_till_free_thread);
mu_lock (&sleep_till_free_thread);
}
