/** Removes "." and ".." from a path */
char *getCanonicalPath(const char *path){
char *prefix = NULL;
char *rest = NULL;
char *tmp = NULL;
size_t offset = 0;
char **src = NULL;
List *dst = NULL;
size_t ii = 0;
Buffer *buf = NULL;
char *result = NULL;
if (path != NULL && strlen(path) > 0) {
tmp = strxreplace(astrcpy(path), '\\', '/');
if (isDosPath(tmp) || isUncPath(tmp)) {
if (isDosPath(tmp)) {
prefix = getPathPart(tmp, PATH_DRIVE);
offset = 0;
}
else if (isUncPath(tmp)) {
prefix = getPathPart(tmp, PATH_HOST);
offset = 2;
}
rest = astrcpy(strchr(&tmp[offset], '/'));
}
else {
rest = astrcpy(tmp);
}
src = astrtok(rest, "/");
dst = new_List();
while (src[ii] != NULL) {
if (strxequals(src[ii], "..")) {
List_remove(dst, -1, false);
}
else if (!strxequals(src[ii], ".")) {
List_append(dst, src[ii]);
}
ii++;
}
buf = new_Buffer(0);
if (prefix != NULL) {
Buffer_appendString(buf, prefix);
}
for (ii = 0; ii < List_length(dst); ++ii) {
Buffer_appendString(buf, List_get(dst, ii));
if (ii != (List_length(dst) - 1)) {
Buffer_appendChar(buf, '/');
}
}
result = astrcpy(buf->data);
delete_Buffer(buf);
delete_List(dst, false);
astrtokfree(src);
mu_free(prefix);
mu_free(rest);
mu_free(tmp);
}
return result;
}
/** Joins two paths together, allowing for relative/absolute paths */
char *getCombinedPath(const char *path1, const char *path2) {
char *tmp1 = NULL;
char *tmp2 = NULL;
char *tmp = NULL;
Buffer *buf = NULL;
char *result = NULL;
char *final = NULL;
if (!strxnullorempty(path1) && !strxnullorempty(path2)) {
tmp1 = strxreplace(astrcpy(path1), '\\', '/');
tmp2 = strxreplace(astrcpy(path2), '\\', '/');
if (isDosPath(tmp2) || isUncPath(tmp2)) {
/* Path2 is a fully-qualified DOS/Windows path; return it */
result = astrcpy(tmp2);
}
else {
buf = new_Buffer(0);
if (tmp2[0] == '/') {
/* Path2 is an absolute path. If Path1 is a DOS/Windows or
** UNC path, prepend the drive letter or hostname; otherwise
** return it.
*/
if (isDosPath(tmp1)) {
tmp = getPathPart(tmp1, PATH_DRIVE);
Buffer_appendString(buf, tmp);
Buffer_appendString(buf, tmp2);
mu_free(tmp);
}
else if (isUncPath(tmp1)) {
tmp = getPathPart(tmp1, PATH_HOST);
Buffer_appendString(buf, "//");
Buffer_appendString(buf, tmp);
Buffer_appendString(buf, tmp2);
mu_free(tmp);
}
else {
Buffer_appendString(buf, tmp2);
}
}
else {
/* Simply concatenate the two paths */
Buffer_appendString(buf, tmp1);
if (tmp1[strlen(tmp1) - 1] != '/') {
Buffer_appendChar(buf, '/');
}
Buffer_appendString(buf, tmp2);
}
result = astrcpy(buf->data);
delete_Buffer(buf);
}
mu_free(tmp1);
mu_free(tmp2);
}
/* Remove any "." and ".." in the path */
final = getCanonicalPath(result);
/** Frees the strings and array returned by astrtok() */
void astrtokfree(char **l) {
size_t ii = 0;
if(l != NULL){
while(l[ii] != NULL) mu_free(l[ii++]);
mu_free(l);
}
else{
Logging_warnf("%s(): NULL argument", __FUNCTION__);
}
}
/**
*** \brief Destroys the specified Dict, freeing the memory associated with it
*** \param d The Dict to destroy
*** \param freeData True if the storage for the items in the Dict is to be freed
*** also
**/
void delete_Dict(Dict *d, bool freeData) {
List *l = (List *)d;
Pair *p = NULL;
while (List_length(l) > 0) {
p = (Pair *)List_get(l, 0);
mu_free(p->key);
if (freeData) mu_free(p->value);
List_remove(l, 0, true);
}
}
bool Dict_remove(Dict *d, const char *key, bool freeData) {
bool retVal = false;
size_t idx = 0;
Pair *p = NULL;
if (keyToIndex(d, key, &idx)) {
p = (Pair *)List_get((List *)d, idx);
if(freeData) mu_free(p->value);
mu_free(p->key);
retVal = List_remove((List *)d, idx, true);
}
return retVal;
}
/** Returns a string consisting of a copy of the file part of path.
*** Does not change path. Backslashes are converted into forward slashes.
**/
char *abasename (const char *path) {
char *result = NULL;
char *tmp = NULL;
size_t ii;
if (strxnullorempty(path)) {
result = astrcpy(".");
}
else {
tmp = strxreplace(astrcpy(path), '\\', '/');
if (strchr(tmp, '/') == NULL) {
result = astrcpy(path);
}
else if (strxequals(tmp, "/") ||
strxequals(tmp, ".") ||
strxequals(tmp, "..")
) {
result = astrcpy(tmp);
}
else {
if (tmp[strlen(tmp) - 1] == '/') tmp[strlen(tmp) - 1] = '\0';
ii = strlen(tmp);
while (ii >= 0) {
if (tmp[ii] == '/') {
result = astrmid(tmp, ii + 1, 0);
break;
}
ii--;
}
}
}
mu_free(tmp);
return result;
}
/** Returns a copy of the string with leading and trailing whitespace removed */
char *astrtrim(const char *s){
char *tmp1 = NULL;
char *tmp2 = NULL;
tmp1 = astrltrim(s);
tmp2 = astrrtrim(tmp1);
mu_free(tmp1);
return tmp2;
}
/** Deallocates a TextFile structure, closing any resources */
void delete_TextFile(TextFile *t){
if (t != NULL){
if (t->f != NULL) fclose(t->f);
if (t->b != NULL) delete_Buffer(t->b);
mu_free(t);
}
else {
Logging_warnNullArg(__FUNCTION__);
}
}
/** Returns True if path is of the form //server/share, False otherwise */
bool isUncPath(const char *path) {
char *tmp = NULL;
bool result = false;
if (path != NULL && strlen(path) > 2) {
tmp = strxreplace(astrcpy(path), '\\', '/');
result = ((tmp[0] == '/') && (tmp[1] == '/') ? true : false);
mu_free(tmp);
}
return result;
}
int main(int argc, char *argv[]){
char *currDir = NULL;
char relPath[] = "../doc";
char testPart[] = "/foo/bar/test.txt";
char *filePart = NULL;
char *newDir = NULL;
Logging_setup(argv[0], LOG_TOSTDERR | LOG_LEVELTRACE, NULL);
currDir = getCurrentDirectory();
Logging_infof("Current directory is: \"%s\"", currDir);
newDir = getCombinedPath(currDir, relPath);
Logging_infof("Doc directory is: \"%s\"", newDir);
filePart = getPathPart(testPart, PATH_FILE);
Logging_infof("The file part of \"%s\" with extension is: \"%s\"",
testPart, filePart);
mu_free(filePart);
filePart = getPathPart(testPart, PATH_FILEONLY);
Logging_infof("The file part of \"%s\" without extension is: \"%s\"",
testPart, filePart);
mu_free(filePart);
mu_free(newDir);
mu_free(currDir);
exit(EXIT_SUCCESS);
}
/** Allocates and initialises a new TextFile structure */
TextFile *new_TextFile(const char *fileName){
TextFile *result = NULL;
FILE *f = NULL;
Buffer *b = NULL;
result = (TextFile *)mu_malloc(sizeof(TextFile));
if ((f = fopen(fileName, "rb")) != NULL) {
result->f = f;
b = new_Buffer(132);
result->b = b;
result->state = 0;
}
if (f == NULL) {
mu_free(result);
Logging_fatalf("%s: Cannot open file \"%s\": %s",
__FUNCTION__, fileName, strerror(errno));
}
return result;
}
int main(int argc, char *argv[]){
char *inputDir = NULL;
char *outputDir = NULL;
char *templateDir = NULL;
char *scriptFile = NULL;
bool verboseMode = false;
bool forceMode = false;
int option;
char *currDir = getCurrentDirectory();
char *logFile = NULL;
unsigned long logFlags = LOG_LEVELWARN | LOG_TOSTDERR;
/* Display usage */
if (argc == 1 || strxequalsi(argv[1], "/?") || strxequalsi(argv[1], "-?")) {
usage(argv[0]);
exit(EXIT_SUCCESS);
}
/* Read command-line args */
while ((option = getopt(argc, argv, "fvi:o:t:l:s:")) != -1){
switch (option){
case 'v': verboseMode = true; break;
case 'f': forceMode = true; break;
case 'i': inputDir = getCombinedPath(currDir, optarg); break;
case 'o': outputDir = getCombinedPath(currDir, optarg); break;
case 't': templateDir = getCombinedPath(currDir, optarg); break;
case 'l': logFile = optarg; break;
case 's': scriptFile = optarg; break;
default:
usage(argv[0]);
exit(EXIT_FAILURE);
}
}
/* Initialise logging */
if (verboseMode) logFlags |= LOG_LEVELDEBUG;
if (logFile != NULL) logFlags |= LOG_TOFILE;
Logging_setup(argv[0], logFlags, logFile);
Logging_infof("BILE build: %s", rcsId);
Logging_infof("Verbose mode: %s", (verboseMode ? "on" : "off"));
Logging_infof("Force mode: %s", (forceMode ? "on" : "off"));
checkDir(inputDir);
Logging_infof("Input directory: %s", inputDir);
checkDir(outputDir);
Logging_infof("Output directory: %s", outputDir);
checkDir(templateDir);
Logging_infof("Template directory: %s", templateDir);
if (!strxnullorempty(scriptFile)) {
Logging_infof("Script file: %s", scriptFile);
}
/* Create the publication */
thePublication = new_Publication(inputDir, outputDir, templateDir,
forceMode, verboseMode, scriptFile);
Publication_build(thePublication);
/* Generate the publication */
Publication_generate(thePublication);
/* Clean up */
mu_free(templateDir);
mu_free(outputDir);
mu_free(inputDir);
mu_free(currDir);
exit(EXIT_SUCCESS);
}
/*==========================================
* main
*========================================== */
int main(int argc, char* argv[])
{
int c, iter, ITER=0, seed=0;
enum dataType data = LdaC;
enum dataType testdata = LdaC;
int dots = 0;
enum GibbsType fix_hold = GibbsNone;
char *stem;
char *resstem;
int topwords = 20;
int noerrorlog = 0;
int displayed = 0;
int load_vocab = 0;
int checkpoint = 0;
int restart = 0;
int dopmi = 0;
int restart_hca = 0;
int load_phi = 0;
int load_mu = 0;
int procs = 1;
int maxW = 0;
enum ScoreType score=ST_idf;
double BM0val=0, BM1val =0, BP0val=0, BP1val=0;
clock_t t1=0, t2=0, t3=0;
double tot_time = 0;
double psample_time = 0;
enum ParType par;
/*
* default values
*/
ddN.T = 10;
ITER = 100;
ddN.TEST = 0;
pctl_init();
while ( (c=getopt(argc, argv,"b:c:C:d:ef:F:g:G:h:K:l:L:N:o:pq:vr:s:S:t:T:vVW:"))>=0 ) {
switch ( c ) {
case 'b':
if ( !optarg || sscanf(optarg,"%d",&ddP.back)!=1 )
yap_quit("Need a valid 'b' argument\n");
break;
case 'c':
if ( !optarg || sscanf(optarg,"%d",&checkpoint)!=1 )
yap_quit("Need a valid 'c' argument\n");
break;
case 'C':
if ( !optarg || sscanf(optarg,"%d",&ITER)!=1 )
yap_quit("Need a valid 'C' argument\n");
break;
case 'd':
if ( !optarg || sscanf(optarg,"%d",&dots)!=1 )
yap_quit("Need a valid 'd' argument\n");
break;
case 'e':
noerrorlog++;
break;
case 'f':
if ( strcmp(optarg,"witdit")==0 )
data = WitDit;
else if ( strcmp(optarg,"docword")==0 )
data = Docword;
else if ( strcmp(optarg,"ldac")==0 )
data = LdaC;
else if ( strcmp(optarg,"bag")==0 )
data = TxtBag;
else if ( strcmp(optarg,"lst")==0 )
data = SeqTxtBag;
else
yap_quit("Illegal data type for -f\n");
break;
case 'F':
if ( strcmp(optarg,"all")==0 ) {
for (par=ParAM; par<=ParBB; par++)
ddT[par].fix = 1;
} else {
par = findpar(optarg);
if ( par==ParNone )
yap_quit("Illegal arg for -F\n");
ddT[par].fix = 1;
}
break;
case 'g':
{
char var[100];
int st=0;
if ( !optarg || sscanf(optarg,"%[^, ],%d", &var[0], &st)<1 )
yap_quit("Need a valid 'g' argument\n");
par = findpar(var);
if ( par==ParBP1 )
ddP.kbatch = st;
else
yap_quit("Illegal var for -g\n");
}
break;
case 'G':
{
char var[100];
int st=0, cy=0;
if ( !optarg || sscanf(optarg,"%[^, ],%d,%d",
&var[0], &cy, &st)<2 || st<0 || cy<0 )
yap_quit("Need a valid 'G' argument\n");
par = findpar(var);
if ( par==ParNone || par==ParB0P || par==ParB0M )
yap_quit("Illegal var for -G\n");
ddT[par].fix = 0;
ddT[par].start = st;
ddT[par].cycles = cy;
}
break;
case 'h':
{
fix_hold = GibbsHold;
if ( !optarg )
yap_quit("Need a valid 'h' argument\n");
if ( strncmp(optarg,"dict,",5)==0 ) {
if ( sscanf(&optarg[5],"%d",&ddP.hold_dict)<1 || ddP.hold_dict<2 )
yap_quit("Need a valid 'hdict' argument\n");
} else if ( strncmp(optarg,"fract,",6)==0 ) {
if ( sscanf(&optarg[6],"%lf",&ddP.hold_fraction)<1
|| ddP.hold_fraction<=0 || ddP.hold_fraction>=1 )
yap_quit("Need a valid 'hfract' argument\n");
} else if ( strncmp(optarg,"doc,",4)==0 ) {
if ( sscanf(&optarg[4],"%d",&ddP.hold_every)<1 || ddP.hold_every<2 )
yap_quit("Need a valid 'hdoc' argument\n");
} else
yap_quit("Need a valid 'h' argument\n");
}
break;
case 'K':
if ( !optarg || sscanf(optarg,"%d",&ddN.T)!=1 )
yap_quit("Need a valid 'K' argument\n");
break;
case 'l':
if ( !optarg )
yap_quit("Need a valid 'l ' argument\n");
if ( strncmp(optarg,"phi,",4)==0 ) {
if ( sscanf(&optarg[4],"%d,%d",&ddP.phiiter, &ddP.phiburn)<2 )
yap_quit("Need a valid 'l word,' argument\n");
} else if ( strncmp(optarg,"theta,",6)==0 ) {
if ( sscanf(&optarg[6],"%d,%d",&ddP.thetaiter, &ddP.thetaburn)<2 )
yap_quit("Need a valid 'l word,' argument\n");
} else if ( strncmp(optarg,"mu,",3)==0 ) {
if ( sscanf(&optarg[3],"%d,%d",&ddP.muiter, &ddP.muburn)<2 )
yap_quit("Need a valid 'l word,' argument\n");
} else if ( strncmp(optarg,"prog,",5)==0 ) {
if ( sscanf(&optarg[5],"%d,%d",&ddP.progiter, &ddP.progburn)<2 )
yap_quit("Need a valid 'l prog,' argument\n");
} else
yap_quit("Need a valid DIAG code in 'l' argument\n");
break;
case 'L':
if ( !optarg )
yap_quit("Need a valid 'L ' argument\n");
if ( strncmp(optarg,"like,",5)==0 ) {
if ( sscanf(&optarg[5],"%d,%d",&ddP.mltiter, &ddP.mltburn)<1 )
yap_quit("Need a valid 'L like' argument\n");
} else
yap_quit("Need a valid DIAG code in 'L' argument\n");
break;
case 'N':
if ( !optarg || sscanf(optarg,"%d,%d", &ddP.maxN, &ddP.maxM)<1 )
yap_quit("Need a valid 'N' argument\n");
break;
case 'o':
{
char *ptr = strchr(optarg, ',');
int len = strlen(optarg);
if ( ptr )
len = ptr - optarg;
if ( strncmp(optarg,"idf",len)==0 )
score = ST_idf;
else if ( strncmp(optarg,"count",len)==0 )
score = ST_count;
else if ( strncmp(optarg,"Q",len)==0 )
score = ST_Q;
else if ( strncmp(optarg,"cost",len)==0 )
score = ST_cost;
else
yap_quit("Need a valid parameter for 'o' argument\n");
if ( ptr ) {
/* there was a second arg */
if ( sscanf(ptr+1, "%d", &topwords) != 1)
yap_quit("Need a valid second 'o' argument\n");
}
break;
}
break;
case 'p':
dopmi++;
break;
case 'q':
if(!optarg || sscanf(optarg, "%d", &procs) != 1)
yap_quit("Need a valid 'q' argument\n");
break;
case 'r':
if(!optarg )
yap_quit("Need a valid 'r' argument\n");
if ( strcmp(optarg,"tca")==0 )
restart++;
else if ( strcmp(optarg,"hca")==0 )
restart_hca++;
else if ( strcmp(optarg,"phi")==0 )
load_phi++;
else if ( strcmp(optarg,"mu")==0 )
load_mu++;
else
yap_quit("Need a valid 'r' argument\n");
break;
case 's':
if ( !optarg || sscanf(optarg,"%d",&seed)!=1 )
yap_quit("Need a valid 's' argument\n");
break;
case 'S':
{
char var[100];
double vin=0;
if ( !optarg || sscanf(optarg,"%[^=, ]=%lf",
&var[0], &vin)<2 )
yap_quit("Need a valid 'S' argument\n");
par = findpar(var);
if ( par==ParNone )
yap_quit("Illegal var for -S\n");
else if ( par==ParBM0 )
BM0val = vin;
else if ( par==ParBM1 )
BM1val = vin;
else if ( par==ParBP0 )
BP0val = vin;
else if ( par==ParBP1 )
BP1val = vin;
else
*(ddT[par].ptr) = vin;
}
break;
case 't':
if ( !optarg || sscanf(optarg,"%d",&ddP.training)!=1 )
yap_quit("Need a valid 't' argument\n");
break;
case 'T':
if ( !optarg )
yap_quit("Need a valid 'T' argument\n");
{
char *tname = data_name(optarg,data);
FILE *fp = fopen(tname,"r");
if ( fp==NULL ) {
free(tname);
tname = data_name(optarg,testdata);
fp = fopen(tname,"r");
} else {
testdata = data;
}
free(tname);
if ( fp!=NULL ) {
/* its a valid test filename */
ddP.teststem = optarg;
fclose(fp);
} else if ( sscanf(optarg,"%d",&ddN.TEST)!=1 )
yap_quit("Need a valid 'T' argument\n");
}
break;
case 'v':
verbose++;
break;
case 'V':
load_vocab = 1;
break;
case 'W':
if ( !optarg || sscanf(optarg,"%d",&maxW)<1 )
yap_quit("Need a valid 'W' argument\n");
break;
default:
yap_quit("Unknown option '%c'\n", c);
}
}
if (argc-optind != 2) {
usage();
exit(-1);
}
if ( optind>=argc ) {
yap_quit("No arguments given\n");
}
stem = strdup(argv[optind++]);
resstem = strdup(argv[optind++]);
if ( dopmi )
load_vocab = 1;
if ( dopmi && verbose !=2 ) {
/*
* due to the use of the ".top" file
* its really multi-purpose
*/
yap_quit("When computing PMI verbose must be exactly 2\n");
}
if ( noerrorlog==0 ) {
char *wname = yap_makename(resstem, ".log");
yap_file(wname);
free(wname);
}
yap_commandline(argc, argv);
#ifdef H_THREADS
yap_message(" Threads,");
#endif
if ( restart || restart_hca ) {
char *fname = yap_makename(resstem,".par");
FILE *fp = fopen(fname,"r");
char *buf;
if ( !fp )
yap_quit("Parameter file '%s' doesn't exist\n", fname);
fclose(fp);
free(fname);
buf = readpar(resstem,"T",50);
if ( !buf )
yap_quit("Parameter file '%s' has no T\n", fname);
ddN.T = atoi(buf);
free(buf);
if ( restart ) {
buf = readpar(resstem,"E",50);
if ( !buf )
yap_quit("Parameter file '%s' has no E\n", fname);
ddN.E = atoi(buf);
free(buf);
pctl_read(resstem);
}
if ( maxW==0 ) {
buf = readpar(resstem,"W",50);
if ( buf ) {
maxW = atoi(buf);
free(buf);
}
}
if ( ddP.training==0 ) {
buf = readpar(resstem,"TRAIN",50);
if ( buf ) {
ddP.training = atoi(buf);
free(buf);
}
}
if ( ddN.TEST==0 ) {
buf = readpar(resstem,"TEST",50);
if ( buf ) {
ddN.TEST = atoi(buf);
free(buf);
}
}
}
assert(ddN.T>0);
assert(ddN.TEST>=0);
assert(restart || restart_hca || ITER>0);
if ( load_phi && ddP.phiiter>0 )
yap_quit("Options '-l phi,...' and '-r phi' incompatible\n");
if ( load_mu && ddP.muiter>0 )
yap_quit("Options '-l mu,...' and '-r mu' incompatible\n");
/*
* set random number generator
*/
if ( seed ) {
rng_seed(rngp,seed);
} else {
rng_time(rngp,&seed);
}
yap_message("Setting seed = %lu\n", seed);
/*
* read data and get dimensions
*/
{
D_bag_t *dbp = data_read(stem, data);
int training = pctl_training(dbp->D);
if ( ddP.teststem ) {
D_bag_t *dbpt = data_read(ddP.teststem, testdata);
/* need to load a separate test set, strip to bare training */
data_shrink(dbp, training);
ddN.TEST = dbpt->D;
data_append(dbp, dbpt);
free(dbpt->w); free(dbpt->d); free(dbpt);
}
if ( maxW>0 ) {
if ( dbp->W <= maxW )
dbp->W = maxW;
if ( dbp->W > maxW )
data_vocabshrink(dbp, maxW);
}
/*
* transfer into system
*/
ddN.D = dbp->D;
ddN.W = dbp->W;
ddN.N = dbp->N;
ddN.NT = dbp->N;
ddN.DT = training;
ddD.w = dbp->w;
ddD.d = dbp->d;
free(dbp);
if ( ddN.DT<ddN.D ) {
/* recompute NT */
int i;
for (i=0; i<ddN.N; i++)
if ( ddD.d[i]>=ddN.DT )
break;
ddN.NT = i;
}
}
data_read_epoch(stem);
/*
* at this point, dimensions are fixed, so load phi and mu if needed
*/
if ( load_phi )
pctl_loadphi(resstem);
if ( load_mu )
pctl_loadmu(resstem);
/*
* correct parameters after command line
*/
pctl_fix(ITER);
if ( BM0val>0 ) {
ddP.b_mu[0] = BM0val;
}
if ( BM1val>0 ) {
int i;
for (i=1; i<ddN.E; i++)
ddP.b_mu[i] = BM1val;
}
if ( BP0val>0 ) {
int i;
for (i=0; i<ddN.T; i++)
ddP.b_phi[0][i] = BP0val;
}
if ( BP1val>0 ) {
int i;
if ( ddN.E==1 )
yap_quit("b_phi[1] invalid when epochs==1\n");
for (i=0; i<ddN.T; i++)
ddP.b_phi[1][i] = BP1val;
}
pctl_samplereport();
/*
* all data structures
*/
data_alloc();
if ( ddP.phiiter>0 )
phi_init(resstem);
else
ddS.phi = NULL;
if ( ddP.muiter>0 )
mu_init(resstem);
else
ddS.mu = NULL;
if ( ddP.thetaiter>0 )
theta_init(resstem);
else
ddS.theta = NULL;
tca_alloc();
if ( PCTL_BURSTY() )
dmi_init(&ddM, ddS.z, ddD.w, ddD.N_dTcum,
ddN.T, ddN.N, ddN.W, ddN.D, ddN.DT,
(fix_hold==GibbsHold)?pctl_hold:NULL);
if ( load_vocab ) {
data_vocab(stem);
}
cache_init();
/*
* yap some details
*/
data_report(ITER, seed);
pctl_report();
/*
* load/init topic assignments and prepare statistics
*/
if ( restart || restart_hca) {
tca_read_z(resstem, 0, ddN.DT);
tca_rand_z(ddN.T, ddN.DT, ddN.D);
} else {
tca_rand_z(ddN.T, 0, ddN.D);
}
tca_reset_stats(resstem, restart, 0);
if ( (restart || restart_hca ) && ITER )
yap_message("Initial log_2(perp)=%lf\n", -M_LOG2E * likelihood()/ddN.NT);
if ( ITER )
yap_report("cycles: ");
for (iter=0; iter<ITER; iter++) {
int pro;
double thislp = 0;
int thisNd = 0;
int doc;
#ifdef H_THREADS
pthread_t thread[procs];
#endif
D_pargs_p parg[procs];
#ifdef MU_CACHE
mu_side_fact_reinit();
#endif
#ifdef PHI_CACHE
phi_cache_reinit();
#endif
t1 = clock();
/*
* sampling
*/
#ifdef IND_STATS
ddP.doc_ind_stats = u32tri(ddN.T,ddN.E,ddN.E);
ddP.word_ind_stats = u32tri(ddN.T,ddN.E,ddN.E);
#endif
/* a bit complex if no threads! */
doc = 0;
for (pro = 0 ; pro < procs ; pro++){
parg[pro].dots=dots;
parg[pro].procs=procs;
parg[pro].doc = &doc;
#ifndef H_THREADS
sampling_p(&parg[pro]);
#else
if ( procs==1 )
sampling_p(&parg[pro]);
else if( pthread_create(&thread[pro],NULL,sampling_p,(void*) &parg[pro]) != 0){
yap_message("thread failed %d\n",pro+1 );
}
#endif
}
#ifdef H_THREADS
if ( procs>1 ) {
//waiting for threads to finish
for (pro = 0; pro < procs; pro++){
pthread_join(thread[pro], NULL);
}
}
#endif
// getting lp, Nd and clock
for(pro = 0; pro < procs; pro++){
thislp += parg[pro].thislp;
thisNd += parg[pro].thisNd;
tot_time += parg[pro].tot_time;
}
#ifdef H_THREADS
if ( procs>1 )
tca_reset_stats(NULL,1,1);
#endif
/*
* full check
*/
#ifndef NDEBUG
{
int e, d;
check_cp_et();
for (e=0; e<ddN.E; e++)
check_m_vte(e);
for (d=0; d<ddN.DT; d++)
check_n_dt(d);
}
#endif
#ifdef IND_STATS
{
char *fname = yap_makename(resstem,".istats");
FILE *ifp = fopen(fname,"a");
int e1, e2, kk;
fprintf(ifp,"Iteration %d\n", iter);
for (kk=0; kk<ddN.T; kk++) {
fprintf(ifp," Topic %d\n", kk);
for (e1=0; e1<ddN.E; e1++) {
fprintf(ifp," Epoch %d\n ", e1);
for (e2=0; e2<ddN.E; e2++)
fprintf(ifp," %u", (unsigned)ddP.doc_ind_stats[kk][e1][e2]);
fprintf(ifp,"\n ");
for (e2=0; e2<ddN.E; e2++)
fprintf(ifp," %u", (unsigned)ddP.word_ind_stats[kk][e1][e2]);
fprintf(ifp,"\n");
}
}
fclose(ifp);
free(ddP.doc_ind_stats[0][0]); free(ddP.doc_ind_stats[0]);
free(ddP.doc_ind_stats);
free(ddP.word_ind_stats[0][0]); free(ddP.word_ind_stats[0]);
free(ddP.word_ind_stats);
free(fname);
}
#endif
/*
* sample hyperparameters
*/
t3 = clock();
pctl_sample(iter, procs);
/*
* do time calcs here to remove diagnostics+reporting
*/
t2 = clock();
tot_time += (double)(t2 - t1) / CLOCKS_PER_SEC;
psample_time += (double)(t2 - t3) / CLOCKS_PER_SEC;
/*
* progress reports
*/
if ( ( iter>ddP.progburn && (iter%ddP.progiter)==0 ) || iter+1>=ITER ) {
yap_message(" %d\nlog_2(perp)=%lf,%lf", iter,
-M_LOG2E * likelihood()/ddN.NT, -M_LOG2E * thislp/thisNd);
pctl_update(iter);
if ( verbose && iter%10==0 )
yap_probs();
if ( iter>0 && verbose>1 ) {
if ( ddN.tokens ) {
tca_displaytopics(resstem,topwords,score);
displayed++;
}
}
if ( iter+1<ITER ) {
// yap_message("\n");
yap_report("cycles: ");
}
} else {
yap_message(" %d", iter);
if ( verbose>1) yap_message("\n");
}
if ( checkpoint>0 && iter>0 && iter%checkpoint==0 ) {
data_checkpoint(resstem, stem, iter+1);
yap_message(" checkpointed\n");
tca_report(resstem, stem, ITER, procs, fix_hold,
(dopmi&&displayed>0)?1:0);
}
if ( ddP.phiiter>0 && iter>ddP.phiburn && (iter%ddP.phiiter)==0 )
phi_update();
if ( ddP.thetaiter>0 && iter>ddP.thetaburn && (iter%ddP.thetaiter)==0 )
theta_update();
if ( ddP.muiter>0 && iter>ddP.muburn && (iter%ddP.muiter)==0 )
mu_update();
} // over iter
if ( ITER )
yap_report("Finished after %d cycles on average of %lf+%lf(s) per cycle\n",
iter, (tot_time-psample_time)/iter, psample_time/iter);
if ( ( verbose==1 || ((iter+1)%5!=0 && verbose>1) ) ) {
if ( ddN.tokens ) {
tca_displaytopics(resstem,topwords,score);
displayed++;
}
}
yap_probs();
if ( ITER>0 )
data_checkpoint(resstem, stem, ITER);
tca_report(resstem, stem, ITER, procs, fix_hold, (dopmi&&displayed>0)?1:0);
if ( ddP.phiiter>0 )
phi_save(resstem);
if ( ddP.thetaiter>0 )
theta_save(resstem);
if ( ddP.muiter>0 )
mu_save(resstem);
/*
* free
*/
phi_free();
theta_free();
mu_free();
cache_free();
pctl_free();
data_free();
dmi_free(&ddM);
tca_free();
free(stem);
free(resstem);
rng_free(rngp);
return 0;
}
/** Returns a part of a directory path */
char *getPathPart(const char *path, PathPart part) {
char *result = NULL;
char *tmp1 = NULL;
char *tmp2 = NULL;
size_t pos = 0;
if (path != NULL && strlen(path) > 0) {
tmp1 = strxreplace(astrcpy(path), '\\', '/');
switch(part){
case PATH_HOST:
if (isUncPath(tmp1)) {
tmp2 = &tmp1[2];
if (strxpos(tmp2, '/', &pos)) {
result = astrleft(tmp2, pos);
}
else {
result = astrcpy(tmp2);
}
}
else {
result = astrcpy("");
}
break;
case PATH_DRIVE:
if (isDosPath(tmp1)) {
result = astrleft(tmp1, 2);
}
else {
result = astrcpy("");
}
break;
case PATH_DIR:
if (isUncPath(tmp1) || isDosPath(tmp1)) {
tmp2 = strchr(&tmp1[2], '/');
}
else {
tmp2 = tmp1;
}
if (tmp2 != NULL) {
if (strxrpos(tmp2, '/', &pos)) {
if (pos == 0) {
result = astrcpy(tmp2);
}
else {
result = astrleft(tmp2, pos);
}
}
else {
result = astrcpy("");
}
}
else {
result = astrcpy("");
}
break;
case PATH_FILE:
case PATH_FILEONLY:
if ((tmp2 = strrchr(tmp1, '/')) != NULL) {
result = &tmp2[1];
}
else {
result = tmp1;
}
if (part == PATH_FILE) {
result = astrcpy(result);
}
else if (part == PATH_FILEONLY) {
/* Strip off file extension */
if (strxrpos(result, '.', &pos)) {
if(pos == 0) {
result = astrcpy(result);
}
else {
result = astrleft(result, pos);
}
}
else {
result = astrcpy(result);
}
}
break;
case PATH_EXT:
if ((tmp2 = strrchr(tmp1, '.')) != NULL) {
if (strchr(tmp2, '/') == NULL) {
result = astrcpy(&tmp2[1]);
}
else {
result = astrcpy("");
}
}
else {
result = astrcpy("");
}
break;
default:
result = astrcpy("");
break;
}
mu_free(tmp1);
}
return result;
}
