extern unsigned long dataAvailable;
/* Function prototypes */
unsigned char mp3_VS1053_attach(VOS_HANDLE spi_master_handle, unsigned char XRESET_GPIO_port, unsigned char XRESET_GPIO_pin, unsigned char DREQ_GPIO_port, unsigned char DREQ_GPIO_pin, unsigned char SDI_CS_identifier, unsigned char SCI_CS_identifier);
int mp3_VS1053_detach();
int mp3_VS1053_select_control(void);
int mp3_VS1053_deselect_control(void);
int mp3_VS1053_select_data(void);
int mp3_VS1053_deselect_data(void);
int mp3_VS1053_hard_reset(void);
int mp3_VS1053_soft_reset(void);
int mp3_VS1053_setup(void);
int mp3_VS1053_wait_for_dreq(void);
int mp3_VS1053_write_reg(unsigned char reg_address, unsigned short data);
int mp3_VS1053_read_reg(unsigned char reg_address, unsigned short *data);
unsigned short mp3_VS1053_read_ram(unsigned short ram_address);
unsigned char mp3_VS1053_write(unsigned char *buf, unsigned short num_to_write, unsigned short *num_written);
unsigned char mp3_VS1053_ioctl(void *cb);
void mp3_VS1053_init(unsigned char mp3VS1053DevNum, mp3_VS1053_context_t *mp3Context)
{
// Set up function pointers for our driver
mp3_VS1053_cb.flags = 0;
mp3_VS1053_cb.read = (PF_IO) NULL;
mp3_VS1053_cb.write = mp3_VS1053_write;
mp3_VS1053_cb.ioctl = mp3_VS1053_ioctl;
mp3_VS1053_cb.interrupt = (PF_INT) NULL;
static DDS_ReturnCode_t sequence_get(qeocore_data_t **value,
const DDS_DynamicData dyndata,
qeocore_member_id_t id,
DDS_DynamicType type)
{
DDS_ReturnCode_t ddsrc;
*value = data_alloc(type, 1);
if (NULL == *value) {
ddsrc = DDS_RETCODE_OUT_OF_RESOURCES;
}
else {
DDS_DynamicData seqdata;
ddsrc = DDS_DynamicData_get_complex_value(dyndata, &seqdata, id);
if (DDS_RETCODE_OK == ddsrc) {
/* actual data was returned */
DDS_DynamicDataFactory_delete_data((*value)->d.dynamic.single_data);
(*value)->d.dynamic.single_data = seqdata;
}
/* in case of DDS_RETCODE_NO_DATA we keep the preallocated data */
else if (DDS_RETCODE_NO_DATA != ddsrc) {
qeo_log_dds_rc("DDS_DynamicData_get_complex_value", ddsrc);
}
}
return ddsrc;
}
static int yield_bench_parse_results(struct plugin *plug)
{
struct yield_bench_data *data = plugin_get_data(plug);
uint64_t abs_time;
uint64_t should_time;
uint64_t loops = 0;
struct data *result;
int i;
abs_time = (data->end.tv_sec - data->start.tv_sec) * 1000000
+ (data->end.tv_nsec - data->start.tv_nsec) / 1000;
should_time = data->seconds * 1000000;
for (i = 0; i != data->nr_threads; ++i) {
loops += data->threads[i].loops;
}
loops = (loops * should_time) / abs_time;
result = data_alloc(DATA_TYPE_RESULT, 1);
data_add_int64(result, loops);
plugin_add_results(plug, result);
return 0;
}
void asf::parameter_float_image::pup(PUP::er &p) {
parameter_pixel_image::pup(p);
// Allocate pixel data (if needed)
if (p.isUnpacking()) data_alloc(src_alloc,0,bands(),bands()*pixel().size_x());
// Pup all pixel data (FIXME: speed up contiguous case?)
ASF_FOR_PIXELS(x,y,pixels())
p(&at(x,y,0),bands());
}
static int plug_parse_results(struct plugin *plug)
{
struct data *data = data_alloc(DATA_TYPE_RESULT, 1);
data_add_float(data, 42.0);
plugin_add_results(plug, data);
printf("%s\n", __func__);
return 0;
}
/// Allow access to these pixels.
void asf::parameter_float_image::pixel_setsize(const pixel_rectangle &pixel_meta,int zoom_meta)
{
super::pixel_setsize(pixel_meta,zoom_meta);
int new_dx=bands(), new_dy=bands()*pixel().size_x();
if ((data!=0) && (new_dx==xd) && (new_dy==yd))
{ /* then re-use existing buffer--it's still OK. */ }
else { /* Have to allocate a new buffer */
data_alloc(src_alloc,0,new_dx,new_dy);
}
}
static void
testBasic (long numVar, long numRecord, long numMaxParent, long percentParent)
{
random_t* randomPtr = random_alloc();
puts("Starting...");
data_t* dataPtr = data_alloc(numVar, numRecord, randomPtr);
assert(dataPtr);
puts("Init:");
data_generate(dataPtr, 0, numMaxParent, percentParent);
long v;
for (v = 0; v < numVar; v++) {
data_sort(dataPtr, 0, numRecord, v);
long s = data_findSplit(dataPtr, 0, numRecord, v);
if (s < numRecord) {
assert(dataPtr->records[numVar * s + v] == 1);
}
if (s > 0) {
assert(dataPtr->records[numVar * (s - 1) + v] == 0);
}
}
memset(dataPtr->records, 0, dataPtr->numVar * dataPtr->numRecord);
for (v = 0; v < numVar; v++) {
data_sort(dataPtr, 0, numRecord, v);
long s = data_findSplit(dataPtr, 0, numRecord, v);
if (s < numRecord) {
assert(dataPtr->records[numVar * s + v] == 1);
}
if (s > 0) {
assert(dataPtr->records[numVar * (s - 1) + v] == 0);
}
assert(s == numRecord);
}
memset(dataPtr->records, 1, dataPtr->numVar * dataPtr->numRecord);
for (v = 0; v < numVar; v++) {
data_sort(dataPtr, 0, numRecord, v);
long s = data_findSplit(dataPtr, 0, numRecord, v);
if (s < numRecord) {
assert(dataPtr->records[numVar * s + v] == 1);
}
if (s > 0) {
assert(dataPtr->records[numVar * (s - 1) + v] == 0);
}
assert(s == 0);
}
data_free(dataPtr);
}
/* executing task for second thread */
void *thread2_run( void *arg ) {
if ( data == NULL ) {
if ( (data = data_alloc()) == NULL ) {
return (void *) (-1);
}
}
data_hold( data );
data->number = 2222;
printf( "data->number is %d\n", data->number );
data_release( data );
return (void *) 0;
}
static void
test (long numVar, long numRecord)
{
random_t* randomPtr = random_alloc();
data_t* dataPtr = data_alloc(numVar, numRecord, randomPtr);
assert(dataPtr);
data_generate(dataPtr, 0, 10, 10);
if (global_doPrint) {
printData(dataPtr);
}
data_t* copyDataPtr = data_alloc(numVar, numRecord, randomPtr);
assert(copyDataPtr);
data_copy(copyDataPtr, dataPtr);
adtree_t* adtreePtr = adtree_alloc();
assert(adtreePtr);
TIMER_T start;
TIMER_READ(start);
adtree_make(adtreePtr, copyDataPtr);
TIMER_T stop;
TIMER_READ(stop);
printf("%lf\n", TIMER_DIFF_SECONDS(start, stop));
if (global_doPrint) {
printAdtree(adtreePtr);
}
testCounts(adtreePtr, dataPtr);
adtree_free(adtreePtr);
random_free(randomPtr);
data_free(dataPtr);
}
static int monitor_schedstat_mon(struct plugin *plug)
{
struct monitor_schedstat_data *d = plugin_get_data(plug);
struct monitor_schedstat schedstat;
struct data *dat;
int ret;
struct timespec time;
double now;
clock_gettime(CLOCK_MONOTONIC, &time);
ret = monitor_schedstat_get(plug, &schedstat);
if (ret)
return ret;
dat = data_alloc(DATA_TYPE_MONITOR, 5);
if (!dat)
return -1;
now = time.tv_sec + time.tv_nsec / 1000000000.0;
if (d->first) {
d->initial = schedstat;
d->start_time = now;
d->first = 0;
}
now -= d->start_time;
schedstat.yields -= d->initial.yields;
schedstat.schedules -= d->initial.schedules;
schedstat.schedules_idle -= d->initial.schedules_idle;
schedstat.ttwus -= d->initial.ttwus;
data_add_double(dat, now);
data_add_int64(dat, schedstat.yields);
data_add_int64(dat, schedstat.schedules);
data_add_int64(dat, schedstat.schedules_idle);
data_add_int64(dat, schedstat.ttwus);
plugin_add_results(plug, dat);
return 0;
}
static int hackbench_parse_results(struct plugin *plug)
{
struct hackbench_data *d = plugin_get_data(plug);
struct data *data = data_alloc(DATA_TYPE_RESULT, 1);
double res;
char *c;
c = strrchr(d->result, ':');
if (!c)
return -1;
res = atof(c+2);
data_add_double(data, res);
plugin_add_results(plug, data);
free(d->result);
d->result = NULL;
return 0;
}
/* =============================================================================
* score
* =============================================================================
*/
static float
score (net_t* netPtr, adtree_t* adtreePtr)
{
/*
* Create dummy data structures to conform to learner_score assumptions
*/
data_t* dataPtr = data_alloc(1, 1, NULL);
learner_t* learnerPtr = learner_alloc(dataPtr, adtreePtr, 1);
net_t* tmpNetPtr = learnerPtr->netPtr;
learnerPtr->netPtr = netPtr;
float score = learner_score(learnerPtr);
learnerPtr->netPtr = tmpNetPtr;
learner_free(learnerPtr);
data_free(dataPtr);
return score;
}
void asf::parameter_float_image::pixel_pointat(parameter_pixel_image *src_p,const pixel_rectangle &pixel_meta,int zoom_meta)
{
parameter_float_image *src=dynamic_cast<parameter_float_image *>(src_p);
if (src==NULL) {
asf::die("asf::parameter_float_image::pixel_pointat called with a non-float image. You can only point at an image of the same type!\n");
}
if (!src->pixel_meta_bounds().contains(pixel_meta)) {
printf("Source image pixel bounds: "); asf::print(src->pixel_meta_bounds());
printf("Requested pixel bounds: "); asf::print(pixel_meta);
asf::die("asf::parameter_float_image::pixel_pointat asked to point at pixels not inside source image!\n");
}
super::pixel_setsize(pixel_meta,zoom_meta);
set_bands(src->bands());
/// Check the zoom factor
int rz=pixel().zoom()-src->pixel().zoom(); /* relative zoom from alloc to pixel */
if (rz<0) asf::die("asf::parameter_float_image::pixel_pointat zoom factor cannot be less than the source image's zoom!\n");
/// Figure out our pixel region's topleft in source coordinates
pixel_location src_tl=src->pixel_from_meta(asf::meta_pixel_location(pixel().source_rect().get_min()));
data_alloc(src_ext, &src->at(src_tl),src->xd<<rz,src->yd<<rz);
}
static void
testAll (long numVar, long numRecord, long numMaxParent, long percentParent)
{
random_t* randomPtr = random_alloc();
puts("Starting...");
data_t* dataPtr = data_alloc(numVar, numRecord, randomPtr);
assert(dataPtr);
puts("Init:");
net_t* netPtr = data_generate(dataPtr, 0, numMaxParent, percentParent);
net_free(netPtr);
printRecords(dataPtr);
puts("Sort first half from 0:");
data_sort(dataPtr, 0, numRecord/2, 0);
printRecords(dataPtr);
puts("Sort second half from 0:");
data_sort(dataPtr, numRecord/2, numRecord-numRecord/2, 0);
printRecords(dataPtr);
puts("Sort all from mid:");
data_sort(dataPtr, 0, numRecord, numVar/2);
printRecords(dataPtr);
long split = data_findSplit(dataPtr, 0, numRecord, numVar/2);
printf("Split = %li\n", split);
long v;
for (v = 0; v < numVar; v++) {
data_sort(dataPtr, 0, numRecord, v);
long s = data_findSplit(dataPtr, 0, numRecord, v);
if (s < numRecord) {
assert(dataPtr->records[numVar * s + v] == 1);
}
if (s > 0) {
assert(dataPtr->records[numVar * (s - 1) + v] == 0);
}
}
memset(dataPtr->records, 0, dataPtr->numVar * dataPtr->numRecord);
for (v = 0; v < numVar; v++) {
data_sort(dataPtr, 0, numRecord, v);
long s = data_findSplit(dataPtr, 0, numRecord, v);
if (s < numRecord) {
assert(dataPtr->records[numVar * s + v] == 1);
}
if (s > 0) {
assert(dataPtr->records[numVar * (s - 1) + v] == 0);
}
assert(s == numRecord);
}
memset(dataPtr->records, 1, dataPtr->numVar * dataPtr->numRecord);
for (v = 0; v < numVar; v++) {
data_sort(dataPtr, 0, numRecord, v);
long s = data_findSplit(dataPtr, 0, numRecord, v);
if (s < numRecord) {
assert(dataPtr->records[numVar * s + v] == 1);
}
if (s > 0) {
assert(dataPtr->records[numVar * (s - 1) + v] == 0);
}
assert(s == 0);
}
data_free(dataPtr);
}
/* =============================================================================
* main
* =============================================================================
*/
MAIN(argc, argv)
{
GOTO_REAL();
/*
* Initialization
*/
parseArgs(argc, (char** const)argv);
long numThread = global_params[PARAM_THREAD];
long numVar = global_params[PARAM_VAR];
long numRecord = global_params[PARAM_RECORD];
long randomSeed = global_params[PARAM_SEED];
long maxNumParent = global_params[PARAM_NUMBER];
long percentParent = global_params[PARAM_PERCENT];
global_insertPenalty = global_params[PARAM_INSERT];
global_maxNumEdgeLearned = global_params[PARAM_EDGE];
SIM_GET_NUM_CPU(numThread);
TM_STARTUP(numThread);
P_MEMORY_STARTUP(numThread);
thread_startup(numThread);
printf("Random seed = %li\n", randomSeed);
printf("Number of vars = %li\n", numVar);
printf("Number of records = %li\n", numRecord);
printf("Max num parents = %li\n", maxNumParent);
printf("%% chance of parent = %li\n", percentParent);
printf("Insert penalty = %li\n", global_insertPenalty);
printf("Max num edge learned / var = %li\n", global_maxNumEdgeLearned);
printf("Operation quality factor = %f\n", global_operationQualityFactor);
fflush(stdout);
/*
* Generate data
*/
printf("Generating data... ");
fflush(stdout);
random_t* randomPtr = random_alloc();
assert(randomPtr);
random_seed(randomPtr, randomSeed);
data_t* dataPtr = data_alloc(numVar, numRecord, randomPtr);
assert(dataPtr);
net_t* netPtr = data_generate(dataPtr, -1, maxNumParent, percentParent);
puts("done.");
fflush(stdout);
/*
* Generate adtree
*/
adtree_t* adtreePtr = adtree_alloc();
assert(adtreePtr);
printf("Generating adtree... ");
fflush(stdout);
TIMER_T adtreeStartTime;
TIMER_READ(adtreeStartTime);
adtree_make(adtreePtr, dataPtr);
TIMER_T adtreeStopTime;
TIMER_READ(adtreeStopTime);
puts("done.");
fflush(stdout);
printf("Adtree time = %f\n",
TIMER_DIFF_SECONDS(adtreeStartTime, adtreeStopTime));
fflush(stdout);
/*
* Score original network
*/
float actualScore = score(netPtr, adtreePtr);
net_free(netPtr);
/*
* Learn structure of Bayesian network
*/
learner_t* learnerPtr = learner_alloc(dataPtr, adtreePtr, numThread);
assert(learnerPtr);
data_free(dataPtr); /* save memory */
printf("Learning structure...");
fflush(stdout);
TIMER_T learnStartTime;
TIMER_READ(learnStartTime);
GOTO_SIM();
learner_run(learnerPtr);
GOTO_REAL();
TIMER_T learnStopTime;
TIMER_READ(learnStopTime);
puts("done.");
fflush(stdout);
printf("Time = %f\n",
TIMER_DIFF_SECONDS(learnStartTime, learnStopTime));
fflush(stdout);
/*
* Check solution
*/
bool_t status = net_isCycle(learnerPtr->netPtr);
assert(!status);
#ifndef SIMULATOR
float learnScore = learner_score(learnerPtr);
printf("Learn score = %f\n", learnScore);
#endif
printf("Actual score = %f\n", actualScore);
/*
* Clean up
*/
fflush(stdout);
random_free(randomPtr);
#ifndef SIMULATOR
adtree_free(adtreePtr);
# if 0
learner_free(learnerPtr);
# endif
#endif
TM_SHUTDOWN();
P_MEMORY_SHUTDOWN();
GOTO_SIM();
thread_shutdown();
MAIN_RETURN(0);
}
void mtclim42_init(int have_dewpt, double elevation, double annual_prcp,
double lat, global_param_struct *vic_global, dmy_struct *dmy,
double *prec, double *tmax, double *tmin, double *hourlyrad,
double *tiny_radfract, control_struct *ctrl,
parameter_struct *p, data_struct *mtclim42_data)
{
int i;
int stepspday;
long tinystepspyear;
/* initialize the control structure */
ctrl->ndays = (vic_global->nrecs*vic_global->dt)/24;
if (have_dewpt)
nrerror("have_dewpt not yet implemented...\n");
else
ctrl->indewpt = 0;
ctrl->outhum = 1; /* output vapor pressure */
ctrl->inyear = 0;
/* initialize the parameter structure. Meteorological variables are only
calculated for the mean grid cell elevation. The temperatures are lapsed
outside of the mtclim code. Therefore p->base_elev and p->site_elev are
set to the same value. The same is true for p->base_isoh and
p->site_isoh. */
p->base_elev = elevation;
p->base_isoh = annual_prcp/10.; /* MTCLIM prcp in cm */
p->site_lat = lat;
p->site_elev = elevation;
p->site_slp = 0.;
p->site_asp = 0.;
p->site_isoh = annual_prcp/10.; /* MTCLIM prcp in cm */
p->site_ehoriz = 0.;
p->site_whoriz = 0.;
p->tmax_lr = T_lapse; /* not used since site_elev == base_elev */
p->tmin_lr = T_lapse; /* not used since site_elev == base_elev */
/* allocate space in the data arrays for input and output data */
if (data_alloc(ctrl, mtclim42_data)) {
nrerror("Error in data_alloc()... exiting\n");
}
/* First populate the solar day array with the tmin, tmax, prcp and possibly
the Tdew values for the corresponding local days. At
this point we will not worry about the first and last incomplete solar days
(if there are any). It could be argued that the mtclim method does not
make all that much sense if you only have data for one or two days
anyway. */
/* in this first version we just take care of the daily data, subdaily data
will be implemented later */
/* initialize the data arrays with the vic input data */
stepspday = 24/vic_global->dt;
for (i = 0; i < ctrl->ndays; i++) {
mtclim42_data->yday[i] = dmy[i*stepspday].day_in_year;
mtclim42_data->tmax[i] = tmax[i];
mtclim42_data->tmin[i] = tmin[i];
/* MTCLIM prcp in cm */
mtclim42_data->prcp[i] = prec[i]/10.;
if (have_dewpt)
nrerror("have_dewpt not yet implemented ...\n");
}
tinystepspyear = 366L*(86400L/(long)SRADDT);
for (i = 0; i < tinystepspyear; i++)
tiny_radfract[i] = 0;
}
/*==========================================
* main
*========================================== */
int main(int argc, char* argv[])
{
int c, iter, ITER=0, seed=0;
enum dataType data = LdaC;
enum dataType testdata = LdaC;
int dots = 0;
enum GibbsType fix_hold = GibbsNone;
char *stem;
char *resstem;
int topwords = 20;
int noerrorlog = 0;
int displayed = 0;
int load_vocab = 0;
int checkpoint = 0;
int restart = 0;
int dopmi = 0;
int restart_hca = 0;
int load_phi = 0;
int load_mu = 0;
int procs = 1;
int maxW = 0;
enum ScoreType score=ST_idf;
double BM0val=0, BM1val =0, BP0val=0, BP1val=0;
clock_t t1=0, t2=0, t3=0;
double tot_time = 0;
double psample_time = 0;
enum ParType par;
/*
* default values
*/
ddN.T = 10;
ITER = 100;
ddN.TEST = 0;
pctl_init();
while ( (c=getopt(argc, argv,"b:c:C:d:ef:F:g:G:h:K:l:L:N:o:pq:vr:s:S:t:T:vVW:"))>=0 ) {
switch ( c ) {
case 'b':
if ( !optarg || sscanf(optarg,"%d",&ddP.back)!=1 )
yap_quit("Need a valid 'b' argument\n");
break;
case 'c':
if ( !optarg || sscanf(optarg,"%d",&checkpoint)!=1 )
yap_quit("Need a valid 'c' argument\n");
break;
case 'C':
if ( !optarg || sscanf(optarg,"%d",&ITER)!=1 )
yap_quit("Need a valid 'C' argument\n");
break;
case 'd':
if ( !optarg || sscanf(optarg,"%d",&dots)!=1 )
yap_quit("Need a valid 'd' argument\n");
break;
case 'e':
noerrorlog++;
break;
case 'f':
if ( strcmp(optarg,"witdit")==0 )
data = WitDit;
else if ( strcmp(optarg,"docword")==0 )
data = Docword;
else if ( strcmp(optarg,"ldac")==0 )
data = LdaC;
else if ( strcmp(optarg,"bag")==0 )
data = TxtBag;
else if ( strcmp(optarg,"lst")==0 )
data = SeqTxtBag;
else
yap_quit("Illegal data type for -f\n");
break;
case 'F':
if ( strcmp(optarg,"all")==0 ) {
for (par=ParAM; par<=ParBB; par++)
ddT[par].fix = 1;
} else {
par = findpar(optarg);
if ( par==ParNone )
yap_quit("Illegal arg for -F\n");
ddT[par].fix = 1;
}
break;
case 'g':
{
char var[100];
int st=0;
if ( !optarg || sscanf(optarg,"%[^, ],%d", &var[0], &st)<1 )
yap_quit("Need a valid 'g' argument\n");
par = findpar(var);
if ( par==ParBP1 )
ddP.kbatch = st;
else
yap_quit("Illegal var for -g\n");
}
break;
case 'G':
{
char var[100];
int st=0, cy=0;
if ( !optarg || sscanf(optarg,"%[^, ],%d,%d",
&var[0], &cy, &st)<2 || st<0 || cy<0 )
yap_quit("Need a valid 'G' argument\n");
par = findpar(var);
if ( par==ParNone || par==ParB0P || par==ParB0M )
yap_quit("Illegal var for -G\n");
ddT[par].fix = 0;
ddT[par].start = st;
ddT[par].cycles = cy;
}
break;
case 'h':
{
fix_hold = GibbsHold;
if ( !optarg )
yap_quit("Need a valid 'h' argument\n");
if ( strncmp(optarg,"dict,",5)==0 ) {
if ( sscanf(&optarg[5],"%d",&ddP.hold_dict)<1 || ddP.hold_dict<2 )
yap_quit("Need a valid 'hdict' argument\n");
} else if ( strncmp(optarg,"fract,",6)==0 ) {
if ( sscanf(&optarg[6],"%lf",&ddP.hold_fraction)<1
|| ddP.hold_fraction<=0 || ddP.hold_fraction>=1 )
yap_quit("Need a valid 'hfract' argument\n");
} else if ( strncmp(optarg,"doc,",4)==0 ) {
if ( sscanf(&optarg[4],"%d",&ddP.hold_every)<1 || ddP.hold_every<2 )
yap_quit("Need a valid 'hdoc' argument\n");
} else
yap_quit("Need a valid 'h' argument\n");
}
break;
case 'K':
if ( !optarg || sscanf(optarg,"%d",&ddN.T)!=1 )
yap_quit("Need a valid 'K' argument\n");
break;
case 'l':
if ( !optarg )
yap_quit("Need a valid 'l ' argument\n");
if ( strncmp(optarg,"phi,",4)==0 ) {
if ( sscanf(&optarg[4],"%d,%d",&ddP.phiiter, &ddP.phiburn)<2 )
yap_quit("Need a valid 'l word,' argument\n");
} else if ( strncmp(optarg,"theta,",6)==0 ) {
if ( sscanf(&optarg[6],"%d,%d",&ddP.thetaiter, &ddP.thetaburn)<2 )
yap_quit("Need a valid 'l word,' argument\n");
} else if ( strncmp(optarg,"mu,",3)==0 ) {
if ( sscanf(&optarg[3],"%d,%d",&ddP.muiter, &ddP.muburn)<2 )
yap_quit("Need a valid 'l word,' argument\n");
} else if ( strncmp(optarg,"prog,",5)==0 ) {
if ( sscanf(&optarg[5],"%d,%d",&ddP.progiter, &ddP.progburn)<2 )
yap_quit("Need a valid 'l prog,' argument\n");
} else
yap_quit("Need a valid DIAG code in 'l' argument\n");
break;
case 'L':
if ( !optarg )
yap_quit("Need a valid 'L ' argument\n");
if ( strncmp(optarg,"like,",5)==0 ) {
if ( sscanf(&optarg[5],"%d,%d",&ddP.mltiter, &ddP.mltburn)<1 )
yap_quit("Need a valid 'L like' argument\n");
} else
yap_quit("Need a valid DIAG code in 'L' argument\n");
break;
case 'N':
if ( !optarg || sscanf(optarg,"%d,%d", &ddP.maxN, &ddP.maxM)<1 )
yap_quit("Need a valid 'N' argument\n");
break;
case 'o':
{
char *ptr = strchr(optarg, ',');
int len = strlen(optarg);
if ( ptr )
len = ptr - optarg;
if ( strncmp(optarg,"idf",len)==0 )
score = ST_idf;
else if ( strncmp(optarg,"count",len)==0 )
score = ST_count;
else if ( strncmp(optarg,"Q",len)==0 )
score = ST_Q;
else if ( strncmp(optarg,"cost",len)==0 )
score = ST_cost;
else
yap_quit("Need a valid parameter for 'o' argument\n");
if ( ptr ) {
/* there was a second arg */
if ( sscanf(ptr+1, "%d", &topwords) != 1)
yap_quit("Need a valid second 'o' argument\n");
}
break;
}
break;
case 'p':
dopmi++;
break;
case 'q':
if(!optarg || sscanf(optarg, "%d", &procs) != 1)
yap_quit("Need a valid 'q' argument\n");
break;
case 'r':
if(!optarg )
yap_quit("Need a valid 'r' argument\n");
if ( strcmp(optarg,"tca")==0 )
restart++;
else if ( strcmp(optarg,"hca")==0 )
restart_hca++;
else if ( strcmp(optarg,"phi")==0 )
load_phi++;
else if ( strcmp(optarg,"mu")==0 )
load_mu++;
else
yap_quit("Need a valid 'r' argument\n");
break;
case 's':
if ( !optarg || sscanf(optarg,"%d",&seed)!=1 )
yap_quit("Need a valid 's' argument\n");
break;
case 'S':
{
char var[100];
double vin=0;
if ( !optarg || sscanf(optarg,"%[^=, ]=%lf",
&var[0], &vin)<2 )
yap_quit("Need a valid 'S' argument\n");
par = findpar(var);
if ( par==ParNone )
yap_quit("Illegal var for -S\n");
else if ( par==ParBM0 )
BM0val = vin;
else if ( par==ParBM1 )
BM1val = vin;
else if ( par==ParBP0 )
BP0val = vin;
else if ( par==ParBP1 )
BP1val = vin;
else
*(ddT[par].ptr) = vin;
}
break;
case 't':
if ( !optarg || sscanf(optarg,"%d",&ddP.training)!=1 )
yap_quit("Need a valid 't' argument\n");
break;
case 'T':
if ( !optarg )
yap_quit("Need a valid 'T' argument\n");
{
char *tname = data_name(optarg,data);
FILE *fp = fopen(tname,"r");
if ( fp==NULL ) {
free(tname);
tname = data_name(optarg,testdata);
fp = fopen(tname,"r");
} else {
testdata = data;
}
free(tname);
if ( fp!=NULL ) {
/* its a valid test filename */
ddP.teststem = optarg;
fclose(fp);
} else if ( sscanf(optarg,"%d",&ddN.TEST)!=1 )
yap_quit("Need a valid 'T' argument\n");
}
break;
case 'v':
verbose++;
break;
case 'V':
load_vocab = 1;
break;
case 'W':
if ( !optarg || sscanf(optarg,"%d",&maxW)<1 )
yap_quit("Need a valid 'W' argument\n");
break;
default:
yap_quit("Unknown option '%c'\n", c);
}
}
if (argc-optind != 2) {
usage();
exit(-1);
}
if ( optind>=argc ) {
yap_quit("No arguments given\n");
}
stem = strdup(argv[optind++]);
resstem = strdup(argv[optind++]);
if ( dopmi )
load_vocab = 1;
if ( dopmi && verbose !=2 ) {
/*
* due to the use of the ".top" file
* its really multi-purpose
*/
yap_quit("When computing PMI verbose must be exactly 2\n");
}
if ( noerrorlog==0 ) {
char *wname = yap_makename(resstem, ".log");
yap_file(wname);
free(wname);
}
yap_commandline(argc, argv);
#ifdef H_THREADS
yap_message(" Threads,");
#endif
if ( restart || restart_hca ) {
char *fname = yap_makename(resstem,".par");
FILE *fp = fopen(fname,"r");
char *buf;
if ( !fp )
yap_quit("Parameter file '%s' doesn't exist\n", fname);
fclose(fp);
free(fname);
buf = readpar(resstem,"T",50);
if ( !buf )
yap_quit("Parameter file '%s' has no T\n", fname);
ddN.T = atoi(buf);
free(buf);
if ( restart ) {
buf = readpar(resstem,"E",50);
if ( !buf )
yap_quit("Parameter file '%s' has no E\n", fname);
ddN.E = atoi(buf);
free(buf);
pctl_read(resstem);
}
if ( maxW==0 ) {
buf = readpar(resstem,"W",50);
if ( buf ) {
maxW = atoi(buf);
free(buf);
}
}
if ( ddP.training==0 ) {
buf = readpar(resstem,"TRAIN",50);
if ( buf ) {
ddP.training = atoi(buf);
free(buf);
}
}
if ( ddN.TEST==0 ) {
buf = readpar(resstem,"TEST",50);
if ( buf ) {
ddN.TEST = atoi(buf);
free(buf);
}
}
}
assert(ddN.T>0);
assert(ddN.TEST>=0);
assert(restart || restart_hca || ITER>0);
if ( load_phi && ddP.phiiter>0 )
yap_quit("Options '-l phi,...' and '-r phi' incompatible\n");
if ( load_mu && ddP.muiter>0 )
yap_quit("Options '-l mu,...' and '-r mu' incompatible\n");
/*
* set random number generator
*/
if ( seed ) {
rng_seed(rngp,seed);
} else {
rng_time(rngp,&seed);
}
yap_message("Setting seed = %lu\n", seed);
/*
* read data and get dimensions
*/
{
D_bag_t *dbp = data_read(stem, data);
int training = pctl_training(dbp->D);
if ( ddP.teststem ) {
D_bag_t *dbpt = data_read(ddP.teststem, testdata);
/* need to load a separate test set, strip to bare training */
data_shrink(dbp, training);
ddN.TEST = dbpt->D;
data_append(dbp, dbpt);
free(dbpt->w); free(dbpt->d); free(dbpt);
}
if ( maxW>0 ) {
if ( dbp->W <= maxW )
dbp->W = maxW;
if ( dbp->W > maxW )
data_vocabshrink(dbp, maxW);
}
/*
* transfer into system
*/
ddN.D = dbp->D;
ddN.W = dbp->W;
ddN.N = dbp->N;
ddN.NT = dbp->N;
ddN.DT = training;
ddD.w = dbp->w;
ddD.d = dbp->d;
free(dbp);
if ( ddN.DT<ddN.D ) {
/* recompute NT */
int i;
for (i=0; i<ddN.N; i++)
if ( ddD.d[i]>=ddN.DT )
break;
ddN.NT = i;
}
}
data_read_epoch(stem);
/*
* at this point, dimensions are fixed, so load phi and mu if needed
*/
if ( load_phi )
pctl_loadphi(resstem);
if ( load_mu )
pctl_loadmu(resstem);
/*
* correct parameters after command line
*/
pctl_fix(ITER);
if ( BM0val>0 ) {
ddP.b_mu[0] = BM0val;
}
if ( BM1val>0 ) {
int i;
for (i=1; i<ddN.E; i++)
ddP.b_mu[i] = BM1val;
}
if ( BP0val>0 ) {
int i;
for (i=0; i<ddN.T; i++)
ddP.b_phi[0][i] = BP0val;
}
if ( BP1val>0 ) {
int i;
if ( ddN.E==1 )
yap_quit("b_phi[1] invalid when epochs==1\n");
for (i=0; i<ddN.T; i++)
ddP.b_phi[1][i] = BP1val;
}
pctl_samplereport();
/*
* all data structures
*/
data_alloc();
if ( ddP.phiiter>0 )
phi_init(resstem);
else
ddS.phi = NULL;
if ( ddP.muiter>0 )
mu_init(resstem);
else
ddS.mu = NULL;
if ( ddP.thetaiter>0 )
theta_init(resstem);
else
ddS.theta = NULL;
tca_alloc();
if ( PCTL_BURSTY() )
dmi_init(&ddM, ddS.z, ddD.w, ddD.N_dTcum,
ddN.T, ddN.N, ddN.W, ddN.D, ddN.DT,
(fix_hold==GibbsHold)?pctl_hold:NULL);
if ( load_vocab ) {
data_vocab(stem);
}
cache_init();
/*
* yap some details
*/
data_report(ITER, seed);
pctl_report();
/*
* load/init topic assignments and prepare statistics
*/
if ( restart || restart_hca) {
tca_read_z(resstem, 0, ddN.DT);
tca_rand_z(ddN.T, ddN.DT, ddN.D);
} else {
tca_rand_z(ddN.T, 0, ddN.D);
}
tca_reset_stats(resstem, restart, 0);
if ( (restart || restart_hca ) && ITER )
yap_message("Initial log_2(perp)=%lf\n", -M_LOG2E * likelihood()/ddN.NT);
if ( ITER )
yap_report("cycles: ");
for (iter=0; iter<ITER; iter++) {
int pro;
double thislp = 0;
int thisNd = 0;
int doc;
#ifdef H_THREADS
pthread_t thread[procs];
#endif
D_pargs_p parg[procs];
#ifdef MU_CACHE
mu_side_fact_reinit();
#endif
#ifdef PHI_CACHE
phi_cache_reinit();
#endif
t1 = clock();
/*
* sampling
*/
#ifdef IND_STATS
ddP.doc_ind_stats = u32tri(ddN.T,ddN.E,ddN.E);
ddP.word_ind_stats = u32tri(ddN.T,ddN.E,ddN.E);
#endif
/* a bit complex if no threads! */
doc = 0;
for (pro = 0 ; pro < procs ; pro++){
parg[pro].dots=dots;
parg[pro].procs=procs;
parg[pro].doc = &doc;
#ifndef H_THREADS
sampling_p(&parg[pro]);
#else
if ( procs==1 )
sampling_p(&parg[pro]);
else if( pthread_create(&thread[pro],NULL,sampling_p,(void*) &parg[pro]) != 0){
yap_message("thread failed %d\n",pro+1 );
}
#endif
}
#ifdef H_THREADS
if ( procs>1 ) {
//waiting for threads to finish
for (pro = 0; pro < procs; pro++){
pthread_join(thread[pro], NULL);
}
}
#endif
// getting lp, Nd and clock
for(pro = 0; pro < procs; pro++){
thislp += parg[pro].thislp;
thisNd += parg[pro].thisNd;
tot_time += parg[pro].tot_time;
}
#ifdef H_THREADS
if ( procs>1 )
tca_reset_stats(NULL,1,1);
#endif
/*
* full check
*/
#ifndef NDEBUG
{
int e, d;
check_cp_et();
for (e=0; e<ddN.E; e++)
check_m_vte(e);
for (d=0; d<ddN.DT; d++)
check_n_dt(d);
}
#endif
#ifdef IND_STATS
{
char *fname = yap_makename(resstem,".istats");
FILE *ifp = fopen(fname,"a");
int e1, e2, kk;
fprintf(ifp,"Iteration %d\n", iter);
for (kk=0; kk<ddN.T; kk++) {
fprintf(ifp," Topic %d\n", kk);
for (e1=0; e1<ddN.E; e1++) {
fprintf(ifp," Epoch %d\n ", e1);
for (e2=0; e2<ddN.E; e2++)
fprintf(ifp," %u", (unsigned)ddP.doc_ind_stats[kk][e1][e2]);
fprintf(ifp,"\n ");
for (e2=0; e2<ddN.E; e2++)
fprintf(ifp," %u", (unsigned)ddP.word_ind_stats[kk][e1][e2]);
fprintf(ifp,"\n");
}
}
fclose(ifp);
free(ddP.doc_ind_stats[0][0]); free(ddP.doc_ind_stats[0]);
free(ddP.doc_ind_stats);
free(ddP.word_ind_stats[0][0]); free(ddP.word_ind_stats[0]);
free(ddP.word_ind_stats);
free(fname);
}
#endif
/*
* sample hyperparameters
*/
t3 = clock();
pctl_sample(iter, procs);
/*
* do time calcs here to remove diagnostics+reporting
*/
t2 = clock();
tot_time += (double)(t2 - t1) / CLOCKS_PER_SEC;
psample_time += (double)(t2 - t3) / CLOCKS_PER_SEC;
/*
* progress reports
*/
if ( ( iter>ddP.progburn && (iter%ddP.progiter)==0 ) || iter+1>=ITER ) {
yap_message(" %d\nlog_2(perp)=%lf,%lf", iter,
-M_LOG2E * likelihood()/ddN.NT, -M_LOG2E * thislp/thisNd);
pctl_update(iter);
if ( verbose && iter%10==0 )
yap_probs();
if ( iter>0 && verbose>1 ) {
if ( ddN.tokens ) {
tca_displaytopics(resstem,topwords,score);
displayed++;
}
}
if ( iter+1<ITER ) {
// yap_message("\n");
yap_report("cycles: ");
}
} else {
yap_message(" %d", iter);
if ( verbose>1) yap_message("\n");
}
if ( checkpoint>0 && iter>0 && iter%checkpoint==0 ) {
data_checkpoint(resstem, stem, iter+1);
yap_message(" checkpointed\n");
tca_report(resstem, stem, ITER, procs, fix_hold,
(dopmi&&displayed>0)?1:0);
}
if ( ddP.phiiter>0 && iter>ddP.phiburn && (iter%ddP.phiiter)==0 )
phi_update();
if ( ddP.thetaiter>0 && iter>ddP.thetaburn && (iter%ddP.thetaiter)==0 )
theta_update();
if ( ddP.muiter>0 && iter>ddP.muburn && (iter%ddP.muiter)==0 )
mu_update();
} // over iter
if ( ITER )
yap_report("Finished after %d cycles on average of %lf+%lf(s) per cycle\n",
iter, (tot_time-psample_time)/iter, psample_time/iter);
if ( ( verbose==1 || ((iter+1)%5!=0 && verbose>1) ) ) {
if ( ddN.tokens ) {
tca_displaytopics(resstem,topwords,score);
displayed++;
}
}
yap_probs();
if ( ITER>0 )
data_checkpoint(resstem, stem, ITER);
tca_report(resstem, stem, ITER, procs, fix_hold, (dopmi&&displayed>0)?1:0);
if ( ddP.phiiter>0 )
phi_save(resstem);
if ( ddP.thetaiter>0 )
theta_save(resstem);
if ( ddP.muiter>0 )
mu_save(resstem);
/*
* free
*/
phi_free();
theta_free();
mu_free();
cache_free();
pctl_free();
data_free();
dmi_free(&ddM);
tca_free();
free(stem);
free(resstem);
rng_free(rngp);
return 0;
}
