void do_balanced_multi_tree_inference(FILE *logfp, mcmc_t *mcmc, node_t **subtrees, node_t **wholetrees, gslws_t *wses, sampman_t *sms, ubertree_t *ut, int burnin, int samples, int lag) {
int i, j;
/* Burn in */
for(i=0; i<burnin; i++) balanced_multi_tree_mcmc_iteration(logfp, mcmc, subtrees, wses);
/* Take samples */
for(i=0; i<samples; i++) {
if(gsl_rng_uniform_int(mcmc->r, 10000) >= 9999) {
/* Random restart! */
random_restart(mcmc);
compute_balanced_multi_tree_probabilities(mcmc, subtrees, wses);
for(j=0; j<burnin; j++) balanced_multi_tree_mcmc_iteration(logfp, mcmc, subtrees, wses);
}
for(j=0; j<lag; j++) {
balanced_multi_tree_mcmc_iteration(logfp, mcmc, subtrees, wses);
}
build_q(mcmc);
for(j=0; j<NUM_TREES; j++) {
upwards_belprop(logfp, wholetrees[j], mcmc->Q, &wses[j]);
process_sample(&sms[j], mcmc, &wses[j], wholetrees[j]);
}
if(ut != NULL) update_ubertree(ut, wholetrees, mcmc->Q, &wses[0]);
}
}
model hydrator::hydrate(std::istream& is) const {
model r;
std::string input_line;
unsigned int line_number(0);
try {
while (std::getline(is, input_line)) {
const auto trimmed(boost::trim_copy(input_line));
if (!trimmed.empty() && trimmed[0] != hash) {
/*
* Ignore lines starting with hash for now until we
* have support for processing headers.
*/
r.samples().push_back(process_sample(trimmed, line_number));
} else
BOOST_LOG_SEV(lg, debug) << "Ignoring line: " << line_number;
++line_number;
}
} catch(boost::exception& e) {
BOOST_LOG_SEV(lg, error) << "Failed to parse line: " << line_number
<< " contents: " << input_line;
e << error_in_line(line_number);
throw;
}
return r;
}
void do_single_tree_inference(FILE *logfp, mcmc_t *mcmc, node_t *tree, gslws_t *ws, sampman_t *sm, int burnin, int samples, int lag) {
int i, j;
/* Burn in */
for(i=0; i<burnin; i++) single_tree_mcmc_iteration(logfp, mcmc, tree, ws);
/* Take samples */
for(i=0; i<samples; i++) {
if(gsl_rng_uniform_int(mcmc->r, 10000) >= 9999) {
/* Random restart! */
random_restart(mcmc);
compute_single_tree_probabilities(mcmc, tree, ws);
for(j=0; j<burnin; j++) single_tree_mcmc_iteration(logfp, mcmc, tree, ws);
}
for(j=0; j<lag; j++) {
single_tree_mcmc_iteration(logfp, mcmc, tree, ws);
}
build_q(mcmc);
upwards_belprop(logfp, tree, mcmc->Q, ws);
/* Record sample */
process_sample(sm, mcmc, ws, tree);
}
/* Do computations for Q matrices sampled from the pior
* and log stuff so we can do importance sampling for
* marginal likelihood later */
for(i=0; i<samples; i++) {
draw_proposal_from_prior(mcmc, ws);
mcmc->log_prior = get_log_prior(mcmc->stabs_dash, mcmc->trans_dash);
mcmc->log_lh = get_model_loglh(tree, mcmc->Q_dash, ws);
mcmc->log_poster = mcmc->log_prior + mcmc->log_lh;
process_prior_sample(sm, mcmc, ws);
}
}
/* Here is the definition of the block processing function */
void process_block(short *clean, short *echo, short *out, int size) {
int i;
for(i = 0;i < size;i++) {
out[i] = toShort(process_sample(toFloat(clean[i]), toFloat(echo[i])));
}
}
GstFlowReturn ofGstVideoUtils::buffer_cb(shared_ptr<GstSample> sample){
GstFlowReturn ret = process_sample(sample);
if(ret==GST_FLOW_OK){
return ofGstUtils::buffer_cb(sample);
}else{
return ret;
}
}
Profile *
tracker_create_profile (tracker_t *tracker)
{
uint8_t *end = tracker->events + tracker->n_event_bytes;
StackStash *resolved_stash;
Profile *profile;
state_t *state;
uint8_t *event;
state = state_new ();
resolved_stash = stack_stash_new (g_free);
event = tracker->events;
while (event < end)
{
event_type_t type = GET_TYPE (*(uint32_t *)event);
switch (type)
{
case NEW_PROCESS:
create_process (state, (new_process_t *)event);
event += sizeof (new_process_t);
break;
case NEW_MAP:
create_map (state, (new_map_t *)event);
event += sizeof (new_map_t);
break;
case FORK:
process_fork (state, (fork_t *)event);
event += sizeof (fork_t);
break;
case EXIT:
process_exit (state, (exit_t *)event);
event += sizeof (exit_t);
break;
case SAMPLE:
process_sample (state, resolved_stash, (sample_t *)event);
event += sizeof (sample_t);
break;
}
}
profile = profile_new (resolved_stash);
state_free (state);
stack_stash_unref (resolved_stash);
return profile;
}
/**
* main method
* contains main run loop
*/
int main() {
//prototype for linear assembly convolution function
extern int convolve_as_func(int x[], int w[], int x_idx, int w_length);
DSK6713_init();
hCodec = DSK6713_AIC23_openCodec(0,&config);
DSK6713_AIC23_setFreq(hCodec, DSK6713_AIC23_FREQ_8KHZ);
// enable interrupts
IRQ_globalEnable();
IRQ_enable(IRQ_EVT_RINT1);
IRQ_enable(IRQ_EVT_XINT1);
reset(); // init the input buffer to zeros
// get the time to make a clock() funciton call; used only for profiling
/*
start = clock();
end = clock();
diff = end - start;
*/
// the first write is needed to trigger the transmit interrupt
while(!DSK6713_AIC23_write(hCodec, 0));
in_channel_flag = 1;
out_channel_flag = 1;
while(1) {
if(input_ready) {
//process sample when input is ready
sig_error = process_sample(in_left, in_right);
input_ready = 0;
}
// set output when ready
if(output_ready) {
out_left = in_left;
out_right = sig_error;
output_ready = 0;
}
};
/* The program will never exit this loop */
/* However, if you _do_ exit the loop (say, using a break
* statement) close the D/A converter properly */
DSK6713_AIC23_closeCodec(hCodec);
exit();
}
void do_multi_tree_inference(FILE *logfp, mcmc_t *mcmc, node_t **subtrees, node_t **wholetrees, gslws_t *wses, sampman_t *sms, ubertree_t *ut, int burnin, int samples, int lag) {
int i, j;
/* Burn in */
for(i=0; i<burnin; i++) multi_tree_mcmc_iteration(logfp, mcmc, subtrees, wses);
/* Take samples */
for(i=0; i<samples; i++) {
if(gsl_rng_uniform_int(mcmc->r, 10000) >= 9999) {
/* Random restart! */
random_restart(mcmc);
compute_multi_tree_probabilities(mcmc, subtrees, wses);
for(j=0; j<burnin; j++) multi_tree_mcmc_iteration(logfp, mcmc, subtrees, wses);
}
for(j=0; j<lag; j++) {
multi_tree_mcmc_iteration(logfp, mcmc, subtrees, wses);
}
build_q(mcmc);
for(j=0; j<NUM_TREES; j++) {
upwards_belprop(logfp, wholetrees[j], mcmc->Q, &wses[j]);
process_sample(&sms[j], mcmc, &wses[j], wholetrees[j]);
}
if(ut != NULL) update_ubertree(ut, wholetrees, mcmc->Q, &wses[0]);
}
/* Do computations for Q matrices sampled from the pior
* and log stuff so we can do importance sampling for
* marginal likelihood later */
for(i=0; i<samples; i++) {
draw_proposal_from_prior(mcmc, wses);
mcmc->log_prior = get_log_prior(mcmc->stabs_dash, mcmc->trans_dash);
mcmc->log_lh = 0;
for(j=0; j<NUM_TREES; j++) mcmc->log_lh += get_model_loglh(wholetrees[j], mcmc->Q_dash, &wses[j]);
mcmc->log_poster = mcmc->log_prior + mcmc->log_lh;
process_prior_sample(&sms[0], mcmc, &wses[0]);
}
}
void split_shared_q(int method, int shuffle, int burnin, int samples, int lag, int treecount, char *outdir, int logging) {
FILE *logfp;
FILE *correlfp;
int treeindex, i, j, k;
char filename[1024];
char methods[][16] = {"geographic", "genetic", "feature", "combination" };
node_t **trees1 = calloc(NUM_TREES, sizeof(node_t*));
node_t **subtrees1 = calloc(NUM_TREES, sizeof(node_t*));
node_t **trees2 = calloc(NUM_TREES, sizeof(node_t*));
node_t **subtrees2 = calloc(NUM_TREES, sizeof(node_t*));
mcmc_t mcmc1, mcmc2;
gslws_t *wses1 = calloc(NUM_TREES, sizeof(gslws_t));
gslws_t *wses2 = calloc(NUM_TREES, sizeof(gslws_t));
sampman_t *sms1 = calloc(NUM_TREES, sizeof(sampman_t));
sampman_t *sms2 = calloc(NUM_TREES, sizeof(sampman_t));
gsl_matrix *q1 = gsl_matrix_alloc(6, 6);
gsl_matrix *q2 = gsl_matrix_alloc(6, 6);
double qcorrel1[6*NUM_TREES], qcorrel2[6*NUM_TREES];
double anccorrel1[6*NUM_TREES], anccorrel2[6*NUM_TREES];
double qcorrelation = 0;
double anccorrelation = 0;
// Open log file
if(logging) {
logfp = fopen("logfile", "w");
} else {
logfp = fopen("/dev/null", "w");
}
initialise_mcmc(&mcmc1);
initialise_mcmc(&mcmc2);
for(i=0; i<NUM_TREES; i++) {
alloc_gslws(&wses1[i]);
alloc_gslws(&wses2[i]);
initialise_sampman(&sms1[i], outdir);
initialise_sampman(&sms2[i], outdir);
// Compute stability sampling rate
sms1[i].stability_sampling_rate = (treecount*samples) / 500.0;
sms2[i].stability_sampling_rate = (treecount*samples) / 500.0;
}
// Loop over trees...
for(treeindex=0; treeindex<treecount; treeindex++) {
/* Build tree(s) */
for(i=0; i<NUM_TREES; i++) {
load_tree(&trees1[i], "../TreeBuilder/generated_trees/split/1/", method, i, treeindex, shuffle);
load_tree(&subtrees1[i], "../TreeBuilder/generated_trees/split/1/", method, i, treeindex, shuffle);
load_tree(&trees2[i], "../TreeBuilder/generated_trees/split/2/", method, i, treeindex, shuffle);
load_tree(&subtrees2[i], "../TreeBuilder/generated_trees/split/2/", method, i, treeindex, shuffle);
}
/* Draw samples for this tree (set) */
compute_multi_tree_probabilities(&mcmc1, subtrees1, wses1);
compute_multi_tree_probabilities(&mcmc2, subtrees2, wses2);
/* Burn in */
for(i=0; i<burnin; i++) {
multi_tree_mcmc_iteration(logfp, &mcmc1, subtrees1, wses1);
multi_tree_mcmc_iteration(logfp, &mcmc2, subtrees2, wses2);
}
gsl_matrix_set_zero(q1);
gsl_matrix_set_zero(q2);
memset(anccorrel1, 0, 6*NUM_TREES*sizeof(double));
memset(anccorrel2, 0, 6*NUM_TREES*sizeof(double));
/* Take samples */
for(i=0; i<samples; i++) {
if(gsl_rng_uniform_int(mcmc1.r, 10000) >= 9999) {
/* Random restart! */
random_restart(&mcmc1);
random_restart(&mcmc2);
compute_multi_tree_probabilities(&mcmc1, subtrees1, wses1);
compute_multi_tree_probabilities(&mcmc2, subtrees2, wses2);
for(j=0; j<burnin; j++) {
multi_tree_mcmc_iteration(logfp, &mcmc1, subtrees1, wses1);
multi_tree_mcmc_iteration(logfp, &mcmc2, subtrees2, wses2);
}
}
for(j=0; j<lag; j++) {
multi_tree_mcmc_iteration(logfp, &mcmc1, subtrees1, wses1);
multi_tree_mcmc_iteration(logfp, &mcmc2, subtrees2, wses2);
}
build_q(&mcmc1);
build_q(&mcmc2);
for(j=0; j<NUM_TREES; j++) {
upwards_belprop(logfp, trees1[j], mcmc1.Q, &wses1[j]);
upwards_belprop(logfp, trees2[j], mcmc2.Q, &wses2[j]);
process_sample(&sms1[j], &mcmc1, &wses1[j], trees1[j]);
process_sample(&sms2[j], &mcmc2, &wses2[j], trees2[j]);
}
/* Combine Q samples for this tree pair */
gsl_matrix_add(q1, mcmc1.Q);
gsl_matrix_add(q2, mcmc2.Q);
for(j=0; j<NUM_TREES; j++) {
for(k=0; k<6; k++) {
anccorrel1[j*6+k] += trees1[j]->dist[k];
anccorrel2[j*6+k] += trees2[j]->dist[k];
}
}
}
gsl_matrix_scale(q1, 1.0 / samples);
gsl_matrix_scale(q2, 1.0 / samples);
for(i=0; i<NUM_TREES; i++) {
for(j=0; j<6; j++) {
qcorrel1[i*6+j] = gsl_matrix_get(q1, i, j);
qcorrel2[i*6+j] = gsl_matrix_get(q2, i, j);
anccorrel1[i*6+j] /= (1.0*samples);
anccorrel2[i*6+j] /= (1.0*samples);
}
}
qcorrelation += gsl_stats_correlation(qcorrel1, 1, qcorrel2, 1, 36);
anccorrelation += gsl_stats_correlation(anccorrel1, 1, anccorrel2, 1, 36);
/***************************************************/
/* Free up tree memory */
for(i=0; i<NUM_TREES; i++) {
free(trees1[i]);
free(subtrees1[i]);
free(trees2[i]);
free(subtrees2[i]);
}
}
qcorrelation /= treecount;
anccorrelation /= treecount;
// Finish up
for(i=0; i<NUM_TREES; i++) {
compute_means(&sms1[i]);
compute_means(&sms2[i]);
}
sprintf(filename, "%s/%s/", outdir, methods[method]);
strcat(filename, "/left-half/");
save_common_q(filename, sms1);
sprintf(filename, "%s/%s/", outdir, methods[method]);
strcat(filename, "/right-half/");
save_common_q(filename, sms2);
sprintf(filename, "%s/%s/correlations", outdir, methods[method]);
correlfp = fopen(filename, "w");
fprintf(correlfp, "Q: %f\n", qcorrelation);
fprintf(correlfp, "Anc: %f\n", anccorrelation);
fclose(correlfp);
fclose(logfp);
}
int gpu_perf_update(struct gpu_perf *gp)
{
const int size = N_PAGES * gp->page_size;
const int mask = size - 1;
uint8_t *buffer = NULL;
int buffer_size = 0;
int n, update = 0;
if (gp->map == NULL)
return 0;
for (n = 0; n < gp->nr_cpus; n++) {
struct perf_event_mmap_page *mmap = gp->map[n];
const uint8_t *data;
uint64_t head, tail;
int wrap = 0;
tail = mmap->data_tail;
head = mmap->data_head;
rmb();
if (head < tail) {
wrap = 1;
tail &= mask;
head &= mask;
head += size;
}
data = (uint8_t *)mmap + gp->page_size;
while (head - tail >= sizeof (struct perf_event_header)) {
const struct perf_event_header *header;
header = (const struct perf_event_header *)(data + (tail & mask));
if (header->size > head - tail)
break;
if ((const uint8_t *)header + header->size > data + size) {
int before;
if (header->size > buffer_size) {
uint8_t *b = realloc(buffer, header->size);
if (b == NULL)
break;
buffer = b;
buffer_size = header->size;
}
before = data + size - (const uint8_t *)header;
memcpy(buffer, header, before);
memcpy(buffer + before, data, header->size - before);
header = (struct perf_event_header *)buffer;
}
if (header->type == PERF_RECORD_SAMPLE)
update += process_sample(gp, n, header);
tail += header->size;
}
if (wrap)
tail &= mask;
mmap->data_tail = tail;
wmb();
}
free(buffer);
return update;
}
static void
process_event (Collector *collector,
counter_t *counter,
counter_event_t *event)
{
char *name;
switch (event->header.type)
{
case PERF_RECORD_MMAP: name = "mmap"; break;
case PERF_RECORD_LOST: name = "lost"; break;
case PERF_RECORD_COMM: name = "comm"; break;
case PERF_RECORD_EXIT: name = "exit"; break;
case PERF_RECORD_THROTTLE: name = "throttle"; break;
case PERF_RECORD_UNTHROTTLE: name = "unthrottle"; break;
case PERF_RECORD_FORK: name = "fork"; break;
case PERF_RECORD_READ: name = "read"; break;
case PERF_RECORD_SAMPLE: name = "samp"; break;
default: name = "unknown"; break;
}
d_print ("cpu %d :: %s :: ", counter->cpu, name);
switch (event->header.type)
{
case PERF_RECORD_MMAP:
process_mmap (collector, &event->mmap);
break;
case PERF_RECORD_LOST:
g_print ("lost event\n");
break;
case PERF_RECORD_COMM:
process_comm (collector, &event->comm);
break;
case PERF_RECORD_EXIT:
process_exit (collector, &event->exit);
break;
case PERF_RECORD_THROTTLE:
g_print ("throttle\n");
break;
case PERF_RECORD_UNTHROTTLE:
g_print ("unthrottle\n");
break;
case PERF_RECORD_FORK:
process_fork (collector, &event->fork);
break;
case PERF_RECORD_READ:
break;
case PERF_RECORD_SAMPLE:
process_sample (collector, &event->sample);
break;
default:
g_warning ("unknown event: %d (%d)\n",
event->header.type, event->header.size);
break;
}
d_print ("\n");
}
int main(int argc, char **argv)
{
sigset_t sigmask;
struct pollfd pollfds[NPFD_COUNT];
int npollfds;
int k;
int64_t wait_ms;
struct paging_data pd0;
struct paging_data pd1;
bool recheck_time;
int init_pass = 0;
/*
* if running interactively, output initially to the terminal,
* otherwise to syslog right away
*/
if (isatty(fileno(stderr)))
{
log_fp = stderr;
log_syslog = false;
}
else
{
log_fp = NULL;
log_syslog = true;
}
if (log_syslog)
openlog(progname, LOG_CONS | LOG_PID, LOG_DAEMON);
/* parse arguments */
handle_cmdline(argc, argv);
if (run_as_daemon)
daemonize();
debug_msg(1, "debug level set to %d", debug_level);
// page_size = sysconf(_SC_PAGESIZE);
// if (page_size <= 0)
// fatal_msg("unable to get system page size");
/*
* Block signals so that they aren't handled according to their
* default dispositions
*/
sigemptyset(&sigmask);
sigaddset(&sigmask, SIGHUP);
sigaddset(&sigmask, SIGTERM);
if (sigprocmask(SIG_BLOCK, &sigmask, NULL) < 0)
fatal_perror("sigprocmask");
/*
* Receive signals on file descriptors
*/
sigemptyset(&sigmask);
sigaddset(&sigmask, SIGHUP);
fd_sighup = signalfd(-1, &sigmask, 0);
if (fd_sighup < 0)
fatal_perror("signalfd(SIGHUP)");
sigemptyset(&sigmask);
sigaddset(&sigmask, SIGTERM);
fd_sigterm = signalfd(-1, &sigmask, 0);
if (fd_sigterm < 0)
fatal_perror("signalfd(SIGTERM)");
/* select clock to use */
select_clk_id();
/* verify we are running under a hypervisor */
verify_hypervisor();
pollfds[NPFD_SIGTERM].fd = fd_sigterm;
pollfds[NPFD_SIGTERM].events = POLLIN|POLLPRI;
pollfds[NPFD_SIGHUP].fd = fd_sighup;
pollfds[NPFD_SIGHUP].events = POLLIN|POLLPRI;
/* keep xs poll fd last in the array */
pollfds[NPFD_XS].fd = -1;
pollfds[NPFD_XS].events = POLLIN|POLLPRI;
/* initialize xenstore structure */
initialize_xs(pollfds);
pollfds[NPFD_XS].fd = xs_fileno(xs);
get_paging_data(&pd0);
for (;;)
{
try_subscribe_membalance_settings();
/* calculate sleep time till next sample point */
if (initialized_xs)
{
wait_ms = timespec_diff_ms(pd0.ts, getnow()) + interval * MSEC_PER_SEC;
}
else
{
/* if xs not initialzied, keep retrying */
wait_ms = interval * MSEC_PER_SEC;
if (++init_pass > 30)
wait_ms *= 2;
}
if (wait_ms >= tolerance_ms)
{
for (k = 0; k < countof(pollfds); k++)
pollfds[k].revents = 0;
/* if settings need to be updated, retry in 1 sec or sooner */
if (need_update_membalance_settings)
wait_ms = min(wait_ms, MSEC_PER_SEC);
/* include xs in the poll only if has initialized xs */
npollfds = countof(pollfds);
if (!initialized_xs)
npollfds--;
if (poll(pollfds, npollfds, wait_ms) < 0)
{
if (errno == EINTR)
continue;
fatal_perror("poll");
}
recheck_time = false;
handle_signals(pollfds, &recheck_time);
if (!initialized_xs)
{
/* try to initialzie xs */
initialize_xs(NULL);
/* if successful, start monitoring page map-in rate */
if (initialized_xs)
get_paging_data(&pd0);
continue;
}
if (pollfds[NPFD_XS].revents & (POLLIN|POLLPRI))
{
/* a watched value in xenstore has been changed */
handle_xs_watch();
recheck_time = true;
}
if (need_update_membalance_settings)
update_membalance_settings();
if (recheck_time)
{
/* go sleep again if wait interval has not expired yet */
if (timespec_diff_ms(getnow(), pd0.ts) < interval * MSEC_PER_SEC - tolerance_ms)
continue;
}
}
if (!initialized_xs)
{
/* try to initialzie xs */
initialize_xs(NULL);
/* if successful, start monitoring page map-in rate */
if (initialized_xs)
get_paging_data(&pd0);
continue;
}
/* process sample and set it as the new "previous" one */
get_paging_data(&pd1);
process_sample(&pd1, &pd0);
pd0 = pd1;
}
/* close connection to xenstore */
shutdown_xs();
return EXIT_SUCCESS;
}