void train_colorizer(char *cfg, char *weight, char *acfg, char *aweight, int clear, int display) { #ifdef GPU //char *train_images = "/home/pjreddie/data/coco/train1.txt"; //char *train_images = "/home/pjreddie/data/coco/trainvalno5k.txt"; char *train_images = "/home/pjreddie/data/imagenet/imagenet1k.train.list"; char *backup_directory = "/home/pjreddie/backup/"; srand(time(0)); char *base = basecfg(cfg); char *abase = basecfg(acfg); printf("%s\n", base); network *net = load_network(cfg, weight, clear); network *anet = load_network(acfg, aweight, clear); int i, j, k; layer imlayer = {0}; for (i = 0; i < net->n; ++i) { if (net->layers[i].out_c == 3) { imlayer = net->layers[i]; break; } } printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net->learning_rate, net->momentum, net->decay); int imgs = net->batch*net->subdivisions; i = *net->seen/imgs; data train, buffer; list *plist = get_paths(train_images); //int N = plist->size; char **paths = (char **)list_to_array(plist); load_args args= get_base_args(net); args.paths = paths; args.n = imgs; args.m = plist->size; args.d = &buffer; args.type = CLASSIFICATION_DATA; args.classes = 1; char *ls[2] = {"imagenet"}; args.labels = ls; pthread_t load_thread = load_data_in_thread(args); clock_t time; int x_size = net->inputs*net->batch; //int y_size = x_size; net->delta = 0; net->train = 1; float *pixs = calloc(x_size, sizeof(float)); float *graypixs = calloc(x_size, sizeof(float)); //float *y = calloc(y_size, sizeof(float)); //int ay_size = anet->outputs*anet->batch; anet->delta = 0; anet->train = 1; float *imerror = cuda_make_array(0, imlayer.outputs*imlayer.batch); float aloss_avg = -1; float gloss_avg = -1; //data generated = copy_data(train); while (get_current_batch(net) < net->max_batches) { i += 1; time=clock(); pthread_join(load_thread, 0); train = buffer; load_thread = load_data_in_thread(args); printf("Loaded: %lf seconds\n", sec(clock()-time)); data gray = copy_data(train); for(j = 0; j < imgs; ++j){ image gim = float_to_image(net->w, net->h, net->c, gray.X.vals[j]); grayscale_image_3c(gim); train.y.vals[j][0] = .95; gray.y.vals[j][0] = .05; } time=clock(); float gloss = 0; for(j = 0; j < net->subdivisions; ++j){ get_next_batch(train, net->batch, j*net->batch, pixs, 0); get_next_batch(gray, net->batch, j*net->batch, graypixs, 0); cuda_push_array(net->input_gpu, graypixs, net->inputs*net->batch); cuda_push_array(net->truth_gpu, pixs, net->truths*net->batch); /* image origi = float_to_image(net->w, net->h, 3, pixs); image grayi = float_to_image(net->w, net->h, 3, graypixs); show_image(grayi, "gray"); show_image(origi, "orig"); cvWaitKey(0); */ *net->seen += net->batch; forward_network_gpu(net); fill_gpu(imlayer.outputs*imlayer.batch, 0, imerror, 1); copy_gpu(anet->inputs*anet->batch, imlayer.output_gpu, 1, anet->input_gpu, 1); fill_gpu(anet->inputs*anet->batch, .95, anet->truth_gpu, 1); anet->delta_gpu = imerror; forward_network_gpu(anet); backward_network_gpu(anet); scal_gpu(imlayer.outputs*imlayer.batch, 1./100., net->layers[net->n-1].delta_gpu, 1); scal_gpu(imlayer.outputs*imlayer.batch, 1, imerror, 1); printf("realness %f\n", cuda_mag_array(imerror, imlayer.outputs*imlayer.batch)); printf("features %f\n", cuda_mag_array(net->layers[net->n-1].delta_gpu, imlayer.outputs*imlayer.batch)); axpy_gpu(imlayer.outputs*imlayer.batch, 1, imerror, 1, net->layers[net->n-1].delta_gpu, 1); backward_network_gpu(net); gloss += *net->cost /(net->subdivisions*net->batch); for(k = 0; k < net->batch; ++k){ int index = j*net->batch + k; copy_cpu(imlayer.outputs, imlayer.output + k*imlayer.outputs, 1, gray.X.vals[index], 1); } } harmless_update_network_gpu(anet); data merge = concat_data(train, gray); //randomize_data(merge); float aloss = train_network(anet, merge); update_network_gpu(net); #ifdef OPENCV if(display){ image im = float_to_image(anet->w, anet->h, anet->c, gray.X.vals[0]); image im2 = float_to_image(anet->w, anet->h, anet->c, train.X.vals[0]); show_image(im, "gen"); show_image(im2, "train"); cvWaitKey(50); } #endif free_data(merge); free_data(train); free_data(gray); if (aloss_avg < 0) aloss_avg = aloss; aloss_avg = aloss_avg*.9 + aloss*.1; gloss_avg = gloss_avg*.9 + gloss*.1; printf("%d: gen: %f, adv: %f | gen_avg: %f, adv_avg: %f, %f rate, %lf seconds, %d images\n", i, gloss, aloss, gloss_avg, aloss_avg, get_current_rate(net), sec(clock()-time), i*imgs); if(i%1000==0){ char buff[256]; sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i); save_weights(net, buff); sprintf(buff, "%s/%s_%d.weights", backup_directory, abase, i); save_weights(anet, buff); } if(i%100==0){ char buff[256]; sprintf(buff, "%s/%s.backup", backup_directory, base); save_weights(net, buff); sprintf(buff, "%s/%s.backup", backup_directory, abase); save_weights(anet, buff); } } char buff[256]; sprintf(buff, "%s/%s_final.weights", backup_directory, base); save_weights(net, buff); #endif }
SearchRequest::~SearchRequest() { Event e(static_cast<JabberPlugin*>(m_client->protocol()->plugin())->EventSearchDone, (void*)m_id.c_str()); e.process(); free_data(jabberSearchData, &data); }
void validate_classifier(char *datacfg, char *filename, char *weightfile) { int i = 0; network net = parse_network_cfg(filename); if(weightfile){ load_weights(&net, weightfile); } srand(time(0)); list *options = read_data_cfg(datacfg); char *label_list = option_find_str(options, "labels", "data/labels.list"); char *valid_list = option_find_str(options, "valid", "data/train.list"); int classes = option_find_int(options, "classes", 2); int topk = option_find_int(options, "top", 1); char **labels = get_labels(label_list); list *plist = get_paths(valid_list); char **paths = (char **)list_to_array(plist); int m = plist->size; free_list(plist); clock_t time; float avg_acc = 0; float avg_topk = 0; int splits = m/1000; int num = (i+1)*m/splits - i*m/splits; data val, buffer; load_args args = {0}; args.w = net.w; args.h = net.h; args.paths = paths; args.classes = classes; args.n = num; args.m = 0; args.labels = labels; args.d = &buffer; args.type = CLASSIFICATION_DATA; pthread_t load_thread = load_data_in_thread(args); for(i = 1; i <= splits; ++i){ time=clock(); pthread_join(load_thread, 0); val = buffer; num = (i+1)*m/splits - i*m/splits; char **part = paths+(i*m/splits); if(i != splits){ args.paths = part; load_thread = load_data_in_thread(args); } printf("Loaded: %d images in %lf seconds\n", val.X.rows, sec(clock()-time)); time=clock(); float *acc = network_accuracies(net, val, topk); avg_acc += acc[0]; avg_topk += acc[1]; printf("%d: top 1: %f, top %d: %f, %lf seconds, %d images\n", i, avg_acc/i, topk, avg_topk/i, sec(clock()-time), val.X.rows); free_data(val); } }
void train_tag(char *cfgfile, char *weightfile, int clear) { srand(time(0)); float avg_loss = -1; char *base = basecfg(cfgfile); char *backup_directory = "/home/pjreddie/backup/"; printf("%s\n", base); network *net = load_network(cfgfile, weightfile, clear); printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net->learning_rate, net->momentum, net->decay); int imgs = 1024; list *plist = get_paths("/home/pjreddie/tag/train.list"); char **paths = (char **)list_to_array(plist); printf("%d\n", plist->size); int N = plist->size; clock_t time; pthread_t load_thread; data train; data buffer; load_args args = {0}; args.w = net->w; args.h = net->h; args.min = net->w; args.max = net->max_crop; args.size = net->w; args.paths = paths; args.classes = net->outputs; args.n = imgs; args.m = N; args.d = &buffer; args.type = TAG_DATA; args.angle = net->angle; args.exposure = net->exposure; args.saturation = net->saturation; args.hue = net->hue; fprintf(stderr, "%d classes\n", net->outputs); load_thread = load_data_in_thread(args); int epoch = (*net->seen)/N; while(get_current_batch(net) < net->max_batches || net->max_batches == 0){ time=clock(); pthread_join(load_thread, 0); train = buffer; load_thread = load_data_in_thread(args); printf("Loaded: %lf seconds\n", sec(clock()-time)); time=clock(); float loss = train_network(net, train); if(avg_loss == -1) avg_loss = loss; avg_loss = avg_loss*.9 + loss*.1; printf("%ld, %.3f: %f, %f avg, %f rate, %lf seconds, %ld images\n", get_current_batch(net), (float)(*net->seen)/N, loss, avg_loss, get_current_rate(net), sec(clock()-time), *net->seen); free_data(train); if(*net->seen/N > epoch){ epoch = *net->seen/N; char buff[256]; sprintf(buff, "%s/%s_%d.weights",backup_directory,base, epoch); save_weights(net, buff); } if(get_current_batch(net)%100 == 0){ char buff[256]; sprintf(buff, "%s/%s.backup",backup_directory,base); save_weights(net, buff); } } char buff[256]; sprintf(buff, "%s/%s.weights", backup_directory, base); save_weights(net, buff); pthread_join(load_thread, 0); free_data(buffer); free_network(net); free_ptrs((void**)paths, plist->size); free_list(plist); free(base); }
int main ( int argc , // Number of command line arguments (includes prog name) char *argv[] // Arguments (prog name is argv[0]) ) { int i, j, k, nvars, ncases, irep, nreps, nbins, nbins_dep, nbins_indep, *count ; int n_indep_vars, idep, icand, *index, *mcpt_max_counts, *mcpt_same_counts, *mcpt_solo_counts ; short int *bins_dep, *bins_indep ; double *data, *work, dtemp, *save_info, criterion, *crits ; double *ab, *bc, *b ; char filename[256], **names, depname[256] ; FILE *fp ; /* Process command line parameters */ #if 1 if (argc != 6) { printf ( "\nUsage: TRANSFER datafile n_indep depname nreps" ) ; printf ( "\n datafile - name of the text file containing the data" ) ; printf ( "\n The first line is variable names" ) ; printf ( "\n Subsequent lines are the data." ) ; printf ( "\n Delimiters can be space, comma, or tab" ) ; printf ( "\n n_indep - Number of independent vars, starting with the first" ) ; printf ( "\n depname - Name of the 'dependent' variable" ) ; printf ( "\n It must be AFTER the first n_indep variables" ) ; printf ( "\n nbins - Number of bins for all variables" ) ; printf ( "\n nreps - Number of Monte-Carlo permutations, including unpermuted" ) ; exit ( 1 ) ; } strcpy ( filename , argv[1] ) ; n_indep_vars = atoi ( argv[2] ) ; strcpy ( depname , argv[3] ) ; nbins = atoi ( argv[4] ) ; nreps = atoi ( argv[5] ) ; #else strcpy ( filename , "..\\SYNTH.TXT" ) ; n_indep_vars = 7 ; strcpy ( depname , "SUM1234" ) ; nbins = 2 ; nreps = 1 ; #endif _strupr ( depname ) ; /* These are used by MEM.CPP for runtime memory validation */ _fullpath ( mem_file_name , "MEM.LOG" , 256 ) ; fp = fopen ( mem_file_name , "wt" ) ; if (fp == NULL) { // Should never happen printf ( "\nCannot open MEM.LOG file for writing!" ) ; return EXIT_FAILURE ; } fclose ( fp ) ; mem_keep_log = 1 ; // Change this to 1 to keep a memory use log (slows execution!) mem_max_used = 0 ; /* Open the text file to which results will be written */ fp = fopen ( "TRANSFER.LOG" , "wt" ) ; if (fp == NULL) { // Should never happen printf ( "\nCannot open TRANSFER.LOG file for writing!" ) ; return EXIT_FAILURE ; } /* Read the file and locate the index of the dependent variable */ if (readfile ( filename , &nvars , &names , &ncases , &data )) return EXIT_FAILURE ; for (idep=0 ; idep<nvars ; idep++) { if (! strcmp ( depname , names[idep] )) break ; } if (idep == nvars) { printf ( "\nERROR... Dependent variable %s is not in file", depname ) ; return EXIT_FAILURE ; } if (idep < n_indep_vars) { printf ( "\nERROR... Dependent variable %s must be beyond independent vars", depname ) ; return EXIT_FAILURE ; } /* Allocate scratch memory crits - Transfer Entropy criterion index - Indices that sort the criterion save_info - Ditto, this is univariate criteria, to be sorted */ MEMTEXT ( "TRANSFER work allocs" ) ; work = (double *) MALLOC ( ncases * sizeof(double) ) ; assert ( work != NULL ) ; crits = (double *) MALLOC ( n_indep_vars * sizeof(double) ) ; assert ( crits != NULL ) ; index = (int *) MALLOC ( n_indep_vars * sizeof(int) ) ; assert ( index != NULL ) ; bins_indep = (short int *) MALLOC ( ncases * sizeof(short int) ) ; assert ( bins_indep != NULL ) ; bins_dep = (short int *) MALLOC ( ncases * sizeof(short int) ) ; assert ( bins_dep != NULL ) ; mcpt_max_counts = (int *) MALLOC ( n_indep_vars * sizeof(int) ) ; assert ( mcpt_max_counts != NULL ) ; mcpt_same_counts = (int *) MALLOC ( n_indep_vars * sizeof(int) ) ; assert ( mcpt_same_counts != NULL ) ; mcpt_solo_counts = (int *) MALLOC ( n_indep_vars * sizeof(int) ) ; assert ( mcpt_solo_counts != NULL ) ; save_info = (double *) MALLOC ( n_indep_vars * sizeof(double) ) ; assert ( save_info != NULL ) ; count = (int *) MALLOC ( nbins * nbins * nbins * sizeof(int) ) ; assert ( count != NULL ) ; ab = (double *) MALLOC ( nbins * nbins * sizeof(double) ) ; assert ( ab != NULL ) ; bc = (double *) MALLOC ( nbins * nbins * sizeof(double) ) ; assert ( bc != NULL ) ; b = (double *) MALLOC ( nbins * sizeof(double) ) ; assert ( b != NULL ) ; /* Get the dependent variable and partition it */ for (i=0 ; i<ncases ; i++) // Get the 'dependent' variable work[i] = data[i*nvars+idep] ; nbins_dep = nbins ; partition ( ncases , work , &nbins_dep , NULL , bins_dep ) ; /* Replication loop is here */ for (irep=0 ; irep<nreps ; irep++) { /* Compute and save the transfer entropy of the dependent variable with each individual independent variable candidate. */ for (icand=0 ; icand<n_indep_vars ; icand++) { // Try all candidates for (i=0 ; i<ncases ; i++) work[i] = data[i*nvars+icand] ; // Shuffle independent variable if in permutation run (irep>0) if (irep) { // If doing permuted runs, shuffle i = ncases ; // Number remaining to be shuffled while (i > 1) { // While at least 2 left to shuffle j = (int) (unifrand () * i) ; if (j >= i) j = i - 1 ; dtemp = work[--i] ; work[i] = work[j] ; work[j] = dtemp ; } } nbins_indep = nbins ; partition ( ncases , work , &nbins_indep , NULL , bins_indep ) ; criterion = trans_ent ( ncases , nbins_indep , nbins_dep , bins_indep , bins_dep , 0 , 1 , 1 , count , ab , bc , b ) ; save_info[icand] = criterion ; // We will sort this when all candidates are done if (irep == 0) { // If doing original (unpermuted), save criterion index[icand] = icand ; // Will need original indices when criteria are sorted crits[icand] = criterion ; mcpt_max_counts[icand] = mcpt_same_counts[icand] = mcpt_solo_counts[icand] = 1 ; // This is >= itself so count it now } else { if (criterion >= crits[icand]) ++mcpt_solo_counts[icand] ; } } // Initial list of all candidates if (irep == 0) // Find the indices that sort the candidates per criterion qsortdsi ( 0 , n_indep_vars-1 , save_info , index ) ; else { qsortd ( 0 , n_indep_vars-1 , save_info ) ; for (icand=0 ; icand<n_indep_vars ; icand++) { if (save_info[icand] >= crits[index[icand]]) ++mcpt_same_counts[index[icand]] ; if (save_info[n_indep_vars-1] >= crits[index[icand]]) // Valid only for largest ++mcpt_max_counts[index[icand]] ; } } } // For all reps fprintf ( fp , "\nTransfer entropy of %s", depname); fprintf ( fp , "\n" ) ; fprintf ( fp , "\n" ) ; fprintf ( fp , "\nPredictors, in order of decreasing transfer entropy" ) ; fprintf ( fp , "\n" ) ; fprintf ( fp , "\n Variable Information Solo pval Min pval Max pval" ) ; for (icand=0 ; icand<n_indep_vars ; icand++) { // Do all candidates k = index[n_indep_vars-1-icand] ; // Index of sorted candidate fprintf ( fp , "\n%31s %11.5lf %12.4lf %10.4lf %10.4lf", names[k], crits[k], (double) mcpt_solo_counts[k] / nreps, (double) mcpt_same_counts[k] / nreps, (double) mcpt_max_counts[k] / nreps ) ; } MEMTEXT ( "TRANSFER: Finish" ) ; fclose ( fp ) ; FREE ( work ) ; FREE ( crits ) ; FREE ( index ) ; FREE ( bins_indep ) ; FREE ( bins_dep ) ; FREE ( mcpt_max_counts ) ; FREE ( mcpt_same_counts ) ; FREE ( mcpt_solo_counts ) ; FREE ( save_info ) ; FREE ( count ) ; FREE ( ab ) ; FREE ( bc ) ; FREE ( b ) ; free_data ( nvars , names , data ) ; MEMCLOSE () ; printf ( "\n\nPress any key..." ) ; _getch () ; return EXIT_SUCCESS ; }
void *gst_worker_cb(void *userdata) { CustomData *data = (CustomData *) userdata; gint64 current = -1; gint mean = 0; signal(SIGILL, sig_func); while (TRUE) { pthread_mutex_lock(&data->mutex); if (data != NULL && data->pipeline != NULL) { data->duration = -1; if (data->state >= GST_STATE_PAUSED) { if (gst_element_query_duration(data->pipeline, GST_FORMAT_TIME, &data->duration)) { if (data->duration > 0) { GPlayerDEBUG("detected duration: %0.3f", ((gfloat) data->duration / SECOND_IN_NANOS)); if ((gfloat) data->duration / SECOND_IN_NANOS < (gfloat) 15) { gint req_buffer_size = data->audio_info.rate * data->audio_info.channels * data->audio_info.finfo->width / 8 * (data->duration / SECOND_IN_NANOS); buffer_size(data, req_buffer_size); } } } if (data->duration == -1) { GPlayerDEBUG("NO duration, assuming stream!"); } } guint maxsizebytes; guint currentlevelbuffers; guint currentlevelbytes; g_object_get(data->buffer, "max-size-bytes", &maxsizebytes, NULL); g_object_get(data->buffer, "current-level-buffers", ¤tlevelbuffers, NULL); g_object_get(data->buffer, "current-level-bytes", ¤tlevelbytes, NULL); if (maxsizebytes > 0) { data->buffering_level = currentlevelbytes * HUNDRED_PERCENT / maxsizebytes; } GstState state; gst_element_get_state(data->pipeline, &state, NULL, GST_TIMEOUT); if (state == GST_STATE_PLAYING && data->target_state == GST_STATE_PAUSED) { GPlayerDEBUG("request GST_STATE_PAUSED"); data->is_live = (gst_element_set_state(data->pipeline, GST_STATE_PAUSED) == GST_STATE_CHANGE_NO_PREROLL); } if ((data->buffering_level >= (data->fast_network ? HUNDRED_PERCENT : HUNDRED_PERCENT / 2) || data->allow_seek || (data->buffering_level > 0 && data->buffering_time >= (data->fast_network ? BUFFERING_TIMEOUT * 2 : BUFFERING_TIMEOUT * 4))) && data->target_state == GST_STATE_PLAYING && (data->state == GST_STATE_PAUSED || (data->state == GST_STATE_READY && data->allow_seek))) { if (state != GST_STATE_PLAYING) { GPlayerDEBUG("request GST_STATE_PLAYING"); data->target_state = GST_STATE_PLAYING; data->buffering_time = 0; data->is_live = (gst_element_set_state(data->pipeline, GST_STATE_PLAYING) == GST_STATE_CHANGE_NO_PREROLL); gplayer_error(BUFFER_FAST, data); } } data->count_buffer_fill++; if ((data->count_buffer_fill == 8) && (data->position > data->last_position + 1999)) { data->no_buffer_fill = 0; data->count_buffer_fill = 0; data->last_position = data->position; } if (data->buffering_level < 5) { if (data->buffering_level == 0) { data->no_buffer_fill++; if (data->no_buffer_fill >= 8 && data->target_state == GST_STATE_PLAYING && state == GST_STATE_PLAYING && data->duration == -1) { GPlayerDEBUG("pausing, NO DATA"); gplayer_error(BUFFER_SLOW, data); data->is_live = (gst_element_set_state(data->pipeline, GST_STATE_PAUSED) == GST_STATE_CHANGE_NO_PREROLL); } } } if (data->count_buffer_fill == 20) { data->count_buffer_fill = 0; if (data->no_buffer_fill >= 16 && data->target_state == GST_STATE_PLAYING) { gplayer_error(ERROR_BUFFERING, data); } data->no_buffer_fill = 0; } data->counter++; if (data->last_buffer_load) { data->deltas[data->delta_index] = currentlevelbytes - data->last_buffer_load; data->delta_index++; data->delta_index %= 5; gint max = 0; gint min = 0; int i; gint acc = 0; for (i = 0; i < 5; i++) { max = MAX(data->deltas[i], max); min = MIN(data->deltas[i], min); acc += data->deltas[i]; } mean = (acc - min - max) / 3; if (data->counter >= 4) { data->counter = 0; if (data->audio_info.finfo == NULL) { break; } gint stream_speed = data->audio_info.channels * data->audio_info.rate * data->audio_info.finfo->width; guint buffer_size = currentlevelbytes * 8; gint buffer_delta = (currentlevelbytes - data->last_buffer_load) * 8; gfloat time_left = INFINITY; if (mean != 0) { time_left = buffer_size / (gfloat) (mean * 4 * 8); } if (time_left < 0) time_left = -time_left; GPlayerDEBUG("stream_speed: %lu Mbit/s, buffer_size: %lu Mbit, buffer_delta: %ld Mbit/s, time left: %.3f s", stream_speed, buffer_size, buffer_delta, time_left); if (data->duration > 0 && time_left != INFINITY) { gint64 position; gst_element_query_position(data->pipeline, GST_FORMAT_TIME, &position); guint64 buffered_ahead = (guint64) ((time_left + 3) * SECOND_IN_NANOS) + position; if (buffered_ahead < data->duration) { data->buffer_is_slow++; if (data->buffer_is_slow >= 5) { gplayer_error(BUFFER_SLOW, data); } } else { if (data->buffer_is_slow > 0) { data->buffer_is_slow = 0; gplayer_error(BUFFER_FAST, data); } } } else if (data->buffering_level < HUNDRED_PERCENT && time_left != INFINITY && time_left < 15 && data->duration == -1) { data->buffer_is_slow++; if (data->buffer_is_slow >= 5) { gplayer_error(BUFFER_SLOW, data); } } else { if (data->buffer_is_slow > 0) { data->buffer_is_slow = 0; gplayer_error(BUFFER_FAST, data); } } } } data->last_buffer_load = currentlevelbytes; data->buffering_time += WORKER_TIMEOUT; GPlayerDEBUG("mean: %8i, errors: %2i, ubuf: %3i, buf: %10i/%10i [%3i]", mean, data->no_buffer_fill, data->buffering_level, currentlevelbytes, maxsizebytes, currentlevelbuffers); } pthread_mutex_unlock(&data->mutex); errno = 0; usleep(WORKER_TIMEOUT * 1000); if (errno == EINTR) { errno = 0; GPlayerDEBUG("Worker destroyed... %p\n", pthread_self()); pthread_exit(NULL); if (pthread_self() == data->gst_worker_thread) free_data(data); return NULL; } } }
/* ====================================================================== */ ROKEN_LIB_FUNCTION int ROKEN_LIB_CALL rk_getifaddrs(struct ifaddrs **ifap) { int sd; struct nlmsg_list *nlmsg_list, *nlmsg_end, *nlm; /* - - - - - - - - - - - - - - - */ int icnt; size_t dlen, xlen, nlen; uint32_t max_ifindex = 0; pid_t pid = getpid(); int seq; int result; int build ; /* 0 or 1 */ /* ---------------------------------- */ /* initialize */ icnt = dlen = xlen = nlen = 0; nlmsg_list = nlmsg_end = NULL; if (ifap) *ifap = NULL; /* ---------------------------------- */ /* open socket and bind */ sd = nl_open(); if (sd < 0) return -1; /* ---------------------------------- */ /* gather info */ if ((seq = nl_getlist(sd, 0, RTM_GETLINK, &nlmsg_list, &nlmsg_end)) < 0){ free_nlmsglist(nlmsg_list); nl_close(sd); return -1; } if ((seq = nl_getlist(sd, seq+1, RTM_GETADDR, &nlmsg_list, &nlmsg_end)) < 0){ free_nlmsglist(nlmsg_list); nl_close(sd); return -1; } /* ---------------------------------- */ /* Estimate size of result buffer and fill it */ for (build=0; build<=1; build++){ struct ifaddrs *ifl = NULL, *ifa = NULL; struct nlmsghdr *nlh, *nlh0; char *data = NULL, *xdata = NULL; void *ifdata = NULL; char *ifname = NULL, **iflist = NULL; uint16_t *ifflist = NULL; struct rtmaddr_ifamap ifamap; if (build){ data = calloc(1, NLMSG_ALIGN(sizeof(struct ifaddrs[icnt])) + dlen + xlen + nlen); ifa = (struct ifaddrs *)data; ifdata = calloc(1, NLMSG_ALIGN(sizeof(char *[max_ifindex+1])) + NLMSG_ALIGN(sizeof(uint16_t [max_ifindex+1]))); if (ifap != NULL) *ifap = (ifdata != NULL) ? ifa : NULL; else{ free_data(data, ifdata); result = 0; break; } if (data == NULL || ifdata == NULL){ free_data(data, ifdata); result = -1; break; } ifl = NULL; data += NLMSG_ALIGN(sizeof(struct ifaddrs)) * icnt; xdata = data + dlen; ifname = xdata + xlen; iflist = ifdata; ifflist = (uint16_t *)(((char *)iflist) + NLMSG_ALIGN(sizeof(char *[max_ifindex+1]))); } for (nlm=nlmsg_list; nlm; nlm=nlm->nlm_next){ int nlmlen = nlm->size; if (!(nlh0 = nlm->nlh)) continue; for (nlh = nlh0; NLMSG_OK(nlh, nlmlen); nlh=NLMSG_NEXT(nlh,nlmlen)){ struct ifinfomsg *ifim = NULL; struct ifaddrmsg *ifam = NULL; struct rtattr *rta; size_t nlm_struct_size = 0; sa_family_t nlm_family = 0; uint32_t nlm_scope = 0, nlm_index = 0; size_t sockaddr_size = 0; uint32_t nlm_prefixlen = 0; size_t rtasize; memset(&ifamap, 0, sizeof(ifamap)); /* check if the message is what we want */ if (nlh->nlmsg_pid != pid || nlh->nlmsg_seq != nlm->seq) continue; if (nlh->nlmsg_type == NLMSG_DONE){ break; /* ok */ } switch (nlh->nlmsg_type){ case RTM_NEWLINK: ifim = (struct ifinfomsg *)NLMSG_DATA(nlh); nlm_struct_size = sizeof(*ifim); nlm_family = ifim->ifi_family; nlm_scope = 0; nlm_index = ifim->ifi_index; nlm_prefixlen = 0; if (build) ifflist[nlm_index] = ifa->ifa_flags = ifim->ifi_flags; break; case RTM_NEWADDR: ifam = (struct ifaddrmsg *)NLMSG_DATA(nlh); nlm_struct_size = sizeof(*ifam); nlm_family = ifam->ifa_family; nlm_scope = ifam->ifa_scope; nlm_index = ifam->ifa_index; nlm_prefixlen = ifam->ifa_prefixlen; if (build) ifa->ifa_flags = ifflist[nlm_index]; break; default: continue; } if (!build){ if (max_ifindex < nlm_index) max_ifindex = nlm_index; } else { if (ifl != NULL) ifl->ifa_next = ifa; } rtasize = NLMSG_PAYLOAD(nlh, nlmlen) - NLMSG_ALIGN(nlm_struct_size); for (rta = (struct rtattr *)(((char *)NLMSG_DATA(nlh)) + NLMSG_ALIGN(nlm_struct_size)); RTA_OK(rta, rtasize); rta = RTA_NEXT(rta, rtasize)){ struct sockaddr **sap = NULL; void *rtadata = RTA_DATA(rta); size_t rtapayload = RTA_PAYLOAD(rta); socklen_t sa_len; switch(nlh->nlmsg_type){ case RTM_NEWLINK: switch(rta->rta_type){ case IFLA_ADDRESS: case IFLA_BROADCAST: if (build){ sap = (rta->rta_type == IFLA_ADDRESS) ? &ifa->ifa_addr : &ifa->ifa_broadaddr; *sap = (struct sockaddr *)data; } sa_len = ifa_sa_len(AF_PACKET, rtapayload); if (rta->rta_type == IFLA_ADDRESS) sockaddr_size = NLMSG_ALIGN(sa_len); if (!build){ dlen += NLMSG_ALIGN(sa_len); } else { memset(*sap, 0, sa_len); ifa_make_sockaddr(AF_PACKET, *sap, rtadata,rtapayload, 0,0); ((struct sockaddr_ll *)*sap)->sll_ifindex = nlm_index; ((struct sockaddr_ll *)*sap)->sll_hatype = ifim->ifi_type; data += NLMSG_ALIGN(sa_len); } break; case IFLA_IFNAME:/* Name of Interface */ if (!build) nlen += NLMSG_ALIGN(rtapayload + 1); else{ ifa->ifa_name = ifname; if (iflist[nlm_index] == NULL) iflist[nlm_index] = ifa->ifa_name; strncpy(ifa->ifa_name, rtadata, rtapayload); ifa->ifa_name[rtapayload] = '\0'; ifname += NLMSG_ALIGN(rtapayload + 1); } break; case IFLA_STATS:/* Statistics of Interface */ if (!build) xlen += NLMSG_ALIGN(rtapayload); else{ ifa->ifa_data = xdata; memcpy(ifa->ifa_data, rtadata, rtapayload); xdata += NLMSG_ALIGN(rtapayload); } break; case IFLA_UNSPEC: break; case IFLA_MTU: break; case IFLA_LINK: break; case IFLA_QDISC: break; default: break; } break; case RTM_NEWADDR: if (nlm_family == AF_PACKET) break; switch(rta->rta_type){ case IFA_ADDRESS: ifamap.address = rtadata; ifamap.address_len = rtapayload; break; case IFA_LOCAL: ifamap.local = rtadata; ifamap.local_len = rtapayload; break; case IFA_BROADCAST: ifamap.broadcast = rtadata; ifamap.broadcast_len = rtapayload; break; #ifdef HAVE_IFADDRS_IFA_ANYCAST case IFA_ANYCAST: ifamap.anycast = rtadata; ifamap.anycast_len = rtapayload; break; #endif case IFA_LABEL: if (!build) nlen += NLMSG_ALIGN(rtapayload + 1); else{ ifa->ifa_name = ifname; if (iflist[nlm_index] == NULL) iflist[nlm_index] = ifname; strncpy(ifa->ifa_name, rtadata, rtapayload); ifa->ifa_name[rtapayload] = '\0'; ifname += NLMSG_ALIGN(rtapayload + 1); } break; case IFA_UNSPEC: break; case IFA_CACHEINFO: break; default: break; } } } if (nlh->nlmsg_type == RTM_NEWADDR && nlm_family != AF_PACKET) { if (!ifamap.local) { ifamap.local = ifamap.address; ifamap.local_len = ifamap.address_len; } if (!ifamap.address) { ifamap.address = ifamap.local; ifamap.address_len = ifamap.local_len; } if (ifamap.address_len != ifamap.local_len || (ifamap.address != NULL && memcmp(ifamap.address, ifamap.local, ifamap.address_len))) { /* p2p; address is peer and local is ours */ ifamap.broadcast = ifamap.address; ifamap.broadcast_len = ifamap.address_len; ifamap.address = ifamap.local; ifamap.address_len = ifamap.local_len; } if (ifamap.address) { #ifndef IFA_NETMASK sockaddr_size = NLMSG_ALIGN(ifa_sa_len(nlm_family,ifamap.address_len)); #endif if (!build) dlen += NLMSG_ALIGN(ifa_sa_len(nlm_family,ifamap.address_len)); else { ifa->ifa_addr = (struct sockaddr *)data; ifa_make_sockaddr(nlm_family, ifa->ifa_addr, ifamap.address, ifamap.address_len, nlm_scope, nlm_index); data += NLMSG_ALIGN(ifa_sa_len(nlm_family, ifamap.address_len)); } } #ifdef IFA_NETMASK if (ifamap.netmask) { if (!build) dlen += NLMSG_ALIGN(ifa_sa_len(nlm_family,ifamap.netmask_len)); else { ifa->ifa_netmask = (struct sockaddr *)data; ifa_make_sockaddr(nlm_family, ifa->ifa_netmask, ifamap.netmask, ifamap.netmask_len, nlm_scope, nlm_index); data += NLMSG_ALIGN(ifa_sa_len(nlm_family, ifamap.netmask_len)); } } #endif if (ifamap.broadcast) { if (!build) dlen += NLMSG_ALIGN(ifa_sa_len(nlm_family,ifamap.broadcast_len)); else { ifa->ifa_broadaddr = (struct sockaddr *)data; ifa_make_sockaddr(nlm_family, ifa->ifa_broadaddr, ifamap.broadcast, ifamap.broadcast_len, nlm_scope, nlm_index); data += NLMSG_ALIGN(ifa_sa_len(nlm_family, ifamap.broadcast_len)); } } #ifdef HAVE_IFADDRS_IFA_ANYCAST if (ifamap.anycast) { if (!build) dlen += NLMSG_ALIGN(ifa_sa_len(nlm_family,ifamap.anycast_len)); else { ifa->ifa_anycast = (struct sockaddr *)data; ifa_make_sockaddr(nlm_family, ifa->ifa_anyaddr, ifamap.anycast, ifamap.anycast_len, nlm_scope, nlm_index); data += NLMSG_ALIGN(ifa_sa_len(nlm_family, ifamap.anycast_len)); } } #endif } if (!build){ #ifndef IFA_NETMASK dlen += sockaddr_size; #endif icnt++; } else { if (ifa->ifa_name == NULL) ifa->ifa_name = iflist[nlm_index]; #ifndef IFA_NETMASK if (ifa->ifa_addr && ifa->ifa_addr->sa_family != AF_UNSPEC && ifa->ifa_addr->sa_family != AF_PACKET){ ifa->ifa_netmask = (struct sockaddr *)data; ifa_make_sockaddr_mask(ifa->ifa_addr->sa_family, ifa->ifa_netmask, nlm_prefixlen); } data += sockaddr_size; #endif ifl = ifa++; } } } if (!build){ if (icnt == 0 && (dlen + nlen + xlen == 0)){ if (ifap != NULL) *ifap = NULL; break; /* cannot found any addresses */ } } else free_data(NULL, ifdata); } /* ---------------------------------- */ /* Finalize */ free_nlmsglist(nlmsg_list); nl_close(sd); return 0; }
int main ( int argc , // Number of command line arguments (includes prog name) char *argv[] // Arguments (prog name is argv[0]) ) { int i, j, k, nvars, ncases, ndiv, maxkept, ivar, nties, ties ; int n_indep_vars, idep, icand, iother, ibest, *sortwork, nkept, *kept ; double *data, *work ; double *save_info, *univar_info, *pair_info, bestredun, redun, bestcrit ; double criterion, relevance, redundancy, *crits, *reduns ; char filename[256], **names, depname[256] ; char trial_name[256], *pair_found ; FILE *fp ; MutualInformationParzen *mi_parzen ; MutualInformationAdaptive *mi_adapt ; /* Process command line parameters */ #if 1 if (argc != 6) { printf ( "\nUsage: MI_CONT datafile n_indep depname ndiv maxkept" ) ; printf ( "\n datafile - name of the text file containing the data" ) ; printf ( "\n The first line is variable names" ) ; printf ( "\n Subsequent lines are the data." ) ; printf ( "\n Delimiters can be space, comma, or tab" ) ; printf ( "\n n_indep - Number of independent vars, starting with the first" ) ; printf ( "\n depname - Name of the 'dependent' variable" ) ; printf ( "\n It must be AFTER the first n_indep variables" ) ; printf ( "\n ndiv - Normally zero, to employ adaptive partitioning" ) ; printf ( "\n Specify 5 (for very few cases) to 15 (for an" ) ; printf ( "\n enormous number of cases) to use Parzen windows" ) ; printf ( "\n maxkept - Stepwise will allow at most this many predictors" ) ; exit ( 1 ) ; } strcpy ( filename , argv[1] ) ; n_indep_vars = atoi ( argv[2] ) ; strcpy ( depname , argv[3] ) ; ndiv = atoi ( argv[4] ) ; maxkept = atoi ( argv[5] ) ; #else strcpy ( filename , "..\\VARS.TXT" ) ; n_indep_vars = 8 ; strcpy ( depname , "DAY_RETURN" ) ; ndiv = 0 ; maxkept = 5 ; #endif _strupr ( depname ) ; /* These are used by MEM.CPP for runtime memory validation */ _fullpath ( mem_file_name , "MEM.LOG" , 256 ) ; fp = fopen ( mem_file_name , "wt" ) ; if (fp == NULL) { // Should never happen printf ( "\nCannot open MEM.LOG file for writing!" ) ; return EXIT_FAILURE ; } fclose ( fp ) ; mem_keep_log = 1 ; mem_max_used = 0 ; /* Open the text file to which results will be written */ fp = fopen ( "MI_CONT.LOG" , "wt" ) ; if (fp == NULL) { // Should never happen printf ( "\nCannot open MI_CONT.LOG file for writing!" ) ; return EXIT_FAILURE ; } /* Read the file and locate the index of the 'dependent' variable */ if (readfile ( filename , &nvars , &names , &ncases , &data )) return EXIT_FAILURE ; for (idep=0 ; idep<nvars ; idep++) { if (! strcmp ( depname , names[idep] )) break ; } if (idep == nvars) { printf ( "\nERROR... Dependent variable %s is not in file", depname ) ; return EXIT_FAILURE ; } if (idep < n_indep_vars) { printf ( "\nERROR... Dependent variable %s must be beyond independent vars", depname ) ; return EXIT_FAILURE ; } /* If adaptive partitioning is specified, check each variable for ties. This is not needed for the algorithm, but it is good to warn the user, because more than a very few tied values in any variable seriously degrades performance of the adaptive partitioning algorithm. */ MEMTEXT ( "MI_CONT: Work" ) ; work = (double *) MALLOC ( ncases * sizeof(double) ) ; assert ( work != NULL ) ; if (ndiv == 0) { // If adaptive partitioning, check for ties ties = 0 ; assert ( work != NULL ) ; for (ivar=0 ; ivar<nvars ; ivar++) { if (ivar > n_indep_vars && ivar != idep) continue ; // Check only the variables selected by the user for (i=0 ; i<ncases ; i++) work[i] = data[i*nvars+ivar] ; qsortd ( 0 , ncases-1 , work ) ; nties = 0 ; for (i=1 ; i<ncases ; i++) { if (work[i] == work[i-1]) ++nties ; } if ((double) nties / (double) ncases > 0.05) { ++ties ; fprintf ( fp , "\nWARNING... %s has %.2lf percent ties!", names[ivar], 100.0 * nties / (double) ncases ) ; } } // For all variables if (ties) { fprintf ( fp , "\nThe presence of ties will seriously degrade" ) ; fprintf ( fp , "\nperformance of the adaptive partitioning algorithm\n\n" ) ; } } // If adaptive partitioning, so testing for ties in the data /* Allocate scratch memory and create the MutualInformation object using the dependent variable kept - Array of indices of variables kept so far crits - Ditto, criterion reduns - Ditto, redundancy sortwork - Temporary use for printing variable's information sorted save_info - Ditto, this is univariate information, to be sorted univar_info - Also univariate information, but not sorted, for use in stepwise pair_found - Flag: is there valid info in the corresponding element of the next array pair_info - Preserve pairwise information of indeps to avoid expensive recalculation mi_parzen - The MutualInformation object, constructed with the 'dependent' variable mi_adapt - Ditto, but used if adaptive partitioning */ MEMTEXT ( "MI_CONT 6 allocs plus MutualInformation" ) ; kept = (int *) MALLOC ( n_indep_vars * sizeof(int) ) ; assert ( kept != NULL ) ; crits = (double *) MALLOC ( n_indep_vars * sizeof(double) ) ; assert ( crits != NULL ) ; reduns = (double *) MALLOC ( n_indep_vars * sizeof(double) ) ; assert ( reduns != NULL ) ; sortwork = (int *) MALLOC ( n_indep_vars * sizeof(int) ) ; assert ( sortwork != NULL ) ; save_info = (double *) MALLOC ( n_indep_vars * sizeof(double) ) ; assert ( save_info != NULL ) ; univar_info = (double *) MALLOC ( n_indep_vars * sizeof(double) ) ; assert ( univar_info != NULL ) ; pair_found = (char *) MALLOC ( (n_indep_vars * (n_indep_vars+1) / 2) * sizeof(char) ) ; assert ( pair_found != NULL ) ; pair_info = (double *) MALLOC ( (n_indep_vars * (n_indep_vars+1) / 2) * sizeof(double) ) ; assert ( pair_info != NULL ) ; for (i=0 ; i<ncases ; i++) // Get the 'dependent' variable work[i] = data[i*nvars+idep] ; if (ndiv > 0) { mi_parzen = new MutualInformationParzen ( ncases , work , ndiv ) ; mi_adapt = NULL ; assert ( mi_parzen != NULL ) ; } else { mi_adapt = new MutualInformationAdaptive ( ncases , work , 0 , 6.0 ) ; mi_parzen = NULL ; assert ( mi_adapt != NULL ) ; } memset ( pair_found , 0 , (n_indep_vars * (n_indep_vars+1) / 2) * sizeof(char) ) ; if (ndiv > 0) fprintf ( fp , "\nParzen mutual information of %s (ndiv=%d)", depname, ndiv); else fprintf ( fp , "\nAdaptive partitioning mutual information of %s", depname); fprintf ( fp , "\n" ) ; fprintf ( fp , "\n---------------------------------------------------------------" ) ; fprintf ( fp , "\n" ) ; /* Compute and save the mutual information for the dependent variable with each individual independent variable candidate. Print the results, sort them, and print them again, this time sorted. */ fprintf ( fp , "\nInitial candidates, in order of appearance in data file" ) ; fprintf ( fp , "\n" ) ; fprintf ( fp , "\n Variable Information" ) ; for (icand=0 ; icand<n_indep_vars ; icand++) { // Try all candidates for (i=0 ; i<ncases ; i++) work[i] = data[i*nvars+icand] ; if (ndiv > 0) criterion = mi_parzen->mut_inf ( work ) ; else criterion = mi_adapt->mut_inf ( work , 0 ) ; printf ( "\n%s = %.5lf", names[icand], criterion ) ; fprintf ( fp , "\n%31s %.5lf", names[icand], criterion ) ; sortwork[icand] = icand ; save_info[icand] = univar_info[icand] = criterion ; } // Initial list of all candidates if (mi_parzen != NULL) { delete mi_parzen ; mi_parzen = NULL ; } if (mi_adapt != NULL) { delete mi_adapt ; mi_adapt = NULL ; } fprintf ( fp , "\n" ) ; fprintf ( fp , "\nInitial candidates, in order of decreasing mutual information" ) ; fprintf ( fp , "\n" ) ; fprintf ( fp , "\n Variable Information" ) ; qsortdsi ( 0 , n_indep_vars-1 , save_info , sortwork ) ; for (icand=0 ; icand<n_indep_vars ; icand++) { // Do all candidates k = sortwork[n_indep_vars-1-icand] ; // Index of sorted candidate fprintf ( fp , "\n%31s %.5lf", names[k], save_info[n_indep_vars-1-icand] ) ; } /* Initialize the 'kept' set to be the best variable, and then begin the main outer loop that adds variables one at a time */ kept[0] = sortwork[n_indep_vars-1] ; // Index of best single candidate crits[0] = save_info[n_indep_vars-1] ; reduns[0] = 0.0 ; nkept = 1 ; if (maxkept > n_indep_vars) // Guard against silly user maxkept = n_indep_vars ; while (nkept < maxkept) { fprintf ( fp , "\n" ) ; fprintf ( fp , "\nVariables so far Relevance Redundancy Criterion" ) ; for (i=0 ; i<nkept ; i++) fprintf ( fp , "\n%31s %10.5lf %10.5lf %10.5lf", names[kept[i]], crits[i] + reduns[i], reduns[i], crits[i] ) ; fprintf ( fp , "\n" ) ; fprintf ( fp , "\nSearching for an additional candidate..." ) ; fprintf ( fp , "\n" ) ; fprintf ( fp , "\n Variable Relevance Redundancy Criterion" ) ; bestcrit = -1.e60 ; for (icand=0 ; icand<n_indep_vars ; icand++) { // Try all candidates for (i=0 ; i<nkept ; i++) { // Is this candidate already kept? if (kept[i] == icand) break ; } if (i < nkept) // If this candidate 'icand' is already kept continue ; // Skip it strcpy ( trial_name , names[icand] ) ; // Its name for printing for (i=0 ; i<ncases ; i++) // Get its cases work[i] = data[i*nvars+icand] ; if (ndiv > 0) { mi_parzen = new MutualInformationParzen ( ncases , work , ndiv ) ; mi_adapt = NULL ; assert ( mi_parzen != NULL ) ; } else { mi_adapt = new MutualInformationAdaptive ( ncases , work , 0 , 6.0 ) ; mi_parzen = NULL ; assert ( mi_adapt != NULL ) ; } relevance = univar_info[icand] ; // We saved it during initial printing printf ( "\n%s relevance = %.5lf", trial_name, relevance ) ; // Compute the redundancy of this candidate // This is the mean of its redundancy with all kept variables redundancy = 0.0 ; for (iother=0 ; iother<nkept ; iother++) { // Process entire kept set j = kept[iother] ; // Index of a variable in the kept set if (icand > j) // pair_found and pair_info are k = icand*(icand+1)/2+j ; // symmetric, so k is the index else // into them k = j*(j+1)/2+icand ; if (pair_found[k]) // If we already computed it redun = pair_info[k] ; // Don't do it again else { // First time for this pair, so compute for (i=0 ; i<ncases ; i++) // Get its cases work[i] = data[i*nvars+j] ; // Variable already in kept set if (ndiv > 0) redun = mi_parzen->mut_inf ( work ) ; else redun = mi_adapt->mut_inf ( work , 0 ) ; pair_found[k] = 1 ; // Flag that this pair has been computed pair_info[k] = redun ; // And save the MI for this pair } // Else must compute redundancy redundancy += redun ; printf ( "\n %s <-> %s redundancy = %.5lf", names[icand], names[j], redun ) ; } // For all kept variables, computing mean redundancy if (mi_parzen != NULL) { delete mi_parzen ; mi_parzen = NULL ; } if (mi_adapt != NULL) { delete mi_adapt ; mi_adapt = NULL ; } redundancy /= nkept ; // It is the mean across all kept printf ( "\nRedundancy = %.5lf", redundancy ) ; criterion = relevance - redundancy ; fprintf ( fp , "\n%31s %10.5lf %10.5lf %10.5lf", trial_name, relevance, redundancy, criterion ) ; if (criterion > bestcrit) { // Did we just set a new record? bestcrit = criterion ; // If so, update the record bestredun = redundancy ; // Needed for printing results later ibest = icand ; // Keep track of the winning candidate } } // For all candidates // We now have the best candidate if (bestcrit <= 0.0) break ; kept[nkept] = ibest ; crits[nkept] = bestcrit ; reduns[nkept] = bestredun ; ++nkept ; } // While adding new variables fprintf ( fp , "\n" ) ; fprintf ( fp , "\nFinal set Relevance Redundancy Criterion" ) ; for (i=0 ; i<nkept ; i++) fprintf ( fp , "\n%31s %10.5lf %10.5lf %10.5lf", names[kept[i]], crits[i] + reduns[i], reduns[i], crits[i] ) ; MEMTEXT ( "MI_CONT: Finish" ) ; fclose ( fp ) ; FREE ( work ) ; FREE ( kept ) ; FREE ( crits ) ; FREE ( reduns ) ; FREE ( sortwork ) ; FREE ( save_info ) ; FREE ( univar_info ) ; FREE ( pair_found ) ; FREE ( pair_info ) ; if (mi_parzen != NULL) delete mi_parzen ; if (mi_adapt != NULL) delete mi_adapt ; free_data ( nvars , names , data ) ; MEMCLOSE () ; printf ( "\n\nPress any key..." ) ; _getch () ; return EXIT_SUCCESS ; }
UpdatePlugin::~UpdatePlugin() { free_data(updateData, &data); if (m_msg) delete m_msg; }
RemotePlugin::~RemotePlugin() { free_data(remoteData, &data); }
void ap_free( struct Arg_parser * const ap ) { free_data( ap ); if( ap->error ) { free( ap->error ); ap->error = 0; } ap->error_size = 0; }
static void save_got_message(CamelFolder *folder, const char *uid, CamelMimeMessage *msg, void *d) { struct _save_data *data = d; GtkDialog *dialog; GtkWidget *w, *tree; GtkTreeStore *model; GtkCellRenderer *renderer; /* not found, the mailer will show an error box for this */ if (msg == NULL) { free_data(data); return; } data->msg = msg; camel_object_ref(msg); dialog = (GtkDialog *)gtk_dialog_new_with_buttons(_("Save attachments"), NULL, /* target->parent? */ 0, GTK_STOCK_CANCEL, GTK_RESPONSE_CANCEL, GTK_STOCK_SAVE, GTK_RESPONSE_OK, NULL); w = gtk_file_chooser_button_new (_("Select save base name"), GTK_FILE_CHOOSER_ACTION_OPEN); data->entry = w; g_object_set(w, "filechooser_action", GTK_FILE_CHOOSER_ACTION_SAVE, NULL); gtk_widget_show(w); gtk_box_pack_start((GtkBox *)dialog->vbox, w, FALSE, TRUE, 6); g_signal_connect(GTK_FILE_CHOOSER_BUTTON (w), "selection-changed", G_CALLBACK(entry_changed), data); model = gtk_tree_store_new(5, G_TYPE_BOOLEAN, G_TYPE_STRING, G_TYPE_STRING, G_TYPE_STRING, G_TYPE_POINTER); data->model = model; fill_model(msg, model); tree = gtk_tree_view_new_with_model((GtkTreeModel *)model); data->tree = tree; gtk_widget_show(tree); gtk_tree_view_expand_all((GtkTreeView *)tree); renderer = gtk_cell_renderer_text_new(); gtk_tree_view_insert_column_with_attributes((GtkTreeView *)tree, -1, _("MIME Type"), renderer, "text", 1, NULL); gtk_tree_view_set_expander_column((GtkTreeView *)tree, gtk_tree_view_get_column((GtkTreeView *)tree, 0)); renderer = gtk_cell_renderer_toggle_new(); g_object_set(renderer, "activatable", TRUE, NULL); g_signal_connect(renderer, "toggled", G_CALLBACK(toggle_changed), data); gtk_tree_view_insert_column_with_attributes((GtkTreeView *)tree, -1, _("Save"), renderer, "active", 0, NULL); renderer = gtk_cell_renderer_text_new(); gtk_tree_view_insert_column_with_attributes((GtkTreeView *)tree, -1, _("Name"), renderer, "text", 2, NULL); w = g_object_new(gtk_frame_get_type(), "shadow_type", GTK_SHADOW_NONE, "label_widget", g_object_new(gtk_label_get_type(), "label", "<span weight=\"bold\">Attachments</span>", "use_markup", TRUE, "xalign", 0.0, NULL), "child", g_object_new(gtk_alignment_get_type(), "left_padding", 12, "top_padding", 6, "child", g_object_new(gtk_scrolled_window_get_type(), "hscrollbar_policy", GTK_POLICY_AUTOMATIC, "vscrollbar_policy", GTK_POLICY_AUTOMATIC, "shadow_type", GTK_SHADOW_IN, "child", tree, NULL), NULL), NULL); gtk_widget_show_all(w); gtk_box_pack_start((GtkBox *)dialog->vbox, w, TRUE, TRUE, 0); g_signal_connect(dialog, "response", G_CALLBACK(save_response), data); gtk_window_set_default_size((GtkWindow *)dialog, 500, 500); gtk_widget_show((GtkWidget *)dialog); }
void train_classifier(char *datacfg, char *cfgfile, char *weightfile, int *gpus, int ngpus, int clear) { int i; float avg_loss = -1; char *base = basecfg(cfgfile); printf("%s\n", base); printf("%d\n", ngpus); network *nets = calloc(ngpus, sizeof(network)); srand(time(0)); int seed = rand(); for(i = 0; i < ngpus; ++i){ srand(seed); #ifdef GPU cuda_set_device(gpus[i]); #endif nets[i] = load_network(cfgfile, weightfile, clear); nets[i].learning_rate *= ngpus; } srand(time(0)); network net = nets[0]; int imgs = net.batch * net.subdivisions * ngpus; printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay); list *options = read_data_cfg(datacfg); char *backup_directory = option_find_str(options, "backup", "/backup/"); char *label_list = option_find_str(options, "labels", "data/labels.list"); char *train_list = option_find_str(options, "train", "data/train.list"); int classes = option_find_int(options, "classes", 2); char **labels = get_labels(label_list); list *plist = get_paths(train_list); char **paths = (char **)list_to_array(plist); printf("%d\n", plist->size); int N = plist->size; double time; load_args args = {0}; args.w = net.w; args.h = net.h; args.threads = 32; args.hierarchy = net.hierarchy; args.min = net.min_crop; args.max = net.max_crop; args.angle = net.angle; args.aspect = net.aspect; args.exposure = net.exposure; args.saturation = net.saturation; args.hue = net.hue; args.size = net.w; args.paths = paths; args.classes = classes; args.n = imgs; args.m = N; args.labels = labels; args.type = CLASSIFICATION_DATA; data train; data buffer; pthread_t load_thread; args.d = &buffer; load_thread = load_data(args); int epoch = (*net.seen)/N; while(get_current_batch(net) < net.max_batches || net.max_batches == 0){ time = what_time_is_it_now(); pthread_join(load_thread, 0); train = buffer; load_thread = load_data(args); printf("Loaded: %lf seconds\n", what_time_is_it_now()-time); time = what_time_is_it_now(); float loss = 0; #ifdef GPU if(ngpus == 1){ loss = train_network(net, train); } else { loss = train_networks(nets, ngpus, train, 4); } #else loss = train_network(net, train); #endif if(avg_loss == -1) avg_loss = loss; avg_loss = avg_loss*.9 + loss*.1; printf("%ld, %.3f: %f, %f avg, %f rate, %lf seconds, %ld images\n", get_current_batch(net), (float)(*net.seen)/N, loss, avg_loss, get_current_rate(net), what_time_is_it_now()-time, *net.seen); free_data(train); if(*net.seen/N > epoch){ epoch = *net.seen/N; char buff[256]; sprintf(buff, "%s/%s_%d.weights",backup_directory,base, epoch); save_weights(net, buff); } if(get_current_batch(net)%1000 == 0){ char buff[256]; sprintf(buff, "%s/%s.backup",backup_directory,base); save_weights(net, buff); } } char buff[256]; sprintf(buff, "%s/%s.weights", backup_directory, base); save_weights(net, buff); pthread_join(load_thread, 0); free_network(net); free_ptrs((void**)labels, classes); free_ptrs((void**)paths, plist->size); free_list(plist); free(base); }
LoggerPlugin::~LoggerPlugin() { if (m_file) delete m_file; free_data(loggerData, &data); }
char ap_init(Arg_parser * ap, const int argc, const char *const argv[], const ap_Option options[], const char in_order) { const char **non_options = 0; // skipped non-options int non_options_size = 0; // number of skipped non-options int argind = 1; // index in argv int i; ap->data = 0; ap->error = 0; ap->data_size = 0; ap->error_size = 0; if (argc < 2 || !argv || !options) return 1; while (argind < argc) { const unsigned char ch1 = argv[argind][0]; const unsigned char ch2 = (ch1 ? argv[argind][1] : 0); if (ch1 == '-' && ch2) // we found an option { const char *const opt = argv[argind]; const char *const arg = (argind + 1 < argc) ? argv[argind + 1] : 0; if (ch2 == '-') { if (!argv[argind][2]) { ++argind; break; } // we found "--" else if (!parse_long_option(ap, opt, arg, options, &argind)) return 0; } else if (!parse_short_option(ap, opt, arg, options, &argind)) return 0; if (ap->error) break; } else { if (!in_order) { if (!ap_resize_buffer((void *) &non_options, (non_options_size + 1) * sizeof(*non_options))) return 0; non_options[non_options_size++] = argv[argind++]; } else if (!push_back_record(ap, 0, argv[argind++])) return 0; } } if (ap->error) free_data(ap); else { for (i = 0; i < non_options_size; ++i) if (!push_back_record(ap, 0, non_options[i])) return 0; while (argind < argc) if (!push_back_record(ap, 0, argv[argind++])) return 0; } if (non_options) free(non_options); return 1; }
int main() { TData data; alloc_data(&data); free_data(data); return 0; }
static void free_node(void *p) { free_data((KEYWORD *) p); }
SearchInfoRequest::~SearchInfoRequest() { free_data(jabberSearchInfo, &data); }
/* Quit the main loop, remove the native thread and free resources */ void gst_native_finalize(JNIEnv* env, jobject thiz) { CustomData *data = GET_CUSTOM_DATA(env, thiz, custom_data_field_id); GPlayerDEBUG("gst_native_finalize() %p", data); free_data(data); }
Handle::~Handle() { free_data(); invalidate(); }
void train_swag(char *cfgfile, char *weightfile) { char *train_images = "data/voc.0712.trainval"; char *backup_directory = "/home/pjreddie/backup/"; srand(time(0)); data_seed = time(0); char *base = basecfg(cfgfile); printf("%s\n", base); float avg_loss = -1; network net = parse_network_cfg(cfgfile); if(weightfile){ load_weights(&net, weightfile); } printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay); int imgs = net.batch*net.subdivisions; int i = *net.seen/imgs; data train, buffer; layer l = net.layers[net.n - 1]; int side = l.side; int classes = l.classes; float jitter = l.jitter; list *plist = get_paths(train_images); //int N = plist->size; char **paths = (char **)list_to_array(plist); load_args args = {0}; args.w = net.w; args.h = net.h; args.paths = paths; args.n = imgs; args.m = plist->size; args.classes = classes; args.jitter = jitter; args.num_boxes = side; args.d = &buffer; args.type = REGION_DATA; pthread_t load_thread = load_data_in_thread(args); clock_t time; //while(i*imgs < N*120){ while(get_current_batch(net) < net.max_batches){ i += 1; time=clock(); pthread_join(load_thread, 0); train = buffer; load_thread = load_data_in_thread(args); printf("Loaded: %lf seconds\n", sec(clock()-time)); time=clock(); float loss = train_network(net, train); if (avg_loss < 0) avg_loss = loss; avg_loss = avg_loss*.9 + loss*.1; printf("%d: %f, %f avg, %f rate, %lf seconds, %d images\n", i, loss, avg_loss, get_current_rate(net), sec(clock()-time), i*imgs); if(i%1000==0 || i == 600){ char buff[256]; sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i); save_weights(net, buff); } free_data(train); } char buff[256]; sprintf(buff, "%s/%s_final.weights", backup_directory, base); save_weights(net, buff); }
void forget_object() { free_data(); invalidate(); object_ = 0; }
JabberClient::~JabberClient() { TCPClient::setStatus(STATUS_OFFLINE, false); free_data(jabberClientData, &data); }
LiveJournalClient::~LiveJournalClient() { if (m_request) delete m_request; free_data(liveJournalClientData, &data); }
ProxyData::~ProxyData() { if (bInit) free_data(_proxyData, this); }
JournalMessage::~JournalMessage() { free_data(journalMessageData, &data); }
AgentRequest::~AgentRequest() { free_data(jabberAgentsInfo, &data); }
uint32_t start_app(uint32_t from, const wchar_t *path, const wchar_t *arguments, const wchar_t *curdir, uint32_t *tid, int show_window) { create_process_t s; memset(&s, 0, sizeof(s)); s.si.cb = sizeof(s.si); // Emulate explorer.exe's startupinfo flags behavior. s.si.dwFlags = STARTF_USESHOWWINDOW; s.si.wShowWindow = show_window; s.create_process_w = resolve_symbol("kernel32", "CreateProcessW"); s.get_last_error = resolve_symbol("kernel32", "GetLastError"); wchar_t *cmd_line = malloc(strsizeW(path) + strsizeW(arguments) + 4 * sizeof(wchar_t)); wsprintfW(cmd_line, L"\"%s\" %s", path, arguments); s.filepath = write_data(from, path, strsizeW(path)); s.cmdline = write_data(from, cmd_line, strsizeW(cmd_line)); s.curdir = write_data(from, curdir, strsizeW(curdir)); create_process_t *settings_addr = write_data(from, &s, sizeof(s)); void *shellcode_addr = write_data(from, &create_process_worker, 0x1000); uint32_t last_error = create_thread_and_wait(from, shellcode_addr, settings_addr); if(last_error != 0) { error("[-] Error launching process: %d\n", last_error); } read_data(from, settings_addr, &s, sizeof(s)); free_data(from, s.curdir, strsizeW(curdir)); free_data(from, s.cmdline, strsizeW(cmd_line)); free_data(from, s.filepath, strsizeW(path)); free_data(from, shellcode_addr, 0x1000); free_data(from, settings_addr, sizeof(s)); free(cmd_line); HANDLE process_handle = open_process(from), object_handle; if(DuplicateHandle(process_handle, s.pi.hThread, GetCurrentProcess(), &object_handle, DUPLICATE_SAME_ACCESS, FALSE, DUPLICATE_CLOSE_SOURCE) != FALSE) { CloseHandle(object_handle); } if(DuplicateHandle(process_handle, s.pi.hProcess, GetCurrentProcess(), &object_handle, DUPLICATE_SAME_ACCESS, FALSE, DUPLICATE_CLOSE_SOURCE) != FALSE) { CloseHandle(object_handle); } CloseHandle(process_handle); if(tid != NULL) { *tid = s.pi.dwThreadId; } return s.pi.dwProcessId; }
void test_classifier(char *datacfg, char *cfgfile, char *weightfile, int target_layer) { int curr = 0; network net = parse_network_cfg(cfgfile); if(weightfile){ load_weights(&net, weightfile); } srand(time(0)); list *options = read_data_cfg(datacfg); char *test_list = option_find_str(options, "test", "data/test.list"); int classes = option_find_int(options, "classes", 1000); list *plist = get_paths(test_list); char **paths = (char **)list_to_array(plist); int m = plist->size; free_list(plist); clock_t time; data val, buffer; load_args args = {0}; args.w = net.w; args.h = net.h; args.paths = paths; args.classes = classes; args.n = net.batch; args.m = 0; args.labels = 0; args.d = &buffer; args.type = CLASSIFICATION_DATA; pthread_t load_thread = load_data_in_thread(args); for(curr = net.batch; curr < m; curr += net.batch){ time=clock(); pthread_join(load_thread, 0); val = buffer; if(curr < m){ args.paths = paths + curr; if (curr + net.batch > m) args.n = m - curr; load_thread = load_data_in_thread(args); } fprintf(stderr, "Loaded: %d images in %lf seconds\n", val.X.rows, sec(clock()-time)); time=clock(); matrix pred = network_predict_data(net, val); int i, j; if (target_layer >= 0){ //layer l = net.layers[target_layer]; } for(i = 0; i < pred.rows; ++i){ printf("%s", paths[curr-net.batch+i]); for(j = 0; j < pred.cols; ++j){ printf("\t%g", pred.vals[i][j]); } printf("\n"); } free_matrix(pred); fprintf(stderr, "%lf seconds, %d images, %d total\n", sec(clock()-time), val.X.rows, curr); free_data(val); } }
void train_dcgan(char *cfg, char *weight, char *acfg, char *aweight, int clear, int display, char *train_images) { #ifdef GPU //char *train_images = "/home/pjreddie/data/coco/train1.txt"; //char *train_images = "/home/pjreddie/data/coco/trainvalno5k.txt"; //char *train_images = "/home/pjreddie/data/imagenet/imagenet1k.train.list"; //char *train_images = "data/64.txt"; //char *train_images = "data/alp.txt"; //char *train_images = "data/cifar.txt"; char *backup_directory = "/home/pjreddie/backup/"; srand(time(0)); char *base = basecfg(cfg); char *abase = basecfg(acfg); printf("%s\n", base); network *gnet = load_network(cfg, weight, clear); network *anet = load_network(acfg, aweight, clear); //float orig_rate = anet->learning_rate; int start = 0; int i, j, k; layer imlayer = {0}; for (i = 0; i < gnet->n; ++i) { if (gnet->layers[i].out_c == 3) { imlayer = gnet->layers[i]; break; } } printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", gnet->learning_rate, gnet->momentum, gnet->decay); int imgs = gnet->batch*gnet->subdivisions; i = *gnet->seen/imgs; data train, buffer; list *plist = get_paths(train_images); //int N = plist->size; char **paths = (char **)list_to_array(plist); load_args args= get_base_args(anet); args.paths = paths; args.n = imgs; args.m = plist->size; args.d = &buffer; args.type = CLASSIFICATION_DATA; args.threads=16; args.classes = 1; char *ls[2] = {"imagenet", "zzzzzzzz"}; args.labels = ls; pthread_t load_thread = load_data_in_thread(args); clock_t time; gnet->train = 1; anet->train = 1; int x_size = gnet->inputs*gnet->batch; int y_size = gnet->truths*gnet->batch; float *imerror = cuda_make_array(0, y_size); //int ay_size = anet->truths*anet->batch; float aloss_avg = -1; //data generated = copy_data(train); while (get_current_batch(gnet) < gnet->max_batches) { start += 1; i += 1; time=clock(); pthread_join(load_thread, 0); train = buffer; //translate_data_rows(train, -.5); //scale_data_rows(train, 2); load_thread = load_data_in_thread(args); printf("Loaded: %lf seconds\n", sec(clock()-time)); data gen = copy_data(train); for (j = 0; j < imgs; ++j) { train.y.vals[j][0] = .95; gen.y.vals[j][0] = .05; } time=clock(); for(j = 0; j < gnet->subdivisions; ++j){ get_next_batch(train, gnet->batch, j*gnet->batch, gnet->truth, 0); int z; for(z = 0; z < x_size; ++z){ gnet->input[z] = rand_normal(); } cuda_push_array(gnet->input_gpu, gnet->input, x_size); cuda_push_array(gnet->truth_gpu, gnet->truth, y_size); *gnet->seen += gnet->batch; forward_network_gpu(gnet); fill_gpu(imlayer.outputs*imlayer.batch, 0, imerror, 1); fill_gpu(anet->truths*anet->batch, .95, anet->truth_gpu, 1); copy_gpu(anet->inputs*anet->batch, imlayer.output_gpu, 1, anet->input_gpu, 1); anet->delta_gpu = imerror; forward_network_gpu(anet); backward_network_gpu(anet); float genaloss = *anet->cost / anet->batch; printf("%f\n", genaloss); scal_gpu(imlayer.outputs*imlayer.batch, 1, imerror, 1); scal_gpu(imlayer.outputs*imlayer.batch, .00, gnet->layers[gnet->n-1].delta_gpu, 1); printf("realness %f\n", cuda_mag_array(imerror, imlayer.outputs*imlayer.batch)); printf("features %f\n", cuda_mag_array(gnet->layers[gnet->n-1].delta_gpu, imlayer.outputs*imlayer.batch)); axpy_gpu(imlayer.outputs*imlayer.batch, 1, imerror, 1, gnet->layers[gnet->n-1].delta_gpu, 1); backward_network_gpu(gnet); for(k = 0; k < gnet->batch; ++k){ int index = j*gnet->batch + k; copy_cpu(gnet->outputs, gnet->output + k*gnet->outputs, 1, gen.X.vals[index], 1); } } harmless_update_network_gpu(anet); data merge = concat_data(train, gen); //randomize_data(merge); float aloss = train_network(anet, merge); //translate_image(im, 1); //scale_image(im, .5); //translate_image(im2, 1); //scale_image(im2, .5); #ifdef OPENCV if(display){ image im = float_to_image(anet->w, anet->h, anet->c, gen.X.vals[0]); image im2 = float_to_image(anet->w, anet->h, anet->c, train.X.vals[0]); show_image(im, "gen"); show_image(im2, "train"); cvWaitKey(50); } #endif /* if(aloss < .1){ anet->learning_rate = 0; } else if (aloss > .3){ anet->learning_rate = orig_rate; } */ update_network_gpu(gnet); free_data(merge); free_data(train); free_data(gen); if (aloss_avg < 0) aloss_avg = aloss; aloss_avg = aloss_avg*.9 + aloss*.1; printf("%d: adv: %f | adv_avg: %f, %f rate, %lf seconds, %d images\n", i, aloss, aloss_avg, get_current_rate(gnet), sec(clock()-time), i*imgs); if(i%10000==0){ char buff[256]; sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i); save_weights(gnet, buff); sprintf(buff, "%s/%s_%d.weights", backup_directory, abase, i); save_weights(anet, buff); } if(i%1000==0){ char buff[256]; sprintf(buff, "%s/%s.backup", backup_directory, base); save_weights(gnet, buff); sprintf(buff, "%s/%s.backup", backup_directory, abase); save_weights(anet, buff); } } char buff[256]; sprintf(buff, "%s/%s_final.weights", backup_directory, base); save_weights(gnet, buff); #endif }