void valid_char_rnn(char *cfgfile, char *weightfile) { char *base = basecfg(cfgfile); fprintf(stderr, "%s\n", base); network net = parse_network_cfg(cfgfile); if(weightfile){ load_weights(&net, weightfile); } int inputs = get_network_input_size(net); int count = 0; int c; float *input = calloc(inputs, sizeof(float)); float sum = 0; c = getc(stdin); float log2 = log(2); while(c != EOF){ int next = getc(stdin); if(next == EOF) break; ++count; input[c] = 1; float *out = network_predict(net, input); input[c] = 0; sum += log(out[next])/log2; c = next; } printf("Perplexity: %f\n", pow(2, -sum/count)); }
void test_char_rnn(char *cfgfile, char *weightfile, int num, char *seed, float temp, int rseed, char *token_file) { char **tokens = 0; if(token_file){ size_t n; tokens = read_tokens(token_file, &n); } srand(rseed); char *base = basecfg(cfgfile); fprintf(stderr, "%s\n", base); network net = parse_network_cfg(cfgfile); if(weightfile){ load_weights(&net, weightfile); } int inputs = get_network_input_size(net); int i, j; for(i = 0; i < net.n; ++i) net.layers[i].temperature = temp; int c = 0; int len = strlen(seed); float *input = calloc(inputs, sizeof(float)); /* fill_cpu(inputs, 0, input, 1); for(i = 0; i < 10; ++i){ network_predict(net, input); } fill_cpu(inputs, 0, input, 1); */ for(i = 0; i < len-1; ++i){ c = seed[i]; input[c] = 1; network_predict(net, input); input[c] = 0; print_symbol(c, tokens); } if(len) c = seed[len-1]; print_symbol(c, tokens); for(i = 0; i < num; ++i){ input[c] = 1; float *out = network_predict(net, input); input[c] = 0; for(j = 32; j < 127; ++j){ //printf("%d %c %f\n",j, j, out[j]); } for(j = 0; j < inputs; ++j){ if (out[j] < .0001) out[j] = 0; } c = sample_array(out, inputs); print_symbol(c, tokens); } printf("\n"); }
void train_char_rnn(char *cfgfile, char *weightfile, char *filename) { FILE *fp = fopen(filename, "rb"); fseek(fp, 0, SEEK_END); size_t size = ftell(fp); fseek(fp, 0, SEEK_SET); unsigned char *text = calloc(size+1, sizeof(char)); fread(text, 1, size, fp); fclose(fp); char *backup_directory = "/home/pjreddie/backup/"; srand(time(0)); data_seed = time(0); char *base = basecfg(cfgfile); fprintf(stderr, "%s\n", base); float avg_loss = -1; network net = parse_network_cfg(cfgfile); if(weightfile){ load_weights(&net, weightfile); } int inputs = get_network_input_size(net); fprintf(stderr, "Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay); int batch = net.batch; int steps = net.time_steps; int i = (*net.seen)/net.batch; clock_t time; while(get_current_batch(net) < net.max_batches){ i += 1; time=clock(); float_pair p = get_rnn_data(text, inputs, size, batch/steps, steps); float loss = train_network_datum(net, p.x, p.y) / (batch); free(p.x); free(p.y); if (avg_loss < 0) avg_loss = loss; avg_loss = avg_loss*.9 + loss*.1; fprintf(stderr, "%d: %f, %f avg, %f rate, %lf seconds\n", i, loss, avg_loss, get_current_rate(net), sec(clock()-time)); if(i%100==0){ char buff[256]; sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i); save_weights(net, buff); } if(i%10==0){ char buff[256]; sprintf(buff, "%s/%s.backup", backup_directory, base); save_weights(net, buff); } } char buff[256]; sprintf(buff, "%s/%s_final.weights", backup_directory, base); save_weights(net, buff); }
void valid_tactic_rnn(char *cfgfile, char *weightfile, char *seed) { char *base = basecfg(cfgfile); fprintf(stderr, "%s\n", base); network net = parse_network_cfg(cfgfile); if(weightfile){ load_weights(&net, weightfile); } int inputs = get_network_input_size(net); int count = 0; int words = 1; int c; int len = strlen(seed); float *input = calloc(inputs, sizeof(float)); int i; for(i = 0; i < len; ++i){ c = seed[i]; input[(int)c] = 1; network_predict(net, input); input[(int)c] = 0; } float sum = 0; c = getc(stdin); float log2 = log(2); int in = 0; while(c != EOF){ int next = getc(stdin); if(next == EOF) break; if(next < 0 || next >= 255) error("Out of range character"); input[c] = 1; float *out = network_predict(net, input); input[c] = 0; if(c == '.' && next == '\n') in = 0; if(!in) { if(c == '>' && next == '>'){ in = 1; ++words; } c = next; continue; } ++count; sum += log(out[next])/log2; c = next; printf("%d %d Perplexity: %4.4f Word Perplexity: %4.4f\n", count, words, pow(2, -sum/count), pow(2, -sum/words)); } }
void vec_char_rnn(char *cfgfile, char *weightfile, char *seed) { char *base = basecfg(cfgfile); fprintf(stderr, "%s\n", base); network net = parse_network_cfg(cfgfile); if(weightfile){ load_weights(&net, weightfile); } int inputs = get_network_input_size(net); int c; int seed_len = strlen(seed); float *input = calloc(inputs, sizeof(float)); int i; char *line; while((line=fgetl(stdin)) != 0){ reset_rnn_state(net, 0); for(i = 0; i < seed_len; ++i){ c = seed[i]; input[(int)c] = 1; network_predict(net, input); input[(int)c] = 0; } strip(line); int str_len = strlen(line); for(i = 0; i < str_len; ++i){ c = line[i]; input[(int)c] = 1; network_predict(net, input); input[(int)c] = 0; } c = ' '; input[(int)c] = 1; network_predict(net, input); input[(int)c] = 0; layer l = net.layers[0]; #ifdef GPU cuda_pull_array(l.output_gpu, l.output, l.outputs); #endif printf("%s", line); for(i = 0; i < l.outputs; ++i){ printf(",%g", l.output[i]); } printf("\n"); } }
std::string ofxDarknet::rnn(int num, std::string seed, float temp ) { int inputs = get_network_input_size( net ); for( int i = 0; i < net.n; ++i ) { net.layers[ i ].temperature = temp; } int c = 0; int len = seed.length(); float *input = ( float* ) calloc( inputs, sizeof( float ) ); std::string sampled_text; for( int i = 0; i < len - 1; ++i ) { c = seed[ i ]; input[ c ] = 1; network_predict( net, input ); input[ c ] = 0; char _c = c; sampled_text += _c; } if( len ) c = seed[ len - 1 ]; char _c = c; sampled_text += _c; for( int i = 0; i < num; ++i ) { input[ c ] = 1; float *out = network_predict( net, input ); input[ c ] = 0; for( int j = 0; j < inputs; ++j ) { if( out[ j ] < .0001 ) out[ j ] = 0; } c = sample_array( out, inputs ); char _c = c; sampled_text += _c; } delete input; return sampled_text; }
void test_tactic_rnn(char *cfgfile, char *weightfile, int num, float temp, int rseed, char *token_file) { char **tokens = 0; if(token_file){ size_t n; tokens = read_tokens(token_file, &n); } srand(rseed); char *base = basecfg(cfgfile); fprintf(stderr, "%s\n", base); network net = parse_network_cfg(cfgfile); if(weightfile){ load_weights(&net, weightfile); } int inputs = get_network_input_size(net); int i, j; for(i = 0; i < net.n; ++i) net.layers[i].temperature = temp; int c = 0; float *input = calloc(inputs, sizeof(float)); float *out = 0; while((c = getc(stdin)) != EOF){ input[c] = 1; out = network_predict(net, input); input[c] = 0; } for(i = 0; i < num; ++i){ for(j = 0; j < inputs; ++j){ if (out[j] < .0001) out[j] = 0; } int next = sample_array(out, inputs); if(c == '.' && next == '\n') break; c = next; print_symbol(c, tokens); input[c] = 1; out = network_predict(net, input); input[c] = 0; } printf("\n"); }
void test_char_rnn(char *cfgfile, char *weightfile, int num, char *seed, float temp, int rseed) { srand(rseed); char *base = basecfg(cfgfile); fprintf(stderr, "%s\n", base); network net = parse_network_cfg(cfgfile); if(weightfile){ load_weights(&net, weightfile); } int inputs = get_network_input_size(net); int i, j; for(i = 0; i < net.n; ++i) net.layers[i].temperature = temp; unsigned char c; int len = strlen(seed); float *input = calloc(inputs, sizeof(float)); for(i = 0; i < len-1; ++i){ c = seed[i]; input[(int)c] = 1; network_predict(net, input); input[(int)c] = 0; printf("%c", c); } c = seed[len-1]; for(i = 0; i < num; ++i){ printf("%c", c); float r = rand_uniform(0,1); float sum = 0; input[(int)c] = 1; float *out = network_predict(net, input); input[(int)c] = 0; for(j = 0; j < inputs; ++j){ sum += out[j]; if(sum > r) break; } c = j; } printf("\n"); }
void train_char_rnn(char *cfgfile, char *weightfile, char *filename, int clear, int tokenized) { srand(time(0)); data_seed = time(0); unsigned char *text = 0; int *tokens = 0; size_t size; if(tokenized){ tokens = read_tokenized_data(filename, &size); } else { FILE *fp = fopen(filename, "rb"); fseek(fp, 0, SEEK_END); size = ftell(fp); fseek(fp, 0, SEEK_SET); text = calloc(size+1, sizeof(char)); fread(text, 1, size, fp); fclose(fp); } char *backup_directory = "/home/pjreddie/backup/"; char *base = basecfg(cfgfile); fprintf(stderr, "%s\n", base); float avg_loss = -1; network net = parse_network_cfg(cfgfile); if(weightfile){ load_weights(&net, weightfile); } int inputs = get_network_input_size(net); fprintf(stderr, "Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay); int batch = net.batch; int steps = net.time_steps; if(clear) *net.seen = 0; int i = (*net.seen)/net.batch; int streams = batch/steps; size_t *offsets = calloc(streams, sizeof(size_t)); int j; for(j = 0; j < streams; ++j){ offsets[j] = rand_size_t()%size; } clock_t time; while(get_current_batch(net) < net.max_batches){ i += 1; time=clock(); float_pair p; if(tokenized){ p = get_rnn_token_data(tokens, offsets, inputs, size, streams, steps); }else{ p = get_rnn_data(text, offsets, inputs, size, streams, steps); } float loss = train_network_datum(net, p.x, p.y) / (batch); free(p.x); free(p.y); if (avg_loss < 0) avg_loss = loss; avg_loss = avg_loss*.9 + loss*.1; int chars = get_current_batch(net)*batch; fprintf(stderr, "%d: %f, %f avg, %f rate, %lf seconds, %f epochs\n", i, loss, avg_loss, get_current_rate(net), sec(clock()-time), (float) chars/size); for(j = 0; j < streams; ++j){ //printf("%d\n", j); if(rand()%10 == 0){ //fprintf(stderr, "Reset\n"); offsets[j] = rand_size_t()%size; reset_rnn_state(net, j); } } if(i%1000==0){ char buff[256]; sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i); save_weights(net, buff); } if(i%10==0){ char buff[256]; sprintf(buff, "%s/%s.backup", backup_directory, base); save_weights(net, buff); } } char buff[256]; sprintf(buff, "%s/%s_final.weights", backup_directory, base); save_weights(net, buff); }
void train_lsd2(char *cfgfile, char *weightfile, char *acfgfile, char *aweightfile, int clear) { #ifdef GPU char *train_images = "/home/pjreddie/data/coco/trainvalno5k.txt"; char *backup_directory = "/home/pjreddie/backup/"; srand(time(0)); char *base = basecfg(cfgfile); printf("%s\n", base); network net = parse_network_cfg(cfgfile); if(weightfile){ load_weights(&net, weightfile); } if(clear) *net.seen = 0; char *abase = basecfg(acfgfile); network anet = parse_network_cfg(acfgfile); if(aweightfile){ load_weights(&anet, aweightfile); } if(clear) *anet.seen = 0; int i, j, k; layer imlayer = {}; for (i = 0; i < net.n; ++i) { if (net.layers[i].out_c == 3) { imlayer = net.layers[i]; break; } } printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay); int imgs = net.batch*net.subdivisions; i = *net.seen/imgs; data train, buffer; list *plist = get_paths(train_images); //int N = plist->size; char **paths = (char **)list_to_array(plist); load_args args = {}; args.w = net.w; args.h = net.h; args.paths = paths; args.n = imgs; args.m = plist->size; args.d = &buffer; args.min = net.min_crop; args.max = net.max_crop; args.angle = net.angle; args.aspect = net.aspect; args.exposure = net.exposure; args.saturation = net.saturation; args.hue = net.hue; args.size = net.w; args.type = CLASSIFICATION_DATA; args.classes = 1; char *ls[1] = {"coco"}; args.labels = ls; pthread_t load_thread = load_data_in_thread(args); clock_t time; network_state gstate = {}; gstate.index = 0; gstate.net = net; int x_size = get_network_input_size(net)*net.batch; int y_size = 1*net.batch; gstate.input = cuda_make_array(0, x_size); gstate.truth = 0; gstate.delta = 0; gstate.train = 1; float *X = (float*)calloc(x_size, sizeof(float)); float *y = (float*)calloc(y_size, sizeof(float)); network_state astate = {}; astate.index = 0; astate.net = anet; int ay_size = get_network_output_size(anet)*anet.batch; astate.input = 0; astate.truth = 0; astate.delta = 0; astate.train = 1; float *imerror = cuda_make_array(0, imlayer.outputs); float *ones_gpu = cuda_make_array(0, ay_size); fill_ongpu(ay_size, 1, ones_gpu, 1); float aloss_avg = -1; float gloss_avg = -1; //data generated = copy_data(train); while (get_current_batch(net) < net.max_batches) { i += 1; time=clock(); pthread_join(load_thread, 0); train = buffer; load_thread = load_data_in_thread(args); printf("Loaded: %lf seconds\n", sec(clock()-time)); data generated = copy_data(train); time=clock(); float gloss = 0; for(j = 0; j < net.subdivisions; ++j){ get_next_batch(train, net.batch, j*net.batch, X, y); cuda_push_array(gstate.input, X, x_size); *net.seen += net.batch; forward_network_gpu(net, gstate); fill_ongpu(imlayer.outputs, 0, imerror, 1); astate.input = imlayer.output_gpu; astate.delta = imerror; astate.truth = ones_gpu; forward_network_gpu(anet, astate); backward_network_gpu(anet, astate); scal_ongpu(imlayer.outputs, 1, imerror, 1); axpy_ongpu(imlayer.outputs, 1, imerror, 1, imlayer.delta_gpu, 1); backward_network_gpu(net, gstate); printf("features %f\n", cuda_mag_array(imlayer.delta_gpu, imlayer.outputs)); printf("realness %f\n", cuda_mag_array(imerror, imlayer.outputs)); gloss += get_network_cost(net) /(net.subdivisions*net.batch); cuda_pull_array(imlayer.output_gpu, imlayer.output, x_size); for(k = 0; k < net.batch; ++k){ int index = j*net.batch + k; copy_cpu(imlayer.outputs, imlayer.output + k*imlayer.outputs, 1, generated.X.vals[index], 1); generated.y.vals[index][0] = 0; } } harmless_update_network_gpu(anet); data merge = concat_data(train, generated); randomize_data(merge); float aloss = train_network(anet, merge); update_network_gpu(net); update_network_gpu(anet); free_data(merge); free_data(train); free_data(generated); if (aloss_avg < 0) aloss_avg = aloss; aloss_avg = aloss_avg*.9 + aloss*.1; gloss_avg = gloss_avg*.9 + gloss*.1; printf("%d: gen: %f, adv: %f | gen_avg: %f, adv_avg: %f, %f rate, %lf seconds, %d images\n", i, gloss, aloss, gloss_avg, aloss_avg, get_current_rate(net), sec(clock()-time), i*imgs); if(i%1000==0){ char buff[256]; sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i); save_weights(net, buff); sprintf(buff, "%s/%s_%d.weights", backup_directory, abase, i); save_weights(anet, buff); } if(i%100==0){ char buff[256]; sprintf(buff, "%s/%s.backup", backup_directory, base); save_weights(net, buff); sprintf(buff, "%s/%s.backup", backup_directory, abase); save_weights(anet, buff); } } char buff[256]; sprintf(buff, "%s/%s_final.weights", backup_directory, base); save_weights(net, buff); #endif }
void train_lsd3(char *fcfg, char *fweight, char *gcfg, char *gweight, char *acfg, char *aweight, int clear) { #ifdef GPU //char *train_images = "/home/pjreddie/data/coco/trainvalno5k.txt"; char *train_images = "/home/pjreddie/data/imagenet/imagenet1k.train.list"; //char *style_images = "/home/pjreddie/data/coco/trainvalno5k.txt"; char *style_images = "/home/pjreddie/zelda.txt"; char *backup_directory = "/home/pjreddie/backup/"; srand(time(0)); network fnet = load_network(fcfg, fweight, clear); network gnet = load_network(gcfg, gweight, clear); network anet = load_network(acfg, aweight, clear); char *gbase = basecfg(gcfg); char *abase = basecfg(acfg); printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", gnet.learning_rate, gnet.momentum, gnet.decay); int imgs = gnet.batch*gnet.subdivisions; int i = *gnet.seen/imgs; data train, tbuffer; data style, sbuffer; list *slist = get_paths(style_images); char **spaths = (char **)list_to_array(slist); list *tlist = get_paths(train_images); char **tpaths = (char **)list_to_array(tlist); load_args targs= get_base_args(gnet); targs.paths = tpaths; targs.n = imgs; targs.m = tlist->size; targs.d = &tbuffer; targs.type = CLASSIFICATION_DATA; targs.classes = 1; char *ls[1] = {"zelda"}; targs.labels = ls; load_args sargs = get_base_args(gnet); sargs.paths = spaths; sargs.n = imgs; sargs.m = slist->size; sargs.d = &sbuffer; sargs.type = CLASSIFICATION_DATA; sargs.classes = 1; sargs.labels = ls; pthread_t tload_thread = load_data_in_thread(targs); pthread_t sload_thread = load_data_in_thread(sargs); clock_t time; float aloss_avg = -1; float floss_avg = -1; network_state fstate = {}; fstate.index = 0; fstate.net = fnet; int x_size = get_network_input_size(fnet)*fnet.batch; int y_size = get_network_output_size(fnet)*fnet.batch; fstate.input = cuda_make_array(0, x_size); fstate.truth = cuda_make_array(0, y_size); fstate.delta = cuda_make_array(0, x_size); fstate.train = 1; float *X = (float*)calloc(x_size, sizeof(float)); float *y = (float*)calloc(y_size, sizeof(float)); float *ones = cuda_make_array(0, anet.batch); float *zeros = cuda_make_array(0, anet.batch); fill_ongpu(anet.batch, .99, ones, 1); fill_ongpu(anet.batch, .01, zeros, 1); network_state astate = {}; astate.index = 0; astate.net = anet; int ax_size = get_network_input_size(anet)*anet.batch; int ay_size = get_network_output_size(anet)*anet.batch; astate.input = 0; astate.truth = ones; astate.delta = cuda_make_array(0, ax_size); astate.train = 1; network_state gstate = {}; gstate.index = 0; gstate.net = gnet; int gx_size = get_network_input_size(gnet)*gnet.batch; int gy_size = get_network_output_size(gnet)*gnet.batch; gstate.input = cuda_make_array(0, gx_size); gstate.truth = 0; gstate.delta = 0; gstate.train = 1; while (get_current_batch(gnet) < gnet.max_batches) { i += 1; time=clock(); pthread_join(tload_thread, 0); pthread_join(sload_thread, 0); train = tbuffer; style = sbuffer; tload_thread = load_data_in_thread(targs); sload_thread = load_data_in_thread(sargs); printf("Loaded: %lf seconds\n", sec(clock()-time)); data generated = copy_data(train); time=clock(); int j, k; float floss = 0; for(j = 0; j < fnet.subdivisions; ++j){ layer imlayer = gnet.layers[gnet.n - 1]; get_next_batch(train, fnet.batch, j*fnet.batch, X, y); cuda_push_array(fstate.input, X, x_size); cuda_push_array(gstate.input, X, gx_size); *gnet.seen += gnet.batch; forward_network_gpu(fnet, fstate); float *feats = fnet.layers[fnet.n - 2].output_gpu; copy_ongpu(y_size, feats, 1, fstate.truth, 1); forward_network_gpu(gnet, gstate); float *gen = gnet.layers[gnet.n-1].output_gpu; copy_ongpu(x_size, gen, 1, fstate.input, 1); fill_ongpu(x_size, 0, fstate.delta, 1); forward_network_gpu(fnet, fstate); backward_network_gpu(fnet, fstate); //HERE astate.input = gen; fill_ongpu(ax_size, 0, astate.delta, 1); forward_network_gpu(anet, astate); backward_network_gpu(anet, astate); float *delta = imlayer.delta_gpu; fill_ongpu(x_size, 0, delta, 1); scal_ongpu(x_size, 100, astate.delta, 1); scal_ongpu(x_size, .00001, fstate.delta, 1); axpy_ongpu(x_size, 1, fstate.delta, 1, delta, 1); axpy_ongpu(x_size, 1, astate.delta, 1, delta, 1); //fill_ongpu(x_size, 0, delta, 1); //cuda_push_array(delta, X, x_size); //axpy_ongpu(x_size, -1, imlayer.output_gpu, 1, delta, 1); //printf("pix error: %f\n", cuda_mag_array(delta, x_size)); printf("fea error: %f\n", cuda_mag_array(fstate.delta, x_size)); printf("adv error: %f\n", cuda_mag_array(astate.delta, x_size)); //axpy_ongpu(x_size, 1, astate.delta, 1, delta, 1); backward_network_gpu(gnet, gstate); floss += get_network_cost(fnet) /(fnet.subdivisions*fnet.batch); cuda_pull_array(imlayer.output_gpu, imlayer.output, x_size); for(k = 0; k < gnet.batch; ++k){ int index = j*gnet.batch + k; copy_cpu(imlayer.outputs, imlayer.output + k*imlayer.outputs, 1, generated.X.vals[index], 1); generated.y.vals[index][0] = .01; } } /* image sim = float_to_image(anet.w, anet.h, anet.c, style.X.vals[j]); show_image(sim, "style"); cvWaitKey(0); */ harmless_update_network_gpu(anet); data merge = concat_data(style, generated); randomize_data(merge); float aloss = train_network(anet, merge); update_network_gpu(gnet); free_data(merge); free_data(train); free_data(generated); free_data(style); if (aloss_avg < 0) aloss_avg = aloss; if (floss_avg < 0) floss_avg = floss; aloss_avg = aloss_avg*.9 + aloss*.1; floss_avg = floss_avg*.9 + floss*.1; printf("%d: gen: %f, adv: %f | gen_avg: %f, adv_avg: %f, %f rate, %lf seconds, %d images\n", i, floss, aloss, floss_avg, aloss_avg, get_current_rate(gnet), sec(clock()-time), i*imgs); if(i%1000==0){ char buff[256]; sprintf(buff, "%s/%s_%d.weights", backup_directory, gbase, i); save_weights(gnet, buff); sprintf(buff, "%s/%s_%d.weights", backup_directory, abase, i); save_weights(anet, buff); } if(i%100==0){ char buff[256]; sprintf(buff, "%s/%s.backup", backup_directory, gbase); save_weights(gnet, buff); sprintf(buff, "%s/%s.backup", backup_directory, abase); save_weights(anet, buff); } } #endif }