int main(int argc, char *argv[]) {
char cmd[CMD_SIZE];
double test_in[IN];
int i;
printf("********** Bpnetwork Console **********\n");
while (TRUE) {
scanf("%s", cmd);
if (!strcmp(cmd, "help")) {
printf("read read neuron\n");
printf("train train network\n");
printf("test test network\n");
printf("exit exit program\n");
} else if (!strcmp(cmd, "read")) {
read_neuron();
} else if (!strcmp(cmd, "train")) {
read_data();
init_bpnetwork();
ga_interface();
train_network();
write_neuron();
} else if (!strcmp(cmd, "test")) {
printf("input test data\n");
for (i = 0; i < IN; i++) {
scanf("%lf", test_in + i);
}
test_network(test_in);
} else if (!strcmp(cmd, "exit")) {
break;
}
}
return 0;
}
void *train_thread(void *ptr)
{
train_args args = *(train_args*)ptr;
free(ptr);
cuda_set_device(args.net.gpu_index);
*args.err = train_network(args.net, args.d);
return 0;
}
void train_imagenet(char *cfgfile, char *weightfile)
{
data_seed = time(0);
srand(time(0));
float avg_loss = -1;
char *base = basecfg(cfgfile);
printf("%s\n", base);
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
//net.seen=0;
int imgs = 1024;
int i = net.seen/imgs;
char **labels = get_labels("data/inet.labels.list");
list *plist = get_paths("/data/imagenet/cls.train.list");
char **paths = (char **)list_to_array(plist);
printf("%d\n", plist->size);
clock_t time;
pthread_t load_thread;
data train;
data buffer;
load_thread = load_data_thread(paths, imgs, plist->size, labels, 1000, 256, 256, &buffer);
while(1){
++i;
time=clock();
pthread_join(load_thread, 0);
train = buffer;
/*
image im = float_to_image(256, 256, 3, train.X.vals[114]);
show_image(im, "training");
cvWaitKey(0);
*/
load_thread = load_data_thread(paths, imgs, plist->size, labels, 1000, 256, 256, &buffer);
printf("Loaded: %lf seconds\n", sec(clock()-time));
time=clock();
float loss = train_network(net, train);
net.seen += imgs;
if(avg_loss == -1) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
printf("%d: %f, %f avg, %lf seconds, %d images\n", i, loss, avg_loss, sec(clock()-time), net.seen);
free_data(train);
if((i % 20000) == 0) net.learning_rate *= .1;
//if(i%100 == 0 && net.learning_rate > .00001) net.learning_rate *= .97;
if(i%1000==0){
char buff[256];
sprintf(buff, "/home/pjreddie/imagenet_backup/%s_%d.weights",base, i);
save_weights(net, buff);
}
}
}
int main(int argx, char **argv)
{
char *nid;
switch (argv[1][1]) {
case 't':
nid = argv[2];
train_network(nid);
break;
case 'v':
nid = argv[2];
validate_network(nid);
break;
}
}
void train_writing(char *cfgfile, char *weightfile)
{
data_seed = time(0);
srand(time(0));
float avg_loss = -1;
char *base = basecfg(cfgfile);
printf("%s\n", base);
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
int imgs = 256;
int i = net.seen/imgs;
list *plist = get_paths("data/train.list");
char **paths = (char **)list_to_array(plist);
printf("%d\n", plist->size);
clock_t time;
while(1){
++i;
time=clock();
data train = load_data_writing(paths, imgs, plist->size, 256, 256, 4);
float loss = train_network(net, train);
#ifdef GPU
float *out = get_network_output_gpu(net);
#else
float *out = get_network_output(net);
#endif
// image pred = float_to_image(32, 32, 1, out);
// print_image(pred);
net.seen += imgs;
if(avg_loss == -1) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
printf("%d: %f, %f avg, %lf seconds, %d images\n", i, loss, avg_loss, sec(clock()-time), net.seen);
free_data(train);
if((i % 20000) == 0) net.learning_rate *= .1;
//if(i%100 == 0 && net.learning_rate > .00001) net.learning_rate *= .97;
if(i%250==0){
char buff[256];
sprintf(buff, "/home/pjreddie/writing_backup/%s_%d.weights", base, i);
save_weights(net, buff);
}
}
}
void train_dice(char *cfgfile, char *weightfile)
{
data_seed = time(0);
srand(time(0));
float avg_loss = -1;
char *base = basecfg(cfgfile);
char *backup_directory = "/home/pjreddie/backup/";
printf("%s\n", base);
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
int imgs = 1024;
int i = *net.seen/imgs;
char **labels = dice_labels;
list *plist = get_paths("data/dice/dice.train.list");
char **paths = (char **)list_to_array(plist);
printf("%d\n", plist->size);
clock_t time;
while(1){
++i;
time=clock();
data train = load_data(paths, imgs, plist->size, labels, 6, net.w, net.h);
printf("Loaded: %lf seconds\n", sec(clock()-time));
time=clock();
float loss = train_network(net, train);
if(avg_loss == -1) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
printf("%d: %f, %f avg, %lf seconds, %d images\n", i, loss, avg_loss, sec(clock()-time), *net.seen);
free_data(train);
if((i % 100) == 0) net.learning_rate *= .1;
if(i%100==0){
char buff[256];
sprintf(buff, "%s/%s_%d.weights",backup_directory,base, i);
save_weights(net, buff);
}
}
}
void train_captcha(char *cfgfile, char *weightfile)
{
float avg_loss = -1;
srand(time(0));
char *base = basecfg(cfgfile);
printf("%s\n", base);
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
int imgs = 1024;
int i = net.seen/imgs;
list *plist = get_paths("/data/captcha/train.auto5");
char **paths = (char **)list_to_array(plist);
printf("%d\n", plist->size);
clock_t time;
while(1){
++i;
time=clock();
data train = load_data_captcha(paths, imgs, plist->size, 10, 200, 60);
translate_data_rows(train, -128);
scale_data_rows(train, 1./128);
printf("Loaded: %lf seconds\n", sec(clock()-time));
time=clock();
float loss = train_network(net, train);
net.seen += imgs;
if(avg_loss == -1) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
printf("%d: %f, %f avg, %lf seconds, %d images\n", i, loss, avg_loss, sec(clock()-time), net.seen);
free_data(train);
if(i%10==0){
char buff[256];
sprintf(buff, "/home/pjreddie/imagenet_backup/%s_%d.weights",base, i);
save_weights(net, buff);
}
}
}
void train_yolo(char *cfgfile, char *weightfile)
{
char *train_images = "/data/voc/train.txt";
char *backup_directory = "/home/kunle12/backup/";
srand(time(0));
char *base = basecfg(cfgfile);
printf("%s\n", base);
float avg_loss = -1;
network * net = load_network(cfgfile, weightfile, 0);
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net->learning_rate, net->momentum, net->decay);
int imgs = net->batch*net->subdivisions;
int i = *net->seen/imgs;
data train, buffer;
layer l = net->layers[net->n - 1];
int side = l.side;
int classes = l.classes;
float jitter = l.jitter;
list *plist = get_paths(train_images);
//int N = plist->size;
char **paths = (char **)list_to_array(plist);
load_args args = {0};
args.w = net->w;
args.h = net->h;
args.paths = paths;
args.n = imgs;
args.m = plist->size;
args.classes = classes;
args.jitter = jitter;
args.num_boxes = side;
args.d = &buffer;
args.type = REGION_DATA;
args.angle = net->angle;
args.exposure = net->exposure;
args.saturation = net->saturation;
args.hue = net->hue;
pthread_t load_thread = load_data_in_thread(args);
clock_t time;
//while(i*imgs < N*120){
while(get_current_batch(net) < net->max_batches){
i += 1;
time=clock();
pthread_join(load_thread, 0);
train = buffer;
load_thread = load_data_in_thread(args);
printf("Loaded: %lf seconds\n", sec(clock()-time));
time=clock();
float loss = train_network(net, train);
if (avg_loss < 0) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
printf("%d: %f, %f avg, %f rate, %lf seconds, %d images\n", i, loss, avg_loss, get_current_rate(net), sec(clock()-time), i*imgs);
if(i%1000==0 || (i < 1000 && i%100 == 0)){
char buff[256];
sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i);
save_weights(net, buff);
}
free_data(train);
}
char buff[256];
sprintf(buff, "%s/%s_final.weights", backup_directory, base);
save_weights(net, buff);
free_network( net );
}
void train_regressor(char *datacfg, char *cfgfile, char *weightfile, int *gpus, int ngpus, int clear)
{
int i;
float avg_loss = -1;
char *base = basecfg(cfgfile);
printf("%s\n", base);
printf("%d\n", ngpus);
network **nets = calloc(ngpus, sizeof(network*));
srand(time(0));
int seed = rand();
for(i = 0; i < ngpus; ++i){
srand(seed);
#ifdef GPU
cuda_set_device(gpus[i]);
#endif
nets[i] = load_network(cfgfile, weightfile, clear);
nets[i]->learning_rate *= ngpus;
}
srand(time(0));
network *net = nets[0];
int imgs = net->batch * net->subdivisions * ngpus;
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net->learning_rate, net->momentum, net->decay);
list *options = read_data_cfg(datacfg);
char *backup_directory = option_find_str(options, "backup", "/backup/");
char *train_list = option_find_str(options, "train", "data/train.list");
int classes = option_find_int(options, "classes", 1);
list *plist = get_paths(train_list);
char **paths = (char **)list_to_array(plist);
printf("%d\n", plist->size);
int N = plist->size;
clock_t time;
load_args args = {0};
args.w = net->w;
args.h = net->h;
args.threads = 32;
args.classes = classes;
args.min = net->min_ratio*net->w;
args.max = net->max_ratio*net->w;
args.angle = net->angle;
args.aspect = net->aspect;
args.exposure = net->exposure;
args.saturation = net->saturation;
args.hue = net->hue;
args.size = net->w;
args.paths = paths;
args.n = imgs;
args.m = N;
args.type = REGRESSION_DATA;
data train;
data buffer;
pthread_t load_thread;
args.d = &buffer;
load_thread = load_data(args);
int epoch = (*net->seen)/N;
while(get_current_batch(net) < net->max_batches || net->max_batches == 0){
time=clock();
pthread_join(load_thread, 0);
train = buffer;
load_thread = load_data(args);
printf("Loaded: %lf seconds\n", sec(clock()-time));
time=clock();
float loss = 0;
#ifdef GPU
if(ngpus == 1){
loss = train_network(net, train);
} else {
loss = train_networks(nets, ngpus, train, 4);
}
#else
loss = train_network(net, train);
#endif
if(avg_loss == -1) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
printf("%ld, %.3f: %f, %f avg, %f rate, %lf seconds, %ld images\n", get_current_batch(net), (float)(*net->seen)/N, loss, avg_loss, get_current_rate(net), sec(clock()-time), *net->seen);
free_data(train);
if(*net->seen/N > epoch){
epoch = *net->seen/N;
char buff[256];
sprintf(buff, "%s/%s_%d.weights",backup_directory,base, epoch);
save_weights(net, buff);
}
if(get_current_batch(net)%100 == 0){
char buff[256];
sprintf(buff, "%s/%s.backup",backup_directory,base);
save_weights(net, buff);
}
}
char buff[256];
sprintf(buff, "%s/%s.weights", backup_directory, base);
save_weights(net, buff);
for(i = 0; i < ngpus; ++i){
free_network(nets[i]);
}
free(nets);
free_ptrs((void**)paths, plist->size);
free_list(plist);
free(base);
}
void train_colorizer(char *cfg, char *weight, char *acfg, char *aweight, int clear, int display)
{
#ifdef GPU
//char *train_images = "/home/kunle12/data/coco/train1.txt";
//char *train_images = "/home/kunle12/data/coco/trainvalno5k.txt";
char *train_images = "/home/kunle12/data/imagenet/imagenet1k.train.list";
char *backup_directory = "/home/kunle12/backup/";
srand(time(0));
char *base = basecfg(cfg);
char *abase = basecfg(acfg);
printf("%s\n", base);
network *net = load_network(cfg, weight, clear);
network *anet = load_network(acfg, aweight, clear);
int i, j, k;
layer imlayer = {0};
for (i = 0; i < net->n; ++i) {
if (net->layers[i].out_c == 3) {
imlayer = net->layers[i];
break;
}
}
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net->learning_rate, net->momentum, net->decay);
int imgs = net->batch*net->subdivisions;
i = *net->seen/imgs;
data train, buffer;
list *plist = get_paths(train_images);
//int N = plist->size;
char **paths = (char **)list_to_array(plist);
load_args args= get_base_args(net);
args.paths = paths;
args.n = imgs;
args.m = plist->size;
args.d = &buffer;
args.type = CLASSIFICATION_DATA;
args.classes = 1;
char *ls[2] = {"imagenet"};
args.labels = ls;
pthread_t load_thread = load_data_in_thread(args);
clock_t time;
int x_size = net->inputs*net->batch;
//int y_size = x_size;
net->delta = 0;
net->train = 1;
float *pixs = calloc(x_size, sizeof(float));
float *graypixs = calloc(x_size, sizeof(float));
//float *y = calloc(y_size, sizeof(float));
//int ay_size = anet->outputs*anet->batch;
anet->delta = 0;
anet->train = 1;
float *imerror = cuda_make_array(0, imlayer.outputs*imlayer.batch);
float aloss_avg = -1;
float gloss_avg = -1;
//data generated = copy_data(train);
while (get_current_batch(net) < net->max_batches) {
i += 1;
time=clock();
pthread_join(load_thread, 0);
train = buffer;
load_thread = load_data_in_thread(args);
printf("Loaded: %lf seconds\n", sec(clock()-time));
data gray = copy_data(train);
for(j = 0; j < imgs; ++j){
image gim = float_to_image(net->w, net->h, net->c, gray.X.vals[j]);
grayscale_image_3c(gim);
train.y.vals[j][0] = .95;
gray.y.vals[j][0] = .05;
}
time=clock();
float gloss = 0;
for(j = 0; j < net->subdivisions; ++j){
get_next_batch(train, net->batch, j*net->batch, pixs, 0);
get_next_batch(gray, net->batch, j*net->batch, graypixs, 0);
cuda_push_array(net->input_gpu, graypixs, net->inputs*net->batch);
cuda_push_array(net->truth_gpu, pixs, net->truths*net->batch);
/*
image origi = float_to_image(net->w, net->h, 3, pixs);
image grayi = float_to_image(net->w, net->h, 3, graypixs);
show_image(grayi, "gray");
show_image(origi, "orig");
cvWaitKey(0);
*/
*net->seen += net->batch;
forward_network_gpu(net);
fill_gpu(imlayer.outputs*imlayer.batch, 0, imerror, 1);
copy_gpu(anet->inputs*anet->batch, imlayer.output_gpu, 1, anet->input_gpu, 1);
fill_gpu(anet->inputs*anet->batch, .95, anet->truth_gpu, 1);
anet->delta_gpu = imerror;
forward_network_gpu(anet);
backward_network_gpu(anet);
scal_gpu(imlayer.outputs*imlayer.batch, 1./100., net->layers[net->n-1].delta_gpu, 1);
scal_gpu(imlayer.outputs*imlayer.batch, 1, imerror, 1);
printf("realness %f\n", cuda_mag_array(imerror, imlayer.outputs*imlayer.batch));
printf("features %f\n", cuda_mag_array(net->layers[net->n-1].delta_gpu, imlayer.outputs*imlayer.batch));
axpy_gpu(imlayer.outputs*imlayer.batch, 1, imerror, 1, net->layers[net->n-1].delta_gpu, 1);
backward_network_gpu(net);
gloss += *net->cost /(net->subdivisions*net->batch);
for(k = 0; k < net->batch; ++k){
int index = j*net->batch + k;
copy_cpu(imlayer.outputs, imlayer.output + k*imlayer.outputs, 1, gray.X.vals[index], 1);
}
}
harmless_update_network_gpu(anet);
data merge = concat_data(train, gray);
//randomize_data(merge);
float aloss = train_network(anet, merge);
update_network_gpu(net);
#ifdef OPENCV
if(display){
image im = float_to_image(anet->w, anet->h, anet->c, gray.X.vals[0]);
image im2 = float_to_image(anet->w, anet->h, anet->c, train.X.vals[0]);
show_image(im, "gen", 1);
show_image(im2, "train", 1);
}
#endif
free_data(merge);
free_data(train);
free_data(gray);
if (aloss_avg < 0) aloss_avg = aloss;
aloss_avg = aloss_avg*.9 + aloss*.1;
gloss_avg = gloss_avg*.9 + gloss*.1;
printf("%d: gen: %f, adv: %f | gen_avg: %f, adv_avg: %f, %f rate, %lf seconds, %d images\n", i, gloss, aloss, gloss_avg, aloss_avg, get_current_rate(net), sec(clock()-time), i*imgs);
if(i%1000==0){
char buff[256];
sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i);
save_weights(net, buff);
sprintf(buff, "%s/%s_%d.weights", backup_directory, abase, i);
save_weights(anet, buff);
}
if(i%100==0){
char buff[256];
sprintf(buff, "%s/%s.backup", backup_directory, base);
save_weights(net, buff);
sprintf(buff, "%s/%s.backup", backup_directory, abase);
save_weights(anet, buff);
}
}
char buff[256];
sprintf(buff, "%s/%s_final.weights", backup_directory, base);
save_weights(net, buff);
#endif
}
void train_dcgan(char *cfg, char *weight, char *acfg, char *aweight, int clear, int display, char *train_images, int maxbatch)
{
#ifdef GPU
char *backup_directory = "/home/kunle12/backup/";
srand(time(0));
char *base = basecfg(cfg);
char *abase = basecfg(acfg);
printf("%s\n", base);
network *gnet = load_network(cfg, weight, clear);
network *anet = load_network(acfg, aweight, clear);
//float orig_rate = anet->learning_rate;
int i, j, k;
layer imlayer = {0};
for (i = 0; i < gnet->n; ++i) {
if (gnet->layers[i].out_c == 3) {
imlayer = gnet->layers[i];
break;
}
}
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", gnet->learning_rate, gnet->momentum, gnet->decay);
int imgs = gnet->batch*gnet->subdivisions;
i = *gnet->seen/imgs;
data train, buffer;
list *plist = get_paths(train_images);
//int N = plist->size;
char **paths = (char **)list_to_array(plist);
load_args args= get_base_args(anet);
args.paths = paths;
args.n = imgs;
args.m = plist->size;
args.d = &buffer;
args.type = CLASSIFICATION_DATA;
args.threads=16;
args.classes = 1;
char *ls[2] = {"imagenet", "zzzzzzzz"};
args.labels = ls;
pthread_t load_thread = load_data_in_thread(args);
clock_t time;
gnet->train = 1;
anet->train = 1;
int x_size = gnet->inputs*gnet->batch;
int y_size = gnet->truths*gnet->batch;
float *imerror = cuda_make_array(0, y_size);
//int ay_size = anet->truths*anet->batch;
float aloss_avg = -1;
//data generated = copy_data(train);
if (maxbatch == 0) maxbatch = gnet->max_batches;
while (get_current_batch(gnet) < maxbatch) {
i += 1;
time=clock();
pthread_join(load_thread, 0);
train = buffer;
//translate_data_rows(train, -.5);
//scale_data_rows(train, 2);
load_thread = load_data_in_thread(args);
printf("Loaded: %lf seconds\n", sec(clock()-time));
data gen = copy_data(train);
for (j = 0; j < imgs; ++j) {
train.y.vals[j][0] = 1;
gen.y.vals[j][0] = 0;
}
time=clock();
for(j = 0; j < gnet->subdivisions; ++j){
get_next_batch(train, gnet->batch, j*gnet->batch, gnet->truth, 0);
int z;
for(z = 0; z < x_size; ++z){
gnet->input[z] = rand_normal();
}
for(z = 0; z < gnet->batch; ++z){
float mag = mag_array(gnet->input + z*gnet->inputs, gnet->inputs);
scale_array(gnet->input + z*gnet->inputs, gnet->inputs, 1./mag);
}
/*
for(z = 0; z < 100; ++z){
printf("%f, ", gnet->input[z]);
}
printf("\n");
printf("input: %f %f\n", mean_array(gnet->input, x_size), variance_array(gnet->input, x_size));
*/
//cuda_push_array(gnet->input_gpu, gnet->input, x_size);
//cuda_push_array(gnet->truth_gpu, gnet->truth, y_size);
*gnet->seen += gnet->batch;
forward_network(gnet);
fill_gpu(imlayer.outputs*imlayer.batch, 0, imerror, 1);
fill_cpu(anet->truths*anet->batch, 1, anet->truth, 1);
copy_cpu(anet->inputs*anet->batch, imlayer.output, 1, anet->input, 1);
anet->delta_gpu = imerror;
forward_network(anet);
backward_network(anet);
//float genaloss = *anet->cost / anet->batch;
//printf("%f\n", genaloss);
scal_gpu(imlayer.outputs*imlayer.batch, 1, imerror, 1);
scal_gpu(imlayer.outputs*imlayer.batch, 0, gnet->layers[gnet->n-1].delta_gpu, 1);
//printf("realness %f\n", cuda_mag_array(imerror, imlayer.outputs*imlayer.batch));
//printf("features %f\n", cuda_mag_array(gnet->layers[gnet->n-1].delta_gpu, imlayer.outputs*imlayer.batch));
axpy_gpu(imlayer.outputs*imlayer.batch, 1, imerror, 1, gnet->layers[gnet->n-1].delta_gpu, 1);
backward_network(gnet);
/*
for(k = 0; k < gnet->n; ++k){
layer l = gnet->layers[k];
cuda_pull_array(l.output_gpu, l.output, l.outputs*l.batch);
printf("%d: %f %f\n", k, mean_array(l.output, l.outputs*l.batch), variance_array(l.output, l.outputs*l.batch));
}
*/
for(k = 0; k < gnet->batch; ++k){
int index = j*gnet->batch + k;
copy_cpu(gnet->outputs, gnet->output + k*gnet->outputs, 1, gen.X.vals[index], 1);
}
}
harmless_update_network_gpu(anet);
data merge = concat_data(train, gen);
//randomize_data(merge);
float aloss = train_network(anet, merge);
//translate_image(im, 1);
//scale_image(im, .5);
//translate_image(im2, 1);
//scale_image(im2, .5);
#ifdef OPENCV
if(display){
image im = float_to_image(anet->w, anet->h, anet->c, gen.X.vals[0]);
image im2 = float_to_image(anet->w, anet->h, anet->c, train.X.vals[0]);
show_image(im, "gen", 1);
show_image(im2, "train", 1);
save_image(im, "gen");
save_image(im2, "train");
}
#endif
/*
if(aloss < .1){
anet->learning_rate = 0;
} else if (aloss > .3){
anet->learning_rate = orig_rate;
}
*/
update_network_gpu(gnet);
free_data(merge);
free_data(train);
free_data(gen);
if (aloss_avg < 0) aloss_avg = aloss;
aloss_avg = aloss_avg*.9 + aloss*.1;
printf("%d: adv: %f | adv_avg: %f, %f rate, %lf seconds, %d images\n", i, aloss, aloss_avg, get_current_rate(gnet), sec(clock()-time), i*imgs);
if(i%10000==0){
char buff[256];
sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i);
save_weights(gnet, buff);
sprintf(buff, "%s/%s_%d.weights", backup_directory, abase, i);
save_weights(anet, buff);
}
if(i%1000==0){
char buff[256];
sprintf(buff, "%s/%s.backup", backup_directory, base);
save_weights(gnet, buff);
sprintf(buff, "%s/%s.backup", backup_directory, abase);
save_weights(anet, buff);
}
}
char buff[256];
sprintf(buff, "%s/%s_final.weights", backup_directory, base);
save_weights(gnet, buff);
#endif
free_network(gnet);
free_network(anet);
}
void train_prog(char *cfg, char *weight, char *acfg, char *aweight, int clear, int display, char *train_images, int maxbatch)
{
#ifdef GPU
char *backup_directory = "/home/kunle12/backup/";
srand(time(0));
char *base = basecfg(cfg);
char *abase = basecfg(acfg);
printf("%s\n", base);
network *gnet = load_network(cfg, weight, clear);
network *anet = load_network(acfg, aweight, clear);
int i, j, k;
layer imlayer = gnet->layers[gnet->n-1];
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", gnet->learning_rate, gnet->momentum, gnet->decay);
int imgs = gnet->batch*gnet->subdivisions;
i = *gnet->seen/imgs;
data train, buffer;
list *plist = get_paths(train_images);
char **paths = (char **)list_to_array(plist);
load_args args= get_base_args(anet);
args.paths = paths;
args.n = imgs;
args.m = plist->size;
args.d = &buffer;
args.type = CLASSIFICATION_DATA;
args.threads=16;
args.classes = 1;
char *ls[2] = {"imagenet", "zzzzzzzz"};
args.labels = ls;
pthread_t load_thread = load_data_in_thread(args);
clock_t time;
gnet->train = 1;
anet->train = 1;
int x_size = gnet->inputs*gnet->batch;
int y_size = gnet->truths*gnet->batch;
float *imerror = cuda_make_array(0, y_size);
float aloss_avg = -1;
if (maxbatch == 0) maxbatch = gnet->max_batches;
while (get_current_batch(gnet) < maxbatch) {
{
int cb = get_current_batch(gnet);
float alpha = (float) cb / (maxbatch/2);
if(alpha > 1) alpha = 1;
float beta = 1 - alpha;
printf("%f %f\n", alpha, beta);
set_network_alpha_beta(gnet, alpha, beta);
set_network_alpha_beta(anet, beta, alpha);
}
i += 1;
time=clock();
pthread_join(load_thread, 0);
train = buffer;
load_thread = load_data_in_thread(args);
printf("Loaded: %lf seconds\n", sec(clock()-time));
data gen = copy_data(train);
for (j = 0; j < imgs; ++j) {
train.y.vals[j][0] = 1;
gen.y.vals[j][0] = 0;
}
time=clock();
for (j = 0; j < gnet->subdivisions; ++j) {
get_next_batch(train, gnet->batch, j*gnet->batch, gnet->truth, 0);
int z;
for(z = 0; z < x_size; ++z){
gnet->input[z] = rand_normal();
}
/*
for(z = 0; z < gnet->batch; ++z){
float mag = mag_array(gnet->input + z*gnet->inputs, gnet->inputs);
scale_array(gnet->input + z*gnet->inputs, gnet->inputs, 1./mag);
}
*/
*gnet->seen += gnet->batch;
forward_network(gnet);
fill_gpu(imlayer.outputs*imlayer.batch, 0, imerror, 1);
fill_cpu(anet->truths*anet->batch, 1, anet->truth, 1);
copy_cpu(anet->inputs*anet->batch, imlayer.output, 1, anet->input, 1);
anet->delta_gpu = imerror;
forward_network(anet);
backward_network(anet);
//float genaloss = *anet->cost / anet->batch;
scal_gpu(imlayer.outputs*imlayer.batch, 1, imerror, 1);
scal_gpu(imlayer.outputs*imlayer.batch, 0, gnet->layers[gnet->n-1].delta_gpu, 1);
axpy_gpu(imlayer.outputs*imlayer.batch, 1, imerror, 1, gnet->layers[gnet->n-1].delta_gpu, 1);
backward_network(gnet);
for(k = 0; k < gnet->batch; ++k){
int index = j*gnet->batch + k;
copy_cpu(gnet->outputs, gnet->output + k*gnet->outputs, 1, gen.X.vals[index], 1);
}
}
harmless_update_network_gpu(anet);
data merge = concat_data(train, gen);
float aloss = train_network(anet, merge);
#ifdef OPENCV
if(display){
image im = float_to_image(anet->w, anet->h, anet->c, gen.X.vals[0]);
image im2 = float_to_image(anet->w, anet->h, anet->c, train.X.vals[0]);
show_image(im, "gen", 1);
show_image(im2, "train", 1);
save_image(im, "gen");
save_image(im2, "train");
}
#endif
update_network_gpu(gnet);
free_data(merge);
free_data(train);
free_data(gen);
if (aloss_avg < 0) aloss_avg = aloss;
aloss_avg = aloss_avg*.9 + aloss*.1;
printf("%d: adv: %f | adv_avg: %f, %f rate, %lf seconds, %d images\n", i, aloss, aloss_avg, get_current_rate(gnet), sec(clock()-time), i*imgs);
if(i%10000==0){
char buff[256];
sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i);
save_weights(gnet, buff);
sprintf(buff, "%s/%s_%d.weights", backup_directory, abase, i);
save_weights(anet, buff);
}
if(i%1000==0){
char buff[256];
sprintf(buff, "%s/%s.backup", backup_directory, base);
save_weights(gnet, buff);
sprintf(buff, "%s/%s.backup", backup_directory, abase);
save_weights(anet, buff);
}
}
char buff[256];
sprintf(buff, "%s/%s_final.weights", backup_directory, base);
save_weights(gnet, buff);
#endif
free_network( gnet );
free_network( anet );
}
void train_imagenet(char *cfgfile, char *weightfile)
{
data_seed = time(0);
srand(time(0));
float avg_loss = -1;
char *base = basecfg(cfgfile);
char *backup_directory = "/home/pjreddie/backup/";
printf("%s\n", base);
network net = parse_network_cfg(cfgfile, 1);
if(weightfile){
load_weights(&net, weightfile);
}
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
int imgs = 1024;
char **labels = get_labels("data/inet.labels.list");
list *plist = get_paths("data/inet.train.list");
char **paths = (char **)list_to_array(plist);
printf("%d\n", plist->size);
int N = plist->size;
clock_t time;
pthread_t load_thread;
data train;
data buffer;
load_args args = {0};
args.w = net.w;
args.h = net.h;
args.paths = paths;
args.classes = 1000;
args.n = imgs;
args.m = N;
args.labels = labels;
args.d = &buffer;
args.type = CLASSIFICATION_DATA;
load_thread = load_data_in_thread(args);
int epoch = (*net.seen)/N;
while(get_current_batch(net) < net.max_batches || net.max_batches == 0){
time=clock();
pthread_join(load_thread, 0);
train = buffer;
load_thread = load_data_in_thread(args);
printf("Loaded: %lf seconds\n", sec(clock()-time));
time=clock();
float loss = train_network(net, train);
if(avg_loss == -1) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
printf("%d, %.3f: %f, %f avg, %f rate, %lf seconds, %d images\n", get_current_batch(net), (float)(*net.seen)/N, loss, avg_loss, get_current_rate(net), sec(clock()-time), *net.seen);
free_data(train);
if(*net.seen/N > epoch){
epoch = *net.seen/N;
char buff[256];
sprintf(buff, "%s/%s_%d.weights",backup_directory,base, epoch);
save_weights(net, buff);
}
if(*net.seen%1000 == 0){
char buff[256];
sprintf(buff, "%s/%s.backup",backup_directory,base);
save_weights(net, buff);
}
}
char buff[256];
sprintf(buff, "%s/%s.weights", backup_directory, base);
save_weights(net, buff);
pthread_join(load_thread, 0);
free_data(buffer);
free_network(net);
free_ptrs((void**)labels, 1000);
free_ptrs((void**)paths, plist->size);
free_list(plist);
free(base);
}
void train_lsd(char *cfgfile, char *weightfile, int clear)
{
char *train_images = "/home/pjreddie/data/coco/trainvalno5k.txt";
char *backup_directory = "/home/pjreddie/backup/";
srand(time(0));
char *base = basecfg(cfgfile);
printf("%s\n", base);
float avg_loss = -1;
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
if(clear) *net.seen = 0;
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
int imgs = net.batch*net.subdivisions;
int i = *net.seen/imgs;
data train, buffer;
list *plist = get_paths(train_images);
//int N = plist->size;
char **paths = (char **)list_to_array(plist);
load_args args = {};
args.w = net.w;
args.h = net.h;
args.paths = paths;
args.n = imgs;
args.m = plist->size;
args.d = &buffer;
args.min = net.min_crop;
args.max = net.max_crop;
args.angle = net.angle;
args.aspect = net.aspect;
args.exposure = net.exposure;
args.saturation = net.saturation;
args.hue = net.hue;
args.size = net.w;
args.type = CLASSIFICATION_DATA;
args.classes = 1;
char *ls[1] = {"coco"};
args.labels = ls;
pthread_t load_thread = load_data_in_thread(args);
clock_t time;
//while(i*imgs < N*120){
while(get_current_batch(net) < net.max_batches){
i += 1;
time=clock();
pthread_join(load_thread, 0);
train = buffer;
load_thread = load_data_in_thread(args);
printf("Loaded: %lf seconds\n", sec(clock()-time));
time=clock();
float loss = train_network(net, train);
if (avg_loss < 0) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
printf("%d: %f, %f avg, %f rate, %lf seconds, %d images\n", i, loss, avg_loss, get_current_rate(net), sec(clock()-time), i*imgs);
if(i%1000==0){
char buff[256];
sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i);
save_weights(net, buff);
}
if(i%100==0){
char buff[256];
sprintf(buff, "%s/%s.backup", backup_directory, base);
save_weights(net, buff);
}
free_data(train);
}
char buff[256];
sprintf(buff, "%s/%s_final.weights", backup_directory, base);
save_weights(net, buff);
}
void train_compare(char *cfgfile, char *weightfile)
{
data_seed = time(0);
srand(time(0));
float avg_loss = -1;
char *base = basecfg(cfgfile);
char *backup_directory = "/home/pjreddie/backup/";
printf("%s\n", base);
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
int imgs = 1024;
list *plist = get_paths("data/compare.train.list");
char **paths = (char **)list_to_array(plist);
int N = plist->size;
printf("%d\n", N);
clock_t time;
#ifndef _MSC_VER
pthread_t load_thread;
#endif
data train;
data buffer;
load_args args = {0};
args.w = net.w;
args.h = net.h;
args.paths = paths;
args.classes = 20;
args.n = imgs;
args.m = N;
args.d = &buffer;
args.type = COMPARE_DATA;
#ifndef _MSC_VER
load_thread = load_data_in_thread(args);
#endif
int epoch = *net.seen/N;
int i = 0;
while(1){
++i;
time=clock();
#ifndef _MSC_VER
pthread_join(load_thread, 0);
#else
load_data_in_thread(args);
#endif
train = buffer;
#ifndef _MSC_VER
load_thread = load_data_in_thread(args);
#endif
printf("Loaded: %lf seconds\n", sec(clock()-time));
time=clock();
float loss = train_network(net, train);
if(avg_loss == -1) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
printf("%.3f: %f, %f avg, %lf seconds, %d images\n", (float)*net.seen/N, loss, avg_loss, sec(clock()-time), *net.seen);
free_data(train);
if(i%100 == 0){
char buff[256];
sprintf(buff, "%s/%s_%d_minor_%d.weights",backup_directory,base, epoch, i);
save_weights(net, buff);
}
if(*net.seen/N > epoch){
epoch = *net.seen/N;
i = 0;
char buff[256];
sprintf(buff, "%s/%s_%d.weights",backup_directory,base, epoch);
save_weights(net, buff);
if(epoch%22 == 0) net.learning_rate *= .1;
}
}
#ifndef _MSC_VER
pthread_join(load_thread, 0);
#endif
free_data(buffer);
free_network(net);
free_ptrs((void**)paths, plist->size);
free_list(plist);
free(base);
}
void train_captcha(char *cfgfile, char *weightfile)
{
srand(time(0));
float avg_loss = -1;
char *base = basecfg(cfgfile);
printf("%s\n", base);
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
int imgs = 1024;
int i = *net.seen/imgs;
int solved = 1;
list *plist;
char **labels = get_labels("/data/captcha/reimgs.labels.list");
if (solved){
plist = get_paths("/data/captcha/reimgs.solved.list");
}else{
plist = get_paths("/data/captcha/reimgs.raw.list");
}
char **paths = (char **)list_to_array(plist);
printf("%d\n", plist->size);
clock_t time;
#if defined __linux__ || defined __APPLE__ || defined PTHREAD_WINDOWS
pthread_t load_thread;
#else
#endif
data train;
data buffer;
load_args args = {0};
args.w = net.w;
args.h = net.h;
args.paths = paths;
args.classes = 26;
args.n = imgs;
args.m = plist->size;
args.labels = labels;
args.d = &buffer;
args.type = CLASSIFICATION_DATA;
#if defined __linux__ || defined __APPLE__ || defined PTHREAD_WINDOWS
load_thread = load_data_in_thread(args);
#endif
while(1){
++i;
time=clock();
#if defined __linux__ || defined __APPLE__ || defined PTHREAD_WINDOWS
pthread_join(load_thread, 0);
#endif
train = buffer;
fix_data_captcha(train, solved);
/*
image im = float_to_image(256, 256, 3, train.X.vals[114]);
show_image(im, "training");
cvWaitKey(0);
*/
#if defined __linux__ || defined __APPLE__ || defined PTHREAD_WINDOWS
load_thread = load_data_in_thread(args);
#endif
printf("Loaded: %lf seconds\n", sec(clock()-time));
time=clock();
float loss = train_network(net, train);
if(avg_loss == -1) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
printf("%d: %f, %f avg, %lf seconds, %d images\n", i, loss, avg_loss, sec(clock()-time), *net.seen);
free_data(train);
if(i%100==0){
char buff[256];
sprintf(buff, "/home/pjreddie/imagenet_backup/%s_%d.weights",base, i);
save_weights(net, buff);
}
}
}
void train_writing(char *cfgfile, char *weightfile)
{
char *backup_directory = "/home/kunle12/backup/";
srand(time(0));
float avg_loss = -1;
char *base = basecfg(cfgfile);
printf("%s\n", base);
network * net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(net, weightfile);
}
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net->learning_rate, net->momentum, net->decay);
int imgs = net->batch*net->subdivisions;
list *plist = get_paths("figures.list");
char **paths = (char **)list_to_array(plist);
clock_t time;
int N = plist->size;
printf("N: %d\n", N);
image out = get_network_image(net);
data train, buffer;
load_args args = {0};
args.w = net->w;
args.h = net->h;
args.out_w = out.w;
args.out_h = out.h;
args.paths = paths;
args.n = imgs;
args.m = N;
args.d = &buffer;
args.type = WRITING_DATA;
pthread_t load_thread = load_data_in_thread(args);
int epoch = (*net->seen)/N;
while(get_current_batch(net) < net->max_batches || net->max_batches == 0){
time=clock();
pthread_join(load_thread, 0);
train = buffer;
load_thread = load_data_in_thread(args);
printf("Loaded %lf seconds\n",sec(clock()-time));
time=clock();
float loss = train_network(net, train);
/*
image pred = float_to_image(64, 64, 1, out);
print_image(pred);
*/
/*
image im = float_to_image(256, 256, 3, train.X.vals[0]);
image lab = float_to_image(64, 64, 1, train.y.vals[0]);
image pred = float_to_image(64, 64, 1, out);
show_image(im, "image");
show_image(lab, "label");
print_image(lab);
show_image(pred, "pred");
cvWaitKey(0);
*/
if(avg_loss == -1) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
printf("%ld, %.3f: %f, %f avg, %f rate, %lf seconds, %ld images\n", get_current_batch(net), (float)(*net->seen)/N, loss, avg_loss, get_current_rate(net), sec(clock()-time), *net->seen);
free_data(train);
if(get_current_batch(net)%100 == 0){
char buff[256];
sprintf(buff, "%s/%s_batch_%ld.weights", backup_directory, base, get_current_batch(net));
save_weights(net, buff);
}
if(*net->seen/N > epoch){
epoch = *net->seen/N;
char buff[256];
sprintf(buff, "%s/%s_%d.weights",backup_directory,base, epoch);
save_weights(net, buff);
}
}
}
void train_lsd2(char *cfgfile, char *weightfile, char *acfgfile, char *aweightfile, int clear)
{
#ifdef GPU
char *train_images = "/home/pjreddie/data/coco/trainvalno5k.txt";
char *backup_directory = "/home/pjreddie/backup/";
srand(time(0));
char *base = basecfg(cfgfile);
printf("%s\n", base);
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
if(clear) *net.seen = 0;
char *abase = basecfg(acfgfile);
network anet = parse_network_cfg(acfgfile);
if(aweightfile){
load_weights(&anet, aweightfile);
}
if(clear) *anet.seen = 0;
int i, j, k;
layer imlayer = {};
for (i = 0; i < net.n; ++i) {
if (net.layers[i].out_c == 3) {
imlayer = net.layers[i];
break;
}
}
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
int imgs = net.batch*net.subdivisions;
i = *net.seen/imgs;
data train, buffer;
list *plist = get_paths(train_images);
//int N = plist->size;
char **paths = (char **)list_to_array(plist);
load_args args = {};
args.w = net.w;
args.h = net.h;
args.paths = paths;
args.n = imgs;
args.m = plist->size;
args.d = &buffer;
args.min = net.min_crop;
args.max = net.max_crop;
args.angle = net.angle;
args.aspect = net.aspect;
args.exposure = net.exposure;
args.saturation = net.saturation;
args.hue = net.hue;
args.size = net.w;
args.type = CLASSIFICATION_DATA;
args.classes = 1;
char *ls[1] = {"coco"};
args.labels = ls;
pthread_t load_thread = load_data_in_thread(args);
clock_t time;
network_state gstate = {};
gstate.index = 0;
gstate.net = net;
int x_size = get_network_input_size(net)*net.batch;
int y_size = 1*net.batch;
gstate.input = cuda_make_array(0, x_size);
gstate.truth = 0;
gstate.delta = 0;
gstate.train = 1;
float *X = (float*)calloc(x_size, sizeof(float));
float *y = (float*)calloc(y_size, sizeof(float));
network_state astate = {};
astate.index = 0;
astate.net = anet;
int ay_size = get_network_output_size(anet)*anet.batch;
astate.input = 0;
astate.truth = 0;
astate.delta = 0;
astate.train = 1;
float *imerror = cuda_make_array(0, imlayer.outputs);
float *ones_gpu = cuda_make_array(0, ay_size);
fill_ongpu(ay_size, 1, ones_gpu, 1);
float aloss_avg = -1;
float gloss_avg = -1;
//data generated = copy_data(train);
while (get_current_batch(net) < net.max_batches) {
i += 1;
time=clock();
pthread_join(load_thread, 0);
train = buffer;
load_thread = load_data_in_thread(args);
printf("Loaded: %lf seconds\n", sec(clock()-time));
data generated = copy_data(train);
time=clock();
float gloss = 0;
for(j = 0; j < net.subdivisions; ++j){
get_next_batch(train, net.batch, j*net.batch, X, y);
cuda_push_array(gstate.input, X, x_size);
*net.seen += net.batch;
forward_network_gpu(net, gstate);
fill_ongpu(imlayer.outputs, 0, imerror, 1);
astate.input = imlayer.output_gpu;
astate.delta = imerror;
astate.truth = ones_gpu;
forward_network_gpu(anet, astate);
backward_network_gpu(anet, astate);
scal_ongpu(imlayer.outputs, 1, imerror, 1);
axpy_ongpu(imlayer.outputs, 1, imerror, 1, imlayer.delta_gpu, 1);
backward_network_gpu(net, gstate);
printf("features %f\n", cuda_mag_array(imlayer.delta_gpu, imlayer.outputs));
printf("realness %f\n", cuda_mag_array(imerror, imlayer.outputs));
gloss += get_network_cost(net) /(net.subdivisions*net.batch);
cuda_pull_array(imlayer.output_gpu, imlayer.output, x_size);
for(k = 0; k < net.batch; ++k){
int index = j*net.batch + k;
copy_cpu(imlayer.outputs, imlayer.output + k*imlayer.outputs, 1, generated.X.vals[index], 1);
generated.y.vals[index][0] = 0;
}
}
harmless_update_network_gpu(anet);
data merge = concat_data(train, generated);
randomize_data(merge);
float aloss = train_network(anet, merge);
update_network_gpu(net);
update_network_gpu(anet);
free_data(merge);
free_data(train);
free_data(generated);
if (aloss_avg < 0) aloss_avg = aloss;
aloss_avg = aloss_avg*.9 + aloss*.1;
gloss_avg = gloss_avg*.9 + gloss*.1;
printf("%d: gen: %f, adv: %f | gen_avg: %f, adv_avg: %f, %f rate, %lf seconds, %d images\n", i, gloss, aloss, gloss_avg, aloss_avg, get_current_rate(net), sec(clock()-time), i*imgs);
if(i%1000==0){
char buff[256];
sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i);
save_weights(net, buff);
sprintf(buff, "%s/%s_%d.weights", backup_directory, abase, i);
save_weights(anet, buff);
}
if(i%100==0){
char buff[256];
sprintf(buff, "%s/%s.backup", backup_directory, base);
save_weights(net, buff);
sprintf(buff, "%s/%s.backup", backup_directory, abase);
save_weights(anet, buff);
}
}
char buff[256];
sprintf(buff, "%s/%s_final.weights", backup_directory, base);
save_weights(net, buff);
#endif
}
void train_classifier(char *datacfg, char *cfgfile, char *weightfile, int *gpus, int ngpus, int clear)
{
int i;
float avg_loss = -1;
char *base = basecfg(cfgfile);
printf("%s\n", base);
printf("%d\n", ngpus);
network **nets = calloc(ngpus, sizeof(network*));
srand(time(0));
int seed = rand();
for(i = 0; i < ngpus; ++i){
srand(seed);
#ifdef GPU
cuda_set_device(gpus[i]);
#endif
nets[i] = load_network(cfgfile, weightfile, clear);
nets[i]->learning_rate *= ngpus;
}
srand(time(0));
network *net = nets[0];
int imgs = net->batch * net->subdivisions * ngpus;
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net->learning_rate, net->momentum, net->decay);
list *options = read_data_cfg(datacfg);
char *backup_directory = option_find_str(options, "backup", "/backup/");
char *label_list = option_find_str(options, "labels", "data/labels.list");
char *train_list = option_find_str(options, "train", "data/train.list");
int classes = option_find_int(options, "classes", 2);
char **labels = get_labels(label_list);
list *plist = get_paths(train_list);
char **paths = (char **)list_to_array(plist);
printf("%d\n", plist->size);
int N = plist->size;
double time;
load_args args = {0};
args.w = net->w;
args.h = net->h;
args.threads = 32;
args.hierarchy = net->hierarchy;
args.min = net->min_ratio*net->w;
args.max = net->max_ratio*net->w;
printf("%d %d\n", args.min, args.max);
args.angle = net->angle;
args.aspect = net->aspect;
args.exposure = net->exposure;
args.saturation = net->saturation;
args.hue = net->hue;
args.size = net->w;
args.paths = paths;
args.classes = classes;
args.n = imgs;
args.m = N;
args.labels = labels;
args.type = CLASSIFICATION_DATA;
data train;
data buffer;
pthread_t load_thread;
args.d = &buffer;
load_thread = load_data(args);
int count = 0;
int epoch = (*net->seen)/N;
while(get_current_batch(net) < net->max_batches || net->max_batches == 0){
if(net->random && count++%40 == 0){
printf("Resizing\n");
int dim = (rand() % 11 + 4) * 32;
//if (get_current_batch(net)+200 > net->max_batches) dim = 608;
//int dim = (rand() % 4 + 16) * 32;
printf("%d\n", dim);
args.w = dim;
args.h = dim;
args.size = dim;
args.min = net->min_ratio*dim;
args.max = net->max_ratio*dim;
printf("%d %d\n", args.min, args.max);
pthread_join(load_thread, 0);
train = buffer;
free_data(train);
load_thread = load_data(args);
for(i = 0; i < ngpus; ++i){
resize_network(nets[i], dim, dim);
}
net = nets[0];
}
time = what_time_is_it_now();
pthread_join(load_thread, 0);
train = buffer;
load_thread = load_data(args);
printf("Loaded: %lf seconds\n", what_time_is_it_now()-time);
time = what_time_is_it_now();
float loss = 0;
#ifdef GPU
if(ngpus == 1){
loss = train_network(net, train);
} else {
loss = train_networks(nets, ngpus, train, 4);
}
#else
loss = train_network(net, train);
#endif
if(avg_loss == -1) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
printf("%ld, %.3f: %f, %f avg, %f rate, %lf seconds, %ld images\n", get_current_batch(net), (float)(*net->seen)/N, loss, avg_loss, get_current_rate(net), what_time_is_it_now()-time, *net->seen);
free_data(train);
if(*net->seen/N > epoch){
epoch = *net->seen/N;
char buff[256];
sprintf(buff, "%s/%s_%d.weights",backup_directory,base, epoch);
save_weights(net, buff);
}
if(get_current_batch(net)%1000 == 0){
char buff[256];
sprintf(buff, "%s/%s.backup",backup_directory,base);
save_weights(net, buff);
}
}
char buff[256];
sprintf(buff, "%s/%s.weights", backup_directory, base);
save_weights(net, buff);
pthread_join(load_thread, 0);
free_network(net);
free_ptrs((void**)labels, classes);
free_ptrs((void**)paths, plist->size);
free_list(plist);
free(base);
}
void train_classifier(char *datacfg, char *cfgfile, char *weightfile)
{
data_seed = time(0);
srand(time(0));
float avg_loss = -1;
char *base = basecfg(cfgfile);
printf("%s\n", base);
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
int imgs = 1024;
list *options = read_data_cfg(datacfg);
char *backup_directory = option_find_str(options, "backup", "/backup/");
char *label_list = option_find_str(options, "labels", "data/labels.list");
char *train_list = option_find_str(options, "train", "data/train.list");
int classes = option_find_int(options, "classes", 2);
char **labels = get_labels(label_list);
list *plist = get_paths(train_list);
char **paths = (char **)list_to_array(plist);
printf("%d\n", plist->size);
int N = plist->size;
clock_t time;
pthread_t load_thread;
data train;
data buffer;
load_args args = {0};
args.w = net.w;
args.h = net.h;
args.min = net.w;
args.max = net.max_crop;
args.size = net.w;
args.paths = paths;
args.classes = classes;
args.n = imgs;
args.m = N;
args.labels = labels;
args.d = &buffer;
args.type = CLASSIFICATION_DATA;
load_thread = load_data_in_thread(args);
int epoch = (*net.seen)/N;
while(get_current_batch(net) < net.max_batches || net.max_batches == 0){
time=clock();
pthread_join(load_thread, 0);
train = buffer;
load_thread = load_data_in_thread(args);
printf("Loaded: %lf seconds\n", sec(clock()-time));
time=clock();
/*
int u;
for(u = 0; u < net.batch; ++u){
image im = float_to_image(net.w, net.h, 3, train.X.vals[u]);
show_image(im, "loaded");
cvWaitKey(0);
}
*/
float loss = train_network(net, train);
if(avg_loss == -1) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
printf("%d, %.3f: %f, %f avg, %f rate, %lf seconds, %d images\n", get_current_batch(net), (float)(*net.seen)/N, loss, avg_loss, get_current_rate(net), sec(clock()-time), *net.seen);
free_data(train);
if(*net.seen/N > epoch){
epoch = *net.seen/N;
char buff[256];
sprintf(buff, "%s/%s_%d.weights",backup_directory,base, epoch);
save_weights(net, buff);
}
if(*net.seen%100 == 0){
char buff[256];
sprintf(buff, "%s/%s.backup",backup_directory,base);
save_weights(net, buff);
}
}
char buff[256];
sprintf(buff, "%s/%s.weights", backup_directory, base);
save_weights(net, buff);
pthread_join(load_thread, 0);
free_data(buffer);
free_network(net);
free_ptrs((void**)labels, classes);
free_ptrs((void**)paths, plist->size);
free_list(plist);
free(base);
}
void train_lsd3(char *fcfg, char *fweight, char *gcfg, char *gweight, char *acfg, char *aweight, int clear)
{
#ifdef GPU
//char *train_images = "/home/pjreddie/data/coco/trainvalno5k.txt";
char *train_images = "/home/pjreddie/data/imagenet/imagenet1k.train.list";
//char *style_images = "/home/pjreddie/data/coco/trainvalno5k.txt";
char *style_images = "/home/pjreddie/zelda.txt";
char *backup_directory = "/home/pjreddie/backup/";
srand(time(0));
network fnet = load_network(fcfg, fweight, clear);
network gnet = load_network(gcfg, gweight, clear);
network anet = load_network(acfg, aweight, clear);
char *gbase = basecfg(gcfg);
char *abase = basecfg(acfg);
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", gnet.learning_rate, gnet.momentum, gnet.decay);
int imgs = gnet.batch*gnet.subdivisions;
int i = *gnet.seen/imgs;
data train, tbuffer;
data style, sbuffer;
list *slist = get_paths(style_images);
char **spaths = (char **)list_to_array(slist);
list *tlist = get_paths(train_images);
char **tpaths = (char **)list_to_array(tlist);
load_args targs= get_base_args(gnet);
targs.paths = tpaths;
targs.n = imgs;
targs.m = tlist->size;
targs.d = &tbuffer;
targs.type = CLASSIFICATION_DATA;
targs.classes = 1;
char *ls[1] = {"zelda"};
targs.labels = ls;
load_args sargs = get_base_args(gnet);
sargs.paths = spaths;
sargs.n = imgs;
sargs.m = slist->size;
sargs.d = &sbuffer;
sargs.type = CLASSIFICATION_DATA;
sargs.classes = 1;
sargs.labels = ls;
pthread_t tload_thread = load_data_in_thread(targs);
pthread_t sload_thread = load_data_in_thread(sargs);
clock_t time;
float aloss_avg = -1;
float floss_avg = -1;
network_state fstate = {};
fstate.index = 0;
fstate.net = fnet;
int x_size = get_network_input_size(fnet)*fnet.batch;
int y_size = get_network_output_size(fnet)*fnet.batch;
fstate.input = cuda_make_array(0, x_size);
fstate.truth = cuda_make_array(0, y_size);
fstate.delta = cuda_make_array(0, x_size);
fstate.train = 1;
float *X = (float*)calloc(x_size, sizeof(float));
float *y = (float*)calloc(y_size, sizeof(float));
float *ones = cuda_make_array(0, anet.batch);
float *zeros = cuda_make_array(0, anet.batch);
fill_ongpu(anet.batch, .99, ones, 1);
fill_ongpu(anet.batch, .01, zeros, 1);
network_state astate = {};
astate.index = 0;
astate.net = anet;
int ax_size = get_network_input_size(anet)*anet.batch;
int ay_size = get_network_output_size(anet)*anet.batch;
astate.input = 0;
astate.truth = ones;
astate.delta = cuda_make_array(0, ax_size);
astate.train = 1;
network_state gstate = {};
gstate.index = 0;
gstate.net = gnet;
int gx_size = get_network_input_size(gnet)*gnet.batch;
int gy_size = get_network_output_size(gnet)*gnet.batch;
gstate.input = cuda_make_array(0, gx_size);
gstate.truth = 0;
gstate.delta = 0;
gstate.train = 1;
while (get_current_batch(gnet) < gnet.max_batches) {
i += 1;
time=clock();
pthread_join(tload_thread, 0);
pthread_join(sload_thread, 0);
train = tbuffer;
style = sbuffer;
tload_thread = load_data_in_thread(targs);
sload_thread = load_data_in_thread(sargs);
printf("Loaded: %lf seconds\n", sec(clock()-time));
data generated = copy_data(train);
time=clock();
int j, k;
float floss = 0;
for(j = 0; j < fnet.subdivisions; ++j){
layer imlayer = gnet.layers[gnet.n - 1];
get_next_batch(train, fnet.batch, j*fnet.batch, X, y);
cuda_push_array(fstate.input, X, x_size);
cuda_push_array(gstate.input, X, gx_size);
*gnet.seen += gnet.batch;
forward_network_gpu(fnet, fstate);
float *feats = fnet.layers[fnet.n - 2].output_gpu;
copy_ongpu(y_size, feats, 1, fstate.truth, 1);
forward_network_gpu(gnet, gstate);
float *gen = gnet.layers[gnet.n-1].output_gpu;
copy_ongpu(x_size, gen, 1, fstate.input, 1);
fill_ongpu(x_size, 0, fstate.delta, 1);
forward_network_gpu(fnet, fstate);
backward_network_gpu(fnet, fstate);
//HERE
astate.input = gen;
fill_ongpu(ax_size, 0, astate.delta, 1);
forward_network_gpu(anet, astate);
backward_network_gpu(anet, astate);
float *delta = imlayer.delta_gpu;
fill_ongpu(x_size, 0, delta, 1);
scal_ongpu(x_size, 100, astate.delta, 1);
scal_ongpu(x_size, .00001, fstate.delta, 1);
axpy_ongpu(x_size, 1, fstate.delta, 1, delta, 1);
axpy_ongpu(x_size, 1, astate.delta, 1, delta, 1);
//fill_ongpu(x_size, 0, delta, 1);
//cuda_push_array(delta, X, x_size);
//axpy_ongpu(x_size, -1, imlayer.output_gpu, 1, delta, 1);
//printf("pix error: %f\n", cuda_mag_array(delta, x_size));
printf("fea error: %f\n", cuda_mag_array(fstate.delta, x_size));
printf("adv error: %f\n", cuda_mag_array(astate.delta, x_size));
//axpy_ongpu(x_size, 1, astate.delta, 1, delta, 1);
backward_network_gpu(gnet, gstate);
floss += get_network_cost(fnet) /(fnet.subdivisions*fnet.batch);
cuda_pull_array(imlayer.output_gpu, imlayer.output, x_size);
for(k = 0; k < gnet.batch; ++k){
int index = j*gnet.batch + k;
copy_cpu(imlayer.outputs, imlayer.output + k*imlayer.outputs, 1, generated.X.vals[index], 1);
generated.y.vals[index][0] = .01;
}
}
/*
image sim = float_to_image(anet.w, anet.h, anet.c, style.X.vals[j]);
show_image(sim, "style");
cvWaitKey(0);
*/
harmless_update_network_gpu(anet);
data merge = concat_data(style, generated);
randomize_data(merge);
float aloss = train_network(anet, merge);
update_network_gpu(gnet);
free_data(merge);
free_data(train);
free_data(generated);
free_data(style);
if (aloss_avg < 0) aloss_avg = aloss;
if (floss_avg < 0) floss_avg = floss;
aloss_avg = aloss_avg*.9 + aloss*.1;
floss_avg = floss_avg*.9 + floss*.1;
printf("%d: gen: %f, adv: %f | gen_avg: %f, adv_avg: %f, %f rate, %lf seconds, %d images\n", i, floss, aloss, floss_avg, aloss_avg, get_current_rate(gnet), sec(clock()-time), i*imgs);
if(i%1000==0){
char buff[256];
sprintf(buff, "%s/%s_%d.weights", backup_directory, gbase, i);
save_weights(gnet, buff);
sprintf(buff, "%s/%s_%d.weights", backup_directory, abase, i);
save_weights(anet, buff);
}
if(i%100==0){
char buff[256];
sprintf(buff, "%s/%s.backup", backup_directory, gbase);
save_weights(gnet, buff);
sprintf(buff, "%s/%s.backup", backup_directory, abase);
save_weights(anet, buff);
}
}
#endif
}
void train_coco(char *cfgfile, char *weightfile)
{
//char *train_images = "/home/pjreddie/data/voc/test/train.txt";
//char *train_images = "/home/pjreddie/data/coco/train.txt";
char *train_images = "data/coco.trainval.txt";
char *backup_directory = "/home/pjreddie/backup/";
srand(time(0));
data_seed = time(0);
char *base = basecfg(cfgfile);
printf("%s\n", base);
float avg_loss = -1;
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
int imgs = net.batch*net.subdivisions;
int i = *net.seen/imgs;
data train, buffer;
layer l = net.layers[net.n - 1];
int side = l.side;
int classes = l.classes;
float jitter = l.jitter;
list *plist = get_paths(train_images);
//int N = plist->size;
char **paths = (char **)list_to_array(plist);
load_args args = {0};
args.w = net.w;
args.h = net.h;
args.paths = paths;
args.n = imgs;
args.m = plist->size;
args.classes = classes;
args.jitter = jitter;
args.num_boxes = side;
args.d = &buffer;
args.type = REGION_DATA;
pthread_t load_thread = load_data_in_thread(args);
clock_t time;
//while(i*imgs < N*120){
while(get_current_batch(net) < net.max_batches){
i += 1;
time=clock();
pthread_join(load_thread, 0);
train = buffer;
load_thread = load_data_in_thread(args);
printf("Loaded: %lf seconds\n", sec(clock()-time));
/*
image im = float_to_image(net.w, net.h, 3, train.X.vals[113]);
image copy = copy_image(im);
draw_coco(copy, train.y.vals[113], 7, "truth");
cvWaitKey(0);
free_image(copy);
*/
time=clock();
float loss = train_network(net, train);
if (avg_loss < 0) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
printf("%d: %f, %f avg, %f rate, %lf seconds, %d images\n", i, loss, avg_loss, get_current_rate(net), sec(clock()-time), i*imgs);
if(i%1000==0 || (i < 1000 && i%100 == 0)){
char buff[256];
sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i);
save_weights(net, buff);
}
free_data(train);
}
char buff[256];
sprintf(buff, "%s/%s_final.weights", backup_directory, base);
save_weights(net, buff);
}
void train_super(char *cfgfile, char *weightfile, int clear)
{
char *train_images = "/data/imagenet/imagenet1k.train.list";
char *backup_directory = "/home/pjreddie/backup/";
srand(time(0));
char *base = basecfg(cfgfile);
printf("%s\n", base);
float avg_loss = -1;
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
if(clear) *net.seen = 0;
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
int imgs = net.batch*net.subdivisions;
int i = *net.seen/imgs;
data train, buffer;
list *plist = get_paths(train_images);
//int N = plist->size;
char **paths = (char **)list_to_array(plist);
load_args args = {0};
args.w = net.w;
args.h = net.h;
args.scale = 4;
args.paths = paths;
args.n = imgs;
args.m = plist->size;
args.d = &buffer;
args.type = SUPER_DATA;
#ifdef __linux__
pthread_t load_thread = load_data_in_thread(args);
#endif
clock_t time;
//while(i*imgs < N*120){
while(get_current_batch(net) < net.max_batches){
i += 1;
time=clock();
#ifdef __linux__
pthread_join(load_thread, 0);
#endif
train = buffer;
#ifdef __linux__
load_thread = load_data_in_thread(args);
#endif
printf("Loaded: %lf seconds\n", sec(clock()-time));
time=clock();
float loss = train_network(net, train);
if (avg_loss < 0) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
printf("%d: %f, %f avg, %f rate, %lf seconds, %d images\n", i, loss, avg_loss, get_current_rate(net), sec(clock()-time), i*imgs);
if(i%1000==0){
char buff[256];
sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i);
save_weights(net, buff);
}
if(i%100==0){
char buff[256];
sprintf(buff, "%s/%s.backup", backup_directory, base);
save_weights(net, buff);
}
free_data(train);
}
char buff[256];
sprintf(buff, "%s/%s_final.weights", backup_directory, base);
save_weights(net, buff);
}
void Workspace::execute(Parser &parser)
{
if(parser.command.compare("demo") == 0)
{
ConsolePrinter::instance().feedback_rewrite(
"Starting Demo... ");
// run_tests();
return;
}
// classic network
else if(parser.command.compare("cn") == 0)
{
NNInfo_uptr ret = classic_network(parser, networks);
if(ret->result == NNOK)
{
ConsolePrinter::instance().network_write_nets(networks);
}
else
{
ConsolePrinter::instance().feedback_rewrite(ret->message);
}
return;
}
else if(parser.command.compare("use") == 0)
{
NNInfo_uptr ret = use_network(parser, networks, current_network);
if(ret->result == NNOK)
{
ConsolePrinter::instance().network_write_active(current_network);
}
else
{
ConsolePrinter::instance().feedback_rewrite(ret->message);
}
return;
}
else if(parser.command.compare("set_io") == 0)
{
NNInfo_uptr ret = set_io(parser, networks, current_network);
if(ret->result == NNOK)
{
ConsolePrinter::instance().feedback_rewrite("Input and Output loaded");
}
else
{
ConsolePrinter::instance().feedback_rewrite(ret->message);
}
return;
}
else if(parser.command.compare("train") == 0)
{
NNInfo_uptr ret = train_network(parser, current_network);
if(ret->result != NNOK)
{
ConsolePrinter::instance().feedback_rewrite(ret->message);
}
return;
}
else if(parser.command.compare("eval") == 0)
{
NNInfo_uptr ret = eval_network(parser, current_network);
if(ret->result != NNOK)
{
ConsolePrinter::instance().feedback_rewrite(ret->message);
}
else
{
std::unique_ptr<double[]> out = current_network->get_output();
size_t out_len = current_network->output_layer_size_;
// TODO:
// Do not output from here.
for(size_t ii = 0; ii < out_len; ii++)
{
std::string outstr = "Output ";
double val = out[ii]*current_network->output_scale;
outstr.append(std::to_string(ii))
.append(": Value: ")
.append(std::to_string(val));
ConsolePrinter::instance().feedback_write(outstr);
}
}
return;
}
else if(parser.command.compare("exit") == 0)
{
abort();
}
else if(parser.command.compare("scale") == 0)
{
NNInfo_uptr ret = scale_network(parser, current_network);
if(ret->result != NNOK)
{
ConsolePrinter::instance().feedback_rewrite(ret->message);
}
else
{
ConsolePrinter::instance().feedback_rewrite("Scales set");
}
return;
}
else if(parser.command.compare("plotSquare") == 0)
{
// NNInfo_uptr ret = plot_square(parser, current_network);
// if(ret->result != NNOK)
// {
// ConsolePrinter::instance().feedback_rewrite(ret->message);
// }
// else
// {
// ConsolePrinter::instance().feedback_rewrite("Plot complete");
// }
return;
}
else if(parser.command.compare("reset") == 0)
{
NNInfo_uptr ret = reset_network(parser, current_network);
if(ret->result != NNOK)
{
ConsolePrinter::instance().feedback_rewrite(ret->message);
}
else
{
ConsolePrinter::instance().feedback_rewrite("Weights are reset");
}
return;
}
else if(parser.command.compare("retrain") == 0)
{
NNInfo_uptr ret = retrain_network(parser, current_network);
if(ret->result != NNOK)
{
ConsolePrinter::instance().feedback_rewrite(ret->message);
}
return;
}
else if(parser.command.compare("validate") == 0)
{
NNInfo_uptr ret = validate_network(parser, current_network);
if(ret->result != NNOK)
{
ConsolePrinter::instance().feedback_rewrite(ret->message);
}
else
{
ConsolePrinter::instance().feedback_rewrite("Output file written");
}
return;
}
else
{
std::string result = "Unrecognized command: ";
result.append(parser.command).append(". Try demo");
ConsolePrinter::instance().feedback_rewrite(result);
}
}
void train_tag(char *cfgfile, char *weightfile, int clear) {
srand(time(0));
real_t avg_loss = -1;
char *base = basecfg(cfgfile);
char *backup_directory = "/home/pjreddie/backup/";
printf("%s\n", base);
network *net = load_network(cfgfile, weightfile, clear);
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net->learning_rate,
net->momentum, net->decay);
int imgs = 1024;
list *plist = get_paths("/home/pjreddie/tag/train.list");
char **paths = (char **) list_to_array(plist);
printf("%d\n", plist->size);
int N = plist->size;
clock_t time;
pthread_t load_thread;
data train;
data buffer;
load_args args = { 0 };
args.w = net->w;
args.h = net->h;
args.min = net->w;
args.max = net->max_crop;
args.size = net->w;
args.paths = paths;
args.classes = net->outputs;
args.n = imgs;
args.m = N;
args.d = &buffer;
args.type = TAG_DATA;
args.angle = net->angle;
args.exposure = net->exposure;
args.saturation = net->saturation;
args.hue = net->hue;
fprintf(stderr, "%d classes\n", net->outputs);
load_thread = load_data_in_thread(args);
int epoch = (*net->seen) / N;
while (get_current_batch(net) < net->max_batches || net->max_batches == 0) {
time = clock();
pthread_join(load_thread, 0);
train = buffer;
load_thread = load_data_in_thread(args);
printf("Loaded: %lf seconds\n", sec(clock() - time));
time = clock();
real_t loss = train_network(net, train);
if (avg_loss == -1)
avg_loss = loss;
avg_loss = avg_loss * .9 + loss * .1;
printf("%ld, %.3f: %f, %f avg, %f rate, %lf seconds, %ld images\n",
get_current_batch(net), (real_t)(*net->seen) / N, loss,
avg_loss, get_current_rate(net), sec(clock() - time),
*net->seen);
free_data(train);
if (*net->seen / N > epoch) {
epoch = *net->seen / N;
char buff[256];
sprintf(buff, "%s/%s_%d.weights", backup_directory, base, epoch);
save_weights(net, buff);
}
if (get_current_batch(net) % 100 == 0) {
char buff[256];
sprintf(buff, "%s/%s.backup", backup_directory, base);
save_weights(net, buff);
}
}
char buff[256];
sprintf(buff, "%s/%s.weights", backup_directory, base);
save_weights(net, buff);
pthread_join(load_thread, 0);
free_data(buffer);
free_network(net);
free_ptrs((void**) paths, plist->size);
free_list(plist);
free(base);
}
void train_yolo(char *datacfg, char *cfgfile, char *weightfile)
{
list *options = read_data_cfg(datacfg);
char *train_list = option_find_str(options, "train", "data/train_list.txt");
//char *test_list = option_find_str(options, "test", "data/test_list.txt");
//char *valid_list = option_find_str(options, "valid", "data/valid_list.txt");
char *backup_directory = option_find_str(options, "backup", "/backup/");
//char *label_list = option_find_str(options, "labels", "data/labels_list.txt");
//int classes = option_find_int(options, "classes", 2);
srand(time(0));
data_seed = time(0);
char *base = basecfg(cfgfile);
printf("%s\n", base);
float avg_loss = -1;
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
int imgs = net.batch*net.subdivisions;
int i = *net.seen/imgs;
data train, buffer;
layer l = net.layers[net.n - 1];
int side = l.side;
int classes = l.classes;
float jitter = l.jitter;
list *plist = get_paths(train_list);
//int N = plist->size;
char **paths = (char **)list_to_array(plist);
load_args args = {0};
args.w = net.w;
args.h = net.h;
args.paths = paths;
args.n = imgs;
args.m = plist->size;
args.classes = classes;
args.jitter = jitter;
args.num_boxes = side;
args.d = &buffer;
args.type = REGION_DATA;
pthread_t load_thread = load_data_in_thread(args);
clock_t time;
//while(i*imgs < N*120){
while(get_current_batch(net) < net.max_batches){
i += 1;
time=clock();
pthread_join(load_thread, 0);
train = buffer;
load_thread = load_data_in_thread(args);
printf("Loaded: %lf seconds\n", sec(clock()-time));
time=clock();
float loss = train_network(net, train);
if (avg_loss < 0) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
printf("%d: %f, %f avg, %f rate, %lf seconds, %d images\n", i, loss, avg_loss, get_current_rate(net), sec(clock()-time), i*imgs);
if(i%1000==0 || (i < 1000 && i%100 == 0)){
char buff[256];
sprintf(buff, "%s/%s_%06d.weights", backup_directory, base, i);
save_weights(net, buff);
}
free_data(train);
}
char buff[256];
sprintf(buff, "%s/%s_final.weights", backup_directory, base);
save_weights(net, buff);
}
void train_go(char *cfgfile, char *weightfile)
{
data_seed = time(0);
srand(time(0));
float avg_loss = -1;
char *base = basecfg(cfgfile);
printf("%s\n", base);
network net = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&net, weightfile);
}
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
char *backup_directory = "/home/pjreddie/backup/";
char buff[256];
sprintf(buff, "/home/pjreddie/go.train.%02d", rand()%10);
data train = load_go(buff);
int N = train.X.rows;
int epoch = (*net.seen)/N;
while(get_current_batch(net) < net.max_batches || net.max_batches == 0){
clock_t time=clock();
data batch = get_random_data(train, net.batch);
int i;
for(i = 0; i < batch.X.rows; ++i){
int flip = rand()%2;
int rotate = rand()%4;
image in = float_to_image(19, 19, 1, batch.X.vals[i]);
image out = float_to_image(19, 19, 1, batch.y.vals[i]);
//show_image_normalized(in, "in");
//show_image_normalized(out, "out");
if(flip){
flip_image(in);
flip_image(out);
}
rotate_image_cw(in, rotate);
rotate_image_cw(out, rotate);
//show_image_normalized(in, "in2");
//show_image_normalized(out, "out2");
//cvWaitKey(0);
}
float loss = train_network(net, batch);
free_data(batch);
if(avg_loss == -1) avg_loss = loss;
avg_loss = avg_loss*.95 + loss*.05;
printf("%d, %.3f: %f, %f avg, %f rate, %lf seconds, %d images\n", get_current_batch(net), (float)(*net.seen)/N, loss, avg_loss, get_current_rate(net), sec(clock()-time), *net.seen);
if(*net.seen/N > epoch){
epoch = *net.seen/N;
char buff[256];
sprintf(buff, "%s/%s_%d.weights",backup_directory,base, epoch);
save_weights(net, buff);
free_data(train);
sprintf(buff, "/home/pjreddie/go.train.%02d", epoch%10);
train = load_go(buff);
}
if(get_current_batch(net)%100 == 0){
char buff[256];
sprintf(buff, "%s/%s.backup",backup_directory,base);
save_weights(net, buff);
}
}
sprintf(buff, "%s/%s.weights", backup_directory, base);
save_weights(net, buff);
free_network(net);
free(base);
free_data(train);
}
void train_detector(char *datacfg, char *cfgfile, char *weightfile, int *gpus, int ngpus, int clear)
{
list *options = read_data_cfg(datacfg);
char *train_images = option_find_str(options, "train", "data/train.list");
char *backup_directory = option_find_str(options, "backup", "/backup/");
srand(time(0));
char *base = basecfg(cfgfile);
printf("%s\n", base);
float avg_loss = -1;
network *nets = calloc(ngpus, sizeof(network));
srand(time(0));
int seed = rand();
int i;
for(i = 0; i < ngpus; ++i){
srand(seed);
#ifdef GPU
cuda_set_device(gpus[i]);
#endif
nets[i] = parse_network_cfg(cfgfile);
if(weightfile){
load_weights(&nets[i], weightfile);
}
if(clear) *nets[i].seen = 0;
nets[i].learning_rate *= ngpus;
}
srand(time(0));
network net = nets[0];
int imgs = net.batch * net.subdivisions * ngpus;
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
data train, buffer;
layer l = net.layers[net.n - 1];
int classes = l.classes;
float jitter = l.jitter;
list *plist = get_paths(train_images);
//int N = plist->size;
char **paths = (char **)list_to_array(plist);
load_args args = {0};
args.w = net.w;
args.h = net.h;
args.paths = paths;
args.n = imgs;
args.m = plist->size;
args.classes = classes;
args.jitter = jitter;
args.num_boxes = l.max_boxes;
args.d = &buffer;
args.type = DETECTION_DATA;
args.threads = 8;
args.angle = net.angle;
args.exposure = net.exposure;
args.saturation = net.saturation;
args.hue = net.hue;
pthread_t load_thread = load_data(args);
clock_t time;
int count = 0;
//while(i*imgs < N*120){
while(get_current_batch(net) < net.max_batches){
if(l.random && count++%10 == 0){
printf("Resizing\n");
//int dim = (rand() % 10 + 10) * 32;
//if (get_current_batch(net)+200 > net.max_batches) dim = 608;
//int dim = (rand() % 4 + 16) * 32;
int dim = (args.w <= args.h ? args.w : args.h);
printf("%d\n", dim);
args.w = dim;
args.h = dim;
pthread_join(load_thread, 0);
train = buffer;
free_data(train);
load_thread = load_data(args);
for(i = 0; i < ngpus; ++i){
resize_network(nets + i, dim, dim);
}
net = nets[0];
}
time=clock();
pthread_join(load_thread, 0);
train = buffer;
load_thread = load_data(args);
/*
int k;
for(k = 0; k < l.max_boxes; ++k){
box b = float_to_box(train.y.vals[10] + 1 + k*5);
if(!b.x) break;
printf("loaded: %f %f %f %f\n", b.x, b.y, b.w, b.h);
}
image im = float_to_image(448, 448, 3, train.X.vals[10]);
int k;
for(k = 0; k < l.max_boxes; ++k){
box b = float_to_box(train.y.vals[10] + 1 + k*5);
printf("%d %d %d %d\n", truth.x, truth.y, truth.w, truth.h);
draw_bbox(im, b, 8, 1,0,0);
}
save_image(im, "truth11");
*/
printf("Loaded: %lf seconds\n", sec(clock()-time));
time=clock();
float loss = 0;
#ifdef GPU
if(ngpus == 1){
loss = train_network(net, train);
} else {
loss = train_networks(nets, ngpus, train, 4);
}
#else
loss = train_network(net, train);
#endif
if (avg_loss < 0) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
i = get_current_batch(net);
printf("%d: %f, %f avg, %f rate, %lf seconds, %d images\n", get_current_batch(net), loss, avg_loss, get_current_rate(net), sec(clock()-time), i*imgs);
if(i%1000==0 || (i < 1000 && i%100 == 0)){
#ifdef GPU
if(ngpus != 1) sync_nets(nets, ngpus, 0);
#endif
char buff[256];
sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i);
save_weights(net, buff);
}
free_data(train);
}
#ifdef GPU
if(ngpus != 1) sync_nets(nets, ngpus, 0);
#endif
char buff[256];
sprintf(buff, "%s/%s_final.weights", backup_directory, base);
save_weights(net, buff);
}
void train_segmenter(char *datacfg, char *cfgfile, char *weightfile, int *gpus, int ngpus, int clear, int display)
{
int i;
float avg_loss = -1;
char *base = basecfg(cfgfile);
printf("%s\n", base);
printf("%d\n", ngpus);
network ** nets = calloc(ngpus, sizeof(network*));
srand(time(0));
int seed = rand();
for(i = 0; i < ngpus; ++i){
srand(seed);
#ifdef GPU
cuda_set_device(gpus[i]);
#endif
nets[i] = load_network(cfgfile, weightfile, clear);
nets[i]->learning_rate *= ngpus;
}
srand(time(0));
network * net = nets[0];
image pred = get_network_image(net);
int div = net->w/pred.w;
assert(pred.w * div == net->w);
assert(pred.h * div == net->h);
int imgs = net->batch * net->subdivisions * ngpus;
printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net->learning_rate, net->momentum, net->decay);
list *options = read_data_cfg(datacfg);
char *backup_directory = option_find_str(options, "backup", "/backup/");
char *train_list = option_find_str(options, "train", "data/train.list");
list *plist = get_paths(train_list);
char **paths = (char **)list_to_array(plist);
printf("%d\n", plist->size);
int N = plist->size;
load_args args = {0};
args.w = net->w;
args.h = net->h;
args.threads = 32;
args.scale = div;
args.min = net->min_crop;
args.max = net->max_crop;
args.angle = net->angle;
args.aspect = net->aspect;
args.exposure = net->exposure;
args.saturation = net->saturation;
args.hue = net->hue;
args.size = net->w;
args.classes = 80;
args.paths = paths;
args.n = imgs;
args.m = N;
args.type = SEGMENTATION_DATA;
data train;
data buffer;
pthread_t load_thread;
args.d = &buffer;
load_thread = load_data(args);
int epoch = (*net->seen)/N;
while(get_current_batch(net) < net->max_batches || net->max_batches == 0){
double time = what_time_is_it_now();
pthread_join(load_thread, 0);
train = buffer;
load_thread = load_data(args);
printf("Loaded: %lf seconds\n", what_time_is_it_now()-time);
time = what_time_is_it_now();
float loss = 0;
#ifdef GPU
if(ngpus == 1){
loss = train_network(net, train);
} else {
loss = train_networks(nets, ngpus, train, 4);
}
#else
loss = train_network(net, train);
#endif
if(display){
image tr = float_to_image(net->w/div, net->h/div, 80, train.y.vals[net->batch*(net->subdivisions-1)]);
image im = float_to_image(net->w, net->h, net->c, train.X.vals[net->batch*(net->subdivisions-1)]);
image mask = mask_to_rgb(tr);
image prmask = mask_to_rgb(pred);
show_image(im, "input", 1);
show_image(prmask, "pred", 1);
show_image(mask, "truth", 100);
free_image(mask);
free_image(prmask);
}
if(avg_loss == -1) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
printf("%ld, %.3f: %f, %f avg, %f rate, %lf seconds, %ld images\n", get_current_batch(net), (float)(*net->seen)/N, loss, avg_loss, get_current_rate(net), what_time_is_it_now()-time, *net->seen);
free_data(train);
if(*net->seen/N > epoch){
epoch = *net->seen/N;
char buff[256];
sprintf(buff, "%s/%s_%d.weights",backup_directory,base, epoch);
save_weights(net, buff);
}
if(get_current_batch(net)%100 == 0){
char buff[256];
sprintf(buff, "%s/%s.backup",backup_directory,base);
save_weights(net, buff);
}
}
char buff[256];
sprintf(buff, "%s/%s.weights", backup_directory, base);
save_weights(net, buff);
for(i = 0; i < ngpus; ++i){
free_network(nets[i]);
}
free(nets);
free_ptrs((void**)paths, plist->size);
free_list(plist);
free(base);
}
