/* main */ int main(int argc, char * argv[]) { Prefs prefs = 0; int o; while((o = getopt(argc, argv, "kP")) != -1) switch(o) { case 'k': prefs |= DF_PREFS_k; break; case 'P': prefs |= DF_PREFS_P; break; default: return _usage(); } return _df(&prefs, argc - optind, &argv[optind]) == 0 ? 0 : 2; }
void operator()( const Range& range ) const { double inv_count = 1./inputs.rows; int ivcount = ann->layer_sizes.front(); int ovcount = ann->layer_sizes.back(); int itype = inputs.type(), otype = outputs.type(); int count = inputs.rows; int i, j, k, l_count = ann->layer_count(); vector<vector<double> > x(l_count); vector<vector<double> > df(l_count); vector<double> _buf(ann->max_lsize*dcount0*2); double* buf[] = { &_buf[0], &_buf[ann->max_lsize*dcount0] }; double E = 0; for( i = 0; i < l_count; i++ ) { x[i].resize(ann->layer_sizes[i]*dcount0); df[i].resize(ann->layer_sizes[i]*dcount0); } for( int si = range.start; si < range.end; si++ ) { int i0 = si*dcount0, i1 = std::min((si + 1)*dcount0, count); int dcount = i1 - i0; const double* w = ann->weights[0].ptr<double>(); // grab and preprocess input data for( i = 0; i < dcount; i++ ) { const uchar* x0data_p = inputs.ptr(i0 + i); const float* x0data_f = (const float*)x0data_p; const double* x0data_d = (const double*)x0data_p; double* xdata = &x[0][i*ivcount]; for( j = 0; j < ivcount; j++ ) xdata[j] = (itype == CV_32F ? (double)x0data_f[j] : x0data_d[j])*w[j*2] + w[j*2+1]; } Mat x1(dcount, ivcount, CV_64F, &x[0][0]); // forward pass, compute y[i]=w*x[i-1], x[i]=f(y[i]), df[i]=f'(y[i]) for( i = 1; i < l_count; i++ ) { Mat x2( dcount, ann->layer_sizes[i], CV_64F, &x[i][0] ); Mat _w = ann->weights[i].rowRange(0, x1.cols); gemm( x1, _w, 1, noArray(), 0, x2 ); Mat _df( x2.size(), CV_64F, &df[i][0] ); ann->calc_activ_func_deriv( x2, _df, ann->weights[i] ); x1 = x2; } Mat grad1(dcount, ovcount, CV_64F, buf[l_count & 1]); w = ann->weights[l_count+1].ptr<double>(); // calculate error for( i = 0; i < dcount; i++ ) { const uchar* udata_p = outputs.ptr(i0+i); const float* udata_f = (const float*)udata_p; const double* udata_d = (const double*)udata_p; const double* xdata = &x[l_count-1][i*ovcount]; double* gdata = grad1.ptr<double>(i); double sweight = sw ? sw[si+i] : inv_count, E1 = 0; for( j = 0; j < ovcount; j++ ) { double t = (otype == CV_32F ? (double)udata_f[j] : udata_d[j])*w[j*2] + w[j*2+1] - xdata[j]; gdata[j] = t*sweight; E1 += t*t; } E += sweight*E1; } for( i = l_count-1; i > 0; i-- ) { int n1 = ann->layer_sizes[i-1], n2 = ann->layer_sizes[i]; Mat _df(dcount, n2, CV_64F, &df[i][0]); multiply(grad1, _df, grad1); { AutoLock lock(ann->mtx); Mat _dEdw = dEdw->at(i).rowRange(0, n1); x1 = Mat(dcount, n1, CV_64F, &x[i-1][0]); gemm(x1, grad1, 1, _dEdw, 1, _dEdw, GEMM_1_T); // update bias part of dEdw double* dst = dEdw->at(i).ptr<double>(n1); for( k = 0; k < dcount; k++ ) { const double* src = grad1.ptr<double>(k); for( j = 0; j < n2; j++ ) dst[j] += src[j]; } } Mat grad2( dcount, n1, CV_64F, buf[i&1] ); if( i > 1 ) { Mat _w = ann->weights[i].rowRange(0, n1); gemm(grad1, _w, 1, noArray(), 0, grad2, GEMM_2_T); } grad1 = grad2; } } { AutoLock lock(ann->mtx); *pE += E; } }
int train_backprop( const Mat& inputs, const Mat& outputs, const Mat& _sw, TermCriteria termCrit ) { int i, j, k; double prev_E = DBL_MAX*0.5, E = 0; int itype = inputs.type(), otype = outputs.type(); int count = inputs.rows; int iter = -1, max_iter = termCrit.maxCount*count; double epsilon = termCrit.epsilon*count; int l_count = layer_count(); int ivcount = layer_sizes[0]; int ovcount = layer_sizes.back(); // allocate buffers vector<vector<double> > x(l_count); vector<vector<double> > df(l_count); vector<Mat> dw(l_count); for( i = 0; i < l_count; i++ ) { int n = layer_sizes[i]; x[i].resize(n+1); df[i].resize(n); dw[i] = Mat::zeros(weights[i].size(), CV_64F); } Mat _idx_m(1, count, CV_32S); int* _idx = _idx_m.ptr<int>(); for( i = 0; i < count; i++ ) _idx[i] = i; AutoBuffer<double> _buf(max_lsize*2); double* buf[] = { _buf, (double*)_buf + max_lsize }; const double* sw = _sw.empty() ? 0 : _sw.ptr<double>(); // run back-propagation loop /* y_i = w_i*x_{i-1} x_i = f(y_i) E = 1/2*||u - x_N||^2 grad_N = (x_N - u)*f'(y_i) dw_i(t) = momentum*dw_i(t-1) + dw_scale*x_{i-1}*grad_i w_i(t+1) = w_i(t) + dw_i(t) grad_{i-1} = w_i^t*grad_i */ for( iter = 0; iter < max_iter; iter++ ) { int idx = iter % count; double sweight = sw ? count*sw[idx] : 1.; if( idx == 0 ) { //printf("%d. E = %g\n", iter/count, E); if( fabs(prev_E - E) < epsilon ) break; prev_E = E; E = 0; // shuffle indices for( i = 0; i < count; i++ ) { j = rng.uniform(0, count); k = rng.uniform(0, count); std::swap(_idx[j], _idx[k]); } } idx = _idx[idx]; const uchar* x0data_p = inputs.ptr(idx); const float* x0data_f = (const float*)x0data_p; const double* x0data_d = (const double*)x0data_p; double* w = weights[0].ptr<double>(); for( j = 0; j < ivcount; j++ ) x[0][j] = (itype == CV_32F ? (double)x0data_f[j] : x0data_d[j])*w[j*2] + w[j*2 + 1]; Mat x1( 1, ivcount, CV_64F, &x[0][0] ); // forward pass, compute y[i]=w*x[i-1], x[i]=f(y[i]), df[i]=f'(y[i]) for( i = 1; i < l_count; i++ ) { int n = layer_sizes[i]; Mat x2(1, n, CV_64F, &x[i][0] ); Mat _w = weights[i].rowRange(0, x1.cols); gemm(x1, _w, 1, noArray(), 0, x2); Mat _df(1, n, CV_64F, &df[i][0] ); calc_activ_func_deriv( x2, _df, weights[i] ); x1 = x2; } Mat grad1( 1, ovcount, CV_64F, buf[l_count&1] ); w = weights[l_count+1].ptr<double>(); // calculate error const uchar* udata_p = outputs.ptr(idx); const float* udata_f = (const float*)udata_p; const double* udata_d = (const double*)udata_p; double* gdata = grad1.ptr<double>(); for( k = 0; k < ovcount; k++ ) { double t = (otype == CV_32F ? (double)udata_f[k] : udata_d[k])*w[k*2] + w[k*2+1] - x[l_count-1][k]; gdata[k] = t*sweight; E += t*t; } E *= sweight; // backward pass, update weights for( i = l_count-1; i > 0; i-- ) { int n1 = layer_sizes[i-1], n2 = layer_sizes[i]; Mat _df(1, n2, CV_64F, &df[i][0]); multiply( grad1, _df, grad1 ); Mat _x(n1+1, 1, CV_64F, &x[i-1][0]); x[i-1][n1] = 1.; gemm( _x, grad1, params.bpDWScale, dw[i], params.bpMomentScale, dw[i] ); add( weights[i], dw[i], weights[i] ); if( i > 1 ) { Mat grad2(1, n1, CV_64F, buf[i&1]); Mat _w = weights[i].rowRange(0, n1); gemm( grad1, _w, 1, noArray(), 0, grad2, GEMM_2_T ); grad1 = grad2; } } } iter /= count; return iter; }