// this does two sampling steps for a rbm. TODO: Do one sampling step less please.
rbm get_next(rbm* r) {
// first sample the hidden layer from the visible one
std::vector<int> hidden = sample_hidden(r, r->visible);
// make a new rbm
rbm r2 = *r;
std::vector<int> new_visible = sample_visible(r, r->hidden);
r2.visible = new_visible;
// you don't have to sample the hidden one
//std::vector<int> new_hidden = sample_hidden(&r2, r2.visible);
//r2.hidden = new_hidden;
return r2;
}
void RBM_P::train_full(bool reset = true, int rbmlayers_id = 0)
{
// Pop parameters
int epochs = *(int*) getParameter("epochs");
int batch_size = *(int*) getParameter("batch_size");
int cd_steps = *(int*) getParameter("cd_steps");
bool verbose = *(bool*) getParameter("verbose");
// ostream* out = *(ostream**) getParameter("log");
// // Pop LS
// Dataset* LS = sets[dataset];
// Dataset* QS = sets["QS"];
// assert(LS != NULL);
// if (conditional) {
// assert(QS != NULL);
// assert(LS->nb_rows == QS->nb_rows);
// }
// Start calculating the running time
struct timeval start;
struct timeval end;
unsigned long usec;
gettimeofday(&start, NULL);
// Reset parameters
if (reset) {
// Reset everything
this->reset();
// // Set vb_ik to the logs of their respective base rates
// // for (int n = 0; n < LS->nb_rows; n++) {
//// for (int m = LS->index[n]; m < LS->index[n] + LS->count[n]; m++) {
//// vb[_ik(LS->ids[m], LS->ratings[m] - 1)] += 1.;
// }
// }
//
// for (int i = 0; i < M; i++) {
// int ik_0 = _ik(i, 0);
// double total = 0.;
//
// for (int k = 0; k < K; k++) {
// total += vb[ik_0 + k];
// }
//
// if (total > 0.) {
// for (int k = 0; k < K; k++) {
// int ik = ik_0 + k;
//
// if (vb[ik] == 0.) {
// vb[ik] = -10E5;
// } else {
// vb[ik] = log(vb[ik] / total);
// }
// }
// }
// }
}
// print running time
gettimeofday(&end, NULL);
usec = 1000000 * (end.tv_sec-start.tv_sec) + end.tv_usec - start.tv_usec;
// cout << "File: " << __FILE__ << ", Function: " << __FUNCTION__ << ", Line: " << __LINE__ << endl;
printf( "File: %s, Function: %s, Line: %d\n", __FILE__, __FUNCTION__, __LINE__);
// cout << "Time of reset(): " << usec << " usec[" << usec / 1000000. <<" sec]." << endl;
printf("Time of reset(): %ld usec[ %lf sec].", usec, usec / 1000000.);
// Print some stats
// if (verbose) {
// *out << toString() << endl;
// *out << "% ---" << endl;
// *out << "% Epoch\tRMSE\tRMSE-TRAIN\tTIME\n";
// out->flush();
// //*out << 0 << "\t" << validate() << "\t" << usec / 1000000. << endl;
// char res[1000];
// sprintf(res, "0\t%lf\t%lf\t%lf\t\n", validate(), test("LS"), usec / 1000000.);
// *out << res;
//// *out << 0 << "\t" << validate() << "\t" << usec / 1000000. << endl;
//
//
// out->flush();
// }
// Start calculating the running time
gettimeofday(&start, NULL);
// Allocate local data structures
double* vs = new double[M * K];
double* vp = new double[M * K];
double* hs = new double[F];
double* hp = new double[F];
double* w_acc = new double[M * K * F];
int* w_count = new int[M * K * F];
double* vb_acc = new double[M * K];
int* vb_count = new int[M * K];
double* hb_acc = new double[F];
// bool* watched = NULL;
// if (conditional) {
// watched = new bool[M];
// }
char ss[1000];
// 只有id=0（最开始那层）的时候才调用这个train函数，其他层调用的是train_full函数
// 读取上一层的hs到这层的vs里面
sprintf(ss, "rbm-hs-%d", rbmlayers_id);
string hs_filename_in = ss;
ifstream in_hs(hs_filename_in.c_str(), ios::in | ios::binary);
if (in_hs.fail()) {
throw runtime_error("I/O exception");
}
in_hs.read((char*) vs, M * K * sizeof (double));
in_hs.close();
for(int dd = 0; dd < M * K; dd++)
printf("after reading from file: vs[%d]: %lf\n", dd, vs[dd]);
// 读取上一层的hb到这层的vb里面
sprintf(ss, "rbm-hb-%d", rbmlayers_id);
string hb_filename_in = ss;
ifstream in_hb(hb_filename_in.c_str(), ios::in | ios::binary);
if (in_hb.fail()) {
throw runtime_error("I/O exception");
}
in_hb.read((char*) vb, M * K * sizeof (double));
in_hb.close();
for(int dd = 0; dd < M * K; dd++)
printf("after reading from file: vb[%d]: %lf\n", dd, vb[dd]);
// Loop through epochs
for (int epoch = 1; epoch <= epochs; epoch++) {
// Start calculating the running time
struct timeval start_train;
struct timeval end_train;
unsigned long usec_train;
gettimeofday(&start_train, NULL);
// Loop through mini-batches of users
// for (int batch = 0; batch < LS->nb_rows; batch += batch_size)//
for (int batch = 0; batch < N; batch += batch_size) {
// Reset the accumulators
zero(w_acc, M * K * F);
zero(w_count, M * K * F);
zero(vb_acc, M * K);
zero(vb_count, M * K);
zero(hb_acc, F);
// if (conditional) {
// zero(watched, M);
// }
// Loop through users of current batch
for (int n = batch; n < min(batch + batch_size, N); n++) {
//for (int n = batch; n < min(batch + batch_size, LS->nb_rows); n++) //
// // Set user n data on the visible units
// for (int m = LS->index[n]; m < LS->index[n] + LS->count[n]; m++) {
// int i = LS->ids[m];
// int ik_0 = _ik(i, 0);
//
// for (int k = 0; k < K; k++) {
// vs[ik_0 + k] = 0.;
// }
//
// vs[ik_0 + LS->ratings[m] - 1] = 1.;
// }
int* mask_hidden = new int[F];
for(int ind = 0; ind < F; ind++) {
mask_hidden[ind] = ind;
}
int* mask_visible = new int[M];
for(int ind = 0; ind < M; ind++) {
mask_visible[ind] = ind;
}
// Compute p(h | V, d) into hp
if (!conditional) {
// update_hidden(vs, &LS->ids[LS->index[n]], LS->count[n], hp);
update_hidden_p(vs, &mask_visible[0], M, hp);
printf("finished first update hidden\n");
} else {
// update_hidden(vs, &LS->ids[LS->index[n]], LS->count[n], &QS->ids[QS->index[n]], QS->count[n], hp);
}
// Accumulate "v_ik * h_j", "v_ik" and "h_j"s (0)
for (int m = 0; m < M; m++) {
// for (int m = LS->index[n]; m < LS->index[n] + LS->count[n]; m++) //
// int i = LS->ids[m];
int i = m;
// int k = LS->ratings[m] - 1;
int k = 0;
int ikj_0 = _ikj(i, k, 0);
for (int j = 0; j < F; j++) {
w_acc[ikj_0 + j] += hp[j];
}
// 原来是对那些受影响的v+=1，现在是要更新所有v
vb_acc[_ik(i, k)] += 1.;
// 学习hb_acc，+= vp[]...不要乱搞。。。
printf("1.vb_acc[%d]: %lf\n", _ik(i, k), vb_acc[_ik(i, k)]);
printf("xx.vp[%d]: %lf\n", _ik(i, k), vp[_ik(i, k)]);
// vb_acc[_ik(i, k)] += vp[_ik(i, k)];
printf("2.vb_acc[%d]: %lf\n", _ik(i, k), vb_acc[_ik(i, k)]);
}
for (int j = 0; j < F; j++) {
hb_acc[j] += hp[j];
printf("hb_acc[%d]: %lf\n", j, hb_acc[j]);
}
// Count seen movies
// if (conditional) {
// for (int m = LS->index[n]; m < LS->index[n] + LS->count[n]; m++) {
// watched[LS->ids[m]] = true;
// }
//
// for (int m = QS->index[n]; m < QS->index[n] + QS->count[n]; m++) {
// watched[QS->ids[m]] = true;
// }
// }
// Gibbs sampling
for (int step = 0; step < cd_steps; step++) {
// Sample from p(h | V, d) into hs
sample_hidden(hp, hs);
// Compute p(V | h) into vp
update_visible_p(hs, vp, &mask_hidden[0], F);
// update_visible(hs, vp, &LS->ids[LS->index[n]], LS->count[n]);
// Sample from p(V | h) into hs
sample_visible_p(vp, vs, &mask_visible[0], M);
// sample_visible(vp, vs, &LS->ids[LS->index[n]], LS->count[n]);
// Compute p(h | V, d) into hp
if (!conditional) {
update_hidden_p(vs, &mask_visible[0], M, hp);
printf("finished second update hidden\n");
// update_hidden(vs, &LS->ids[LS->index[n]], LS->count[n], hp);
} else {
// update_hidden(vs, &LS->ids[LS->index[n]], LS->count[n], &QS->ids[QS->index[n]], QS->count[n], hp);
}
}
// Accumulate "-v_ik * h_j", "-v_ik" and "-h_j"s (T)
// Count the number of terms in the accumulators
for (int m = 0; m < M; m++) {
// for (int m = LS->index[n]; m < LS->index[n] + LS->count[n]; m++) //
// int i = LS->ids[m];
int i = m;
int ik_0 = _ik(i, 0);
for (int k = 0; k < K; k++) {
int ik = ik_0 + k;
printf("after cd, vs[%d]: %lf\n", ik, vs[ik]);
for (int j = 0; j < F; j++) {
int ikj = _ikj(i, k, j);
w_acc[ikj] -= vs[ik] * hp[j];
// w_count[ikj]++;
w_count[ikj] = M * K * F;
}
vb_acc[ik] -= vs[ik];
vb_count[ik]++;
//vb_count[ik] = M * K;
printf("vb_count[%d]: %d\n", ik, vb_count[ik]);
}
}
for (int j = 0; j < F; j++) {
hb_acc[j] -= hp[j];
}
}
// Update weights and biases
// 参数3是用来算annealing rate用的
update_w_p(w_acc, w_count, (epoch - 1) * N + batch);
printf("last update_vb\n");
update_vb_p(vb_acc, vb_count, (epoch - 1) * N + batch);
update_hb_p(hb_acc, (epoch - 1) * N + batch);
// if (conditional) {
// update_d(hb_acc, watched, (epoch - 1) * LS->nb_rows + batch);
// }
}
// // Log RMSE
// if (verbose) {
//
// gettimeofday(&end_train, NULL);
// usec_train = 1000000 * (end_train.tv_sec-start_train.tv_sec) + end_train.tv_usec - start_train.tv_usec;
// char res[1000];
// sprintf(res, "%d\t%lf\t%lf\t%lf\n", epoch, validate(), test("LS"), usec_train / 1000000.);
// *out << res;
//// *out << epoch << "\t" << validate() << "\t" << usec_train / 1000000. << endl;
// out->flush();
// }
}
// Added by daiwenkai
// 把hs的状态输出到文件，供下一层当做vs的输入
sprintf(ss, "rbm-hs-%d", rbmlayers_id + 1);
string hs_filename_out = ss;
ofstream out_hs(hs_filename_out.c_str(), ios::out | ios::binary);
if (out_hs.fail()) {
throw runtime_error("I/O exception! In openning rbm-hs-%d");
}
out_hs.write((char*) hs, F * sizeof (double));
out_hs.close();
// 把hb的状态输出到文件，供下一层当做vb的输入
sprintf(ss, "rbm-hb-%d", rbmlayers_id + 1);
string hb_filename_out = ss;
ofstream out_hb(hb_filename_out.c_str(), ios::out | ios::binary);
if (out_hb.fail()) {
throw runtime_error("I/O exception! In openning rbm-hs-%d");
}
out_hb.write((char*) hb, F * sizeof (double));
out_hb.close();
for(int dd = 0; dd < M; dd++)
printf("after train_full: vb[%d]: %lf\n", dd, vb[dd]);
for(int dd = 0; dd < F; dd++)
printf("after train_full: hb[%d]: %lf\n", dd, hb[dd]);
// Deallocate data structures
if (vs != NULL) delete[] vs;
if (vp != NULL) delete[] vp;
if (hs != NULL) delete[] hs;
if (hp != NULL) delete[] hp;
if (w_acc != NULL) delete[] w_acc;
if (w_count != NULL) delete[] w_count;
if (vb_acc != NULL) delete[] vb_acc;
if (vb_count != NULL) delete[] vb_count;
if (hb_acc != NULL) delete[] hb_acc;
// if (watched != NULL) delete[] watched;
// print running time
gettimeofday(&end, NULL);
usec = 1000000 * (end.tv_sec-start.tv_sec) + end.tv_usec - start.tv_usec;
// cout << "File: " << __FILE__ << ", Function: " << __FUNCTION__ << ", Line: " << __LINE__ << endl;
printf( "File: %s, Function: %s, Line: %d\n", __FILE__, __FUNCTION__, __LINE__);
// cout << "Time of train(): " << usec << " usec[" << usec / 1000000. <<" sec]." << endl;
printf("Time of train_full(): %ld usec[ %lf sec].", usec, usec / 1000000.);
}
void train_crbm(dataset_blas *train_set, int nvisible, int nhidden, int nlabel,
int epoch, double lr,
int minibatch, double momentum, char *model_file){
crbm m;
int i, j, k;
int nepoch;
double *v1, *y1;
uint8_t *l;
int batch_size, niter;
int wc;
int err;
double lik;
double delta;
int *idx;
time_t start_t, end_t;
init_crbm(&m, nvisible, nhidden, nlabel);
//wc = (int)cblas_dasum(train_set->N * train_set->n_feature, train_set->input, 1);
niter = (train_set->N-1)/minibatch + 1;
delta = lr / (1.0*minibatch);
/*
* shuffle training data
v1 = (double*)malloc(minibatch * m.nvisible * sizeof(double));
y1 = (double*)malloc(minibatch * m.ncat * sizeof(double));
l = (uint8_t*)malloc(minibatch * sizeof(uint8_t));
idx = (int*)malloc(train_set->N * sizeof(int));
for(i = 0; i < train_set->N; i++)
idx[i] = i;*/
bzero(w_u, sizeof(w_u));
bzero(u_u, sizeof(u_u));
bzero(bh_u, sizeof(bh_u));
bzero(bv_u, sizeof(bv_u));
bzero(by_u, sizeof(by_u));
//shuffle(idx, train_set->N);
for(nepoch = 0; nepoch < epoch; nepoch++){
lik = 0;
err = 0;
start_t = time(NULL);
for(k = 0; k < niter; k++){
#ifdef DEBUG
if((k+1) % 200 == 0){
printf("batch %d\n", k+1);
}
#endif
if(k == niter - 1){
batch_size = train_set->N - minibatch * (niter-1);
}else{
batch_size = minibatch;
}
v1 = train_set->input + train_set->n_feature * minibatch * k;
y1 = train_set->label + train_set->nlabel * minibatch * k;
l = train_set->output + minibatch * k;
/*
* shuffle training data
for(i = 0; i < batch_size; i++){
cblas_dcopy(m.nvisible, train_set->input + m.nvisible * idx[k*minibatch+i],
1, v1 + m.nvisible * i, 1);
cblas_dcopy(m.ncat, train_set->label + m.ncat * idx[k*minibatch+i],
1, y1 + m.ncat * i, 1);
l[i] = train_set->output[idx[k*minibatch+i]];
}*/
get_hidden(&m, v1, y1, ph1, batch_size);
sample_hidden(&m, ph1, h1, batch_size);
get_visible(&m, h1, pv, batch_size);
sample_visible(&m, pv, v2, batch_size);
get_class(&m, h1, py, batch_size);
sample_class(&m, py, y2, batch_size);
get_hidden(&m, v2, y2, ph2, batch_size);
sample_hidden(&m, ph2, h2, batch_size);
//lik += get_likelihood(&m, v1, pv, batch_size);
err += get_error(&m, y2, l, batch_size);
//update w_u
cblas_dgemm(CblasRowMajor, CblasTrans, CblasNoTrans,
m.nhidden, m.nvisible, batch_size,
1.0, ph2, m.nhidden, v2, m.nvisible,
0, a, m.nvisible);
cblas_dgemm(CblasRowMajor, CblasTrans, CblasNoTrans,
m.nhidden, m.nvisible, batch_size,
1.0, ph1, m.nhidden, v1, m.nvisible,
-1, a, m.nvisible);
cblas_daxpy(m.nvisible * m.nhidden, momentum, w_u, 1,
a, 1);
cblas_dcopy(m.nvisible * m.nhidden, a, 1, w_u, 1);
//update u_u
cblas_dgemm(CblasRowMajor, CblasTrans, CblasNoTrans,
m.nhidden, m.ncat, batch_size,
1.0, ph2, m.nhidden, y2, m.ncat,
0, a, m.ncat);
cblas_dgemm(CblasRowMajor, CblasTrans, CblasNoTrans,
m.nhidden, m.ncat, batch_size,
1.0, ph1, m.nhidden, y1, m.ncat,
-1, a, m.ncat);
cblas_daxpy(m.ncat * m.nhidden, momentum, u_u, 1,
a, 1);
cblas_dcopy(m.ncat * m.nhidden, a, 1, u_u, 1);
//update bv_u
cblas_dgemm(CblasRowMajor, CblasTrans, CblasNoTrans,
m.nvisible, 1, batch_size,
1.0, v2, m.nvisible, I, 1,
0, a, 1);
cblas_dgemm(CblasRowMajor, CblasTrans, CblasNoTrans,
m.nvisible, 1, batch_size,
1.0, v1, m.nvisible, I, 1,
-1, a, 1);
cblas_daxpy(m.nvisible, momentum, bv_u, 1,
a, 1);
cblas_dcopy(m.nvisible, a, 1, bv_u, 1);
//update by_u
cblas_dgemm(CblasRowMajor, CblasTrans, CblasNoTrans,
m.ncat, 1, batch_size,
1.0, y2, m.ncat, I, 1,
0, a, 1);
cblas_dgemm(CblasRowMajor, CblasTrans, CblasNoTrans,
m.ncat, 1, batch_size,
1.0, y1, m.ncat, I, 1,
-1, a, 1);
cblas_daxpy(m.ncat, momentum, by_u, 1,
a, 1);
cblas_dcopy(m.ncat, a, 1, by_u, 1);
//update bh_u
cblas_dgemm(CblasRowMajor, CblasTrans, CblasNoTrans,
m.nhidden, 1, batch_size,
1.0, ph2, m.nhidden, I, 1,
0, a, 1);
cblas_dgemm(CblasRowMajor, CblasTrans, CblasNoTrans,
m.nhidden, 1, batch_size,
1.0, ph1, m.nhidden, I, 1,
-1, a, 1);
cblas_daxpy(m.nhidden, momentum, bh_u, 1,
a, 1);
cblas_dcopy(m.nhidden, a, 1, bh_u, 1);
//change parameter
cblas_daxpy(m.nvisible * m.nhidden, delta, w_u, 1,
m.w, 1);
cblas_daxpy(m.ncat * m.nhidden, delta, u_u, 1,
m.u, 1);
cblas_daxpy(m.nvisible, delta, bv_u, 1,
m.bv, 1);
cblas_daxpy(m.ncat, delta, by_u, 1,
m.by, 1);
cblas_daxpy(m.nhidden, delta, bh_u, 1,
m.bh, 1);
}
end_t = time(NULL);
printf("[epoch %d] error:%.5lf%%\ttime:%.2fmin\n", nepoch + 1,
err * 100.0 / train_set->N, (end_t - start_t) / 60.0);
}
dump_model(&m, model_file);
//print_prob(&m, train_set, "../data/rsm/test.prob");
/*
* shuffle training data
free(v1);
free(y1);
free(l);*/
free_crbm(&m);
}