void HLayeredBlWStructure::AtVkV( gsl_vector *u, long k, const gsl_matrix *A,
const gsl_vector *v, gsl_vector *tmpVkV, double beta ) const {
gsl_blas_dcopy(v, tmpVkV);
gsl_blas_dtrmv(CblasLower, (k > 0 ? CblasNoTrans : CblasTrans),
CblasNonUnit, getWk(abs(k)), tmpVkV);
gsl_blas_dgemv(CblasTrans, 1.0, A, tmpVkV, beta, u);
}
double lda_m_step(lda* model, lda_suff_stats* ss) {
int k, w;
double lhood = 0;
for (k = 0; k < model->ntopics; k++)
{
gsl_vector ss_k = gsl_matrix_column(ss->topics_ss, k).vector;
gsl_vector log_p = gsl_matrix_column(model->topics, k).vector;
if (LDA_USE_VAR_BAYES == 0)
{
gsl_blas_dcopy(&ss_k, &log_p);
normalize(&log_p);
vct_log(&log_p);
}
else
{
double digsum = sum(&ss_k)+model->nterms*LDA_TOPIC_DIR_PARAM;
digsum = gsl_sf_psi(digsum);
double param_sum = 0;
for (w = 0; w < model->nterms; w++)
{
double param = vget(&ss_k, w) + LDA_TOPIC_DIR_PARAM;
param_sum += param;
double elogprob = gsl_sf_psi(param) - digsum;
vset(&log_p, w, elogprob);
lhood += (LDA_TOPIC_DIR_PARAM - param) * elogprob + gsl_sf_lngamma(param);
}
lhood -= gsl_sf_lngamma(param_sum);
}
}
return(lhood);
}
double normal_null_maximum(gsl_vector *means,gsl_vector *s) {
assert(means->size == s->size);
// Baskara coefs.
double a,b,c;
gsl_vector *s2=gsl_vector_alloc(means->size);
gsl_blas_dcopy(s,s2);
gsl_vector_mul(s2,s2);
gsl_vector *B=gsl_vector_alloc(means->size);
gsl_blas_dcopy(means,B);
gsl_blas_dscal(-2.0,B);
//printf("B=%lg %lg\n",ELTd(B,0),ELTd(B,1));
gsl_vector *C=gsl_vector_alloc(means->size);
gsl_blas_dcopy(means,C);
gsl_vector_mul(C,C);
gsl_vector *prods=gsl_vector_alloc(means->size);
mutual_prod(s2,prods);
a=gsl_blas_dasum(prods);
gsl_blas_ddot(B,prods,&b);
gsl_blas_ddot(C,prods,&c);
printf("null max: a=%lf b=%lf c=%lf\n",a,b,c);
double delta=b*b-4*a*c;
double x0=-b/(2.0*a);
printf("null max: delta=%lg\n",delta);
if (fabs(delta) < 1e-5) {
return x0;
} else {
if (delta > 0) {
double x1=(-b-sqrt(delta))/(2.0*a);
double x2=(-b-sqrt(delta))/(2.0*a);
printf("null max: x1=%lg x2=%lg\n",x1,x2);
return x1;
} else {
printf("WARNING: Null max not found!\n");
return x0;
}
}
}
tnn_error tnn_module_bprop_bias(tnn_module *m){
//Routine check
if(m->t != TNN_MODULE_TYPE_BIAS){
return TNN_ERROR_MODULE_MISTYPE;
}
if(m->input->valid != true || m->output->valid != true || m->w.valid != true){
return TNN_ERROR_STATE_INVALID;
}
//bprop to input
TNN_MACRO_GSLTEST(gsl_blas_dcopy(&m->output->dx, &m->input->dx));
//bprop to dw
TNN_MACRO_GSLTEST(gsl_blas_dcopy(&m->output->dx, &m->w.dx));
return TNN_ERROR_SUCCESS;
}
tnn_error tnn_loss_bprop_euclidean(tnn_loss *l){
//Routine check
if(l->t != TNN_LOSS_TYPE_EUCLIDEAN){
return TNN_ERROR_LOSS_MISTYPE;
}
if(l->input1->valid != true || l->input2->valid != true || l->output->valid != true){
return TNN_ERROR_STATE_INVALID;
}
//bprop to input1 and input2 dx = dl 2 (x-y); dy = dl 2 (y-x)
TNN_MACRO_GSLTEST(gsl_blas_dcopy(&l->input1->x, &l->input1->dx));
TNN_MACRO_GSLTEST(gsl_blas_daxpy(-1.0, &l->input2->x, &l->input1->dx));
gsl_blas_dscal(2.0*gsl_vector_get(&l->output->dx, 0), &l->input1->dx);
TNN_MACRO_GSLTEST(gsl_blas_dcopy(&l->input1->dx, &l->input2->dx));
gsl_blas_dscal(-1.0, &l->input2->dx);
return TNN_ERROR_SUCCESS;
}
//Learn one sample using naive stochastic gradient descent
tnn_error tnn_trainer_class_learn_nsgd(tnn_trainer_class *t, gsl_vector *input, size_t label){
tnn_error ret;
tnn_state *sin;
tnn_param *p;
gsl_vector_view lb;
//Routine check
if(t->t != TNN_TRAINER_CLASS_TYPE_NSGD){
return TNN_ERROR_TRAINER_CLASS_MISTYPE;
}
//Check the input and label
TNN_MACRO_ERRORTEST(tnn_machine_get_sin(&t->m, &sin),ret);
if(label >= t->lset->size1 || input->size != sin->size){
return TNN_ERROR_STATE_INCOMP;
}
lb = gsl_matrix_row(t->lset, label);
//Set the loss output dx to be 1
gsl_vector_set(&t->l.output->dx, 0, 1.0);
//Copy the data into the input/label and do forward and backward propagation
TNN_MACRO_GSLTEST(gsl_blas_dcopy(input, &sin->x));
TNN_MACRO_GSLTEST(gsl_blas_dcopy(&lb.vector, &t->label->x));
TNN_MACRO_ERRORTEST(tnn_machine_fprop(&t->m), ret);
TNN_MACRO_ERRORTEST(tnn_loss_fprop(&t->l), ret);
TNN_MACRO_ERRORTEST(tnn_loss_bprop(&t->l), ret);
TNN_MACRO_ERRORTEST(tnn_machine_bprop(&t->m), ret);
//Compute the accumulated regularization paramter
TNN_MACRO_ERRORTEST(tnn_machine_get_param(&t->m, &p), ret);
TNN_MACRO_ERRORTEST(tnn_reg_addd(&t->r, p->x, p->dx, t->lambda), ret);
//Compute the parameter update
TNN_MACRO_GSLTEST(gsl_blas_daxpy(-((tnn_trainer_class_nsgd*)t->c)->eta, p->dx, p->x));
//Set the titer parameter
((tnn_trainer_class_nsgd*)t->c)->titer = 1;
return TNN_ERROR_SUCCESS;
}
tnn_error tnn_module_bprop_sum(tnn_module *m){
tnn_state **t;
//Routine check
if(m->t != TNN_MODULE_TYPE_SUM){
return TNN_ERROR_MODULE_MISTYPE;
}
if(m->input->valid != true || m->output->valid != true){
return TNN_ERROR_STATE_INVALID;
}
//bprop to each input
for(t = (tnn_state **)utarray_front(((tnn_module_sum*)m->c)->sarray);
t != NULL;
t = (tnn_state **)utarray_next(((tnn_module_sum*)m->c)->sarray, t)){
TNN_MACRO_GSLTEST(gsl_blas_dcopy(&m->output->dx, &(*t)->dx));
}
return TNN_ERROR_SUCCESS;
}
tnn_error tnn_loss_fprop_euclidean(tnn_loss *l){
gsl_vector *diff;
double loss;
//Routine check
if(l->t != TNN_LOSS_TYPE_EUCLIDEAN){
return TNN_ERROR_LOSS_MISTYPE;
}
if(l->input1->valid != true || l->input2->valid != true || l->output->valid != true){
return TNN_ERROR_STATE_INVALID;
}
//Do the forward propagation
if((diff = gsl_vector_alloc(l->input1->size)) == NULL){
return TNN_ERROR_GSL;
}
TNN_MACRO_GSLTEST(gsl_blas_dcopy(&l->input1->x, diff));
TNN_MACRO_GSLTEST(gsl_blas_daxpy(-1.0, &l->input2->x, diff));
loss = gsl_blas_dnrm2(diff);
gsl_vector_set(&l->output->x, 0, loss*loss);
gsl_vector_free(diff);
return TNN_ERROR_SUCCESS;
}
/**
* C++ version of gsl_blas_dcopy().
* @param X A vector
* @param Y A vector
* @return Error code on failure
*/
int dcopy( vector const& X, vector& Y ){ return gsl_blas_dcopy( X.get(), Y.get() ); }
static VALUE rb_gsl_blas_dcopy(int argc, VALUE *argv, VALUE obj)
{
gsl_vector *x = NULL, *y = NULL;
get_vector2(argc, argv, obj, &x, &y);
return INT2FIX(gsl_blas_dcopy(x, y));
}
static int
bundle_method_iterate (void *vstate, gsl_multimin_function_fsdf * fsdf, gsl_vector * x, double * f,
gsl_vector * subgradient, gsl_vector * dx, double * eps)
{
bundle_method_state_t *state = (bundle_method_state_t *) vstate;
bundle_element *item;
size_t i, debug=0;
int status;
double tmp_d, t_old, t_int_l; /* local variables */
gsl_vector *y; /* a trial point (the next iteration point by the serios step) */
gsl_vector *sgr_y; /* subgradient at y */
double f_y; /* the function value at y */
gsl_vector *p; /* the aggregate subgradient */
double p_norm, lin_error_p; /* norm of p, the aggregate linear. error */
gsl_vector *tmp_v;
/* data for the convex quadratic problem (for the dual problem) */
gsl_vector *q; /* elements of the array are the linearization errors */
gsl_matrix *Q; /* Q=G^T*G (G is matrix which collumns are subgradients) */
gsl_vector *lambda; /* the convex combination coefficients of the subgradients (solution of the dual problem) */
lambda = gsl_vector_alloc(state->bundle_size);
if(lambda == 0)
{
GSL_ERROR_VAL ("failed to allocate workspace", GSL_ENOMEM, 0);
}
q = gsl_vector_alloc(lambda->size);
if(q == 0)
{
gsl_vector_free(lambda);
GSL_ERROR_VAL ("failed to allocate workspace", GSL_ENOMEM, 0);
}
y = gsl_vector_calloc(x->size);
if(y == 0)
{
gsl_vector_free(q);
gsl_vector_free(lambda);
GSL_ERROR_VAL ("failed to allocate workspace", GSL_ENOMEM, 0);
}
sgr_y = gsl_vector_calloc(x->size);
if(sgr_y == 0)
{
gsl_vector_free(y);
gsl_vector_free(q);
gsl_vector_free(lambda);
GSL_ERROR_VAL ("failed to allocate workspace", GSL_ENOMEM, 0);
}
Q = gsl_matrix_alloc(state->bundle_size, state->bundle_size);
if(Q == 0)
{
gsl_vector_free(sgr_y);
gsl_vector_free(y);
gsl_vector_free(q);
gsl_vector_free(lambda);
GSL_ERROR_VAL ("failed to allocate workspace", GSL_ENOMEM, 0);
}
p = gsl_vector_calloc(x->size);
if(p == 0)
{
gsl_matrix_free(Q);
gsl_vector_free(sgr_y);
gsl_vector_free(y);
gsl_vector_free(q);
gsl_vector_free(lambda);
GSL_ERROR_VAL ("failed to allocate workspace", GSL_ENOMEM, 0);
}
tmp_v = gsl_vector_calloc(x->size);
if(tmp_v == 0)
{
gsl_vector_free(p);
gsl_matrix_free(Q);
gsl_vector_free(sgr_y);
gsl_vector_free(y);
gsl_vector_free(q);
gsl_vector_free(lambda);
GSL_ERROR_VAL ("failed to allocate workspace", GSL_ENOMEM, 0);
}
/* solve the dual problem */
status = build_cqp_data(state, Q, q);
status = solve_qp_pdip(Q, q, lambda);
gsl_matrix_free(Q);
gsl_vector_free(q);
/* compute the aggregate subgradient (it is called p in the documantation)*/
/* and the appropriated linearization error */
lin_error_p = 0.0;
item = state->head;
for(i=0; i<lambda->size; i++)
{
status = gsl_blas_daxpy(gsl_vector_get(lambda,i), item->sgr, p);
lin_error_p += gsl_vector_get(lambda,i)*(item->lin_error);
item = item->next;
}
if(debug)
{
printf("the dual problem solution:\n");
for(i=0;i<lambda->size;i++)
printf("%7.6e ",gsl_vector_get(lambda,i));
printf("\n\n");
printf("the aggregate subgradient: \n");
for(i=0;i<p->size;i++)
printf("%.6e ",gsl_vector_get(p,i));
printf("\n");
printf("lin. error for aggr subgradient = %e\n",lin_error_p);
}
/* the norm of the aggr subgradient */
p_norm = gsl_blas_dnrm2(p);
/* search direction dx=-t*p (t is the length of step) */
status = gsl_vector_memcpy(dx,p);
status = gsl_vector_scale(dx,-1.0*state->t);
/* v =-t*norm(p)^2-alpha_p */
state->v = -gsl_pow_2(p_norm)*(state->t)-lin_error_p;
/* the subgradient is the aggegate sungradient */
status = gsl_blas_dcopy(p,subgradient);
/* iteration step */
/* y=x+dx */
status = gsl_blas_dcopy(dx,y);
status = gsl_blas_daxpy(1.0,x,y);
/* function value at y */
f_y = GSL_MULTIMIN_FN_EVAL_F(fsdf, y);
state->f_eval++;
/* for t-update */
if(!state->fixed_step_length)
{
t_old = state->t;
if(fabs(state->v-(f_y-*f)) < state->rg || state->v-(f_y-*f) > state->rg)
t_int_l = state->t_max;
else
t_int_l = 0.5*t_old*(state->v)/(state->v-(f_y-*f));
}
else
{
t_old = state->t;
t_int_l = state->t;
}
if( f_y-*f <= state->m_ss*state->v ) /* Serious-Step */
{
if(debug)
printf("\nSerious-Step\n");
/* the relaxation step */
if(state->relaxation)
{
if(f_y-*f <= state->v*state->m_rel)
{
double f_z;
gsl_vector * z = gsl_vector_alloc(y->size);
/* z = y+dx = x+2*dx */
status = gsl_blas_dcopy(x,z);
status = gsl_blas_daxpy(2.0,dx,z);
f_z = GSL_MULTIMIN_FN_EVAL_F(fsdf, z);
state->f_eval++;
if(0.5*f_z-f_y+0.5*(*f) > state->rg)
state->rel_parameter = GSL_MIN_DBL(-0.5*(-0.5*f_z+2.0*f_y-1.5*(*f))/(0.5*f_z-f_y+0.5*(*f)),1.999);
else if (fabs(0.5*f_z-f_y+0.5*(*f)) > state->rg)
state->rel_parameter = 1.999;
else
/* something is wrong */
state->rel_parameter = 1.0;
/* save the old iteration point */
status = gsl_blas_dcopy(y,z);
/* y = (1-rel_parameter)*x+rel_parameter*y */
gsl_blas_dscal(state->rel_parameter,y);
status = gsl_blas_daxpy(1.0-state->rel_parameter,x,y);
/* f(y) und sgr_f(y) */
tmp_d = GSL_MULTIMIN_FN_EVAL_F(fsdf, y);
state->f_eval++;
if(tmp_d > f_y)
{
/* keep y as the current point */
status = gsl_blas_dcopy(z,y);
state->rel_counter++;
}
else
{
f_y = tmp_d;
/* dx = y-x */
status = gsl_blas_dcopy(y,dx);
status = gsl_blas_daxpy(-1.0,x,dx);
/* if iteration points bevor and after the rel. step are closly,
the rel_step counte will be increased */
/* |1-rel_parameter| <= 0.1*/
if( fabs(1.0-state->rel_parameter) < 0.1)
state->rel_counter++;
}
GSL_MULTIMIN_FN_EVAL_SDF(fsdf, y, sgr_y);
state->sgr_eval++;
if(state->rel_counter > state->rel_counter_max)
state->relaxation = 0;
/* */
status = gsl_blas_daxpy(-1.0,y,z);
status = gsl_blas_ddot(p, z, &tmp_d);
*eps = f_y-*f-(state->v)+tmp_d;
gsl_vector_free(z);
}
else
{
*eps = f_y-(state->v)-*f;
GSL_MULTIMIN_FN_EVAL_SDF(fsdf, y, sgr_y);
state->sgr_eval++;
}
}
else
{
*eps = f_y-(state->v)-*f;
GSL_MULTIMIN_FN_EVAL_SDF(fsdf, y, sgr_y);
state->sgr_eval++;
}
/* calculate linearization errors at new iteration point */
item = state->head;
for(i=0; i<state->bundle_size; i++)
{
status = gsl_blas_ddot(item->sgr, dx, &tmp_d);
item->lin_error += f_y-*f-tmp_d;
item = item->next;
}
/* linearization error at new iteration point */
status = gsl_blas_ddot(p, dx, &tmp_d);
lin_error_p += f_y-*f-tmp_d;
/* update the bundle */
status = update_bundle(state, sgr_y, 0.0, lambda, p, lin_error_p, 1);
/* adapt the step length */
if(!state->fixed_step_length)
{
if(f_y-*f <= state->v*state->m_t && state->step_counter > 0)
state->t = t_int_l;
else if(state->step_counter>3)
state->t=2.0*t_old;
state->t = GSL_MIN_DBL(GSL_MIN_DBL(state->t,10.0*t_old),state->t_max);
/*state->eps_v = GSL_MAX_DBL(state->eps_v,-2.0*state->v);*/
state->step_counter = GSL_MAX_INT(state->step_counter+1,1);
if(fabs(state->t-t_old) > state->rg)
state->step_counter=1;
}
/* x=y, f=f(y) */
status = gsl_blas_dcopy(y,x);
*f = f_y;
}
else /* Null-Step */
{
if(debug)
printf("\nNull-Step\n");
GSL_MULTIMIN_FN_EVAL_SDF(fsdf, y, sgr_y);
state->sgr_eval++;
/* eps for the eps_subdifferential */
*eps = lin_error_p;
/*calculate the liniarization error at y */
status = gsl_blas_ddot(sgr_y,dx,&tmp_d);
tmp_d += *f-f_y;
/* Bundle update */
status = update_bundle(state, sgr_y, tmp_d, lambda, p, lin_error_p, 0);
/* adapt the step length */
if(!state->fixed_step_length)
{
/*state->eps_v = GSL_MIN_DBL(state->eps_v,lin_error_p);*/
if(tmp_d > GSL_MAX_DBL(p_norm,lin_error_p) && state->step_counter < -1)
state->t = t_int_l;
else if(state->step_counter < -3)
state->t = 0.5*t_old;
state->t = GSL_MAX_DBL(GSL_MAX_DBL(0.1*t_old,state->t),state->t_min);
state->step_counter = GSL_MIN_INT(state->step_counter-1,-1);
if(fabs(state->t-t_old) > state->rg)
state->step_counter = -1;
}
}
state->lambda_min = p_norm * state->lm_accuracy;
if(debug)
{
printf("\nthe new bundle:\n");
bundle_out_liste(state);
printf("\n\n");
printf("the curent itarationspoint (1 x %d)\n",x->size);
for(i=0;i<x->size;i++)
printf("%12.6f ",gsl_vector_get(x,i));
printf("\n\n");
printf("functions value at current point: f=%.8f\n",*f);
printf("\nstep length t=%.5e\n",state->t);
printf("\nstep_counter sc=%d\n",state->step_counter);
printf("\naccuracy: v=%.5e\n",state->v);
printf("\nlambda_min=%e\n",state->lambda_min);
printf("\n");
}
gsl_vector_free(lambda);
gsl_vector_free(y);
gsl_vector_free(sgr_y);
gsl_vector_free(p);
return GSL_SUCCESS;
}
static int
update_bundle(bundle_method_state_t *state, const gsl_vector *new_sgr, const double lin_error_sgr, const gsl_vector *lambda,
const gsl_vector *aggr_sgr, const double lin_error_aggr_sgr, const size_t serious_step)
{
bundle_element *current;
bundle_element *item;
bundle_element *item_aggr_sgr;
bundle_element *item_largest_lin_error;
size_t i;
int status;
/* at first: drop all inactive bundle elements, i.e. elements with lambda(j)=0, because they do not contribute to the trial point y */
/* second: drop the bundle elemen (if necessary ) with the largest linearization error */
current = state->head;
item_largest_lin_error = NULL;
item_aggr_sgr = NULL;
for(i=0; i<lambda->size; i++)
{
if(!serious_step && current->state == 2)
{
item_aggr_sgr = current;
current = current->next;
}
else if((fabs(gsl_vector_get(lambda,i))<state->lambda_min && current->state != 0) || (serious_step && current->state == 2))
{
item = current;
current = current->next;
status = remove_element(item, &(state->head), &(state->tail));
(state->bundle_size)--;
}
else
{
if(item_largest_lin_error == NULL || fabs(current->lin_error) > fabs(item_largest_lin_error->lin_error))
item_largest_lin_error = current;
if (current->state ==0 && serious_step)
current->state = 1;
current = current->next;
}
}
if(state->bundle_size >= state->bundle_size_max)
{
status = remove_element(item_largest_lin_error, &(state->head), &(state->tail));
(state->bundle_size)--;
if(state->bundle_size >= state->bundle_size_max-1 && !serious_step && item_aggr_sgr == NULL)
{
if(state->head->state != 0)
item_largest_lin_error = state->head;
else
item_largest_lin_error = state->head->next;
for(item = item_largest_lin_error->next; item != NULL; item=item->next)
{
if(fabs(item->lin_error) > fabs(item_largest_lin_error->lin_error) && item->state != 0)
item_largest_lin_error = item;
}
status = remove_element(item_largest_lin_error, &(state->head), &(state->tail));
(state->bundle_size)--;
}
}
/* add the new element to the bundle */
item = malloc(sizeof(bundle_element));
if (item == 0)
{
GSL_ERROR ("failed to allocate space for a new bundle element", GSL_ENOMEM);
}
item->sgr = gsl_vector_alloc(new_sgr->size);
if (item->sgr == 0)
{
free(item);
GSL_ERROR ("failed to allocate space for a subgradient in the new bundle element", GSL_ENOMEM);
}
status = gsl_blas_dcopy(new_sgr,item->sgr);
item->lin_error = lin_error_sgr;
if(serious_step)
item->state = 0;
else
item->state = 1;
status = insert_element(item, &(state->head), &(state->tail));
(state->bundle_size)++;
if(!serious_step)
{
if(item_aggr_sgr == NULL)
{
item = malloc(sizeof(bundle_element));
if (item == 0)
{
GSL_ERROR ("failed to allocate space for a new bundle element", GSL_ENOMEM);
}
item->sgr = gsl_vector_alloc(new_sgr->size);
if (item->sgr == 0)
{
free(item);
GSL_ERROR ("failed to allocate space for a subgradient in the new bundle element", GSL_ENOMEM);
}
status = gsl_blas_dcopy(aggr_sgr,item->sgr);
item->lin_error = lin_error_aggr_sgr;
item->state = 2;
status = insert_element(item, &(state->head), &(state->tail));
(state->bundle_size)++;
}
else
{
status = gsl_blas_dcopy(aggr_sgr,item_aggr_sgr->sgr);
item_aggr_sgr->lin_error = lin_error_aggr_sgr;
}
}
return GSL_SUCCESS;
}
static int
bundle_method_set(void *vstate, gsl_multimin_function_fsdf * fsdf,
const gsl_vector * x, double *f, gsl_vector * subgradient,
double *eps, size_t bundle_size_max)
{
bundle_method_state_t *state = (bundle_method_state_t *) vstate;
int status, debug=0;
state->rg = 1.0e-20;
/* initialize the first bundle element with a subgradient at the start point,
the lin. error at this point (it is of course zero),
the state of the bundle elemet is seted to 0 (current iteration point) */
GSL_MULTIMIN_FN_EVAL_F_SDF (fsdf, x, f, subgradient);
status = gsl_blas_dcopy(subgradient, state->head->sgr);
state->head->lin_error = 0.0;
state->head->state = 0;
*eps = 0.0;
state->f_eval = 1;
state->sgr_eval = 1;
state->serious_step = 0;
if(debug)
{
size_t i;
printf("\n the start point:\n");
for(i=0; i<x->size; i++)
printf("%12.8f ", gsl_vector_get(x,i));
printf("\n");
printf("the function value at the start point: f=%f\n",*f);
printf("\n the subgradient at the start point:\n");
for(i=0; i<subgradient->size; i++)
printf("%12.8f ", gsl_vector_get(subgradient,i));
printf("\n");
}
/* the initial length step, and lower and upper bound for t */
state->fixed_step_length = 0;
if(state->fixed_step_length)
{
state->t_max = 1;
state->t = 1;
state->t_min = 1;
}
else
{
state->t_max = 1.0e+10;
if(gsl_blas_dnrm2(subgradient) > state->rg)
state->t = 1.0/gsl_blas_dnrm2(subgradient);
else
state->t = state->t_max;
state->t_min = (state->t)*(1.0e-10);
}
state->step_counter = 0;
state->m_ss = 1.0e-1;
state->m_t = 5.0e-1;
state->m_rel = 3.0e-1;
if(bundle_size_max < 3)
{
GSL_ERROR ("the maximal number of bundle elements must be greater or equal to 3", GSL_ENOMEM);
}
state->bundle_size_max = bundle_size_max;
state->bundle_size = 1;
state->lm_accuracy = 1.0e-12;
state->lambda_min = (1.0/(state->t))*state->lm_accuracy;
state->relaxation = 1;
state->fixed_relaxation = 0;
state->rel_parameter = 1.2;
state->rel_counter = 0;
state->rel_counter_max = 3;
return GSL_SUCCESS;
}
//Train all the samples using naive stochastic gradient descent
tnn_error tnn_trainer_class_train_nsgd(tnn_trainer_class *t, gsl_matrix *inputs, size_t *labels){
tnn_error ret;
tnn_state *sin;
tnn_param *p;
gsl_vector *rd;
gsl_vector *pw;
gsl_vector_view in;
gsl_vector_view lb;
double eps;
size_t i,j;
//Routine check
if(t->t != TNN_TRAINER_CLASS_TYPE_NSGD){
return TNN_ERROR_TRAINER_CLASS_MISTYPE;
}
//Check the input
TNN_MACRO_ERRORTEST(tnn_machine_get_sin(&t->m, &sin),ret);
if(inputs->size2 != sin->size){
return TNN_ERROR_STATE_INCOMP;
}
//Set the loss output dx to be 1
gsl_vector_set(&t->l.output->dx, 0, 1.0);
//Get the parameter and allocate rd and pw
TNN_MACRO_ERRORTEST(tnn_machine_get_param(&t->m, &p), ret);
rd = gsl_vector_alloc(p->size);
pw = gsl_vector_alloc(p->size);
if(rd == NULL || pw == NULL){
return TNN_ERROR_GSL;
}
//Into the main loop
for(eps = DBL_MAX, ((tnn_trainer_class_nsgd*)t->c)->titer = 0;
eps > ((tnn_trainer_class_nsgd*)t->c)->epsilon && ((tnn_trainer_class_nsgd*)t->c)->titer < ((tnn_trainer_class_nsgd*)t->c)->niter;
((tnn_trainer_class_nsgd*)t->c)->titer = ((tnn_trainer_class_nsgd*)t->c)->titer + ((tnn_trainer_class_nsgd*)t->c)->eiter){
//Copy the previous pw
TNN_MACRO_GSLTEST(gsl_blas_dcopy(p->x, pw));
for(i = 0; i < ((tnn_trainer_class_nsgd*)t->c)->eiter; i = i + 1){
j = (((tnn_trainer_class_nsgd*)t->c)->titer + i)%inputs->size1;
//Check the label
if(labels[j] >= t->lset->size1){
return TNN_ERROR_STATE_INCOMP;
}
//Get the inputs and label vector
lb = gsl_matrix_row(t->lset, labels[j]);
in = gsl_matrix_row(inputs, j);
//Copy the data into the input/label and do forward and backward propagation
TNN_MACRO_GSLTEST(gsl_blas_dcopy(&in.vector, &sin->x));
TNN_MACRO_GSLTEST(gsl_blas_dcopy(&lb.vector, &t->label->x));
TNN_MACRO_ERRORTEST(tnn_machine_fprop(&t->m), ret);
TNN_MACRO_ERRORTEST(tnn_loss_fprop(&t->l), ret);
TNN_MACRO_ERRORTEST(tnn_loss_bprop(&t->l), ret);
TNN_MACRO_ERRORTEST(tnn_machine_bprop(&t->m), ret);
//Compute the accumulated regularization paramter
TNN_MACRO_ERRORTEST(tnn_reg_d(&t->r, p->x, rd), ret);
TNN_MACRO_GSLTEST(gsl_blas_daxpy(t->lambda, rd, p->dx));
//Compute the parameter update
TNN_MACRO_GSLTEST(gsl_blas_daxpy(-((tnn_trainer_class_nsgd*)t->c)->eta, p->dx, p->x));
}
//Compute the 2 square norm of difference of p as eps
TNN_MACRO_GSLTEST(gsl_blas_daxpy(-1.0, p->x, pw));
eps = gsl_blas_dnrm2(pw);
}
return TNN_ERROR_SUCCESS;
}
