Exemplo n.º 1
0
int run_em(EM_ENG_PTR em_ptr)
{
    int     r, iterate, old_valid, converged, saved = 0;
    double  likelihood, log_prior;
    double  lambda, old_lambda = 0.0;

    config_em(em_ptr);

    for (r = 0; r < num_restart; r++) {
        SHOW_PROGRESS_HEAD("#em-iters", r);

        initialize_params();
        itemp = daem ? itemp_init : 1.0;
        iterate = 0;

        /* [21 Aug 2007, by yuizumi]
         * while-loop for inversed temperature (DAEM).  Note that this
         * loop is evaluated only once for EM without annealing, since
         * itemp initially set to 1.0 by the code above.
         */
        while (1) {
            if (daem) {
                SHOW_PROGRESS_TEMP(itemp);
            }
            old_valid = 0;

            while (1) {
                if (CTRLC_PRESSED) {
                    SHOW_PROGRESS_INTR();
                    RET_ERR(err_ctrl_c_pressed);
                }

                RET_ON_ERR(em_ptr->compute_inside());
                RET_ON_ERR(em_ptr->examine_inside());

                likelihood = em_ptr->compute_likelihood();
                log_prior  = em_ptr->smooth ? em_ptr->compute_log_prior() : 0.0;
                lambda = likelihood + log_prior;

                if (verb_em) {
                    if (em_ptr->smooth) {
                        prism_printf("Iteration #%d:\tlog_likelihood=%.9f\tlog_prior=%.9f\tlog_post=%.9f\n", iterate, likelihood, log_prior, lambda);
                    }
                    else {
                        prism_printf("Iteration #%d:\tlog_likelihood=%.9f\n", iterate, likelihood);
                    }
                }

                if (debug_level) {
                    prism_printf("After I-step[%d]:\n", iterate);
                    prism_printf("likelihood = %.9f\n", likelihood);
                    print_egraph(debug_level, PRINT_EM);
                }

                if (!isfinite(lambda)) {
                    emit_internal_error("invalid log likelihood or log post: %s (at iteration #%d)",
                                        isnan(lambda) ? "NaN" : "infinity", iterate);
                    RET_ERR(ierr_invalid_likelihood);
                }
                if (old_valid && old_lambda - lambda > prism_epsilon) {
                    emit_error("log likelihood or log post decreased [old: %.9f, new: %.9f] (at iteration #%d)",
                               old_lambda, lambda, iterate);
                    RET_ERR(err_invalid_likelihood);
                }
                if (itemp == 1.0 && likelihood > 0.0) {
                    emit_error("log likelihood greater than zero [value: %.9f] (at iteration #%d)",
                               likelihood, iterate);
                    RET_ERR(err_invalid_likelihood);
                }

                converged = (old_valid && lambda - old_lambda <= prism_epsilon);
                if (converged || REACHED_MAX_ITERATE(iterate)) {
                    break;
                }

                old_lambda = lambda;
                old_valid  = 1;

                RET_ON_ERR(em_ptr->compute_expectation());

                if (debug_level) {
                    prism_printf("After O-step[%d]:\n", iterate);
                    print_egraph(debug_level, PRINT_EM);
                }

                SHOW_PROGRESS(iterate);
                RET_ON_ERR(em_ptr->update_params());
                iterate++;
            }

            /* [21 Aug 2007, by yuizumi]
             * Note that 1.0 can be represented exactly in IEEE 754.
             */
            if (itemp == 1.0) {
                break;
            }
            itemp *= itemp_rate;
            if (itemp >= 1.0) {
                itemp = 1.0;
            }
        }

        SHOW_PROGRESS_TAIL(converged, iterate, lambda);

        if (r == 0 || lambda > em_ptr->lambda) {
            em_ptr->lambda     = lambda;
            em_ptr->likelihood = likelihood;
            em_ptr->iterate    = iterate;

            saved = (r < num_restart - 1);
            if (saved) {
                save_params();
            }
        }
    }

    if (saved) {
        restore_params();
    }

    em_ptr->bic = compute_bic(em_ptr->likelihood);
    em_ptr->cs  = em_ptr->smooth ? compute_cs(em_ptr->likelihood) : 0.0;

    return BP_TRUE;
}
Exemplo n.º 2
0
/* main loop */
static int run_grd(CRF_ENG_PTR crf_ptr) {
	int r,iterate,old_valid,converged,conv_time,saved = 0;
	double likelihood,old_likelihood = 0.0;
	double crf_max_iterate = 0.0;
	double tmp_epsilon,alpha0,gf_sd,old_gf_sd = 0.0;

	config_crf(crf_ptr);

	initialize_weights();

	if (crf_learn_mode == 1) {
		initialize_LBFGS();
		printf("L-BFGS mode\n");
	}

	if (crf_learning_rate==1) {
		printf("learning rate:annealing\n");
	} else if (crf_learning_rate==2) {
		printf("learning rate:backtrack\n");
	} else if (crf_learning_rate==3) {
		printf("learning rate:golden section\n");
	}

	if (max_iterate == -1) {
		crf_max_iterate = DEFAULT_MAX_ITERATE;
	} else if (max_iterate >= +1) {
		crf_max_iterate = max_iterate;
	}

	for (r = 0; r < num_restart; r++) {
		SHOW_PROGRESS_HEAD("#crf-iters", r);

		initialize_crf_count();
		initialize_lambdas();
		initialize_visited_flags();

		old_valid = 0;
		iterate = 0;
		tmp_epsilon = crf_epsilon;

		LBFGS_index = 0;
		conv_time = 0;

		while (1) {
			if (CTRLC_PRESSED) {
				SHOW_PROGRESS_INTR();
				RET_ERR(err_ctrl_c_pressed);
			}

			RET_ON_ERR(crf_ptr->compute_feature());

			crf_ptr->compute_crf_probs();

			likelihood = crf_ptr->compute_likelihood();

			if (verb_em) {
				prism_printf("Iteration #%d:\tlog_likelihood=%.9f\n", iterate, likelihood);
			}

			if (debug_level) {
				prism_printf("After I-step[%d]:\n", iterate);
				prism_printf("likelihood = %.9f\n", likelihood);
				print_egraph(debug_level, PRINT_EM);
			}

			if (!isfinite(likelihood)) {
				emit_internal_error("invalid log likelihood: %s (at iteration #%d)",
				                    isnan(likelihood) ? "NaN" : "infinity", iterate);
				RET_ERR(ierr_invalid_likelihood);
			}
			/*        if (old_valid && old_likelihood - likelihood > prism_epsilon) {
					  emit_error("log likelihood decreased [old: %.9f, new: %.9f] (at iteration #%d)",
					  old_likelihood, likelihood, iterate);
					  RET_ERR(err_invalid_likelihood);
					  }*/
			if (likelihood > 0.0) {
				emit_error("log likelihood greater than zero [value: %.9f] (at iteration #%d)",
				           likelihood, iterate);
				RET_ERR(err_invalid_likelihood);
			}

			if (crf_learn_mode == 1 && iterate > 0) restore_old_gradient();

			RET_ON_ERR(crf_ptr->compute_gradient());

			if (crf_learn_mode == 1 && iterate > 0) {
				compute_LBFGS_y_rho();
				compute_hessian(iterate);
			} else if (crf_learn_mode == 1 && iterate == 0) {
				initialize_LBFGS_q();
			}

			converged = (old_valid && fabs(likelihood - old_likelihood) <= prism_epsilon);

			if (converged || REACHED_MAX_ITERATE(iterate)) {
				break;
			}

			old_likelihood = likelihood;
			old_valid = 1;

			if (debug_level) {
				prism_printf("After O-step[%d]:\n", iterate);
				print_egraph(debug_level, PRINT_EM);
			}

			SHOW_PROGRESS(iterate);

			if (crf_learning_rate == 1) { // annealing
				tmp_epsilon = (annealing_weight / (annealing_weight + iterate)) * crf_epsilon;
			} else if (crf_learning_rate == 2) { // line-search(backtrack)
				if (crf_learn_mode == 1) {
					gf_sd = compute_gf_sd_LBFGS();
				} else {
					gf_sd = compute_gf_sd();
				}
				if (iterate==0) {
					alpha0 = 1;
				} else {
					alpha0 = tmp_epsilon * old_gf_sd / gf_sd;
				}
				if (crf_learn_mode == 1) {
					tmp_epsilon = line_search_LBFGS(crf_ptr,alpha0,crf_ls_rho,crf_ls_c1,likelihood,gf_sd);
				} else {
					tmp_epsilon = line_search(crf_ptr,alpha0,crf_ls_rho,crf_ls_c1,likelihood,gf_sd);
				}

				if (tmp_epsilon < EPS) {
					emit_error("invalid alpha in line search(=0.0) (at iteration #%d)",iterate);
					RET_ERR(err_line_search);
				}
				old_gf_sd = gf_sd;
			} else if (crf_learning_rate == 3) { // line-search(golden section)
				if (crf_learn_mode == 1) {
					tmp_epsilon = golden_section_LBFGS(crf_ptr,0,crf_golden_b);
				} else {
					tmp_epsilon = golden_section(crf_ptr,0,crf_golden_b);
				}
			}
			crf_ptr->update_lambdas(tmp_epsilon);

			iterate++;
		}

		SHOW_PROGRESS_TAIL(converged, iterate, likelihood);

		if (r == 0 || likelihood > crf_ptr->likelihood) {
			crf_ptr->likelihood = likelihood;
			crf_ptr->iterate    = iterate;

			saved = (r < num_restart - 1);
			if (saved) {
				save_params();
			}
		}
	}

	if (crf_learn_mode == 1) clean_LBFGS();
	INIT_VISITED_FLAGS;
	return BP_TRUE;
}
Exemplo n.º 3
0
int mpm_run_em(EM_ENG_PTR emptr) {
	int     r, iterate, old_valid, converged, saved=0;
	double  likelihood, log_prior;
	double  lambda, old_lambda=0.0;

	config_em(emptr);

	for (r = 0; r < num_restart; r++) {
		SHOW_PROGRESS_HEAD("#em-iters", r);

		initialize_params();
		mpm_bcast_inside();
		clear_sw_msg_send();

		itemp = daem ? itemp_init : 1.0;
		iterate = 0;

		while (1) {
			if (daem) {
				SHOW_PROGRESS_TEMP(itemp);
			}
			old_valid = 0;

			while (1) {
				if (CTRLC_PRESSED) {
					SHOW_PROGRESS_INTR();
					RET_ERR(err_ctrl_c_pressed);
				}

				if (failure_observed) {
					inside_failure = mp_sum_value(0.0);
				}

				log_prior  = emptr->smooth ? emptr->compute_log_prior() : 0.0;
				lambda = mp_sum_value(log_prior);
				likelihood = lambda - log_prior;

				mp_debug("local lambda = %.9f, lambda = %.9f", log_prior, lambda);

				if (verb_em) {
					if (emptr->smooth) {
						prism_printf("Iteration #%d:\tlog_likelihood=%.9f\tlog_prior=%.9f\tlog_post=%.9f\n", iterate, likelihood, log_prior, lambda);
					} else {
						prism_printf("Iteration #%d:\tlog_likelihood=%.9f\n", iterate, likelihood);
					}
				}

				if (!isfinite(lambda)) {
					emit_internal_error("invalid log likelihood or log post: %s (at iterateion #%d)",
					                    isnan(lambda) ? "NaN" : "infinity", iterate);
					RET_ERR(ierr_invalid_likelihood);
				}
				if (old_valid && old_lambda - lambda > prism_epsilon) {
					emit_error("log likelihood or log post decreased [old: %.9f, new: %.9f] (at iteration #%d)",
					           old_lambda, lambda, iterate);
					RET_ERR(err_invalid_likelihood);
				}
				if (itemp == 1.0 && likelihood > 0.0) {
					emit_error("log likelihood greater than zero [value: %.9f] (at iteration #%d)",
					           likelihood, iterate);
					RET_ERR(err_invalid_likelihood);
				}

				converged = (old_valid && lambda - old_lambda <= prism_epsilon);
				if (converged || REACHED_MAX_ITERATE(iterate)) {
					break;
				}

				old_lambda = lambda;
				old_valid  = 1;

				mpm_share_expectation();

				SHOW_PROGRESS(iterate);
				RET_ON_ERR(emptr->update_params());
				iterate++;
			}

			if (itemp == 1.0) {
				break;
			}
			itemp *= itemp_rate;
			if (itemp >= 1.0) {
				itemp = 1.0;
			}
		}

		SHOW_PROGRESS_TAIL(converged, iterate, lambda);

		if (r == 0 || lambda > emptr->lambda) {
			emptr->lambda     = lambda;
			emptr->likelihood = likelihood;
			emptr->iterate    = iterate;

			saved = (r < num_restart - 1);
			if (saved) {
				save_params();
			}
		}
	}

	if (saved) {
		restore_params();
	}

	emptr->bic = compute_bic(emptr->likelihood);
	emptr->cs  = emptr->smooth ? compute_cs(emptr->likelihood) : 0.0;

	return BP_TRUE;
}