// [[Rcpp::export]] arma::mat BeQTL2(const arma::mat & A, const arma::mat & B, const arma::umat & Bootmat){ int bsi= Bootmat.n_rows; arma::mat C(A.n_cols*B.n_cols,Bootmat.n_rows); arma::mat tC(A.n_rows,B.n_rows); for(int i=0; i<bsi; i++){ tC = cor(A.rows(Bootmat.row(i)),B.rows(Bootmat.row(i))); C.col(i) = vectorise(tC,0); } C.elem(find_nonfinite(C)).zeros(); return reshape(median(C,1),A.n_cols,B.n_cols); }
//Script that takes two matrices, performs bootstrapped correlation, and returns the median // [[Rcpp::export]] arma::mat BeQTL(const arma::mat & A, const arma::mat & B, const arma::umat & Bootmat){ int bsi= Bootmat.n_rows; Rcpp::Rcout<<"Starting Bootstrap!"<<std::endl; arma::mat C(A.n_cols*B.n_cols,Bootmat.n_rows); arma::mat tA(A.n_rows,A.n_cols); arma::mat tB(B.n_rows,B.n_cols); arma::mat tC(A.n_rows,B.n_rows); for(int i=0; i<bsi; i++){ tA = A.rows(Bootmat.row(i)); tB = B.rows(Bootmat.row(i)); tC = cor(tA,tB); C.col(i) = vectorise(tC,0); } C.elem(find_nonfinite(C)).zeros(); return reshape(median(C,1),A.n_cols,B.n_cols); }
// bootstrap class Boot::Boot(const Data& data, Estimator* estimator, const arma::umat& rep) : data(data), n(data.n), p(data.p), rep(rep), r(rep.n_cols) { coef.set_size(p, r); fitted.set_size(n, r); resid.set_size(n, r); scale.set_size(1, r); arma::uvec indices; Data resamp; // for each replicate, construct the dataset then estimate for (int i = 0; i < r; i++) { indices = rep.col(i); resamp.x = data.x.rows(indices); resamp.y = data.y.elem(indices); (*estimator)(resamp, coef.colptr(i), fitted.colptr(i), resid.colptr(i), scale.colptr(i)); } }
List objectivex(const arma::mat& transition, const arma::cube& emission, const arma::vec& init, const arma::ucube& obs, const arma::umat& ANZ, const arma::ucube& BNZ, const arma::uvec& INZ, const arma::uvec& nSymbols, const arma::mat& coef, const arma::mat& X, arma::uvec& numberOfStates, unsigned int threads) { unsigned int q = coef.n_rows; arma::vec grad( arma::accu(ANZ) + arma::accu(BNZ) + arma::accu(INZ) + (numberOfStates.n_elem- 1) * q, arma::fill::zeros); arma::mat weights = exp(X * coef).t(); if (!weights.is_finite()) { grad.fill(-arma::datum::inf); return List::create(Named("objective") = arma::datum::inf, Named("gradient") = wrap(grad)); } weights.each_row() /= sum(weights, 0); arma::mat initk(emission.n_rows, obs.n_slices); for (unsigned int k = 0; k < obs.n_slices; k++) { initk.col(k) = init % reparma(weights.col(k), numberOfStates); } arma::uvec cumsumstate = arma::cumsum(numberOfStates); unsigned int error = 0; double ll = 0; #pragma omp parallel for if(obs.n_slices >= threads) schedule(static) reduction(+:ll) num_threads(threads) \ default(none) shared(q, grad, nSymbols, ANZ, BNZ, INZ, \ numberOfStates, cumsumstate, obs, init, initk, X, weights, transition, emission, error) for (unsigned int k = 0; k < obs.n_slices; k++) { if (error == 0) { arma::mat alpha(emission.n_rows, obs.n_cols); //m,n arma::vec scales(obs.n_cols); //n arma::sp_mat sp_trans(transition); uvForward(sp_trans.t(), emission, initk.col(k), obs.slice(k), alpha, scales); arma::mat beta(emission.n_rows, obs.n_cols); //m,n uvBackward(sp_trans, emission, obs.slice(k), beta, scales); int countgrad = 0; arma::vec grad_k(grad.n_elem, arma::fill::zeros); // transitionMatrix if (arma::accu(ANZ) > 0) { for (unsigned int jj = 0; jj < numberOfStates.n_elem; jj++) { arma::vec gradArow(numberOfStates(jj)); arma::mat gradA(numberOfStates(jj), numberOfStates(jj)); int ind_jj = cumsumstate(jj) - numberOfStates(jj); for (unsigned int i = 0; i < numberOfStates(jj); i++) { arma::uvec ind = arma::find(ANZ.row(ind_jj + i).subvec(ind_jj, cumsumstate(jj) - 1)); if (ind.n_elem > 0) { gradArow.zeros(); gradA.eye(); gradA.each_row() -= transition.row(ind_jj + i).subvec(ind_jj, cumsumstate(jj) - 1); gradA.each_col() %= transition.row(ind_jj + i).subvec(ind_jj, cumsumstate(jj) - 1).t(); for (unsigned int j = 0; j < numberOfStates(jj); j++) { for (unsigned int t = 0; t < (obs.n_cols - 1); t++) { double tmp = alpha(ind_jj + i, t); for (unsigned int r = 0; r < obs.n_rows; r++) { tmp *= emission(ind_jj + j, obs(r, t + 1, k), r); } gradArow(j) += tmp * beta(ind_jj + j, t + 1); } } gradArow = gradA * gradArow; grad_k.subvec(countgrad, countgrad + ind.n_elem - 1) = gradArow.rows(ind); countgrad += ind.n_elem; } } } } if (arma::accu(BNZ) > 0) { // emissionMatrix for (unsigned int r = 0; r < obs.n_rows; r++) { arma::vec gradBrow(nSymbols(r)); arma::mat gradB(nSymbols(r), nSymbols(r)); for (unsigned int i = 0; i < emission.n_rows; i++) { arma::uvec ind = arma::find(BNZ.slice(r).row(i)); if (ind.n_elem > 0) { gradBrow.zeros(); gradB.eye(); gradB.each_row() -= emission.slice(r).row(i).subvec(0, nSymbols(r) - 1); gradB.each_col() %= emission.slice(r).row(i).subvec(0, nSymbols(r) - 1).t(); for (unsigned int j = 0; j < nSymbols(r); j++) { if (obs(r, 0, k) == j) { double tmp = initk(i, k); for (unsigned int r2 = 0; r2 < obs.n_rows; r2++) { if (r2 != r) { tmp *= emission(i, obs(r2, 0, k), r2); } } gradBrow(j) += tmp * beta(i, 0); } for (unsigned int t = 0; t < (obs.n_cols - 1); t++) { if (obs(r, t + 1, k) == j) { double tmp = beta(i, t + 1); for (unsigned int r2 = 0; r2 < obs.n_rows; r2++) { if (r2 != r) { tmp *= emission(i, obs(r2, t + 1, k), r2); } } gradBrow(j) += arma::dot(alpha.col(t), transition.col(i)) * tmp; } } } gradBrow = gradB * gradBrow; grad_k.subvec(countgrad, countgrad + ind.n_elem - 1) = gradBrow.rows(ind); countgrad += ind.n_elem; } } } } if (arma::accu(INZ) > 0) { for (unsigned int i = 0; i < numberOfStates.n_elem; i++) { int ind_i = cumsumstate(i) - numberOfStates(i); arma::uvec ind = arma::find( INZ.subvec(ind_i, cumsumstate(i) - 1)); if (ind.n_elem > 0) { arma::vec gradIrow(numberOfStates(i), arma::fill::zeros); for (unsigned int j = 0; j < numberOfStates(i); j++) { double tmp = weights(i, k); for (unsigned int r = 0; r < obs.n_rows; r++) { tmp *= emission(ind_i + j, obs(r, 0, k), r); } gradIrow(j) += tmp * beta(ind_i + j, 0); } arma::mat gradI(numberOfStates(i), numberOfStates(i), arma::fill::zeros); gradI.eye(); gradI.each_row() -= init.subvec(ind_i, cumsumstate(i) - 1).t(); gradI.each_col() %= init.subvec(ind_i, cumsumstate(i) - 1); gradIrow = gradI * gradIrow; grad_k.subvec(countgrad, countgrad + ind.n_elem - 1) = gradIrow.rows(ind); countgrad += ind.n_elem; } } } for (unsigned int jj = 1; jj < numberOfStates.n_elem; jj++) { unsigned int ind_jj = (cumsumstate(jj) - numberOfStates(jj)); for (unsigned int j = 0; j < emission.n_rows; j++) { double tmp = 1.0; for (unsigned int r = 0; r < obs.n_rows; r++) { tmp *= emission(j, obs(r, 0, k), r); } if ((j >= ind_jj) & (j < cumsumstate(jj))) { grad_k.subvec(countgrad + q * (jj - 1), countgrad + q * jj - 1) += tmp * beta(j, 0) * initk(j, k) * X.row(k).t() * (1.0 - weights(jj, k)); } else { grad_k.subvec(countgrad + q * (jj - 1), countgrad + q * jj - 1) -= tmp * beta(j, 0) * initk(j, k) * X.row(k).t() * weights(jj, k); } } } if (!scales.is_finite() || !beta.is_finite()) { #pragma omp atomic error++; } else { ll -= arma::sum(log(scales)); #pragma omp critical grad += grad_k; } } } if(error > 0){ ll = -arma::datum::inf; grad.fill(-arma::datum::inf); } return List::create(Named("objective") = -ll, Named("gradient") = wrap(-grad)); }
// [[Rcpp::export]] Rcpp::List objective(const arma::mat& transition, const arma::cube& emission, const arma::vec& init, arma::ucube& obs, const arma::umat& ANZ, const arma::ucube& BNZ, const arma::uvec& INZ, const arma::uvec& nSymbols, unsigned int threads) { arma::vec grad(arma::accu(ANZ) + arma::accu(BNZ) + arma::accu(INZ), arma::fill::zeros); unsigned int error = 0; double ll = 0; #pragma omp parallel for if(obs.n_slices >= threads) schedule(static) reduction(+:ll) num_threads(threads) \ default(shared) //shared(grad, nSymbols, ANZ, BNZ, INZ, obs, init, transition, emission, error, arma::fill::zeros) for (unsigned int k = 0; k < obs.n_slices; k++) { if (error == 0) { arma::mat alpha(emission.n_rows, obs.n_cols); //m,n arma::vec scales(obs.n_cols); //n arma::sp_mat sp_trans(transition); uvForward(sp_trans.t(), emission, init, obs.slice(k), alpha, scales); arma::mat beta(emission.n_rows, obs.n_cols); //m,n uvBackward(sp_trans, emission, obs.slice(k), beta, scales); int countgrad = 0; arma::vec grad_k(grad.n_elem, arma::fill::zeros); // transitionMatrix arma::vec gradArow(emission.n_rows); arma::mat gradA(emission.n_rows, emission.n_rows); for (unsigned int i = 0; i < emission.n_rows; i++) { arma::uvec ind = arma::find(ANZ.row(i)); if (ind.n_elem > 0) { gradArow.zeros(); gradA.eye(); gradA.each_row() -= transition.row(i); gradA.each_col() %= transition.row(i).t(); for (unsigned int t = 0; t < (obs.n_cols - 1); t++) { for (unsigned int j = 0; j < emission.n_rows; j++) { double tmp = 1.0; for (unsigned int r = 0; r < obs.n_rows; r++) { tmp *= emission(j, obs(r, t + 1, k), r); } gradArow(j) += alpha(i, t) * tmp * beta(j, t + 1); } } gradArow = gradA * gradArow; grad_k.subvec(countgrad, countgrad + ind.n_elem - 1) = gradArow.rows(ind); countgrad += ind.n_elem; } } // emissionMatrix for (unsigned int r = 0; r < obs.n_rows; r++) { arma::vec gradBrow(nSymbols(r)); arma::mat gradB(nSymbols(r), nSymbols(r)); for (unsigned int i = 0; i < emission.n_rows; i++) { arma::uvec ind = arma::find(BNZ.slice(r).row(i)); if (ind.n_elem > 0) { gradBrow.zeros(); gradB.eye(); gradB.each_row() -= emission.slice(r).row(i).subvec(0, nSymbols(r) - 1); gradB.each_col() %= emission.slice(r).row(i).subvec(0, nSymbols(r) - 1).t(); for (unsigned int j = 0; j < nSymbols(r); j++) { if (obs(r, 0, k) == j) { double tmp = 1.0; for (unsigned int r2 = 0; r2 < obs.n_rows; r2++) { if (r2 != r) { tmp *= emission(i, obs(r2, 0, k), r2); } } gradBrow(j) += init(i) * tmp * beta(i, 0); } for (unsigned int t = 0; t < (obs.n_cols - 1); t++) { if (obs(r, t + 1, k) == j) { double tmp = 1.0; for (unsigned int r2 = 0; r2 < obs.n_rows; r2++) { if (r2 != r) { tmp *= emission(i, obs(r2, t + 1, k), r2); } } gradBrow(j) += arma::dot(alpha.col(t), transition.col(i)) * tmp * beta(i, t + 1); } } } gradBrow = gradB * gradBrow; grad_k.subvec(countgrad, countgrad + ind.n_elem - 1) = gradBrow.rows(ind); countgrad += ind.n_elem; } } } // InitProbs arma::uvec ind = arma::find(INZ); if (ind.n_elem > 0) { arma::vec gradIrow(emission.n_rows); arma::mat gradI(emission.n_rows, emission.n_rows); gradIrow.zeros(); gradI.zeros(); gradI.eye(); gradI.each_row() -= init.t(); gradI.each_col() %= init; for (unsigned int j = 0; j < emission.n_rows; j++) { double tmp = 1.0; for (unsigned int r = 0; r < obs.n_rows; r++) { tmp *= emission(j, obs(r, 0, k), r); } gradIrow(j) += tmp * beta(j, 0); } gradIrow = gradI * gradIrow; grad_k.subvec(countgrad, countgrad + ind.n_elem - 1) = gradIrow.rows(ind); countgrad += ind.n_elem; } if (!scales.is_finite() || !beta.is_finite()) { #pragma omp atomic error++; } else { ll -= arma::sum(log(scales)); #pragma omp critical grad += grad_k; // gradmat.col(k) = grad_k; } // for (unsigned int ii = 0; ii < grad_k.n_elem; ii++) { // #pragma omp atomic // grad(ii) += grad_k(ii); // } } } if(error > 0){ ll = -arma::datum::inf; grad.fill(-arma::datum::inf); } // } else { // grad = sum(gradmat, 1); // } return Rcpp::List::create(Rcpp::Named("objective") = -ll, Rcpp::Named("gradient") = Rcpp::wrap(-grad)); }
double ung_ssm::bsf_filter(const unsigned int nsim, arma::cube& alpha, arma::mat& weights, arma::umat& indices) { arma::uvec nonzero = arma::find(P1.diag() > 0); arma::mat L_P1(m, m, arma::fill::zeros); if (nonzero.n_elem > 0) { L_P1.submat(nonzero, nonzero) = arma::chol(P1.submat(nonzero, nonzero), "lower"); } std::normal_distribution<> normal(0.0, 1.0); for (unsigned int i = 0; i < nsim; i++) { arma::vec um(m); for(unsigned int j = 0; j < m; j++) { um(j) = normal(engine); } alpha.slice(i).col(0) = a1 + L_P1 * um; } std::uniform_real_distribution<> unif(0.0, 1.0); arma::vec normalized_weights(nsim); double loglik = 0.0; if(arma::is_finite(y(0))) { weights.col(0) = log_obs_density(0, alpha); double max_weight = weights.col(0).max(); weights.col(0) = arma::exp(weights.col(0) - max_weight); double sum_weights = arma::accu(weights.col(0)); if(sum_weights > 0.0){ normalized_weights = weights.col(0) / sum_weights; } else { return -std::numeric_limits<double>::infinity(); } loglik = max_weight + std::log(sum_weights / nsim); } else { weights.col(0).ones(); normalized_weights.fill(1.0 / nsim); } for (unsigned int t = 0; t < n; t++) { arma::vec r(nsim); for (unsigned int i = 0; i < nsim; i++) { r(i) = unif(engine); } indices.col(t) = stratified_sample(normalized_weights, r, nsim); arma::mat alphatmp(m, nsim); for (unsigned int i = 0; i < nsim; i++) { alphatmp.col(i) = alpha.slice(indices(i, t)).col(t); } for (unsigned int i = 0; i < nsim; i++) { arma::vec uk(k); for(unsigned int j = 0; j < k; j++) { uk(j) = normal(engine); } alpha.slice(i).col(t + 1) = C.col(t * Ctv) + T.slice(t * Ttv) * alphatmp.col(i) + R.slice(t * Rtv) * uk; } if ((t < (n - 1)) && arma::is_finite(y(t + 1))) { weights.col(t + 1) = log_obs_density(t + 1, alpha); double max_weight = weights.col(t + 1).max(); weights.col(t + 1) = arma::exp(weights.col(t + 1) - max_weight); double sum_weights = arma::accu(weights.col(t + 1)); if(sum_weights > 0.0){ normalized_weights = weights.col(t + 1) / sum_weights; } else { return -std::numeric_limits<double>::infinity(); } loglik += max_weight + std::log(sum_weights / nsim); } else { weights.col(t + 1).ones(); normalized_weights.fill(1.0/nsim); } } // constant part of the log-likelihood switch(distribution) { case 0 : loglik += arma::uvec(arma::find_finite(y)).n_elem * norm_log_const(phi); break; case 1 : { arma::uvec finite_y(find_finite(y)); loglik += poisson_log_const(y(finite_y), u(finite_y)); } break; case 2 : { arma::uvec finite_y(find_finite(y)); loglik += binomial_log_const(y(finite_y), u(finite_y)); } break; case 3 : { arma::uvec finite_y(find_finite(y)); loglik += negbin_log_const(y(finite_y), u(finite_y), phi); } break; } return loglik; }
double ung_ssm::psi_filter(const ugg_ssm& approx_model, const double approx_loglik, const arma::vec& scales, const unsigned int nsim, arma::cube& alpha, arma::mat& weights, arma::umat& indices) { arma::mat alphahat(m, n + 1); arma::cube Vt(m, m, n + 1); arma::cube Ct(m, m, n + 1); approx_model.smoother_ccov(alphahat, Vt, Ct); conditional_cov(Vt, Ct); std::normal_distribution<> normal(0.0, 1.0); for (unsigned int i = 0; i < nsim; i++) { arma::vec um(m); for(unsigned int j = 0; j < m; j++) { um(j) = normal(engine); } alpha.slice(i).col(0) = alphahat.col(0) + Vt.slice(0) * um; } std::uniform_real_distribution<> unif(0.0, 1.0); arma::vec normalized_weights(nsim); double loglik = 0.0; if(arma::is_finite(y(0))) { weights.col(0) = arma::exp(log_weights(approx_model, 0, alpha) - scales(0)); double sum_weights = arma::accu(weights.col(0)); if(sum_weights > 0.0){ normalized_weights = weights.col(0) / sum_weights; } else { return -std::numeric_limits<double>::infinity(); } loglik = approx_loglik + std::log(sum_weights / nsim); } else { weights.col(0).ones(); normalized_weights.fill(1.0 / nsim); loglik = approx_loglik; } for (unsigned int t = 0; t < n; t++) { arma::vec r(nsim); for (unsigned int i = 0; i < nsim; i++) { r(i) = unif(engine); } indices.col(t) = stratified_sample(normalized_weights, r, nsim); arma::mat alphatmp(m, nsim); // for (unsigned int i = 0; i < nsim; i++) { // alphatmp.col(i) = alpha.slice(i).col(t); // } for (unsigned int i = 0; i < nsim; i++) { alphatmp.col(i) = alpha.slice(indices(i, t)).col(t); //alpha.slice(i).col(t) = alphatmp.col(indices(i, t)); } for (unsigned int i = 0; i < nsim; i++) { arma::vec um(m); for(unsigned int j = 0; j < m; j++) { um(j) = normal(engine); } alpha.slice(i).col(t + 1) = alphahat.col(t + 1) + Ct.slice(t + 1) * (alphatmp.col(i) - alphahat.col(t)) + Vt.slice(t + 1) * um; } if ((t < (n - 1)) && arma::is_finite(y(t + 1))) { weights.col(t + 1) = arma::exp(log_weights(approx_model, t + 1, alpha) - scales(t + 1)); double sum_weights = arma::accu(weights.col(t + 1)); if(sum_weights > 0.0){ normalized_weights = weights.col(t + 1) / sum_weights; } else { return -std::numeric_limits<double>::infinity(); } loglik += std::log(sum_weights / nsim); } else { weights.col(t + 1).ones(); normalized_weights.fill(1.0 / nsim); } } return loglik; }