int main(int argc, char *argv[]) { paracel::main_env comm_main_env(argc, argv); paracel::Comm comm(MPI_COMM_WORLD); google::SetUsageMessage("[options]\n\t--server_info\n\t--cfg_file\n"); google::ParseCommandLineFlags(&argc, &argv, true); paracel::json_parser pt(FLAGS_cfg_file); std::string rating_input = pt.check_parse<std::string>("rating_input"); std::string factor_input = pt.check_parse<std::string>("factor_input"); std::string validate_input = pt.check_parse<std::string>("validate_input"); std::string output = pt.parse<std::string>("output"); std::string pattern = pt.parse<std::string>("pattern"); double lambda = pt.parse<double>("lambda"); paracel::alg::alternating_least_square_standard H_solver(comm, FLAGS_server_info, rating_input, factor_input, validate_input, output, pattern, lambda); H_solver.solve(); H_solver.dump_result(); double train_err = H_solver.cal_rmse(); if(comm.get_rank() == 0) std::cout << "train error: " << train_err << std::endl; double test_err = H_solver.validate(); if(comm.get_rank() == 0) std::cout << "test error:" << test_err << std::endl; return 0; }
int main(int argc, char *argv[]) { paracel::main_env comm_main_env(argc, argv); paracel::Comm comm(MPI_COMM_WORLD); google::SetUsageMessage("[options]\n\t--server_info\n\t--cfg_file\n"); google::ParseCommandLineFlags(&argc, &argv, true); paracel::json_parser pt(FLAGS_cfg_file); std::string user_fac_input = pt.parse<std::string>("user_fac_input"); std::string item_fac_input = pt.parse<std::string>("item_fac_input"); std::string output = pt.parse<std::string>("output"); std::string handle_fn = pt.parse<std::string>("handle_file"); int level = pt.parse<int>("level"); int tree_start_indx = pt.parse<int>("tree_start_index"); int tree_end_indx = pt.parse<int>("tree_end_index"); paracel::alg::decision_tree_builder_factor solver(comm, FLAGS_server_info, user_fac_input, item_fac_input, paracel::add_folder_suffix_with_date(output), handle_fn, level, tree_start_indx, tree_end_indx); solver.solve(); solver.dump(); return 0; }
int main(int argc, char *argv[]) { paracel::main_env comm_main_env(argc, argv); paracel::Comm comm(MPI_COMM_WORLD); google::SetUsageMessage("[options]\n\t--server_info\n\t--cfg_file\n"); google::ParseCommandLineFlags(&argc, &argv, true); paracel::json_parser pt(FLAGS_cfg_file); std::string input = pt.check_parse<std::string>("input"); std::string output = pt.parse<std::string>("output"); int topk = pt.parse<int>("topk"); std::string handle_fn = pt.check_parse<std::string>("handle_file"); std::string update_fcn = pt.parse<std::string>("update_function"); std::string filter_fcn = pt.parse<std::string>("filter_function"); paracel::alg::word_count wc_solver(comm, FLAGS_server_info, input, output, topk, handle_fn, update_fcn, filter_fcn); wc_solver.solve(); wc_solver.print(); return 0; }
int main(int argc, char *argv[]) { paracel::main_env comm_main_env(argc, argv); paracel::Comm comm(MPI_COMM_WORLD); google::SetUsageMessage("[options]\n\t--server_info\n\t--cfg_file\n"); google::ParseCommandLineFlags(&argc, &argv, true); paracel::json_parser pt(FLAGS_cfg_file); std::string input = pt.check_parse<std::string>("input"); std::string pred_input = pt.check_parse<std::string>("predict_input"); std::string output = pt.parse<std::string>("output"); std::string update_fn = pt.check_parse<std::string>("update_file"); std::vector<std::string> update_funcs = pt.parse_v<std::string>("update_functions"); std::string filter_fn = pt.check_parse<std::string>("filter_file"); std::vector<std::string> filter_funcs = pt.parse_v<std::string>("filter_functions"); int k = pt.parse<int>("k"); int rounds = pt.parse<int>("rounds"); double alpha = pt.parse<double>("alpha"); double beta = pt.parse<double>("beta"); bool debug = pt.parse<bool>("debug"); bool ssp_switch = pt.parse<bool>("ssp_switch"); int limit_s = pt.parse<int>("limit_s"); paracel::alg::matrix_factorization mf_solver(comm, FLAGS_server_info, input, pred_input, output, update_fn, update_funcs, filter_fn, filter_funcs, k, rounds, alpha, beta, debug, ssp_switch, limit_s); mf_solver.solve(); std::cout << "rmse: " << mf_solver.cal_rmse() << std::endl; mf_solver.predict(); mf_solver.dump_result(); return 0; }
int main(int argc, char *argv[]) { paracel::main_env comm_main_env(argc, argv); paracel::Comm comm(MPI_COMM_WORLD); google::SetUsageMessage("[options]\n\t--server_info\n\t--cfg_file\n"); google::ParseCommandLineFlags(&argc, &argv, true); paracel::json_parser pt(FLAGS_cfg_file); std::string training_input, test_input, predict_input, output, update_file, update_func, method; try { training_input = pt.check_parse<std::string>("training_input"); test_input = pt.check_parse<std::string>("test_input"); predict_input = pt.check_parse<std::string>("predict_input"); output = pt.parse<std::string>("output"); update_file = pt.check_parse<std::string>("update_file"); update_func = pt.parse<std::string>("update_func"); method = pt.parse<std::string>("method"); } catch (const std::invalid_argument & e) { std::cerr << e.what(); return 1; } int rounds = pt.parse<int>("rounds"); double alpha = pt.parse<double>("alpha"); double beta = pt.parse<double>("beta"); bool debug = pt.parse<bool>("debug"); paracel::alg::logistic_regression lr_solver(comm, FLAGS_server_info, training_input, output, update_file, update_func, method, rounds, alpha, beta, debug); lr_solver.solve(); std::cout << "final loss: " << lr_solver.calc_loss() << std::endl; lr_solver.test(test_input); lr_solver.predict(predict_input); lr_solver.dump_result(); return 0; }
int main(int argc, char *argv[]) { paracel::main_env comm_main_env(argc, argv); paracel::Comm comm(MPI_COMM_WORLD); google::SetUsageMessage("[options]\n\t--server_info\n\t--cfg_file\n"); google::ParseCommandLineFlags(&argc, &argv, true); paracel::json_parser pt(FLAGS_cfg_file); std::string rating_input = pt.check_parse<std::string>("rating_input"); std::string factor_input = pt.check_parse<std::string>("factor_input"); std::string validate_input = pt.check_parse<std::string>("validate_input"); std::string output = pt.parse<std::string>("output"); std::string pattern = pt.parse<std::string>("pattern"); std::vector<double> lambdas = pt.parse_v<double>("lambdas"); // model selection { std::vector<double> validate_errs, train_errs; paracel::alg::alternating_least_square_validate H_solver(comm, FLAGS_server_info, rating_input, factor_input, validate_input, output, pattern); H_solver.init(); for(auto & lambda : lambdas) { H_solver.learning(lambda); train_errs.push_back(H_solver.cal_rmse()); validate_errs.push_back(H_solver.validate()); } if(comm.get_rank() == 0) { for(size_t i = 0; i < train_errs.size(); ++i) { std::cout << "###################################" << std::endl; std::cout << "lambda: " << lambdas[i] << std::endl; std::cout << "train error: " << train_errs[i] << std::endl; std::cout << "validate error: " << validate_errs[i] << std::endl; std::cout << "###################################" << std::endl; } } } return 0; }
int main(int argc, char* argv[]) { paracel::main_env comm_main_env(argc, argv); paracel::Comm comm(MPI_COMM_WORLD); google::InitGoogleLogging(argv[0]); int num_cookies = 7; LOG(INFO) << "Found " << num_cookies << " cookies"; LOG_IF(INFO, comm.get_rank() == 0) << "Got lots of cookies"; DLOG(INFO) << "Found cookies"; DLOG_IF(INFO, comm.get_rank() == 0) << "Got lots of cookies"; VLOG_IF(1, comm.get_rank() == 0) << "I'm printed when size is more than 1024 and when you run the " "program with --v=1 or more"; return 0; }
int main(int argc, char *argv[]) { paracel::main_env comm_main_env(argc, argv); paracel::Comm comm(MPI_COMM_WORLD); google::SetUsageMessage("[options]\n\t--cfg_file\n"); google::ParseCommandLineFlags(&argc, &argv, true); paracel::json_parser pt(FLAGS_cfg_file); std::string input = pt.check_parse<std::string>("input"); std::string output = pt.parse<std::string>("output"); double lambda = pt.parse<double>("lambda"); int rounds = pt.parse<int>("rounds"); paracel::tool::lasso solver(comm, input, output, lambda, rounds); solver.solve(); solver.dump_result(); solver.check(); return 0; }
int main(int argc, char *argv[]) { paracel::main_env comm_main_env(argc, argv); paracel::Comm comm(MPI_COMM_WORLD); google::SetUsageMessage("[options]\n\t--server_info\n\t--cfg_file\n"); google::ParseCommandLineFlags(&argc, &argv, true); paracel::json_parser pt(FLAGS_cfg_file); std::string rating_input = pt.check_parse<std::string>("rating_input"); std::string sim_input = pt.check_parse<std::string>("sim_input"); std::string output = pt.parse<std::string>("output"); paracel::alg::adjust_ktop_s solver(comm, FLAGS_server_info, rating_input, sim_input, output); solver.solve(); solver.dump_result(); return 0; }
int main(int argc, char *argv[]) { paracel::main_env comm_main_env(argc, argv); paracel::Comm comm(MPI_COMM_WORLD); google::SetUsageMessage("[options]\n\t--server_info\n\t--cfg_file\n"); google::ParseCommandLineFlags(&argc, &argv, true); paracel::json_parser pt(FLAGS_cfg_file); std::string input, output, handle_fn; double alpha, beta; int k_topics, iters, top_words; bool debug; try { input = pt.check_parse<std::string>("input"); output = pt.parse<std::string>("output"); alpha = pt.parse<double>("alpha"); beta = pt.parse<double>("beta"); k_topics = pt.parse<int>("k_topics"); iters = pt.parse<int>("iters"); top_words = pt.parse<int>("top_words"); handle_fn = pt.check_parse<std::string>("handle_file"); debug = pt.parse<bool>("debug"); } catch (const std::invalid_argument & e) { std::cerr << e.what(); return 1; } paracel::alg::LDAmodel solver(comm, FLAGS_server_info, input, output, alpha, beta, k_topics, iters, top_words, handle_fn, debug); solver.train(); solver.save(); return 0; }
int main(int argc, char *argv[]) { paracel::main_env comm_main_env(argc, argv); paracel::Comm comm(MPI_COMM_WORLD); int rk = comm.get_rank(); int sz = comm.get_size(); paracel::scheduler scheduler_obj(comm); paracel::lt_type slotslst; if(rk == 0) { slotslst.push_back(std::make_tuple("64", "6", 3.)); slotslst.push_back(std::make_tuple("64", "42", 4.)); } if(rk == 1) { slotslst.push_back(std::make_tuple("27", "28", 1.)); slotslst.push_back(std::make_tuple("27", "42", 2.)); slotslst.push_back(std::make_tuple("37", "42", 3.)); slotslst.push_back(std::make_tuple("29", "28", 1.)); slotslst.push_back(std::make_tuple("29", "6", 3.)); } paracel::dict_type<size_t, paracel::str_type> rm, cm; paracel::list_type<std::tuple<size_t, size_t, double> > stf; paracel::dict_type<size_t, int> dm; paracel::dict_type<size_t, int> col_dm; scheduler_obj.index_mapping(slotslst, stf, rm, cm, dm, col_dm); if(rk == 0) { for(auto & item : stf) std::cout << std::get<0>(item) << " | " << std::get<1>(item) << " | "<< std::get<2>(item) << std::endl; for(auto & item : dm) std::cout << item.first << " - " << item.second << std::endl; for(auto & item : col_dm) std::cout << item.first << " * " << item.second << std::endl; } return 0; }
int main(int argc, char *argv[]) { paracel::main_env comm_main_env(argc, argv); paracel::Comm comm(MPI_COMM_WORLD); int rk = comm.get_rank(); int sz = comm.get_size(); auto f = []() { int b = 1; }; comm.get_rank(); { // builtin send + recv if(rk == 0) { int a = 7; comm.send(a, 1, 2014); } else if(rk == 1) { int b = 0; comm.recv(b, 0, 2014); std::cout << b << std::endl; } } { // isend + recv if(rk == 0) { int a = 7; MPI_Request req; req = comm.isend(a, 1, 2014); } else if(rk == 1) { int b = 0; comm.recv(b, 0, 2014); std::cout << b << std::endl; } } { // container send + recv if(rk == 0) { std::vector<int> aa {77, 88}; comm.send(aa, 1, 2014); } else if(rk == 1) { std::vector<int> bb; comm.recv(bb, 0, 2014); for(auto & item : bb) std::cout << item << std::endl; } } { // container isend + recv if(rk == 0) { std::vector<int> aa {77, 88}; MPI_Request req; req = comm.isend(aa, 1, 2014); } else if(rk == 1) { std::vector<int> bb; comm.recv(bb, 0, 2014); for(auto & item : bb) std::cout << item << std::endl; } } { // paracel triple send + recv if(rk == 0) { std::tuple<std::string, std::string, double> aa; std::get<0>(aa) = "abc"; std::get<1>(aa) = "def"; std::get<2>(aa) = 3.14; MPI_Request req; req = comm.isend(aa, 1, 2014); } else if(rk == 1) { std::tuple<std::string, std::string, double> bb; comm.recv(bb, 0, 2014); std::cout << "deniug" << std::get<0>(bb) << "--" << std::get<1>(bb) << "--" << std::get<2>(bb) << std::endl; } } { // paracel list of triple send + recv if(rk == 0) { std::vector<std::tuple<std::string, std::string, double> > aa; std::tuple<std::string, std::string, double> tmp1; std::get<0>(tmp1) = "abc"; std::get<1>(tmp1) = "def"; std::get<2>(tmp1) = 4.15; aa.push_back(tmp1); std::tuple<std::string, std::string, double> tmp2; std::get<0>(tmp2) = "cba"; std::get<1>(tmp2) = "fed"; std::get<2>(tmp2) = 5.16; aa.push_back(tmp2); MPI_Request req; req = comm.isend(aa, 1, 2014); } else if(rk == 1) { std::vector<std::tuple<std::string, std::string, double> > bb; comm.recv(bb, 0, 2014); for(auto & item : bb) { std::cout << "test" << std::get<0>(item) << "--" << std::get<1>(item) << "--" << std::get<2>(item) << std::endl; } } } { // another paracel list of triple send + recv if(rk == 0) { std::vector<std::tuple<std::string, std::string, double> > aa; std::tuple<std::string, std::string, double> tmp1; std::get<0>(tmp1) = "abc"; std::get<1>(tmp1) = "def"; std::get<2>(tmp1) = 4.15; aa.push_back(tmp1); std::tuple<std::string, std::string, double> tmp2; aa.push_back(tmp2); MPI_Request req; req = comm.isend(aa, 1, 2014); } else if(rk == 1) { std::vector<std::tuple<std::string, std::string, double> > bb; comm.recv(bb, 0, 2014); for(auto & item : bb) { std::cout << "check " << std::get<0>(item) << "--" << std::get<1>(item) << "--" << std::get<2>(item) << std::endl; } } } { // builtin sendrecv int a = 8; int b; int left, right; right = (rk + 1) % sz; left = rk - 1; if(left < 0) left = sz - 1; comm.sendrecv(a, b, left, 2014, right, 2014); std::cout << "b" << b << std::endl; } { // container sendrecv std::vector<int> aaa{1,2,3}; std::vector<int> bbb; int left, right; right = (rk + 1) % sz; left = rk - 1; if(left < 0) left = sz - 1; comm.sendrecv(aaa, bbb, left, 2014, right, 2014); for(auto & item : bbb) std::cout << item << std::endl; } { // paracel triple sendrecv std::tuple<std::string, std::string, double> aa; std::tuple<std::string, std::string, double> bb; std::get<0>(aa) = "abc"; std::get<1>(aa) = "def"; std::get<2>(aa) = 3.14; int left, right; right = (rk + 1) % sz; left = rk - 1; if(left < 0) left = sz - 1; comm.sendrecv(aa, bb, left, 2014, right, 2014); std::cout << "triple sendrecv" << std::get<0>(bb) << "--" << std::get<1>(bb) << "--" << std::get<2>(bb) << std::endl; } { // paracel list of triple sendrecv std::vector<std::tuple<std::string, std::string, double> > bb; std::vector<std::tuple<std::string, std::string, double> > aa; std::tuple<std::string, std::string, double> tmp1; std::get<0>(tmp1) = "abc"; std::get<1>(tmp1) = "def"; std::get<2>(tmp1) = 4.15; aa.push_back(tmp1); std::tuple<std::string, std::string, double> tmp2; std::get<0>(tmp2) = "cba"; std::get<1>(tmp2) = "fed"; std::get<2>(tmp2) = 5.16; aa.push_back(tmp2); int left, right; right = (rk + 1) % sz; left = rk - 1; if(left < 0) left = sz - 1; comm.sendrecv(aa, bb, left, 2014, right, 2014); for(auto & item : bb) { std::cout << "test sendrecv" << std::get<0>(item) << "--" << std::get<1>(item) << "--" << std::get<2>(item) << std::endl; } } { // another paracel list of triple sendrecv std::vector<std::tuple<std::string, std::string, double> > bb; std::vector<std::tuple<std::string, std::string, double> > aa; std::tuple<std::string, std::string, double> tmp1; std::get<0>(tmp1) = "abc"; std::get<1>(tmp1) = "def"; std::get<2>(tmp1) = 4.15; std::tuple<std::string, std::string, double> tmp2; aa.push_back(tmp2); aa.push_back(tmp1); int left, right; right = (rk + 1) % sz; left = rk - 1; if(left < 0) left = sz - 1; comm.sendrecv(aa, bb, left, 2014, right, 2014); for(auto & item : bb) { std::cout << "another test sendrecv" << std::get<0>(item) << "--" << std::get<1>(item) << "--" << std::get<2>(item) << std::endl; } } { // debug for list of triple sendrecv std::vector<std::vector<std::tuple<std::string, std::string, double> > > aa(2); std::vector<std::tuple<std::string, std::string, double> > bb; int t, f; if(rk == 0) { std::vector<std::tuple<std::string, std::string, double> > aaa; std::tuple<std::string, std::string, double> tmp1; std::get<0>(tmp1) = "a"; std::get<1>(tmp1) = "b"; std::get<2>(tmp1) = 1.; aaa.push_back(tmp1); std::get<0>(tmp1) = "a"; std::get<1>(tmp1) = "c"; std::get<2>(tmp1) = 1.; aaa.push_back(tmp1); std::get<0>(tmp1) = "a"; std::get<1>(tmp1) = "d"; std::get<2>(tmp1) = 1.; aaa.push_back(tmp1); std::get<0>(tmp1) = "b"; std::get<1>(tmp1) = "a"; std::get<2>(tmp1) = 1.; aaa.push_back(tmp1); aa[1] = aaa; t = 1; f = 1; } else if(rk == 1) { std::vector<std::tuple<std::string, std::string, double> > aaa; std::tuple<std::string, std::string, double> tmp1; std::get<0>(tmp1) = "e"; std::get<1>(tmp1) = "a"; std::get<2>(tmp1) = 1.; aaa.push_back(tmp1); std::get<0>(tmp1) = "e"; std::get<1>(tmp1) = "d"; std::get<2>(tmp1) = 1.; aaa.push_back(tmp1); aa[0] = aaa; std::vector<std::tuple<std::string, std::string, double> > aaaa; std::tuple<std::string, std::string, double> tmp2; std::get<0>(tmp2) = "b"; std::get<1>(tmp2) = "d"; std::get<2>(tmp2) = 1.; aaaa.push_back(tmp2); std::get<0>(tmp2) = "d"; std::get<1>(tmp2) = "c"; std::get<2>(tmp2) = 1.; aaaa.push_back(tmp2); aa[1] = aaaa; f = 0; t = 0; } comm.sendrecv(aa[t], bb, t, 2014, f, 2014); } { // builtin bcast int a; if(rk == 0) a = 7; comm.bcast(a, 0); std::cout << "rk " << rk << " a " << a << std::endl; } { // container bcast std::vector<int> aa(2); if(rk == 0) { aa[0] = 3; aa[1] = 4; } comm.bcast(aa, 0); std::cout << "rk " << rk << " aa0 " << aa[0] << " aa1 " << aa[1] << std::endl; } { // builtin alltoall std::vector<int> a(2), b(2); if(rk == 0) { a[0] = 1; a[1] = 3; } if(rk == 1) { a[0] = 2; a[1] = 4; } comm.alltoall(a, b); std::cout << " rk " << rk << b[0] << " " << b[1] << std::endl; } { // container alltoall std::vector< std::vector<int> > a(2), b; if(rk == 0) { std::vector<int> tmp1{1, 5}; std::vector<int> tmp2{3}; a[0] = tmp1; a[1] = tmp2; } if(rk == 1) { std::vector<int> tmp1{2}; std::vector<int> tmp2{4, 7}; a[0] = tmp1; a[1] = tmp2; } comm.alltoall(a, b); if(rk == 1) { for(auto & item : b[0]) std::cout << " test " << item << " "; std::cout << "sep" << std::endl; for (auto & item : b[1]) std::cout << " test " << item << " "; std::cout << std::endl; } } { // builtin allreduce int aaa; if(rk == 0) { aaa = 1; } if(rk == 1) { aaa = 2; } comm.allreduce(aaa); std::cout << " rk " << rk << " result " << aaa << std::endl; } { // container allreduce std::vector<int> aaa(3); if(rk == 0) { aaa[0] = 1; aaa[1] = 2; aaa[2] = 3; } if(rk == 1) { aaa[0] = 3; aaa[1] = 2; aaa[2] = 1; } comm.allreduce(aaa); for(auto & item : aaa) std::cout << " rk " << rk << " result aaa " << item << std::endl; } { // bcastring std::vector<int> a; if(rk == 0) { a.push_back(6); a.push_back(42); } if(rk == 1) { a.push_back(28); a.push_back(42); a.push_back(42); a.push_back(28); a.push_back(6); } std::set<int> result; auto func = [&](std::vector<int> tmp) { for(auto & stf : tmp) result.insert(stf); }; comm.bcastring(a, func); for(auto & item : result) { std::cout << rk << " : "<< item << std::endl; } } { // dict_type<size_t, int> isend if(rk == 0) { std::unordered_map<size_t, int> aa; aa[0] = 1; aa[1] = 2; MPI_Request req; req = comm.isend(aa, 1, 2014); } else if(rk == 1) { std::unordered_map<size_t, int> bb; comm.recv(bb, 0, 2014); for(auto & item : bb) std::cout << item.first << " * " << item.second << std::endl; } } { // alltoall } return 0; }