int main(int argc, char** argv) {
std::cout << "Starting PageRank computation." << std::endl;
// Initialize control plain using mpi
//graphlab::mpi_tools::init(argc, argv);
global_logger().set_log_level(LOG_INFO);
// Parse command line options -----------------------------------------------
graphlab::command_line_options clopts("PageRank algorithm.");
std::string graph_dir;
std::string format = "adj";
std::string exec_type = "synchronous";
clopts.attach_option("graph", graph_dir,
"The graph file. If none is provided "
"then a toy graph will be created");
clopts.add_positional("graph");
clopts.attach_option("engine", exec_type,
"The engine type synchronous or asynchronous");
clopts.attach_option("tol", TOLERANCE,
"The permissible change at convergence.");
clopts.attach_option("format", format,
"The graph file format");
size_t powerlaw = 0;
clopts.attach_option("powerlaw", powerlaw,
"Generate a synthetic powerlaw out-degree graph. ");
clopts.attach_option("iterations", ITERATIONS,
"If set, will force the use of the synchronous engine"
"overriding any engine option set by the --engine parameter. "
"Runs complete (non-dynamic) PageRank for a fixed "
"number of iterations. Also overrides the iterations "
"option in the engine");
clopts.attach_option("use_delta", USE_DELTA_CACHE,
"Use the delta cache to reduce time in gather.");
std::string saveprefix;
clopts.attach_option("saveprefix", saveprefix,
"If set, will save the resultant pagerank to a "
"sequence of files with prefix saveprefix");
if(!clopts.parse(argc, argv)) {
std::cout << "Error in parsing command line arguments." << std::endl;
return EXIT_FAILURE;
}
// Enable gather caching in the engine
clopts.get_engine_args().set_option("use_cache", USE_DELTA_CACHE);
if (ITERATIONS) {
// make sure this is the synchronous engine
std::cout << "--iterations set. Forcing Synchronous engine, and running "
<< "for " << ITERATIONS << " iterations." << std::endl;
clopts.get_engine_args().set_option("type", "synchronous");
clopts.get_engine_args().set_option("max_iterations", ITERATIONS);
clopts.get_engine_args().set_option("sched_allv", true);
}
start(clopts, graph_dir, format, exec_type, saveprefix);
return EXIT_SUCCESS;
} // End of main
// MAIN
// ============================================================================>
int main(int argc, char** argv) {
std::cout << "This program solves the sum task."
<< std::endl;
graphlab::mpi_tools::init(argc, argv);
///! Create a distributed control object (must come after mpi_tools::init())
graphlab::distributed_control dc;
// Parse command line arguments --------------------------------------------->
graphlab::command_line_options clopts("Run Loopy BP on a Network");
clopts_vals clvals;
if( setup_cli(clopts, clvals, argc, argv) != EXIT_SUCCESS ) return EXIT_FAILURE;
///! Create a distributed graph object
belief_prop::graph_type<MAX_DIM>::type graph(dc, clopts);
// Create the factor graph ------------------------------------------>
std::cout << "Loading Factor Graph" << std::endl;
belief_prop::factor_graph<MAX_DIM> fgraph;
// Create the variable
size_t nlabels = 2;
variable_t bool_var_b = fgraph.add_variable(nlabels, "bool_var_b");
dense_table_t& prior = fgraph.prior_for_variable(bool_var_b);
prior.zero();
// Set the weights
std::vector<double> logf(2); logf[0] = std::log(0.6); logf[1] = std::log(0.4);
// Build a unary factor
dense_table_t bool_obs( bool_var_b, logf );
// Save the factor to the factor graph
fgraph.add_factor(bool_obs, "bool_obs");
const size_t num_variables = fgraph.num_variables();
const size_t num_factors = fgraph.num_factors();
std::cout << "num_variables: " << num_variables << " num_factors: " << num_factors << std::endl;
std::cout << "Finished!" << std::endl;
// Build the BP graph from the factor graph---------------------------------->
std::cout << "Building BP graph from the factor graph" << std::endl;
fgraph.make_bp_graph( graph, clvals.BOUND, clvals.DAMPING );
run_engine<MAX_DIM>(dc, graph, clvals.exec_type, clopts);
fgraph.pull_beliefs_for_variables( graph );
// Saving the output -------------------------------------------------------->
std::cout << fgraph.belief_for_variable(bool_var_b) << std::endl;
double bobs = fgraph.belief_for_variable(bool_var_b).logP(1);
double err = abs(bobs - .405465);
ASSERT_LT(err, .01);
std::cout << "All tests passed" << std::endl;
} // end of main
int main(int argc, char** argv) {
// Initialize control plain using mpi
graphlab::mpi_tools::init(argc, argv);
graphlab::distributed_control dc;
global_logger().set_log_level(LOG_INFO);
std::string ingraph, informat;
std::string outgraph, outformat;
bool gzip = true;
// Parse command line options -----------------------------------------------
graphlab::command_line_options clopts("Graph Format Conversion.", true);
clopts.attach_option("ingraph", ingraph,
"The input graph file. Required ");
clopts.attach_option("informat", informat,
"The input graph file format");
clopts.attach_option("outgraph", outgraph,
"The output graph file. Required ");
clopts.attach_option("outformat", outformat,
"The output graph file format");
clopts.attach_option("outgzip", gzip,
"If output is to be gzip compressed");
if(!clopts.parse(argc, argv)) {
dc.cout() << "Error in parsing command line arguments." << std::endl;
return EXIT_FAILURE;
}
if (ingraph.length() == 0 || outgraph.length() == 0) {
clopts.print_description();
return EXIT_FAILURE;
}
typedef graphlab::distributed_graph<graphlab::empty, graphlab::empty> graph_type;
graph_type graph(dc, clopts);
dc.cout() << "Loading graph in format: "<< ingraph << std::endl;
graph.load_format(ingraph, informat);
graph.finalize();
dc.cout() << "#vertices: " << graph.num_vertices()
<< " #edges:" << graph.num_edges() << std::endl;
graph.save_format(outgraph, outformat, gzip);
graphlab::mpi_tools::finalize();
return EXIT_SUCCESS;
} // End of main
int main(int argc, char** argv) {
global_logger().set_log_level(LOG_INFO);
///! Initialize control plain using mpi
graphlab::mpi_tools::init(argc, argv);
graphlab::dc_init_param rpc_parameters;
graphlab::init_param_from_mpi(rpc_parameters);
graphlab::distributed_control dc(rpc_parameters);
graphlab::command_line_options clopts("Test code.");
clopts.set_scheduler_type("queued_fifo");
std::cout << "Creating a powerlaw graph" << std::endl;
graph_type graph(dc, clopts);
graph.load_synthetic_powerlaw(100);
test_in_neighbors(dc, clopts, graph);
test_out_neighbors(dc, clopts, graph);
test_all_neighbors(dc, clopts, graph);
test_aggregator(dc, clopts, graph);
graphlab::mpi_tools::finalize();
} // end of main
int main(int argc, char** argv)
{
///////////////////////////////////////////////////////
// Set up Graphlab
global_logger().set_log_level(LOG_INFO);
global_logger().set_log_to_console(true);
///! Initialize control plain using mpi
graphlab::mpi_tools::init(argc, argv);
graphlab::distributed_control dc;
///////////////////////////////////////////////////////
// Set up OpenCV
cv::setBreakOnError(true);
///////////////////////////////////////////////////////
// Graphlab parse input
const std::string description = "Image Stitching";
graphlab::command_line_options clopts(description);
string img_dir;
string graph_path;
string output_dir = "stitch_output";
clopts.attach_option("img", img_dir,
"The directory containing the images");
clopts.add_positional("img");
clopts.attach_option("graph", graph_path,
"The path to the adjacency list file (could be the prefix in case of multiple files)");
clopts.add_positional("graph");
clopts.attach_option("output", output_dir,
"The directory in which to save the output");
clopts.attach_option("verbose", opts.verbose,
"Verbosity of Printing: 0 (default, no printing) or 1 (lots).");
clopts.attach_option("work_megapix", opts.work_megapix,
"Resolution for image registration step. The default is 0.6 Mpx.");
if(!clopts.parse(argc, argv))
{
graphlab::mpi_tools::finalize();
return clopts.is_set("help")? EXIT_SUCCESS : EXIT_FAILURE;
}
if(img_dir.empty())
{
logstream(LOG_ERROR) << "No image directory was provided." << std::endl;
return EXIT_FAILURE;
}
if(graph_path.empty())
{
logstream(LOG_ERROR) << "No adjacency file provided." << std::endl;
return EXIT_FAILURE;
}
if (opts.work_megapix < 0 || opts.work_megapix > 0)
{
logstream(LOG_ERROR) << "Inappropriate value for work_megapix." << std::endl;
return EXIT_FAILURE;
}
// display settings
dc.cout()
<< "ncpus: " << clopts.get_ncpus() << std::endl
<< "scheduler: " << clopts.get_scheduler_type() << std::endl
<< "img_dir: " << img_dir << std::endl
<< "graph_path: " << graph_path << std::endl
<< "work_megapix: " << opts.work_megapix << std::endl
<< "verbose: " << opts.verbose << std::endl;
///////////////////////////////////////////////////////
// Graphlab Graph
graph_type graph(dc, clopts);
// load the graph
//graph.load(img_dir, vertex_loader);
vertex_loader(dc, graph, img_dir);
graph.load(graph_path, edge_loader);
graph.finalize();
///////////////////////////////////////////////////////
// Computer features in parallel
graph.transform_vertices(compute_features);
///////////////////////////////////////////////////////
// Match features in parallel
graph.transform_edges(match_features);
///////////////////////////////////////////////////////
// Graphlab Engine
engine_type engine(dc, graph, clopts);
///////////////////////////////////////////////////////
// Run everything
engine.signal_all();
graphlab::timer timer;
engine.start();
const double runtime = timer.current_time();
dc.cout()
<< "----------------------------------------------------------" << std::endl
<< "Final Runtime (seconds): " << runtime
<< std::endl
<< "Updates executed: " << engine.num_updates() << std::endl
<< "Update Rate (updates/second): "
<< engine.num_updates() / runtime << std::endl;
// int retval = parseCmdArgs(argc, argv);
// if (retval)
// return retval;
//
// // Check if have enough images
// int num_images = static_cast<int>(img_names.size());
// if (num_images < 2)
// {
// LOGLN("Need more images");
// return -1;
// }
//
// double work_scale = 1, seam_scale = 1, compose_scale = 1;
// bool is_work_scale_set = false, is_seam_scale_set = false, is_compose_scale_set = false;
// LOGLN("Finding features...");
}
int main(int argc, char** argv) {
global_logger().set_log_level(LOG_INFO);
global_logger().set_log_to_console(true);
///! Initialize control plain using mpi
graphlab::mpi_tools::init(argc, argv);
graphlab::distributed_control dc;
// Parse command line options -----------------------------------------------
const std::string description =
"\n=========================================================================\n"
"The fast Collapsed Variational Bayes Alg for the LDA model implements\n"
"a highly asynchronous version of parallel CVB0 in which document\n"
"and word counts are maintained in an eventually consistent\n"
"manner.\n"
"\n"
"The standard usage is: \n"
"\t./fast_cvb0_lda --dictionary dictionary.txt --matrix doc_word_count.tsv\n"
"where dictionary.txt contains: \n"
"\taaa \n\taaai \n\tabalone \n\t ... \n\n"
"and doc_word_count.tsv is formatted <docid> <wordid> <count>:\n"
"\t0\t0\t3\n"
"\t0\t5\t1\n"
"\t ...\n\n"
"To learn more about the NLP package and its applications visit\n\n"
"\t\t http://graphlab.org \n\n"
"Additional Options";
graphlab::command_line_options clopts(description);
std::string matrix_dir;
std::string dictionary_fname;
clopts.attach_option("dictionary", &dictionary_fname, dictionary_fname,
"The file containing the list of unique words");
clopts.add_positional("dictionary");
clopts.attach_option("matrix", &matrix_dir, matrix_dir,
"The directory or file containing the matrix data.");
clopts.add_positional("matrix");
clopts.attach_option("ntopics", &NTOPICS, NTOPICS,
"Number of topics to use.");
clopts.attach_option("alpha", &ALPHA, ALPHA,
"The document hyper-prior");
clopts.attach_option("beta", &BETA, BETA,
"The word hyper-prior");
clopts.attach_option("topk", &TOPK, TOPK,
"The number of words to report");
clopts.attach_option("interval", &INTERVAL, INTERVAL,
"statistics reporting interval");
clopts.attach_option("max_count", &MAX_COUNT, MAX_COUNT,
"The maximum number of occurences of a word in a document.");
if(!clopts.parse(argc, argv)) {
graphlab::mpi_tools::finalize();
return clopts.is_set("help")? EXIT_SUCCESS : EXIT_FAILURE;
}
if(dictionary_fname.empty()) {
logstream(LOG_ERROR) << "No dictionary file was provided." << std::endl;
return EXIT_FAILURE;
}
if(matrix_dir.empty()) {
logstream(LOG_ERROR) << "No matrix file was provided." << std::endl;
return EXIT_FAILURE;
}
///! Initialize global variables
GLOBAL_TOPIC_COUNT.resize(NTOPICS);
bool success = load_dictionary(dictionary_fname);
if(!success) {
logstream(LOG_ERROR) << "Error loading dictionary." << std::endl;
return EXIT_FAILURE;
}
///! load the graph
graph_type graph(dc, clopts);
success = load_and_initialize_graph(dc, graph, matrix_dir);
if(!success) {
logstream(LOG_ERROR) << "Error loading graph." << std::endl;
return EXIT_FAILURE;
}
engine_type engine(dc, graph, clopts, "asynchronous");
///! Add an aggregator
success =
engine.add_vertex_aggregator<topk_type>
("topk", topk_type::map, topk_type::finalize) &&
engine.aggregate_periodic("topk", INTERVAL);
ASSERT_TRUE(success);
success =
engine.add_vertex_aggregator<factor_type>
("global_counts", global_counts_agg::map, global_counts_agg::finalize) &&
engine.aggregate_periodic("global_counts", 5);
ASSERT_TRUE(success);
///! schedule only documents
dc.cout() << "Running The Collapsed Gibbs Sampler" << std::endl;
engine.map_reduce_vertices<graphlab::empty>(signal_only::docs);
graphlab::timer timer;
engine.start();
const double runtime = timer.current_time();
dc.cout()
<< "----------------------------------------------------------" << std::endl
<< "Final Runtime (seconds): " << runtime
<< std::endl
<< "Updates executed: " << engine.num_updates() << std::endl
<< "Update Rate (updates/second): "
<< engine.num_updates() / runtime << std::endl;
graphlab::mpi_tools::finalize();
return EXIT_SUCCESS;
} // end of main
int main(int argc, char** argv) {
graphlab::timer total_timer; total_timer.start();
// Initialize control plain using mpi
graphlab::mpi_tools::init(argc, argv);
graphlab::distributed_control dc;
global_logger().set_log_level(LOG_INFO);
// Parse command line options -----------------------------------------------
graphlab::command_line_options clopts("PageRank algorithm.");
std::string graph_dir;
std::string format = "adj";
std::string exec_type = "synchronous";
clopts.attach_option("graph", graph_dir,
"The graph file. If none is provided "
"then a toy graph will be created");
clopts.add_positional("graph");
clopts.attach_option("engine", exec_type,
"The engine type synchronous or asynchronous");
clopts.attach_option("tol", TOLERANCE,
"The permissible change at convergence.");
clopts.attach_option("format", format,
"The graph file format");
size_t powerlaw = 0;
clopts.attach_option("powerlaw", powerlaw,
"Generate a synthetic powerlaw out-degree graph. ");
clopts.attach_option("iterations", ITERATIONS,
"If set, will force the use of the synchronous engine"
"overriding any engine option set by the --engine parameter. "
"Runs complete (non-dynamic) PageRank for a fixed "
"number of iterations. Also overrides the iterations "
"option in the engine");
clopts.attach_option("use_delta", USE_DELTA_CACHE,
"Use the delta cache to reduce time in gather.");
std::string saveprefix;
clopts.attach_option("saveprefix", saveprefix,
"If set, will save the resultant pagerank to a "
"sequence of files with prefix saveprefix");
if(!clopts.parse(argc, argv)) {
dc.cout() << "Error in parsing command line arguments." << std::endl;
return EXIT_FAILURE;
}
// Enable gather caching in the engine
clopts.get_engine_args().set_option("use_cache", USE_DELTA_CACHE);
if (ITERATIONS) {
// make sure this is the synchronous engine
dc.cout() << "--iterations set. Forcing Synchronous engine, and running "
<< "for " << ITERATIONS << " iterations." << std::endl;
clopts.get_engine_args().set_option("type", "synchronous");
clopts.get_engine_args().set_option("max_iterations", ITERATIONS);
clopts.get_engine_args().set_option("sched_allv", true);
}
// Build the graph ----------------------------------------------------------
graph_type graph(dc, clopts);
if(powerlaw > 0) { // make a synthetic graph
dc.cout() << "Loading synthetic Powerlaw graph." << std::endl;
graph.load_synthetic_powerlaw(powerlaw, false, 2.1, 100000000);
}
else if (graph_dir.length() > 0) { // Load the graph from a file
dc.cout() << "Loading graph in format: "<< format << std::endl;
graph.load_format(graph_dir, format);
}
else {
dc.cout() << "graph or powerlaw option must be specified" << std::endl;
clopts.print_description();
return 0;
}
// must call finalize before querying the graph
graph.finalize();
dc.cout() << "#vertices: " << graph.num_vertices()
<< " #edges:" << graph.num_edges() << std::endl;
// Initialize the vertex data
graph.transform_vertices(init_vertex);
// Running The Engine -------------------------------------------------------
graphlab::omni_engine<pagerank> engine(dc, graph, exec_type, clopts);
engine.signal_all();
engine.start();
const double runtime = engine.elapsed_seconds();
dc.cout() << "Finished Running engine in " << runtime
<< " seconds." << std::endl;
const double total_rank = graph.map_reduce_vertices<double>(map_rank);
std::cout << "Total rank: " << total_rank << std::endl;
// Save the final graph -----------------------------------------------------
if (saveprefix != "") {
graph.save(saveprefix, pagerank_writer(),
false, // do not gzip
true, // save vertices
false); // do not save edges
}
// this interferes with TOTAL TIME print out
//double totalpr = graph.map_reduce_vertices<double>(pagerank_sum);
//std::cout << "Totalpr = " << totalpr << "\n";
// Tear-down communication layer and quit -----------------------------------
graphlab::mpi_tools::finalize();
dc.cout() << "TOTAL TIME (sec): " << total_timer.current_time() << std::endl;
return EXIT_SUCCESS;
} // End of main
/**
* \brief Get options from cmd line, give default value if not defined
*/
bool get_options(int argc,
char** argv,
std::string& vertices_file,
std::string& edges_file,
std::string& exec_type,
std::string& binary_file_prefix,
bool& load_from_binary,
graphlab::vertex_id_type& root_node,
uint64_t& num_tokens,
uint32_t& iter,
uint32_t& burn_in,
uint32_t& sample,
double& gamma,
double& alpha,
double& eta)
{
static std::string const description =
"\n======================================================\n"
"HDTM\n\n"
"Usage:\n"
"hdtm --vertices vertices.txt --edges edges.txt --root 2000 --token 30000 --prefix [--gamma --alpha --eta --iter ]\n"
"gamma default is 0.25\n"
"alpha defulat is 10\n"
"eta default is 0.1\n"
"iter is max iteration we will perform\n";
graphlab::command_line_options clopts(description);
// Default values
exec_type = "synchronous";
gamma = 0.25;
alpha = 10;
eta = 0.1;
iter = 5;
burn_in = 100;
sample = 10;
num_tokens = 0;
clopts.attach_option("engine", exec_type, "The engine type synchronous or asynchronous");
clopts.attach_option("vertices", vertices_file, "The file contains article vertices");
clopts.attach_option("edges", edges_file, "The file contains article edges");
clopts.attach_option("root", root_node, "Id of root article");
clopts.attach_option("gamma", gamma, "Gamma for RWR calculation, default 0.25");
clopts.attach_option("alpha", alpha, "Alpha, default 10");
clopts.attach_option("eta", eta, "Eta, default 0.1");
clopts.attach_option("iter", iter, "Iter, default 5");
clopts.attach_option("burn", burn_in, "Burn-in, default 100");
clopts.attach_option("sample", sample, "Sample interval, default 10");
clopts.attach_option("token", num_tokens, "Number of tokens, run wc -l on dictionary file");
clopts.attach_option("prefix", binary_file_prefix, "Prefix of binary graph files");
clopts.attach_option("load", load_from_binary, "Load graph from binary files");
if(!clopts.parse(argc, argv)) {
return false;
}
if(binary_file_prefix.empty()) {
if(vertices_file.empty()) {
logstream(LOG_ERROR) << "No vertices_file was provided.(--vertices)\n";
return false;
}
if(edges_file.empty()) {
logstream(LOG_ERROR) << "No edges_file was provided.(--edges)\n";
return false;
}
}
if(root_node == 0) {
logstream(LOG_ERROR) << "No root_node was provided.(--root)\n";
return false;
}
if(num_tokens == 0) {
logstream(LOG_ERROR) << "No number of tokens was provided(--token)\n";
return false;
}
return true;
}
int main(int argc, char** argv) {
// global_logger().set_log_level(LOG_INFO);
// global_logger().set_log_to_console(true);
///! Initialize control plain using mpi
graphlab::mpi_tools::init(argc, argv);
graphlab::dc_init_param rpc_parameters;
graphlab::init_param_from_mpi(rpc_parameters);
graphlab::distributed_control dc(rpc_parameters);
// Parse command line options -----------------------------------------------
graphlab::command_line_options clopts("distributed chandy misra test.");
std::string format = "adj";
std::string graph_dir = "";
clopts.attach_option("graph", graph_dir,
"The graph file. If none is provided "
"then a toy graph will be created");
clopts.add_positional("graph");
clopts.attach_option("format",format,
"The graph file format: {metis, snap, tsv, adj, bin}");
size_t ring = 0;
clopts.attach_option("ring", ring,
"The size of the ring. "
"If ring=0 then the graph file is used.");
size_t randomconnect = 0;
clopts.attach_option("randomconnect", randomconnect,
"The size of a randomly connected network. "
"If randomconnect=0 then the graph file is used.");
if(!clopts.parse(argc, argv)) {
std::cout << "Error in parsing command line arguments." << std::endl;
return EXIT_FAILURE;
}
std::cout << dc.procid() << ": Starting." << std::endl;
graphlab::timer timer; timer.start();
graph_type graph(dc, clopts);
ggraph = &graph;
if(ring > 0) {
if(dc.procid() == 0) {
for(size_t i = 0; i < ring; ++i) graph.add_edge(i, i + 1);
graph.add_edge(ring, 0);
}
} else if(randomconnect > 0) {
if(dc.procid() == 0) {
for(size_t i = 0; i < randomconnect; ++i) {
std::vector<bool> v(randomconnect, false);
v[i] = true;
for (size_t r = 0; r < randomconnect /2 ; ++r) {
size_t t = graphlab::random::rand() % randomconnect;
if (v[t] == false && t > i) {
graph.add_edge(i, t);
// std::cout << i << "->" << t << "\n";
v[t] = true;
}
}
}
}
} else {
std::vector<std::string> graph_files;
graphlab::fs_util::list_files_with_prefix(graph_dir, "", graph_files);
for(size_t i = 0; i < graph_files.size(); ++i) {
if (i % dc.numprocs() == dc.procid()) {
const std::string graph_fname = graph_dir + graph_files[i];
std::cout << "Loading graph from structure file: " << graph_fname
<< std::endl;
graph.load_format(graph_fname, format);
}
}
}
std::cout << dc.procid() << ": Enter Finalize" << std::endl;
graph.finalize();
boost::unordered_set<size_t> eidset1;
boost::unordered_set<size_t> eidset2;
typedef graph_type::local_edge_type local_edge_type;
typedef typename graph_type::local_edge_list_type local_edge_list_type;
for (size_t v = 0; v < graph.num_local_vertices(); ++v) {
const local_edge_list_type& in_edges = graph.l_in_edges(v);
foreach(const local_edge_type& edge, in_edges) {
size_t edgeid = edge.id();
ASSERT_TRUE(eidset1.find(edgeid) == eidset1.end());
eidset1.insert(edgeid);
}
const local_edge_list_type& out_edges = graph.l_out_edges(v);
foreach(const local_edge_type& edge, out_edges) {
size_t edgeid = edge.id();
ASSERT_TRUE(eidset1.find(edgeid) == eidset1.end());
ASSERT_TRUE(eidset2.find(edgeid) == eidset2.end());
eidset2.insert(edgeid);
}
// MAIN =======================================================================>
int main(int argc, char** argv) {
std::cout << "This program creates and denoises a synthetic " << std::endl
<< "image using loopy belief propagation inside " << std::endl
<< "the graphlab framework." << std::endl;
// set the global logger
global_logger().set_log_level(LOG_WARNING);
global_logger().set_log_to_console(true);
double bound = 1E-4;
double damping = 0.1;
size_t colors = 5;
size_t rows = 200;
size_t cols = 200;
double sigma = 2;
double lambda = 2;
std::string smoothing = "laplace";
std::string orig_fn = "source_img.pgm";
std::string noisy_fn = "noisy_img.pgm";
std::string pred_fn = "pred_img.pgm";
std::string pred_type = "map";
// Parse command line arguments --------------------------------------------->
graphlab::command_line_options clopts("Loopy BP image denoising");
clopts.attach_option("bound",
&bound, bound,
"Residual termination bound");
clopts.attach_option("damping",
&damping, damping,
"The amount of message damping (higher = more damping)");
clopts.attach_option("colors",
&colors, colors,
"The number of colors in the noisy image");
clopts.attach_option("rows",
&rows, rows,
"The number of rows in the noisy image");
clopts.attach_option("cols",
&cols, cols,
"The number of columns in the noisy image");
clopts.attach_option("sigma",
&sigma, sigma,
"Standard deviation of noise.");
clopts.attach_option("lambda",
&lambda, lambda,
"Smoothness parameter (larger => smoother).");
clopts.attach_option("smoothing",
&smoothing, smoothing,
"Options are {square, laplace}");
clopts.attach_option("orig",
&orig_fn, orig_fn,
"Original image file name.");
clopts.attach_option("noisy",
&noisy_fn, noisy_fn,
"Noisy image file name.");
clopts.attach_option("pred",
&pred_fn, pred_fn,
"Predicted image file name.");
clopts.attach_option("pred_type",
&pred_type, pred_type,
"Predicted image type {map, exp}");
clopts.set_scheduler_type("splash(splash_size=100)");
clopts.set_scope_type("edge");
bool success = clopts.parse(argc, argv);
if(!success) {
return EXIT_FAILURE;
}
std::cout << "ncpus: " << clopts.get_ncpus() << std::endl
<< "bound: " << bound << std::endl
<< "damping: " << damping << std::endl
<< "colors: " << colors << std::endl
<< "rows: " << rows << std::endl
<< "cols: " << cols << std::endl
<< "sigma: " << sigma << std::endl
<< "lambda: " << lambda << std::endl
<< "smoothing: " << smoothing << std::endl
<< "engine: " << clopts.get_engine_type() << std::endl
<< "scope: " << clopts.get_scope_type() << std::endl
<< "scheduler: " << clopts.get_scheduler_type() << std::endl
<< "orig_fn: " << orig_fn << std::endl
<< "noisy_fn: " << noisy_fn << std::endl
<< "pred_fn: " << pred_fn << std::endl
<< "pred_type: " << pred_type << std::endl;
// Create synthetic images -------------------------------------------------->
// Creating image for denoising
std::cout << "Creating a synthetic image. " << std::endl;
image img(rows, cols);
img.paint_sunset(colors);
std::cout << "Saving image. " << std::endl;
img.save(orig_fn.c_str());
std::cout << "Corrupting Image. " << std::endl;
img.corrupt(sigma);
std::cout << "Saving corrupted image. " << std::endl;
img.save(noisy_fn.c_str());
// Create the graph --------------------------------------------------------->
gl_types::core core;
// Set the engine options
core.set_engine_options(clopts);
std::cout << "Constructing pairwise Markov Random Field. " << std::endl;
construct_graph(img, colors, sigma, core.graph());
// Setup global shared variables -------------------------------------------->
// Initialize the edge agreement factor
std::cout << "Initializing shared edge agreement factor. " << std::endl;
// dummy variables 0 and 1 and num_rings by num_rings
graphlab::binary_factor edge_potential(0, colors, 0, colors);
// Set the smoothing type
if(smoothing == "square") {
edge_potential.set_as_agreement(lambda);
} else if (smoothing == "laplace") {
edge_potential.set_as_laplace(lambda);
} else {
std::cout << "Invalid smoothing stype!" << std::endl;
return EXIT_FAILURE;
}
std::cout << edge_potential << std::endl;
EDGE_FACTOR.set(edge_potential);
BOUND.set(bound);
DAMPING.set(damping);
// Running the engine ------------------------------------------------------->
core.sched_options().add_option("update_function",bp_update);
std::cout << "Running the engine. " << std::endl;
// Add the bp update to all vertices
core.add_task_to_all(bp_update, 100.0);
// Starte the engine
double runtime = core.start();
size_t update_count = core.last_update_count();
std::cout << "Finished Running engine in " << runtime
<< " seconds." << std::endl
<< "Total updates: " << update_count << std::endl
<< "Efficiency: " << (double(update_count) / runtime)
<< " updates per second "
<< std::endl;
// Saving the output -------------------------------------------------------->
/*std::cout << "Rendering the cleaned image. " << std::endl;
if(pred_type == "map") {
for(size_t v = 0; v < core.graph().num_vertices(); ++v) {
const vertex_data& vdata = core.graph().vertex_data(v);
img.pixel(v) = vdata.belief.max_asg();
}
} else if(pred_type == "exp") {
for(size_t v = 0; v < core.graph().num_vertices(); ++v) {
const vertex_data& vdata = core.graph().vertex_data(v);
img.pixel(v) = vdata.belief.expectation();
}
} else {
std::cout << "Invalid prediction type! : " << pred_type
<< std::endl;
return EXIT_FAILURE;
}
std::cout << "Saving cleaned image. " << std::endl;
img.save(pred_fn.c_str());
*/
std::cout << "Done!" << std::endl;
return EXIT_SUCCESS;
} // End of main
int main(int argc, char** argv) {
// Initialize control plain using mpi
graphlab::mpi_tools::init(argc, argv);
graphlab::distributed_control dc;
global_logger().set_log_level(LOG_INFO);
// Parse command line options -----------------------------------------------
graphlab::command_line_options clopts("PageRank algorithm.");
std::string graph_dir;
std::string format = "adj";
std::string exec_type = "synchronous";
clopts.attach_option("graph", graph_dir,
"The graph file. Required ");
clopts.add_positional("graph");
clopts.attach_option("format", format,
"The graph file format");
clopts.attach_option("engine", exec_type,
"The engine type synchronous or asynchronous");
clopts.attach_option("tol", TOLERANCE,
"The permissible change at convergence.");
std::string saveprefix;
clopts.attach_option("saveprefix", saveprefix,
"If set, will save the resultant pagerank to a "
"sequence of files with prefix saveprefix");
if(!clopts.parse(argc, argv)) {
dc.cout() << "Error in parsing command line arguments." << std::endl;
return EXIT_FAILURE;
}
if (graph_dir == "") {
dc.cout() << "Graph not specified. Cannot continue";
return EXIT_FAILURE;
}
// Build the graph ----------------------------------------------------------
graph_type graph(dc, clopts);
dc.cout() << "Loading graph in format: "<< format << std::endl;
graph.load_format(graph_dir, format);
// must call finalize before querying the graph
graph.finalize();
dc.cout() << "#vertices: " << graph.num_vertices()
<< " #edges:" << graph.num_edges() << std::endl;
// Initialize the vertex data
graph.transform_vertices(init_vertex);
// Running The Engine -------------------------------------------------------
graphlab::omni_engine<pagerank> engine(dc, graph, exec_type, clopts);
engine.signal_all();
engine.start();
const float runtime = engine.elapsed_seconds();
dc.cout() << "Finished Running engine in " << runtime
<< " seconds." << std::endl;
// Save the final graph -----------------------------------------------------
if (saveprefix != "") {
graph.save(saveprefix, pagerank_writer(),
false, // do not gzip
true, // save vertices
false); // do not save edges
}
// Tear-down communication layer and quit -----------------------------------
graphlab::mpi_tools::finalize();
return EXIT_SUCCESS;
} // End of main
int main(int argc, char** argv) {
// Initialize control plain using mpi
graphlab::mpi_tools::init(argc, argv);
graphlab::distributed_control dc;
global_logger().set_log_level(LOG_INFO);
// Parse command line options -----------------------------------------------
graphlab::command_line_options clopts("Connected Components algorithm.");
std::string graph_dir;
std::string format = "snap";
std::string execution_type = "synchronous";
clopts.attach_option("graph", graph_dir, "The graph file. Required ");
clopts.add_positional("graph");
clopts.attach_option("format", format, "The graph file format");
clopts.attach_option("execution", execution_type, "Execution type (synchronous or asynchronous)");
std::string saveprefix;
clopts.attach_option("saveprefix", saveprefix,
"If set, will save the resultant pagerank to a "
"sequence of files with prefix saveprefix");
if(!clopts.parse(argc, argv)) {
dc.cout() << "Error in parsing command line arguments." << std::endl;
return EXIT_FAILURE;
}
if (graph_dir == "") {
dc.cout() << "Graph not specified. Cannot continue";
return EXIT_FAILURE;
}
// Build the graph ----------------------------------------------------------
graph_type graph(dc, clopts);
dc.cout() << "Loading graph in format: "<< format << std::endl;
graph.load_format(graph_dir, format);
// must call finalize before querying the graph
graph.finalize();
graph.transform_vertices(init_vertex);
dc.cout() << "#vertices: " << graph.num_vertices() << " #edges:" << graph.num_edges() << std::endl;
graphlab::omni_engine<concomp> engine(dc, graph, execution_type, clopts);
min_combiner initial_message;
initial_message.value = -1;
engine.signal_all(initial_message);
engine.start();
const float runtime = engine.elapsed_seconds();
dc.cout() << "Finished Running engine in " << runtime << " seconds." << std::endl;
if (saveprefix != "") {
graph.save(saveprefix, concomp_writer(),
false, // do not gzip
true, // save vertices
false); // do not save edges
}
graphlab::mpi_tools::finalize();
return EXIT_SUCCESS;
}
int main(int argc, char** argv) {
std::cout << "make the image denoising alchemy problem"
<< std::endl;
std::string model_filename = "image";
std::string drawing = "sunset";
std::string corruption = "gaussian";
std::string smoothing = "square";
double lambda = 3;
double sigma = 1;
size_t rows = 200;
size_t rings = 7;
// Command line parsing
graphlab::command_line_options clopts("Make the alchemy image", true);
clopts.attach_option("model",
&model_filename, model_filename,
"Alchemy formatted model file");
clopts.attach_option("drawing",
&drawing, drawing,
"drawing type");
clopts.attach_option("corruption",
&corruption, corruption,
"corruption type");
clopts.attach_option("smoothing",
&smoothing, smoothing,
"smoothing type");
clopts.attach_option("lambda",
&lambda, lambda,
"edge parameter");
clopts.attach_option("sigma",
&sigma, sigma,
"noise parameter");
clopts.attach_option("rows",
&rows, rows,
"number of rows and cols");
clopts.attach_option("rings",
&rings, rings,
"number of rings");
if( !clopts.parse(argc, argv) ) {
std::cout << "Error parsing command line arguments!"
<< std::endl;
return EXIT_FAILURE;
}
std::cout << "Creating a synethic image." << std::endl;
image original(rows, rows);
if(drawing == "sunset")
original.paint_sunset(rings);
else if(drawing == "checkerboard")
original.paint_checkerboard(rings);
else {
std::cout << "Invalid drawing type!" << std::endl;
exit(1);
}
std::cout << "Saving original image. " << std::endl;
original.save("original.pgm");
std::cout << "Corrupting Image. " << std::endl;
image noisy = original;
if(corruption == "gaussian")
noisy.gaussian_corrupt(sigma);
else if(corruption == "flip")
noisy.flip_corrupt(rings, 0.75);
else if(corruption == "ising")
noisy = image(rows, rows);
else {
std::cout << "Invalid corruption type!" << std::endl;
exit(1);
}
std::cout << "Saving corrupted image. " << std::endl;
noisy.save("corrupted.pgm");
// dummy variables 0 and 1 and num_rings by num_rings
std::cout << "Creating edge factor" << std::endl;
factor_t edge_factor(domain_t(variable_t(0, rings), variable_t(1, rings)));
// Set the smoothing type
if(smoothing == "square") {
edge_factor.set_as_agreement(lambda);
} else if (smoothing == "laplace") {
edge_factor.set_as_laplace(lambda);
} else {
std::cout << "Invalid smoothing stype!" << std::endl;
assert(false);
}
std::cout << edge_factor << std::endl;
std::cout << "Constructing factor graph." << std::endl;
factorized_model model;
// Add all the node factors
double sigmaSq = sigma*sigma;
for(size_t i = 0; i < noisy.rows(); ++i) {
for(size_t j = 0; j < noisy.cols(); ++j) {
// initialize the potential and belief
uint32_t pixel_id = noisy.vertid(i, j);
variable_t var(pixel_id, rings);
factor_t factor(var);
// Set the node potential
double obs = noisy.pixel(i, j);
if(corruption == "gaussian") {
for(size_t pred = 0; pred < rings; ++pred) {
factor.logP(pred) =
-(obs - pred)*(obs - pred) / (2.0 * sigmaSq);
}
} else if(corruption == "flip") {
for(size_t pred = 0; pred < rings; ++pred) {
factor.logP(pred) = obs == pred? 0 : -sigma;
}
} else if(corruption == "ising") {
// Do nothing since we want a uniform node potential
factor.uniform();
} else {
std::cout << "Invalid corruption!" << std::endl;
exit(1);
}
factor.normalize();
model.add_factor(factor);
} // end of for j in cols
} // end of for i in rows
// Construct edge_factors
for(size_t i = 0; i < noisy.rows(); ++i) {
for(size_t j = 0; j < noisy.cols(); ++j) {
size_t source = noisy.vertid(i,j);
variable_t source_var(source, rings);
if(i+1 < noisy.rows()) {
vertex_id_t target = noisy.vertid(i+1, j);
variable_t target_var(target, rings);
domain_t dom(source_var, target_var);
edge_factor.set_args(dom);
model.add_factor(edge_factor);
}
if(j+1 < noisy.cols()) {
vertex_id_t target = noisy.vertid(i, j+1);
variable_t target_var(target, rings);
domain_t dom(source_var, target_var);
edge_factor.set_args(dom);
model.add_factor(edge_factor);
}
} // end of for j in cols
} // end of for i in rows
std::cout << "Saving model in alchemy format" << std::endl;
model.save_alchemy(model_filename + ".alchemy");
return EXIT_SUCCESS;
} // end of main
int main(int argc, char** argv) {
// Initialize control plain using mpi
graphlab::mpi_tools::init(argc, argv);
graphlab::distributed_control dc;
global_logger().set_log_level(LOG_FATAL);
// Parse command line options -------------------------------------
graphlab::command_line_options
clopts("Distributed Path Finding Algorithm.");
std::string graph_dir;
std::string format = "tsv";
std::string exec_type = "synchronous";
clopts.attach_option("graph", graph_dir,
"The graph file. If none is provided "
"then a toy graph will be created");
clopts.add_positional("graph");
clopts.attach_option("engine", exec_type,
"The engine type synchronous or asynchronous");
std::string saveprefix;
clopts.attach_option("saveprefix", saveprefix,
"If set, will save the resultant pagerank to a "
"sequence of files with prefix saveprefix");
size_t n_tree = 1;
clopts.attach_option("num_tree", n_tree,
"Number of trees.");
size_t n_query = 0;
clopts.attach_option("num_query", n_query,
"Preset number of experiments.");
bool stepy = false;
clopts.attach_option("stepy", stepy,
"Regular mode or stepy mode.");
std::string input_file = "";
clopts.attach_option("input_file", input_file,
"Read from file or randomly generate (default).");
if(!clopts.parse(argc, argv)) {
dc.cout() <<
"Error in parsing command line arguments." << std::endl;
return EXIT_FAILURE;
}
// Build the graph ---------------------------------------------
dc.cout() << "\n0 Loading graph..." << std::endl;
graph_type graph(dc, clopts);
dc.cout() << "\tLoading graph in format: "<< format << std::endl;
graph.load_format(graph_dir, format);
// must call finalize before querying the graph
graph.finalize();
dc.cout() << "\t#vertices: " << graph.num_vertices() <<
"\t#edges: " << graph.num_edges() << std::endl;
// start the handler here
#ifdef TIE_FULL
#ifdef TIE_HEUR
size_t n_query_batch = 10000;
#else
size_t n_query_batch = 1000;
#endif
#else
//size_t n_query_batch = n_query * 2;
size_t n_query_batch = 50000;
#endif
#ifdef BIDIRECTIONAL_SEARCH
n_query_batch /= 2;
#endif
query_handler qh(n_tree, stepy, n_query,
&dc, &clopts, &graph,
input_file, exec_type,
saveprefix, graph_dir,
n_query_batch);
#ifndef CALC_REAL
qh.ds_query_batch();
#else
qh.real_query_serial();
#endif
// Tear-down communication layer and quit ----------------------
graphlab::mpi_tools::finalize();
return EXIT_SUCCESS;
} // End of main
int main(int argc, char** argv) {
global_logger().set_log_level(LOG_INFO);
global_logger().set_log_to_console(true);
// Parse command line options -----------------------------------------------
const std::string description =
"Creates a folder with synthetic training data";
graphlab::command_line_options clopts(description, false);
std::string output_folder = "synthetic_data";
size_t nfiles = 5;
size_t D = 20;
size_t nusers = 1000;
size_t nmovies = 10000;
size_t nvalidate = 2;
size_t npredict = 1;
double noise = 0.1;
double stdev = 2;
double alpha = 1.8;
clopts.attach_option("dir", output_folder,
"Location to create the data files");
clopts.attach_option("nfiles", nfiles,
"The number of files to generate.");
clopts.attach_option("D", D, "Number of latent dimensions.");
clopts.attach_option("nusers", nusers,
"The number of users.");
clopts.attach_option("nmovies", nmovies,
"The number of movies.");
clopts.attach_option("alpha", alpha,
"The power-law constant.");
clopts.attach_option("nvalidate", nvalidate,
"The validate ratings pers user");
clopts.attach_option("npredict", npredict,
"The predict ratings pers user");
clopts.attach_option("noise", noise,
"The standard deviation noise parameter");
clopts.attach_option("stdev", stdev,
"The standard deviation in latent factor values");
if(!clopts.parse(argc, argv)) {
std::cout << "Error in parsing command line arguments." << std::endl;
return EXIT_FAILURE;
}
std::cout << "Creating data directory: " << output_folder << std::endl;
boost::filesystem::path directory(output_folder);
if(!boost::filesystem::create_directory(output_folder)) {
logstream(LOG_ERROR)
<< "Error creating directory: " << directory << std::endl;
return EXIT_FAILURE;
}
std::cout << "Opening files:" << std::endl;
std::vector< std::ofstream* > train_files(nfiles);
std::vector< std::ofstream* > validate_files(nfiles);
std::vector< std::ofstream* > predict_files(nfiles);
for(size_t i = 0; i < nfiles; ++i) {
const std::string train_fname =
output_folder + "/graph_" + graphlab::tostr(i) + ".tsv";
train_files[i] = new std::ofstream(train_fname.c_str());
if(!train_files[i]->good()) {
logstream(LOG_ERROR)
<< "Error creating file: " << train_fname;
}
const std::string validate_fname =
output_folder + "/graph_" + graphlab::tostr(i) + ".tsv.validate";
validate_files[i] = new std::ofstream(validate_fname.c_str());
if(!validate_files[i]->good()){
logstream(LOG_ERROR)
<< "Error creating file: " << train_fname;
}
const std::string predict_fname =
output_folder + "/graph_" + graphlab::tostr(i) + ".tsv.predict";
predict_files[i] = new std::ofstream(predict_fname.c_str());
if(!predict_files[i]->good()){
logstream(LOG_ERROR)
<< "Error creating file: " << train_fname;
}
}
// Make synthetic latent factors
std::vector< vec_type > user_factors(nusers);
std::vector< vec_type > movie_factors(nmovies);
// Create a shared random number generator
graphlab::random::generator gen; gen.seed(31413);
std::cout << "Constructing latent user factors" << std::endl;
foreach(vec_type& factor, user_factors) {
factor.resize(D);
// Randomize the factor
for(size_t d = 0; d < D; ++d)
factor(d) = gen.gaussian(0, stdev);
}
int ToolRunner::Run_Capture(LPCTSTR command, LPCTSTR params, LPCTSTR dir)
{
tstring clopts(_T("\"\" "));
clopts.insert(1, command);
clopts += params;
if (!m_pWrapper->IsTextFilter())
{
tstring tempstr(clopts);
tempstr.insert(0, _T("> "));
tempstr += _T("\n");
m_pWrapper->_AddToolOutput(tempstr.c_str());
}
if (_tcslen(dir) == 0)
dir = NULL;
OSVERSIONINFO osv = {sizeof(OSVERSIONINFO), 0, 0, 0, 0, _T("")};
::GetVersionEx(&osv);
bool bWin9x = osv.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS;
SECURITY_ATTRIBUTES sa = {sizeof(SECURITY_ATTRIBUTES), 0, 0};
sa.bInheritHandle = TRUE;
sa.lpSecurityDescriptor = NULL;
SECURITY_DESCRIPTOR sd;
if (!bWin9x)
{
// On NT we can have a proper security descriptor...
::InitializeSecurityDescriptor(&sd, SECURITY_DESCRIPTOR_REVISION);
::SetSecurityDescriptorDacl(&sd, TRUE, NULL, FALSE);
sa.lpSecurityDescriptor = &sd;
}
HANDLE hWritePipe, hReadPipe;
HANDLE hStdInWrite, hStdInRead;
if (!::CreatePipe(&hReadPipe, &hWritePipe, &sa, 0))
{
CLastErrorInfo lei;
m_pWrapper->_AddToolOutput(_T("\n> Failed to create StdOut and StdErr Pipe: "));
m_pWrapper->_AddToolOutput((LPCTSTR)lei);
return lei.GetErrorCode();
}
// read handle, write handle, security attributes, number of bytes reserved for pipe - 0 default
if (!::CreatePipe(&hStdInRead, &hStdInWrite, &sa, 0))
{
CLastErrorInfo lei;
m_pWrapper->_AddToolOutput(_T("\n> Failed to create StdIn Pipe: "));
m_pWrapper->_AddToolOutput((LPCTSTR)lei);
return lei.GetErrorCode();
}
::SetHandleInformation(hReadPipe, HANDLE_FLAG_INHERIT, 0);
::SetHandleInformation(hStdInWrite, HANDLE_FLAG_INHERIT, 0);
STARTUPINFO si;
memset(&si, 0, sizeof(STARTUPINFO));
si.cb = sizeof(STARTUPINFO);
si.dwFlags = STARTF_USESHOWWINDOW | STARTF_USESTDHANDLES;
si.wShowWindow = SW_HIDE;
si.hStdInput = hStdInRead;
si.hStdOutput = hWritePipe;
si.hStdError = hWritePipe;
PROCESS_INFORMATION pi = {0, 0, 0, 0};
DWORD dwCreationFlags = CREATE_NEW_CONSOLE; // CREATE_NEW_PROCESS_GROUP;
std::vector<TCHAR> cmdbuf(clopts.length() + 1);
memcpy(&cmdbuf[0], clopts.c_str(), clopts.size() * sizeof(TCHAR));
bool bCreated = ::CreateProcess(
NULL,
&cmdbuf[0],
&sa, /*LPSECURITY_ATTRIBUTES lpProcessAttributes*/
NULL, /*LPSECURITYATTRIBUTES lpThreadAttributes*/
TRUE, /*BOOL bInheritHandles*/
dwCreationFlags, /*DWORD dwCreationFlags*/
NULL, /*LPVOID lpEnvironment*/
dir, /*LPCTSTR lpWorkingDir*/
&si, /*LPSTARTUPINFO lpStartupInfo*/
&pi /*LPPROCESS_INFORMATION lpProcessInformation*/
) != 0;
if (!bCreated)
{
::CloseHandle(hReadPipe);
::CloseHandle(hWritePipe);
::CloseHandle(hStdInRead);
::CloseHandle(hStdInWrite);
CLastErrorInfo lei;
m_pWrapper->_AddToolOutput(_T("\n> Failed to create process: "));
m_pWrapper->_AddToolOutput((LPCTSTR)lei);
return lei.GetErrorCode();
}
unsigned int dwBytesToWrite;
unsigned char* stdinbuf;
stdinbuf = m_pWrapper->GetStdIOBuffer(dwBytesToWrite);
if (!dwBytesToWrite)
stdinbuf = NULL;
DWORD dwBytesAvail, dwBytesRead, exitCode, timeDeathDetected;
dwBytesAvail = dwBytesRead = exitCode = timeDeathDetected = 0;
DWORD dwBytesWritten = 0;
bool bCompleted = false;
BYTE buffer[TOOLS_BUFFER_SIZE];
while (!bCompleted)
{
Sleep(50);
if (dwBytesToWrite > 0)
{
// We have a filter (we think), so write the filter data into the stdin
// of the process we started.
DWORD dwWrote;
::WriteFile(hStdInWrite, stdinbuf+dwBytesWritten, dwBytesToWrite, &dwWrote, NULL);
dwBytesWritten += dwWrote;
dwBytesToWrite -= dwWrote;
if(dwBytesToWrite == 0)
{
// Now close stdin so that the filter doesn't wait forever for more data.
::CloseHandle(hStdInWrite);
hStdInWrite = NULL;
}
}
//The PeekNamedPipe function copies data from a named or
// anonymous pipe into a buffer without removing it from the pipe.
if (!::PeekNamedPipe(hReadPipe, NULL, 0, NULL, &dwBytesAvail, NULL) )
{
dwBytesAvail = 0;
}
if (dwBytesAvail > 0)
{
BOOL bRead = ::ReadFile(hReadPipe, buffer, sizeof(buffer)-1, &dwBytesRead, NULL);
buffer[dwBytesRead] = NULL;
if (bRead && dwBytesRead)
{
CA2CT conv(reinterpret_cast<const char*>(buffer));
m_pWrapper->_AddToolOutput(&conv[0], -1);
}
else
{
// Couldn't read from the pipe, must be finished...
bCompleted = true;
}
}
else
{
exitCode = STILL_ACTIVE;
// No data from the process, is it still active?
::GetExitCodeProcess(pi.hProcess, &exitCode);
if (STILL_ACTIVE != exitCode)
{
if (bWin9x)
{
// If we're running on Windows 9x then we give the
// process some time to return the remainder of its data.
// We wait until a pre-set amount of time has elapsed and
// then exit.
if (timeDeathDetected == 0)
{
timeDeathDetected = ::GetTickCount();
}
else
{
///@todo Get this value from the registry...
if ((::GetTickCount() - timeDeathDetected) > 500)
{
bCompleted = true;
}
}
}
else
{
// If NT, then the process is already dead.
bCompleted = true;
}
}
}
// While we're here, we check to see if we've been told to close.
if (!GetCanRun())
{
if (WAIT_OBJECT_0 != ::WaitForSingleObject(pi.hProcess, 500))
{
// We should do this only if the GUI process is stuck and
// don't answer to a normal termination command.
// This function is dangerous: dependant DLLs don't know the process
// is terminated, and memory isn't released.
m_pWrapper->_AddToolOutput(_T("\n> Forcefully terminating process...\n"));
::TerminateProcess(pi.hProcess, 1);
}
bCompleted = true;
}
} // while (!bCompleted)
if (WAIT_OBJECT_0 != ::WaitForSingleObject(pi.hProcess, 1000))
{
m_pWrapper->_AddToolOutput(_T("\n> Process failed to respond; forcing abrupt termination..."));
::TerminateProcess(pi.hProcess, 2);
}
::GetExitCodeProcess(pi.hProcess, &exitCode);
m_RetCode = exitCode;
::CloseHandle(pi.hProcess);
::CloseHandle(pi.hThread);
::CloseHandle(hReadPipe);
::CloseHandle(hWritePipe);
::CloseHandle(hStdInRead);
if (hStdInWrite) // might already be closed
::CloseHandle(hStdInWrite);
return m_RetCode;
}
int main(int argc, char** argv) {
// Initialize control plain using mpi
// graphlab::mpi_tools::init(argc, argv);
graphlab::distributed_control dc;
global_logger().set_log_level(LOG_INFO);
// Parse command line options -----------------------------------------------
graphlab::command_line_options clopts("PageRank algorithm.");
std::string graph_dir;
std::string format = "adj";
std::string exec_type = "synchronous";
clopts.attach_option("graph", graph_dir,
"The graph file. If none is provided "
"then a toy graph will be created");
clopts.add_positional("graph");
clopts.attach_option("engine", exec_type,
"The engine type synchronous or asynchronous");
clopts.attach_option("tol", TOLERANCE,
"The permissible change at convergence.");
clopts.attach_option("format", format,
"The graph file format");
size_t powerlaw = 0;
clopts.attach_option("powerlaw", powerlaw,
"Generate a synthetic powerlaw out-degree graph. ");
clopts.attach_option("iterations", ITERATIONS,
"If set, will force the use of the synchronous engine"
"overriding any engine option set by the --engine parameter. "
"Runs complete (non-dynamic) PageRank for a fixed "
"number of iterations. Also overrides the iterations "
"option in the engine");
clopts.attach_option("use_delta", USE_DELTA_CACHE,
"Use the delta cache to reduce time in gather.");
std::string saveprefix;
clopts.attach_option("saveprefix", saveprefix,
"If set, will save the resultant pagerank to a "
"sequence of files with prefix saveprefix");
if(!clopts.parse(argc, argv)) {
dc.cout() << "Error in parsing command line arguments." << std::endl;
return EXIT_FAILURE;
}
// Enable gather caching in the engine
clopts.get_engine_args().set_option("use_cache", USE_DELTA_CACHE);
if (ITERATIONS) {
// make sure this is the synchronous engine
dc.cout() << "--iterations set. Forcing Synchronous engine, and running "
<< "for " << ITERATIONS << " iterations." << std::endl;
clopts.get_engine_args().set_option("type", "synchronous");
clopts.get_engine_args().set_option("max_iterations", ITERATIONS);
clopts.get_engine_args().set_option("sched_allv", true);
}
graphlab::timer timer;
// Build the graph ----------------------------------------------------------
graph_type graph(dc, clopts);
if(powerlaw > 0) { // make a synthetic graph
dc.cout() << "Loading synthetic Powerlaw graph." << std::endl;
graph.load_synthetic_powerlaw(powerlaw, false, 2.1, 100000000);
}
else if (graph_dir.length() > 0) { // Load the graph from a file
dc.cout() << "Loading graph in format: "<< format << std::endl;
graph.load_format(graph_dir, format);
}
else {
dc.cout() << "graph or powerlaw option must be specified" << std::endl;
clopts.print_description();
return 0;
}
double load_time = timer.current_time();
timer.start();
// must call finalize before querying the graph
graph.finalize();
double finalize_time = timer.current_time();
double ingress_time = load_time + finalize_time;
dc.cout() << "#vertices: " << graph.num_vertices()
<< " #edges:" << graph.num_edges() << std::endl;
// Initialize the vertex data
graph.transform_vertices(init_vertex);
// Running The Engine -------------------------------------------------------
graphlab::omni_engine<pagerank> engine(dc, graph, exec_type, clopts);
engine.signal_all();
timer.start();
engine.start();
const double runtime = timer.current_time();
dc.cout() << "Finished Running engine in " << runtime
<< " seconds." << std::endl;
const double total_rank = graph.map_reduce_vertices<double>(map_rank);
std::cout << "Total rank: " << total_rank << std::endl;
const double replication_factor = (double)graph.num_replicas()/graph.num_vertices();
// Save the final graph -----------------------------------------------------
if (saveprefix != "") {
graph.save(saveprefix, pagerank_writer(),
false, // do not gzip
true, // save vertices
false); // do not save edges
}
double totalpr = graph.map_reduce_vertices<double>(pagerank_sum);
std::cout << "Totalpr = " << totalpr << "\n";
if (dc.procid() == 0) {
const std::string output_filename = "/projects/sciteam/jsb/shin1/output.csv";
std::ofstream ofs;
ofs.open(output_filename.c_str(), std::ios::out | std::ios::app);
if (!ofs.is_open()) {
std::cout << "Failed to open output file.\n";
return EXIT_FAILURE;
}
std::string ingress_method = "";
clopts.get_graph_args().get_option("ingress", ingress_method);
bool topology_aware = dc.topology_aware();
double num_master2mirror_hops = (double) graph.num_master2mirror_hops() / graph.num_vertices();
double average_local_own_nverts = graph.average_num_local_own_vertices();
double variance_local_own_nverts = graph.variance_num_local_own_vertices();
// algorithm,partitioning_strategy,num_iterations,replication_factor,load_time,finalize_time,ingress_time,computation_time,total_time,topology_aware,master2mirror_hops,average_local_own_nverts,variance_local_own_nverts
ofs << "pagerank," << ingress_method << "," << ITERATIONS << "," << replication_factor << "," << load_time << "," << finalize_time << "," << ingress_time << "," << runtime << "," << (ingress_time + runtime) << "," << topology_aware << "," << num_master2mirror_hops<< "," << average_local_own_nverts<< "," << variance_local_own_nverts << std::endl;
ofs.close();
std::cout << "Topologies:\n";
for (size_t i = 0; i < dc.topologies().size(); ++i) {
std::cout << "procid: " << i << ": ";
std::vector<int> coord = dc.topologies()[i];
for (size_t j = 0; j < coord.size(); ++j) {
std::cout << coord[j] << " ";
}
std::cout << std::endl;
}
}
// Tear-down communication layer and quit -----------------------------------
// graphlab::mpi_tools::finalize();
return EXIT_SUCCESS;
} // End of main
int main(int argc, char** argv) {
std::cout << "Spectral clustering\n\n";
time_t start, end, mid;
std::vector<std::pair<std::string,time_t> > times;
time(&start);
std::string datafile;
std::string graph_analytics_dir = "../graph_analytics/";
std::string svd_dir = "../collaborative_filtering/";
std::string kmeans_dir = "./";
std::string mpi_args;
size_t num_clusters = 0;
size_t num_nearests = 30;
float sigma = 0.1;
float epsilon = 0.0;
size_t pre_kmeans_clusters = 0;
//parse command line
graphlab::command_line_options clopts(
"Spectral clustering. The input data file is provided by the "
"--data argument which is non-optional. The format of the data file is a "
"collection of lines, where each line contains a data id followed by a "
"comma or white-space separated list of numeric values representing a vector. "
"Every line must have the same number of values. The required --clusters=N "
"argument denotes the number of clusters to generate.");
clopts.attach_option("data", datafile,
"Input file. Each line holds a data id followed by a white-space "
"or comma separated numeric vector");
clopts.attach_option("clusters", num_clusters,
"The number of clusters to create");
clopts.attach_option("sigma", sigma,
"Scale parameter for Gaussian kernel");
clopts.attach_option("t-nearest", num_nearests,
"Number of nearest neighbors (=t). Will use only the t-nearest similarities "
"for each datapoint. If set at 0, will use all similarities.");
clopts.attach_option("similarity-thres", epsilon,
"Threshold to discard small similarities");
clopts.attach_option("svd-dir", svd_dir,
"Path to the directory where Graphlab svd is located");
clopts.attach_option("kmeans-dir", kmeans_dir,
"Path to the directory where Graphlab kmeans is located");
clopts.attach_option("graph-analytics-dir", graph_analytics_dir,
"Path to the directory where Graphlab eigen_vector_normalization is located");
clopts.attach_option("pre-kmeans-clusters", pre_kmeans_clusters,
"If set, will perform kmeans as a preprocess with the given cluster number.");
clopts.attach_option("mpi-args", mpi_args,
"If set, will execute mipexec with the given arguments. "
"For example, --mpi-args=\"-n [N machines] --hostfile [host file]\"");
if (!clopts.parse(argc, argv))
return EXIT_FAILURE;
if (datafile == "") {
std::cout << "--data is not optional\n";
return EXIT_FAILURE;
}
if (num_clusters == 0) {
std::cout << "--cluster is not optional\n";
return EXIT_FAILURE;
}
std::vector<std::string> remove_opts;
remove_opts.push_back("--data");
remove_opts.push_back("--svd-dir");
remove_opts.push_back("--graph-analytics-dir");
remove_opts.push_back("--kmeans-dir");
remove_opts.push_back("--clusters");
remove_opts.push_back("--sigma");
remove_opts.push_back("--similarity-thres");
remove_opts.push_back("--mpi-args");
remove_opts.push_back("--t-nearest");
remove_opts.push_back("--pre-kmeans-clusters");
std::string other_args = get_arg_str_without(argc, argv, remove_opts);
//preprocess by kmeans for fast clustering
if(pre_kmeans_clusters > 0){
if(pre_kmeans_clusters < num_clusters){
std::cout << "the number of --pre-kmeans-clusters must be bigger than the number of clusters\n";
return EXIT_FAILURE;
}
time(&mid);
if(call_kmeans_as_preprocess(mpi_args, datafile, kmeans_dir, pre_kmeans_clusters, other_args) == false)
return EXIT_FAILURE;
//modify settings
datafile = datafile + ".pre.centers";
num_nearests = 0;
time(&end);
times.push_back(std::pair<std::string, time_t>("kmeans preprocess",(end - mid)));
}
//construct graph laplacian
time(&mid);
if (call_graph_laplacian_construction(mpi_args, datafile, sigma, epsilon,
num_nearests, other_args) == false) {
return EXIT_FAILURE;
}
time(&end);
times.push_back(std::pair<std::string, time_t>("graph laplacian",(end - mid)));
//eigen value decomposition
//read number of data
size_t num_data = 0;
const std::string datanum_filename = datafile + ".datanum";
std::ifstream ifs(datanum_filename.c_str());
if (!ifs) {
std::cout << "can't read number of data." << std::endl;
return EXIT_FAILURE;
}
ifs >> num_data;
//determine the rank of Lanczos method
size_t rank = get_lanczos_rank(num_clusters, num_data);
time(&mid);
if (call_svd(mpi_args, datafile, svd_dir, num_clusters, rank, num_data,
other_args) == false) {
return EXIT_FAILURE;
}
if (call_eigen_vector_normalization(mpi_args, datafile, graph_analytics_dir,
num_clusters, rank, num_data, other_args) == false) {
return EXIT_FAILURE;
}
time(&end);
times.push_back(std::pair<std::string, time_t>("eigen decomposition",(end - mid)));
//run kmeans
time(&mid);
if (call_kmeans(mpi_args, datafile, kmeans_dir, num_clusters, other_args)
== false) {
return EXIT_FAILURE;
}
time(&end);
times.push_back(std::pair<std::string, time_t>("kmeans",(end - mid)));
//recover cluster membership if preprocess with kmeans was done
if(pre_kmeans_clusters > 0){
//remove ".pre.centers"
datafile = datafile.substr(0, datafile.size() - 12);
recover_labels(datafile);
}
time(&end);
std::cout << "computation times:\n";
for(size_t i=0;i<times.size();++i){
std::cout << "process " << i+1 << "\t" << times[i].first << "\t" << times[i].second << " sec\n";
}
std::cout << "Overall processing time of spectral clustering is " << (end - start) << " sec\n";
return EXIT_SUCCESS;
}
int main(int argc, char** argv) {
// Initialize control plain using mpi
graphlab::mpi_tools::init(argc, argv);
graphlab::distributed_control dc;
global_logger().set_log_level(LOG_INFO);
// Parse command line options -----------------------------------------------
graphlab::command_line_options
clopts("Single Source Shortest Path Algorithm.");
std::string graph_dir;
std::string format = "adj";
std::string exec_type = "synchronous";
size_t powerlaw = 0;
std::vector<unsigned int> sources;
bool max_degree_source = false;
clopts.attach_option("graph", graph_dir,
"The graph file. If none is provided "
"then a toy graph will be created");
clopts.add_positional("graph");
clopts.attach_option("format", format,
"graph format");
clopts.attach_option("source", sources,
"The source vertices");
clopts.attach_option("max_degree_source", max_degree_source,
"Add the vertex with maximum degree as a source");
clopts.add_positional("source");
clopts.attach_option("directed", DIRECTED_SSSP,
"Treat edges as directed.");
clopts.attach_option("engine", exec_type,
"The engine type synchronous or asynchronous");
clopts.attach_option("powerlaw", powerlaw,
"Generate a synthetic powerlaw out-degree graph. ");
std::string saveprefix;
clopts.attach_option("saveprefix", saveprefix,
"If set, will save the resultant pagerank to a "
"sequence of files with prefix saveprefix");
if(!clopts.parse(argc, argv)) {
dc.cout() << "Error in parsing command line arguments." << std::endl;
return EXIT_FAILURE;
}
graphlab::timer load_timer;
// Build the graph ----------------------------------------------------------
graph_type graph(dc, clopts);
if(powerlaw > 0) { // make a synthetic graph
dc.cout() << "Loading synthetic Powerlaw graph." << std::endl;
graph.load_synthetic_powerlaw(powerlaw, false, 2, 100000000);
} else if (graph_dir.length() > 0) { // Load the graph from a file
dc.cout() << "Loading graph in format: "<< format << std::endl;
graph.load_format(graph_dir, format);
} else {
dc.cout() << "graph or powerlaw option must be specified" << std::endl;
clopts.print_description();
return EXIT_FAILURE;
}
// must call finalize before querying the graph
graph.finalize();
dc.cout() << "#vertices: " << graph.num_vertices() << std::endl
<< "#edges: " << graph.num_edges() << std::endl;
double load_elapsed_secs = load_timer.current_time();
std::cout << "Load: " << load_elapsed_secs << " seconds." << std::endl;
if(sources.empty()) {
if (max_degree_source == false) {
dc.cout()
<< "No source vertex provided. Adding vertex 0 as source"
<< std::endl;
sources.push_back(0);
}
}
if (max_degree_source) {
max_deg_vertex_reducer v = graph.map_reduce_vertices<max_deg_vertex_reducer>(find_max_deg_vertex);
dc.cout()
<< "No source vertex provided. Using highest degree vertex " << v.vid << " as source."
<< std::endl;
sources.push_back(v.vid);
}
// Running The Engine -------------------------------------------------------
graphlab::omni_engine<sssp> engine(dc, graph, exec_type, clopts);
// Signal all the vertices in the source set
for(size_t i = 0; i < sources.size(); ++i) {
engine.signal(sources[i], min_distance_type(0));
}
engine.start();
const float runtime = engine.elapsed_seconds();
dc.cout() << "Finished Running engine in " << runtime
<< " seconds." << std::endl;
// Save the final graph -----------------------------------------------------
if (saveprefix != "") {
graph.save(saveprefix, shortest_path_writer(),
false, // do not gzip
true, // save vertices
false); // do not save edges
}
// Tear-down communication layer and quit -----------------------------------
graphlab::mpi_tools::finalize();
const std::string output_filename = "/home/yosub_shin_0/output.csv";
std::ofstream ofs;
ofs.open(output_filename.c_str(), std::ios::out | std::ios::app);
if (!ofs.is_open()) {
std::cout << "Failed to open output file.\n";
return EXIT_FAILURE;
}
std::string ingress_method = "";
clopts.get_graph_args().get_option("ingress", ingress_method);
// algorithm, partitioning_strategy, num_iterations, loading_time, partitioning_time, computation_time, total_time
ofs << "sssp," << ingress_method << "," << sources.size() << "," << load_elapsed_secs << ",0," << runtime << "," << (load_elapsed_secs + runtime) << std::endl;
ofs.close();
return EXIT_SUCCESS;
} // End of main
int main(int argc, char** argv) {
// Initialize control plain using mpi
graphlab::mpi_tools::init(argc, argv);
graphlab::distributed_control dc;
dc_ptr = &dc;
global_logger().set_log_level(LOG_INFO);
// Parse command line options -----------------------------------------------
graphlab::command_line_options clopts("./ldademo");
std::string lda_edges;
std::string lda_dictionary;
bool USE_SYNC=false;
// The dir of the link graph
clopts.attach_option("lda_edges", lda_edges, "The lda graph (edges). Required ");
clopts.attach_option("lda_dictionary", lda_dictionary, "The lda word dictionary. Required ");
clopts.attach_option("wordid_offset", lda::WORDID_OFFSET, "The starting id of the words in the lda input.");
clopts.attach_option("ntopics", lda::NTOPICS, "Number of topics to use");
clopts.attach_option("topklda", lda::TOPK, "Top k words in each topic for display");
clopts.attach_option("force_lock", lda::FORCE_LOCK, "force locked words");
clopts.attach_option("use_sync", USE_SYNC, "Use Synchronous LDA");
if(!clopts.parse(argc, argv)) {
dc.cout() << "Error in parsing command line arguments." << std::endl;
return EXIT_FAILURE;
}
if(!clopts.is_set("lda_edges")) {
std::cout << "LDA edge file not provided" << std::endl;
clopts.print_description();
return EXIT_FAILURE;
}
if(!clopts.is_set("lda_dictionary")) {
std::cout << "LDA dictionary file not provided" << std::endl;
clopts.print_description();
return EXIT_FAILURE;
}
lda::INITIAL_NTOPICS = NTOPICS;
// Build the lda (right) graph
// The global lda_graph points to rgraph
lda::graph_type ldagraph(dc);
load_ldagraph(ldagraph, lda_edges, lda_dictionary);
// Run lda -----------------------------------------------------------------
launch_metric_server();
graphlab::graphlab_options opts;
std::string engine_type = "sync";
if (!USE_SYNC) {
opts.get_engine_args().set_option("factorized",true);
engine_type = "async";
}
//opts2.get_engine_args().set_option("handler_intercept",true);
graphlab::omni_engine<lda::cgs_lda_vertex_program> engine(dc, ldagraph, engine_type, opts);
lda::initialize_global();
{
const bool success =
engine.add_vertex_aggregator<lda::topk_aggregator>
("topk", lda::topk_aggregator::map, lda::topk_aggregator::finalize) &&
engine.aggregate_periodic("topk", 1);
ASSERT_TRUE(success);
}
{ // Add the Global counts aggregator
const bool success =
engine.add_vertex_aggregator<lda::factor_type>
("global_counts",
lda::global_counts_aggregator::map,
lda::global_counts_aggregator::finalize) &&
engine.aggregate_periodic("global_counts", 1);
ASSERT_TRUE(success);
}
fn_run_lda(engine);
// ------------FINALIZE------------------
// --------------------------------------
graphlab::stop_metric_server_on_eof();
graphlab::mpi_tools::finalize();
return EXIT_SUCCESS;
}
