int main(int argc, char *argv[])
{
unsigned long threads, chunk_size;
if(argc < 3) {
std::cerr << "Usage " << argv[0] << " chunk_size threads" << std::endl;
exit(-1);
}
else {
chunk_size = atol(argv[1]);
threads = atol(argv[2]);
}
boost::thread ** thread_array = new boost::thread * [threads];
mem_speed ** obj = new mem_speed *[threads];
boost::barrier * thread_sync = new boost::barrier(threads);
unsigned long alloc_bytes =
((BUFFER_SIZE + chunk_size - 1) /chunk_size)*chunk_size;
for(unsigned long i=0;i<threads;i++) {
unsigned char *buffer1 = (unsigned char *)
map_anon_memory(alloc_bytes, true, "buffer");
unsigned char *buffer2 = (unsigned char *)
map_anon_memory(alloc_bytes, true, "buffer");
obj[i] = new mem_speed(chunk_size, thread_sync, buffer1, buffer2);
thread_array[i] = new boost::thread
(boost::ref(*obj[i]));
}
for(unsigned long i=0;i<threads;i++) {
thread_array[i]->join();
}
return 0;
}
int main(int argc, const char **argv)
{
slipstore::io *streams;
rtc_clock prefill_clock;
rtc_clock drain_clock;
/* Get options */
setup_options(argc, argv);
slipchunk = vm["slipchunk"].as<unsigned long>();
BOOST_LOG_TRIVIAL(info) << "SLIPBENCH::SLIPCHUNK "
<< slipchunk;
unsigned long slipfile_size = vm["slipbench_fsize"].as<unsigned long>();
unsigned long blockmem = vm["blocked_memory"].as<unsigned long>();
unsigned long prefill = vm["test_prefill"].as<unsigned long>();
/* Read in slipstore information */
init_slipstore_desc();
unsigned long fanout = vm["ext_fanout"].as<unsigned long>();
BOOST_LOG_TRIVIAL(info) << "SLIPBENCH::FANOUT_UNDER_TEST "
<< fanout;
streams = new slipstore::io(1, 1, fanout, NULL);
slipstore::init(streams, 100, 1); // tilesize does not matter
unsigned char *buffer = new unsigned char[slipchunk];
choice = new slipstore::cyclic(fanout,
slipstore::slipstore_client_drain->get_me());
if(blockmem > 0) {
BOOST_LOG_TRIVIAL(info) << "Blocking " << blockmem;
(void)map_anon_memory(blockmem, true, "Blockmem");
BOOST_LOG_TRIVIAL(info) << "Done";
}
unsigned long drain_bytes = slipfile_size;
if(drain_bytes < 2*fanout*slipchunk) {
drain_bytes = 2*fanout*slipchunk;
}
BOOST_LOG_TRIVIAL(info) << "SLIPBENCH::PREFILL "
<< prefill;
BOOST_LOG_TRIVIAL(info) << "SLIPBENCH::DRAIN_AMOUNT "
<< drain_bytes;
/////////// Prefill
control_barrier(); // everyone started
prefill_clock.start();
drain_test(buffer, prefill);
prefill_clock.stop();
control_barrier(); // everyone done
prefill_clock.print("SLIPBENCH::PREFILL_TIME ");
////////// Drain test
control_barrier(); // everyone started
drain_clock.start();
drain_test(buffer, drain_bytes);
control_barrier(); // everyone done
// Flush the remaining blocks
for(unsigned long i=0;i<fanout;i++) {
streams->rewind(slipstore::STREAM_VERTEX_STATE, 0, i);
}
drain_clock.stop();
drain_clock.print("SLIPBENCH::DRAIN_TIME ");
///////////////////////
slipstore::shutdown();
delete buffer;
delete streams;
}
disk_io(unsigned long stream_unit_in, ioq *wq_in)
:stream_unit(stream_unit_in),
wq(wq_in)
{
bounce_buffer = (unsigned char *)
map_anon_memory(stream_unit, true, "Bounce buffer");
}
int main(int argc, char *argv[]) {
rtc_clock wall_clock;
desc.add_options()
("help,h", "Produce help message")
("algorithm,a",
boost::program_options::value<std::string>()->default_value("quick_sort"),
"Sort algorithm: quick_sort (default), counting_sort")
("source,i", boost::program_options::value<std::string>()->required(),
"Name of the input to sort (need input.ini)");
try {
boost::program_options::store(boost::program_options::parse_command_line(argc,
argv,
desc),
vm);
boost::program_options::notify(vm);
}
catch (boost::program_options::error &e) {
if (vm.count("help") || argc == 1) {
std::cerr << desc << "\n";
}
std::cerr << "Error:" << e.what() << std::endl;
std::cerr << "Try: " << argv[0] << " --help" << std::endl;
exit(-1);
}
init_graph_desc(vm["source"].as<std::string>());
unsigned char *stream;
unsigned long items, record_size;
std::stringstream fname;
fname << pt.get<std::string>("graph.name");
items = pt.get < unsigned
long > ("graph.edges");
int fd_edges = open(fname.str().c_str(), O_RDONLY | O_LARGEFILE);
if (fd_edges == -1) {
BOOST_LOG_TRIVIAL(fatal) << "Unable to open edges file " <<
fname.str() << ":" << strerror(errno);
exit(-1);
}
if (pt.get < unsigned
long > ("graph.type") == 1) {
record_size = format::type1::format_utils::split_size_bytes();
stream = (unsigned char *)
map_anon_memory(items * record_size,
true, "Sort buffer");
read_from_file(fd_edges, stream, items * record_size);
close(fd_edges);
}
else {
BOOST_LOG_TRIVIAL(fatal) << "Unknown file type";
exit(-1);
}
if (vm["algorithm"].as<std::string>() == "quick_sort") {
wall_clock.start();
qsort(stream, items,
record_size, compare<format::type1::format_utils>);
wall_clock.stop();
}
else if (vm["algorithm"].as<std::string>() == "counting_sort") {
wall_clock.start();
unsigned long keys = pt.get < unsigned
long > ("graph.vertices");
counting_sort<format::type1::format_utils>
(stream,
(unsigned char *) map_anon_memory(items * record_size,
true, "Output buffer"),
items,
(unsigned long *) map_anon_memory(keys * sizeof(unsigned long),
true, "Index"),
keys);
wall_clock.stop();
}
else {
BOOST_LOG_TRIVIAL(fatal) << "Don't know how to do " <<
vm["algorithm"].as<std::string>();
exit(-1);
}
wall_clock.print("TIME WALL ");
return 0;
}
int main(int argc, const char **argv) {
slipstore::io *streams;
rtc_clock fill_clock;
rtc_clock drain_clock;
slipstore::slipstore_req_t req;
bool silent;
/* Get options */
setup_options(argc, argv);
unsigned long slipchunk = vm["slipchunk"].as < unsigned
long > ();
BOOST_LOG_TRIVIAL(info) << "SLIPBENCH::SLIPCHUNK "
<< slipchunk;
unsigned long slipfile_size = vm["slipbench_fsize"].as < unsigned
long > ();
BOOST_LOG_TRIVIAL(info) << "SLIPBENCH::SLIPBENCH_FIZE "
<< slipfile_size;
unsigned long background_threads =
vm["slipbench_background_threads"].as < unsigned
long > ();
unsigned long align = vm["slipbench_align"].as < unsigned
long > ();
slipchunk = (slipchunk / align) * align;
unsigned long blockmem = vm["blocked_memory"].as < unsigned
long > ();
/* Read in slipstore information */
init_slipstore_desc();
bool run_drain_test = (vm.count("drain_test") > 0);
bool run_fill_test = (vm.count("fill_test") > 0);
streams = new slipstore::io(1, 1, 1, NULL);
slipstore::init(streams, 100, 1); // tilesize does not matter
unsigned char *buffer = new unsigned char[slipchunk];
unsigned long fill_bytes = 0;
unsigned char *big_buffer;
unsigned long big_bufsize = 512 * 1024 * 1024;
if (blockmem > 0) {
BOOST_LOG_TRIVIAL(info) << "Blocking " << blockmem;
}
else {
blockmem = big_bufsize;
BOOST_LOG_TRIVIAL(info) << "Allocating " << blockmem;
}
if (blockmem < big_bufsize) {
blockmem = big_bufsize;
}
big_buffer = (unsigned char *) map_anon_memory(blockmem, true, "Blockmem");
BOOST_LOG_TRIVIAL(info) << "Done ";
bool centralized = (vm.count("centralized") > 0);
bool batching = (vm.count("request_batching") > 0);
unsigned long silenced_clients = vm["slipbench_silence"].as < unsigned
long > ();
if (silenced_clients != ULONG_MAX &&
slipstore::slipstore_client_drain->get_me() < silenced_clients) {
silent = true;
}
else {
silent = false;
}
if (background_threads > 0) {
BOOST_LOG_TRIVIAL(info) << "Starting background threads";
for (unsigned long i = 0; i < background_threads; i++) {
(void) new boost::thread(boost::ref(*new interference()));
}
while (bthread_running != background_threads);
BOOST_LOG_TRIVIAL(info) << "done";
}
control_barrier(); // everyone started
if (run_drain_test) {
if (!silent) {
BOOST_LOG_TRIVIAL(info) << "Starting drain test";
drain_clock.start();
unsigned long drain_bytes = slipfile_size;
if (batching) {
while (drain_bytes) {
req.cmd = slipstore::CMD_DRAIN;
req.stream = slipstore::STREAM_VERTEX_STATE;
req.partition = 0;
req.tile = 0;
unsigned long bytes_to_drain = drain_bytes;
if (bytes_to_drain > big_bufsize) {
bytes_to_drain = big_bufsize;
}
if (slipstore::slipstore_client_drain->batch_access_store(&req,
big_buffer,
slipchunk,
bytes_to_drain)) {
BOOST_LOG_TRIVIAL(fatal) << "Unable to write to slipstore";
exit(-1);
}
drain_bytes -= bytes_to_drain;
}
}
else {
while (drain_bytes) {
unsigned long bytes;
bytes = (drain_bytes > slipchunk) ?
slipchunk : drain_bytes;
if (centralized) {
req.cmd = slipstore::CMD_NOP;
req.stream = slipstore::STREAM_VERTEX_STATE;
req.partition = 0;
req.tile = 0;
req.size = 0;
if (!slipstore::slipstore_client_drain->access_store(&req, buffer, 0)) {
BOOST_LOG_TRIVIAL(fatal) << "Unable to write to slipstore";
exit(-1);
}
}
req.cmd = slipstore::CMD_DRAIN;
req.stream = slipstore::STREAM_VERTEX_STATE;
req.partition = 0;
req.tile = 0;
req.size = bytes;
if (!slipstore::slipstore_client_drain->access_store(&req, buffer)) {
BOOST_LOG_TRIVIAL(fatal) << "Unable to write to slipstore";
exit(-1);
}
drain_bytes -= bytes;
}
}
drain_clock.stop();
BOOST_LOG_TRIVIAL(info) << "Completed drain test";
}
else {
BOOST_LOG_TRIVIAL(info) << "SILENCED !";
}
if (run_fill_test) {
control_barrier();
streams->rewind(0, 0, 0);
control_barrier();
}
}
if (run_fill_test) {
streams->fill_test_prep(slipstore::STREAM_VERTEX_STATE);
control_barrier();
if (!silent) {
BOOST_LOG_TRIVIAL(info) << "Starting fill test";
fill_clock.start();
if (batching) {
unsigned long filled_bytes;
do {
req.cmd = slipstore::CMD_FILL;
req.stream = slipstore::STREAM_VERTEX_STATE;
req.partition = 0;
req.tile = 0;
filled_bytes =
slipstore::slipstore_client_fill->batch_access_store(&req,
big_buffer,
slipchunk,
big_bufsize);
fill_bytes += filled_bytes;
} while (filled_bytes > 0);
}
else {
do {
if (centralized) {
req.cmd = slipstore::CMD_NOP;
req.stream = slipstore::STREAM_VERTEX_STATE;
req.partition = 0;
req.tile = 0;
req.size = 0;
if (!slipstore::slipstore_client_drain->access_store(&req, buffer, 0)) {
BOOST_LOG_TRIVIAL(fatal) << "Unable to write to slipstore";
exit(-1);
}
}
req.cmd = slipstore::CMD_FILL;
req.stream = slipstore::STREAM_VERTEX_STATE;
req.partition = 0;
req.tile = 0;
req.size = slipchunk;
if (!slipstore::slipstore_client_fill->access_store(&req, buffer)) {
break;
}
fill_bytes += req.size;
} while (true);
}
fill_clock.stop();
BOOST_LOG_TRIVIAL(info) << "Completed fill test";
}
else {
BOOST_LOG_TRIVIAL(info) << "SILENCED !";
}
}
control_barrier(); // everyone done
slipstore::shutdown();
delete buffer;
delete streams;
BOOST_LOG_TRIVIAL(info) << "SLIPBENCH::DRAIN_BYTES "
<< slipfile_size;
drain_clock.print("SLIPBENCH::DRAIN_TIME ");
BOOST_LOG_TRIVIAL(info) << "SLIPBENCH::FILL_BYTES "
<< fill_bytes;
fill_clock.print("SLIPBENCH::FILL_TIME ");
}
interference() {
buffer = (unsigned char *) map_anon_memory(512 * 1024 * 1024, true, "Dummy buf");
}
int main(int argc, const char* argv[])
{
/* Parse the cmd line */
setup_options(argc, argv);
BOOST_LOG_TRIVIAL(info) << "STARTUP";
BOOST_LOG_TRIVIAL(info) << "CORE::GRAPH " << vm["graph"].as<std::string>();
BOOST_LOG_TRIVIAL(info) << "CORE::ALGORITHM " << vm["benchmark"].as<std::string>();
/* Read in key graph information */
init_graph_desc(vm["graph"].as<std::string>());
unsigned long blocked_memory = vm["blocked_memory"].as<unsigned long>();
if(blocked_memory > 0) {
BOOST_LOG_TRIVIAL(info) << clock::timestamp()
<< " Blocking memory amount "
<< blocked_memory;
(void)map_anon_memory(blocked_memory, true, "Blockmem");
BOOST_LOG_TRIVIAL(info) << clock::timestamp()
<< " Done blocking memory";
}
/* Only certain combinations of algorithms and formats
* are supported. This is encoded below.
* Algorithms that require the edge value cannot use format type2
*/
if(false);
ADD_SG_ALGORITHM(bfs, algorithm::sg_simple::bfs, 1, false);
ADD_SG_ALGORITHM(bfs, algorithm::sg_simple::bfs, 2, false);
ADD_SGASYNC_ALGORITHM(bfs_async, algorithm::sg_simple::bfs_async, 1, false);
ADD_SGASYNC_ALGORITHM(bfs_async, algorithm::sg_simple::bfs_async, 2, false);
ADD_SG_ALGORITHM(bfs_forest, algorithm::sg_simple::bfs_forest, 1, false);
ADD_SG_ALGORITHM(bfs_forest, algorithm::sg_simple::bfs_forest, 2, false);
ADD_SG_ALGORITHM(degree_cnt, algorithm::sg_simple::degree_cnt, 1, false);
ADD_SG_ALGORITHM(degree_cnt, algorithm::sg_simple::degree_cnt, 2, false);
ADD_SG_ALGORITHM(conductance, algorithm::sg_simple::conductance, 1, false);
ADD_SG_ALGORITHM(conductance, algorithm::sg_simple::conductance, 2, false);
ADD_SG_ALGORITHM(range_check, algorithm::sg_simple::range_check, 1, false);
ADD_SG_ALGORITHM(range_check, algorithm::sg_simple::range_check, 2, false);
ADD_ALGORITHM(reverse, algorithm::reverse::reverse, 1, false);
ADD_ALGORITHM(reverse, algorithm::reverse::reverse, 2, false);
ADD_SG_ALGORITHM(spmv, algorithm::sg_simple::spmv, 1, false);
ADD_SG_ALGORITHM(sssp, algorithm::sg_simple::sssp, 1, false);
ADD_SG_ALGORITHM(sssp_forest, algorithm::sg_simple::sssp_forest, 1, false);
ADD_SG_ALGORITHM(cc, algorithm::sg_simple::cc, 1, false);
ADD_SG_ALGORITHM(cc, algorithm::sg_simple::cc, 2, false);
ADD_ALGORITHM(cc_online, algorithm::cc_online_ns::cc_online, 1, false);
ADD_ALGORITHM(cc_online, algorithm::cc_online_ns::cc_online, 2, false);
// Uncomment for ALS
//ADD_SG_ALGORITHM(als, algorithm::sg_simple::als_factorization, 1, false);
//ADD_SGASYNC_ALGORITHM(als_async, algorithm::sg_simple::als_async_factorization, 1, false);
//
ADD_SG_ALGORITHM(checkbip, algorithm::sg_simple::check_if_bipartite, 1, false);
ADD_SG_ALGORITHM(mis, algorithm::sg_simple::mis, 1, false);
ADD_SG_ALGORITHM(mis, algorithm::sg_simple::mis, 2, false);
ADD_SG_ALGORITHM(pagerank, algorithm::sg_simple::pagerank, 1, false);
ADD_SG_ALGORITHM(pagerank, algorithm::sg_simple::pagerank, 2, false);
ADD_SG_ALGORITHM(pagerank_ddf, algorithm::sg_simple::pagerank_ddf, 1, false);
ADD_SG_ALGORITHM(pagerank_ddf, algorithm::sg_simple::pagerank_ddf, 2, false);
ADD_ALGORITHM(belief_propagation, algorithm::belief_prop::standard::belief_propagation, 1, false);
ADD_ALGORITHM(belief_propagation_graphchi, algorithm::belief_prop::graphchi::belief_propagation_graphchi, 1, false);
ADD_ALGORITHM(belief_propagation_graphchi, algorithm::belief_prop::graphchi::belief_propagation_graphchi, 2, false);
ADD_ALGORITHM(mcst, algorithm::mcst::mcst, 1, false);
ADD_SG_ALGORITHM(hyperanf, algorithm::sg_simple::hyperanf, 1, false);
ADD_SG_ALGORITHM(hyperanf, algorithm::sg_simple::hyperanf, 2, false);
ADD_ALGORITHM(scc, algorithm::scc::scc, 1, false);
ADD_ALGORITHM(scc, algorithm::scc::scc, 2, false);
ADD_ALGORITHM(triangle_counting, algorithm::triangle_counting::triangle_counting, 1, false);
ADD_ALGORITHM(triangle_counting, algorithm::triangle_counting::triangle_counting, 2, false);
ADD_ALGORITHM(bc, algorithm::bc::bc, 1, false);
else {
