void run_threaded_string_sends_0(size_t length, size_t numthreads) {
if (rmi.procid() == 1) {
rmi.full_barrier();
return;
}
timer ti;
std::cout << numthreads << " threaded " << SEND_LIMIT_PRINT <<" sends, "
<< length << " bytes\n";
ti.start();
size_t numsends = SEND_LIMIT / (length * numthreads);
size_t rd = rdtsc();
thread_group thrgrp;
for (size_t i = 0; i < numthreads; ++i) {
thrgrp.launch(boost::bind(&teststruct::perform_string_sends_0, this, length, numsends));
}
thrgrp.join();
size_t rd2 = rdtsc();
std::cout << (rd2 - rd) / (numthreads * numsends) << " cycles per call\n";
double t1 = ti.current_time();
rmi.dc().flush();
double t2 = ti.current_time();
rmi.full_barrier();
double t3 = ti.current_time();
print_res(t1,t2,t3);
}
void task(size_t i) {
if (i < 5) std::cout << "Task " << i << std::endl;
if (i > 0) {
if (rmi.numprocs() == 1) {
add_task_local(i - 1);
}
else {
rmi.remote_call((procid_t)((rmi.procid() + 1) % rmi.numprocs()),
&simple_engine_test::add_task_local,
i - 1);
}
}
}
std::vector<std::vector<size_t> >
get_procs_with_keys(const std::vector<size_t>& local_key_list, Graph& g) {
// this machine will get all keys from each processor where
// key = procid mod numprocs
std::vector<std::vector<size_t> > procs_with_keys(rmi.numprocs());
for (size_t i = 0; i < local_key_list.size(); ++i) {
if (g.l_vertex(i).owned() && local_key_list[i] != (size_t)(-1)) {
procid_t target_procid = local_key_list[i] % rmi.numprocs();
procs_with_keys[target_procid].push_back(local_key_list[i]);
}
}
rmi.all_to_all(procs_with_keys);
return procs_with_keys;
}
/**
* Sets the newval to be the value associated with the key
*/
void set(const KeyType &key, const ValueType &newval) {
// who owns the data?
const size_t hashvalue = hasher(key);
const size_t owningmachine = hashvalue % rpc.numprocs();
// if it is me, set it
if (owningmachine == rpc.dc().procid()) {
lock.lock();
storage[hashvalue] = newval;
lock.unlock();
} else {
rpc.remote_call(owningmachine,
&dht<KeyType,ValueType>::set,
key, newval);
}
}
void prepare_injective_join(LeftEmitKey left_emit_key,
RightEmitKey right_emit_key) {
typedef std::pair<size_t, vertex_id_type> key_vertex_pair;
// Basically, what we are trying to do is to figure out, for each vertex
// on one side of the graph, which vertices for the other graph
// (and on on which machines) emitted the same key.
//
// The target datastructure is:
// vtx_to_key[vtx]: The key for each vertex
// opposing_join_proc[vtx]: Machines which hold a vertex on the opposing
// graph which emitted the same key
// key_to_vtx[key] Mapping of keys to vertices.
// resize the left index
// resize the right index
reset_and_fill_injective_index(left_inj_index,
left_graph,
left_emit_key, "left graph");
reset_and_fill_injective_index(right_inj_index,
right_graph,
right_emit_key, "right graph");
rmi.barrier();
// now, we need cross join across all machines to figure out the
// opposing join proc
// we need to do this twice. Once for left, and once for right.
compute_injective_join();
}
void run_short_sends_0() {
if (rmi.procid() == 1) {
rmi.full_barrier();
return;
}
timer ti;
std::cout << "Single Threaded " << SEND_LIMIT_PRINT << " sends, 4 integer blocks\n";
ti.start();
size_t numsends = SEND_LIMIT / (sizeof(size_t) * 4);
perform_short_sends_0(numsends);
double t1 = ti.current_time();
rmi.dc().flush();
double t2 = ti.current_time();
rmi.full_barrier();
double t3 = ti.current_time();
print_res(t1,t2,t3);
}
void injective_join(injective_join_index& target,
TargetGraph& target_graph,
injective_join_index& source,
SourceGraph& source_graph,
JoinOp joinop) {
// build up the exchange structure.
// move right vertex data to left
std::vector<
std::vector<
std::pair<size_t, typename SourceGraph::vertex_data_type> > >
source_data(rmi.numprocs());
for (size_t i = 0; i < source.opposing_join_proc.size(); ++i) {
if (source_graph.l_vertex(i).owned()) {
procid_t target_proc = source.opposing_join_proc[i];
if (target_proc >= 0 && target_proc < rmi.numprocs()) {
source_data[target_proc].push_back(
std::make_pair(source.vtx_to_key[i],
source_graph.l_vertex(i).data()));
}
}
}
// exchange
rmi.all_to_all(source_data);
// ok. now join against left
#ifdef _OPENMP
#pragma omp parallel for
#endif
for (size_t p = 0;p < source_data.size(); ++p) {
for (size_t i = 0;i < source_data[p].size(); ++i) {
// find the target vertex with the matching key
hopscotch_map<size_t, vertex_id_type>::const_iterator iter =
target.key_to_vtx.find(source_data[p][i].first);
ASSERT_TRUE(iter != target.key_to_vtx.end());
// found it!
typename TargetGraph::local_vertex_type
lvtx = target_graph.l_vertex(iter->second);
typename TargetGraph::vertex_type vtx(lvtx);
joinop(vtx, source_data[p][i].second);
}
}
target_graph.synchronize();
}
/**
* gets the value associated with a key.
* Returns (true, Value) if the entry is available.
* Returns (false, undefined) otherwise.
*/
std::pair<bool, ValueType> get(const KeyType &key) const {
// who owns the data?
const size_t hashvalue = hasher(key);
const size_t owningmachine = hashvalue % rpc.numprocs();
std::pair<bool, ValueType> retval;
// if it is me, we can return it
if (owningmachine == rpc.dc().procid()) {
lock.lock();
typename storage_type::const_iterator iter = storage.find(hashvalue);
retval.first = iter != storage.end();
if (retval.first) retval.second = iter->second;
lock.unlock();
} else {
retval = rpc.remote_request(owningmachine,
&dht<KeyType,ValueType>::get,
key);
}
return retval;
}
void run_threaded_short_pod_sends_0(size_t numthreads) {
if (rmi.procid() == 1) {
rmi.full_barrier();
return;
}
timer ti;
std::cout << numthreads << " threaded "<< SEND_LIMIT_PRINT <<" POD sends, 4 integers\n";
size_t numsends = SEND_LIMIT / (sizeof(size_t) * 4 * numthreads);
ti.start();
thread_group thrgrp;
for (size_t i = 0; i < numthreads; ++i) {
thrgrp.launch(boost::bind(&teststruct::perform_short_pod_sends_0, this, numsends));
}
thrgrp.join();
double t1 = ti.current_time();
rmi.dc().flush();
double t2 = ti.current_time();
rmi.full_barrier();
double t3 = ti.current_time();
print_res(t1,t2,t3);
}
void run_string_sends_0(size_t length) {
if (rmi.procid() == 1) {
rmi.full_barrier();
return;
}
timer ti;
size_t numsends = SEND_LIMIT / (length);
std::cout << "Single Threaded " << SEND_LIMIT_PRINT <<" sends, " << length << " bytes * "<< numsends << "\n";
ti.start();
size_t rd = rdtsc();
perform_string_sends_0(length, numsends);
size_t rd2 = rdtsc();
std::cout << "Completed in: " << ti.current_time() << " seconds\n";
std::cout << (rd2 - rd) / numsends << " cycles per call\n";
double t1 = ti.current_time();
rmi.dc().flush();
std::cout << "Flush in: " << ti.current_time() << " seconds\n";
double t2 = ti.current_time();
rmi.full_barrier();
std::cout << "Receive Complete in: " << ti.current_time() << " seconds\n";
double t3 = ti.current_time();
print_res(t1,t2,t3);
}
void run_synchronous(MemberFunction member_fun, const vertex_set& vset) {
shared_lvid_counter = 0;
if (threads.size() <= 1) {
(this->*(member_fun))(0, vset);
}
else {
// launch the initialization threads
for(size_t i = 0; i < threads.size(); ++i) {
boost::function<void(void)> invoke = boost::bind(member_fun, this, i, vset);
threads.launch(invoke, i);
}
}
// Wait for all threads to finish
threads.join();
rmi.barrier();
} // end of run_synchronous
inline void all_reduce(U& data, bool control = false) {
rmi.all_reduce(data, control);
}
void compute_injective_join() {
std::vector<std::vector<size_t> > left_keys =
get_procs_with_keys(left_inj_index.vtx_to_key, left_graph);
std::vector<std::vector<size_t> > right_keys =
get_procs_with_keys(right_inj_index.vtx_to_key, right_graph);
// now. for each key on the right, I need to figure out which proc it
// belongs in. and vice versa. This is actually kind of annoying.
// but since it is one-to-one, I only need to make a hash map of one side.
hopscotch_map<size_t, procid_t> left_key_to_procs;
// construct a hash table of keys to procs
// clear frequently to use less memory
for (size_t p = 0; p < left_keys.size(); ++p) {
for (size_t i = 0; i < left_keys[p].size(); ++i) {
ASSERT_MSG(left_key_to_procs.count(left_keys[p][i]) == 0,
"Duplicate keys not permitted for left graph keys in injective join");
left_key_to_procs.insert(std::make_pair(left_keys[p][i], p));
}
std::vector<size_t>().swap(left_keys[p]);
}
left_keys.clear();
std::vector<
std::vector<
std::pair<size_t, procid_t> > > left_match(rmi.numprocs());
std::vector<
std::vector<
std::pair<size_t, procid_t> > > right_match(rmi.numprocs());
// now for each key on the right, find the matching key on the left
for (size_t p = 0; p < right_keys.size(); ++p) {
for (size_t i = 0; i < right_keys[p].size(); ++i) {
size_t key = right_keys[p][i];
hopscotch_map<size_t, procid_t>::iterator iter =
left_key_to_procs.find(key);
if (iter != left_key_to_procs.end()) {
ASSERT_MSG(iter->second != (procid_t)(-1),
"Duplicate keys not permitted for right graph keys in injective join");
// we have a match
procid_t left_proc = iter->second;
procid_t right_proc = p;
// now. left has to be told about right and right
// has to be told about left
left_match[left_proc].push_back(std::make_pair(key, right_proc));
right_match[right_proc].push_back(std::make_pair(key, left_proc));
// set the map entry to -1
// so we know if it is ever reused
iter->second = (procid_t)(-1);
}
}
std::vector<size_t>().swap(right_keys[p]);
}
right_keys.clear();
rmi.all_to_all(left_match);
rmi.all_to_all(right_match);
// fill in the index
// go through the left match and set up the opposing index to based
// on the match result
#ifdef _OPENMP
#pragma omp parallel for
#endif
for (size_t p = 0;p < left_match.size(); ++p) {
for (size_t i = 0;i < left_match[p].size(); ++i) {
// search for the key in the left index
hopscotch_map<size_t, vertex_id_type>::const_iterator iter =
left_inj_index.key_to_vtx.find(left_match[p][i].first);
ASSERT_TRUE(iter != left_inj_index.key_to_vtx.end());
// fill in the match
left_inj_index.opposing_join_proc[iter->second] = left_match[p][i].second;
}
}
left_match.clear();
// repeat for the right match
#ifdef _OPENMP
#pragma omp parallel for
#endif
for (size_t p = 0;p < right_match.size(); ++p) {
for (size_t i = 0;i < right_match[p].size(); ++i) {
// search for the key in the right index
hopscotch_map<size_t, vertex_id_type>::const_iterator iter =
right_inj_index.key_to_vtx.find(right_match[p][i].first);
ASSERT_TRUE(iter != right_inj_index.key_to_vtx.end());
// fill in the match
right_inj_index.opposing_join_proc[iter->second] = right_match[p][i].second;
}
}
right_match.clear();
// ok done.
}
/// \copydoc distributed_control::full_barrier()
inline void full_barrier() {
rmi.full_barrier();
}
/**
Must be called by all machines simultaneously
*/
void clear() {
rpc.barrier();
storage.clear();
}
void all_reduce2(U& data, PlusEqual plusequal, bool control = false) {
rmi.all_reduce2(data, plusequal, control);
}
inline void broadcast(U& data, bool originator, bool control = false) {
rmi.broadcast(data, originator, control);
}
inline void recv_from(procid_t source, U& t, bool control = false) {
rmi.recv_from(source, t, control);
}
inline void send_to(procid_t target, U& t, bool control = false) {
rmi.send_to(target, t, control);
}
void print_stats() const {
std::cerr << rpc.calls_sent() << " calls sent\n";
std::cerr << rpc.calls_received() << " calls received\n";
}
inline void all_gather(std::vector<U>& data, bool control = false) {
rmi.all_gather(data, control);
}
/**
* Get the owner of the key
*/
procid_t owner(const KeyType& key) const {
return hasher(key) % rpc.dc().numprocs();
}
void perform_string_sends_0(size_t length, size_t number) {
std::string s(length, 1);
for (size_t i = 0;i < number; ++i) {
rmi.remote_call(1, &teststruct::receive_string, s);
}
}
void perform_short_pod_sends_0(size_t number) {
for (size_t i = 0;i < number; ++i) {
rmi.pod_call(1, &teststruct::receive_ints, 100,100,1000,5000000);
}
}
/// \copydoc distributed_control::barrier()
inline void barrier() {
rmi.barrier();
}
void perform_long_sends_0(size_t length, size_t number) {
std::vector<size_t> v(length, 5000000);
for (size_t i = 0;i < number; ++i) {
rmi.remote_call(1, &teststruct::receive_vector, v);
}
}
inline void gather(std::vector<U>& data, procid_t sendto, bool control = false) {
rmi.gather(data, sendto, control);
}
