void distributed_control::close_all_connections() {
mpi_barrier();
logger(LOG_INFO, "Closing sockets");
done = 1;
mpi_barrier();
// wake all receivers up everyone up;
for (procid_t j = 0 ;j < numprocs(); ++j) {
if (j != procid()) {
char *a = (char*)malloc(16);
memset(a, 0, 16);
send_to_sock(j, a, 1);
}
}
procthreads.join();
mpi_barrier();
// socket closing order.to avoid TIME_WAITs
// close initiators first
for (procid_t j = procid()+1;j < numprocs(); ++j) {
close(socks[j]);
}
mpi_barrier();
// then close everything else
for (procid_t j = 0 ;j < procid(); ++j) {
close(socks[j]);
}
close(listensock);
// wake up all threads
procthreads.signalall(SIGURG);
logger(LOG_INFO, "All sockets closed");
}
void distributed_control::connect_udt() {
open_listening_udt();
socks.resize(numprocs());
socks[procid()] = 0;
mpi_barrier();
// this is a little tricky. both connect() and accept() blocks.
// so I have to do this in phases
for (procid_t i = 0;i < numprocs(); ++i) {
if (procid() == i) {
// my turn to connect to everyone. I only need to connect to those with
// id above me
for (procid_t j = i+1;j < numprocs(); ++j) {
sockfd_t newsock = socket(AF_INET, SOCK_STREAM, 0);
sockaddr_in serv_addr;
serv_addr.sin_family = AF_INET;
// set the target port
serv_addr.sin_port = htons(DC_LOCAL_BASE_PORT_NUM + j);
// set the target address
serv_addr.sin_addr = *(struct in_addr*)&(all_addrs[j]);
memset(&(serv_addr.sin_zero), '\0', 8);
// Connect!
logstream(LOG_INFO) << "Trying to connect from "
<< i << " -> " << j
<< " on port " << DC_LOCAL_BASE_PORT_NUM + j << "\n";
logger(LOG_INFO, "Destination IP = %s", inet_ntoa(serv_addr.sin_addr));
if (connect(newsock, (sockaddr*)&serv_addr, sizeof(serv_addr)) < 0) {
logstream(LOG_FATAL) << "connect " << i << " to " << j << ": "
<< strerror(errno) << "\n";
ASSERT_TRUE(false);
}
logstream(LOG_INFO) << "Sent connection " << i << " -> " << j << "\n";
// store in the socks table
set_socket_options(newsock);
socks[j] = newsock;
}
}
else if (procid() > i) {
logstream(LOG_INFO) << "Proc " << procid() << " Waiting to accept\n";
sockaddr_in their_addr;
socklen_t namelen = sizeof(sockaddr_in);
// this is a socket to i
socks[i] = accept(listensock, (sockaddr*)&their_addr, &namelen);
set_socket_options(socks[i]);
logstream(LOG_INFO) << "Accepted connection " << i << " -> " << procid() << "\n";
}
mpi_barrier();
}
// allocate the locks
sendlocks.resize(socks.size());
recvlocks.resize(socks.size());
logstream(LOG_INFO) << "All connections constructed\n";
}
distributed_control::~distributed_control() {
logger(LOG_INFO, "Shutting down distributed control");
close_all_connections();
// close all sockets
mpi_barrier();
// kill the send thread
send_requests.stop_blocking();
send_thread->join();
logstream(LOG_INFO) << "Total Bytes Transmitted: " << send_thread->bytes_sent << std::endl;
delete send_thread;
// kill the message handling thread
dispatch_requests.stop_blocking();
for (size_t i = 0; i < dispatch_thread.size(); ++i) {
dispatch_thread[i]->join();
delete dispatch_thread[i];
}
delete [] all_addrs;
// delete buffers
for (size_t i = 0; i <buffer.size(); ++i) {
if (buffer[i].buffer != NULL) free(buffer[i].buffer);
}
logger(LOG_INFO, "MPI_Finalize");
MPI_Finalize();
}
void distributed_control::comms_barrier() {
logger(LOG_INFO, "%d comms barrier", procid());
logstream(LOG_INFO) << msgsent.value << " " << msgprocessed.value << std::endl;
terminator->reset();
mpi_barrier();
while (!terminator->done(msgsent.value, msgprocessed.value)) {
sched_yield();
}
terminator->reset();
}
void
init_timer()
{
double start;
mpi_barrier(MPI_COMM_WORLD);
start = mpi_wtime();
mpi_allreduce(&start, &_zero, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);
_skew = start - _zero;
_initialized = 1;
}
distributed_control::distributed_control(int *pargc, char*** pargv) {
assert(dc_singleton_ptr == NULL);
msgsent.value = 0;
msgprocessed.value = 0;
done = 0;
int provided;
MPI_Init_thread(pargc, pargv, MPI_THREAD_MULTIPLE, &provided);
if (provided >= MPI_THREAD_MULTIPLE) {
logger(LOG_INFO, "MPI threading available");
}
else {
logger(LOG_FATAL, "MPI threading not available. Threading level %d < %d",
provided, MPI_THREAD_MULTIPLE);
ASSERT_TRUE(provided);
}
// create the thread specific data
MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN);
// get the number of processes
int i_nprocs;
MPI_Comm_size(MPI_COMM_WORLD, &i_nprocs);
nprocs = (procid_t)i_nprocs;
logger(LOG_INFO, "%d Processes", nprocs);
int i_id;
MPI_Comm_rank(MPI_COMM_WORLD, &i_id);
id = (procid_t)i_id;
// initialize the local UDT socket
// Now to pick up the IP address of everyone
sync_ip_list();
connect_udt();
send_thread = new background_send_thread(*this);
send_thread->start();
dc_singleton_ptr = this;
terminator = new distributed_terminator(*this);
terminator->set_use_control_packets(true);
mpi_barrier();
}
void distributed_control::init_message_processing(size_t nummsgthreads) {
ASSERT_GT(nummsgthreads, 0);
create_receive_buffers(131072);
// begin message processing threads
dispatch_thread.resize(nummsgthreads);
for (size_t i = 0; i < nummsgthreads; ++i) {
dispatch_thread[i] = new message_dispatch_thread(*this);
dispatch_thread[i]->start();
}
// begin socket receive threads
size_t numrecvthreads = socks.size() - 1;
procs.resize(numrecvthreads );
for (size_t i = 0;i < numrecvthreads ; ++i) {
procs[i].done = &done;
procs[i].dc = this;
procthreads.launch(&(procs[i]));
}
mpi_barrier();
}
