/**
* This is Jie's original constructor (with the effect of some statements moved
* to the initialization list). It's kept because it's referenced by
* `SenderStatusProxy::InitializeExecutionProcess()`. If that method is deleted,
* then so can this constructor.
*
* @param buf_size Ignored.
*/
FMTPSender::FMTPSender(
int buf_size)
:
FMTPComm(),
retrans_tcp_server(new TcpServer(BUFFER_TCP_SEND_PORT, this)),
cur_session_id(0),
/*
* Set the initial maximum size of the retransmission buffer to be 32 * 8 MB
* = 256 MB
*/
max_num_retrans_buffs(32),
retrans_scheme(RETRANS_SERIAL), // set default retransmission scheme
num_retrans_threads(1),
status_proxy(0),
send_rate_in_mbps(10000) // 10 Gbps initial send rate limit
{
bzero(&send_stats, sizeof(send_stats));
rate_shaper.SetRate(send_rate_in_mbps * 1000000.0);
AccessCPUCounter(&global_timer.hi, &global_timer.lo);
// Set CPU affinity
/*cpu_set_t cpu_mask;
CPU_SET(1, &cpu_mask);
if (sched_setaffinity(0, sizeof(cpu_set_t), &cpu_mask) == -1)
SysError("FMTPSender::FMTPSender()::sched_setaffinity() error");
*/
//event_queue_manager = new FmtpEventQueueManager();
}
//=========== Functions related to TCP file transfer ================
void FMTPSender::TcpSendMemoryData(void* data, size_t length) {
AccessCPUCounter(&cpu_counter.hi, &cpu_counter.lo);
// Send a notification to all receivers before starting the memory transfer
char msg_packet[500];
FmtpHeader* header = (FmtpHeader*)msg_packet;
header->session_id = cur_session_id;
header->seq_number = 0;
header->data_len = sizeof(FmtpSenderMessage);
header->flags = FMTP_SENDER_MSG_EXP;
FmtpSenderMessage* msg = (FmtpSenderMessage*)(msg_packet + FMTP_HLEN);
msg->msg_type = TCP_MEMORY_TRANSFER_START;
msg->session_id = cur_session_id;
msg->data_len = length;
retrans_tcp_server->SendToAll(&msg_packet, FMTP_HLEN + sizeof(FmtpSenderMessage));
retrans_tcp_server->SendToAll(data, length);
// Record memory data multicast time
double trans_time = GetElapsedSeconds(cpu_counter);
double send_rate = length / 1024.0 / 1024.0 * 8.0 * 1514.0 / 1460.0 / trans_time;
char str[256];
sprintf(str, "***** TCP Send Info *****\nTotal transfer time: %.2f\nThroughput: %.2f\n", trans_time, send_rate);
status_proxy->SendMessageLocal(EXP_RESULT_REPORT, str);
cur_session_id++;
}
/**
* This new constructor takes specifications of the TCP server.
*
* @param[in] tcpAddr Dotted-decimal IPv4 address of the TCP server. The
* caller may delete.
* @param[in] tcpPort Port number of the TCP server.
* @param[in] fileId Initial product-index. The first multicast file will
* have this as its product-index.
*/
FMTPSender::FMTPSender(
const string& tcpAddr,
const unsigned short tcpPort,
const u_int32_t fileId)
:
FMTPComm(),
retrans_tcp_server(new TcpServer(BUFFER_TCP_SEND_PORT, this)),
cur_session_id(fileId),
/*
* Set the initial maximum size of the retransmission buffer to 32 * 8 MB =
* 256 MB
*/
max_num_retrans_buffs(32),
retrans_scheme(RETRANS_SERIAL), // set default retransmission scheme
num_retrans_threads(1),
status_proxy(0),
send_rate_in_mbps(10000) // 10 Gbps initial send rate limit
{
// This constructor is unfinished.
bzero(&send_stats, sizeof(send_stats));
rate_shaper.SetRate(send_rate_in_mbps * 1000000.0);
AccessCPUCounter(&global_timer.hi, &global_timer.lo);
}
void RawSocketComm::WaitForNewToken() {
//timeval cur_time;
double time_diff;
double diff_unit = RATE_CHECK_PERIOD / 1000.0;
int new_token = unit_size_token;
bool isConstrained = false;
//gettimeofday(&cur_time, NULL);
//time_diff = (cur_time.tv_sec - last_check_time.tv_sec) * 1000000
// + (cur_time.tv_usec - last_check_time.tv_usec);
time_diff = GetElapsedSeconds(last_checked_counter);
while (time_diff < diff_unit) {
isConstrained = true;
usleep(5000);
time_diff = GetElapsedSeconds(last_checked_counter);
// gettimeofday(&cur_time, NULL);
// time_diff = (cur_time.tv_sec - last_check_time.tv_sec) * 1000000
// + (cur_time.tv_usec - last_check_time.tv_usec);
}
if (isConstrained) {
new_token = (int)(new_token * 1.0 * time_diff / diff_unit);
}
current_size_token += new_token;
AccessCPUCounter(&last_checked_counter.hi, &last_checked_counter.lo);
//last_check_time = cur_time;
}
void RawSocketComm::SetSendRate(int num_mbps) {
send_rate_in_mbps = num_mbps;
unit_size_token = (size_t)(RATE_CHECK_PERIOD / 1000.0 * num_mbps * 1024 * 1024 / 8.0); // change to number of bytes
current_size_token = unit_size_token;
AccessCPUCounter(&last_checked_counter.hi, &last_checked_counter.lo);
//gettimeofday(&last_check_time, NULL);
}
void FMTPSender::ResetAllReceiverStats() {
AccessCPUCounter(&global_timer.hi, &global_timer.lo);
char msg_packet[500];
FmtpHeader* header = (FmtpHeader*)msg_packet;
header->session_id = cur_session_id;
header->seq_number = 0;
header->data_len = sizeof(FmtpSenderMessage);
header->flags = FMTP_SENDER_MSG_EXP;
FmtpSenderMessage* msg = (FmtpSenderMessage*)(msg_packet + FMTP_HLEN);
msg->msg_type = RESET_HISTORY_STATISTICS;
msg->session_id = cur_session_id;
msg->data_len = 0;
retrans_tcp_server->SendToAll(&msg_packet, FMTP_HLEN + sizeof(FmtpSenderMessage));
}
void FMTPSender::SendMemoryData(void* data, size_t length) {
ResetSessionStatistics();
AccessCPUCounter(&cpu_counter.hi, &cpu_counter.lo);
// Send a notification to all receivers before starting the memory transfer
struct FmtpSenderMessage msg;
msg.msg_type = MEMORY_TRANSFER_START;
msg.session_id = cur_session_id;
msg.data_len = length;
retrans_tcp_server->SendToAll(&msg, sizeof(msg));
DoMemoryTransfer(data, length, 0);
// Record memory data multicast time
send_stats.session_trans_time = GetElapsedSeconds(cpu_counter);
// Sleep for a few milliseconds to allow receivers to
// empty their multicast socket buffers
//usleep(50000);
// Send a notification to all receivers to start retransmission
msg.msg_type = MEMORY_TRANSFER_FINISH;
cout << "Memory to memory transfer finished." << endl;
retrans_tcp_server->SendToAll(&msg, sizeof(msg));
cout << "Start retransmission..." << endl;
DoMemoryDataRetransmission(data);
// collect experiment results from receivers
CollectExpResults();
// Record total transfer and retransmission time
send_stats.session_total_time = GetElapsedSeconds(cpu_counter);
send_stats.session_retrans_time = send_stats.session_total_time - send_stats.session_trans_time;
send_stats.session_retrans_percentage = send_stats.session_retrans_packets * 1.0
/ (send_stats.session_sent_packets + send_stats.session_retrans_packets);
// Increase the session id for the next transfer
cur_session_id++;
SendSessionStatistics();
}
void FMTPSender::TcpSendFile(const char* file_name) {
AccessCPUCounter(&cpu_counter.hi, &cpu_counter.lo);
struct stat file_status;
stat(file_name, &file_status);
ulong file_size = file_status.st_size;
ulong remained_size = file_size;
// Send a notification to all receivers before starting the memory transfer
char msg_packet[500];
FmtpHeader* header = (FmtpHeader*)msg_packet;
header->session_id = cur_session_id;
header->seq_number = 0;
header->data_len = sizeof(FmtpSenderMessage);
header->flags = FMTP_SENDER_MSG_EXP;
FmtpSenderMessage* msg = (FmtpSenderMessage*)(msg_packet + FMTP_HLEN);
msg->msg_type = TCP_FILE_TRANSFER_START;
msg->session_id = cur_session_id;
msg->data_len = file_size;
strcpy(msg->text, file_name);
retrans_tcp_server->SendToAll(&msg_packet, FMTP_HLEN + sizeof(FmtpSenderMessage));
PerformanceCounter cpu_info(100);
cpu_info.SetCPUFlag(true);
cpu_info.Start();
cout << "Start TCP file transferring..." << endl;
list<int> sock_list = retrans_tcp_server->GetSocketList();
list<TcpThreadInfo*> thread_info_list;
list<pthread_t*> thread_list;
int file_name_len = strlen(file_name);
for (list<int>::iterator it = sock_list.begin(); it != sock_list.end(); it++) {
TcpThreadInfo* info = new TcpThreadInfo();
info->ptr = this;
info->sock_fd = *it;
memcpy(info->file_name, file_name, file_name_len);
thread_info_list.push_back(info);
pthread_t * t = new pthread_t();
pthread_create(t, NULL, &FMTPSender::StartTcpSendThread, info);
thread_list.push_back(t);
}
for (list<pthread_t*>::iterator it = thread_list.begin(); it != thread_list.end(); it++) {
pthread_join(**it, NULL);
}
for (list<pthread_t*>::iterator it = thread_list.begin(); it != thread_list.end(); it++) {
delete (*it);
}
for (list<TcpThreadInfo*>::iterator it = thread_info_list.begin(); it != thread_info_list.end(); it++) {
delete (*it);
}
cpu_info.Stop();
int cpu_usage = cpu_info.GetAverageCpuUsage();
// Record memory data multicast time
double trans_time = GetElapsedSeconds(cpu_counter);
double send_rate = file_size / 1024.0 / 1024.0 * 8.0 * 1514.0 / 1460.0 / trans_time;
char str[256];
sprintf(str, "***** TCP Send Info *****\nTotal transfer time: %.2f seconds\nThroughput: %.2f Mbps\nAvg. CPU Usage: %d\%\n",
trans_time, send_rate, cpu_usage);
status_proxy->SendMessageLocal(INFORMATIONAL, str);
cur_session_id++;
}
//==================================== Old Functions ====================================
void FMTPSender::SendFileBufferedIO(const char* file_name) {
PerformanceCounter cpu_info(50);
cpu_info.SetCPUFlag(true);
cpu_info.Start();
ResetSessionStatistics();
AccessCPUCounter(&cpu_counter.hi, &cpu_counter.lo);
struct stat file_status;
stat(file_name, &file_status);
ulong file_size = file_status.st_size;
ulong remained_size = file_size;
// Send a notification to all receivers before starting the memory transfer
struct FmtpSenderMessage msg;
msg.session_id = cur_session_id;
msg.msg_type = FILE_TRANSFER_START;
msg.data_len = file_size;
strcpy(msg.text, file_name);
retrans_tcp_server->SendToAll(&msg, sizeof(msg));
//cout << "Start file transferring..." << endl;
// Transfer the file using memory mapped I/O
int fd = open(file_name, O_RDWR);
if (fd < 0) {
SysError("FMTPSender()::SendFile(): File open error!");
}
char* buffer = (char *)malloc(FMTP_DATA_LEN);
off_t offset = 0;
while (remained_size > 0) {
uint read_size = remained_size < FMTP_DATA_LEN ? remained_size
: FMTP_DATA_LEN;
ssize_t res = read(fd, buffer, read_size);
if (res < 0) {
SysError("FMTPSender::SendFileBufferedIO()::read() error");
}
DoMemoryTransfer(buffer, read_size, offset);
offset += read_size;
remained_size -= read_size;
}
free(buffer);
// Record memory data multicast time
send_stats.session_trans_time = GetElapsedSeconds(cpu_counter);
AccessCPUCounter(&cpu_counter.hi, &cpu_counter.lo);
// Send a notification to all receivers to start retransmission
msg.msg_type = FILE_TRANSFER_FINISH;
retrans_tcp_server->SendToAll(&msg, sizeof(msg));
//cout << "File transfer finished. Start retransmission..." << endl;
if (retrans_scheme == RETRANS_SERIAL)
DoFileRetransmissionSerial(fd);
else if (retrans_scheme == RETRANS_SERIAL_RR)
DoFileRetransmissionSerialRR(fd);
else if (retrans_scheme == RETRANS_PARALLEL)
DoFileRetransmissionParallel(file_name);
close(fd);
// collect experiment results from receivers
CollectExpResults();
// Record total transfer and retransmission time
send_stats.session_retrans_time = GetElapsedSeconds(cpu_counter); //send_stats.session_total_time - send_stats.session_trans_time;
send_stats.session_total_time = send_stats.session_trans_time + send_stats.session_retrans_time; //GetElapsedSeconds(cpu_counter);
send_stats.session_retrans_percentage = send_stats.session_retrans_packets * 1.0
/ (send_stats.session_sent_packets + send_stats.session_retrans_packets);
// Increase the session id for the next transfer
cur_session_id++;
SendSessionStatistics();
cpu_info.Stop();
}
uint FMTPSender::SendFile(const char* file_name, int retx_timeout_ratio) {
ResetSessionStatistics();
// Increase the session id for new file transfer
cur_session_id++;
struct stat file_status;
stat(file_name, &file_status);
ulong file_size = file_status.st_size;
ulong remained_size = file_size;
// Create metadata for the new file
FileMessageMetadata* meta = new FileMessageMetadata();
if (meta == NULL)
SysError("FMTPSender::SendFile()::cannot allocate memory for FileMessageMetadata");
meta->msg_id = cur_session_id;
meta->msg_length = file_size;
meta->file_name = file_name;
meta->retx_timeout_ratio = retx_timeout_ratio;
// add all current receiver sockets into the unfinished receiver map
list<int> sock_list = retrans_tcp_server->GetSocketList();
for (list<int>::iterator it = sock_list.begin(); it != sock_list.end(); it++) {
meta->unfinished_recvers.insert(*it);
}
metadata.AddMessageMetadata(meta);
AccessCPUCounter(&cpu_counter.hi, &cpu_counter.lo);
meta->multicast_start_cpu_time = cpu_counter;
// Send the BOF message to all receivers before starting the file transfer
char msg_packet[500];
FmtpHeader* header = (FmtpHeader*)msg_packet;
header->session_id = cur_session_id;
header->seq_number = 0;
header->data_len = sizeof(FmtpSenderMessage);
header->flags = FMTP_BOF;
FmtpSenderMessage* msg = (FmtpSenderMessage*)(msg_packet + FMTP_HLEN);
msg->session_id = cur_session_id;
msg->msg_type = FILE_TRANSFER_START;
msg->data_len = file_size;
msg->time_stamp = GetElapsedSeconds(global_timer);
strcpy(msg->text, file_name);
//retrans_tcp_server->SendToAll(&msg_packet, FMTP_HLEN + sizeof(FmtpSenderMessage));
if (ptr_multicast_comm->SendData(&msg_packet, FMTP_HLEN + sizeof(FmtpSenderMessage), 0, NULL) < 0) {
SysError("FMTPSender::SendFile()::SendData() error");
}
//PerformanceCounter cpu_info(100);
//cpu_info.SetCPUFlag(true);
//cpu_info.Start();
//cout << "Start file transferring..." << endl;
// Transfer the file using memory mapped I/O
int fd = open(file_name, O_RDONLY);
if (fd < 0) {
SysError("FMTPSender()::SendFile(): File open error!");
}
char* buffer;
off_t offset = 0;
while (remained_size > 0) {
uint map_size = remained_size < MAX_MAPPED_MEM_SIZE ? remained_size
: MAX_MAPPED_MEM_SIZE;
buffer = (char*) mmap(0, map_size, PROT_READ, MAP_FILE | MAP_SHARED, fd,
offset);
if (buffer == MAP_FAILED) {
SysError("FMTPSender::SendFile()::mmap() error");
}
DoMemoryTransfer(buffer, map_size, offset);
munmap(buffer, map_size);
offset += map_size;
remained_size -= map_size;
}
// Record memory data multicast time
send_stats.session_trans_time = meta->msg_length * 8.0 / 100000000.0; //GetElapsedSeconds(cpu_counter);
meta->stats.session_trans_time = send_stats.session_trans_time;
double default_timeout = meta->stats.session_trans_time * (meta->retx_timeout_ratio / 100.0 + 1.0);
meta->retx_timeout_seconds = default_timeout > (MIN_RETX_TIMEOUT + meta->stats.session_retrans_time) ?
default_timeout : (MIN_RETX_TIMEOUT + meta->stats.session_retrans_time);
// TODO: remove this in real implementation
// For test ONLY: clear system cache before doing retransmission
// system("sudo sync && sudo echo 3 > /proc/sys/vm/drop_caches");
// Send a notification to all receivers to start retransmission
header->flags = FMTP_EOF;
header->data_len = 0;
//retrans_tcp_server->SendToAll(header, FMTP_HLEN);
if (ptr_multicast_comm->SendData(header, FMTP_HLEN, 0, NULL) < 0) {
SysError("FMTPSender::SendFile()::SendData() error");
}
close(fd);
//SendSessionStatistics();
return meta->msg_id;
}
