void RawSocketComm::WaitForNewToken() {
//timeval cur_time;
double time_diff;
double diff_unit = RATE_CHECK_PERIOD / 1000.0;
int new_token = unit_size_token;
bool isConstrained = false;
//gettimeofday(&cur_time, NULL);
//time_diff = (cur_time.tv_sec - last_check_time.tv_sec) * 1000000
// + (cur_time.tv_usec - last_check_time.tv_usec);
time_diff = GetElapsedSeconds(last_checked_counter);
while (time_diff < diff_unit) {
isConstrained = true;
usleep(5000);
time_diff = GetElapsedSeconds(last_checked_counter);
// gettimeofday(&cur_time, NULL);
// time_diff = (cur_time.tv_sec - last_check_time.tv_sec) * 1000000
// + (cur_time.tv_usec - last_check_time.tv_usec);
}
if (isConstrained) {
new_token = (int)(new_token * 1.0 * time_diff / diff_unit);
}
current_size_token += new_token;
AccessCPUCounter(&last_checked_counter.hi, &last_checked_counter.lo);
//last_check_time = cur_time;
}
//=========== Functions related to TCP file transfer ================
void FMTPSender::TcpSendMemoryData(void* data, size_t length) {
AccessCPUCounter(&cpu_counter.hi, &cpu_counter.lo);
// Send a notification to all receivers before starting the memory transfer
char msg_packet[500];
FmtpHeader* header = (FmtpHeader*)msg_packet;
header->session_id = cur_session_id;
header->seq_number = 0;
header->data_len = sizeof(FmtpSenderMessage);
header->flags = FMTP_SENDER_MSG_EXP;
FmtpSenderMessage* msg = (FmtpSenderMessage*)(msg_packet + FMTP_HLEN);
msg->msg_type = TCP_MEMORY_TRANSFER_START;
msg->session_id = cur_session_id;
msg->data_len = length;
retrans_tcp_server->SendToAll(&msg_packet, FMTP_HLEN + sizeof(FmtpSenderMessage));
retrans_tcp_server->SendToAll(data, length);
// Record memory data multicast time
double trans_time = GetElapsedSeconds(cpu_counter);
double send_rate = length / 1024.0 / 1024.0 * 8.0 * 1514.0 / 1460.0 / trans_time;
char str[256];
sprintf(str, "***** TCP Send Info *****\nTotal transfer time: %.2f\nThroughput: %.2f\n", trans_time, send_rate);
status_proxy->SendMessageLocal(EXP_RESULT_REPORT, str);
cur_session_id++;
}
void FMTPSender::SendMemoryData(void* data, size_t length) {
ResetSessionStatistics();
AccessCPUCounter(&cpu_counter.hi, &cpu_counter.lo);
// Send a notification to all receivers before starting the memory transfer
struct FmtpSenderMessage msg;
msg.msg_type = MEMORY_TRANSFER_START;
msg.session_id = cur_session_id;
msg.data_len = length;
retrans_tcp_server->SendToAll(&msg, sizeof(msg));
DoMemoryTransfer(data, length, 0);
// Record memory data multicast time
send_stats.session_trans_time = GetElapsedSeconds(cpu_counter);
// Sleep for a few milliseconds to allow receivers to
// empty their multicast socket buffers
//usleep(50000);
// Send a notification to all receivers to start retransmission
msg.msg_type = MEMORY_TRANSFER_FINISH;
cout << "Memory to memory transfer finished." << endl;
retrans_tcp_server->SendToAll(&msg, sizeof(msg));
cout << "Start retransmission..." << endl;
DoMemoryDataRetransmission(data);
// collect experiment results from receivers
CollectExpResults();
// Record total transfer and retransmission time
send_stats.session_total_time = GetElapsedSeconds(cpu_counter);
send_stats.session_retrans_time = send_stats.session_total_time - send_stats.session_trans_time;
send_stats.session_retrans_percentage = send_stats.session_retrans_packets * 1.0
/ (send_stats.session_sent_packets + send_stats.session_retrans_packets);
// Increase the session id for the next transfer
cur_session_id++;
SendSessionStatistics();
}
float GetFPS()
{
/*
static float ReturnFPS = 0;
static float FPSCounter = 0;
static float ElapsedSecTime = 0;
ElapsedSecTime += GetElapsedMilliseconds();
FPSCounter++;
if (ElapsedSecTime >= 1000)
{
ElapsedSecTime = 0;
ReturnFPS = FPSCounter;
FPSCounter = 0;
}
return ReturnFPS;
*/
static const int NUM_FPS_SAMPLES = 100;
static float fpsSamples[NUM_FPS_SAMPLES] = { 0 };
static int currentSample = 0;
fpsSamples[currentSample] = 1.0f / (float)GetElapsedSeconds();
float fps = 0;
for (int i = 0; i < NUM_FPS_SAMPLES; i++)
fps += fpsSamples[i];
fps /= NUM_FPS_SAMPLES;
currentSample++;
if (currentSample > NUM_FPS_SAMPLES)
currentSample = 0;
return fps;
}
double GetElapsedMilliseconds()
{
return (GetElapsedSeconds() * 1000);
}
float CStopWatch::GetElapsedMilliseconds() const
{
return GetElapsedSeconds() * 1000.0f;
}
void FMTPSender::TcpSendFile(const char* file_name) {
AccessCPUCounter(&cpu_counter.hi, &cpu_counter.lo);
struct stat file_status;
stat(file_name, &file_status);
ulong file_size = file_status.st_size;
ulong remained_size = file_size;
// Send a notification to all receivers before starting the memory transfer
char msg_packet[500];
FmtpHeader* header = (FmtpHeader*)msg_packet;
header->session_id = cur_session_id;
header->seq_number = 0;
header->data_len = sizeof(FmtpSenderMessage);
header->flags = FMTP_SENDER_MSG_EXP;
FmtpSenderMessage* msg = (FmtpSenderMessage*)(msg_packet + FMTP_HLEN);
msg->msg_type = TCP_FILE_TRANSFER_START;
msg->session_id = cur_session_id;
msg->data_len = file_size;
strcpy(msg->text, file_name);
retrans_tcp_server->SendToAll(&msg_packet, FMTP_HLEN + sizeof(FmtpSenderMessage));
PerformanceCounter cpu_info(100);
cpu_info.SetCPUFlag(true);
cpu_info.Start();
cout << "Start TCP file transferring..." << endl;
list<int> sock_list = retrans_tcp_server->GetSocketList();
list<TcpThreadInfo*> thread_info_list;
list<pthread_t*> thread_list;
int file_name_len = strlen(file_name);
for (list<int>::iterator it = sock_list.begin(); it != sock_list.end(); it++) {
TcpThreadInfo* info = new TcpThreadInfo();
info->ptr = this;
info->sock_fd = *it;
memcpy(info->file_name, file_name, file_name_len);
thread_info_list.push_back(info);
pthread_t * t = new pthread_t();
pthread_create(t, NULL, &FMTPSender::StartTcpSendThread, info);
thread_list.push_back(t);
}
for (list<pthread_t*>::iterator it = thread_list.begin(); it != thread_list.end(); it++) {
pthread_join(**it, NULL);
}
for (list<pthread_t*>::iterator it = thread_list.begin(); it != thread_list.end(); it++) {
delete (*it);
}
for (list<TcpThreadInfo*>::iterator it = thread_info_list.begin(); it != thread_info_list.end(); it++) {
delete (*it);
}
cpu_info.Stop();
int cpu_usage = cpu_info.GetAverageCpuUsage();
// Record memory data multicast time
double trans_time = GetElapsedSeconds(cpu_counter);
double send_rate = file_size / 1024.0 / 1024.0 * 8.0 * 1514.0 / 1460.0 / trans_time;
char str[256];
sprintf(str, "***** TCP Send Info *****\nTotal transfer time: %.2f seconds\nThroughput: %.2f Mbps\nAvg. CPU Usage: %d\%\n",
trans_time, send_rate, cpu_usage);
status_proxy->SendMessageLocal(INFORMATIONAL, str);
cur_session_id++;
}
//==================================== Old Functions ====================================
void FMTPSender::SendFileBufferedIO(const char* file_name) {
PerformanceCounter cpu_info(50);
cpu_info.SetCPUFlag(true);
cpu_info.Start();
ResetSessionStatistics();
AccessCPUCounter(&cpu_counter.hi, &cpu_counter.lo);
struct stat file_status;
stat(file_name, &file_status);
ulong file_size = file_status.st_size;
ulong remained_size = file_size;
// Send a notification to all receivers before starting the memory transfer
struct FmtpSenderMessage msg;
msg.session_id = cur_session_id;
msg.msg_type = FILE_TRANSFER_START;
msg.data_len = file_size;
strcpy(msg.text, file_name);
retrans_tcp_server->SendToAll(&msg, sizeof(msg));
//cout << "Start file transferring..." << endl;
// Transfer the file using memory mapped I/O
int fd = open(file_name, O_RDWR);
if (fd < 0) {
SysError("FMTPSender()::SendFile(): File open error!");
}
char* buffer = (char *)malloc(FMTP_DATA_LEN);
off_t offset = 0;
while (remained_size > 0) {
uint read_size = remained_size < FMTP_DATA_LEN ? remained_size
: FMTP_DATA_LEN;
ssize_t res = read(fd, buffer, read_size);
if (res < 0) {
SysError("FMTPSender::SendFileBufferedIO()::read() error");
}
DoMemoryTransfer(buffer, read_size, offset);
offset += read_size;
remained_size -= read_size;
}
free(buffer);
// Record memory data multicast time
send_stats.session_trans_time = GetElapsedSeconds(cpu_counter);
AccessCPUCounter(&cpu_counter.hi, &cpu_counter.lo);
// Send a notification to all receivers to start retransmission
msg.msg_type = FILE_TRANSFER_FINISH;
retrans_tcp_server->SendToAll(&msg, sizeof(msg));
//cout << "File transfer finished. Start retransmission..." << endl;
if (retrans_scheme == RETRANS_SERIAL)
DoFileRetransmissionSerial(fd);
else if (retrans_scheme == RETRANS_SERIAL_RR)
DoFileRetransmissionSerialRR(fd);
else if (retrans_scheme == RETRANS_PARALLEL)
DoFileRetransmissionParallel(file_name);
close(fd);
// collect experiment results from receivers
CollectExpResults();
// Record total transfer and retransmission time
send_stats.session_retrans_time = GetElapsedSeconds(cpu_counter); //send_stats.session_total_time - send_stats.session_trans_time;
send_stats.session_total_time = send_stats.session_trans_time + send_stats.session_retrans_time; //GetElapsedSeconds(cpu_counter);
send_stats.session_retrans_percentage = send_stats.session_retrans_packets * 1.0
/ (send_stats.session_sent_packets + send_stats.session_retrans_packets);
// Increase the session id for the next transfer
cur_session_id++;
SendSessionStatistics();
cpu_info.Stop();
}
// The execution function for the retransmission thread
void FMTPSender::RunRetransThread(int sock, map<uint, int>& retrans_fd_map, set<uint>& timeout_set) {
int sock_fd = sock;
map<uint, int>::iterator it;
char recv_buf[FMTP_PACKET_LEN];
FmtpHeader* recv_header = (FmtpHeader*)recv_buf;
char* recv_packet_data = recv_buf + FMTP_HLEN;
FmtpRetransRequest* retx_request = (FmtpRetransRequest*)recv_packet_data;
char send_buf[FMTP_PACKET_LEN];
FmtpHeader* send_header = (FmtpHeader*)send_buf;
char* send_packet_data = send_buf + FMTP_HLEN;
while (true) {
if (retrans_tcp_server->Receive(sock_fd, recv_header, FMTP_HLEN) <= 0) {
SysError("FMTPSender::RunRetransThread()::receive header error");
}
// Handle a retransmission request
if (recv_header->flags & FMTP_RETRANS_REQ) {
if (retrans_tcp_server->Receive(sock_fd, retx_request, recv_header->data_len) < 0) {
SysError("FMTPSender::RunRetransThread()::receive retx request data error");
}
MessageMetadata* meta = metadata.GetMetadata(retx_request->msg_id);
if (meta == NULL) {
//cout << "Error: could not find metadata for file " << retx_request->msg_id << endl;
continue;
} else if (timeout_set.find(retx_request->msg_id) != timeout_set.end()) {
continue;
}
// check whether the retransmission for the file has already time out
if (GetElapsedSeconds(meta->multicast_start_cpu_time) > meta->retx_timeout_seconds) {
//cout << "Retx timeout for file " << retx_request->msg_id << ". Elapsed Time: "
// << GetElapsedSeconds(meta->multicast_start_cpu_time) << " Timeout: " << meta->retx_timeout_seconds << endl;
send_header->session_id = retx_request->msg_id;
send_header->flags = FMTP_RETRANS_TIMEOUT;
send_header->data_len = 0;
retrans_tcp_server->SelectSend(sock_fd, send_buf, FMTP_HLEN);
timeout_set.insert(retx_request->msg_id);
}
else if (meta->is_disk_file) { // is disk file transfer
FileMessageMetadata* file_meta = (FileMessageMetadata*)meta;
// get the file descriptor to read data from the file
int fd;
if ( (it = retrans_fd_map.find(recv_header->session_id)) != retrans_fd_map.end()) {
fd = it->second;
}
else {
if ( (fd = open(file_meta->file_name.c_str(), O_RDONLY)) < 0)
SysError("FMTPSender::RunRetransThread() file open error");
else
retrans_fd_map[recv_header->session_id] = fd;
}
// send the missing blocks to the receiver
lseek(fd, retx_request->seq_num, SEEK_SET);
size_t remained_size = retx_request->data_len;
size_t curr_pos = retx_request->seq_num;
send_header->session_id = recv_header->session_id;
send_header->flags = FMTP_RETRANS_DATA;
while (remained_size > 0) {
size_t data_length =
remained_size > FMTP_DATA_LEN ? FMTP_DATA_LEN
: remained_size;
send_header->seq_number = curr_pos;
send_header->data_len = data_length;
read(fd, send_packet_data, send_header->data_len);
retrans_tcp_server->SelectSend(sock_fd, send_buf, FMTP_HLEN + send_header->data_len);
curr_pos += data_length;
remained_size -= data_length;
// Update statistics
send_stats.total_retrans_packets++;
send_stats.total_retrans_bytes += send_header->data_len;
//file_meta->stats.session_retrans_packets++;
//file_meta->stats.session_retrans_bytes += header->data_len;
}
}
else { // is memory data transfer
}
}
else if (recv_header->flags & FMTP_RETRANS_END) {
if (retrans_tcp_server->Receive(sock_fd, retx_request, recv_header->data_len) < 0) {
SysError("FMTPSender::RunRetransThread()::receive retx end msg error");
}
// send back the retransmission end message to the receiver
send_header->session_id = recv_header->session_id;
send_header->seq_number = 0;
send_header->data_len = 0;
send_header->flags = FMTP_RETRANS_END;
retrans_tcp_server->SelectSend(sock_fd, send_header, FMTP_HLEN);
map<uint, int>::iterator it = retrans_fd_map.find(recv_header->session_id);
if ( it != retrans_fd_map.end() ) {
close(it->second);
retrans_fd_map.erase(it);
}
if ( timeout_set.find(recv_header->session_id) != timeout_set.end() )
timeout_set.erase(recv_header->session_id);
// mark the completion of retransmission to one receiver
metadata.RemoveFinishedReceiver(recv_header->session_id, sock_fd);
}
else if (recv_header->flags & FMTP_HISTORY_STATISTICS) {
char* buf = new char[recv_header->data_len + 1];
if (retrans_tcp_server->Receive(sock_fd, buf, recv_header->data_len) < 0) {
break;
}
buf[recv_header->data_len] = '\0';
status_proxy->SendMessageLocal(EXP_RESULT_REPORT, buf);
delete[] buf;
cout << "Received a history statistics from socket " << sock_fd << endl;
}
}
cout << "Retransmission thread exited for socket " << sock_fd << endl;
}
uint FMTPSender::SendFile(const char* file_name, int retx_timeout_ratio) {
ResetSessionStatistics();
// Increase the session id for new file transfer
cur_session_id++;
struct stat file_status;
stat(file_name, &file_status);
ulong file_size = file_status.st_size;
ulong remained_size = file_size;
// Create metadata for the new file
FileMessageMetadata* meta = new FileMessageMetadata();
if (meta == NULL)
SysError("FMTPSender::SendFile()::cannot allocate memory for FileMessageMetadata");
meta->msg_id = cur_session_id;
meta->msg_length = file_size;
meta->file_name = file_name;
meta->retx_timeout_ratio = retx_timeout_ratio;
// add all current receiver sockets into the unfinished receiver map
list<int> sock_list = retrans_tcp_server->GetSocketList();
for (list<int>::iterator it = sock_list.begin(); it != sock_list.end(); it++) {
meta->unfinished_recvers.insert(*it);
}
metadata.AddMessageMetadata(meta);
AccessCPUCounter(&cpu_counter.hi, &cpu_counter.lo);
meta->multicast_start_cpu_time = cpu_counter;
// Send the BOF message to all receivers before starting the file transfer
char msg_packet[500];
FmtpHeader* header = (FmtpHeader*)msg_packet;
header->session_id = cur_session_id;
header->seq_number = 0;
header->data_len = sizeof(FmtpSenderMessage);
header->flags = FMTP_BOF;
FmtpSenderMessage* msg = (FmtpSenderMessage*)(msg_packet + FMTP_HLEN);
msg->session_id = cur_session_id;
msg->msg_type = FILE_TRANSFER_START;
msg->data_len = file_size;
msg->time_stamp = GetElapsedSeconds(global_timer);
strcpy(msg->text, file_name);
//retrans_tcp_server->SendToAll(&msg_packet, FMTP_HLEN + sizeof(FmtpSenderMessage));
if (ptr_multicast_comm->SendData(&msg_packet, FMTP_HLEN + sizeof(FmtpSenderMessage), 0, NULL) < 0) {
SysError("FMTPSender::SendFile()::SendData() error");
}
//PerformanceCounter cpu_info(100);
//cpu_info.SetCPUFlag(true);
//cpu_info.Start();
//cout << "Start file transferring..." << endl;
// Transfer the file using memory mapped I/O
int fd = open(file_name, O_RDONLY);
if (fd < 0) {
SysError("FMTPSender()::SendFile(): File open error!");
}
char* buffer;
off_t offset = 0;
while (remained_size > 0) {
uint map_size = remained_size < MAX_MAPPED_MEM_SIZE ? remained_size
: MAX_MAPPED_MEM_SIZE;
buffer = (char*) mmap(0, map_size, PROT_READ, MAP_FILE | MAP_SHARED, fd,
offset);
if (buffer == MAP_FAILED) {
SysError("FMTPSender::SendFile()::mmap() error");
}
DoMemoryTransfer(buffer, map_size, offset);
munmap(buffer, map_size);
offset += map_size;
remained_size -= map_size;
}
// Record memory data multicast time
send_stats.session_trans_time = meta->msg_length * 8.0 / 100000000.0; //GetElapsedSeconds(cpu_counter);
meta->stats.session_trans_time = send_stats.session_trans_time;
double default_timeout = meta->stats.session_trans_time * (meta->retx_timeout_ratio / 100.0 + 1.0);
meta->retx_timeout_seconds = default_timeout > (MIN_RETX_TIMEOUT + meta->stats.session_retrans_time) ?
default_timeout : (MIN_RETX_TIMEOUT + meta->stats.session_retrans_time);
// TODO: remove this in real implementation
// For test ONLY: clear system cache before doing retransmission
// system("sudo sync && sudo echo 3 > /proc/sys/vm/drop_caches");
// Send a notification to all receivers to start retransmission
header->flags = FMTP_EOF;
header->data_len = 0;
//retrans_tcp_server->SendToAll(header, FMTP_HLEN);
if (ptr_multicast_comm->SendData(header, FMTP_HLEN, 0, NULL) < 0) {
SysError("FMTPSender::SendFile()::SendData() error");
}
close(fd);
//SendSessionStatistics();
return meta->msg_id;
}
//-----------------------------------------------------------------------------
// Name: GetElapsedMilliseconds()
// Desc: The amount of time the watch has been running since it was last
// reset, in mS
//-----------------------------------------------------------------------------
FLOAT CXBStopWatch::GetElapsedMilliseconds() const
{
return( GetElapsedSeconds() * 1000.0f );
}
unsigned User::GetIdleSeconds() const
{
return GetElapsedSeconds( m_LastActive );
}
