/*!
* \brief UTCからローカル時間に変換
*/
void DateTime::toLocal()
{
#if defined(_WINDOWS)
TIME_ZONE_INFORMATION timeZone;
GetTimeZoneInformation(&timeZone);
int32_t bias = -timeZone.Bias / 60;
#else
struct timezone tz;
gettimeofday(nullptr, &tz);
int32_t bias = -tz.tz_minuteswest / 60;
#endif
struct tm tm;
tm.tm_year = getYear() - 1900;
tm.tm_mon = getMonth() - 1;
tm.tm_mday = getDay();
tm.tm_hour = getHour() + bias;
tm.tm_min = getMinute();
tm.tm_sec = getSecond();
tm.tm_yday =
tm.tm_wday =
tm.tm_isdst = 0;
#if defined(_WINDOWS)
time_t time = mktime(&tm);
#else
time_t time = mktime(&tm);
// time_t time = timegm(&tm);
#endif
struct tm* tmLocal = localtime(&time);
toValue(tmLocal, getMilliSecond());
}
unsigned TimeOutReceiver::TimeOutWait(unsigned expected_message_count,unsigned time_out_in_ms){
unsigned long long int start=curtick();
unsigned count(0);
while(count<expected_message_count&&getMilliSecond(start)<time_out_in_ms){
count+=Consume(expected_message_count-count);
}
return count;
}
/*!
* \brief 日付時間をミリ秒に変換
*/
int64_t DateTime::toMilliSeconds() const
{
enum
{
January = 0,
February = January + 31,
March = February + 28,
April = March + 31,
May = April + 30,
June = May + 31,
July = June + 30,
August = July + 31,
September = August + 31,
October = September + 30,
November = October + 31,
December = November + 30,
};
static const int64_t daySpan[] =
{
January,
February,
March,
April,
May,
June,
July,
August,
September,
October,
November,
December,
};
int64_t y = getYear();
int64_t m = getMonth();
int64_t d = getDay();
int64_t dy = (y - 1) * 365; // 年数分の日数
int64_t dl = (y / 4) - (y / 100) + (y / 400); // 閏年分の日数
int64_t dm = daySpan[m - 1]; // 1月1日からm月1日までの日数
int64_t result = dy + dl + dm + d - 1;
result =
(result * 24 * 60 * 60 * 1000) +
(getHour() * 60 * 60 * 1000) +
(getMinute() * 60 * 1000) +
(getSecond() * 1000) +
getMilliSecond();
return result;
}
// receive each one block from all sender
void* ExpandableBlockStreamExchangeEpoll::receiver(void* arg){
ExpandableBlockStreamExchangeEpoll* Pthis=(ExpandableBlockStreamExchangeEpoll*)arg;
struct epoll_event event;
struct epoll_event *events;
int status;
/** create epoll **/
Pthis->epoll_fd_ = epoll_create1(0);
if (Pthis->epoll_fd_ == -1)
{
Pthis->logging_->elog("epoll create error!\n");
return 0;
}
event.data.fd = Pthis->sock_fd;
event.events = EPOLLIN | EPOLLET;
status = epoll_ctl(Pthis->epoll_fd_, EPOLL_CTL_ADD, Pthis->sock_fd, &event);
if (status == -1)
{
Pthis->logging_->elog("epoll ctl error!\n");
return 0;
}
events=(epoll_event*)calloc(Pthis->nlowers,sizeof(epoll_event));
int fd_cur=0;
ticks start=curtick();
std::vector<int> finish_times;//in ms
while(true){
usleep(1);
const int event_count = epoll_wait(Pthis->epoll_fd_, events, Pthis->nlowers, -1);
for (int i = 0; i < event_count; i++)
{
if ((events[i].events & EPOLLERR) || (events[i].events & EPOLLHUP) || (!(events[i].events & EPOLLIN)))
{
if (errno == EINTR)
{
continue;
}
Pthis->logging_->elog("[%ld] epoll error,reason:%s\n", Pthis->state.exchange_id_, strerror(errno));
FileClose(events[i].data.fd);
std::cout << "in " << __FILE__ << ":" << __LINE__;
printf("-----for debug:close fd %d.\n", events[i].data.fd);
continue;
}
else if (Pthis->sock_fd == events[i].data.fd)
{
/* We have a notification on the listening socket, which means one or more incoming connections.*/
while (true)
{
sockaddr in_addr;
socklen_t in_len;
int infd;
char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
in_len = sizeof in_addr;
infd = accept(Pthis->sock_fd, &in_addr, &in_len);
if (infd == -1)
{
if ((errno == EAGAIN) || (errno == EWOULDBLOCK))
{
/* all the incoming connections are processed.*/
break;
}
else
{
Pthis->logging_->elog("accept error! ");
break;
}
}
status=getnameinfo(&in_addr,in_len,hbuf,sizeof(hbuf),sbuf,sizeof(sbuf),NI_NUMERICHOST|NI_NUMERICSERV);
if(status==0){
Pthis->logging_->log("[%ld] Accepted connection on descriptor %d (host=%s, port=%s),id=%d\n",Pthis->state.exchange_id_, infd, hbuf, sbuf,Pthis->state.exchange_id_);
Pthis->lower_ip_array.push_back(hbuf);
Pthis->lower_sock_fd_to_index[infd]=Pthis->lower_ip_array.size()-1;
assert(Pthis->lower_ip_array.size()<=Pthis->state.lower_id_list_.size());
}
/*Make the incoming socket non-blocking and add it to the list of fds to monitor.*/
if (!Pthis->SetSocketNonBlocking(infd))
{
return 0;
}
event.data.fd = infd;
event.events = EPOLLIN | EPOLLET;
status = epoll_ctl(Pthis->epoll_fd_, EPOLL_CTL_ADD, infd, &event);
if (status == -1)
{
Pthis->logging_->elog("epoll_ctl");
return 0;
}
}
continue;
}
else
{
/* We have data on the fd waiting to be read.*/
int done = 0;
while (true)
{
int byte_received;
int socket_fd_index=Pthis->lower_sock_fd_to_index[events[i].data.fd];
byte_received=read(events[i].data.fd,
(char*)Pthis->block_for_socket_[socket_fd_index]->getBlock()+Pthis->block_for_socket_[socket_fd_index]->GetCurSize(),
Pthis->block_for_socket_[socket_fd_index]->GetRestSize());
if(byte_received==-1){
if(errno==EAGAIN){
/*We have read all the data,so go back to the loop.*/
break;
}
Pthis->logging_->elog("read error!\n");
done = 1;
}
else if (byte_received == 0)
{
/* End of file. The remote has closed the connection.*/
done = 1;
break;
}
/* The data is successfully read.*/
Pthis->block_for_socket_[socket_fd_index]->IncreaseActualSize(byte_received);
if (Pthis->block_for_socket_[socket_fd_index]->GetRestSize() > 0)
{
/** the current block is not read entirely from the sender, so continue the loop to read.**/
continue;
}
/** a block is completely read. **/
Pthis->logging_->log("[%ld] The %d-th block is received from Lower[%s]", Pthis->state.exchange_id_, Pthis->debug_received_block[socket_fd_index],
Pthis->lower_ip_array[socket_fd_index].c_str());
Pthis->debug_received_block[socket_fd_index]++;
/** deserialize the data block from sender to the blockstreambase (received_block_stream_) **/
Pthis->received_block_stream_->deserialize((Block*) Pthis->block_for_socket_[socket_fd_index]);
/** mark block_for_socket_[socket_fd_index] to be empty so that it can accommodate the subsequent data **/
Pthis->block_for_socket_[socket_fd_index]->reset();
/** In the current implementation, a empty block stream means End-Of-File**/
const bool eof=Pthis->received_block_stream_->Empty();
if(!eof){
/** the newly obtained data block is validate, so we insert it into the buffer and post
* sem_new_block_or_eof_ so that all the threads waiting for the semaphore continue. **/
Pthis->buffer->insertBlock(Pthis->received_block_stream_);
//??? why is all ,not 1
// multiple threads will still compete with lock
Pthis->sem_new_block_or_eof_.post(Pthis->number_of_registered_expanded_threads_);
}
else
{
/** The newly obtained data block is the end-of-file. **/
Pthis->logging_->log("[%ld] *****This block is the last one.", Pthis->state.exchange_id_);
finish_times.push_back((int)getMilliSecond(start));
/** update the exhausted senders count and post sem_new_block_or_eof_ so that all the
* threads waiting for the semaphore continue.
**/
Pthis->nexhausted_lowers++;
Pthis->sem_new_block_or_eof_.post(Pthis->number_of_registered_expanded_threads_);
if (Pthis->nexhausted_lowers == Pthis->nlowers)
{
/*
* When all the exchange lowers are exhausted, notify the buffer
* that the input data is completely received.
*/
Pthis->buffer->setInputComplete();
/* print the finish times */
for(unsigned i=0;i<finish_times.size();i++){
printf("%d\t",finish_times[i]);
}
printf("\t Var:%5.4f\n",get_stddev(finish_times));
}
Pthis->logging_->log(
"[%ld] <<<<<<<<<<<<<<<<nexhausted_lowers=%d>>>>>>>>>>>>>>>>exchange=(%d,%d)",
Pthis->state.exchange_id_, Pthis->nexhausted_lowers,
Pthis->state.exchange_id_, Pthis->partition_offset);
/** tell the sender that all the block are consumed so that the sender can close the socket**/
Pthis->SendBlockAllConsumedNotification(events[i].data.fd);
Pthis->logging_->log("[%ld] This notification (all the blocks in the socket buffer are consumed) is send to the lower[%s] exchange=(%d,%d).\n",
Pthis->state.exchange_id_, Pthis->lower_ip_array[socket_fd_index].c_str(), Pthis->state.exchange_id_, Pthis->partition_offset);
}
}
if (done)
{
Pthis->logging_->log("[%ld] Closed connection on descriptor %d[%s]\n", Pthis->state.exchange_id_, events[i].data.fd,
Pthis->lower_ip_array[Pthis->lower_sock_fd_to_index[events[i].data.fd]].c_str());
/* Closing the descriptor will make epoll remove it
from the set of descriptors which are monitored. */
FileClose(events[i].data.fd);
}
}
}
}
}
/**
* first, block_for_socket_ for receive data from senders, then if one block is
* enough, next serialize it and put it into all_merged_block_buffer.
* epoll is good at listening every coming block for different socket.
*
*/
void* ExchangeMerger::Receiver(void* arg) {
ExchangeMerger* Pthis = reinterpret_cast<ExchangeMerger*>(arg);
struct epoll_event event;
struct epoll_event* events;
int status;
// create epoll
Pthis->epoll_fd_ = epoll_create1(0);
if (Pthis->epoll_fd_ == -1) {
LOG(ERROR) << " exchange_id = " << Pthis->state_.exchange_id_
<< " partition_offset = " << Pthis->partition_offset_
<< " merger fail to create epoll!" << std::endl;
return NULL;
}
event.data.fd = Pthis->sock_fd_;
event.events = EPOLLIN | EPOLLET;
status = epoll_ctl(Pthis->epoll_fd_, EPOLL_CTL_ADD, Pthis->sock_fd_, &event);
if (-1 == status) {
LOG(ERROR) << " exchange_id = " << Pthis->state_.exchange_id_
<< " partition_offset = " << Pthis->partition_offset_
<< " merger fail to create epoll_ctl!" << std::endl;
return NULL;
}
events = reinterpret_cast<epoll_event*>(
calloc(Pthis->lower_num_, sizeof(epoll_event)));
int fd_cur = 0;
ticks start = curtick();
std::vector<int> finish_times; // in ms
while (true) {
usleep(1);
const int event_count =
epoll_wait(Pthis->epoll_fd_, events, Pthis->lower_num_, -1);
for (int i = 0; i < event_count; i++) {
if ((events[i].events & EPOLLERR) || (events[i].events & EPOLLHUP) ||
(!(events[i].events & EPOLLIN))) {
if (errno == EINTR) {
continue;
}
LOG(WARNING) << " exchange_id = " << Pthis->state_.exchange_id_
<< " partition_offset = " << Pthis->partition_offset_
<< " epoll error,reason: " << strerror(errno)
<< " close fd = " << events[i].data.fd << std::endl;
FileClose(events[i].data.fd);
continue;
} else if (Pthis->sock_fd_ == events[i].data.fd) {
/* We have a notification on the listening socket, which means one or
* more incoming connections.
*/
while (true) {
sockaddr in_addr;
socklen_t in_len;
int infd;
char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
in_len = sizeof in_addr;
infd = accept(Pthis->sock_fd_, &in_addr, &in_len);
if (infd == -1) {
if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) {
/* all the incoming connections are processed.*/
break;
} else {
LOG(WARNING) << " exchange_id = " << Pthis->state_.exchange_id_
<< " partition_offset = " << Pthis->partition_offset_
<< " epoll accept error, try again!" << std::endl;
break;
}
}
status = getnameinfo(&in_addr, in_len, hbuf, sizeof(hbuf), sbuf,
sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV);
if (0 == status) {
LOG(INFO) << " exchange_id = " << Pthis->state_.exchange_id_
<< " partition_offset = " << Pthis->partition_offset_
<< " Accepted connection on descriptor " << infd
<< " host= " << hbuf << " port= " << sbuf << std::endl;
Pthis->lower_ip_list_.push_back(hbuf);
Pthis->lower_sock_fd_to_id_[infd] =
Pthis->lower_ip_list_.size() - 1;
assert(Pthis->lower_ip_list_.size() <=
Pthis->state_.lower_id_list_.size());
}
/*Make the incoming socket non-blocking and add it to the list of fds
* to monitor.*/
if (!Pthis->SetSocketNonBlocking(infd)) {
return 0;
}
event.data.fd = infd;
event.events = EPOLLIN | EPOLLET;
status = epoll_ctl(Pthis->epoll_fd_, EPOLL_CTL_ADD, infd, &event);
if (-1 == status) {
LOG(ERROR) << " exchange_id = " << Pthis->state_.exchange_id_
<< " partition_offset = " << Pthis->partition_offset_
<< " epoll_ctl error2" << std::endl;
return NULL;
}
}
continue;
} else {
/* We have data on the fd waiting to be read.*/
int done = 0;
while (true) {
int byte_received;
int socket_fd_index = Pthis->lower_sock_fd_to_id_[events[i].data.fd];
byte_received =
read(events[i].data.fd,
(reinterpret_cast<char*>(
Pthis->block_for_socket_[socket_fd_index]->getBlock())) +
Pthis->block_for_socket_[socket_fd_index]->GetCurSize(),
Pthis->block_for_socket_[socket_fd_index]->GetRestSize());
if (byte_received == -1) {
if (errno == EAGAIN) {
/*We have read all the data,so go back to the loop.*/
break;
}
LOG(WARNING) << " exchange_id = " << Pthis->state_.exchange_id_
<< " partition_offset = " << Pthis->partition_offset_
<< " merger read error!" << std::endl;
done = 1;
} else if (byte_received == 0) {
/* End of file. The remote has closed the connection.*/
done = 1;
break;
}
/* The data is successfully read.*/
Pthis->block_for_socket_[socket_fd_index]->IncreaseActualSize(
byte_received);
if (Pthis->block_for_socket_[socket_fd_index]->GetRestSize() > 0) {
/** the current block is not read entirely from the Sender, so
* continue the loop to read.**/
continue;
}
/** a block is completely read. **/
/** deserialize the data block from Sender to the blockstreambase
* (block_for_deserialization) **/
Pthis->block_for_deserialization->deserialize(
reinterpret_cast<Block*>(
Pthis->block_for_socket_[socket_fd_index]));
/** mark block_for_socket_[socket_fd_index] to be empty so that it can
* accommodate the subsequent data **/
Pthis->block_for_socket_[socket_fd_index]->reset();
/**
* In the current implementation, a empty block stream means
* End-Of-File
*/
const bool eof = Pthis->block_for_deserialization->Empty();
if (!eof) {
/** the newly obtained data block is validate, so we insert it into
* the all_merged_block_buffer_ and post sem_new_block_or_eof_ so
* that all the threads waiting for the semaphore continue.
*/
Pthis->all_merged_block_buffer_->insertBlock(
Pthis->block_for_deserialization);
//??? why is all ,not 1
// multiple threads will still compete with lock
Pthis->sem_new_block_or_eof_.post(
Pthis->number_of_registered_expanded_threads_);
} else {
/** The newly obtained data block is the end-of-file. **/
LOG(INFO) << " exchange_id = " << Pthis->state_.exchange_id_
<< " partition_offset = " << Pthis->partition_offset_
<< " This block is the last one." << std::endl;
finish_times.push_back(static_cast<int>(getMilliSecond(start)));
/** update the exhausted senders count and post
*sem_new_block_or_eof_ so that all the
* threads waiting for the semaphore continue.
**/
Pthis->exhausted_lowers++;
Pthis->sem_new_block_or_eof_.post(
Pthis->number_of_registered_expanded_threads_);
if (Pthis->exhausted_lowers == Pthis->lower_num_) {
/*
* When all the exchange lowers are exhausted, notify the
* all_merged_block_buffer_
* that the input data is completely received.
*/
Pthis->all_merged_block_buffer_->setInputComplete();
/* print the finish times */
// for (unsigned i = 0; i < finish_times.size(); i++)
// {
// printf("%d\t", finish_times[i]);
// }
// printf("\t Var:%5.4f\n", get_stddev(finish_times));
}
LOG(INFO) << " exchange_id = " << Pthis->state_.exchange_id_
<< " partition_offset = " << Pthis->partition_offset_
<< " exhausted lowers = " << Pthis->exhausted_lowers
<< " senders have exhausted" << std::endl;
/** tell the Sender that all the block are consumed so that the
* Sender can close the socket**/
Pthis->ReplyAllBlocksConsumed(events[i].data.fd);
LOG(INFO)
<< " exchange_id = " << Pthis->state_.exchange_id_
<< " partition_offset = " << Pthis->partition_offset_
<< " This notification (all the blocks in the socket buffer "
"are consumed) is replied to the lower "
<< Pthis->lower_ip_list_[socket_fd_index].c_str() << std::endl;
}
}
if (done) {
LOG(INFO)
<< " exchange_id = " << Pthis->state_.exchange_id_
<< " partition_offset = " << Pthis->partition_offset_
<< " Closed connection on descriptor " << events[i].data.fd << " "
<< Pthis->lower_ip_list_
[Pthis->lower_sock_fd_to_id_[events[i].data.fd]].c_str()
<< std::endl;
/* Closing the descriptor will make epoll remove it
from the set of descriptors which are monitored. */
FileClose(events[i].data.fd);
}
}
}
}
}
