int SRLockClient::start_client () {
SRClientContext ctx;
char temp_char;
ctx.ib_port = SERVER_IB_PORT;
srand (generate_random_seed()); // initialize random seed
TEST_NZ (establish_tcp_connection(SERVER_ADDR.c_str(), SERVER_TCP_PORT, &(ctx.sockfd)));
DEBUG_COUT("[Comm] Client connected to LM on sock " << ctx.sockfd);
TEST_NZ (ctx.create_context());
DEBUG_COUT("[Info] Context Created " << ctx.sockfd);
TEST_NZ (RDMACommon::connect_qp (&(ctx.qp), ctx.ib_port, ctx.port_attr.lid, ctx.sockfd));
DEBUG_COUT("[Conn] QP connected!");
start_operation(ctx);
// Sync so server will know that client is done mucking with its memory
DEBUG_COUT("[Info] Client is done, and is ready to destroy its resources!");
TEST_NZ (sock_sync_data (ctx.sockfd, 1, "W", &temp_char)); /* just send a dummy char back and forth */
TEST_NZ(ctx.destroy_context());
}
int RDMAServer::initialize_data_structures(){
global_items_region = new ItemVersion[ITEM_CNT * MAX_ITEM_VERSIONS];
//OrdersVersion* RDMAServer::orders_region = new OrdersVersion[MAX_ORDERS_CNT * MAX_ORDERS_VERSIONS];
global_orders_region = new OrdersVersion[MAX_BUFFER_SIZE]; // TODO
//OrderLineVersion* RDMAServer::order_line_region = new OrderLineVersion[ORDERLINE_PER_ORDER * MAX_ORDERS_CNT * MAX_ORDERLINE_VERSIONS];
global_order_line_region = new OrderLineVersion[MAX_BUFFER_SIZE]; // TODO
//CCXactsVersion* RDMAServer::cc_xacts_region = new CCXactsVersion[MAX_CCXACTS_CNT * MAX_CCXACTS_VERSIONS];
global_cc_xacts_region = new CCXactsVersion[MAX_BUFFER_SIZE]; // TODO
global_timestamp_region = new Timestamp[1];
global_lock_items_region = new uint64_t[ITEM_CNT];
RDMAServer::global_timestamp_region[0].value = 0ULL; // the timestamp counter is initially set to 0
DEBUG_COUT("[Info] Timestamp set to " << RDMAServer::global_timestamp_region[0].value);
TEST_NZ(load_tables_from_files(global_items_region));
DEBUG_COUT("[Info] Tables loaded successfully");
int i;
uint32_t stat;
uint32_t version;
for (i=0; i < ITEM_CNT; i++) {
stat = (uint32_t)0; // 0 for free, 1 for locked
version = (uint32_t)0; // the first version of each item is 0
global_lock_items_region[i] = Lock::set_lock(stat, version);
}
DEBUG_COUT("[Info] All locks set free");
return 0;
}
ClientGroup::~ClientGroup() {
DEBUG_COUT(CLASS_NAME, __func__, "[Info] Deconstructor called ");
for (auto& c : clients_)
delete c;
if (config::SNAPSHOT_CLUSTER_MODE == true)
delete oracleReader_;
}
int Coordinator::start () {
CoordinatorContext ctx[SERVER_CNT];
struct sockaddr_in returned_addr;
socklen_t len = sizeof(returned_addr);
char temp_char;
srand (generate_random_seed()); // initialize random seed
// Call socket(), bind() and listen()
TEST_NZ (server_socket_setup(&server_sockfd, SERVER_CNT));
// accept connections from Cohorts
std::cout << "[Info] Waiting for " << SERVER_CNT << " cohort(s) on port " << TRX_MANAGER_TCP_PORT << std::endl;
for (int s = 0; s < SERVER_CNT; s++) {
ctx[s].sockfd = accept (server_sockfd, (struct sockaddr *) &returned_addr, &len);
if (ctx[s].sockfd < 0){
std::cerr << "ERROR on accept() for RM #" << s << std::endl;
return -1;
}
std::cout << "[Conn] Received Cohort #" << s << " on sock " << ctx[s].sockfd << std::endl;
// create all resources
ctx[s].ib_port = TRX_MANAGER_IB_PORT;
TEST_NZ (ctx[s].create_context());
DEBUG_COUT("[Info] Context for cohort " << s << " created");
// connect the QPs
TEST_NZ (RDMACommon::connect_qp (&(ctx[s].qp), ctx[s].ib_port, ctx[s].port_attr.lid, ctx[s].sockfd));
DEBUG_COUT("[Conn] QPed to cohort " << s);
}
std::cout << "[Info] Established connection to all " << SERVER_CNT << " resource-manager(s)." << std::endl;
start_benchmark(ctx);
DEBUG_COUT("[Info] Coordinator is done, and is ready to destroy its resources!");
for (int i = 0; i < SERVER_CNT; i++) {
TEST_NZ (sock_sync_data (ctx[i].sockfd, 1, "W", &temp_char)); /* just send a dummy char back and forth */
DEBUG_COUT("[Conn] Notified cohort " << i << " it's done");
TEST_NZ ( ctx[i].destroy_context());
}
}
ErrorType MemoryRequestDispatcher::handleCASRequest(std::shared_ptr<CASRequest> &casReqPtr) {
ErrorType eType;
size_t msNum = casReqPtr->getMemoryServerNumber();
std::shared_ptr<CASRequest::CASParameters> paramPtr = casReqPtr->getParameters();
primitive::pointer_size_t expected = paramPtr->expectedHead_.toULL();
primitive::pointer_size_t desired = paramPtr->desiredHead_.toULL();
size_t bucketID = paramPtr->offset_;
eType = replicas_.at(msNum).getHashRegion()->CAS(&expected, desired, paramPtr->offset_);
if (eType == error::SUCCESS)
DEBUG_COUT(CLASS_NAME, __func__, "CASing bucket " << bucketID << " on MS " << msNum << " succeeded (expected: "
<< paramPtr->expectedHead_.toHexString() << ", changed to: " << paramPtr->desiredHead_.toHexString() << ")");
else {
const Pointer& actualHead = Pointer::makePointer(expected);
DEBUG_COUT(CLASS_NAME, __func__, "CASing bucket " << bucketID << " on MS " << msNum << " failed (expected: "
<< paramPtr->expectedHead_.toHexString() << ", real head: " << actualHead.toHexString() << ")");
}
return eType;
}
ErrorType MemoryRequestDispatcher::handleReadEntryRequest(std::shared_ptr<EntryReadRequest> &entryReadReqPtr){
ErrorType eType;
size_t msNum = entryReadReqPtr->getMemoryServerNumber();
primitive::coordinator_num_t cID = entryReadReqPtr->getCoordinatorID();
std::shared_ptr<EntryReadRequest::EntryReadParameters> paramPtr = entryReadReqPtr->getParameters();
LogEntry &entry = paramPtr->localEntry_;
char readBuffer[paramPtr->length_];
eType = replicas_.at(msNum).getLogRegion(cID)->read(readBuffer, paramPtr->remoteBufferOffset_, paramPtr->length_);
if (eType == error::SUCCESS) {
std::string tempStr(readBuffer, paramPtr->length_);
std::istringstream is(tempStr);
LogEntry::deserialize(is, entry);
DEBUG_COUT(CLASS_NAME, __func__, "Log entry " << entry.getCurrentP().toHexString() << " read from log journal[" << (int)cID << "] from MS " << msNum);
}
else
DEBUG_COUT(CLASS_NAME, __func__, "Failure in reading Log entry " << entry.getCurrentP().toHexString() << " from log journal[" << (int)cID << "] from MS " << msNum);
return eType;
}
void MemoryRequestDispatcher::run() {
ErrorType eType;
while (true) {
//std::this_thread::sleep_for(std::chrono::seconds(1));
std::shared_ptr<Request> req = reqBufferPtr_->remove();
if (req->getRequestType() == Request::RequestType::CAS) {
DEBUG_COUT(CLASS_NAME, __func__, "Received CAS request");
std::shared_ptr<CASRequest> casReqPtr = std::dynamic_pointer_cast<CASRequest> (req);
eType = handleCASRequest(casReqPtr);
}
else if (req->getRegionType() == Request::RegionType::LOG) {
if (req->getRequestType() == Request::RequestType::READ) {
DEBUG_COUT(CLASS_NAME, __func__, "Received Log Read request");
std::shared_ptr<EntryReadRequest> entryReadReqPtr = std::dynamic_pointer_cast<EntryReadRequest> (req);
eType = handleReadEntryRequest(entryReadReqPtr);
}
else { // entryReqPtr->getRequestType() == Request::RequestType::WRITE
DEBUG_COUT(CLASS_NAME, __func__, "Received Log Write request");
std::shared_ptr<EntryWriteRequest> entryWriteReqPtr = std::dynamic_pointer_cast<EntryWriteRequest> (req);
eType = handleWriteEntryRequest(entryWriteReqPtr);
}
}
else if (req->getRegionType() == Request::RegionType::HASH) {
if (req->getRequestType() == Request::RequestType::READ) {
DEBUG_COUT(CLASS_NAME, __func__, "Received Bucket Read request");
std::shared_ptr<BucketReadRequest> bucketReadReqPtr = std::dynamic_pointer_cast<BucketReadRequest> (req);
eType = handleReadBucketRequest(bucketReadReqPtr);
}
else // bucketReqPtr->getRequestType() == Request::RequestType::WRITE, which is an error
eType = error::UNKNOWN_REQUEST_TYPE;
}
else if (req->getRegionType() == Request::RegionType::ENTRY_STATE) {
if (req->getRequestType() == Request::RequestType::READ) {
DEBUG_COUT(CLASS_NAME, __func__, "Received State Read request");
std::shared_ptr<StateReadRequest> stateReadReqPtr = std::dynamic_pointer_cast<StateReadRequest> (req);
eType = handleReadStateRequest(stateReadReqPtr);
}
else {
DEBUG_COUT(CLASS_NAME, __func__, "Received State Write request");
std::shared_ptr<StateWriteRequest> stateWriteReqPtr = std::dynamic_pointer_cast<StateWriteRequest> (req);
eType = handleWriteStateRequest(stateWriteReqPtr);
}
}
else {
DEBUG_CERR(CLASS_NAME, __func__, "Received request with unknown type");
eType = error::UNKNOWN_REQUEST_TYPE;
}
// We finally set the std::promise variable
req->setProm(eType);
}
}
ErrorType MemoryRequestDispatcher::handleWriteEntryRequest(std::shared_ptr<EntryWriteRequest> &entryReqPtr){
ErrorType eType;
size_t msNum = entryReqPtr->getMemoryServerNumber();
primitive::coordinator_num_t cID = entryReqPtr->getCoordinatorID();
std::shared_ptr<EntryWriteRequest::EntryWriteParameters> paramPtr = entryReqPtr->getParameters();
const LogEntry &entry = paramPtr->localEntry_;
std::ostringstream os;
entry.serialize(os);
const std::string& tmp = os.str();
const char* cstr = tmp.c_str();
eType = replicas_.at(msNum).getLogRegion(cID)->write(cstr, paramPtr->remoteBufferOffset_, paramPtr->length_);
if (eType == error::SUCCESS)
DEBUG_COUT(CLASS_NAME, __func__, "Log Entry " << entry.getCurrentP().toHexString() << " written on log journal[" << (int)cID << "] on MS " << msNum);
else
DEBUG_COUT(CLASS_NAME, __func__, "Failure in writing log Entry " << entry.getCurrentP().toHexString() << " on log journal[" << (int)cID << "] on MS " << msNum);
return eType;
}
int SRLockClient::release_lock (SRClientContext &ctx, struct LockRequest &req, struct LockResponse &res) {
if (req.request_type == LockRequest::RELEASE){
TEST_NZ (RDMACommon::post_RECEIVE(ctx.qp, ctx.lock_res_mr, (uintptr_t)&res, sizeof(struct LockResponse)));
DEBUG_COUT("[Info] receive posted to the queue");
//TEST_NZ (sock_write(ctx.sockfd, (char *)&req, sizeof(struct LockRequest)));
TEST_NZ (RDMACommon::post_SEND(ctx.qp, ctx.lock_req_mr, (uintptr_t)&req, sizeof(struct LockRequest), true));
TEST_NZ (RDMACommon::poll_completion(ctx.cq)); // Ack for SEND (false)
DEBUG_COUT("[Sent] LockRequest::Request (RELEASE) to LM.");
//TEST_NZ (sock_read(ctx.sockfd, (char *)&res, sizeof(struct LockResponse)));
TEST_NZ (RDMACommon::poll_completion(ctx.cq)); // Receive LockResponse
if (res.response_type == LockResponse::RELEASED)
DEBUG_COUT("[Recv] RELEASE LockResponse (result: released)");
else {
DEBUG_COUT("[Error] RELEASE LockResponse result: " << res.response_type);
}
return 0;
}
return 1;
}
void* DQClientCentricServer::handle_client(void *param) {
DQServerContext *ctx = (DQServerContext *) param;
char temp_char;
DEBUG_COUT("[in handle client]");
// TEST_NZ (RDMACommon::post_RECEIVE(ctx->qp, ctx->recv_data_mr, (uintptr_t)&ctx->recv_data_msg, sizeof(int)));
TEST_NZ (sock_sync_data (ctx->sockfd, 1, "W", &temp_char)); // just send a dummy char back and forth
DEBUG_COUT("[Synced with client]");
/*
int iteration = 0;
while (iteration < OPERATIONS_CNT) {
TEST_NZ (RDMACommon::poll_completion(ctx->cq));
DEBUG_COUT("[Recv] request from client");
TEST_NZ (RDMACommon::post_RECEIVE(ctx->qp, ctx->recv_data_mr, (uintptr_t)&ctx->recv_data_msg, sizeof(int))); // for
DEBUG_COUT("[Info] receive posted to the queue");
if (iteration % 1000 == 0) {
TEST_NZ (RDMACommon::post_SEND(ctx->qp, ctx->send_data_mr, (uintptr_t)ctx->send_data_msg, 10 * sizeof(char), true));
DEBUG_COUT("[Sent] response to client");
TEST_NZ (RDMACommon::poll_completion(ctx->cq)); // for SEND
DEBUG_COUT("[Info] completion received");
}
else {
TEST_NZ (RDMACommon::post_SEND(ctx->qp, ctx->send_data_mr, (uintptr_t)ctx->send_data_msg, 10 * sizeof(char), false));
DEBUG_COUT("[Sent] response to client (without completion)");
}
iteration++;
}
*/
DEBUG_COUT("[Sent] buffer info to client");
}
ErrorType MemoryRequestDispatcher::handleReadBucketRequest(std::shared_ptr<BucketReadRequest> &bucketReadReqPtr) {
ErrorType eType;
size_t msNum = bucketReadReqPtr->getMemoryServerNumber();
std::shared_ptr<BucketReadRequest::BucketReadParameters> paramPtr = bucketReadReqPtr->getParameters();
size_t bucketID = paramPtr->remoteBufferOffset_;
Pointer &pointer = paramPtr->localPointer_;
primitive::pointer_size_t readBuffer[1];
eType = replicas_.at(msNum).getHashRegion()->read(readBuffer, paramPtr->remoteBufferOffset_, sizeof(primitive::pointer_size_t));
if (eType != error::SUCCESS) {
DEBUG_COUT(CLASS_NAME, __func__, "Failure in reading bucket " << bucketID << " from MS " << msNum);
return eType;
}
pointer = Pointer::makePointer(readBuffer[0]);
DEBUG_COUT(CLASS_NAME, __func__, "Bucket " << bucketID << " read from MS " << msNum << " and is pointing to " << pointer.toHexString());
// TODO: what if the bucket is empty
// if (pointer.isNull())
// eType = error::KEY_NOT_FOUND;
return eType;
}
int IPCohort::start_server () {
IPCohortContext ctx;
char temp_char;
TEST_NZ (establish_tcp_connection(TRX_MANAGER_ADDR.c_str(), TRX_MANAGER_TCP_PORT, &ctx.sockfd));
TEST_NZ (ctx.create_context());
srand (time(NULL)); // initialize random seed
start_benchmark(ctx);
/* Sync so server will know that client is done mucking with its memory */
DEBUG_COUT("[Info] Cohort client is done, and is ready to destroy its resources!");
TEST_NZ (sock_sync_data (ctx.sockfd, 1, "W", &temp_char)); /* just send a dummy char back and forth */
TEST_NZ(ctx.destroy_context());
}
ClientGroup::ClientGroup(unsigned instanceID, uint32_t clientsCnt, uint32_t homeServerID, size_t ibPortsCnt)
: clientsCnt_(clientsCnt){
DEBUG_COUT(CLASS_NAME, __func__, "[Info] Constructor called ");
srand ((unsigned int)utils::generate_random_seed()); // initialize random seed
if (config::SNAPSHOT_CLUSTER_MODE == true){
oracleReader_ = new OracleReader(instanceID, clientsCnt, 1);
}
for (uint32_t i = 0; i < clientsCnt; i++) {
uint8_t ibPort = (uint8_t)(i % ibPortsCnt + 1);
uint16_t homeWarehouseID = (uint16_t)(homeServerID * config::tpcc_settings::WAREHOUSE_PER_SERVER + rand() % config::tpcc_settings::WAREHOUSE_PER_SERVER);
uint8_t homeDistrictID = (uint8_t)(rand() % config::tpcc_settings::DISTRICT_PER_WAREHOUSE);
clients_.push_back(new TPCC::TPCCClient(instanceID, homeWarehouseID, homeDistrictID, ibPort, oracleReader_));
}
}
LogJournal::~LogJournal() {
delete[] content_;
delete region_;
DEBUG_COUT (CLASS_NAME, __func__, "LogJournal destroyed!");
}
int IPCohort::start_benchmark(IPCohortContext &ctx) {
char temp_char;
unsigned long long cpu_checkpoint_start, cpu_checkpoint_finish;
int iteration = 0;
struct rusage usage;
struct timeval start_user_usage, start_kernel_usage, end_user_usage, end_kernel_usage;
DEBUG_COUT("[in handle client]");
TEST_NZ (sock_sync_data (ctx.sockfd, 1, "W", &temp_char)); // just send a dummy char back and forth
DEBUG_COUT("[Synced with client]");
// For CPU usage in terms of time
getrusage(RUSAGE_SELF, &usage);
start_kernel_usage = usage.ru_stime;
start_user_usage = usage.ru_utime;
// For CPU usage in terms of clocks (ticks)
cpu_checkpoint_start = rdtsc();
while (iteration < OPERATIONS_CNT) {
//TEST_NZ (RDMACommon::poll_completion(ctx.cq)); // for Receive
TEST_NZ (sock_read(ctx.sockfd, (char *)&(ctx.recv_data_msg), sizeof(int)));
DEBUG_COUT("[READ] --READY-- request from coordinator");
//TEST_NZ (RDMACommon::post_SEND(ctx.qp, ctx.send_data_mr, (uintptr_t)&ctx.send_data_msg, sizeof(int), false));
TEST_NZ (sock_write(ctx.sockfd, (char *)&(ctx.send_data_msg), sizeof(int)));
DEBUG_COUT("[WRIT] --YES-- response to coordinator (without completion)");
//TEST_NZ (RDMACommon::poll_completion(ctx.cq)); // for Receive
TEST_NZ (sock_read(ctx.sockfd, (char *)&(ctx.recv_data_msg), sizeof(int)));
DEBUG_COUT("[READ] --COMMIT-- request from coordinator");
//TEST_NZ (RDMACommon::post_SEND(ctx.qp, ctx.send_data_mr, (uintptr_t)&ctx.send_data_msg, sizeof(int), true));
TEST_NZ (sock_write(ctx.sockfd, (char *)&(ctx.send_data_msg), sizeof(int)));
DEBUG_COUT("[WRIT] --DONE-- response to coordinator");
iteration++;
}
cpu_checkpoint_finish = rdtsc();
getrusage(RUSAGE_SELF, &usage);
end_user_usage = usage.ru_utime;
end_kernel_usage = usage.ru_stime;
double user_cpu_microtime = ( end_user_usage.tv_sec - start_user_usage.tv_sec ) * 1E6 + ( end_user_usage.tv_usec - start_user_usage.tv_usec );
double kernel_cpu_microtime = ( end_kernel_usage.tv_sec - start_kernel_usage.tv_sec ) * 1E6 + ( end_kernel_usage.tv_usec - start_kernel_usage.tv_usec );
DEBUG_COUT("[Info] Cohort Client Done");
unsigned long long average_cpu_clocks = (cpu_checkpoint_finish - cpu_checkpoint_start) / OPERATIONS_CNT;
std::cout << "[Stat] AVG total CPU elapsed time (u sec): " << (user_cpu_microtime + kernel_cpu_microtime) / OPERATIONS_CNT << std::endl;
std::cout << "[Stat] Average CPU clocks: " << average_cpu_clocks << std::endl;
return 0;
}
int DQClientCentricServer::start_server () {
tcp_port = SERVER_TCP_PORT;
ib_port = SERVER_IB_PORT;
DQServerContext ctx[CLIENTS_CNT];
struct sockaddr_in serv_addr, cli_addr;
socklen_t clilen = sizeof(cli_addr);
char temp_char;
std::string clients_addresses; // separated by the delimiter '|'
std::string clients_tcp_port; // separated by the delimiter '|'
TEST_NZ(initialize_data_structures());
std::cout << "[Info] Server is waiting for " << CLIENTS_CNT
<< " client(s) on tcp port: " << tcp_port << ", ib port: " << ib_port << std::endl;
// Open Socket
server_sockfd = socket (AF_INET, SOCK_STREAM, 0);
if (server_sockfd < 0) {
std::cerr << "Error opening socket" << std::endl;
return -1;
}
// Bind
memset(&serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = INADDR_ANY;
serv_addr.sin_port = htons(tcp_port);
TEST_NZ(bind(server_sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)));
// listen
TEST_NZ(listen (server_sockfd, CLIENTS_CNT));
// accept connections
for (int i = 0; i < CLIENTS_CNT; i++){
initialize_context(ctx[i]);
ctx[i].sockfd = accept (server_sockfd, (struct sockaddr *) &cli_addr, &clilen);
if (ctx[i].sockfd < 0){
std::cerr << "ERROR on accept" << std::endl;
return -1;
}
std::cout << "[Conn] Received client #" << i << " on socket " << ctx[i].sockfd << std::endl;
ctx[i].client_ip = "";
ctx[i].client_ip += std::string(inet_ntoa (cli_addr.sin_addr));
clients_addresses.append(ctx[i].client_ip);
clients_tcp_port.append(std::to_string(8000 + i));
// if the client is not the last client, add the delimiter
if (i != CLIENTS_CNT - 1){
clients_addresses.append("|");
clients_tcp_port.append("|");
}
// create all resources
TEST_NZ (ctx[i].create_context());
DEBUG_COUT("[Info] Context for client " << i << " (" << ctx[i].client_ip << ") created");
// connect the QPs
TEST_NZ (RDMACommon::connect_qp (&(ctx[i].qp), ctx[i].ib_port, ctx[i].port_attr.lid, ctx[i].sockfd));
DEBUG_COUT("[Conn] QPed to client " << i);
}
for (int i = 0; i < CLIENTS_CNT; i++){
// send memory locations using SEND
memcpy(&(ctx[i].send_message_msg.peer_mr), ctx[i].locks_mr, sizeof(struct ibv_mr));
ctx[i].send_message_msg.client_id = static_cast<uint32_t>(i + 1);
strcpy(ctx[i].send_message_msg.clients_addr, clients_addresses.c_str());
strcpy(ctx[i].send_message_msg.clients_tcp_port, clients_tcp_port.c_str());
DEBUG_COUT("[Sent] Cheeck: " << ctx[i].send_message_msg.clients_addr);
TEST_NZ (RDMACommon::post_SEND (ctx[i].qp, ctx[i].send_message_mr, (uintptr_t)&ctx[i].send_message_msg, sizeof (struct MemoryKeys), true));
TEST_NZ (RDMACommon::poll_completion(ctx[i].cq));
DEBUG_COUT("[Sent] buffer info to client " << i);
}
// Server waits for the client to muck with its memory
/*************** THIS IS FOR SEND
// accept connections
for (int i = 0; i < CLIENTS_CNT; i++){
initialize_context(ctx[i]);
ctx[i].sockfd = accept (server_sockfd, (struct sockaddr *) &cli_addr, &clilen);
if (ctx[i].sockfd < 0){
std::cerr << "ERROR on accept" << std::endl;
return -1;
}
std::cout << "[Conn] Received client #" << i << " on socket " << ctx[i].sockfd << std::endl;
// create all resources
TEST_NZ (ctx[i].create_context());
DEBUG_COUT("[Info] Context for client " << i << " created");
// connect the QPs
TEST_NZ (RDMACommon::connect_qp (&(ctx[i].qp), ctx[i].ib_port, ctx[i].port_attr.lid, ctx[i].sockfd));
DEBUG_COUT("[Conn] QPed to client " << i);
pthread_create(&master_threads[i], NULL, BenchmarkServer::handle_client, &ctx[i]);
}
std::cout << "[Info] Established connection to all " << CLIENTS_CNT << " client(s)." << std::endl;
//wait for handlers to finish
for (int i = 0; i < CLIENTS_CNT; i++) {
pthread_join(master_threads[i], NULL);
}
*/
for (int i = 0; i < CLIENTS_CNT; i++) {
TEST_NZ (sock_sync_data (ctx[i].sockfd, 1, "W", &temp_char)); // just send a dummy char back and forth
DEBUG_COUT("[Conn] Client " << i << " notified it's finished");
TEST_NZ (ctx[i].destroy_context());
std::cout << "[Info] Destroying client " << i << " resources" << std::endl;
}
std::cout << "[Info] Server's ready to gracefully get destroyed" << std::endl;
}
int SRLockClient::start_operation (SRClientContext &ctx) {
struct timespec firstRequestTime, lastRequestTime, beforeAcquisition, afterAcquisition, beforeRelease, afterRelease;
char temp_char;
struct LockRequest req_cpy;
double sumExclusiveAcqTime= 0.0, sumSharedAcqTime= 0.0, sumExclusiveRelTime= 0.0, sumSharedRelime= 0.0;
double acquisitionTime, releaseTime;
int exclusiveCount= 0, sharedCount= 0;
clock_gettime(CLOCK_REALTIME, &firstRequestTime); // Fire the timer
ctx.op_num = 0;
while (ctx.op_num < OPERATIONS_CNT) {
ctx.op_num = ctx.op_num + 1;
DEBUG_COUT (std::endl << "[Info] Submitting lock request #" << ctx.op_num);
// ************************************************************************
// Client request a lock on the selected item and waits for a response.
select_item(ctx.lock_request);
req_cpy.request_item = ctx.lock_request.request_item; // to keep track of the initial request type, because ctx.lock_request is going to change
req_cpy.request_type = ctx.lock_request.request_type; // to keep track of the initial request type, because ctx.lock_request is going to change
clock_gettime(CLOCK_REALTIME, &beforeAcquisition); // Fire the timer
acquire_lock (ctx, ctx.lock_request, ctx.lock_response);
clock_gettime(CLOCK_REALTIME, &afterAcquisition); // Fire the timer
acquisitionTime = ( afterAcquisition.tv_sec - beforeAcquisition.tv_sec ) * 1E6 + ( afterAcquisition.tv_nsec - beforeAcquisition.tv_nsec )/1E3;
// hold lock for a specified time period (in a specific way).
hold_lock();
// ************************************************************************
// Clients release the acquired lock.
ctx.lock_request.request_type = LockRequest::RELEASE;
clock_gettime(CLOCK_REALTIME, &beforeRelease); // Fire the timer
release_lock (ctx, ctx.lock_request, ctx.lock_response);
clock_gettime(CLOCK_REALTIME, &afterRelease); // Fire the timer
releaseTime = ( afterRelease.tv_sec - beforeRelease.tv_sec ) * 1E6 + ( afterRelease.tv_nsec - beforeRelease.tv_nsec )/1E3;
if(req_cpy.request_type == LockRequest::EXCLUSIVE){
sumExclusiveAcqTime += acquisitionTime;
sumExclusiveRelTime += releaseTime;
exclusiveCount++;
}
else if(req_cpy.request_type == LockRequest::SHARED){
sumSharedAcqTime += acquisitionTime;
sumSharedRelime += releaseTime;
sharedCount++;
}
}
clock_gettime(CLOCK_REALTIME, &lastRequestTime); // Fire the timer
double micro_elapsed_time = ( lastRequestTime.tv_sec - firstRequestTime.tv_sec ) * 1E6 + ( lastRequestTime.tv_nsec - firstRequestTime.tv_nsec )/1E3;
double lock_per_sec = (double)(OPERATIONS_CNT / (double)(micro_elapsed_time / 1000000));
// std::cout << std::endl << "[Stat] Locks (acquire & release) per sec: " << lock_per_sec << std::endl;
// std::cout << "[STAT] Avg time per Exclusive acquisition (us) " << sumExclusiveAcqTime / exclusiveCount << std::endl;
// std::cout << "[STAT] Avg time per Exclusive release (us) " << sumExclusiveRelTime / exclusiveCount << std::endl;
// std::cout << "[STAT] Avg time per Shared acquisition (us) " << sumSharedAcqTime / sharedCount << std::endl;
// std::cout << "[STAT] Avg time per Shared release (us) " << sumSharedRelime / sharedCount << std::endl;
std::cout << "[Summary] " << lock_per_sec << ", " << sumExclusiveAcqTime / exclusiveCount << ", " << sumExclusiveRelTime / exclusiveCount << ", " << sumSharedAcqTime / sharedCount << ", " << sumSharedRelime / sharedCount << std::endl;
return 0;
}
int BaseTradTrxManager::initialize_data_structures(){
timestamp.value = 0ULL; // the timestamp counter is initially set to 0
DEBUG_COUT("[Info] Timestamp set to " << timestamp.value);
return 0;
}
MemoryRequestDispatcher::~MemoryRequestDispatcher() {
DEBUG_COUT (CLASS_NAME, __func__, "MemoryRequestDispatcher destroyed!");
}
int Coordinator::start_benchmark(CoordinatorContext *ctx) {
int signaledPosts = 0;
struct timespec firstRequestTime, lastRequestTime; // for calculating TPMS
struct timespec beforeSending, afterSending; // for calculating TPMS
struct timespec before_read_ts, after_read_ts, after_fetch_info, after_commit_ts, after_lock, after_decrement, after_unlock;
struct rusage usage;
struct timeval start_user_usage, start_kernel_usage, end_user_usage, end_kernel_usage;
char temp_char;
unsigned long long cpu_checkpoint_start, cpu_checkpoint_2, cpu_checkpoint_3, cpu_checkpoint_finish ;
bool signalled_flag = false;
for (int i = 0; i < SERVER_CNT; i++) {
TEST_NZ (sock_sync_data (ctx[i].sockfd, 1, "W", &temp_char)); // just send a dummy char back and forth
}
DEBUG_COUT ("[Info] Benchmark now gets started");
clock_gettime(CLOCK_REALTIME, &firstRequestTime); // Fire the timer
getrusage(RUSAGE_SELF, &usage);
start_kernel_usage = usage.ru_stime;
start_user_usage = usage.ru_utime;
int iteration = 0;
cpu_checkpoint_start = rdtsc();
while (iteration < OPERATIONS_CNT) {
DEBUG_COUT("iteration " << iteration);
// Sent: Ready??
for (int i = 0; i < SERVER_CNT; i++) {
TEST_NZ (RDMACommon::post_RECEIVE(ctx[i].qp, ctx[i].recv_data_mr, (uintptr_t)&ctx[i].recv_data_msg, sizeof(int)));
DEBUG_COUT("[Info] Receive posted");
TEST_NZ (RDMACommon::post_SEND(ctx[i].qp, ctx[i].send_data_mr, (uintptr_t)&ctx[i].send_data_msg, sizeof(int), false));
DEBUG_COUT("[Sent] --READY-- request sent to cohort " << i);
}
// Received: Ready!
for (int i = 0; i < SERVER_CNT; i++) {
TEST_NZ (RDMACommon::poll_completion(ctx[i].cq)); // for RECV
//TEST_NZ (RDMACommon::event_based_poll_completion(ctx[i].comp_channel, ctx[i].cq)); // for RECV
DEBUG_COUT("[Recv] --YES-- response received from cohort " << i);
}
// Sent: Commit
for (int i = 0; i < SERVER_CNT; i++) {
TEST_NZ (RDMACommon::post_RECEIVE(ctx[i].qp, ctx[i].recv_data_mr, (uintptr_t)&ctx[i].recv_data_msg, sizeof(int)));
DEBUG_COUT("[Info] Receive posted");
if (iteration % 500 == 0) {
signalled_flag = true;
TEST_NZ (RDMACommon::post_SEND(ctx[i].qp, ctx[i].send_data_mr, (uintptr_t)&ctx[i].send_data_msg, sizeof(int), true));
DEBUG_COUT("[Sent] --COMMIT-- request sent to cohort " << i);
}
else{
signalled_flag = false;
TEST_NZ (RDMACommon::post_SEND(ctx[i].qp, ctx[i].send_data_mr, (uintptr_t)&ctx[i].send_data_msg, sizeof(int), false));
DEBUG_COUT("[Sent] --COMMIT-- request sent to cohort (unsignalled) " << i);
}
}
// collect completion events for SEND
if (signalled_flag){
for (int i = 0; i < SERVER_CNT; i++) {
TEST_NZ (RDMACommon::poll_completion(ctx[i].cq)); // for SENT
//TEST_NZ (RDMACommon::event_based_poll_completion(ctx[i].comp_channel, ctx[i].cq)); // for SENT
}
}
// Received: Done!
for (int i = 0; i < SERVER_CNT; i++) {
TEST_NZ (RDMACommon::poll_completion(ctx[i].cq)); // for RECV
//TEST_NZ (RDMACommon::event_based_poll_completion(ctx[i].comp_channel, ctx[i].cq)); // for RECV
DEBUG_COUT("[Recv] --DONE-- response received from cohort " << i);
}
iteration++;
}
cpu_checkpoint_finish = rdtsc();
getrusage(RUSAGE_SELF, &usage);
end_user_usage = usage.ru_utime;
end_kernel_usage = usage.ru_stime;
clock_gettime(CLOCK_REALTIME, &lastRequestTime); // Fire the timer
double user_cpu_microtime = ( end_user_usage.tv_sec - start_user_usage.tv_sec ) * 1E6 + ( end_user_usage.tv_usec - start_user_usage.tv_usec );
double kernel_cpu_microtime = ( end_kernel_usage.tv_sec - start_kernel_usage.tv_sec ) * 1E6 + ( end_kernel_usage.tv_usec - start_kernel_usage.tv_usec );
double micro_elapsed_time = ( ( lastRequestTime.tv_sec - firstRequestTime.tv_sec ) * 1E9 + ( lastRequestTime.tv_nsec - firstRequestTime.tv_nsec ) ) / 1000;
double latency_in_micro = (double)(micro_elapsed_time / OPERATIONS_CNT);
//double latency_in_micro = (double)(cumulative_latency / signaledPosts) / 1000;
double mega_byte_per_sec = ((sizeof(int) * OPERATIONS_CNT / 1E6 ) / (micro_elapsed_time / 1E6) );
double operations_per_sec = OPERATIONS_CNT / (micro_elapsed_time / 1E6);
double cpu_utilization = (user_cpu_microtime + kernel_cpu_microtime) / micro_elapsed_time;
unsigned long long average_cpu_clocks = (cpu_checkpoint_finish - cpu_checkpoint_start) / OPERATIONS_CNT;
std::cout << "[Stat] Avg latency(u sec): " << latency_in_micro << std::endl;
std::cout << "[Stat] MegaByte per Sec: " << mega_byte_per_sec << std::endl;
std::cout << "[Stat] Operations per Sec: " << operations_per_sec << std::endl;
std::cout << "[Stat] CPU utilization: " << cpu_utilization << std::endl;
std::cout << "[Stat] USER CPU utilization: " << user_cpu_microtime / micro_elapsed_time << std::endl;
std::cout << "[Stat] KERNEL CPU utilization: " << kernel_cpu_microtime / micro_elapsed_time << std::endl;
std::cout << "[Stat] AVG USER CPU elapsed time (u sec): " << user_cpu_microtime / OPERATIONS_CNT << std::endl;
std::cout << "[Stat] AVG KERNEL CPU elapsed time (u sec): " << kernel_cpu_microtime / OPERATIONS_CNT << std::endl;
std::cout << "[Stat] AVG total CPU elapsed time (u sec): " << (user_cpu_microtime + kernel_cpu_microtime) / OPERATIONS_CNT << std::endl;
std::cout << "[Stat] Average CPU clocks: " << average_cpu_clocks << std::endl;
std::cout << latency_in_micro << '\t' << mega_byte_per_sec << '\t' << operations_per_sec << '\t' << cpu_utilization << std::endl;
return 0;
}
int RDMAServer::start_server (int server_num) {
tcp_port = TCP_PORT[server_num];
ib_port = IB_PORT[server_num];
RDMAServerContext ctx[CLIENTS_CNT];
char temp_char;
TEST_NZ(initialize_data_structures());
std::cout << "[Info] Server " << server_num << " is waiting for " << CLIENTS_CNT
<< " client(s) on tcp port: " << tcp_port << ", ib port: " << ib_port << std::endl;
server_sockfd = -1;
struct sockaddr_in serv_addr, cli_addr;
socklen_t clilen = sizeof(cli_addr);
pthread_t master_threads[CLIENTS_CNT];
// Open Socket
server_sockfd = socket (AF_INET, SOCK_STREAM, 0);
if (server_sockfd < 0) {
std::cerr << "Error opening socket" << std::endl;
return -1;
}
// Bind
memset(&serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = INADDR_ANY;
serv_addr.sin_port = htons(tcp_port);
TEST_NZ(bind(server_sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)));
// listen
TEST_NZ(listen (server_sockfd, CLIENTS_CNT));
// accept connections
for (int i = 0; i < CLIENTS_CNT; i++){
initialize_context(ctx[i]);
ctx[i].sockfd = accept (server_sockfd, (struct sockaddr *) &cli_addr, &clilen);
if (ctx[i].sockfd < 0){
std::cerr << "ERROR on accept" << std::endl;
return -1;
}
std::cout << "[Conn] Received client #" << i << " on socket " << ctx[i].sockfd << std::endl;
// pthread_create(&master_threads[i], NULL, RDMAServer::handle_client, &client_socks[i]);
// create all resources
TEST_NZ (ctx[i].create_context());
DEBUG_COUT("[Info] Context for client " << i << " created");
// connect the QPs
TEST_NZ (RDMACommon::connect_qp (&(ctx[i].qp), ctx[i].ib_port, ctx[i].port_attr.lid, ctx[i].sockfd));
DEBUG_COUT("[Conn] QPed to client " << i);
// prepare server buffer with read message
memcpy(&(ctx[i].send_msg.mr_items), ctx[i].mr_items, sizeof(struct ibv_mr));
memcpy(&(ctx[i].send_msg.mr_orders), ctx[i].mr_orders, sizeof(struct ibv_mr));
memcpy(&(ctx[i].send_msg.mr_order_line),ctx[i].mr_order_line, sizeof(struct ibv_mr));
memcpy(&(ctx[i].send_msg.mr_cc_xacts), ctx[i].mr_cc_xacts, sizeof(struct ibv_mr));
memcpy(&(ctx[i].send_msg.mr_timestamp), ctx[i].mr_timestamp, sizeof(struct ibv_mr));
memcpy(&(ctx[i].send_msg.mr_lock_items),ctx[i].mr_lock_items, sizeof(struct ibv_mr));
}
for (int i = 0; i < CLIENTS_CNT; i++){
// send memory locations using SEND
TEST_NZ (RDMACommon::post_SEND (ctx[i].qp, ctx[i].send_mr, (uintptr_t)&ctx[i].send_msg, sizeof(struct MemoryKeys), true));
TEST_NZ (RDMACommon::poll_completion(ctx[i].cq));
DEBUG_COUT("[Sent] buffer info to client " << i);
}
/*
Server waits for the client to muck with its memory
*/
for (int i = 0; i < CLIENTS_CNT; i++) {
TEST_NZ (sock_sync_data (ctx[i].sockfd, 1, "W", &temp_char)); /* just send a dummy char back and forth */
DEBUG_COUT("[Conn] Client " << i << " notified it's finished");
TEST_NZ (ctx[i].destroy_context());
std::cout << "[Info] Destroying client " << i << " resources" << std::endl;
}
std::cout << "[Info] Server's ready to gracefully get destroyed" << std::endl;
}
MemoryRequestDispatcher::MemoryRequestDispatcher()
: replicas_(config::MEMORY_SERVER_CNT){
DEBUG_COUT (CLASS_NAME, __func__, "MemoryRequestDispatcher created!");
//reqBufferPtr_.reset(new LIFORequestBuffer());
reqBufferPtr_.reset(new RandomRequestBuffer());
}