/* a function that creates a ZMQ socket and bind an open port in the configured range * @param start_port the start port range * @param end_port the end port range * @param ctx ZeroMQ socket context * @param port_in_use an out parameter that returns the port address that is bind to a socket * @param type the socket type. default is ZMQ_PULL * @ret a socket that is created from the function - THIS HAS TO BE FREED AFTER USAGE!!!!!! * @throws PrestoWarningException when there is no available open port in the configured range */ socket_t* CreateBindedSocket(int start_port, int end_port, context_t* ctx, int* port_in_use, int type) { socket_t* sock = new socket_t(*ctx, type); #ifdef ZMQ_LINGER int linger_period = SOCK_LINGER_TIME; sock->setsockopt(ZMQ_LINGER, &linger_period, sizeof(linger_period)); #endif int num_port_cand = end_port - start_port + 1; bool bind_succeed = false; port_bind_mutex.lock(); for (int i = 0; i < num_port_cand; ++i){ cur_port_assigned = (cur_port_assigned >= start_port && cur_port_assigned <= end_port) ? cur_port_assigned : start_port; // to make the value circulate after hitting the end_port string endpoint = "tcp://*:"+int_to_string(cur_port_assigned); try { sock->bind(endpoint.c_str()); bind_succeed = true; *port_in_use = (cur_port_assigned++); break; } catch(const std::exception& e) { ++cur_port_assigned; continue; } } port_bind_mutex.unlock(); if (bind_succeed == false) { LOG_ERROR("CreateBindedSocket ZMQ - fail to bind a port (no available port)"); throw PrestoWarningException("There is no open port in the configured range"); } return sock; }
/** * Appends data bytes to main buffer * - Mallocs required buffer size. Appends received data bytes to the buffer. * - Keeps track to total number of rows and total data size received * - When total number of rows is > or equal to partition size, * creates a shared memory segment for the partition * - Shared memory segment created contains data in comma-separated format. * **/ void DataLoader::AppendDataBytes(const char* shm_file, uint64_t data_size, uint32_t nrows) { unique_lock<recursive_mutex> lock(metadata_mutex_); char* p = (char*)shm_file; total_data_size += data_size; total_nrows += nrows; if(total_data_size > buffer.buffersize) { buffer.buf = (char*) realloc ((void*)buffer.buf, total_data_size); if(buffer.buf == NULL) throw PrestoWarningException("Error in reserving memory for loading data files from Vertica"); buffer.buffersize = total_data_size; } while((p!=NULL) && (*p!=0x0)) { size_t len = strlen(p); memcpy(buffer.buf + buffer.size, p, len); buffer.size += len; p += len; } if(total_nrows >= DR_partition_size) { LOG_DEBUG("Creating a shared memory segment of %d rows", total_nrows); std::string shm_name = getNextShmName(); CreateCsvShm(shm_name, total_data_size); total_data_size = 0; total_nrows = 0; buffer.size = 0; } lock.unlock(); boost::this_thread::interruption_point(); }
/* a function that creates a Berkeley socket and bind an open port in the configured range * @param start_port the start port range * @param end_port the end port range * @param port_in_use an out parameter that returns the port address that is bind to a socket * @ret a socket that is created from the function * @throws PrestoWarningException when there is no available open port in the configured range */ int32_t CreateBindedSocket(int start_port, int end_port, int* port_in_use) { int32_t serverfd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); if (serverfd < 0) { LOG_ERROR("CreateBindedSocket - fail to create a socket"); throw PrestoWarningException("CreateBindedSocket - fail to open a socket"); } int optval = 1; // set SO_REUSEADDR to allow reusing a port number that is in close_wait state setsockopt(serverfd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)); int num_port_cand = end_port - start_port + 1; bool bind_succeed = false; port_bind_mutex.lock(); for (int i = 0; i < num_port_cand; ++i){ struct sockaddr_in addr; addr.sin_family = AF_INET; addr.sin_addr.s_addr = htonl(INADDR_ANY); cur_port_assigned = (cur_port_assigned >= start_port && cur_port_assigned <= end_port) ? cur_port_assigned : start_port; // to make the value circulate after hitting the end_port addr.sin_port = htons(cur_port_assigned); int ret = ::bind(serverfd, (struct sockaddr*) &addr, sizeof(addr)); if (ret < 0) { ++cur_port_assigned; // try the next large value continue; } else { *port_in_use = (cur_port_assigned++); bind_succeed = true; break; } } port_bind_mutex.unlock(); if (bind_succeed == false) { LOG_ERROR("CreateBindedSocket - fail to bind a port (no available port)"); throw PrestoWarningException("There is no open port in the configured range"); } return serverfd; }