static void
stream_close(struct vconn *vconn)
{
struct stream_vconn *s = stream_vconn_cast(vconn);
poll_cancel(s->tx_waiter);
stream_clear_txbuf(s);
ofpbuf_delete(s->rxbuf);
close(s->fd);
free(s);
}
static void
ssl_close(struct vconn *vconn)
{
struct ssl_vconn *sslv = ssl_vconn_cast(vconn);
poll_cancel(sslv->tx_waiter);
ssl_clear_txbuf(sslv);
ofpbuf_delete(sslv->rxbuf);
SSL_free(sslv->ssl);
close(sslv->fd);
free(sslv);
}
/* Destroys 'server' and stops listening for connections. */
void
vlog_server_close(struct vlog_server *server)
{
if (server) {
poll_cancel(server->waiter);
close(server->fd);
unlink(server->path);
fatal_signal_remove_file_to_unlink(server->path);
free(server->path);
free(server);
}
}
void Defrag::poll()
{
dprintf(D_FULLDEBUG,"Evaluating defragmentation policy.\n");
// If we crash during this polling cycle, we will have saved
// the time of the last poll, so the next cycle will be
// scheduled on the false assumption that a cycle ran now. In
// this way, we error on the side of draining too little
// rather than too much.
time_t now = time(NULL);
time_t prev = m_last_poll;
m_last_poll = now;
saveState();
m_stats.Tick();
int num_to_drain = m_draining_per_poll;
time_t last_hour = (prev / 3600)*3600;
time_t current_hour = (now / 3600)*3600;
time_t last_day = (prev / (3600*24))*3600*24;
time_t current_day = (now / (3600*24))*3600*24;
if( current_hour != last_hour ) {
num_to_drain += prorate(m_draining_per_poll_hour,now-current_hour,3600,m_polling_interval);
}
if( current_day != last_day ) {
num_to_drain += prorate(m_draining_per_poll_day,now-current_day,3600*24,m_polling_interval);
}
int num_draining = countMachines(DRAINING_CONSTRAINT,"<InternalDrainingConstraint>");
m_stats.MachinesDraining = num_draining;
MachineSet whole_machines;
int num_whole_machines = countMachines(m_whole_machine_expr.c_str(),"DEFRAG_WHOLE_MACHINE_EXPR",&whole_machines);
m_stats.WholeMachines = num_whole_machines;
dprintf(D_ALWAYS,"There are currently %d draining and %d whole machines.\n",
num_draining,num_whole_machines);
queryDrainingCost();
// If possible, cancel some drains.
MachineSet cancelled_machines;
poll_cancel(cancelled_machines);
if( num_to_drain <= 0 ) {
dprintf(D_ALWAYS,"Doing nothing, because number to drain in next %ds is calculated to be 0.\n",
m_polling_interval);
return;
}
if( (int)ceil(m_draining_per_hour) <= 0 ) {
dprintf(D_ALWAYS,"Doing nothing, because DEFRAG_DRAINING_MACHINES_PER_HOUR=%f\n",
m_draining_per_hour);
return;
}
if( m_max_draining == 0 ) {
dprintf(D_ALWAYS,"Doing nothing, because DEFRAG_MAX_CONCURRENT_DRAINING=0\n");
return;
}
if( m_max_whole_machines == 0 ) {
dprintf(D_ALWAYS,"Doing nothing, because DEFRAG_MAX_WHOLE_MACHINES=0\n");
return;
}
if( m_max_draining >= 0 ) {
if( num_draining >= m_max_draining ) {
dprintf(D_ALWAYS,"Doing nothing, because DEFRAG_MAX_CONCURRENT_DRAINING=%d and there are %d draining machines.\n",
m_max_draining, num_draining);
return;
}
else if( num_draining < 0 ) {
dprintf(D_ALWAYS,"Doing nothing, because DEFRAG_MAX_CONCURRENT_DRAINING=%d and the query to count draining machines failed.\n",
m_max_draining);
return;
}
}
if( m_max_whole_machines >= 0 ) {
if( num_whole_machines >= m_max_whole_machines ) {
dprintf(D_ALWAYS,"Doing nothing, because DEFRAG_MAX_WHOLE_MACHINES=%d and there are %d whole machines.\n",
m_max_whole_machines, num_whole_machines);
return;
}
}
// Even if m_max_whole_machines is -1 (infinite), we still need
// the list of whole machines in order to filter them out in
// the draining selection algorithm, so abort now if the
// whole machine query failed.
if( num_whole_machines < 0 ) {
dprintf(D_ALWAYS,"Doing nothing, because the query to find whole machines failed.\n");
return;
}
dprintf(D_ALWAYS,"Looking for %d machines to drain.\n",num_to_drain);
ClassAdList startdAds;
std::string requirements;
sprintf(requirements,"(%s) && Draining =!= true",m_defrag_requirements.c_str());
if( !queryMachines(requirements.c_str(),"DEFRAG_REQUIREMENTS",startdAds) ) {
dprintf(D_ALWAYS,"Doing nothing, because the query to select machines matching DEFRAG_REQUIREMENTS failed.\n");
return;
}
startdAds.Shuffle();
startdAds.Sort(StartdSortFunc,&m_rank_ad);
startdAds.Open();
int num_drained = 0;
ClassAd *startd_ad_ptr;
MachineSet machines_done;
while( (startd_ad_ptr=startdAds.Next()) ) {
if (!startd_ad_ptr) continue;
ClassAd &startd_ad = *startd_ad_ptr;
std::string machine;
std::string name;
startd_ad.LookupString(ATTR_NAME,name);
slotNameToDaemonName(name,machine);
// If we have already cancelled draining on this machine, ignore it for this cycle.
if( cancelled_machines.count(machine) ) {
dprintf(D_FULLDEBUG,
"Skipping %s: already cancelled draining of %s in this cycle.\n",
name.c_str(),machine.c_str());
continue;
}
if( machines_done.count(machine) ) {
dprintf(D_FULLDEBUG,
"Skipping %s: already attempted to drain %s in this cycle.\n",
name.c_str(),machine.c_str());
continue;
}
if( whole_machines.count(machine) ) {
dprintf(D_FULLDEBUG,
"Skipping %s: because it is already running as a whole machine.\n",
name.c_str());
continue;
}
if( drain(startd_ad) ) {
machines_done.insert(machine);
if( ++num_drained >= num_to_drain ) {
dprintf(D_ALWAYS,
"Drained maximum number of machines allowed in this cycle (%d).\n",
num_to_drain);
break;
}
}
}
startdAds.Close();
dprintf(D_ALWAYS,"Drained %d machines (wanted to drain %d machines).\n",
num_drained,num_drained);
dprintf(D_FULLDEBUG,"Done evaluating defragmentation policy.\n");
}
/* Blocks until one or more of the events registered with poll_fd_wait()
* occurs, or until the minimum duration registered with poll_timer_wait()
* elapses, or not at all if poll_immediate_wake() has been called.
*
* Also executes any autonomous subroutines registered with poll_fd_callback(),
* if their file descriptors have become ready. */
void
poll_block(void)
{
static struct pollfd *pollfds;
static size_t max_pollfds;
struct poll_waiter *pw;
struct list *node;
int n_pollfds;
int retval;
assert(!running_cb);
if (max_pollfds < n_waiters) {
max_pollfds = n_waiters;
pollfds = xrealloc(pollfds, max_pollfds * sizeof *pollfds);
}
n_pollfds = 0;
LIST_FOR_EACH (pw, struct poll_waiter, node, &waiters) {
pw->pollfd = &pollfds[n_pollfds];
pollfds[n_pollfds].fd = pw->fd;
pollfds[n_pollfds].events = pw->events;
pollfds[n_pollfds].revents = 0;
n_pollfds++;
}
retval = time_poll(pollfds, n_pollfds, timeout);
if (retval < 0) {
static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
VLOG_ERR_RL(&rl, "poll: %s", strerror(-retval));
} else if (!retval && VLOG_IS_DBG_ENABLED()) {
log_wakeup(&timeout_backtrace, "%d-ms timeout", timeout);
}
for (node = waiters.next; node != &waiters; ) {
pw = CONTAINER_OF(node, struct poll_waiter, node);
if (!pw->pollfd || !pw->pollfd->revents) {
if (pw->function) {
node = node->next;
continue;
}
} else {
if (VLOG_IS_DBG_ENABLED()) {
log_wakeup(pw->backtrace, "%s%s%s%s%s on fd %d",
pw->pollfd->revents & POLLIN ? "[POLLIN]" : "",
pw->pollfd->revents & POLLOUT ? "[POLLOUT]" : "",
pw->pollfd->revents & POLLERR ? "[POLLERR]" : "",
pw->pollfd->revents & POLLHUP ? "[POLLHUP]" : "",
pw->pollfd->revents & POLLNVAL ? "[POLLNVAL]" : "",
pw->fd);
}
if (pw->function) {
#ifndef NDEBUG
running_cb = pw;
#endif
pw->function(pw->fd, pw->pollfd->revents, pw->aux);
#ifndef NDEBUG
running_cb = NULL;
#endif
}
}
node = node->next;
poll_cancel(pw);
}
timeout = -1;
timeout_backtrace.n_frames = 0;
}
static int
ssl_recv(struct vconn *vconn, struct ofpbuf **bufferp)
{
struct ssl_vconn *sslv = ssl_vconn_cast(vconn);
struct ofpbuf *rx;
size_t want_bytes;
int old_state;
ssize_t ret;
if (sslv->rxbuf == NULL) {
sslv->rxbuf = ofpbuf_new(1564);
}
rx = sslv->rxbuf;
again:
if (sizeof(struct ofp_header) > rx->size) {
want_bytes = sizeof(struct ofp_header) - rx->size;
} else {
struct ofp_header *oh = rx->data;
size_t length = ntohs(oh->length);
if (length < sizeof(struct ofp_header)) {
VLOG_ERR_RL(&rl, "received too-short ofp_header (%zu bytes)",
length);
return EPROTO;
}
want_bytes = length - rx->size;
if (!want_bytes) {
*bufferp = rx;
sslv->rxbuf = NULL;
return 0;
}
}
ofpbuf_prealloc_tailroom(rx, want_bytes);
/* Behavior of zero-byte SSL_read is poorly defined. */
assert(want_bytes > 0);
old_state = SSL_get_state(sslv->ssl);
ret = SSL_read(sslv->ssl, ofpbuf_tail(rx), want_bytes);
if (old_state != SSL_get_state(sslv->ssl)) {
sslv->tx_want = SSL_NOTHING;
if (sslv->tx_waiter) {
poll_cancel(sslv->tx_waiter);
ssl_tx_poll_callback(sslv->fd, POLLIN, vconn);
}
}
sslv->rx_want = SSL_NOTHING;
if (ret > 0) {
rx->size += ret;
if (ret == want_bytes) {
if (rx->size > sizeof(struct ofp_header)) {
*bufferp = rx;
sslv->rxbuf = NULL;
return 0;
} else {
goto again;
}
}
return EAGAIN;
} else {
int error = SSL_get_error(sslv->ssl, ret);
if (error == SSL_ERROR_ZERO_RETURN) {
/* Connection closed (EOF). */
if (rx->size) {
VLOG_WARN_RL(&rl, "SSL_read: unexpected connection close");
return EPROTO;
} else {
return EOF;
}
} else {
return interpret_ssl_error("SSL_read", ret, error, &sslv->rx_want);
}
}
}
void Defrag::poll()
{
dprintf(D_FULLDEBUG,"Evaluating defragmentation policy.\n");
// If we crash during this polling cycle, we will have saved
// the time of the last poll, so the next cycle will be
// scheduled on the false assumption that a cycle ran now. In
// this way, we error on the side of draining too little
// rather than too much.
time_t now = time(NULL);
time_t prev = m_last_poll;
m_last_poll = now;
saveState();
m_stats.Tick();
int num_to_drain = m_draining_per_poll;
time_t last_hour = (prev / 3600)*3600;
time_t current_hour = (now / 3600)*3600;
time_t last_day = (prev / (3600*24))*3600*24;
time_t current_day = (now / (3600*24))*3600*24;
if( current_hour != last_hour ) {
num_to_drain += prorate(m_draining_per_poll_hour,now-current_hour,3600,m_polling_interval);
}
if( current_day != last_day ) {
num_to_drain += prorate(m_draining_per_poll_day,now-current_day,3600*24,m_polling_interval);
}
MachineSet draining_machines;
int num_draining = countMachines(DRAINING_CONSTRAINT,"<InternalDrainingConstraint>", &draining_machines);
m_stats.MachinesDraining = num_draining;
MachineSet whole_machines;
int num_whole_machines = countMachines(m_whole_machine_expr.c_str(),"DEFRAG_WHOLE_MACHINE_EXPR",&whole_machines);
m_stats.WholeMachines = num_whole_machines;
dprintf(D_ALWAYS,"There are currently %d draining and %d whole machines.\n",
num_draining,num_whole_machines);
// Calculate arrival rate of fully drained machines. This is a bit tricky because we poll.
// We count by finding the newly-arrived
// fully drained machines, and add to that count machines which are no-longer draining.
// This allows us to find machines that have fully drained, but were then claimed between
// polling cycles.
MachineSet new_machines;
MachineSet no_longer_whole_machines;
// Find newly-arrived machines
std::set_difference(whole_machines.begin(), whole_machines.end(),
m_prev_whole_machines.begin(), m_prev_whole_machines.end(),
std::inserter(new_machines, new_machines.begin()));
// Now, newly-departed machines
std::set_difference(m_prev_draining_machines.begin(), m_prev_draining_machines.end(),
draining_machines.begin(), draining_machines.end(),
std::inserter(no_longer_whole_machines, no_longer_whole_machines.begin()));
dprintf_set("Set of current whole machines is ", &whole_machines);
dprintf_set("Set of current draining machine is ", &draining_machines);
dprintf_set("Newly Arrived whole machines is ", &new_machines);
dprintf_set("Newly departed draining machines is ", &no_longer_whole_machines);
m_prev_draining_machines = draining_machines;
m_prev_whole_machines = whole_machines;
int newly_drained = new_machines.size() + no_longer_whole_machines.size();
double arrival_rate = 0.0;
// If there is an arrival...
if (newly_drained > 0) {
time_t current = time(0);
// And it isn't the first one since defrag boot...
if (m_last_whole_machine_arrival > 0) {
m_whole_machines_arrived += newly_drained;
time_t arrival_time = current - m_last_whole_machine_arrival;
if (arrival_time < 1) arrival_time = 1; // very unlikely, but just in case
m_whole_machine_arrival_sum += newly_drained * arrival_time;
arrival_rate = newly_drained / ((double)arrival_time);
dprintf(D_ALWAYS, "Arrival rate is %g machines/hour\n", arrival_rate * 3600.0);
}
m_last_whole_machine_arrival = current;
}
dprintf(D_ALWAYS, "Lifetime whole machines arrived: %d\n", m_whole_machines_arrived);
if (m_whole_machine_arrival_sum > 0) {
double lifetime_mean = m_whole_machines_arrived / m_whole_machine_arrival_sum;
dprintf(D_ALWAYS, "Lifetime mean arrival rate: %g machines / hour\n", 3600.0 * lifetime_mean);
if (newly_drained > 0) {
double diff = arrival_rate - lifetime_mean;
m_whole_machine_arrival_mean_squared += diff * diff;
}
double sd = sqrt(m_whole_machine_arrival_mean_squared / m_whole_machines_arrived);
dprintf(D_ALWAYS, "Lifetime mean arrival rate sd: %g\n", sd * 3600);
m_stats.MeanDrainedArrival = lifetime_mean;
m_stats.MeanDrainedArrivalSD = sd;
m_stats.DrainedMachines = m_whole_machines_arrived;
}
queryDrainingCost();
// If possible, cancel some drains.
MachineSet cancelled_machines;
poll_cancel(cancelled_machines);
if( num_to_drain <= 0 ) {
dprintf(D_ALWAYS,"Doing nothing, because number to drain in next %ds is calculated to be 0.\n",
m_polling_interval);
return;
}
if( (int)ceil(m_draining_per_hour) <= 0 ) {
dprintf(D_ALWAYS,"Doing nothing, because DEFRAG_DRAINING_MACHINES_PER_HOUR=%f\n",
m_draining_per_hour);
return;
}
if( m_max_draining == 0 ) {
dprintf(D_ALWAYS,"Doing nothing, because DEFRAG_MAX_CONCURRENT_DRAINING=0\n");
return;
}
if( m_max_whole_machines == 0 ) {
dprintf(D_ALWAYS,"Doing nothing, because DEFRAG_MAX_WHOLE_MACHINES=0\n");
return;
}
if( m_max_draining >= 0 ) {
if( num_draining >= m_max_draining ) {
dprintf(D_ALWAYS,"Doing nothing, because DEFRAG_MAX_CONCURRENT_DRAINING=%d and there are %d draining machines.\n",
m_max_draining, num_draining);
return;
}
else if( num_draining < 0 ) {
dprintf(D_ALWAYS,"Doing nothing, because DEFRAG_MAX_CONCURRENT_DRAINING=%d and the query to count draining machines failed.\n",
m_max_draining);
return;
}
}
if( m_max_whole_machines >= 0 ) {
if( num_whole_machines >= m_max_whole_machines ) {
dprintf(D_ALWAYS,"Doing nothing, because DEFRAG_MAX_WHOLE_MACHINES=%d and there are %d whole machines.\n",
m_max_whole_machines, num_whole_machines);
return;
}
}
// Even if m_max_whole_machines is -1 (infinite), we still need
// the list of whole machines in order to filter them out in
// the draining selection algorithm, so abort now if the
// whole machine query failed.
if( num_whole_machines < 0 ) {
dprintf(D_ALWAYS,"Doing nothing, because the query to find whole machines failed.\n");
return;
}
dprintf(D_ALWAYS,"Looking for %d machines to drain.\n",num_to_drain);
ClassAdList startdAds;
std::string requirements;
formatstr(requirements,"(%s) && Draining =!= true",m_defrag_requirements.c_str());
if( !queryMachines(requirements.c_str(),"DEFRAG_REQUIREMENTS",startdAds) ) {
dprintf(D_ALWAYS,"Doing nothing, because the query to select machines matching DEFRAG_REQUIREMENTS failed.\n");
return;
}
startdAds.Shuffle();
startdAds.Sort(StartdSortFunc,&m_rank_ad);
startdAds.Open();
int num_drained = 0;
ClassAd *startd_ad_ptr;
MachineSet machines_done;
while( (startd_ad_ptr=startdAds.Next()) ) {
ClassAd &startd_ad = *startd_ad_ptr;
std::string machine;
std::string name;
startd_ad.LookupString(ATTR_NAME,name);
slotNameToDaemonName(name,machine);
// If we have already cancelled draining on this machine, ignore it for this cycle.
if( cancelled_machines.count(machine) ) {
dprintf(D_FULLDEBUG,
"Skipping %s: already cancelled draining of %s in this cycle.\n",
name.c_str(),machine.c_str());
continue;
}
if( machines_done.count(machine) ) {
dprintf(D_FULLDEBUG,
"Skipping %s: already attempted to drain %s in this cycle.\n",
name.c_str(),machine.c_str());
continue;
}
if( whole_machines.count(machine) ) {
dprintf(D_FULLDEBUG,
"Skipping %s: because it is already running as a whole machine.\n",
name.c_str());
continue;
}
if( drain(startd_ad) ) {
machines_done.insert(machine);
if( ++num_drained >= num_to_drain ) {
dprintf(D_ALWAYS,
"Drained maximum number of machines allowed in this cycle (%d).\n",
num_to_drain);
break;
}
}
}
startdAds.Close();
dprintf(D_ALWAYS,"Drained %d machines (wanted to drain %d machines).\n",
num_drained,num_to_drain);
dprintf(D_FULLDEBUG,"Done evaluating defragmentation policy.\n");
}
