void _reap_hportal(host_portal_t *hp, int quick)
{
host_connection_t *hc;
apr_status_t value;
int count;
tbx_stack_move_to_top(hp->closed_que);
while ((hc = (host_connection_t *)tbx_stack_get_current_data(hp->closed_que)) != NULL) {
apr_thread_join(&value, hc->recv_thread);
log_printf(5, "hp=%s ns=%d\n", hp->skey, tbx_ns_getid(hc->ns));
for (count=0; ((quick == 0) || (count < 2)); count++) {
lock_hc(hc); //** Make sure that no one is running close_hc() while we're trying to close it
if (hc->closing != 1) { //** Ok to to remove it
unlock_hc(hc);
tbx_stack_delete_current(hp->closed_que, 0, 0);
destroy_host_connection(hc);
break;
} else { //** Got somone trying ot close it so wait a little bit
unlock_hc(hc);
apr_sleep(apr_time_from_msec(10));
}
}
tbx_stack_move_down(hp->closed_que);
if (tbx_stack_get_current_data(hp->closed_que) == NULL) tbx_stack_move_to_top(hp->closed_que); //** Restart it needed
}
}
lio_data_block_attr_t *db_find_key(tbx_stack_t *stack, char *key)
{
lio_data_block_attr_t *attr;
if (stack == NULL) return(NULL);
tbx_stack_move_to_top(stack);
while ((attr = (lio_data_block_attr_t *)tbx_stack_get_current_data(stack)) != NULL) {
if (strcmp(attr->key, key) == 0) {
return(attr);
}
tbx_stack_move_down(stack);
}
return(NULL);
}
void rsrs_client_notify(resource_service_fn_t *rs, int everyone)
{
rs_remote_server_priv_t *rsrs = (rs_remote_server_priv_t *)rs->priv;
rsrs_update_handle_t *h;
apr_time_t now, new_wakeup_time;
char *config;
char version[128];
int clen, vlen;
now = apr_time_now();
new_wakeup_time = 0;
//** Get the config
config = rs_get_rid_config(rsrs->rs_child);
clen = strlen(config);
apr_thread_mutex_lock(rsrs->lock);
//** and the version. Note the "\n" for the version. This preserves space for a NULL term on the receiver
snprintf(version, sizeof(version), "%d %d\n", rsrs->my_map_version.map_version, rsrs->my_map_version.status_version);
vlen = strlen(version);
//** Cycle through looking for commands about to expire and sending a response
tbx_stack_move_to_top(rsrs->pending);
while ((h = tbx_stack_get_current_data(rsrs->pending)) != NULL) {
if ((h->reply_time < now) || (everyone == 1)) {
log_printf(5, "sending update to a client everyone=%d\n", everyone);
mq_frame_set(h->version_frame, strdup(version), vlen, MQF_MSG_AUTO_FREE);
mq_frame_set(h->config_frame, strdup(config), clen, MQF_MSG_AUTO_FREE);
mq_submit(rsrs->server_portal, mq_task_new(rsrs->mqc, h->msg, NULL, NULL, 30));
tbx_stack_delete_current(rsrs->pending, 0, 0);
free(h); //** The msg is auto destroyed after being sent
} else if ((new_wakeup_time > h->reply_time) || (new_wakeup_time == 0)) {
new_wakeup_time = h->reply_time;
}
tbx_stack_move_down(rsrs->pending);
}
//** Change our wakeup time
rsrs->wakeup_time = (new_wakeup_time > 0) ? new_wakeup_time : apr_time_now() + apr_time_from_sec(3600);
apr_thread_mutex_unlock(rsrs->lock);
free(config);
}
void compact_hportal_direct(host_portal_t *hp)
{
host_portal_t *shp;
if (tbx_stack_count(hp->direct_list) == 0) return;
tbx_stack_move_to_top(hp->direct_list);
while ((shp = (host_portal_t *)tbx_stack_get_current_data(hp->direct_list)) != NULL) {
hportal_lock(shp);
_reap_hportal(shp, 1); //** Clean up any closed connections
if ((shp->n_conn == 0) && (shp->closing_conn == 0) && (tbx_stack_count(shp->que) == 0) && (tbx_stack_count(shp->closed_que) == 0)) { //** if not used so remove it
tbx_stack_delete_current(hp->direct_list, 0, 0);
hportal_unlock(shp);
destroy_hportal(shp);
} else {
hportal_unlock(shp);
tbx_stack_move_down(hp->direct_list);
}
}
}
void rsrs_abort_cb(void *arg, mq_task_t *task)
{
resource_service_fn_t *rs = (resource_service_fn_t *)arg;
rs_remote_server_priv_t *rsrs = (rs_remote_server_priv_t *)rs->priv;
mq_frame_t *f, *fid;
rsrs_update_handle_t *h;
mq_msg_t *msg;
char *data;
int bufsize = 1024;
char buffer[bufsize];
int n;
log_printf(5, "Processing incoming request\n");
//** Parse the command
msg = task->msg; //** Don't have to worry about msg cleanup. It's handled at a higher level
mq_msg_first(msg);
f = mq_msg_pop(msg);
mq_get_frame(f, (void **)&data, &n);
if (n != 0) { //** SHould be an empty frame
log_printf(0, " ERROR: Missing initial empty frame!\n");
goto fail;
}
mq_frame_destroy(f);
f = mq_msg_pop(msg);
mq_get_frame(f, (void **)&data, &n);
if (mq_data_compare(data, n, MQF_VERSION_KEY, MQF_VERSION_SIZE) != 0) {
log_printf(0, "ERROR: Missing version frame!\n");
goto fail;
}
mq_frame_destroy(f);
//** This is the low level command
f = mq_msg_pop(msg);
mq_get_frame(f, (void **)&data, &n);
if (mq_data_compare(data, n, MQF_EXEC_KEY, MQF_TRACKEXEC_SIZE) != 0) {
log_printf(0, "ERROR: Invalid command type!\n");
goto fail;
}
mq_frame_destroy(f);
//** Get the ID frame
fid = mq_msg_pop(msg);
mq_get_frame(fid, (void **)&data, &n);
if (n == 0) {
log_printf(0, " ERROR: Bad ID size! Got %d should be greater than 0\n", n);
goto fail;
}
log_printf(5, "Looking for mqid=%s\n", mq_id2str(data, n, buffer, bufsize));
//** Scan through the list looking for the id
apr_thread_mutex_lock(rsrs->lock);
tbx_stack_move_to_top(rsrs->pending);
while ((h = tbx_stack_get_current_data(rsrs->pending)) != NULL) {
if (mq_data_compare(data, n, h->id, h->id_size) == 0) { //** Found a match
log_printf(5, "Aborting task\n");
tbx_stack_delete_current(rsrs->pending, 0, 0);
mq_submit(rsrs->server_portal, mq_task_new(rsrs->mqc, h->msg, NULL, NULL, 30));
free(h); //** The msg is deleted after sending
break;
}
tbx_stack_move_down(rsrs->pending);
}
apr_thread_mutex_unlock(rsrs->lock);
mq_frame_destroy(fid); //** Destroy the id frame
fail:
log_printf(5, "END incoming request\n");
return;
}
int submit_hp_direct_op(portal_context_t *hpc, op_generic_t *op)
{
int status;
host_portal_t *hp, *shp;
host_connection_t *hc;
command_op_t *hop = &(op->op->cmd);
apr_thread_mutex_lock(hpc->lock);
//** Check if we should do a garbage run **
if (hpc->next_check < time(NULL)) {
hpc->next_check = time(NULL) + hpc->compact_interval;
apr_thread_mutex_unlock(hpc->lock);
compact_hportals(hpc);
apr_thread_mutex_lock(hpc->lock);
}
//** Find it in the list or make a new one
hp = _lookup_hportal(hpc, hop->hostport);
if (hp == NULL) {
log_printf(15, "submit_hp_direct_op: New host: %s\n", hop->hostport);
hp = create_hportal(hpc, hop->connect_context, hop->hostport, 1, 1, apr_time_from_sec(1));
if (hp == NULL) {
log_printf(15, "submit_hp_direct_op: create_hportal failed!\n");
apr_thread_mutex_unlock(hpc->lock);
return(-1);
}
apr_hash_set(hpc->table, hp->skey, APR_HASH_KEY_STRING, (const void *)hp);
}
apr_thread_mutex_unlock(hpc->lock);
log_printf(15, "submit_hp_direct_op: start opid=%d\n", op->base.id);
//** Scan the direct list for a free connection
hportal_lock(hp);
tbx_stack_move_to_top(hp->direct_list);
while ((shp = (host_portal_t *)tbx_stack_get_current_data(hp->direct_list)) != NULL) {
if (hportal_trylock(shp) == 0) {
log_printf(15, "submit_hp_direct_op: opid=%d shp->wl=" I64T " stack_size=%d\n", op->base.id, shp->workload, tbx_stack_count(shp->que));
if (tbx_stack_count(shp->que) == 0) {
if (tbx_stack_count(shp->conn_list) > 0) {
tbx_stack_move_to_top(shp->conn_list);
hc = (host_connection_t *)tbx_stack_get_current_data(shp->conn_list);
if (trylock_hc(hc) == 0) {
if ((tbx_stack_count(hc->pending_stack) == 0) && (hc->curr_workload == 0)) {
log_printf(15, "submit_hp_direct_op(A): before submit ns=%d opid=%d wl=%d\n",tbx_ns_getid(hc->ns), op->base.id, hc->curr_workload);
unlock_hc(hc);
hportal_unlock(shp);
status = submit_hportal(shp, op, 1, 0);
log_printf(15, "submit_hp_direct_op(A): after submit ns=%d opid=%d\n",tbx_ns_getid(hc->ns), op->base.id);
hportal_unlock(hp);
return(status);
}
unlock_hc(hc);
}
} else {
hportal_unlock(shp);
log_printf(15, "submit_hp_direct_op(B): opid=%d\n", op->base.id);
status = submit_hportal(shp, op, 1, 0);
hportal_unlock(hp);
return(status);
}
}
hportal_unlock(shp);
}
tbx_stack_move_down(hp->direct_list); //** Move to the next hp in the list
}
//** If I made it here I have to add a new hportal
shp = create_hportal(hpc, hop->connect_context, hop->hostport, 1, 1, apr_time_from_sec(1));
if (shp == NULL) {
log_printf(15, "submit_hp_direct_op: create_hportal failed!\n");
hportal_unlock(hp);
return(-1);
}
tbx_stack_push(hp->direct_list, (void *)shp);
status = submit_hportal(shp, op, 1, 0);
hportal_unlock(hp);
return(status);
}
host_connection_t *find_hc_to_close(portal_context_t *hpc)
{
apr_hash_index_t *hi;
host_portal_t *hp, *shp;
host_connection_t *hc, *best_hc;
void *val;
int best_workload, hold_lock;
hc = NULL;
best_hc = NULL;
best_workload = -1;
apr_thread_mutex_lock(hpc->lock);
for (hi=apr_hash_first(hpc->pool, hpc->table); hi != NULL; hi = apr_hash_next(hi)) {
apr_hash_this(hi, NULL, NULL, &val);
hp = (host_portal_t *)val;
hportal_lock(hp);
//** Scan the async connections
tbx_stack_move_to_top(hp->conn_list);
while ((hc = (host_connection_t *)tbx_stack_get_current_data(hp->conn_list)) != NULL) {
hold_lock = 0;
lock_hc(hc);
if (hc->closing == 0) {
if (best_workload<0) best_workload = hc->curr_workload+1;
if (hc->curr_workload < best_workload) {
if (best_hc != NULL) unlock_hc(best_hc);
hold_lock = 1;
best_workload = hc->curr_workload;
best_hc = hc;
}
}
tbx_stack_move_down(hp->conn_list);
if (hold_lock == 0) unlock_hc(hc);
}
//** Scan the direct connections
tbx_stack_move_to_top(hp->direct_list);
while ((shp = (host_portal_t *)tbx_stack_get_current_data(hp->direct_list)) != NULL) {
hportal_lock(shp);
if (tbx_stack_count(shp->conn_list) > 0) {
tbx_stack_move_to_top(shp->conn_list);
hc = (host_connection_t *)tbx_stack_get_current_data(shp->conn_list);
hold_lock = 0;
lock_hc(hc);
if (hc->closing == 0) {
if (best_workload<0) best_workload = hc->curr_workload+1;
if (hc->curr_workload < best_workload) {
if (best_hc != NULL) unlock_hc(best_hc);
hold_lock = 1;
best_workload = hc->curr_workload;
best_hc = hc;
}
}
if (hold_lock == 0) unlock_hc(hc);
}
hportal_unlock(shp);
tbx_stack_move_down(hp->direct_list);
}
hportal_unlock(hp);
}
if (best_hc != NULL) { //** Flag it as being closed so we ignore it next round and release the lock
best_hc->closing = 1;
unlock_hc(best_hc);
}
apr_thread_mutex_unlock(hpc->lock);
return(best_hc);
}
void shutdown_hportal(portal_context_t *hpc)
{
host_portal_t *hp;
host_connection_t *hc;
apr_hash_index_t *hi;
void *val;
log_printf(15, "shutdown_hportal: Shutting down the whole system\n");
//IFFY apr_thread_mutex_lock(hpc->lock);
//** First tell everyone to shutdown
for (hi=apr_hash_first(hpc->pool, hpc->table); hi != NULL; hi = apr_hash_next(hi)) {
apr_hash_this(hi, NULL, NULL, &val);
hp = (host_portal_t *)val;
hportal_lock(hp);
log_printf(5, "before wait n_conn=%d tbx_stack_count(conn_list)=%d host=%s\n", hp->n_conn, tbx_stack_count(hp->conn_list), hp->skey);
while (tbx_stack_count(hp->conn_list) != hp->n_conn) {
hportal_unlock(hp);
log_printf(5, "waiting for connections to finish starting. host=%s closing_conn=%d n_conn=%d tbx_stack_count(conn_list)=%d\n", hp->skey, hp->closing_conn, hp->n_conn, tbx_stack_count(hp->conn_list));
usleep(10000);
hportal_lock(hp);
}
log_printf(5, "after wait n_conn=%d tbx_stack_count(conn_list)=%d\n", hp->n_conn, tbx_stack_count(hp->conn_list));
tbx_stack_move_to_top(hp->conn_list);
while ((hc = (host_connection_t *)tbx_stack_get_current_data(hp->conn_list)) != NULL) {
tbx_stack_free(hp->que, 1); //** Empty the que so we don't respawn connections
hp->que = tbx_stack_new();
// hportal_unlock(hp);
lock_hc(hc);
hc->shutdown_request = 1;
apr_thread_cond_signal(hc->recv_cond);
unlock_hc(hc);
// hportal_lock(hp);
tbx_stack_move_down(hp->conn_list);
}
hportal_unlock(hp);
}
//** Now go and clean up
for (hi=apr_hash_first(hpc->pool, hpc->table); hi != NULL; hi = apr_hash_next(hi)) {
apr_hash_this(hi, NULL, NULL, &val);
hp = (host_portal_t *)val;
apr_hash_set(hpc->table, hp->skey, APR_HASH_KEY_STRING, NULL); //** This removes the key
log_printf(15, "shutdown_hportal: Shutting down host=%s\n", hp->skey);
hportal_lock(hp);
log_printf(5, "closing_conn=%d n_conn=%d host=%s\n", hp->closing_conn, hp->n_conn, hp->host);
_reap_hportal(hp, 0); //** clean up any closed connections
log_printf(5, "closing_conn=%d n_conn=%d\n", hp->closing_conn, hp->n_conn);
while ((hp->closing_conn > 0) || (hp->n_conn > 0)) {
log_printf(5, "waiting for connections to close. host=%s closing_conn=%d n_conn=%d tbx_stack_count(conn_list)=%d\n", hp->skey, hp->closing_conn, hp->n_conn, tbx_stack_count(hp->conn_list));
hportal_unlock(hp);
usleep(10000);
hportal_lock(hp);
}
shutdown_direct(hp); //** Shutdown any direct connections
tbx_stack_move_to_top(hp->conn_list);
while ((hc = (host_connection_t *)tbx_stack_get_current_data(hp->conn_list)) != NULL) {
tbx_stack_free(hp->que, 1); //** Empty the que so we don't respawn connections
hp->que = tbx_stack_new();
hportal_unlock(hp);
apr_thread_mutex_unlock(hpc->lock);
close_hc(hc, 0);
apr_thread_mutex_lock(hpc->lock);
hportal_lock(hp);
tbx_stack_move_to_top(hp->conn_list);
}
hportal_unlock(hp);
destroy_hportal(hp);
}
//IFFY apr_thread_mutex_unlock(hpc->lock);
return;
}
