int ibp_sync_command(ibp_op_t *iop)
{
op_generic_t *gop = ibp_get_gop(iop);
op_status_t status;
//** Submit if for execution
// gop->base.started_execution = 1; //** Mark it as being submitted
// submit_hportal_sync(gop->op->pc, gop);
gop_set_exec_mode(gop, OP_EXEC_DIRECT);
gop_start_execution(gop);
//** and Wait for completion
gop_waitany(gop);
status = gop_get_status(gop);
IBP_errno = status.error_code;
if (gop->op->cmd.hostport != NULL) {
free(gop->op->cmd.hostport);
gop->op->cmd.hostport = NULL;
}
log_printf(10, "ibp_sync_command: IBP_errno=%d\n", IBP_errno); //tbx_log_flush();
return(IBP_errno);
}
op_status_t gop_sync_exec_status(op_generic_t *gop)
{
int err;
op_status_t status;
if (gop->type == Q_TYPE_OPERATION) { //** Got an operation so see if we can directly exec it
if (gop->base.pc->fn->sync_exec != NULL) { //** Yup we can!
log_printf(15, "sync_exec -- waiting for gid=%d to complete\n", gop_id(gop));
gop->base.pc->fn->sync_exec(gop->base.pc, gop);
status = gop->base.status;
log_printf(15, "sync_exec -- gid=%d completed with err=%d\n", gop_id(gop), status.op_status);
gop_free(gop, OP_DESTROY);
return(status);
}
}
log_printf(15, "waiting for gid=%d to complete\n", gop_id(gop));
err = gop_waitall(gop);
status = gop_get_status(gop);
log_printf(15, "gid=%d completed with err=%d\n", gop_id(gop), err);
gop_free(gop, OP_DESTROY);
log_printf(15, "After gop destruction\n");
return(status);
}
int gop_wait(op_generic_t *gop)
{
op_status_t status;
//log_printf(15, "gop_wait: START gid=%d state=%d\n", gop_id(gop), gop->base.state);
lock_gop(gop);
//log_printf(15, "gop_wait: after lock gid=%d state=%d\n", gop_id(gop), gop->base.state);
while (gop->base.state == 0) {
log_printf(15, "gop_wait: WHILE gid=%d state=%d\n", gop_id(gop), gop->base.state);
apr_thread_cond_wait(gop->base.ctl->cond, gop->base.ctl->lock); //** Sleep until something completes
}
status = gop_get_status(gop);
// state = _gop_completed_successfully(gop);
log_printf(15, "gop_wait: FINISHED gid=%d status=%d err=%d\n", gop_id(gop), status.op_status, status.error_code);
unlock_gop(gop);
return(status.op_status);
}
int main(int argc, char **argv)
{
int i, start_option;
int mode;
char *clone_arg = NULL;
char *sfname = NULL;
char *cfname = NULL;
lio_exnode_t *ex, *cex;
lio_exnode_exchange_t *exp, *exp_out;
gop_op_generic_t *gop;
gop_op_status_t status;
FILE *fd;
if (argc < 3) {
printf("\n");
printf("ex_clone LIO_COMMON_OPTIONS [-structure|-data] [-a clone_attr] source_file.ex3 clone_file.ex3\n");
lio_print_options(stdout);
printf(" -structure - Clone the structure only [default mode]\n");
printf(" -data - Clone the structure and copy the data\n");
printf(" -a clone_attr - Segment specific attribute passed to the clone routine. Not used for all Segment types.\n");
printf(" source_file.ex3 - File to clone\n");
printf(" clone_file.ex3 - DEstination cloned file\n");
return(1);
}
lio_init(&argc, &argv);
mode = CLONE_STRUCTURE;
//*** Parse the args
i=1;
do {
start_option = i;
if (strcmp(argv[i], "-structure") == 0) { //** Clone the structure only
i++;
mode = CLONE_STRUCTURE;
} else if (strcmp(argv[i], "-data") == 0) { //** Clone the structure and the data
i++;
mode = CLONE_STRUCT_AND_DATA;
} else if (strcmp(argv[i], "-a") == 0) { //** Alternate query attribute
i++;
clone_arg = argv[i];
i++;
}
} while (start_option < i);
//** This is the source file
sfname = argv[i];
i++;
if (sfname == NULL) {
printf("Missing source file!\n");
return(2);
}
//** This is the cloned file
cfname = argv[i];
i++;
if (cfname == NULL) {
printf("Missing cloned file!\n");
return(2);
}
//** Load the source
exp = lio_exnode_exchange_load_file(sfname);
ex = lio_exnode_create();
lio_exnode_deserialize(ex, exp, lio_gc->ess);
//** Execute the clone operation
gop = lio_exnode_clone_gop(lio_gc->tpc_unlimited, ex, lio_gc->da, &cex, (void *)clone_arg, mode, lio_gc->timeout);
gop_waitany(gop);
status = gop_get_status(gop);
gop_free(gop, OP_DESTROY);
if (status.op_status != OP_STATE_SUCCESS) {
printf("Error with clone! source=%s mode=%d\n", sfname, mode);
abort();
}
//** Store the updated exnode back to disk
exp_out = lio_exnode_exchange_create(EX_TEXT);
lio_exnode_serialize(cex, exp_out);
fd = fopen(cfname, "w");
fprintf(fd, "%s", exp_out->text.text);
fclose(fd);
lio_exnode_exchange_destroy(exp_out);
//** Clean up
lio_exnode_exchange_destroy(exp);
lio_exnode_destroy(ex);
lio_exnode_destroy(cex);
lio_shutdown();
return(0);
}
void *ongoing_heartbeat_thread(apr_thread_t *th, void *data)
{
mq_ongoing_t *on = (mq_ongoing_t *)data;
apr_time_t timeout = apr_time_make(on->check_interval, 0);
op_generic_t *gop;
mq_msg_t *msg;
ongoing_hb_t *oh;
ongoing_table_t *table;
apr_hash_index_t *hi, *hit;
opque_t *q;
char *id;
mq_msg_hash_t *remote_hash;
apr_time_t now;
apr_ssize_t id_len;
int n, k;
char *remote_host_string;
apr_thread_mutex_lock(on->lock);
n = 0;
do {
now = apr_time_now() - apr_time_from_sec(5); //** Give our selves a little buffer
log_printf(5, "Loop Start now=" TT "\n", apr_time_now());
q = new_opque();
// opque_start_execution(q);
for (hit = apr_hash_first(NULL, on->table); hit != NULL; hit = apr_hash_next(hit)) {
apr_hash_this(hit, (const void **)&remote_hash, &id_len, (void **)&table);
k = apr_hash_count(table->table);
if (log_level() > 1) {
remote_host_string = mq_address_to_string(table->remote_host);
log_printf(1, "host=%s count=%d\n", remote_host_string, k);
free(remote_host_string);
}
for (hi = apr_hash_first(NULL, table->table); hi != NULL; hi = apr_hash_next(hi)) {
apr_hash_this(hi, (const void **)&id, &id_len, (void **)&oh);
log_printf(1, "id=%s now=" TT " next_check=" TT "\n", oh->id, apr_time_sec(apr_time_now()), apr_time_sec(oh->next_check));
if (now > oh->next_check) {
log_printf(1, "id=%s sending HEARTBEAT EXEC SUBMIT nows=" TT " hb=%d\n", oh->id, apr_time_sec(apr_time_now()), oh->heartbeat);
flush_log();
//** Form the message
msg = mq_make_exec_core_msg(table->remote_host, 1);
mq_msg_append_mem(msg, ONGOING_KEY, ONGOING_SIZE, MQF_MSG_KEEP_DATA);
mq_msg_append_mem(msg, oh->id, oh->id_len, MQF_MSG_KEEP_DATA);
mq_msg_append_mem(msg, NULL, 0, MQF_MSG_KEEP_DATA);
//** Make the gop
gop = new_mq_op(on->mqc, msg, ongoing_response_status, NULL, NULL, oh->heartbeat);
gop_set_private(gop, table);
opque_add(q, gop);
oh->in_progress = 1; //** Flag it as in progress so it doesn't get deleted behind the scenes
}
}
}
log_printf(5, "Loop end now=" TT "\n", apr_time_now());
//** Wait for it to complete
apr_thread_mutex_unlock(on->lock);
opque_waitall(q);
apr_thread_mutex_lock(on->lock);
//** Dec the counters
while ((gop = opque_waitany(q)) != NULL) {
log_printf(0, "gid=%d gotone status=%d now=" TT "\n", gop_id(gop), (gop_get_status(gop)).op_status, apr_time_sec(apr_time_now()));
table = gop_get_private(gop);
table->count--;
//** Update the next check
for (hi = apr_hash_first(NULL, table->table); hi != NULL; hi = apr_hash_next(hi)) {
apr_hash_this(hi, (const void **)&id, &id_len, (void **)&oh);
oh->next_check = apr_time_now() + apr_time_from_sec(oh->heartbeat);
//** Check if we get rid of it
oh->in_progress = 0;
if (oh->count <= 0) { //** Need to delete it
apr_hash_set(table->table, id, id_len, NULL);
free(oh->id);
free(oh);
}
}
gop_free(gop, OP_DESTROY);
}
opque_free(q, OP_DESTROY);
now = apr_time_now();
log_printf(2, "sleeping %d now=" TT "\n", on->check_interval, now);
//** Sleep until time for the next heartbeat or time to exit
if (on->shutdown == 0) apr_thread_cond_timedwait(on->cond, on->lock, timeout);
n = on->shutdown;
now = apr_time_now() - now;
log_printf(2, "main loop bottom n=%d dt=" TT " sec=" TT "\n", n, now, apr_time_sec(now));
} while (n == 0);
log_printf(2, "CLEANUP\n");
for (hit = apr_hash_first(NULL, on->table); hit != NULL; hit = apr_hash_next(hit)) {
apr_hash_this(hit, (const void **)&remote_hash, &id_len, (void **)&table);
for (hi = apr_hash_first(NULL, table->table); hi != NULL; hi = apr_hash_next(hi)) {
apr_hash_this(hi, (const void **)&id, &id_len, (void **)&oh);
apr_hash_set(table->table, id, id_len, NULL);
free(oh->id);
free(oh);
}
apr_hash_set(on->table, &(table->remote_host_hash), sizeof(mq_msg_hash_t), NULL);
mq_msg_destroy(table->remote_host);
free(table);
}
log_printf(2, "EXITING\n");
apr_thread_mutex_unlock(on->lock);
return(NULL);
}
int mq_stream_read_wait(mq_stream_t *mqs)
{
int err = 0;
apr_interval_time_t dt;
op_status_t status;
//** If 1st time make all the variables
if (mqs->mpool == NULL) {
apr_pool_create(&mqs->mpool, NULL);
apr_thread_mutex_create(&(mqs->lock), APR_THREAD_MUTEX_DEFAULT, mqs->mpool);
apr_thread_cond_create(&(mqs->cond), mqs->mpool);
}
dt = apr_time_from_sec(1);
//** Flag the just processed gop to clean up
apr_thread_mutex_lock(mqs->lock);
log_printf(5, "START msid=%d waiting=%d processed=%d gop_processed=%p\n", mqs->msid, mqs->waiting, mqs->processed, mqs->gop_processed);
if (mqs->gop_processed != NULL) mqs->processed = 1;
apr_thread_cond_broadcast(mqs->cond);
if (mqs->data) {
if (mqs->data[MQS_STATE_INDEX] != MQS_MORE) err = 1;
}
apr_thread_mutex_unlock(mqs->lock);
if (mqs->gop_processed != NULL) {
gop_waitany(mqs->gop_processed);
gop_free(mqs->gop_processed, OP_DESTROY);
mqs->gop_processed = NULL;
}
if (err != 0) {
log_printf(2, "ERROR no more data available!\n");
return(-1);
}
//** Now handle the waiting gop
apr_thread_mutex_lock(mqs->lock);
log_printf(5, "before loop msid=%d waiting=%d processed=%d\n", mqs->msid, mqs->waiting, mqs->processed);
while (mqs->waiting == 1) {
log_printf(5, "LOOP msid=%d waiting=%d processed=%d\n", mqs->msid, mqs->waiting, mqs->processed);
if (gop_will_block(mqs->gop_waiting) == 0) { //** Oops! failed request
status = gop_get_status(mqs->gop_waiting);
log_printf(2, "msid=%d gid=%d status=%d\n", mqs->msid, gop_id(mqs->gop_waiting), status.op_status);
if (status.op_status != OP_STATE_SUCCESS) {
mqs->waiting = -3;
err = 1;
} else {
apr_thread_cond_timedwait(mqs->cond, mqs->lock, dt);
}
} else {
apr_thread_cond_timedwait(mqs->cond, mqs->lock, dt);
}
}
if (mqs->waiting == 0) { //** Flag the receiver to accept the data
mqs->waiting = -1;
apr_thread_cond_broadcast(mqs->cond); //** Let the receiver know we're ready to accept the data
//** Wait for the data to be accepted
while (mqs->waiting != -2) {
apr_thread_cond_wait(mqs->cond, mqs->lock);
}
} else if (mqs->waiting == -3) { //**error occured
err = 1;
}
apr_thread_mutex_unlock(mqs->lock);
//** Flip states
mqs->gop_processed = mqs->gop_waiting;
mqs->gop_waiting = NULL;
//** This shouldn't get triggered but just in case lets throw an error.
if ((mqs->gop_processed == NULL) && (mqs->data != NULL)) {
if ((mqs->data[MQS_STATE_INDEX] == MQS_MORE) && (mqs->want_more == MQS_MORE)) {
err = 3;
log_printf(0, "ERROR: MQS gop processed=waiting=NULL want_more set!!!!!! err=%d\n", err);
fprintf(stderr, "ERROR: MQS gop processed=waiting=NULL want_more set!!!!!! err=%d\n", err);
}
}
//** Check if we need to fire off the next request
if (mqs->data != NULL) {
if ((mqs->data[MQS_STATE_INDEX] == MQS_MORE) && (mqs->want_more == MQS_MORE)) {
mq_stream_read_request(mqs);
}
}
log_printf(5, "err=%d\n", err);
return(err);
}
int rss_perform_check(resource_service_fn_t *rs)
{
rs_simple_priv_t *rss = (rs_simple_priv_t *)rs->priv;
apr_hash_index_t *hi;
rss_check_entry_t *ce;
int prev_status, status_change;
char *rid;
apr_ssize_t klen;
op_status_t status;
opque_t *q;
op_generic_t *gop;
log_printf(5, "START\n");
//** Generate the task list
q = new_opque();
status_change = 0;
apr_thread_mutex_lock(rss->lock);
for (hi = apr_hash_first(NULL, rss->rid_mapping); hi != NULL; hi = apr_hash_next(hi)) {
apr_hash_this(hi, (const void **)&rid, &klen, (void **)&ce);
// if (ce->re->status != RS_STATUS_IGNORE) {
gop = ds_res_inquire(rss->ds, ce->ds_key, rss->da, ce->space, rss->check_timeout);
gop_set_private(gop, ce);
opque_add(q, gop);
// }
}
apr_thread_mutex_unlock(rss->lock);
//** Wait for them all to complete
opque_waitall(q);
//** Process the results
apr_thread_mutex_lock(rss->lock);
if (rss->modify_time == rss->current_check) { //** Only process the results if not updated
for (gop = opque_get_next_finished(q); gop != NULL; gop = opque_get_next_finished(q)) {
status = gop_get_status(gop);
ce = gop_get_private(gop);
prev_status = ce->re->status;
if (status.op_status == OP_STATE_SUCCESS) { //** Got a valid response
ce->re->space_free = ds_res_inquire_get(rss->ds, DS_INQUIRE_FREE, ce->space);
ce->re->space_used = ds_res_inquire_get(rss->ds, DS_INQUIRE_USED, ce->space);
ce->re->space_total = ds_res_inquire_get(rss->ds, DS_INQUIRE_TOTAL, ce->space);
if (ce->re->status != RS_STATUS_IGNORE) {
if (ce->re->space_free <= rss->min_free) {
ce->re->status = RS_STATUS_OUT_OF_SPACE;
} else {
ce->re->status = RS_STATUS_UP;
}
}
} else { //** No response so mark it as down
if (ce->re->status != RS_STATUS_IGNORE) ce->re->status = RS_STATUS_DOWN;
}
if (prev_status != ce->re->status) status_change = 1;
log_printf(15, "ds_key=%s prev_status=%d new_status=%d\n", ce->ds_key, prev_status, ce->re->status);
gop_free(gop, OP_DESTROY);
}
}
opque_free(q, OP_DESTROY);
apr_thread_mutex_unlock(rss->lock);
log_printf(5, "END status_change=%d\n", status_change);
return(status_change);
}
double write_allocs(ibp_capset_t *caps, int qlen, int n, int asize, int block_size)
{
int count, i, j, nleft, nblocks, rem, len, err, block_start, alloc_start;
int *slot;
op_status_t status;
int64_t nbytes, last_bytes, print_bytes, delta_bytes;
apr_int32_t nfds, finished;
double dbytes, r1, r2;
apr_pool_t *pool;
apr_file_t *fd_in;
opque_t *q;
op_generic_t *op;
char *buffer = (char *)malloc(block_size);
apr_interval_time_t dt = 10;
apr_pollfd_t pfd;
apr_time_t stime, dtime;
int *tbuf_index;
tbuffer_t *buf;
Stack_t *tbuf_free;
tbuf_free = new_stack();
type_malloc_clear(tbuf_index, int, qlen);
type_malloc_clear(buf, tbuffer_t, qlen);
for (i=0; i<qlen; i++) {
tbuf_index[i] = i;
push(tbuf_free, &(tbuf_index[i]));
}
//** Make the stuff to capture the kbd
apr_pool_create(&pool, NULL);
nfds = 1;
apr_file_open_stdin(&fd_in, pool);
pfd.p = pool;
pfd.desc_type = APR_POLL_FILE;
pfd.reqevents = APR_POLLIN|APR_POLLHUP;
pfd.desc.f = fd_in;
pfd.client_data = NULL;
//** Init the ibp stuff
init_buffer(buffer, 'W', block_size);
q = new_opque();
opque_start_execution(q);
nblocks = asize / block_size;
rem = asize % block_size;
if (rem > 0) nblocks++;
block_start = 0;
alloc_start = 0;
finished = 0;
apr_poll(&pfd, nfds, &finished, dt);
count = 0;
nbytes=0;
last_bytes = 0;
delta_bytes = 1024 * 1024 * 1024;
print_bytes = delta_bytes;
stime = apr_time_now();
while (finished == 0) {
// nleft = qlen - opque_tasks_left(q);
nleft = stack_size(tbuf_free);
// printf("\nLOOP: nleft=%d qlen=%d\n", nleft, qlen);
if (nleft > 0) {
for (j=block_start; j < nblocks; j++) {
for (i=alloc_start; i<n; i++) {
nleft--;
if (nleft <= 0) {
block_start = j;
alloc_start = i;
goto skip_submit;
}
slot = (int *)pop(tbuf_free);
if ((j==(nblocks-1)) && (rem > 0)) {
len = rem;
} else {
len = block_size;
}
// printf("%d=(%d,%d) ", count, j, i);
tbuffer_single(&(buf[*slot]), len, buffer);
op = new_ibp_write_op(ic, get_ibp_cap(&(caps[i]), IBP_WRITECAP), j*block_size, &(buf[*slot]), 0, len, ibp_timeout);
gop_set_id(op, *slot);
ibp_op_set_cc(ibp_get_iop(op), cc);
ibp_op_set_ncs(ibp_get_iop(op), ncs);
opque_add(q, op);
}
alloc_start = 0;
}
block_start = 0;
}
skip_submit:
finished = 1;
apr_poll(&pfd, nfds, &finished, dt);
do { //** Empty the finished queue. Always wait for for at least 1 to complete
op = opque_waitany(q);
status = gop_get_status(op);
if (status.error_code != IBP_OK) {
printf("ERROR: Aborting with error code %d\n", status.error_code);
finished = 0;
}
count++;
i = gop_get_id(op);
nbytes = nbytes + tbuffer_size(&(buf[i]));
if (nbytes > print_bytes) {
dbytes = nbytes / (1.0*1024*1024*1024);
dtime = apr_time_now() - stime;
r2 = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = nbytes - last_bytes;
r1 = r1 / (r2 * 1024.0 * 1024.0);
printf("%.2lfGB written (%.2lfMB/s : %.2lf secs)\n", dbytes, r1, r2);
print_bytes = print_bytes + delta_bytes;
last_bytes = nbytes;
stime = apr_time_now();
}
push(tbuf_free, &(tbuf_index[i]));
gop_free(op, OP_DESTROY);
} while (opque_tasks_finished(q) > 0);
}
err = opque_waitall(q);
if (err != OP_STATE_SUCCESS) {
printf("write_allocs: At least 1 error occured! * ibp_errno=%d * nfailed=%d\n", err, opque_tasks_failed(q));
}
opque_free(q, OP_DESTROY);
free_stack(tbuf_free, 0);
free(tbuf_index);
free(buf);
free(buffer);
apr_pool_destroy(pool);
dbytes = nbytes;
return(dbytes);
}
int main(int argc, char **argv)
{
char *path;
int i, mode, n, nfailed;
os_fsck_iter_t *it;
char *fname;
op_generic_t *gop;
op_status_t status;
int ftype, err;
if (argc < 2) {
printf("\n");
printf("os_fsck LIO_COMMON_OPTIONS [-fix manual|delete|repair] path\n");
lio_print_options(stdout);
printf(" -fix - How to handle issues. Default is manual. Can also be delete or repair.\n");
printf(" path - Path prefix to use\n");
printf("\n");
return(1);
}
lio_init(&argc, &argv);
if (argc < 2) {
printf("Missing path!\n");
return(1);
}
mode = OS_FSCK_MANUAL;
i = 1;
if (strcmp(argv[i], "-fix") == 0) {
i++;
if (strcmp(argv[i], "delete") == 0) {
mode = OS_FSCK_REMOVE;
} else if (strcmp(argv[i], "repair") == 0) {
mode = OS_FSCK_REPAIR;
}
i++;
}
path = argv[i];
info_printf(lio_ifd, 0, "--------------------------------------------------------------------\n");
info_printf(lio_ifd, 0, "Using path=%s and mode=%d (%d=manual, %d=delete, %d=repair)\n", path, mode, OS_FSCK_MANUAL, OS_FSCK_REMOVE, OS_FSCK_REPAIR);
info_printf(lio_ifd, 0, "Possible error states: %d=missing attr, %d=missing object\n", OS_FSCK_MISSING_ATTR, OS_FSCK_MISSING_OBJECT);
info_printf(lio_ifd, 0, "--------------------------------------------------------------------\n");
tbx_info_flush(lio_ifd);
n = 0;
nfailed = 0;
it = os_create_fsck_iter(lio_gc->os, lio_gc->creds, path, OS_FSCK_MANUAL); //** WE use reolve to clean up so we can see the problem objects
while ((err = os_next_fsck(lio_gc->os, it, &fname, &ftype)) != OS_FSCK_GOOD) {
info_printf(lio_ifd, 0, "err:%d type:%d object:%s\n", err, ftype, fname);
if (err == OS_FSCK_ERROR) { //** Internal error so abort!
info_printf(lio_ifd, 0, "Internal FSCK error! Aborting!\n");
break;
}
if (mode != OS_FSCK_MANUAL) {
gop = os_fsck_object(lio_gc->os, lio_gc->creds, fname, ftype, mode);
gop_waitany(gop);
status = gop_get_status(gop);
gop_free(gop, OP_DESTROY);
if (status.error_code != OS_FSCK_GOOD) nfailed++;
info_printf(lio_ifd, 0, " resolve:%d object:%s\n", status.error_code, fname);
}
free(fname);
fname = NULL;
n++;
}
os_destroy_fsck_iter(lio_gc->os, it);
info_printf(lio_ifd, 0, "--------------------------------------------------------------------\n");
info_printf(lio_ifd, 0, "Problem objects: %d Repair Failed count: %d\n", n, nfailed);
info_printf(lio_ifd, 0, "--------------------------------------------------------------------\n");
lio_shutdown();
return(0);
}
