op_status_t gop_sync_exec_status(op_generic_t *gop)
{
int err;
op_status_t status;
if (gop->type == Q_TYPE_OPERATION) { //** Got an operation so see if we can directly exec it
if (gop->base.pc->fn->sync_exec != NULL) { //** Yup we can!
log_printf(15, "sync_exec -- waiting for gid=%d to complete\n", gop_id(gop));
gop->base.pc->fn->sync_exec(gop->base.pc, gop);
status = gop->base.status;
log_printf(15, "sync_exec -- gid=%d completed with err=%d\n", gop_id(gop), status.op_status);
gop_free(gop, OP_DESTROY);
return(status);
}
}
log_printf(15, "waiting for gid=%d to complete\n", gop_id(gop));
err = gop_waitall(gop);
status = gop_get_status(gop);
log_printf(15, "gid=%d completed with err=%d\n", gop_id(gop), err);
gop_free(gop, OP_DESTROY);
log_printf(15, "After gop destruction\n");
return(status);
}
ibp_depotinfo_t *IBP_status(ibp_depot_t *depot, int cmd, ibp_timer_t *timer, char *password,
unsigned long int hard, unsigned long int soft, long duration)
{
int err;
ibp_op_t op;
ibp_depotinfo_t *di = NULL;
make_ibp_sync_context();
init_ibp_op(_ibp_sync, &op);
if (cmd == IBP_ST_INQ) {
di = (ibp_depotinfo_t *)malloc(sizeof(ibp_depotinfo_t));
assert(di != NULL);
set_ibp_depot_inq_op(&op, depot, password, di, timer->ClientTimeout);
} else {
set_ibp_depot_modify_op(&op, depot, password, hard, soft, duration, timer->ClientTimeout);
}
err = ibp_sync_command(&op);
gop_free(ibp_get_gop(&op), OP_FINALIZE);
if (err != IBP_OK) {
free(di);
return(NULL);
}
return(di);
}
void single_gop_mark_completed(op_generic_t *gop, op_status_t status)
{
op_common_t *base = &(gop->base);
int mode;
log_printf(15, "gop_mark_completed: START gid=%d status=%d\n", gop_id(gop), status.op_status);
lock_gop(gop);
log_printf(15, "gop_mark_completed: after lock gid=%d\n", gop_id(gop));
//** Store the status
base->status = status;
//** and trigger any callbacks
log_printf(15, "gop_mark_completed: before cb gid=%d op_status=%d\n", gop_id(gop), base->status.op_status);
callback_execute(base->cb, base->status.op_status);
log_printf(15, "gop_mark_completed: after cb gid=%d op_success=%d\n", gop_id(gop), base->status.op_status);
base->state = 1;
//** Lastly trigger the signal. for anybody listening
apr_thread_cond_broadcast(gop->base.ctl->cond);
log_printf(15, "gop_mark_completed: after brodcast gid=%d\n", gop_id(gop));
mode = gop_get_auto_destroy(gop); //** Get the auto destroy status w/in the lock
unlock_gop(gop);
//** Check if we do an auto cleanop
if (mode == 1) gop_free(gop, OP_DESTROY);
}
void gop_set_auto_destroy(op_generic_t *gop, int val)
{
int state;
lock_gop(gop);
gop->base.auto_destroy = val;
state = gop->base.state;
unlock_gop(gop);
//** Already completed go ahead and destroy it
if (state == 1) gop_free(gop, OP_DESTROY);
}
unsigned long int IBP_phoebus_copy(char *path, ibp_cap_t *srccap, ibp_cap_t *destcap, ibp_timer_t *src_timer, ibp_timer_t *dest_timer,
ibp_off_t size, ibp_off_t offset)
{
ibp_op_t op;
int err;
make_ibp_sync_context();
init_ibp_op(_ibp_sync, &op);
set_ibp_copyappend_op(&op, NS_TYPE_PHOEBUS, path, srccap, destcap, offset, size, src_timer->ClientTimeout,
dest_timer->ServerSync, dest_timer->ClientTimeout);
err = ibp_sync_command(&op);
gop_free(ibp_get_gop(&op), OP_FINALIZE);
if (err != IBP_OK) return(0);
return(size);
}
ibp_depot_t *generate_depot_list(int *n_depots, ibp_depot_t *depot_list)
{
int n, i, j, err;
rid_t skip_rid, rid;
ibp_depot_t *dl;
op_generic_t *gop;
ibp_ridlist_t rlist;
//*** 1st Query the depot resources ***
gop = new_ibp_query_resources_op(ic, depot_list, &rlist, ibp_timeout);
err = ibp_sync_command(ibp_get_iop(gop));
if (err != IBP_OK) {
printf("Can't query the depot RID list! err=%d\n", err);
abort();
}
// printf("Number of resources: %d\n", ridlist_get_size(&rlist));
// for (i=0; i<ridlist_get_size(&rlist); i++) {
// printf(" %d: %s\n", i, ibp_rid2str(ridlist_get_element(&rlist, i), rbuf));
// }
//** Now generate the list
n = ridlist_get_size(&rlist);
dl = (ibp_depot_t *)malloc(sizeof(ibp_depot_t)*n);
if (n < 2) {
printf("RID list to short! n=%d\n", n);
abort();
}
skip_rid = depot_list->rid;
j = 0;
for (i=0; i<n; i++) {
rid = ridlist_get_element(&rlist, i);
if (ibp_compare_rid(skip_rid, rid) != 0) { //** See if we skip it
set_ibp_depot(&(dl[j]), depot_list->host, depot_list->port, rid);
j++;
}
}
gop_free(gop, OP_DESTROY);
*n_depots = j;
return(dl);
}
int IBP_manage(ibp_cap_t *cap, ibp_timer_t *timer, int cmd, int captype, ibp_capstatus_t *cs)
{
ibp_op_t op;
int err;
make_ibp_sync_context();
init_ibp_op(_ibp_sync, &op);
log_printf(15, "IBP_manage: cmd=%d cap=%s cctype=%d\n", cmd, cap, op.ic->cc[IBP_MANAGE].type);
fflush(stdout);
err = 0;
switch (cmd) {
case IBP_INCR:
case IBP_DECR:
set_ibp_modify_count_op(&op, cap, cmd, captype, timer->ClientTimeout);
break;
case IBP_PROBE:
set_ibp_probe_op(&op, cap, cs, timer->ClientTimeout);
break;
case IBP_CHNG:
set_ibp_modify_alloc_op(&op, cap, cs->maxSize, cs->attrib.duration, cs->attrib.reliability, timer->ClientTimeout);
break;
default:
err = 1;
log_printf(0, "IBP_manage: Invalid command: %d\n", cmd);
}
log_printf(15, "AFTER IBP_manage: cmd=%d cap=%s cctype=%d\n", cmd, cap, op.ic->cc[IBP_MANAGE].type);
fflush(stdout);
if (err == 0) {
ibp_sync_command(&op);
} else {
IBP_errno = IBP_E_INVALID_PARAMETER;
}
gop_free(ibp_get_gop(&op), OP_FINALIZE);
if (IBP_errno != IBP_OK) return(-1);
return(0);
}
unsigned long int IBP_load(ibp_cap_t *cap, ibp_timer_t *timer, char *data,
unsigned long int size, unsigned long int offset)
{
ibp_op_t op;
int err;
tbx_tbuf_t buf;
tbx_tbuf_single(&buf, size, data);
make_ibp_sync_context();
init_ibp_op(_ibp_sync, &op);
set_ibp_read_op(&op, cap, offset, &buf, 0, size, timer->ClientTimeout);
err = ibp_sync_command(&op);
gop_free(ibp_get_gop(&op), OP_FINALIZE);
if (err != IBP_OK) return(0);
return(size);
}
ibp_capset_t *IBP_allocate(ibp_depot_t *depot, ibp_timer_t *timer, unsigned long int size, ibp_attributes_t *attr)
{
ibp_op_t op;
int err;
make_ibp_sync_context();
ibp_capset_t *cs = (ibp_capset_t *)malloc(sizeof(ibp_capset_t));
assert(cs != NULL);
init_ibp_op(_ibp_sync, &op);
set_ibp_alloc_op(&op, cs, size, depot, attr, CHKSUM_DEFAULT, 0, timer->ClientTimeout);
err = ibp_sync_command(&op);
gop_free(ibp_get_gop(&op), OP_FINALIZE);
if (err != IBP_OK) {
free(cs);
cs = NULL;
}
return(cs);
}
op_status_t segment_get_func(void *arg, int id)
{
segment_copy_t *sc = (segment_copy_t *)arg;
tbuffer_t *wbuf, *rbuf, *tmpbuf;
tbuffer_t tbuf1, tbuf2;
char *rb, *wb, *tb;
ex_off_t bufsize;
int err;
ex_off_t rpos, wpos, rlen, wlen, tlen, nbytes, got, total;
ex_iovec_t rex;
apr_time_t loop_start, file_start;
double dt_loop, dt_file;
op_generic_t *gop;
op_status_t status;
//** Set up the buffers
bufsize = sc->bufsize / 2; //** The buffer is split for R/W
rb = sc->buffer;
wb = &(sc->buffer[bufsize]);
tbuffer_single(&tbuf1, bufsize, rb);
tbuffer_single(&tbuf2, bufsize, wb);
rbuf = &tbuf1;
wbuf = &tbuf2;
status = op_success_status;
//** Read the initial block
rpos = sc->src_offset;
wpos = 0;
nbytes = segment_size(sc->src) - sc->src_offset;
if (nbytes < 0) {
rlen = bufsize;
} else {
rlen = (nbytes > bufsize) ? bufsize : nbytes;
}
log_printf(5, "FILE fd=%p\n", sc->fd);
ex_iovec_single(&rex, rpos, rlen);
wlen = 0;
rpos += rlen;
nbytes -= rlen;
loop_start = apr_time_now();
gop = segment_read(sc->src, sc->da, sc->rw_hints, 1, &rex, rbuf, 0, sc->timeout);
err = gop_waitall(gop);
if (err != OP_STATE_SUCCESS) {
log_printf(1, "Intial read failed! src=" XIDT " rpos=" XOT " len=" XOT "\n", segment_id(sc->src), rpos, rlen);
gop_free(gop, OP_DESTROY);
return(op_failure_status);
}
gop_free(gop, OP_DESTROY);
total = 0;
do {
//** Swap the buffers
tb = rb;
rb = wb;
wb = tb;
tmpbuf = rbuf;
rbuf = wbuf;
wbuf = tmpbuf;
tlen = rlen;
rlen = wlen;
wlen = tlen;
log_printf(1, "sseg=" XIDT " rpos=" XOT " wpos=" XOT " rlen=" XOT " wlen=" XOT " nbytes=" XOT "\n", segment_id(sc->src), rpos, wpos, rlen, wlen, nbytes);
//** Read in the next block
if (nbytes < 0) {
rlen = bufsize;
} else {
rlen = (nbytes > bufsize) ? bufsize : nbytes;
}
if (rlen > 0) {
ex_iovec_single(&rex, rpos, rlen);
loop_start = apr_time_now();
gop = segment_read(sc->src, sc->da, sc->rw_hints, 1, &rex, rbuf, 0, sc->timeout);
gop_start_execution(gop); //** Start doing the transfer
rpos += rlen;
nbytes -= rlen;
}
//** Start the write
file_start = apr_time_now();
got = fwrite(wb, 1, wlen, sc->fd);
dt_file = apr_time_now() - file_start;
dt_file /= (double)APR_USEC_PER_SEC;
total += got;
log_printf(5, "sid=" XIDT " fwrite(wb,1," XOT ", sc->fd)=" XOT " total=" XOT "\n", segment_id(sc->src), wlen, got, total);
if (wlen != got) {
log_printf(1, "ERROR from fread=%d dest sid=" XIDT "\n", errno, segment_id(sc->dest));
status = op_failure_status;
gop_waitall(gop);
gop_free(gop, OP_DESTROY);
goto fail;
}
wpos += wlen;
//** Wait for the read to complete
if (rlen > 0) {
err = gop_waitall(gop);
gop_free(gop, OP_DESTROY);
if (err != OP_STATE_SUCCESS) {
log_printf(1, "ERROR write(dseg=" XIDT ") failed! wpos=" XOT " len=" XOT "\n", segment_id(sc->dest), wpos, wlen);
status = op_failure_status;
goto fail;
}
}
dt_loop = apr_time_now() - loop_start;
dt_loop /= (double)APR_USEC_PER_SEC;
log_printf(1, "dt_loop=%lf dt_file=%lf\n", dt_loop, dt_file);
} while (rlen > 0);
fail:
return(status);
}
int main(int argc, char **argv)
{
int i, start_option;
int n_rid;
char *query_text = NULL;
rs_query_t *rq;
ex_off_t block_size, total_size;
lio_exnode_t *ex;
lio_segment_create_fn_t *screate;
char *fname_out = NULL;
lio_exnode_exchange_t *exp;
lio_segment_t *seg = NULL;
gop_op_generic_t *gop;
if (argc < 5) {
printf("\n");
printf("mk_linear LIO_COMMON_OPTIONS -q rs_query_string n_rid block_size total_size file.ex3\n");
lio_print_options(stdout);
printf("\n");
return(1);
}
lio_init(&argc, &argv);
//*** Parse the args
i=1;
do {
start_option = i;
if (strcmp(argv[i], "-q") == 0) { //** Load the query
i++;
query_text = argv[i];
i++;
}
} while (start_option - i < 0);
//** Load the fixed options
n_rid = atoi(argv[i]);
i++;
block_size = atoi(argv[i]);
i++;
total_size = atoi(argv[i]);
i++;
fname_out = argv[i];
i++;
//** Do some simple sanity checks
//** Make sure we loaded a simple res service
if (fname_out == NULL) {
printf("Missing output filename!\n");
return(2);
}
//** Create an empty linear segment
screate = lio_lookup_service(lio_gc->ess, SEG_SM_CREATE, SEGMENT_TYPE_LINEAR);
seg = (*screate)(lio_gc->ess);
//** Parse the query
rq = rs_query_parse(lio_gc->rs, query_text);
if (rq == NULL) {
printf("Error parsing RS query: %s\n", query_text);
printf("Exiting!\n");
exit(1);
}
//** Make the actual segment
gop = lio_segment_linear_make_gop(seg, NULL, rq, n_rid, block_size, total_size, lio_gc->timeout);
i = gop_waitall(gop);
if (i != 0) {
printf("ERROR making segment! nerr=%d\n", i);
return(-1);
}
gop_free(gop, OP_DESTROY);
//** Make an empty exnode
ex = lio_exnode_create();
//** and insert it
lio_view_insert(ex, seg);
//** Print it
exp = lio_exnode_exchange_create(EX_TEXT);
lio_exnode_serialize(ex, exp);
printf("%s", exp->text.text);
//** and Save if back to disk
FILE *fd = fopen(fname_out, "w");
fprintf(fd, "%s", exp->text.text);
fclose(fd);
lio_exnode_exchange_destroy(exp);
//** Clean up
lio_exnode_destroy(ex);
rs_query_destroy(lio_gc->rs, rq);
lio_shutdown();
return(0);
}
double write_allocs(ibp_capset_t *caps, int qlen, int n, int asize, int block_size)
{
int count, i, j, nleft, nblocks, rem, len, err, block_start, alloc_start;
int *slot;
op_status_t status;
int64_t nbytes, last_bytes, print_bytes, delta_bytes;
apr_int32_t nfds, finished;
double dbytes, r1, r2;
apr_pool_t *pool;
apr_file_t *fd_in;
opque_t *q;
op_generic_t *op;
char *buffer = (char *)malloc(block_size);
apr_interval_time_t dt = 10;
apr_pollfd_t pfd;
apr_time_t stime, dtime;
int *tbuf_index;
tbuffer_t *buf;
Stack_t *tbuf_free;
tbuf_free = new_stack();
type_malloc_clear(tbuf_index, int, qlen);
type_malloc_clear(buf, tbuffer_t, qlen);
for (i=0; i<qlen; i++) {
tbuf_index[i] = i;
push(tbuf_free, &(tbuf_index[i]));
}
//** Make the stuff to capture the kbd
apr_pool_create(&pool, NULL);
nfds = 1;
apr_file_open_stdin(&fd_in, pool);
pfd.p = pool;
pfd.desc_type = APR_POLL_FILE;
pfd.reqevents = APR_POLLIN|APR_POLLHUP;
pfd.desc.f = fd_in;
pfd.client_data = NULL;
//** Init the ibp stuff
init_buffer(buffer, 'W', block_size);
q = new_opque();
opque_start_execution(q);
nblocks = asize / block_size;
rem = asize % block_size;
if (rem > 0) nblocks++;
block_start = 0;
alloc_start = 0;
finished = 0;
apr_poll(&pfd, nfds, &finished, dt);
count = 0;
nbytes=0;
last_bytes = 0;
delta_bytes = 1024 * 1024 * 1024;
print_bytes = delta_bytes;
stime = apr_time_now();
while (finished == 0) {
// nleft = qlen - opque_tasks_left(q);
nleft = stack_size(tbuf_free);
// printf("\nLOOP: nleft=%d qlen=%d\n", nleft, qlen);
if (nleft > 0) {
for (j=block_start; j < nblocks; j++) {
for (i=alloc_start; i<n; i++) {
nleft--;
if (nleft <= 0) {
block_start = j;
alloc_start = i;
goto skip_submit;
}
slot = (int *)pop(tbuf_free);
if ((j==(nblocks-1)) && (rem > 0)) {
len = rem;
} else {
len = block_size;
}
// printf("%d=(%d,%d) ", count, j, i);
tbuffer_single(&(buf[*slot]), len, buffer);
op = new_ibp_write_op(ic, get_ibp_cap(&(caps[i]), IBP_WRITECAP), j*block_size, &(buf[*slot]), 0, len, ibp_timeout);
gop_set_id(op, *slot);
ibp_op_set_cc(ibp_get_iop(op), cc);
ibp_op_set_ncs(ibp_get_iop(op), ncs);
opque_add(q, op);
}
alloc_start = 0;
}
block_start = 0;
}
skip_submit:
finished = 1;
apr_poll(&pfd, nfds, &finished, dt);
do { //** Empty the finished queue. Always wait for for at least 1 to complete
op = opque_waitany(q);
status = gop_get_status(op);
if (status.error_code != IBP_OK) {
printf("ERROR: Aborting with error code %d\n", status.error_code);
finished = 0;
}
count++;
i = gop_get_id(op);
nbytes = nbytes + tbuffer_size(&(buf[i]));
if (nbytes > print_bytes) {
dbytes = nbytes / (1.0*1024*1024*1024);
dtime = apr_time_now() - stime;
r2 = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = nbytes - last_bytes;
r1 = r1 / (r2 * 1024.0 * 1024.0);
printf("%.2lfGB written (%.2lfMB/s : %.2lf secs)\n", dbytes, r1, r2);
print_bytes = print_bytes + delta_bytes;
last_bytes = nbytes;
stime = apr_time_now();
}
push(tbuf_free, &(tbuf_index[i]));
gop_free(op, OP_DESTROY);
} while (opque_tasks_finished(q) > 0);
}
err = opque_waitall(q);
if (err != OP_STATE_SUCCESS) {
printf("write_allocs: At least 1 error occured! * ibp_errno=%d * nfailed=%d\n", err, opque_tasks_failed(q));
}
opque_free(q, OP_DESTROY);
free_stack(tbuf_free, 0);
free(tbuf_index);
free(buf);
free(buffer);
apr_pool_destroy(pool);
dbytes = nbytes;
return(dbytes);
}
void *lru_dirty_thread(apr_thread_t *th, void *data)
{
cache_t *c = (cache_t *)data;
cache_lru_t *cp = (cache_lru_t *)c->fn.priv;
double df;
int n, i;
ex_id_t *id;
segment_t *seg;
opque_t *q;
op_generic_t *gop;
cache_segment_t *s;
skiplist_iter_t it;
segment_t **flush_list;
cache_lock(c);
log_printf(15, "Dirty thread launched\n");
while (c->shutdown_request == 0) {
apr_thread_cond_timedwait(cp->dirty_trigger, c->lock, cp->dirty_max_wait);
df = cp->max_bytes;
df = c->stats.dirty_bytes / df;
log_printf(15, "Dirty thread running. dirty fraction=%lf dirty bytes=" XOT " inprogress=%d cached segments=%d\n", df, c->stats.dirty_bytes, cp->flush_in_progress, list_key_count(c->segments));
cp->flush_in_progress = 1;
q = new_opque();
n = list_key_count(c->segments);
type_malloc(flush_list, segment_t *, n);
it = list_iter_search(c->segments, NULL, 0);
list_next(&it, (list_key_t **)&id, (list_data_t **)&seg);
i = 0;
while (seg != NULL) {
log_printf(15, "Flushing seg=" XIDT " i=%d\n", *id, i);
flush_log();
flush_list[i] = seg;
s = (cache_segment_t *)seg->priv;
atomic_set(s->cache_check_in_progress, 1); //** Flag it as being checked
gop = cache_flush_range(seg, s->c->da, 0, -1, s->c->timeout);
gop_set_myid(gop, i);
opque_add(q, gop);
i++;
list_next(&it, (list_key_t **)&id, (list_data_t **)&seg);
}
cache_unlock(c);
//** Flag the tasks as they complete
opque_start_execution(q);
while ((gop = opque_waitany(q)) != NULL) {
i = gop_get_myid(gop);
segment_lock(flush_list[i]);
s = (cache_segment_t *)flush_list[i]->priv;
log_printf(15, "Flush completed seg=" XIDT " i=%d\n", segment_id(flush_list[i]), i);
flush_log();
atomic_set(s->cache_check_in_progress, 0); //** Flag it as being finished
segment_unlock(flush_list[i]);
gop_free(gop, OP_DESTROY);
}
opque_free(q, OP_DESTROY);
cache_lock(c);
cp->flush_in_progress = 0;
free(flush_list);
df = cp->max_bytes;
df = c->stats.dirty_bytes / df;
log_printf(15, "Dirty thread sleeping. dirty fraction=%lf dirty bytes=" XOT " inprogress=%d\n", df, c->stats.dirty_bytes, cp->flush_in_progress);
// apr_thread_cond_timedwait(cp->dirty_trigger, c->lock, cp->dirty_max_wait);
}
log_printf(15, "Dirty thread Exiting\n");
cache_unlock(c);
return(NULL);
}
op_status_t segment_copy_func(void *arg, int id)
{
segment_copy_t *sc = (segment_copy_t *)arg;
tbuffer_t *wbuf, *rbuf, *tmpbuf;
tbuffer_t tbuf1, tbuf2;
int err;
ex_off_t bufsize;
ex_off_t rpos, wpos, rlen, wlen, tlen, nbytes, dend;
ex_iovec_t rex, wex;
opque_t *q;
op_generic_t *rgop, *wgop;
op_status_t status;
//** Set up the buffers
bufsize = sc->bufsize / 2; //** The buffer is split for R/W
tbuffer_single(&tbuf1, bufsize, sc->buffer);
tbuffer_single(&tbuf2, bufsize, &(sc->buffer[bufsize]));
rbuf = &tbuf1;
wbuf = &tbuf2;
//** Check the length
nbytes = segment_size(sc->src) - sc->src_offset;
if (nbytes < 0) {
rlen = bufsize;
} else {
rlen = (nbytes > bufsize) ? bufsize : nbytes;
}
if ((sc->len != -1) && (sc->len < nbytes)) nbytes = sc->len;
//** Go ahead and reserve the space in the destintaion
dend = sc->dest_offset + nbytes;
log_printf(1, "reserving space=" XOT "\n", dend);
gop_sync_exec(segment_truncate(sc->dest, sc->da, -dend, sc->timeout));
//** Read the initial block
rpos = sc->src_offset;
wpos = sc->dest_offset;
// rlen = (nbytes > bufsize) ? bufsize : nbytes;
wlen = 0;
ex_iovec_single(&rex, rpos, rlen);
rpos += rlen;
nbytes -= rlen;
rgop = segment_read(sc->src, sc->da, sc->rw_hints, 1, &rex, rbuf, 0, sc->timeout);
err = gop_waitall(rgop);
if (err != OP_STATE_SUCCESS) {
log_printf(1, "Intial read failed! src=%" PRIu64 " rpos=" XOT " len=" XOT "\n", segment_id(sc->src), rpos, rlen);
gop_free(rgop, OP_DESTROY);
return(op_failure_status);
}
gop_free(rgop, OP_DESTROY);
q = new_opque();
do {
//** Swap the buffers
tmpbuf = rbuf;
rbuf = wbuf;
wbuf = tmpbuf;
tlen = rlen;
rlen = wlen;
wlen = tlen;
log_printf(1, "sseg=" XIDT " dseg=" XIDT " wpos=%" PRId64 " rlen=%" PRId64 " wlen=%" PRId64 "\n", segment_id(sc->src), segment_id(sc->dest), wpos, rlen, wlen);
//** Start the write
ex_iovec_single(&wex, wpos, wlen);
wpos += wlen;
wgop = segment_write(sc->dest, sc->da, sc->rw_hints, 1, &wex, wbuf, 0, sc->timeout);
opque_add(q, wgop);
//** Read in the next block
// rlen = (nbytes > bufsize) ? bufsize : nbytes;
if (nbytes < 0) {
rlen = bufsize;
} else {
rlen = (nbytes > bufsize) ? bufsize : nbytes;
}
if (rlen > 0) {
ex_iovec_single(&rex, rpos, rlen);
rpos += rlen;
nbytes -= rlen;
rgop = segment_read(sc->src, sc->da, sc->rw_hints, 1, &rex, rbuf, 0, sc->timeout);
opque_add(q, rgop);
}
err = opque_waitall(q);
if (err != OP_STATE_SUCCESS) {
log_printf(1, "ERROR read/write failed! src=" XIDT " rpos=" XOT " len=" XOT "\n", segment_id(sc->src), rpos, rlen);
opque_free(q, OP_DESTROY);
return(op_failure_status);
}
} while (rlen > 0);
opque_free(q, OP_DESTROY);
if (sc->truncate == 1) { //** Truncate if wanted
gop_sync_exec(segment_truncate(sc->dest, sc->da, wpos, sc->timeout));
}
status = op_success_status;
status.error_code = rpos;
return(status);
}
op_status_t segment_put_func(void *arg, int id)
{
segment_copy_t *sc = (segment_copy_t *)arg;
tbuffer_t *wbuf, *rbuf, *tmpbuf;
tbuffer_t tbuf1, tbuf2;
char *rb, *wb, *tb;
ex_off_t bufsize;
int err;
ex_off_t rpos, wpos, rlen, wlen, tlen, nbytes, got, dend;
ex_iovec_t wex;
op_generic_t *gop;
op_status_t status;
apr_time_t loop_start, file_start;
double dt_loop, dt_file;
//** Set up the buffers
bufsize = sc->bufsize / 2; //** The buffer is split for R/W
rb = sc->buffer;
wb = &(sc->buffer[bufsize]);
tbuffer_single(&tbuf1, bufsize, rb);
tbuffer_single(&tbuf2, bufsize, wb);
rbuf = &tbuf1;
wbuf = &tbuf2;
nbytes = sc->len;
status = op_success_status;
//** Go ahead and reserve the space in the destintaion
dend = sc->dest_offset + nbytes;
gop_sync_exec(segment_truncate(sc->dest, sc->da, -dend, sc->timeout));
//** Read the initial block
rpos = 0;
wpos = sc->dest_offset;
if (nbytes < 0) {
rlen = bufsize;
} else {
rlen = (nbytes > bufsize) ? bufsize : nbytes;
}
wlen = 0;
rpos += rlen;
nbytes -= rlen;
log_printf(0, "FILE fd=%p bufsize=" XOT " rlen=" XOT " nbytes=" XOT "\n", sc->fd, bufsize, rlen, nbytes);
loop_start = apr_time_now();
got = fread(rb, 1, rlen, sc->fd);
dt_file = apr_time_now() - loop_start;
dt_file /= (double)APR_USEC_PER_SEC;
if (got == 0) {
if (feof(sc->fd) == 0) {
log_printf(1, "ERROR from fread=%d dest sid=" XIDT " rlen=" XOT " got=" XOT "\n", errno, segment_id(sc->dest), rlen, got);
status = op_failure_status;
}
goto finished;
}
rlen = got;
do {
//** Swap the buffers
tb = rb;
rb = wb;
wb = tb;
tmpbuf = rbuf;
rbuf = wbuf;
wbuf = tmpbuf;
tlen = rlen;
rlen = wlen;
wlen = tlen;
log_printf(1, "dseg=" XIDT " wpos=" XOT " rlen=" XOT " wlen=" XOT "\n", segment_id(sc->dest), wpos, rlen, wlen);
//** Start the write
ex_iovec_single(&wex, wpos, wlen);
wpos += wlen;
loop_start = apr_time_now();
gop = segment_write(sc->dest, sc->da, sc->rw_hints, 1, &wex, wbuf, 0, sc->timeout);
gop_start_execution(gop); //** Start doing the transfer
//** Read in the next block
if (nbytes < 0) {
rlen = bufsize;
} else {
rlen = (nbytes > bufsize) ? bufsize : nbytes;
}
if (rlen > 0) {
file_start = apr_time_now();
got = fread(rb, 1, rlen, sc->fd);
dt_file = apr_time_now() - file_start;
dt_file /= (double)APR_USEC_PER_SEC;
if (got == 0) {
if (feof(sc->fd) == 0) {
log_printf(1, "ERROR from fread=%d dest sid=" XIDT " got=" XOT " rlen=" XOT "\n", errno, segment_id(sc->dest), got, rlen);
status = op_failure_status;
gop_waitall(gop);
gop_free(gop, OP_DESTROY);
goto finished;
}
}
rlen = got;
rpos += rlen;
nbytes -= rlen;
}
//** Wait for it to complete
err = gop_waitall(gop);
dt_loop = apr_time_now() - loop_start;
dt_loop /= (double)APR_USEC_PER_SEC;
log_printf(1, "dt_loop=%lf dt_file=%lf\n", dt_loop, dt_file);
if (err != OP_STATE_SUCCESS) {
log_printf(1, "ERROR write(dseg=" XIDT ") failed! wpos=" XOT " len=" XOT "\n", segment_id(sc->dest), wpos, wlen);
status = op_failure_status;
gop_free(gop, OP_DESTROY);
goto finished;
}
gop_free(gop, OP_DESTROY);
} while (rlen > 0);
if (sc->truncate == 1) { //** Truncate if wanted
gop_sync_exec(segment_truncate(sc->dest, sc->da, wpos, sc->timeout));
}
finished:
// status.error_code = rpos;
return(status);
}
int mq_stream_read_wait(mq_stream_t *mqs)
{
int err = 0;
apr_interval_time_t dt;
op_status_t status;
//** If 1st time make all the variables
if (mqs->mpool == NULL) {
apr_pool_create(&mqs->mpool, NULL);
apr_thread_mutex_create(&(mqs->lock), APR_THREAD_MUTEX_DEFAULT, mqs->mpool);
apr_thread_cond_create(&(mqs->cond), mqs->mpool);
}
dt = apr_time_from_sec(1);
//** Flag the just processed gop to clean up
apr_thread_mutex_lock(mqs->lock);
log_printf(5, "START msid=%d waiting=%d processed=%d gop_processed=%p\n", mqs->msid, mqs->waiting, mqs->processed, mqs->gop_processed);
if (mqs->gop_processed != NULL) mqs->processed = 1;
apr_thread_cond_broadcast(mqs->cond);
if (mqs->data) {
if (mqs->data[MQS_STATE_INDEX] != MQS_MORE) err = 1;
}
apr_thread_mutex_unlock(mqs->lock);
if (mqs->gop_processed != NULL) {
gop_waitany(mqs->gop_processed);
gop_free(mqs->gop_processed, OP_DESTROY);
mqs->gop_processed = NULL;
}
if (err != 0) {
log_printf(2, "ERROR no more data available!\n");
return(-1);
}
//** Now handle the waiting gop
apr_thread_mutex_lock(mqs->lock);
log_printf(5, "before loop msid=%d waiting=%d processed=%d\n", mqs->msid, mqs->waiting, mqs->processed);
while (mqs->waiting == 1) {
log_printf(5, "LOOP msid=%d waiting=%d processed=%d\n", mqs->msid, mqs->waiting, mqs->processed);
if (gop_will_block(mqs->gop_waiting) == 0) { //** Oops! failed request
status = gop_get_status(mqs->gop_waiting);
log_printf(2, "msid=%d gid=%d status=%d\n", mqs->msid, gop_id(mqs->gop_waiting), status.op_status);
if (status.op_status != OP_STATE_SUCCESS) {
mqs->waiting = -3;
err = 1;
} else {
apr_thread_cond_timedwait(mqs->cond, mqs->lock, dt);
}
} else {
apr_thread_cond_timedwait(mqs->cond, mqs->lock, dt);
}
}
if (mqs->waiting == 0) { //** Flag the receiver to accept the data
mqs->waiting = -1;
apr_thread_cond_broadcast(mqs->cond); //** Let the receiver know we're ready to accept the data
//** Wait for the data to be accepted
while (mqs->waiting != -2) {
apr_thread_cond_wait(mqs->cond, mqs->lock);
}
} else if (mqs->waiting == -3) { //**error occured
err = 1;
}
apr_thread_mutex_unlock(mqs->lock);
//** Flip states
mqs->gop_processed = mqs->gop_waiting;
mqs->gop_waiting = NULL;
//** This shouldn't get triggered but just in case lets throw an error.
if ((mqs->gop_processed == NULL) && (mqs->data != NULL)) {
if ((mqs->data[MQS_STATE_INDEX] == MQS_MORE) && (mqs->want_more == MQS_MORE)) {
err = 3;
log_printf(0, "ERROR: MQS gop processed=waiting=NULL want_more set!!!!!! err=%d\n", err);
fprintf(stderr, "ERROR: MQS gop processed=waiting=NULL want_more set!!!!!! err=%d\n", err);
}
}
//** Check if we need to fire off the next request
if (mqs->data != NULL) {
if ((mqs->data[MQS_STATE_INDEX] == MQS_MORE) && (mqs->want_more == MQS_MORE)) {
mq_stream_read_request(mqs);
}
}
log_printf(5, "err=%d\n", err);
return(err);
}
void *ongoing_heartbeat_thread(apr_thread_t *th, void *data)
{
mq_ongoing_t *on = (mq_ongoing_t *)data;
apr_time_t timeout = apr_time_make(on->check_interval, 0);
op_generic_t *gop;
mq_msg_t *msg;
ongoing_hb_t *oh;
ongoing_table_t *table;
apr_hash_index_t *hi, *hit;
opque_t *q;
char *id;
mq_msg_hash_t *remote_hash;
apr_time_t now;
apr_ssize_t id_len;
int n, k;
char *remote_host_string;
apr_thread_mutex_lock(on->lock);
n = 0;
do {
now = apr_time_now() - apr_time_from_sec(5); //** Give our selves a little buffer
log_printf(5, "Loop Start now=" TT "\n", apr_time_now());
q = new_opque();
// opque_start_execution(q);
for (hit = apr_hash_first(NULL, on->table); hit != NULL; hit = apr_hash_next(hit)) {
apr_hash_this(hit, (const void **)&remote_hash, &id_len, (void **)&table);
k = apr_hash_count(table->table);
if (log_level() > 1) {
remote_host_string = mq_address_to_string(table->remote_host);
log_printf(1, "host=%s count=%d\n", remote_host_string, k);
free(remote_host_string);
}
for (hi = apr_hash_first(NULL, table->table); hi != NULL; hi = apr_hash_next(hi)) {
apr_hash_this(hi, (const void **)&id, &id_len, (void **)&oh);
log_printf(1, "id=%s now=" TT " next_check=" TT "\n", oh->id, apr_time_sec(apr_time_now()), apr_time_sec(oh->next_check));
if (now > oh->next_check) {
log_printf(1, "id=%s sending HEARTBEAT EXEC SUBMIT nows=" TT " hb=%d\n", oh->id, apr_time_sec(apr_time_now()), oh->heartbeat);
flush_log();
//** Form the message
msg = mq_make_exec_core_msg(table->remote_host, 1);
mq_msg_append_mem(msg, ONGOING_KEY, ONGOING_SIZE, MQF_MSG_KEEP_DATA);
mq_msg_append_mem(msg, oh->id, oh->id_len, MQF_MSG_KEEP_DATA);
mq_msg_append_mem(msg, NULL, 0, MQF_MSG_KEEP_DATA);
//** Make the gop
gop = new_mq_op(on->mqc, msg, ongoing_response_status, NULL, NULL, oh->heartbeat);
gop_set_private(gop, table);
opque_add(q, gop);
oh->in_progress = 1; //** Flag it as in progress so it doesn't get deleted behind the scenes
}
}
}
log_printf(5, "Loop end now=" TT "\n", apr_time_now());
//** Wait for it to complete
apr_thread_mutex_unlock(on->lock);
opque_waitall(q);
apr_thread_mutex_lock(on->lock);
//** Dec the counters
while ((gop = opque_waitany(q)) != NULL) {
log_printf(0, "gid=%d gotone status=%d now=" TT "\n", gop_id(gop), (gop_get_status(gop)).op_status, apr_time_sec(apr_time_now()));
table = gop_get_private(gop);
table->count--;
//** Update the next check
for (hi = apr_hash_first(NULL, table->table); hi != NULL; hi = apr_hash_next(hi)) {
apr_hash_this(hi, (const void **)&id, &id_len, (void **)&oh);
oh->next_check = apr_time_now() + apr_time_from_sec(oh->heartbeat);
//** Check if we get rid of it
oh->in_progress = 0;
if (oh->count <= 0) { //** Need to delete it
apr_hash_set(table->table, id, id_len, NULL);
free(oh->id);
free(oh);
}
}
gop_free(gop, OP_DESTROY);
}
opque_free(q, OP_DESTROY);
now = apr_time_now();
log_printf(2, "sleeping %d now=" TT "\n", on->check_interval, now);
//** Sleep until time for the next heartbeat or time to exit
if (on->shutdown == 0) apr_thread_cond_timedwait(on->cond, on->lock, timeout);
n = on->shutdown;
now = apr_time_now() - now;
log_printf(2, "main loop bottom n=%d dt=" TT " sec=" TT "\n", n, now, apr_time_sec(now));
} while (n == 0);
log_printf(2, "CLEANUP\n");
for (hit = apr_hash_first(NULL, on->table); hit != NULL; hit = apr_hash_next(hit)) {
apr_hash_this(hit, (const void **)&remote_hash, &id_len, (void **)&table);
for (hi = apr_hash_first(NULL, table->table); hi != NULL; hi = apr_hash_next(hi)) {
apr_hash_this(hi, (const void **)&id, &id_len, (void **)&oh);
apr_hash_set(table->table, id, id_len, NULL);
free(oh->id);
free(oh);
}
apr_hash_set(on->table, &(table->remote_host_hash), sizeof(mq_msg_hash_t), NULL);
mq_msg_destroy(table->remote_host);
free(table);
}
log_printf(2, "EXITING\n");
apr_thread_mutex_unlock(on->lock);
return(NULL);
}
int rss_perform_check(resource_service_fn_t *rs)
{
rs_simple_priv_t *rss = (rs_simple_priv_t *)rs->priv;
apr_hash_index_t *hi;
rss_check_entry_t *ce;
int prev_status, status_change;
char *rid;
apr_ssize_t klen;
op_status_t status;
opque_t *q;
op_generic_t *gop;
log_printf(5, "START\n");
//** Generate the task list
q = new_opque();
status_change = 0;
apr_thread_mutex_lock(rss->lock);
for (hi = apr_hash_first(NULL, rss->rid_mapping); hi != NULL; hi = apr_hash_next(hi)) {
apr_hash_this(hi, (const void **)&rid, &klen, (void **)&ce);
// if (ce->re->status != RS_STATUS_IGNORE) {
gop = ds_res_inquire(rss->ds, ce->ds_key, rss->da, ce->space, rss->check_timeout);
gop_set_private(gop, ce);
opque_add(q, gop);
// }
}
apr_thread_mutex_unlock(rss->lock);
//** Wait for them all to complete
opque_waitall(q);
//** Process the results
apr_thread_mutex_lock(rss->lock);
if (rss->modify_time == rss->current_check) { //** Only process the results if not updated
for (gop = opque_get_next_finished(q); gop != NULL; gop = opque_get_next_finished(q)) {
status = gop_get_status(gop);
ce = gop_get_private(gop);
prev_status = ce->re->status;
if (status.op_status == OP_STATE_SUCCESS) { //** Got a valid response
ce->re->space_free = ds_res_inquire_get(rss->ds, DS_INQUIRE_FREE, ce->space);
ce->re->space_used = ds_res_inquire_get(rss->ds, DS_INQUIRE_USED, ce->space);
ce->re->space_total = ds_res_inquire_get(rss->ds, DS_INQUIRE_TOTAL, ce->space);
if (ce->re->status != RS_STATUS_IGNORE) {
if (ce->re->space_free <= rss->min_free) {
ce->re->status = RS_STATUS_OUT_OF_SPACE;
} else {
ce->re->status = RS_STATUS_UP;
}
}
} else { //** No response so mark it as down
if (ce->re->status != RS_STATUS_IGNORE) ce->re->status = RS_STATUS_DOWN;
}
if (prev_status != ce->re->status) status_change = 1;
log_printf(15, "ds_key=%s prev_status=%d new_status=%d\n", ce->ds_key, prev_status, ce->re->status);
gop_free(gop, OP_DESTROY);
}
}
opque_free(q, OP_DESTROY);
apr_thread_mutex_unlock(rss->lock);
log_printf(5, "END status_change=%d\n", status_change);
return(status_change);
}
int main(int argc, char **argv)
{
int i, start_option;
int mode;
char *clone_arg = NULL;
char *sfname = NULL;
char *cfname = NULL;
lio_exnode_t *ex, *cex;
lio_exnode_exchange_t *exp, *exp_out;
gop_op_generic_t *gop;
gop_op_status_t status;
FILE *fd;
if (argc < 3) {
printf("\n");
printf("ex_clone LIO_COMMON_OPTIONS [-structure|-data] [-a clone_attr] source_file.ex3 clone_file.ex3\n");
lio_print_options(stdout);
printf(" -structure - Clone the structure only [default mode]\n");
printf(" -data - Clone the structure and copy the data\n");
printf(" -a clone_attr - Segment specific attribute passed to the clone routine. Not used for all Segment types.\n");
printf(" source_file.ex3 - File to clone\n");
printf(" clone_file.ex3 - DEstination cloned file\n");
return(1);
}
lio_init(&argc, &argv);
mode = CLONE_STRUCTURE;
//*** Parse the args
i=1;
do {
start_option = i;
if (strcmp(argv[i], "-structure") == 0) { //** Clone the structure only
i++;
mode = CLONE_STRUCTURE;
} else if (strcmp(argv[i], "-data") == 0) { //** Clone the structure and the data
i++;
mode = CLONE_STRUCT_AND_DATA;
} else if (strcmp(argv[i], "-a") == 0) { //** Alternate query attribute
i++;
clone_arg = argv[i];
i++;
}
} while (start_option < i);
//** This is the source file
sfname = argv[i];
i++;
if (sfname == NULL) {
printf("Missing source file!\n");
return(2);
}
//** This is the cloned file
cfname = argv[i];
i++;
if (cfname == NULL) {
printf("Missing cloned file!\n");
return(2);
}
//** Load the source
exp = lio_exnode_exchange_load_file(sfname);
ex = lio_exnode_create();
lio_exnode_deserialize(ex, exp, lio_gc->ess);
//** Execute the clone operation
gop = lio_exnode_clone_gop(lio_gc->tpc_unlimited, ex, lio_gc->da, &cex, (void *)clone_arg, mode, lio_gc->timeout);
gop_waitany(gop);
status = gop_get_status(gop);
gop_free(gop, OP_DESTROY);
if (status.op_status != OP_STATE_SUCCESS) {
printf("Error with clone! source=%s mode=%d\n", sfname, mode);
abort();
}
//** Store the updated exnode back to disk
exp_out = lio_exnode_exchange_create(EX_TEXT);
lio_exnode_serialize(cex, exp_out);
fd = fopen(cfname, "w");
fprintf(fd, "%s", exp_out->text.text);
fclose(fd);
lio_exnode_exchange_destroy(exp_out);
//** Clean up
lio_exnode_exchange_destroy(exp);
lio_exnode_destroy(ex);
lio_exnode_destroy(cex);
lio_shutdown();
return(0);
}
int main(int argc, char **argv)
{
char *path;
int i, mode, n, nfailed;
os_fsck_iter_t *it;
char *fname;
op_generic_t *gop;
op_status_t status;
int ftype, err;
if (argc < 2) {
printf("\n");
printf("os_fsck LIO_COMMON_OPTIONS [-fix manual|delete|repair] path\n");
lio_print_options(stdout);
printf(" -fix - How to handle issues. Default is manual. Can also be delete or repair.\n");
printf(" path - Path prefix to use\n");
printf("\n");
return(1);
}
lio_init(&argc, &argv);
if (argc < 2) {
printf("Missing path!\n");
return(1);
}
mode = OS_FSCK_MANUAL;
i = 1;
if (strcmp(argv[i], "-fix") == 0) {
i++;
if (strcmp(argv[i], "delete") == 0) {
mode = OS_FSCK_REMOVE;
} else if (strcmp(argv[i], "repair") == 0) {
mode = OS_FSCK_REPAIR;
}
i++;
}
path = argv[i];
info_printf(lio_ifd, 0, "--------------------------------------------------------------------\n");
info_printf(lio_ifd, 0, "Using path=%s and mode=%d (%d=manual, %d=delete, %d=repair)\n", path, mode, OS_FSCK_MANUAL, OS_FSCK_REMOVE, OS_FSCK_REPAIR);
info_printf(lio_ifd, 0, "Possible error states: %d=missing attr, %d=missing object\n", OS_FSCK_MISSING_ATTR, OS_FSCK_MISSING_OBJECT);
info_printf(lio_ifd, 0, "--------------------------------------------------------------------\n");
tbx_info_flush(lio_ifd);
n = 0;
nfailed = 0;
it = os_create_fsck_iter(lio_gc->os, lio_gc->creds, path, OS_FSCK_MANUAL); //** WE use reolve to clean up so we can see the problem objects
while ((err = os_next_fsck(lio_gc->os, it, &fname, &ftype)) != OS_FSCK_GOOD) {
info_printf(lio_ifd, 0, "err:%d type:%d object:%s\n", err, ftype, fname);
if (err == OS_FSCK_ERROR) { //** Internal error so abort!
info_printf(lio_ifd, 0, "Internal FSCK error! Aborting!\n");
break;
}
if (mode != OS_FSCK_MANUAL) {
gop = os_fsck_object(lio_gc->os, lio_gc->creds, fname, ftype, mode);
gop_waitany(gop);
status = gop_get_status(gop);
gop_free(gop, OP_DESTROY);
if (status.error_code != OS_FSCK_GOOD) nfailed++;
info_printf(lio_ifd, 0, " resolve:%d object:%s\n", status.error_code, fname);
}
free(fname);
fname = NULL;
n++;
}
os_destroy_fsck_iter(lio_gc->os, it);
info_printf(lio_ifd, 0, "--------------------------------------------------------------------\n");
info_printf(lio_ifd, 0, "Problem objects: %d Repair Failed count: %d\n", n, nfailed);
info_printf(lio_ifd, 0, "--------------------------------------------------------------------\n");
lio_shutdown();
return(0);
}
