void write_allocs(ibp_capset_t *caps, int n, int asize, int block_size)
{
int i, j, nblocks, rem, len, err;
oplist_t *iolist;
ibp_op_t *op;
char *buffer = (char *)malloc(asize);
memset(buffer, 'W', asize);
iolist = new_ibp_oplist(NULL);
nblocks = asize / block_size;
rem = asize % block_size;
if (rem > 0) nblocks++;
//for (j=0; j<nblocks; j++) {
for (j=nblocks-1; j>= 0; j--) {
for (i=0; i<n; i++) {
if ((j==(nblocks-1)) && (rem > 0)) { len = rem; } else { len = block_size; }
op = new_ibp_write_op(get_ibp_cap(&(caps[i]), IBP_WRITECAP), j*block_size, len, buffer, ibp_timeout, NULL, cc);
add_ibp_oplist(iolist, op);
}
}
io_start(iolist);
err = io_waitall(iolist);
if (err != IBP_OK) {
printf("write_allocs: At least 1 error occured! * ibp_errno=%d * nfailed=%d\n", err, oplist_nfailed(iolist));
}
free_oplist(iolist);
free(buffer);
}
double small_random_allocs(ibp_capset_t *caps, int n, int asize, double readfrac, int small_count, int min_size, int max_size)
{
int i, io_size, offset, slot, err;
oplist_t *iolist;
ibp_op_t *op;
double rnd, lmin, lmax;
double nbytes;
iolist = new_ibp_oplist(NULL);
lmin = log(min_size); lmax = log(max_size);
if (asize < max_size) {
max_size = asize;
log_printf(0, "small_random_allocs: Adjusting max_size=%d\n", max_size);
}
char *buffer = (char *)malloc(max_size);
memset(buffer, 0, max_size);
nbytes = 0;
for (i=0; i<small_count; i++) {
rnd = rand()/(RAND_MAX+1.0);
slot = n * rnd;
rnd = rand()/(RAND_MAX+1.0);
rnd = lmin + (lmax - lmin) * rnd;
io_size = exp(rnd);
if (io_size == 0) io_size = 1;
nbytes = nbytes + io_size;
rnd = rand()/(RAND_MAX+1.0);
offset = (asize - io_size) * rnd;
// log_printf(15, "small_random_allocs: slot=%d offset=%d size=%d\n", slot, offset, io_size);
rnd = rand()/(RAND_MAX+1.0);
if (rnd < readfrac) {
op = new_ibp_read_op(get_ibp_cap(&(caps[slot]), IBP_READCAP), offset, io_size, buffer, ibp_timeout, NULL, cc);
} else {
op = new_ibp_write_op(get_ibp_cap(&(caps[slot]), IBP_WRITECAP), offset, io_size, buffer, ibp_timeout, NULL, cc);
}
add_ibp_oplist(iolist, op);
}
io_start(iolist);
err = io_waitall(iolist);
if (err != IBP_OK) {
printf("small_random_allocs: At least 1 error occured! * ibp_errno=%d * nfailed=%d\n", err, oplist_nfailed(iolist));
}
free_oplist(iolist);
free(buffer);
return(nbytes);
}
void random_allocs(ibp_capset_t *caps, int n, int asize, int block_size, double rfrac)
{
int i, err, bslot;
int j, nblocks, rem, len;
opque_t *q;
op_generic_t *op;
double rnd;
tbuffer_t *buf;
char *rbuffer = (char *)malloc(block_size);
char *wbuffer = (char *)malloc(block_size);
init_buffer(rbuffer, 'r', block_size);
init_buffer(wbuffer, 'w', block_size);
q = new_opque();
nblocks = asize / block_size;
rem = asize % block_size;
if (rem > 0) nblocks++;
type_malloc_clear(buf, tbuffer_t, n*nblocks);
for (j=0; j<nblocks; j++) {
for (i=0; i<n; i++) {
rnd = rand()/(RAND_MAX + 1.0);
if ((j==(nblocks-1)) && (rem > 0)) {
len = rem;
} else {
len = block_size;
}
bslot = j*n + i;
if (rnd < rfrac) {
tbuffer_single(&(buf[bslot]), len, rbuffer);
op = new_ibp_read_op(ic, get_ibp_cap(&(caps[i]), IBP_READCAP), j*block_size, &(buf[bslot]), 0, len, ibp_timeout);
} else {
tbuffer_single(&(buf[bslot]), len, wbuffer);
op = new_ibp_write_op(ic, get_ibp_cap(&(caps[i]), IBP_WRITECAP), j*block_size, &(buf[bslot]), 0, len, ibp_timeout);
}
opque_add(q, op);
}
}
io_start(q);
err = io_waitall(q);
if (err != 0) {
printf("random_allocs: At least 1 error occured! * ibp_errno=%d * nfailed=%d\n", err, opque_tasks_failed(q));
}
opque_free(q, OP_DESTROY);
free(buf);
free(rbuffer);
free(wbuffer);
}
double small_write_allocs(ibp_capset_t *caps, int n, int asize, int small_count, int min_size, int max_size)
{
int i, io_size, offset, slot, err;
opque_t *q;
op_generic_t *op;
double rnd, lmin, lmax;
double nbytes;
tbuffer_t *buf;
q = new_opque();
if (asize < max_size) {
max_size = asize;
log_printf(0, "small_write_allocs: Adjusting max_size=%d\n", max_size);
}
lmin = log(min_size);
lmax = log(max_size);
char *buffer = (char *)malloc(max_size);
init_buffer(buffer, 'a', max_size);
type_malloc_clear(buf, tbuffer_t, small_count);
nbytes = 0;
for (i=0; i<small_count; i++) {
rnd = rand()/(RAND_MAX+1.0);
slot = n * rnd;
rnd = rand()/(RAND_MAX+1.0);
rnd = lmin + (lmax - lmin) * rnd;
io_size = exp(rnd);
if (io_size == 0) io_size = 1;
nbytes = nbytes + io_size;
rnd = rand()/(RAND_MAX+1.0);
offset = (asize - io_size) * rnd;
tbuffer_single(&(buf[i]), io_size, buffer);
op = new_ibp_write_op(ic, get_ibp_cap(&(caps[slot]), IBP_WRITECAP), offset, &(buf[i]), 0, io_size, ibp_timeout);
opque_add(q, op);
}
io_start(q);
err = io_waitall(q);
if (err != 0) {
printf("small_write_allocs: At least 1 error occured! * ibp_errno=%d * nfailed=%d\n", err, opque_tasks_failed(q));
}
opque_free(q, OP_DESTROY);
free(buf);
free(buffer);
return(nbytes);
}
op_generic_t *ds_ibp_write(data_service_fn_t *dsf, data_attr_t *dattr, data_cap_t *wcap, ds_int_t off, tbuffer_t *dwrite, ds_int_t boff, ds_int_t size, int timeout)
{
ds_ibp_priv_t *ds = (ds_ibp_priv_t *)(dsf->priv);
ds_ibp_attr_t *attr = (ds_ibp_attr_t *)dattr;
ds_ibp_op_t *iop = ds_ibp_op_create(ds, attr);
//** Create the op
iop->gop = new_ibp_write_op(ds->ic, wcap, off, dwrite, boff, size, timeout);
ds_ibp_setup_finish(iop);
return(iop->gop);
}
void random_allocs(ibp_capset_t *caps, int n, int asize, int block_size, double rfrac)
{
int i, slot, err;
int j, nblocks, rem, len;
oplist_t *iolist;
ibp_op_t *op;
double rnd;
char *buffer = (char *)malloc(asize);
memset(buffer, 'R', asize);
iolist = new_ibp_oplist(NULL);
nblocks = asize / block_size;
rem = asize % block_size;
if (rem > 0) nblocks++;
for (j=0; j<nblocks; j++) {
for (i=0; i<n; i++) {
rnd = rand()/(RAND_MAX+1.0);
slot = n * rnd;
rnd = rand()/(RAND_MAX + 1.0);
if ((j==(nblocks-1)) && (rem > 0)) { len = rem; } else { len = block_size; }
// op = new_ibp_read_op(get_ibp_cap(&(caps[i]), IBP_READCAP), j*block_size, len, buffer, ibp_timeout, NULL, cc);
if (rnd < rfrac) {
op = new_ibp_read_op(get_ibp_cap(&(caps[i]), IBP_READCAP), j*block_size, len, buffer, ibp_timeout, NULL, cc);
} else {
op = new_ibp_write_op(get_ibp_cap(&(caps[i]), IBP_WRITECAP), j*block_size, len, buffer, ibp_timeout, NULL, cc);
}
add_ibp_oplist(iolist, op);
}
}
io_start(iolist);
err = io_waitall(iolist);
if (err != IBP_OK) {
printf("random_allocs: At least 1 error occured! * ibp_errno=%d * nfailed=%d\n", err, oplist_nfailed(iolist));
}
free_oplist(iolist);
free(buffer);
}
void write_allocs(ibp_capset_t *caps, int n, int asize, int block_size)
{
int i, j, nblocks, rem, len, err, slot;
opque_t *q;
op_generic_t *op;
tbuffer_t *buf;
char *buffer = (char *)malloc(block_size);
init_buffer(buffer, 'W', block_size);
q = new_opque();
nblocks = asize / block_size;
rem = asize % block_size;
if (rem > 0) nblocks++;
type_malloc_clear(buf, tbuffer_t, n*nblocks);
//for (j=0; j<nblocks; j++) {
for (j=nblocks-1; j>= 0; j--) {
for (i=0; i<n; i++) {
if ((j==(nblocks-1)) && (rem > 0)) {
len = rem;
} else {
len = block_size;
}
slot = j*n + i;
tbuffer_single(&(buf[slot]), len, buffer);
op = new_ibp_write_op(ic, get_ibp_cap(&(caps[i]), IBP_WRITECAP), j*block_size, &(buf[slot]), 0, len, ibp_timeout);
opque_add(q, op);
}
}
io_start(q);
err = io_waitall(q);
if (err != 0) {
printf("write_allocs: At least 1 error occured! * ibp_errno=%d * nfailed=%d\n", err, opque_tasks_failed(q));
}
opque_free(q, OP_DESTROY);
free(buf);
free(buffer);
}
void base_async_test(int nthreads, ibp_depot_t *depot)
{
// int size = 115;
// int block = 10;
int size = 10*1024*1024;
int block = 5000;
// char buffer[size+1];
// char buffer_cmp[size+1];
char *buffer, *buffer_cmp;
char c;
char block_buf[block+1];
ibp_attributes_t attr;
ibp_capset_t caps;
ibp_cap_t *cap;
int err, i, offset, bcount, remainder;
ibp_op_t *op;
oplist_t *iol;
buffer = (char *)malloc(size+1);
buffer_cmp = (char *)malloc(size+1);
assert(buffer != NULL);
assert(buffer_cmp != NULL);
printf("base_async_test: Starting simple test\n"); fflush(stdout);
//** Create the list for handling the commands
iol = new_ibp_oplist(NULL);
oplist_start_execution(iol); //** and start executing the commands
//** Create the allocation used for test
set_ibp_attributes(&attr, time(NULL) + A_DURATION, IBP_HARD, IBP_BYTEARRAY);
op = new_ibp_alloc_op(&caps, size, depot, &attr, ibp_timeout, NULL, NULL);
add_ibp_oplist(iol, op);
err = oplist_waitall(iol);
if (err != IBP_OK) {
printf("base_async_test: ibp_allocate error! * ibp_errno=%d\n", err);
abort();
}
printf("base_async_test: rcap=%s\n", get_ibp_cap(&caps, IBP_READCAP));
printf("base_async_test: wcap=%s\n", get_ibp_cap(&caps, IBP_WRITECAP));
printf("base_async_test: mcap=%s\n", get_ibp_cap(&caps, IBP_MANAGECAP));
//** Init the buffers
buffer[size] = '\0'; memset(buffer, '*', size);
buffer_cmp[size] = '\0'; memset(buffer_cmp, '_', size);
block_buf[block] = '\0'; memset(block_buf, '0', block);
//-------------------------------
//** Do the initial upload
// op = new_ibp_write_op(get_ibp_cap(&caps, IBP_WRITECAP), 0, size, buffer_cmp, ibp_timeout, NULL, NULL);
// add_ibp_oplist(iol, op);
// err = oplist_waitall(iol);
// if (err != IBP_OK) {
// printf("base_async_test: Initial ibp_write error! * ibp_errno=%d\n", err);
// abort();
// }
bcount = size / (2*block);
remainder = size - bcount * (2*block);
//** Now do the striping **
offset = (bcount-1)*(2*block); // Now store the data in chunks
cap = get_ibp_cap(&caps, IBP_WRITECAP);
for (i=0; i<bcount; i++) {
c = 'A' + (i%27);
memset(&(buffer_cmp[offset]), c, 2*block);
op = new_ibp_write_op(cap, offset, 2*block, &(buffer_cmp[offset]), ibp_timeout, NULL, NULL);
add_ibp_oplist(iol, op);
offset = offset - 2*block;
}
if (remainder>0) {
offset = bcount*2*block;
memset(&(buffer_cmp[offset]), '@', remainder);
op = new_ibp_write_op(cap, offset, remainder, &(buffer_cmp[offset]), ibp_timeout, NULL, NULL);
add_ibp_oplist(iol, op);
}
//** Now wait for them to complete
err = oplist_waitall(iol);
if (err != IBP_OK) {
printf("base_async_test: Error in stripe write! * ibp_errno=%d\n", err);
abort();
}
//-------------------------------
bcount = size / block;
remainder = size % block;
//** Generate the Read list
offset = 0; // Now read the data in chunks
cap = get_ibp_cap(&caps, IBP_READCAP);
for (i=0; i<bcount; i++) {
op = new_ibp_read_op(cap, offset, block, &(buffer[offset]), ibp_timeout, NULL, NULL);
add_ibp_oplist(iol, op);
offset = offset + block;
}
if (remainder>0) {
//printf("read remainder: rem=%d offset=%d\n", remainder, offset);
op = new_ibp_read_op(cap, offset, remainder, &(buffer[offset]), ibp_timeout, NULL, NULL);
add_ibp_oplist(iol, op);
}
//** Now wait for them to complete
err = oplist_waitall(iol);
if (err != IBP_OK) {
printf("base_async_test: Error in stripe read! * ibp_errno=%d\n", err);
abort();
}
//-------------------------------
//** Do the comparison **
i = strcmp(buffer, buffer_cmp);
if (i == 0) {
printf("base_async_test: Success!\n");
} else {
failed_tests++;
printf("base_async_test: Failed! strcmp = %d\n", i);
}
// printf("base_async_test: buffer_cmp=%s\n", buffer_cmp);
// printf("base_async_test: buffer=%s\n", buffer);
// printf("base_async_test:block_buffer=%s\n", block_buf);
//-------------------------------
//** Remove the allocation **
op = new_ibp_remove_op(get_ibp_cap(&caps, IBP_MANAGECAP), ibp_timeout, NULL, NULL);
add_ibp_oplist(iol, op);
err = oplist_waitall(iol);
if (err != IBP_OK) {
printf("base_async_test: Error removing the allocation! ibp_errno=%d\n", err);
abort();
}
// free_oplist(iol);
oplist_finished_submission(iol, OPLIST_AUTO_FREE);
free(buffer);
free(buffer_cmp);
}
double write_allocs(ibp_capset_t *caps, int qlen, int n, int asize, int block_size)
{
int count, i, j, nleft, nblocks, rem, len, err, block_start, alloc_start;
int *slot;
op_status_t status;
int64_t nbytes, last_bytes, print_bytes, delta_bytes;
apr_int32_t nfds, finished;
double dbytes, r1, r2;
apr_pool_t *pool;
apr_file_t *fd_in;
opque_t *q;
op_generic_t *op;
char *buffer = (char *)malloc(block_size);
apr_interval_time_t dt = 10;
apr_pollfd_t pfd;
apr_time_t stime, dtime;
int *tbuf_index;
tbuffer_t *buf;
Stack_t *tbuf_free;
tbuf_free = new_stack();
type_malloc_clear(tbuf_index, int, qlen);
type_malloc_clear(buf, tbuffer_t, qlen);
for (i=0; i<qlen; i++) {
tbuf_index[i] = i;
push(tbuf_free, &(tbuf_index[i]));
}
//** Make the stuff to capture the kbd
apr_pool_create(&pool, NULL);
nfds = 1;
apr_file_open_stdin(&fd_in, pool);
pfd.p = pool;
pfd.desc_type = APR_POLL_FILE;
pfd.reqevents = APR_POLLIN|APR_POLLHUP;
pfd.desc.f = fd_in;
pfd.client_data = NULL;
//** Init the ibp stuff
init_buffer(buffer, 'W', block_size);
q = new_opque();
opque_start_execution(q);
nblocks = asize / block_size;
rem = asize % block_size;
if (rem > 0) nblocks++;
block_start = 0;
alloc_start = 0;
finished = 0;
apr_poll(&pfd, nfds, &finished, dt);
count = 0;
nbytes=0;
last_bytes = 0;
delta_bytes = 1024 * 1024 * 1024;
print_bytes = delta_bytes;
stime = apr_time_now();
while (finished == 0) {
// nleft = qlen - opque_tasks_left(q);
nleft = stack_size(tbuf_free);
// printf("\nLOOP: nleft=%d qlen=%d\n", nleft, qlen);
if (nleft > 0) {
for (j=block_start; j < nblocks; j++) {
for (i=alloc_start; i<n; i++) {
nleft--;
if (nleft <= 0) {
block_start = j;
alloc_start = i;
goto skip_submit;
}
slot = (int *)pop(tbuf_free);
if ((j==(nblocks-1)) && (rem > 0)) {
len = rem;
} else {
len = block_size;
}
// printf("%d=(%d,%d) ", count, j, i);
tbuffer_single(&(buf[*slot]), len, buffer);
op = new_ibp_write_op(ic, get_ibp_cap(&(caps[i]), IBP_WRITECAP), j*block_size, &(buf[*slot]), 0, len, ibp_timeout);
gop_set_id(op, *slot);
ibp_op_set_cc(ibp_get_iop(op), cc);
ibp_op_set_ncs(ibp_get_iop(op), ncs);
opque_add(q, op);
}
alloc_start = 0;
}
block_start = 0;
}
skip_submit:
finished = 1;
apr_poll(&pfd, nfds, &finished, dt);
do { //** Empty the finished queue. Always wait for for at least 1 to complete
op = opque_waitany(q);
status = gop_get_status(op);
if (status.error_code != IBP_OK) {
printf("ERROR: Aborting with error code %d\n", status.error_code);
finished = 0;
}
count++;
i = gop_get_id(op);
nbytes = nbytes + tbuffer_size(&(buf[i]));
if (nbytes > print_bytes) {
dbytes = nbytes / (1.0*1024*1024*1024);
dtime = apr_time_now() - stime;
r2 = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = nbytes - last_bytes;
r1 = r1 / (r2 * 1024.0 * 1024.0);
printf("%.2lfGB written (%.2lfMB/s : %.2lf secs)\n", dbytes, r1, r2);
print_bytes = print_bytes + delta_bytes;
last_bytes = nbytes;
stime = apr_time_now();
}
push(tbuf_free, &(tbuf_index[i]));
gop_free(op, OP_DESTROY);
} while (opque_tasks_finished(q) > 0);
}
err = opque_waitall(q);
if (err != OP_STATE_SUCCESS) {
printf("write_allocs: At least 1 error occured! * ibp_errno=%d * nfailed=%d\n", err, opque_tasks_failed(q));
}
opque_free(q, OP_DESTROY);
free_stack(tbuf_free, 0);
free(tbuf_index);
free(buf);
free(buffer);
apr_pool_destroy(pool);
dbytes = nbytes;
return(dbytes);
}
void simple_test()
{
int size = 1024*1024;
int block = 5000;
char buffer[size+1];
char buffer_cmp[size+1];
char block_buf[block+1];
ibp_attributes_t attr;
ibp_depot_t *depot;
ibp_capset_t caps;
ibp_cap_t *cap;
int err, i, offset, bcount, remainder;
ibp_op_t *op;
oplist_t *iol;
//** Make the allocation ***
depot = &(depot_list[0]);
//** Create the list for handling the commands
iol = new_ibp_oplist(NULL);
oplist_start_execution(iol); //** and start executing the commands
//** Create the allocation used for test
set_ibp_attributes(&attr, time(NULL) + a_duration, IBP_HARD, IBP_BYTEARRAY);
op = new_ibp_alloc_op(&caps, size, depot, &attr, ibp_timeout, NULL, NULL);
add_ibp_oplist(iol, op);
err = oplist_waitall(iol);
if (err != IBP_OK) {
printf("simple_test: ibp_allocate error! * ibp_errno=%d\n", err);
abort();
}
printf("simple_test: rcap=%s\n", get_ibp_cap(&caps, IBP_READCAP));
printf("simple_test: wcap=%s\n", get_ibp_cap(&caps, IBP_WRITECAP));
printf("simple_test: mcap=%s\n", get_ibp_cap(&caps, IBP_MANAGECAP));
//** Init the buffers
buffer[size] = '\0'; memset(buffer, '*', size);
buffer_cmp[size] = '\0'; memset(buffer_cmp, '_', size);
block_buf[block] = '\0'; memset(block_buf, '0', block);
//-------------------------------
//** Do the initial upload
op = new_ibp_write_op(get_ibp_cap(&caps, IBP_WRITECAP), 0, size, buffer_cmp, ibp_timeout, NULL, NULL);
add_ibp_oplist(iol, op);
err = oplist_waitall(iol);
if (err != IBP_OK) {
printf("simple_test: Initial ibp_write error! * ibp_errno=%d\n", err);
abort();
}
bcount = size / (2*block);
remainder = size - bcount * (2*block);
//** Now do the striping **
offset = 0; // Now store the data in chunks
cap = get_ibp_cap(&caps, IBP_WRITECAP);
for (i=0; i<bcount; i++) {
op = new_ibp_write_op(cap, offset, block, block_buf, ibp_timeout, NULL, NULL);
add_ibp_oplist(iol, op);
memset(&(buffer_cmp[offset]), '0', block);
offset = offset + 2*block;
}
if (remainder>0) {
if (remainder > block) remainder = block;
op = new_ibp_write_op(cap, offset, remainder, block_buf, ibp_timeout, NULL, NULL);
add_ibp_oplist(iol, op);
memset(&(buffer_cmp[offset]), '0', remainder);
}
//** Now wait for them to complete
err = oplist_waitall(iol);
if (err != IBP_OK) {
printf("simple_test: Error in stripe write! * ibp_errno=%d\n", err);
abort();
}
//-------------------------------
bcount = size / block;
remainder = size - bcount * block;
//** Generate the Read list
offset = 0; // Now store the data in chunks
cap = get_ibp_cap(&caps, IBP_READCAP);
for (i=0; i<bcount; i++) {
op = new_ibp_read_op(cap, offset, block, &(buffer[offset]), ibp_timeout, NULL, NULL);
add_ibp_oplist(iol, op);
offset = offset + block;
}
if (remainder>0) {
op = new_ibp_read_op(cap, offset, remainder, &(buffer[offset]), ibp_timeout, NULL, NULL);
add_ibp_oplist(iol, op);
}
//** Now wait for them to complete
err = oplist_waitall(iol);
if (err != IBP_OK) {
printf("simple_test: Error in stripe read! * ibp_errno=%d\n", err);
abort();
}
//-------------------------------
//** Do the comparison **
i = strcmp(buffer, buffer_cmp);
if (i == 0) {
printf("simple_test: Success!\n");
} else {
printf("simple_test: Failed! strcmp = %d\n", i);
}
// printf("simple_test: buffer_cmp=%s\n", buffer_cmp);
// printf("simple_test: buffer=%s\n", buffer);
//-------------------------------
//** Remove the allocation **
op = new_ibp_remove_op(get_ibp_cap(&caps, IBP_MANAGECAP), ibp_timeout, NULL, NULL);
add_ibp_oplist(iol, op);
err = oplist_waitall(iol);
if (err != IBP_OK) {
printf("simple_test: Error removing the allocation! ibp_errno=%d\n", err);
abort();
}
free_oplist(iol);
}