int main(int argc, char **argv)
{
double r1, r2;
int i, automode, doskip, start_option, tcpsize, cs_type;
int q_len, n_allocs, alloc_size, block_size, llevel;
ibp_capset_t *caps_list;
rid_t rid;
int port;
char buffer[1024];
apr_time_t stime, dtime;
double dt;
char *net_cs_name, *disk_cs_name;
phoebus_t pcc;
char pstr[2048];
chksum_t cs;
ns_chksum_t ns_cs;
int blocksize = 0;
if (argc < 4) {
printf("\n");
printf("ibp_find_drive [-d log_level] [-network_chksum type blocksize] [-disk_chksum type blocksize]\n");
printf(" [-config ibp.cfg] [-duration duration] [-tcpsize tcpbufsize]\n");
printf(" [-skip] [-auto]\n");
printf(" depot1 port1 resource_id1\n");
printf(" nthreads ibp_timeout\n");
printf(" q_len n_alloc alloc_size block_size\n");
printf("\n");
printf("-d loglevel - Enable debug output (5,6 provide minimal output an 20 provides full output\n");
printf("-network_chksum type blocksize - Enable network checksumming for transfers.\n");
printf(" type should be SHA256, SHA512, SHA1, or MD5.\n");
printf(" blocksize determines how many bytes to send between checksums in kbytes.\n");
printf("-disk_chksum type blocksize - Enable Disk checksumming.\n");
printf(" type should be NONE, SHA256, SHA512, SHA1, or MD5.\n");
printf(" blocksize determines how many bytes to send between checksums in kbytes.\n");
printf("-config ibp.cfg - Use the IBP configuration defined in file ibp.cfg.\n");
printf(" nthreads overrides value in cfg file unless -1.\n");
printf("-tcpsize tcpbufsize - Use this value, in KB, for the TCP send/recv buffer sizes\n");
printf("-duration duration - Allocation duration in sec. Needs to be big enough to last the entire\n");
printf("-skip - Hit all drives on the depot *except* the resource_id given\n");
printf("-auto - Automatically configure nthreads, ibp_timeout, q_len, n_alloc, alloc_size, and block_Size\n");
printf("depot - Depot hostname\n");
printf("port - IBP port on depot\n");
printf("resource_id - Resource ID to use on depot\n");
printf("nthreads - Max Number of simultaneous threads to use. Use -1 for defaults or value in ibp.cfg\n");
printf("ibp_timeout - Timeout(sec) for each IBP copmmand\n");
printf("q_len - Number of commands queued at any thime. Controls how long it takes to stop\n");
printf("n_alloc - Number of allocations to use\n");
printf("alloc_size - Size of each allocation in KB for sequential and random tests\n");
printf("block_size - Size of each write operation in KB for sequential and random tests\n");
printf("\n");
return(-1);
}
ic = ibp_create_context(); //** Initialize IBP
i = 1;
net_cs_name = NULL;
disk_cs_name = NULL;
doskip = 0;
sync_transfer = 0;
use_alias = 0;
print_progress = 0;
automode = 0;
ibp_timeout = 10;
do {
start_option = i;
if (strcmp(argv[i], "-d") == 0) { //** Enable debugging
i++;
llevel = atoi(argv[i]);
set_log_level(llevel);
i++;
} else if (strcmp(argv[i], "-network_chksum") == 0) { //** Add checksum capability
i++;
net_cs_name = argv[i];
cs_type = chksum_name_type(argv[i]);
if (cs_type == -1) {
printf("Invalid chksum type. Got %s should be SHA1, SHA256, SHA512, or MD5\n", argv[i]);
abort();
}
chksum_set(&cs, cs_type);
i++;
blocksize = atoi(argv[i])*1024;
i++;
ns_chksum_set(&ns_cs, &cs, blocksize);
ncs = &ns_cs;
ibp_set_chksum(ic, ncs);
} else if (strcmp(argv[i], "-disk_chksum") == 0) { //** Add checksum capability
i++;
disk_cs_name = argv[i];
disk_cs_type = chksum_name_type(argv[i]);
if (disk_cs_type < CHKSUM_DEFAULT) {
printf("Invalid chksum type. Got %s should be NONE, SHA1, SHA256, SHA512, or MD5\n", argv[i]);
abort();
}
i++;
disk_blocksize = atoi(argv[i])*1024;
i++;
} else if (strcmp(argv[i], "-config") == 0) { //** Read the config file
i++;
ibp_load_config_file(ic, argv[i], NULL);
i++;
} else if (strcmp(argv[i], "-tcpsize") == 0) { //** Change the tcp buffer size
i++;
tcpsize = atoi(argv[i]) * 1024;
ibp_set_tcpsize(ic, tcpsize);
i++;
} else if (strcmp(argv[i], "-duration") == 0) { //** Adjust the duration for a longer run
i++;
a_duration = atoi(argv[i]);
i++;
} else if (strcmp(argv[i], "-skip") == 0) { //** Check if we want skip the RID
i++;
doskip = 1;
} else if (strcmp(argv[i], "-auto") == 0) { //** Check if we want automatically generate the load
i++;
automode = 1;
}
} while (start_option < i);
n_depots = 1;
depot_list = (ibp_depot_t *)malloc(sizeof(ibp_depot_t)*n_depots);
int j;
for (j=0; j<n_depots; j++) {
port = atoi(argv[i+1]);
rid = ibp_str2rid(argv[i+2]);
set_ibp_depot(&(depot_list[j]), argv[i], port, rid);
i = i + 3;
}
//** Tweak the list if needed **
if ((doskip == 1) || (ibp_rid_is_empty(depot_list[0].rid) == 1)) {
depot_list = generate_depot_list(&n_depots, depot_list);
//** The depot thread count is fixed when the host portal is created so to change
//** it I have to restart IBP.
// ibp_context_t *ic2 = ic;
// ic = ibp_create_context();
// ibp_destroy_context(ic2);
}
if (automode == 1) {
nthreads = 2 * n_depots;
ibp_set_max_depot_threads(ic, nthreads);
// ibp_set_min_depot_threads(ic, nthreads);
q_len = (n_depots == 1) ? 10 : 10*n_depots;
n_allocs = (n_depots == 1) ? 10 : 10*n_depots;
alloc_size = 81920*1024;
block_size = 4096*1024;
} else {
//*** Get thread count ***
nthreads = atoi(argv[i]);
if (nthreads <= 0) {
nthreads = ibp_get_max_depot_threads(ic);
} else {
ibp_set_max_depot_threads(ic, nthreads);
}
i++;
ibp_timeout = atoi(argv[i]);
i++;
q_len = atol(argv[i]);
i++;
n_allocs = atol(argv[i]);
i++;
alloc_size = atol(argv[i])*1024;
i++;
block_size = atol(argv[i])*1024;
i++;
}
//*** Print the ibp client version ***
printf("\n");
printf("================== IBP Client Version =================\n");
printf("%s\n", ibp_version());
//*** Print summary of options ***
printf("\n");
printf("======= Base options =======\n");
printf("n_depots: %d\n", n_depots);
for (i=0; i<n_depots; i++) {
printf("depot %d: %s:%d rid:%s\n", i, depot_list[i].host, depot_list[i].port, ibp_rid2str(depot_list[i].rid, buffer));
}
printf("\n");
printf("IBP timeout: %d\n", ibp_timeout);
printf("IBP duration: %d\n", a_duration);
printf("Max Threads: %d\n", nthreads);
if (cc != NULL) {
switch (cc->type) {
case NS_TYPE_SOCK:
printf("Connection Type: SOCKET\n");
break;
case NS_TYPE_PHOEBUS:
phoebus_path_to_string(pstr, sizeof(pstr), &pcc);
printf("Connection Type: PHOEBUS (%s)\n", pstr);
break;
case NS_TYPE_1_SSL:
printf("Connection Type: Single SSL\n");
break;
case NS_TYPE_2_SSL:
printf("Connection Type: Dual SSL\n");
break;
}
} else {
printf("Connection Type: SOCKET\n");
}
if (net_cs_name == NULL) {
printf("Network Checksum Type: NONE\n");
} else {
printf("Network Checksum Type: %s Block size: %dkb\n", net_cs_name, (blocksize/1024));
}
if (disk_cs_name == NULL) {
printf("Disk Checksum Type: NONE\n");
} else {
printf("Disk Checksum Type: %s Block size: " I64T "kb\n", disk_cs_name, (disk_blocksize/1024));
if (do_validate == 1) {
printf("Disk Validation: Enabled\n");
} else {
printf("Disk Validation: Disabled\n");
}
}
printf("TCP buffer size: %dkb (0 defaults to OS)\n", ibp_get_tcpsize(ic)/1024);
printf("\n");
printf("======= Bulk transfer options =======\n");
printf("Number of allocations: %d\n", n_allocs);
printf("Allocation size: %d KB\n", alloc_size/1024);
r1 = n_allocs;
r1 = r1 * alloc_size / (1024.0*1024.0);
printf("Total space allocated: %.2lf MB\n", r1);
printf("Transfer block_size: %d KB\n", block_size/1024);
printf("q_len: %d\n", q_len);
printf("\n");
io_set_mode(sync_transfer, print_progress, nthreads);
printf("Creating allocations....");
fflush(stdout);
stime = apr_time_now();
caps_list = create_allocs(n_allocs, alloc_size);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*n_allocs/dt;
printf(" %lf creates/sec (%.2lf sec total) \n", r1, dt);
printf("Starting depot writing. Press ENTER to exit.\n");
stime = apr_time_now();
double dbytes = write_allocs(caps_list, q_len, n_allocs, alloc_size, block_size);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = dbytes/(dt*1024*1024);
r2 = dbytes / (1.0*1024*1024);
printf("Average for entire run: %lf MB/sec (%.2lfMB written in %.2lf sec) \n", r1, r2, dt);
printf("Removing allocations....");
fflush(stdout);
stime = apr_time_now();
remove_allocs(caps_list, n_allocs);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*n_allocs/dt;
printf(" %lf removes/sec (%.2lf sec total) \n", r1, dt);
printf("\n");
// printf("min_threads=%d max_threads=%d\n", ibp_get_min_depot_threads(), ibp_get_max_depot_threads());
printf("Final network connection counter: %d\n", network_counter(NULL));
ibp_destroy_context(ic); //** Shutdown IBP
return(0);
}
int main(int argc, char **argv)
{
double r1, r2, r3;
int i, start_option, tcpsize, cs_type;
ibp_capset_t *caps_list, *base_caps;
rid_t rid;
int port, fd_special;
char buffer[1024];
apr_time_t stime, dtime;
double dt;
char *ppath, *net_cs_name, *disk_cs_name, *out_fname;
FILE *fd_out;
phoebus_t pcc;
char pstr[2048];
chksum_t cs;
ns_chksum_t ns_cs;
int blocksize;
base_caps = NULL;
if (argc < 12) {
printf("\n");
printf("ibp_perf [-d|-dd] [-network_chksum type blocksize] [-disk_chksum type blocksize]\n");
printf(" [-validate] [-config ibp.cfg] [-phoebus gateway_list] [-tcpsize tcpbufsize]\n");
printf(" [-duration duration] [-sync] [-alias] [-progress] [-random]\n");
printf(" n_depots depot1 port1 resource_id1 ... depotN portN ridN\n");
printf(" nthreads ibp_timeout\n");
printf(" alias_createremove_count createremove_count\n");
printf(" readwrite_count readwrite_alloc_size rw_block_size read_mix_fraction\n");
printf(" smallio_count small_min_size small_max_size small_read_fraction\n");
printf("\n");
printf("-d - Enable *minimal* debug output\n");
printf("-dd - Enable *FULL* debug output\n");
printf("-network_chksum type blocksize - Enable network checksumming for transfers.\n");
printf(" type should be SHA256, SHA512, SHA1, or MD5.\n");
printf(" blocksize determines how many bytes to send between checksums in kbytes.\n");
printf("-disk_chksum type blocksize - Enable Disk checksumming.\n");
printf(" type should be NONE, SHA256, SHA512, SHA1, or MD5.\n");
printf(" blocksize determines how many bytes to send between checksums in kbytes.\n");
printf("-validate - Validate disk chksum data. Option is ignored unless disk chksumming is enabled.\n");
printf("-config ibp.cfg - Use the IBP configuration defined in file ibp.cfg.\n");
printf(" nthreads overrides value in cfg file unless -1.\n");
printf("-phoebus - Use Phoebus protocol for data transfers.\n");
printf(" gateway_list - Comma separated List of phoebus hosts/ports, eg gateway1/1234,gateway2/4321\n");
printf("-tcpsize tcpbufsize - Use this value, in KB, for the TCP send/recv buffer sizes\n");
printf("-duration duration - Allocation duration in sec. Needs to be big enough to last the entire\n");
printf(" run. The default duration is %d sec.\n", a_duration);
printf("-save fname - Don't delete the R/W allocations on completion. Instead save them to the given filename.\n");
printf(" Can use 'stdout' and 'stderr' as the filename to redirect output\n");
printf("-sync - Use synchronous protocol. Default uses async.\n");
printf("-alias - Use alias allocations for all I/O operations\n");
printf("-progress - Print completion progress.\n");
printf("-random - Initializes the transfer buffers with quasi-random data.\n");
printf(" Disabled if network chksums are enabled.\n");
printf("n_depots - Number of depot tuplets\n");
printf("depot - Depot hostname\n");
printf("port - IBP port on depot\n");
printf("resource_id - Resource ID to use on depot\n");
printf("nthreads - Max Number of simultaneous threads to use. Use -1 for defaults or value in ibp.cfg\n");
printf("ibp_timeout - Timeout(sec) for each IBP copmmand\n");
printf("alias_createremove_count* - Number of 0 byte files to create and remove using alias allocations\n");
printf("createremove_count* - Number of 0 byte files to create and remove to test metadata performance\n");
printf("readwrite_count* - Number of files to write sequentially then read sequentially\n");
printf("readwrite_alloc_size - Size of each allocation in KB for sequential and random tests\n");
printf("rw_block_size - Size of each R/W operation in KB for sequential and random tests\n");
printf("read_mix_fraction - Fraction of Random I/O operations that are READS\n");
printf("smallio_count* - Number of random small I/O operations\n");
printf("small_min_size - Minimum size of each small I/O operation(kb)\n");
printf("small_max_size - Max size of each small I/O operation(kb)\n");
printf("small_read_fraction - Fraction of small random I/O operations that are READS\n");
printf("\n");
printf("*If the variable is set to 0 then the test is skipped\n");
printf("\n");
return(-1);
}
set_log_level(-1);
ic = ibp_create_context(); //** Initialize IBP
i = 1;
net_cs_name = NULL;
disk_cs_name = NULL;
sync_transfer = 0;
use_alias = 0;
print_progress = 0;
fd_special = 0;
fd_out = NULL;
out_fname = NULL;
do {
start_option = i;
if (strcmp(argv[i], "-d") == 0) { //** Enable debugging
set_log_level(5);
i++;
} else if (strcmp(argv[i], "-dd") == 0) { //** Enable debugging
set_log_level(20);
i++;
} else if (strcmp(argv[i], "-random") == 0) { //** Random buffers
i++;
identical_buffers = 0;
} else if (strcmp(argv[i], "-network_chksum") == 0) { //** Add checksum capability
i++;
net_cs_name = argv[i];
cs_type = chksum_name_type(argv[i]);
if (cs_type == -1) {
printf("Invalid chksum type. Got %s should be SHA1, SHA256, SHA512, or MD5\n", argv[i]);
abort();
}
chksum_set(&cs, cs_type);
i++;
blocksize = atoi(argv[i])*1024;
i++;
ns_chksum_set(&ns_cs, &cs, blocksize);
ncs = &ns_cs;
ibp_set_chksum(ic, ncs);
} else if (strcmp(argv[i], "-disk_chksum") == 0) { //** Add checksum capability
i++;
disk_cs_name = argv[i];
disk_cs_type = chksum_name_type(argv[i]);
if (disk_cs_type < CHKSUM_DEFAULT) {
printf("Invalid chksum type. Got %s should be NONE, SHA1, SHA256, SHA512, or MD5\n", argv[i]);
abort();
}
i++;
disk_blocksize = atoi(argv[i])*1024;
i++;
} else if (strcmp(argv[i], "-validate") == 0) { //** Enable validation
i++;
do_validate=1;
} else if (strcmp(argv[i], "-config") == 0) { //** Read the config file
i++;
ibp_load_config_file(ic, argv[i], NULL);
i++;
} else if (strcmp(argv[i], "-save") == 0) { //** Save the allocations and don't delete them
i++;
out_fname = argv[i];
i++;
if (strcmp(out_fname, "stderr") == 0) {
fd_out = stderr;
fd_special = 1;
} else if (strcmp(out_fname, "stdout") == 0) {
fd_out = stdout;
fd_special = 1;
} else {
{
fd_out = fopen(out_fname, "w");
assert(fd_out != NULL);
}
}
} else if (strcmp(argv[i], "-phoebus") == 0) { //** Check if we want Phoebus transfers
cc = (ibp_connect_context_t *)malloc(sizeof(ibp_connect_context_t));
cc->type = NS_TYPE_PHOEBUS;
i++;
ppath = argv[i];
phoebus_path_set(&pcc, ppath);
// printf("ppath=%s\n", ppath);
cc->data = &pcc;
ibp_set_read_cc(ic, cc);
ibp_set_write_cc(ic, cc);
i++;
} else if (strcmp(argv[i], "-tcpsize") == 0) { //** Check if we want sync tests
i++;
tcpsize = atoi(argv[i]) * 1024;
ibp_set_tcpsize(ic, tcpsize);
i++;
} else if (strcmp(argv[i], "-duration") == 0) { //** Check if we want sync tests
i++;
a_duration = atoi(argv[i]);
i++;
} else if (strcmp(argv[i], "-sync") == 0) { //** Check if we want sync tests
sync_transfer = 1;
i++;
} else if (strcmp(argv[i], "-alias") == 0) { //** Check if we want to use alias allocation
use_alias = 1;
i++;
} else if (strcmp(argv[i], "-progress") == 0) { //** Check if we want to print the progress
print_progress = 1;
i++;
}
} while (start_option < i);
if (net_cs_name != NULL) identical_buffers = 1;
n_depots = atoi(argv[i]);
i++;
depot_list = (ibp_depot_t *)malloc(sizeof(ibp_depot_t)*n_depots);
int j;
for (j=0; j<n_depots; j++) {
port = atoi(argv[i+1]);
rid = ibp_str2rid(argv[i+2]);
set_ibp_depot(&(depot_list[j]), argv[i], port, rid);
i = i + 3;
}
//*** Get thread count ***
nthreads = atoi(argv[i]);
if (nthreads <= 0) {
nthreads = ibp_get_max_depot_threads(ic);
} else {
ibp_set_max_depot_threads(ic, nthreads);
}
i++;
ibp_timeout = atoi(argv[i]);
i++;
//****** Get the different Stream counts *****
int aliascreateremove_count = atol(argv[i]);
i++;
int createremove_count = atol(argv[i]);
i++;
int readwrite_count = atol(argv[i]);
i++;
int readwrite_size = atol(argv[i])*1024;
i++;
int rw_block_size = atol(argv[i])*1024;
i++;
double read_mix_fraction = atof(argv[i]);
i++;
//****** Get the different small I/O counts *****
int smallio_count = atol(argv[i]);
i++;
int small_min_size = atol(argv[i])*1024;
i++;
int small_max_size = atol(argv[i])*1024;
i++;
double small_read_fraction = atof(argv[i]);
i++;
//*** Print the ibp client version ***
printf("\n");
printf("================== IBP Client Version =================\n");
printf("%s\n", ibp_version());
//*** Print summary of options ***
printf("\n");
printf("======= Base options =======\n");
printf("n_depots: %d\n", n_depots);
for (i=0; i<n_depots; i++) {
printf("depot %d: %s:%d rid:%s\n", i, depot_list[i].host, depot_list[i].port, ibp_rid2str(depot_list[i].rid, buffer));
}
printf("\n");
printf("IBP timeout: %d\n", ibp_timeout);
printf("IBP duration: %d\n", a_duration);
printf("Max Threads: %d\n", nthreads);
if (sync_transfer == 1) {
printf("Transfer_mode: SYNC\n");
} else {
printf("Transfer_mode: ASYNC\n");
}
printf("Use alias: %d\n", use_alias);
if (cc != NULL) {
switch (cc->type) {
case NS_TYPE_SOCK:
printf("Connection Type: SOCKET\n");
break;
case NS_TYPE_PHOEBUS:
phoebus_path_to_string(pstr, sizeof(pstr), &pcc);
printf("Connection Type: PHOEBUS (%s)\n", pstr);
break;
case NS_TYPE_1_SSL:
printf("Connection Type: Single SSL\n");
break;
case NS_TYPE_2_SSL:
printf("Connection Type: Dual SSL\n");
break;
}
} else {
printf("Connection Type: SOCKET\n");
}
if (identical_buffers == 1) {
printf("Identical buffers being used.\n");
} else {
printf("Quasi-random buffers being used.\n");
}
if (net_cs_name == NULL) {
printf("Network Checksum Type: NONE\n");
} else {
printf("Network Checksum Type: %s Block size: %dkb\n", net_cs_name, (blocksize/1024));
}
if (disk_cs_name == NULL) {
printf("Disk Checksum Type: NONE\n");
} else {
printf("Disk Checksum Type: %s Block size: " I64T "kb\n", disk_cs_name, (disk_blocksize/1024));
if (do_validate == 1) {
printf("Disk Validation: Enabled\n");
} else {
printf("Disk Validation: Disabled\n");
}
}
if (fd_out != NULL) {
printf("Saving allocations to %s\n", out_fname);
}
printf("TCP buffer size: %dkb (0 defaults to OS)\n", ibp_get_tcpsize(ic)/1024);
printf("\n");
printf("======= Bulk transfer options =======\n");
printf("aliascreateremove_count: %d\n", aliascreateremove_count);
printf("createremove_count: %d\n", createremove_count);
printf("readwrite_count: %d\n", readwrite_count);
printf("readwrite_alloc_size: %dkb\n", readwrite_size/1024);
printf("rw_block_size: %dkb\n", rw_block_size/1024);
printf("read_mix_fraction: %lf\n", read_mix_fraction);
printf("\n");
printf("======= Small Random I/O transfer options =======\n");
printf("smallio_count: %d\n", smallio_count);
printf("small_min_size: %dkb\n", small_min_size/1024);
printf("small_max_size: %dkb\n", small_max_size/1024);
printf("small_read_fraction: %lf\n", small_read_fraction);
printf("\n");
r1 = 1.0 * readwrite_size/1024.0/1024.0;
r1 = readwrite_count * r1;
printf("Approximate I/O for sequential tests: %lfMB\n", r1);
printf("\n");
io_set_mode(sync_transfer, print_progress, nthreads);
//**************** Create/Remove tests ***************************
if (aliascreateremove_count > 0) {
i = aliascreateremove_count/nthreads;
printf("Starting Alias create test (total files: %d, approx per thread: %d)\n",aliascreateremove_count, i);
base_caps = create_allocs(1, 1);
stime = apr_time_now();
caps_list = create_alias_allocs(aliascreateremove_count, base_caps, 1);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*aliascreateremove_count/dt;
printf("Alias create : %lf creates/sec (%.2lf sec total) \n", r1, dt);
stime = apr_time_now();
alias_remove_allocs(caps_list, base_caps, aliascreateremove_count, 1);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*aliascreateremove_count/dt;
printf("Alias remove : %lf removes/sec (%.2lf sec total) \n", r1, dt);
printf("\n");
printf("-----------------------------\n");
fflush(stdout);
remove_allocs(base_caps, 1);
printf("\n");
}
if (createremove_count > 0) {
i = createremove_count/nthreads;
printf("Starting Create test (total files: %d, approx per thread: %d)\n",createremove_count, i);
stime = apr_time_now();
caps_list = create_allocs(createremove_count, 1);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*createremove_count/dt;
printf("Create : %lf creates/sec (%.2lf sec total) \n", r1, dt);
stime = apr_time_now();
remove_allocs(caps_list, createremove_count);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*createremove_count/dt;
printf("Remove : %lf removes/sec (%.2lf sec total) \n", r1, dt);
printf("\n");
}
//**************** Read/Write tests ***************************
if (readwrite_count > 0) {
i = readwrite_count/nthreads;
printf("Starting Bulk tests (total files: %d, approx per thread: %d", readwrite_count, i);
r1 = 1.0*readwrite_count*readwrite_size/1024.0/1024.0;
r2 = r1 / nthreads;
printf(" -- total size: %lfMB, approx per thread: %lfMB\n", r1, r2);
printf("Creating allocations....");
fflush(stdout);
stime = apr_time_now();
caps_list = create_allocs(readwrite_count, readwrite_size);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*readwrite_count/dt;
printf(" %lf creates/sec (%.2lf sec total) \n", r1, dt);
if (use_alias) {
base_caps = caps_list;
printf("Creating alias allocations....");
fflush(stdout);
stime = apr_time_now();
caps_list = create_alias_allocs(readwrite_count, base_caps, readwrite_count);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*readwrite_count/dt;
printf(" %lf creates/sec (%.2lf sec total) \n", r1, dt);
}
stime = apr_time_now();
write_allocs(caps_list, readwrite_count, readwrite_size, rw_block_size);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*readwrite_count*readwrite_size/(dt*1024*1024);
printf("Write: %lf MB/sec (%.2lf sec total) \n", r1, dt);
stime = apr_time_now();
read_allocs(caps_list, readwrite_count, readwrite_size, rw_block_size);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*readwrite_count*readwrite_size/(dt*1024*1024);
printf("Read: %lf MB/sec (%.2lf sec total) \n", r1, dt);
stime = apr_time_now();
random_allocs(caps_list, readwrite_count, readwrite_size, rw_block_size, read_mix_fraction);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*readwrite_count*readwrite_size/(dt*1024*1024);
printf("Random: %lf MB/sec (%.2lf sec total) \n", r1, dt);
//**************** Small I/O tests ***************************
if (smallio_count > 0) {
if (small_min_size == 0) small_min_size = 1;
if (small_max_size == 0) small_max_size = 1;
printf("\n");
printf("Starting Small Random I/O tests...\n");
stime = apr_time_now();
r1 = small_write_allocs(caps_list, readwrite_count, readwrite_size, smallio_count, small_min_size, small_max_size);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = r1/(1024.0*1024.0);
r2 = r1/dt;
r3 = smallio_count;
r3 = r3 / dt;
printf("Small Random Write: %lf MB/sec (%.2lf sec total using %lfMB or %.2lf ops/sec) \n", r2, dt, r1, r3);
stime = apr_time_now();
r1 = small_read_allocs(caps_list, readwrite_count, readwrite_size, smallio_count, small_min_size, small_max_size);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = r1/(1024.0*1024.0);
r2 = r1/dt;
r3 = smallio_count;
r3 = r3 / dt;
printf("Small Random Read: %lf MB/sec (%.2lf sec total using %lfMB or %.2lf ops/sec) \n", r2, dt, r1, r3);
stime = apr_time_now();
r1 = small_random_allocs(caps_list, readwrite_count, readwrite_size, small_read_fraction, smallio_count, small_min_size, small_max_size);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = r1/(1024.0*1024.0);
r2 = r1/dt;
r3 = smallio_count;
r3 = r3 / dt;
printf("Small Random R/W: %lf MB/sec (%.2lf sec total using %lfMB or %.2lf ops/sec) \n", r2, dt, r1, r3);
}
if (use_alias) {
printf("Removing alias allocations....");
fflush(stdout);
stime = apr_time_now();
alias_remove_allocs(caps_list, base_caps, readwrite_count, readwrite_count);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*readwrite_count/dt;
printf(" %lf removes/sec (%.2lf sec total) \n", r1, dt);
caps_list = base_caps;
}
//** If disk chksumming is enabled then validate it as well
if ((chksum_valid_type(disk_cs_type) == 1) && (do_validate == 1)) {
printf("Validating allocations....");
fflush(stdout);
stime = apr_time_now();
validate_allocs(caps_list, readwrite_count);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*readwrite_count/dt;
printf(" %lf validates/sec (%.2lf sec total) \n", r1, dt);
printf("\n");
}
if (fd_out == NULL) {
printf("Removing allocations....");
fflush(stdout);
stime = apr_time_now();
remove_allocs(caps_list, readwrite_count);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*readwrite_count/dt;
printf(" %lf removes/sec (%.2lf sec total) \n", r1, dt);
printf("\n");
} else {
printf("Saving allocations to %s instead of deleting them\n", out_fname);
save_allocs(fd_out, caps_list, readwrite_count);
if (fd_special == 0) fclose(fd_out);
}
}
printf("Final network connection counter: %d\n", network_counter(NULL));
ibp_destroy_context(ic); //** Shutdown IBP
return(0);
}
int main(int argc, char **argv)
{
double r1, r2, r3;
int i, do_simple_test, tcpsize;
ibp_capset_t *caps_list, *base_caps;
rid_t rid;
int port;
char buffer[1024];
apr_time_t stime, dtime;
double dt;
char *ppath;
phoebus_t pcc;
char pstr[2048];
base_caps = NULL;
if (argc < 12) {
printf("\n");
printf("ibp_perf [-d|-dd] [-config ibp.cfg] [-phoebus gateway_list] [-tcpsize tcpbufsize]\n");
printf(" [-duration duration] [-sync] [-alias]\n");
printf(" n_depots depot1 port1 resource_id1 ... depotN portN ridN\n");
printf(" nthreads ibp_timeout\n");
printf(" alias_createremove_count createremove_count\n");
printf(" readwrite_count readwrite_alloc_size rw_block_size read_mix_fraction\n");
printf(" smallio_count small_min_size small_max_size small_read_fraction\n");
printf("\n");
printf("-d - Enable *minimal* debug output\n");
printf("-dd - Enable *FULL* debug output\n");
printf("-config ibp.cfg - Use the IBP configuration defined in file ibp.cfg.\n");
printf(" nthreads overrides value in cfg file unless -1.\n");
printf("-phoebus - Use Phoebus protocol for data transfers.\n");
printf(" gateway_list - Comma separated List of phoebus hosts/ports, eg gateway1/1234,gateway2/4321\n");
printf("-tcpsize tcpbufsize - Use this value, in KB, for the TCP send/recv buffer sizes\n");
printf("-duration duration - Allocation duration in sec. Needs to be big enough to last the entire\n");
printf(" run. The default duration is %d sec.\n", a_duration);
printf("-sync - Use synchronous protocol. Default uses async.\n");
printf("-alias - Use alias allocations for all I/O operations\n");
printf("n_depots - Number of depot tuplets\n");
printf("depot - Depot hostname\n");
printf("port - IBP port on depot\n");
printf("resource_id - Resource ID to use on depot\n");
printf("nthreads - Max Number of simultaneous threads to use. Use -1 for defaults or value in ibp.cfg\n");
printf("ibp_timeout - Timeout(sec) for each IBP copmmand\n");
printf("alias_createremove_count* - Number of 0 byte files to create and remove using alias allocations\n");
printf("createremove_count* - Number of 0 byte files to create and remove to test metadata performance\n");
printf("readwrite_count* - Number of files to write sequentially then read sequentially\n");
printf("readwrite_alloc_size - Size of each allocation in KB for sequential and random tests\n");
printf("rw_block_size - Size of each R/W operation in KB for sequential and random tests\n");
printf("read_mix_fraction - Fraction of Random I/O operations that are READS\n");
printf("smallio_count* - Number of random small I/O operations\n");
printf("small_min_size - Minimum size of each small I/O operation(kb)\n");
printf("small_max_size - Max size of each small I/O operation(kb)\n");
printf("small_read_fraction - Fraction of small random I/O operations that are READS\n");
printf("\n");
printf("*If the variable is set to 0 then the test is skipped\n");
printf("\n");
return(-1);
}
ibp_init(); //** Initialize IBP
i = 1;
if (strcmp(argv[i], "-d") == 0) { //** Enable debugging
set_log_level(5);
i++;
}
if (strcmp(argv[i], "-dd") == 0) { //** Enable debugging
set_log_level(20);
i++;
}
if (strcmp(argv[i], "-config") == 0) { //** Read the config file
i++;
ibp_load_config(argv[i]);
i++;
}
if (strcmp(argv[i], "-phoebus") == 0) { //** Check if we want Phoebus transfers
cc = (ibp_connect_context_t *)malloc(sizeof(ibp_connect_context_t));
cc->type = NS_TYPE_PHOEBUS;
i++;
ppath = argv[i];
phoebus_path_set(&pcc, ppath);
cc->data = &pcc;
i++;
}
if (strcmp(argv[i], "-tcpsize") == 0) { //** Check if we want sync tests
i++;
tcpsize = atoi(argv[i]) * 1024;
ibp_set_tcpsize(tcpsize);
i++;
}
if (strcmp(argv[i], "-duration") == 0) { //** Check if we want sync tests
i++;
a_duration = atoi(argv[i]);
i++;
}
sync_transfer = 0;
if (strcmp(argv[i], "-sync") == 0) { //** Check if we want sync tests
sync_transfer = 1;
i++;
}
use_alias = 0;
if (strcmp(argv[i], "-alias") == 0) { //** Check if we want to use alias allocation
use_alias = 1;
i++;
}
do_simple_test = 0;
if (strcmp(argv[i], "-simpletest") == 0) { //** Just do the simple test
do_simple_test = 1;
i++;
}
n_depots = atoi(argv[i]);
i++;
depot_list = (ibp_depot_t *)malloc(sizeof(ibp_depot_t)*n_depots);
int j;
for (j=0; j<n_depots; j++) {
port = atoi(argv[i+1]);
rid = ibp_str2rid(argv[i+2]);
set_ibp_depot(&(depot_list[j]), argv[i], port, rid);
i = i + 3;
}
//*** Get thread count ***
nthreads = atoi(argv[i]);
if (nthreads <= 0) {
nthreads = ibp_get_max_depot_threads();
} else {
ibp_set_max_depot_threads(nthreads);
}
i++;
ibp_timeout = atoi(argv[i]); i++;
//****** Get the different Stream counts *****
int aliascreateremove_count = atoi(argv[i]); i++;
int createremove_count = atoi(argv[i]); i++;
int readwrite_count = atoi(argv[i]); i++;
int readwrite_size = atoi(argv[i])*1024; i++;
int rw_block_size = atoi(argv[i])*1024; i++;
double read_mix_fraction = atof(argv[i]); i++;
//****** Get the different small I/O counts *****
int smallio_count = atoi(argv[i]); i++;
int small_min_size = atoi(argv[i])*1024; i++;
int small_max_size = atoi(argv[i])*1024; i++;
double small_read_fraction = atof(argv[i]); i++;
//*** Print the ibp client version ***
printf("\n");
printf("================== IBP Client Version =================\n");
printf("%s\n", ibp_client_version());
//*** Print summary of options ***
printf("\n");
printf("======= Base options =======\n");
printf("n_depots: %d\n", n_depots);
for (i=0; i<n_depots; i++) {
printf("depot %d: %s:%d rid:%s\n", i, depot_list[i].host, depot_list[i].port, ibp_rid2str(depot_list[i].rid, buffer));
}
printf("\n");
printf("IBP timeout: %d\n", ibp_timeout);
printf("Max Threads: %d\n", nthreads);
if (sync_transfer == 1) {
printf("Transfer_mode: SYNC\n");
} else {
printf("Transfer_mode: ASYNC\n");
}
printf("Use alias: %d\n", use_alias);
if (cc != NULL) {
switch (cc->type) {
case NS_TYPE_SOCK:
printf("Connection Type: SOCKET\n"); break;
case NS_TYPE_PHOEBUS:
phoebus_path_to_string(pstr, sizeof(pstr), &pcc);
printf("Connection Type: PHOEBUS (%s)\n", pstr); break;
case NS_TYPE_1_SSL:
printf("Connection Type: Single SSL\n"); break;
case NS_TYPE_2_SSL:
printf("Connection Type: Dual SSL\n"); break;
}
} else {
printf("Connection Type: SOCKET\n");
}
printf("TCP buffer size: %dkb (0 defaults to OS)\n", ibp_get_tcpsize()/1024);
printf("\n");
printf("======= Bulk transfer options =======\n");
printf("aliascreateremove_count: %d\n", aliascreateremove_count);
printf("createremove_count: %d\n", createremove_count);
printf("readwrite_count: %d\n", readwrite_count);
printf("readwrite_alloc_size: %dkb\n", readwrite_size/1024);
printf("rw_block_size: %dkb\n", rw_block_size/1024);
printf("read_mix_fraction: %lf\n", read_mix_fraction);
printf("\n");
printf("======= Small Random I/O transfer options =======\n");
printf("smallio_count: %d\n", smallio_count);
printf("small_min_size: %dkb\n", small_min_size/1024);
printf("small_max_size: %dkb\n", small_max_size/1024);
printf("small_read_fraction: %lf\n", small_read_fraction);
printf("\n");
r1 = 1.0 * readwrite_size/1024.0/1024.0;
r1 = readwrite_count * r1;
printf("Approximate I/O for sequential tests: %lfMB\n", r1);
printf("\n");
if (do_simple_test == 1) {
simple_test(nthreads);
return(0);
}
//**************** Create/Remove tests ***************************
if (aliascreateremove_count > 0) {
i = aliascreateremove_count/nthreads;
printf("Starting Alias create test (total files: %d, approx per thread: %d)\n",aliascreateremove_count, i);
base_caps = create_allocs(1, 1);
stime = apr_time_now();
caps_list = create_alias_allocs(aliascreateremove_count, base_caps, 1);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*aliascreateremove_count/dt;
printf("Alias create : %lf creates/sec (%.2lf sec total) \n", r1, dt);
stime = apr_time_now();
alias_remove_allocs(caps_list, base_caps, aliascreateremove_count, 1);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*aliascreateremove_count/dt;
printf("Alias remove : %lf removes/sec (%.2lf sec total) \n", r1, dt);
printf("\n");
printf("-----------------------------\n"); fflush(stdout);
remove_allocs(base_caps, 1);
printf("\n");
}
if (createremove_count > 0) {
i = createremove_count/nthreads;
printf("Starting Create test (total files: %d, approx per thread: %d)\n",createremove_count, i);
stime = apr_time_now();
caps_list = create_allocs(createremove_count, 1);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*createremove_count/dt;
printf("Create : %lf creates/sec (%.2lf sec total) \n", r1, dt);
stime = apr_time_now();
remove_allocs(caps_list, createremove_count);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*createremove_count/dt;
printf("Remove : %lf removes/sec (%.2lf sec total) \n", r1, dt);
printf("\n");
}
//**************** Read/Write tests ***************************
if (readwrite_count > 0) {
i = readwrite_count/nthreads;
printf("Starting Bulk tests (total files: %d, approx per thread: %d", readwrite_count, i);
r1 = 1.0*readwrite_count*readwrite_size/1024.0/1024.0;
r2 = r1 / nthreads;
printf(" -- total size: %lfMB, approx per thread: %lfMB\n", r1, r2);
printf("Creating allocations...."); fflush(stdout);
stime = apr_time_now();
caps_list = create_allocs(readwrite_count, readwrite_size);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*readwrite_count/dt;
printf(" %lf creates/sec (%.2lf sec total) \n", r1, dt);
if (use_alias) {
base_caps = caps_list;
printf("Creating alias allocations...."); fflush(stdout);
stime = apr_time_now();
caps_list = create_alias_allocs(readwrite_count, base_caps, readwrite_count);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*readwrite_count/dt;
printf(" %lf creates/sec (%.2lf sec total) \n", r1, dt);
}
stime = apr_time_now();
write_allocs(caps_list, readwrite_count, readwrite_size, rw_block_size);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*readwrite_count*readwrite_size/(dt*1024*1024);
printf("Write: %lf MB/sec (%.2lf sec total) \n", r1, dt);
stime = apr_time_now();
read_allocs(caps_list, readwrite_count, readwrite_size, rw_block_size);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*readwrite_count*readwrite_size/(dt*1024*1024);
printf("Read: %lf MB/sec (%.2lf sec total) \n", r1, dt);
stime = apr_time_now();
random_allocs(caps_list, readwrite_count, readwrite_size, rw_block_size, read_mix_fraction);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*readwrite_count*readwrite_size/(dt*1024*1024);
printf("Random: %lf MB/sec (%.2lf sec total) \n", r1, dt);
//**************** Small I/O tests ***************************
if (smallio_count > 0) {
if (small_min_size == 0) small_min_size = 1;
if (small_max_size == 0) small_max_size = 1;
printf("\n");
printf("Starting Small Random I/O tests...\n");
stime = apr_time_now();
r1 = small_write_allocs(caps_list, readwrite_count, readwrite_size, smallio_count, small_min_size, small_max_size);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = r1/(1024.0*1024.0);
r2 = r1/dt;
r3 = smallio_count; r3 = r3 / dt;
printf("Small Random Write: %lf MB/sec (%.2lf sec total using %lfMB or %.2lf ops/sec) \n", r2, dt, r1, r3);
stime = apr_time_now();
r1 = small_read_allocs(caps_list, readwrite_count, readwrite_size, smallio_count, small_min_size, small_max_size);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = r1/(1024.0*1024.0);
r2 = r1/dt;
r3 = smallio_count; r3 = r3 / dt;
printf("Small Random Read: %lf MB/sec (%.2lf sec total using %lfMB or %.2lf ops/sec) \n", r2, dt, r1, r3);
stime = apr_time_now();
r1 = small_random_allocs(caps_list, readwrite_count, readwrite_size, small_read_fraction, smallio_count, small_min_size, small_max_size);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = r1/(1024.0*1024.0);
r2 = r1/dt;
r3 = smallio_count; r3 = r3 / dt;
printf("Small Random R/W: %lf MB/sec (%.2lf sec total using %lfMB or %.2lf ops/sec) \n", r2, dt, r1, r3);
}
if (use_alias) {
printf("Removing alias allocations...."); fflush(stdout);
stime = apr_time_now();
alias_remove_allocs(caps_list, base_caps, readwrite_count, readwrite_count);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*readwrite_count/dt;
printf(" %lf removes/sec (%.2lf sec total) \n", r1, dt);
caps_list = base_caps;
}
printf("Removing allocations...."); fflush(stdout);
stime = apr_time_now();
remove_allocs(caps_list, readwrite_count);
dtime = apr_time_now() - stime;
dt = dtime / (1.0 * APR_USEC_PER_SEC);
r1 = 1.0*readwrite_count/dt;
printf(" %lf removes/sec (%.2lf sec total) \n", r1, dt);
printf("\n");
}
printf("Final network connection counter: %d\n", network_counter(NULL));
ibp_finalize(); //** Shutdown IBP
return(0);
}