/**
********************************************************************************
** \brief print errno and exit
** \details
** An IO has failed, print the errno and exit
*******************************************************************************/
void io_error(int id, int err)
{
fprintf(stderr, "io_error: errno:%d\n", err);
cblk_close(id,0);
cblk_term(NULL,0);
exit(err);
}
int ea_delete(EA *ea)
{
int rc = 0;
if ( ea->st_type == EA_STORE_TYPE_MEMORY )
{
KV_TRC(pAT, "free ea %p ea->st_memory %p", ea, ea->st_memory);
// Simple free the block of store
if (ea->st_memory)
{
free(ea->st_memory);
ea->st_memory=NULL;
}
}
else
{
// Call to close out the chunk and free the space
// for the device name
rc = cblk_close(ea->st_flash, 0);
am_free(ea->st_device);
}
if ( rc == 0 )
{
KV_TRC(pAT, "free ea %p", ea);
am_free(ea);
}
return rc;
}
/**
********************************************************************************
** \brief main
** \details
** process input parms \n
** open device \n
** alloc memory \n
** loop running IO until secs expire \n
** print IO stats \n
** cleanup
*******************************************************************************/
int main(int argc, char **argv)
{
struct timeval start, delta;
long int mil = 1000000;
float esecs = 0;
uint8_t *rbuf = NULL;
uint8_t *wbuf = NULL;
OP_t *op = NULL;
char *dev = NULL;
char FF = 0xFF;
char c = '\0';
chunk_ext_arg_t ext = 0;
int flags = 0;
int i, rc = 0;
int id = 0;
char *_secs = NULL;
char *_QD = NULL;
char *_RD = NULL;
char *_nblocks = NULL;
uint32_t plun = 0;
uint32_t nsecs = 4;
uint32_t QD = 256;
uint32_t nRD = 100;
uint32_t RD = 0;
uint32_t WR = 0;
uint32_t intrp_thds = 0;
int rtag = 0;
int htag = 0;
uint32_t lba = 0;
size_t nblks = 0;
uint32_t nblocks = 1;
uint32_t cnt = 0;
uint32_t tmiss = 0;
uint64_t status = 0;
uint32_t TI = TIME_INTERVAL;
uint32_t N = 0;
uint32_t TIME = 1;
uint32_t COMP = 0;
uint32_t miss = 0;
uint64_t tlat = 0;
double ns_per_tick = 0;
/*--------------------------------------------------------------------------
* process and verify input parms
*------------------------------------------------------------------------*/
while (FF != (c=getopt(argc, argv, "d:r:q:n:s:phi")))
{
switch (c)
{
case 'd': dev = optarg; break;
case 'r': _RD = optarg; break;
case 'q': _QD = optarg; break;
case 'n': _nblocks = optarg; break;
case 's': _secs = optarg; break;
case 'p': plun = 1; break;
case 'i': intrp_thds = 1; break;
case 'h':
case '?': usage(); break;
}
}
if (_secs) nsecs = atoi(_secs);
if (_QD) QD = atoi(_QD);
if (_nblocks) nblocks = atoi(_nblocks);
if (_RD) nRD = atoi(_RD);
if (QD > _8K) QD = _8K;
if (nRD > 100) nRD = 100;
if (!plun && nblocks > 1)
{
printf("error: <-n %d> can only be used with a physical lun\n",nblocks);
usage();
}
if (dev == NULL) usage();
srand48(time(0));
ns_per_tick = time_per_tick(1000, 100);
N = QD;
COMP = QD < 8 ? 1 : QD/8;
/*--------------------------------------------------------------------------
* open device and set lun size
*------------------------------------------------------------------------*/
rc = cblk_init(NULL,0);
if (rc)
{
fprintf(stderr,"cblk_init failed with rc = %d and errno = %d\n",
rc,errno);
exit(1);
}
if (!plun) flags = CBLK_OPN_VIRT_LUN;
if (!intrp_thds) flags |= CBLK_OPN_NO_INTRP_THREADS;
id = cblk_open(dev, QD, O_RDWR, ext, flags);
if (id == NULL_CHUNK_ID)
{
if (ENOSPC == errno) fprintf(stderr,"cblk_open: ENOSPC\n");
else if (ENODEV == errno) fprintf(stderr,"cblk_open: ENODEV\n");
else fprintf(stderr,"cblk_open: errno:%d\n",errno);
cblk_term(NULL,0);
exit(errno);
}
rc = cblk_get_lun_size(id, &nblks, 0);
if (rc)
{
fprintf(stderr, "cblk_get_lun_size failed: errno: %d\n", errno);
exit(errno);
}
if (!plun)
{
nblks = nblks > SET_NBLKS ? SET_NBLKS : nblks;
rc = cblk_set_size(id, nblks, 0);
if (rc)
{
fprintf(stderr, "cblk_set_size failed, errno: %d\n", errno);
exit(errno);
}
}
lba = lrand48() % nblks;
/*--------------------------------------------------------------------------
* alloc data for IO
*------------------------------------------------------------------------*/
op = malloc(QD*sizeof(OP_t));
if ((rc=posix_memalign((void**)&rbuf, _4K, _4K*nblocks)))
{
fprintf(stderr,"posix_memalign failed, size=%d, rc=%d\n",
_4K*nblocks, rc);
cblk_close(id,0);
cblk_term(NULL,0);
exit(0);
}
if ((rc=posix_memalign((void**)&wbuf, _4K, _4K*nblocks)))
{
fprintf(stderr,"posix_memalign failed, size=%d, rc=%d\n",
_4K*nblocks, rc);
cblk_close(id,0);
cblk_term(NULL,0);
exit(0);
}
memset(wbuf,0x79,_4K*nblocks);
memset(op, 0, QD*sizeof(OP_t));
/*--------------------------------------------------------------------------
* loop running IO until secs expire
*------------------------------------------------------------------------*/
gettimeofday(&start, NULL);
do
{
/* setup #read ops and #write ops to send before completing ops */
if (!RD && !WR) {RD=nRD; WR=100-RD;}
/*----------------------------------------------------------------------
* send up to RD reads, as long as the queuedepth N is not max
*--------------------------------------------------------------------*/
while (TIME && RD && N)
{
rc = cblk_aread(id, rbuf, lba, nblocks, &rtag, NULL,
CBLK_ARW_WAIT_CMD_FLAGS);
if (0 == rc) {OP_BEG(rtag); --RD; --N; BMP_LBA();}
else if (EBUSY == errno) {break;}
else {io_error(id,errno);}
}
/*----------------------------------------------------------------------
* send up to WR writes, as long as the queuedepth N is not max
*--------------------------------------------------------------------*/
while (TIME && WR && N)
{
rc = cblk_awrite(id, wbuf, lba, nblocks, &rtag, NULL,
CBLK_ARW_WAIT_CMD_FLAGS);
if (0 == rc) {OP_BEG(rtag); --WR; --N; BMP_LBA();}
else if (EBUSY == errno) {break;}
else {io_error(id,errno);}
}
/*----------------------------------------------------------------------
* complete cmds
*--------------------------------------------------------------------*/
for (i=0; i<QD && N<COMP; i++, htag++)
{
if (intrp_thds)
{
rc = cblk_aresult(id, &htag, &status, CBLK_ARESULT_BLOCKING |
CBLK_ARESULT_NEXT_TAG);
if (rc != nblocks) {io_error(id,errno);}
OP_END(htag); ++cnt; ++N;
continue;
}
if (htag>=QD) htag=0;
if (!op[htag].iss) {continue;}
rc = cblk_aresult(id, &htag, &status, 0);
if (rc == 0)
{
if (QD==1 && ++miss==1) {usleep(80);}
++tmiss; continue;
}
else if (rc < 0)
{io_error(id,errno);}
OP_END(htag); ++cnt; ++N; miss=0;
}
/*----------------------------------------------------------------------
* at an interval which does not impact performance, check if secs
* have expired, and randomize lba
*--------------------------------------------------------------------*/
if (cnt > TI)
{
TI += TIME_INTERVAL;
gettimeofday(&delta, NULL);
if (delta.tv_sec - start.tv_sec >= nsecs) {TIME=0; COMP=QD;}
lba = lrand48() % nblks;
}
}
while (TIME || QD-N);
/*--------------------------------------------------------------------------
* print IO stats
*------------------------------------------------------------------------*/
gettimeofday(&delta, NULL);
esecs = ((float)((delta.tv_sec*mil + delta.tv_usec) -
(start.tv_sec*mil + start.tv_usec))) / (float)mil;
printf("d:%s r:%d q:%d s:%d p:%d n:%d i:%d miss:%d lat:%d mbps:%d iops:%d",
dev, nRD, QD, nsecs, plun, nblocks, intrp_thds, tmiss,
(uint32_t)((tlat*ns_per_tick)/cnt/1000),
(uint32_t)((float)((cnt*nblocks*4)/1024)/esecs),
(uint32_t)((float)(cnt/esecs)));
if (plun && nblocks > 1)
printf(" 4k-iops:%d", (uint32_t)((float)(cnt*nblocks)/esecs));
printf("\n");
/*--------------------------------------------------------------------------
* cleanup
*------------------------------------------------------------------------*/
free(op);
free(rbuf);
free(wbuf);
cblk_close(id,0);
cblk_term(NULL,0);
return 0;
}
EA *ea_new(const char *path, uint64_t bsize, int basyncs,
uint64_t *size, uint64_t *bcount, uint64_t vlun)
{
int rc = 0;
size_t plen = 0;
uint8_t *store = NULL;
EA *ea = NULL;
chunk_id_t chkid = NULL_CHUNK_ID;
chunk_ext_arg_t ext = 0;
if (!(fetch_and_or(&cflsh_blk_lib_init,1)))
{
// We need to call cblk_init once before
// we use any other cblk_ interfaces
rc = cblk_init(NULL,0);
if (rc)
{
KV_TRC_FFDC(pAT, "cblk_init failed path %s bsize %"PRIu64" "
"size %"PRIu64" bcount %"PRIu64", errno = %d",
path, bsize, *size, *bcount, errno);
goto error_exit;
}
}
ea = am_malloc(sizeof(EA));
if (NULL == ea)
{
KV_TRC_FFDC(pAT, "Out of memory path %s bsize %"PRIu64" size %"PRIu64" "
"bcount %"PRIu64", errno = %d",
path, bsize, *size, *bcount, errno);
goto error_exit;
}
// We need to check the path parameter to see if
// we are going to use memory or a file/capi
// device (to be determined by the block layer)
if ( (NULL == path) || (strlen(path) == 0) )
{
KV_TRC(pAT, "EA_STORE_TYPE_MEMORY");
// Using memory for store
ea->st_type = EA_STORE_TYPE_MEMORY;
store = malloc(*size);
if (NULL == store)
{
errno = ENOMEM;
KV_TRC_FFDC(pAT, "Out of memory for store path %s bsize %"PRIu64" "
"size %"PRIu64" bcount %"PRIu64", errno = %d",
path, bsize, *size, *bcount, errno);
goto error_exit;
}
*bcount = ((*size) / bsize);
ea->st_memory = store;
}
else
{
KV_TRC(pAT, "EA_STORE_TYPE_FILE(%s)", path);
// Using a file. We don't care if it's an actual
// file or a CAPI device, we let block layer
// decide and we just use the chunk ID that is
// passed back from the cblk_open call.
ea->st_type = EA_STORE_TYPE_FILE;
// Check to see if we need to create the store on a
// physical or virtual LUN. Previously, in GA1,
// we keyed off the size and if it was 0, then we
// asked for the LUN to be physical. Now, the user
// can specify with a flag.
if ( vlun == 0 )
{
KV_TRC(pAT, "cblk_open PHYSICAL LUN: %s", path);
chkid = cblk_open(path, basyncs, O_RDWR, ext,
CBLK_OPN_NO_INTRP_THREADS);
if (NULL_CHUNK_ID == chkid)
{
printf("cblk_open physical lun failed\n");
KV_TRC_FFDC(pAT, "cblk_open phys lun failed path:%s bsize:%ld "
"size:%ld bcount:%ld, errno:%d",
path, bsize, *size, *bcount, errno);
goto error_exit;
}
rc = cblk_get_size(chkid, (size_t *)bcount, 0);
if ( (rc != 0) || (*bcount == 0) )
{
// An error was encountered, close the chunk
cblk_close(chkid, 0);
chkid = NULL_CHUNK_ID;
KV_TRC_FFDC(pAT, "cblk_get_size failed path %s bsize %"PRIu64" "
"size %"PRIu64" bcount %"PRIu64", errno = %d",
path, bsize, *size, *bcount, errno);
goto error_exit;
}
// Set the size to be returned
*size = *bcount * bsize;
}
else
{
KV_TRC(pAT, "cblk_open VIRTUAL LUN: %s", path);
chkid = cblk_open(path, basyncs, O_RDWR, ext,
CBLK_OPN_VIRT_LUN|CBLK_OPN_NO_INTRP_THREADS);
if (NULL_CHUNK_ID == chkid)
{
printf("cblk_open virtual lun failed\n");
KV_TRC_FFDC(pAT, "cblk_open virt lun failed path:%s bsize:%ld "
"size:%ld bcount:%ld, errno:%d",
path, bsize, *size, *bcount, errno);
goto error_exit;
}
// A specific size was passed in so we try to set the
// size of the chunk.
*bcount = *size / bsize;
rc = cblk_set_size(chkid, (size_t)*bcount, 0);
if ( rc != 0 )
{
printf("cblk_set_size failed for %ld\n", *bcount);
// An error was encountered, close the chunk
cblk_close(chkid, 0);
chkid = NULL_CHUNK_ID;
KV_TRC_FFDC(pAT, "cblk_set_size failed path %s bsize %"PRIu64" "
"size %"PRIu64" bcount %"PRIu64", errno = %d",
path, bsize, *size, *bcount, errno);
goto error_exit;
}
}
// Save off the chunk ID and the device name
ea->st_flash = chkid;
plen = strlen(path) + 1;
ea->st_device = (char *)am_malloc(plen);
if (!ea->st_device)
{
cblk_close(chkid, 0);
KV_TRC_FFDC(pAT, "MALLOC st_device failed (%s) plen=%ld errno:%d",
path, plen, errno);
goto error_exit;
}
memset(ea->st_device, 0, plen);
strncpy(ea->st_device, path, plen);
}
// Fill in the EA struct
pthread_rwlock_init(&(ea->ea_rwlock), NULL);
ea->bsize = bsize;
ea->bcount = *bcount;
ea->size = *size;
KV_TRC(pAT, "path %s bsize %"PRIu64" size %"PRIu64" bcount %"PRIu64"",
path, bsize, *size, *bcount);
goto done;
error_exit:
am_free(ea);
ea = NULL;
if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=ENOSPC;}
done:
return ea;
}
/**
********************************************************************************
** \brief main
** \details
** process input parms \n
** open device \n
** alloc memory \n
** loop running IO until secs expire \n
** print IO stats \n
** cleanup
*******************************************************************************/
int main(int argc, char **argv)
{
struct timeval start, delta;
long int mil = 1000000;
float esecs = 0;
uint8_t **rbuf = NULL;
uint8_t **wbuf = NULL;
int *tags = NULL;
char *dev = NULL;
char FF = 0xFF;
char c = '\0';
chunk_ext_arg_t ext = 0;
int flags = 0;
int rc = 0;
int id = 0;
char *_secs = NULL;
char *_QD = NULL;
char *_RD = NULL;
char *_nblocks = NULL;
uint32_t plun = 0;
uint32_t nsecs = 4;
uint32_t QD = 500;
uint32_t nRD = 100;
uint32_t RD = 0;
uint32_t WR = 0;
uint32_t intrp_thds = 0;
int tag = 0;
int rtag = 0;
uint32_t lba = 0;
size_t nblks = 0;
uint32_t nblocks = 1;
uint32_t cnt = 0;
uint32_t pollN = 0;
uint64_t status = 0;
uint32_t TI = TIME_INTERVAL;
uint32_t N = 0;
uint32_t TIME = 1;
uint32_t COMP = 0;
/*--------------------------------------------------------------------------
* process and verify input parms
*------------------------------------------------------------------------*/
while (FF != (c=getopt(argc, argv, "d:r:q:n:s:phi")))
{
switch (c)
{
case 'd': dev = optarg; break;
case 'r': _RD = optarg; break;
case 'q': _QD = optarg; break;
case 'n': _nblocks = optarg; break;
case 's': _secs = optarg; break;
case 'p': plun = 1; break;
case 'i': intrp_thds = 1; break;
case 'h':
case '?': usage(); break;
}
}
if (_secs) nsecs = atoi(_secs);
if (_QD) QD = atoi(_QD);
if (_nblocks) nblocks = atoi(_nblocks);
if (_RD) nRD = atoi(_RD);
if (QD > _8K) QD = _8K;
if (nRD > 100) nRD = 100;
if (!plun) nblocks = 1;
if (dev == NULL) usage();
srand48(time(0));
N = QD;
COMP = QD < 5 ? 1 : QD/5;
/*--------------------------------------------------------------------------
* open device and set lun size
*------------------------------------------------------------------------*/
rc = cblk_init(NULL,0);
if (rc)
{
fprintf(stderr,"cblk_init failed with rc = %d and errno = %d\n",
rc,errno);
exit(1);
}
if (!plun) flags = CBLK_OPN_VIRT_LUN;
if (!intrp_thds) flags |= CBLK_OPN_NO_INTRP_THREADS;
id = cblk_open(dev, QD, O_RDWR, ext, flags);
if (id == NULL_CHUNK_ID)
{
if (ENOSPC == errno) fprintf(stderr,"cblk_open: ENOSPC\n");
else if (ENODEV == errno) fprintf(stderr,"cblk_open: ENODEV\n");
else fprintf(stderr,"cblk_open: errno:%d\n",errno);
cblk_term(NULL,0);
exit(errno);
}
rc = cblk_get_lun_size(id, &nblks, 0);
if (rc)
{
fprintf(stderr, "cblk_get_lun_size failed: errno: %d\n", errno);
exit(errno);
}
if (!plun)
{
nblks = nblks > SET_NBLKS ? SET_NBLKS : nblks;
rc = cblk_set_size(id, nblks, 0);
if (rc)
{
fprintf(stderr, "cblk_set_size failed, errno: %d\n", errno);
exit(errno);
}
}
/*--------------------------------------------------------------------------
* alloc data for IO
*------------------------------------------------------------------------*/
tags = malloc(QD*sizeof(int));
rbuf = malloc(QD*sizeof(uint8_t*));
wbuf = malloc(QD*sizeof(uint8_t*));
for (tag=0; tag<QD; tag++)
{
if ((rc=posix_memalign((void**)&(rbuf[tag]), _4K, _4K*nblocks)))
{
fprintf(stderr,"posix_memalign failed, size=%d, rc=%d\n",
_4K*nblocks, rc);
cblk_close(id,0);
cblk_term(NULL,0);
exit(0);
}
if ((rc=posix_memalign((void**)&(wbuf[tag]), _4K, _4K*nblocks)))
{
fprintf(stderr,"posix_memalign failed, size=%d, rc=%d\n",
_4K*nblocks, rc);
cblk_close(id,0);
cblk_term(NULL,0);
exit(0);
}
memset(wbuf[tag],0x79,_4K*nblocks);
tags[tag] = tag;
}
/*--------------------------------------------------------------------------
* loop running IO until secs expire
*------------------------------------------------------------------------*/
gettimeofday(&start, NULL);
do
{
/* setup #read ops and #write ops to send before completing ops */
if (!RD && !WR) {RD=nRD; WR=100-RD;}
/*----------------------------------------------------------------------
* send up to RD reads, as long as the queuedepth N is not max
*--------------------------------------------------------------------*/
while (TIME && RD && N)
{
tag = tags[--N];
rc = cblk_aread(id, rbuf[tag], lba, nblocks, &tag, NULL,
CBLK_ARW_WAIT_CMD_FLAGS | CBLK_ARW_USER_TAG_FLAG);
if (0 == rc) {--RD; lba+=2; if (lba >= nblks) lba=cnt%2;}
else if (EBUSY == errno) {++N; usleep(USLEEP); continue;}
else {io_error(id,errno);}
}
/*----------------------------------------------------------------------
* send up to WR writes, as long as the queuedepth N is not max
*--------------------------------------------------------------------*/
while (TIME && WR && N)
{
tag = tags[--N];
rc = cblk_awrite(id, wbuf[tag], lba, nblocks, &tag, NULL,
CBLK_ARW_WAIT_CMD_FLAGS | CBLK_ARW_USER_TAG_FLAG);
if (0 == rc) {--WR; lba+=2; if (lba >= nblks) lba=cnt%2;}
else if (EBUSY == errno) {++N; usleep(USLEEP); continue;}
else {io_error(id,errno);}
}
/* if the queuedepth is 1, don't immediately pound aresult */
if (QD==1) usleep(USLEEP);
/*----------------------------------------------------------------------
* complete cmds until queue depth is QD-COMP
*--------------------------------------------------------------------*/
while (N < COMP)
{
rtag=0;
rc = cblk_aresult(id, &rtag, &status, CBLK_ARESULT_BLOCKING|
CBLK_ARESULT_NEXT_TAG);
if (rc == 0) {++pollN; usleep(USLEEP); continue;}
else if (rc < 0) {io_error(id,errno);}
++cnt; tags[N++] = rtag;
}
/*----------------------------------------------------------------------
* at an interval which does not impact performance, check if secs
* have expired, and randomize lba
*--------------------------------------------------------------------*/
if (cnt > TI)
{
TI += TIME_INTERVAL;
gettimeofday(&delta, NULL);
if (delta.tv_sec - start.tv_sec >= nsecs) {TIME=0; COMP = QD;}
lba = lrand48() % TIME_INTERVAL;
}
}
while (TIME || QD-N);
/*--------------------------------------------------------------------------
* print IO stats
*------------------------------------------------------------------------*/
gettimeofday(&delta, NULL);
esecs = ((float)((delta.tv_sec*mil + delta.tv_usec) -
(start.tv_sec*mil + start.tv_usec))) / (float)mil;
printf("d:%s r:%d q:%d s:%d p:%d n:%d i:%d pollN:%d mbps:%d iops:%d",
dev, nRD, QD, nsecs, plun, nblocks, intrp_thds, pollN,
(uint32_t)((float)((cnt*nblocks*4)/1024)/esecs),
(uint32_t)((float)(cnt/esecs)));
if (plun && nblocks > 1)
printf(" 4k-iops:%d", (uint32_t)((float)(cnt*nblocks)/esecs));
printf("\n");
/*--------------------------------------------------------------------------
* cleanup
*------------------------------------------------------------------------*/
for (cnt=0; cnt<QD; cnt++)
{
free(rbuf[cnt]);
free(wbuf[cnt]);
}
free(rbuf);
free(wbuf);
cblk_close(id,0);
cblk_term(NULL,0);
return 0;
}