Beispiel #1
0
/**
********************************************************************************
** \brief print errno and exit
** \details
**   An IO has failed, print the errno and exit
*******************************************************************************/
void io_error(int id, int err)
{
    fprintf(stderr, "io_error: errno:%d\n", err);
    cblk_close(id,0);
    cblk_term(NULL,0);
    exit(err);
}
Beispiel #2
0
int ea_delete(EA *ea)
{
    int rc = 0;

    if ( ea->st_type == EA_STORE_TYPE_MEMORY )
    {
        KV_TRC(pAT, "free ea %p ea->st_memory %p", ea, ea->st_memory);
        // Simple free the block of store
        if (ea->st_memory)
        {
            free(ea->st_memory);
            ea->st_memory=NULL;
        }
    }
    else
    {
        // Call to close out the chunk and free the space
        // for the device name
        rc = cblk_close(ea->st_flash, 0);
        am_free(ea->st_device);
    }

    if ( rc == 0 )
    {
        KV_TRC(pAT, "free ea %p", ea);
        am_free(ea);
    }

    return rc;
}
Beispiel #3
0
/**
********************************************************************************
** \brief main
** \details
**   process input parms                                                      \n
**   open device                                                              \n
**   alloc memory                                                             \n
**   loop running IO until secs expire                                        \n
**   print IO stats                                                           \n
**   cleanup
*******************************************************************************/
int main(int argc, char **argv)
{
    struct timeval  start, delta;
    long int        mil         = 1000000;
    float           esecs       = 0;
    uint8_t        *rbuf        = NULL;
    uint8_t        *wbuf        = NULL;
    OP_t           *op          = NULL;
    char           *dev         = NULL;
    char            FF          = 0xFF;
    char            c           = '\0';
    chunk_ext_arg_t ext         = 0;
    int             flags       = 0;
    int             i, rc       = 0;
    int             id          = 0;
    char           *_secs       = NULL;
    char           *_QD         = NULL;
    char           *_RD         = NULL;
    char           *_nblocks    = NULL;
    uint32_t        plun        = 0;
    uint32_t        nsecs       = 4;
    uint32_t        QD          = 256;
    uint32_t        nRD         = 100;
    uint32_t        RD          = 0;
    uint32_t        WR          = 0;
    uint32_t        intrp_thds  = 0;
    int             rtag        = 0;
    int             htag        = 0;
    uint32_t        lba         = 0;
    size_t          nblks       = 0;
    uint32_t        nblocks     = 1;
    uint32_t        cnt         = 0;
    uint32_t        tmiss       = 0;
    uint64_t        status      = 0;
    uint32_t        TI          = TIME_INTERVAL;
    uint32_t        N           = 0;
    uint32_t        TIME        = 1;
    uint32_t        COMP        = 0;
    uint32_t        miss        = 0;
    uint64_t        tlat        = 0;
    double          ns_per_tick = 0;

    /*--------------------------------------------------------------------------
     * process and verify input parms
     *------------------------------------------------------------------------*/
    while (FF != (c=getopt(argc, argv, "d:r:q:n:s:phi")))
    {
        switch (c)
        {
            case 'd': dev        = optarg; break;
            case 'r': _RD        = optarg; break;
            case 'q': _QD        = optarg; break;
            case 'n': _nblocks   = optarg; break;
            case 's': _secs      = optarg; break;
            case 'p': plun       = 1;      break;
            case 'i': intrp_thds = 1;      break;
            case 'h':
            case '?': usage();             break;
        }
    }
    if (_secs)      nsecs   = atoi(_secs);
    if (_QD)        QD      = atoi(_QD);
    if (_nblocks)   nblocks = atoi(_nblocks);
    if (_RD)        nRD     = atoi(_RD);

    if (QD > _8K)   QD      = _8K;
    if (nRD > 100)  nRD     = 100;

    if (!plun && nblocks > 1)
    {
        printf("error: <-n %d> can only be used with a physical lun\n",nblocks);
        usage();
    }
    if (dev == NULL) usage();

    srand48(time(0));
    ns_per_tick = time_per_tick(1000, 100);

    N    = QD;
    COMP = QD < 8 ? 1 : QD/8;

    /*--------------------------------------------------------------------------
     * open device and set lun size
     *------------------------------------------------------------------------*/
    rc = cblk_init(NULL,0);
    if (rc)
    {
        fprintf(stderr,"cblk_init failed with rc = %d and errno = %d\n",
                rc,errno);
        exit(1);
    }
    if (!plun)       flags  = CBLK_OPN_VIRT_LUN;
    if (!intrp_thds) flags |= CBLK_OPN_NO_INTRP_THREADS;
    id = cblk_open(dev, QD, O_RDWR, ext, flags);
    if (id == NULL_CHUNK_ID)
    {
        if      (ENOSPC == errno) fprintf(stderr,"cblk_open: ENOSPC\n");
        else if (ENODEV == errno) fprintf(stderr,"cblk_open: ENODEV\n");
        else                      fprintf(stderr,"cblk_open: errno:%d\n",errno);
        cblk_term(NULL,0);
        exit(errno);
    }

    rc = cblk_get_lun_size(id, &nblks, 0);
    if (rc)
    {
        fprintf(stderr, "cblk_get_lun_size failed: errno: %d\n", errno);
        exit(errno);
    }
    if (!plun)
    {
        nblks = nblks > SET_NBLKS ? SET_NBLKS : nblks;
        rc = cblk_set_size(id, nblks, 0);
        if (rc)
        {
            fprintf(stderr, "cblk_set_size failed, errno: %d\n", errno);
            exit(errno);
        }
    }
    lba = lrand48() % nblks;

    /*--------------------------------------------------------------------------
     * alloc data for IO
     *------------------------------------------------------------------------*/
    op = malloc(QD*sizeof(OP_t));
    if ((rc=posix_memalign((void**)&rbuf, _4K, _4K*nblocks)))
    {
        fprintf(stderr,"posix_memalign failed, size=%d, rc=%d\n",
                _4K*nblocks, rc);
        cblk_close(id,0);
        cblk_term(NULL,0);
        exit(0);
    }
    if ((rc=posix_memalign((void**)&wbuf, _4K, _4K*nblocks)))
    {
        fprintf(stderr,"posix_memalign failed, size=%d, rc=%d\n",
                _4K*nblocks, rc);
        cblk_close(id,0);
        cblk_term(NULL,0);
        exit(0);
    }
    memset(wbuf,0x79,_4K*nblocks);
    memset(op,  0,   QD*sizeof(OP_t));

    /*--------------------------------------------------------------------------
     * loop running IO until secs expire
     *------------------------------------------------------------------------*/
    gettimeofday(&start, NULL);

    do
    {
        /* setup #read ops and #write ops to send before completing ops */
        if (!RD && !WR) {RD=nRD; WR=100-RD;}

        /*----------------------------------------------------------------------
         * send up to RD reads, as long as the queuedepth N is not max
         *--------------------------------------------------------------------*/
        while (TIME && RD && N)
        {
            rc = cblk_aread(id, rbuf, lba, nblocks, &rtag, NULL,
                            CBLK_ARW_WAIT_CMD_FLAGS);
            if (0 == rc) {OP_BEG(rtag); --RD; --N; BMP_LBA();}
            else if (EBUSY == errno) {break;}
            else                     {io_error(id,errno);}
        }
        /*----------------------------------------------------------------------
         * send up to WR writes, as long as the queuedepth N is not max
         *--------------------------------------------------------------------*/
        while (TIME && WR && N)
        {
            rc = cblk_awrite(id, wbuf, lba, nblocks, &rtag, NULL,
                            CBLK_ARW_WAIT_CMD_FLAGS);
            if (0 == rc) {OP_BEG(rtag); --WR; --N; BMP_LBA();}
            else if (EBUSY == errno) {break;}
            else                     {io_error(id,errno);}
        }

        /*----------------------------------------------------------------------
         * complete cmds
         *--------------------------------------------------------------------*/
        for (i=0; i<QD && N<COMP; i++, htag++)
        {
            if (intrp_thds)
            {
                rc = cblk_aresult(id, &htag, &status, CBLK_ARESULT_BLOCKING |
                                                      CBLK_ARESULT_NEXT_TAG);
                if (rc != nblocks) {io_error(id,errno);}
                OP_END(htag); ++cnt; ++N;
                continue;
            }

            if (htag>=QD) htag=0;
            if (!op[htag].iss) {continue;}
            rc = cblk_aresult(id, &htag, &status, 0);
            if (rc == 0)
            {
                if (QD==1 && ++miss==1) {usleep(80);}
                ++tmiss; continue;
            }
            else if (rc < 0)
                {io_error(id,errno);}

            OP_END(htag); ++cnt; ++N; miss=0;
        }

        /*----------------------------------------------------------------------
         * at an interval which does not impact performance, check if secs
         * have expired, and randomize lba
         *--------------------------------------------------------------------*/
        if (cnt > TI)
        {
            TI += TIME_INTERVAL;
            gettimeofday(&delta, NULL);
            if (delta.tv_sec - start.tv_sec >= nsecs) {TIME=0; COMP=QD;}
            lba = lrand48() % nblks;
        }
    }
    while (TIME || QD-N);

    /*--------------------------------------------------------------------------
     * print IO stats
     *------------------------------------------------------------------------*/
    gettimeofday(&delta, NULL);
    esecs = ((float)((delta.tv_sec*mil + delta.tv_usec) -
                     (start.tv_sec*mil + start.tv_usec))) / (float)mil;
    printf("d:%s r:%d q:%d s:%d p:%d n:%d i:%d miss:%d lat:%d mbps:%d iops:%d",
            dev, nRD, QD, nsecs, plun, nblocks, intrp_thds, tmiss,
            (uint32_t)((tlat*ns_per_tick)/cnt/1000),
            (uint32_t)((float)((cnt*nblocks*4)/1024)/esecs),
            (uint32_t)((float)(cnt/esecs)));
    if (plun && nblocks > 1)
        printf(" 4k-iops:%d", (uint32_t)((float)(cnt*nblocks)/esecs));
    printf("\n");

    /*--------------------------------------------------------------------------
     * cleanup
     *------------------------------------------------------------------------*/
    free(op);
    free(rbuf);
    free(wbuf);
    cblk_close(id,0);
    cblk_term(NULL,0);
    return 0;
}
Beispiel #4
0
EA *ea_new(const char *path, uint64_t bsize, int basyncs,
           uint64_t *size, uint64_t *bcount, uint64_t vlun)
{
    int             rc    = 0;
    size_t          plen  = 0;
    uint8_t        *store = NULL;
    EA             *ea    = NULL;
    chunk_id_t      chkid = NULL_CHUNK_ID;
    chunk_ext_arg_t ext   = 0;

    if (!(fetch_and_or(&cflsh_blk_lib_init,1)))
    {
        // We need to call cblk_init once before
        // we use any other cblk_ interfaces
        rc = cblk_init(NULL,0);
        if (rc)
        {
            KV_TRC_FFDC(pAT, "cblk_init failed path %s bsize %"PRIu64" "
                             "size %"PRIu64" bcount %"PRIu64", errno = %d",
                             path, bsize, *size, *bcount, errno);
            goto error_exit;
        }
    }

    ea = am_malloc(sizeof(EA));
    if (NULL == ea)
    {
        KV_TRC_FFDC(pAT, "Out of memory path %s bsize %"PRIu64" size %"PRIu64" "
                         "bcount %"PRIu64", errno = %d",
                         path, bsize, *size, *bcount, errno);
        goto error_exit;
    }

    // We need to check the path parameter to see if
    // we are going to use memory or a file/capi
    // device (to be determined by the block layer)
    if ( (NULL == path) || (strlen(path) == 0) )
    {
        KV_TRC(pAT, "EA_STORE_TYPE_MEMORY");
        // Using memory for store
        ea->st_type = EA_STORE_TYPE_MEMORY;

        store = malloc(*size);
        if (NULL == store)
        {
            errno = ENOMEM;
            KV_TRC_FFDC(pAT, "Out of memory for store path %s bsize %"PRIu64" "
                             "size %"PRIu64" bcount %"PRIu64", errno = %d",
                             path, bsize, *size, *bcount, errno);
            goto error_exit;
        }

        *bcount = ((*size) / bsize);
        ea->st_memory = store;
    }
    else
    {
        KV_TRC(pAT, "EA_STORE_TYPE_FILE(%s)", path);

        // Using a file.  We don't care if it's an actual
        // file or a CAPI device, we let block layer
        // decide and we just use the chunk ID that is
        // passed back from the cblk_open call.
        ea->st_type = EA_STORE_TYPE_FILE;

        // Check to see if we need to create the store on a
        // physical or virtual LUN.  Previously, in GA1,
        // we keyed off the size and if it was 0, then we
        // asked for the LUN to be physical.  Now, the user
        // can specify with a flag.
        if ( vlun == 0 )
        {
            KV_TRC(pAT, "cblk_open PHYSICAL LUN: %s", path);
            chkid = cblk_open(path, basyncs, O_RDWR, ext,
                              CBLK_OPN_NO_INTRP_THREADS);

            if (NULL_CHUNK_ID == chkid)
            {
                printf("cblk_open physical lun failed\n");
                KV_TRC_FFDC(pAT, "cblk_open phys lun failed path:%s bsize:%ld "
                                 "size:%ld bcount:%ld, errno:%d",
                                 path, bsize, *size, *bcount, errno);
                goto error_exit;
            }

            rc = cblk_get_size(chkid, (size_t *)bcount, 0);
            if ( (rc != 0) || (*bcount == 0) )
            {
                // An error was encountered, close the chunk
                cblk_close(chkid, 0);
                chkid = NULL_CHUNK_ID;
                KV_TRC_FFDC(pAT, "cblk_get_size failed path %s bsize %"PRIu64" "
                                 "size %"PRIu64" bcount %"PRIu64", errno = %d",
                                 path, bsize, *size, *bcount, errno);
                goto error_exit;
            }

            // Set the size to be returned
            *size = *bcount * bsize;
        }
        else
        {
            KV_TRC(pAT, "cblk_open VIRTUAL LUN: %s", path);
            chkid = cblk_open(path, basyncs, O_RDWR, ext,
                              CBLK_OPN_VIRT_LUN|CBLK_OPN_NO_INTRP_THREADS);

            if (NULL_CHUNK_ID == chkid)
            {
                printf("cblk_open virtual lun failed\n");
                KV_TRC_FFDC(pAT, "cblk_open virt lun failed path:%s bsize:%ld "
                                 "size:%ld bcount:%ld, errno:%d",
                                 path, bsize, *size, *bcount, errno);
                goto error_exit;
            }

            // A specific size was passed in so we try to set the
            // size of the chunk.
            *bcount = *size / bsize;
            rc = cblk_set_size(chkid, (size_t)*bcount, 0);
            if ( rc != 0 )
            {
                printf("cblk_set_size failed for %ld\n", *bcount);
                // An error was encountered, close the chunk
                cblk_close(chkid, 0);
                chkid = NULL_CHUNK_ID;
                KV_TRC_FFDC(pAT, "cblk_set_size failed path %s bsize %"PRIu64" "
                                 "size %"PRIu64" bcount %"PRIu64", errno = %d",
                                 path, bsize, *size, *bcount, errno);
                goto error_exit;
            }
        }

        // Save off the chunk ID and the device name
        ea->st_flash  = chkid;
        plen          = strlen(path) + 1;
        ea->st_device = (char *)am_malloc(plen);
        if (!ea->st_device)
        {
            cblk_close(chkid, 0);
            KV_TRC_FFDC(pAT, "MALLOC st_device failed (%s) plen=%ld errno:%d",
                        path, plen, errno);
            goto error_exit;
        }

        memset(ea->st_device, 0, plen);
        strncpy(ea->st_device, path, plen);
    }

    // Fill in the EA struct
    pthread_rwlock_init(&(ea->ea_rwlock), NULL);
    ea->bsize  = bsize;
    ea->bcount = *bcount;
    ea->size   = *size;

    KV_TRC(pAT, "path %s bsize %"PRIu64" size %"PRIu64" bcount %"PRIu64"",
           path, bsize, *size, *bcount);
    goto done;

error_exit:
    am_free(ea);
    ea = NULL;
    if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=ENOSPC;}

done:
    return ea;
}
Beispiel #5
0
/**
********************************************************************************
** \brief main
** \details
**   process input parms                                                      \n
**   open device                                                              \n
**   alloc memory                                                             \n
**   loop running IO until secs expire                                        \n
**   print IO stats                                                           \n
**   cleanup
*******************************************************************************/
int main(int argc, char **argv)
{
    struct timeval  start, delta;
    long int        mil        = 1000000;
    float           esecs      = 0;
    uint8_t        **rbuf      = NULL;
    uint8_t        **wbuf      = NULL;
    int            *tags       = NULL;
    char           *dev        = NULL;
    char            FF         = 0xFF;
    char            c          = '\0';
    chunk_ext_arg_t ext        = 0;
    int             flags      = 0;
    int             rc         = 0;
    int             id         = 0;
    char           *_secs      = NULL;
    char           *_QD        = NULL;
    char           *_RD        = NULL;
    char           *_nblocks   = NULL;
    uint32_t        plun       = 0;
    uint32_t        nsecs      = 4;
    uint32_t        QD         = 500;
    uint32_t        nRD        = 100;
    uint32_t        RD         = 0;
    uint32_t        WR         = 0;
    uint32_t        intrp_thds = 0;
    int             tag        = 0;
    int             rtag       = 0;
    uint32_t        lba        = 0;
    size_t          nblks      = 0;
    uint32_t        nblocks    = 1;
    uint32_t        cnt        = 0;
    uint32_t        pollN      = 0;
    uint64_t        status     = 0;
    uint32_t        TI         = TIME_INTERVAL;
    uint32_t        N          = 0;
    uint32_t        TIME       = 1;
    uint32_t        COMP       = 0;

    /*--------------------------------------------------------------------------
     * process and verify input parms
     *------------------------------------------------------------------------*/
    while (FF != (c=getopt(argc, argv, "d:r:q:n:s:phi")))
    {
        switch (c)
        {
            case 'd': dev        = optarg; break;
            case 'r': _RD        = optarg; break;
            case 'q': _QD        = optarg; break;
            case 'n': _nblocks   = optarg; break;
            case 's': _secs      = optarg; break;
            case 'p': plun       = 1;      break;
            case 'i': intrp_thds = 1;      break;
            case 'h':
            case '?': usage();             break;
        }
    }
    if (_secs)      nsecs   = atoi(_secs);
    if (_QD)        QD      = atoi(_QD);
    if (_nblocks)   nblocks = atoi(_nblocks);
    if (_RD)        nRD     = atoi(_RD);

    if (QD > _8K)   QD      = _8K;
    if (nRD > 100)  nRD     = 100;
    if (!plun)      nblocks = 1;
    if (dev == NULL) usage();

    srand48(time(0));

    N = QD;
    COMP  = QD < 5 ? 1 : QD/5;

    /*--------------------------------------------------------------------------
     * open device and set lun size
     *------------------------------------------------------------------------*/
    rc = cblk_init(NULL,0);
    if (rc)
    {
        fprintf(stderr,"cblk_init failed with rc = %d and errno = %d\n",
                rc,errno);
        exit(1);
    }
    if (!plun)       flags  = CBLK_OPN_VIRT_LUN;
    if (!intrp_thds) flags |= CBLK_OPN_NO_INTRP_THREADS;
    id = cblk_open(dev, QD, O_RDWR, ext, flags);
    if (id == NULL_CHUNK_ID)
    {
        if      (ENOSPC == errno) fprintf(stderr,"cblk_open: ENOSPC\n");
        else if (ENODEV == errno) fprintf(stderr,"cblk_open: ENODEV\n");
        else                      fprintf(stderr,"cblk_open: errno:%d\n",errno);
        cblk_term(NULL,0);
        exit(errno);
    }

    rc = cblk_get_lun_size(id, &nblks, 0);
    if (rc)
    {
        fprintf(stderr, "cblk_get_lun_size failed: errno: %d\n", errno);
        exit(errno);
    }
    if (!plun)
    {
        nblks = nblks > SET_NBLKS ? SET_NBLKS : nblks;
        rc = cblk_set_size(id, nblks, 0);
        if (rc)
        {
            fprintf(stderr, "cblk_set_size failed, errno: %d\n", errno);
            exit(errno);
        }
    }

    /*--------------------------------------------------------------------------
     * alloc data for IO
     *------------------------------------------------------------------------*/
    tags = malloc(QD*sizeof(int));
    rbuf = malloc(QD*sizeof(uint8_t*));
    wbuf = malloc(QD*sizeof(uint8_t*));

    for (tag=0; tag<QD; tag++)
    {
        if ((rc=posix_memalign((void**)&(rbuf[tag]), _4K, _4K*nblocks)))
        {
            fprintf(stderr,"posix_memalign failed, size=%d, rc=%d\n",
                    _4K*nblocks, rc);
            cblk_close(id,0);
            cblk_term(NULL,0);
            exit(0);
        }
        if ((rc=posix_memalign((void**)&(wbuf[tag]), _4K, _4K*nblocks)))
        {
            fprintf(stderr,"posix_memalign failed, size=%d, rc=%d\n",
                    _4K*nblocks, rc);
            cblk_close(id,0);
            cblk_term(NULL,0);
            exit(0);
        }
        memset(wbuf[tag],0x79,_4K*nblocks);
        tags[tag] = tag;
    }

    /*--------------------------------------------------------------------------
     * loop running IO until secs expire
     *------------------------------------------------------------------------*/
    gettimeofday(&start, NULL);

    do
    {
        /* setup #read ops and #write ops to send before completing ops */
        if (!RD && !WR) {RD=nRD; WR=100-RD;}

        /*----------------------------------------------------------------------
         * send up to RD reads, as long as the queuedepth N is not max
         *--------------------------------------------------------------------*/
        while (TIME && RD && N)
        {
            tag = tags[--N];
            rc = cblk_aread(id, rbuf[tag], lba, nblocks, &tag, NULL,
                            CBLK_ARW_WAIT_CMD_FLAGS | CBLK_ARW_USER_TAG_FLAG);
            if      (0 == rc) {--RD; lba+=2; if (lba >= nblks) lba=cnt%2;}
            else if (EBUSY == errno) {++N; usleep(USLEEP); continue;}
            else                     {io_error(id,errno);}
        }
        /*----------------------------------------------------------------------
         * send up to WR writes, as long as the queuedepth N is not max
         *--------------------------------------------------------------------*/
        while (TIME && WR && N)
        {
            tag = tags[--N];
            rc = cblk_awrite(id, wbuf[tag], lba, nblocks, &tag, NULL,
                             CBLK_ARW_WAIT_CMD_FLAGS | CBLK_ARW_USER_TAG_FLAG);
            if      (0 == rc) {--WR; lba+=2; if (lba >= nblks) lba=cnt%2;}
            else if (EBUSY == errno) {++N; usleep(USLEEP); continue;}
            else                     {io_error(id,errno);}
        }

        /* if the queuedepth is 1, don't immediately pound aresult */
        if (QD==1) usleep(USLEEP);

        /*----------------------------------------------------------------------
         * complete cmds until queue depth is QD-COMP
         *--------------------------------------------------------------------*/
        while (N < COMP)
        {
            rtag=0;
            rc = cblk_aresult(id, &rtag, &status, CBLK_ARESULT_BLOCKING|
                                                  CBLK_ARESULT_NEXT_TAG);
            if      (rc == 0) {++pollN; usleep(USLEEP); continue;}
            else if (rc < 0)  {io_error(id,errno);}
            ++cnt; tags[N++] = rtag;
        }

        /*----------------------------------------------------------------------
         * at an interval which does not impact performance, check if secs
         * have expired, and randomize lba
         *--------------------------------------------------------------------*/
        if (cnt > TI)
        {
            TI += TIME_INTERVAL;
            gettimeofday(&delta, NULL);
            if (delta.tv_sec - start.tv_sec >= nsecs) {TIME=0; COMP = QD;}
            lba = lrand48() % TIME_INTERVAL;
        }
    }
    while (TIME || QD-N);

    /*--------------------------------------------------------------------------
     * print IO stats
     *------------------------------------------------------------------------*/
    gettimeofday(&delta, NULL);
    esecs = ((float)((delta.tv_sec*mil + delta.tv_usec) -
                     (start.tv_sec*mil + start.tv_usec))) / (float)mil;
    printf("d:%s r:%d q:%d s:%d p:%d n:%d i:%d pollN:%d mbps:%d iops:%d",
            dev, nRD, QD, nsecs, plun, nblocks, intrp_thds, pollN,
            (uint32_t)((float)((cnt*nblocks*4)/1024)/esecs),
            (uint32_t)((float)(cnt/esecs)));
    if (plun && nblocks > 1)
        printf(" 4k-iops:%d", (uint32_t)((float)(cnt*nblocks)/esecs));
    printf("\n");

    /*--------------------------------------------------------------------------
     * cleanup
     *------------------------------------------------------------------------*/
    for (cnt=0; cnt<QD; cnt++)
    {
        free(rbuf[cnt]);
        free(wbuf[cnt]);
    }
    free(rbuf);
    free(wbuf);
    cblk_close(id,0);
    cblk_term(NULL,0);
    return 0;
}