Example #1
0
void
WriteDataPat_r10b::SendToIOSQ(SharedIOSQPtr iosq, SharedIOCQPtr iocq,
    SharedWritePtr writeCmd, string qualifier)
{
    uint32_t numCE;
    uint32_t ceRemain;
    uint32_t numReaped;
    uint32_t isrCount;
    uint16_t uniqueId;


    LOG_NRM("Send the cmd to hdw via %s IOSQ", qualifier.c_str());
    iosq->Send(writeCmd, uniqueId);
    iosq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iosq", qualifier),
        "Just B4 ringing SQ doorbell, dump entire IOSQ contents");
    iosq->Ring();


    LOG_NRM("Wait for the CE to arrive in IOCQ");
    if (iocq->ReapInquiryWaitSpecify(CALC_TIMEOUT_ms(1), 1, numCE, isrCount)
        == false) {

        iocq->Dump(
            FileSystem::PrepDumpFile(mGrpName, mTestName, "iocq", qualifier),
            "Unable to see any CE's in IOCQ, dump entire CQ contents");
        throw FrmwkEx(HERE, "Unable to see completion of cmd");
    } else if (numCE != 1) {
        iocq->Dump(
            FileSystem::PrepDumpFile(mGrpName, mTestName, "iocq", qualifier),
            "Unable to see any CE's in IOCQ, dump entire CQ contents");
        throw FrmwkEx(HERE,
            "The IOCQ should only have 1 CE as a result of a cmd");
    }
    iocq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iocq", qualifier),
        "Just B4 reaping IOCQ, dump entire CQ contents");


    LOG_NRM("The CQ's metrics B4 reaping holds head_ptr needed");
    struct nvme_gen_cq iocqMetrics = iocq->GetQMetrics();
    KernelAPI::LogCQMetrics(iocqMetrics);

    LOG_NRM("Reaping CE from IOCQ, requires memory to hold reaped CE");
    SharedMemBufferPtr ceMemIOCQ = SharedMemBufferPtr(new MemBuffer());
    if ((numReaped = iocq->Reap(ceRemain, ceMemIOCQ, isrCount, numCE, true))
        != 1) {

        throw FrmwkEx(HERE, "Verified there was 1 CE, but reaping produced %d",
            numReaped);
    }
    LOG_NRM("The reaped CE is...");
    iocq->LogCE(iocqMetrics.head_ptr);

    union CE ce = iocq->PeekCE(iocqMetrics.head_ptr);
    ProcessCE::Validate(ce);  // throws upon error
}
Example #2
0
void
IOQRollChkDiff_r10b::ReapAndVerifyCE(SharedIOCQPtr iocq, uint32_t expectedVal)
{
    uint32_t numCE;
    uint32_t ceRemain;
    uint32_t numReaped;
    uint32_t isrCount;

    LOG_NRM("Wait for the CE to arrive in IOCQ %d", iocq->GetQId());
    if (iocq->ReapInquiryWaitSpecify(DEFAULT_CMD_WAIT_ms, 1, numCE, isrCount)
        == false) {

        iocq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iocq",
            "reapInq"),
            "Unable to see any CE's in IOCQ, dump entire CQ contents");
        throw FrmwkEx(HERE, "Unable to see completion of cmd");
    } else if (numCE != 1) {
        throw FrmwkEx(HERE, "The IOCQ should only have 1 CE as a result of a cmd");
    }

    LOG_NRM("The CQ's metrics before reaping holds head_ptr");
    struct nvme_gen_cq iocqMetrics = iocq->GetQMetrics();

    LOG_NRM("Reaping CE from IOCQ, requires memory to hold reaped CE");
    SharedMemBufferPtr ceMemIOCQ = SharedMemBufferPtr(new MemBuffer());
    if ((numReaped = iocq->Reap(ceRemain, ceMemIOCQ, isrCount, numCE, true))
        != 1) {

        throw FrmwkEx(HERE, 
            "Verified there was 1 CE, but reaping produced %d", numReaped);
    }

    union CE ce = iocq->PeekCE(iocqMetrics.head_ptr);
    ProcessCE::Validate(ce, CESTAT_SUCCESS);  // throws upon error

    if (ce.n.SQID != IOQ_ID) {
        iocq->Dump(
            FileSystem::PrepDumpFile(mGrpName, mTestName, "iocq", "CE.SQID"),
            "CE SQ ID Inconsistent");
        throw FrmwkEx(HERE, "Expected CE.SQID = 0x%04X in IOCQ CE but actual "
            "CE.SQID  = 0x%04X", IOQ_ID, ce.n.SQID);
    }

    if (ce.n.SQHD != expectedVal) {
        iocq->Dump(
            FileSystem::PrepDumpFile(mGrpName, mTestName, "iocq", "CE.SQHD"),
            "CE SQ Head Pointer Inconsistent");
        throw FrmwkEx(HERE, "Expected CE.SQHD = 0x%04X in IOCQ CE but actual "
            "CE.SQHD  = 0x%04X", expectedVal, ce.n.SQHD);
    }
}
Example #3
0
void
CIDAcceptedIOSQ_r10b::ReapVerifyCID(SharedIOSQPtr iosq, SharedIOCQPtr iocq,
    uint16_t expCID)
{
    uint32_t isrCount;
    uint32_t ceRemain;
    uint32_t numReaped;
    uint32_t numCE;

    if (iocq->ReapInquiryWaitSpecify(CALC_TIMEOUT_ms(1), 1, numCE,
        isrCount) == false) {
        iocq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iocq.fail"),
            "Dump Entire IOCQ");
        iosq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iosq.fail"),
            "Dump Entire IOSQ");
        throw FrmwkEx(HERE, "Unable to see CEs for issued cmd");
    }

    SharedMemBufferPtr ceMem = SharedMemBufferPtr(new MemBuffer());
    if ((numReaped = iocq->Reap(ceRemain, ceMem, isrCount, numCE, true)) != 1) {
        iocq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iocq.fail"),
            "Dump Entire IOCQ");
        iosq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iosq.fail"),
            "Dump Entire IOSQ");
        throw FrmwkEx(HERE, "Unable to reap on IOCQ");
    }

    union CE *ce = (union CE *)ceMem->GetBuffer();
    ProcessCE::Validate(*ce);  // throws upon error

    if (ce->n.CID != expCID) {
        iocq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iocq.fail"),
            "Dump Entire IOCQ");
        iosq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iosq.fail"),
            "Dump Entire IOSQ");
        throw FrmwkEx(HERE, "Received CID %d but expected %d", ce->n.CID,
            expCID);
    }

    if (ce->n.SQID != iosq->GetQId()) {
        iocq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iocq.fail"),
            "Dump Entire IOCQ");
        iosq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iosq.fail"),
            "Dump Entire IOSQ");
        throw FrmwkEx(HERE, "Rx'd SDID %d but expt'd %d", ce->n.SQID,
            iosq->GetQId());
    }

}
Example #4
0
void
QIDVariations_r10b::ReapVerifyOnCQ(SharedIOCQPtr iocq, SharedIOSQPtr iosq)
{
    uint32_t numCE;
    uint32_t ceRemain;
    uint32_t numReaped;
    uint32_t isrCount;

    LOG_NRM("Reap and verify CE.");
    SharedMemBufferPtr ceMemIOCQ = SharedMemBufferPtr(new MemBuffer());
    for (uint32_t nCmds = 1; nCmds < iosq->GetNumEntries(); nCmds++) {
        LOG_NRM("Wait for the CE to arrive in IOCQ");
        if (iocq->ReapInquiryWaitSpecify(CALC_TIMEOUT_ms(1), 1, numCE,
            isrCount) == false) {
            iocq->Dump(
                FileSystem::PrepDumpFile(mGrpName, mTestName, "iocq", "reapInq"),
                "Unable to see any CE's in IOCQ, dump entire CQ contents");
            throw FrmwkEx(HERE, "Unable to see completion of cmd");
        }

        LOG_NRM("The CQ's metrics B4 reaping holds head_ptr needed");
        struct nvme_gen_cq iocqMetrics = iocq->GetQMetrics();
        KernelAPI::LogCQMetrics(iocqMetrics);

        if ((numReaped = iocq->Reap(ceRemain, ceMemIOCQ, isrCount, 1, true))
            != 1) {
            iocq->Dump(
                FileSystem::PrepDumpFile(mGrpName, mTestName, "iocq", "reap"),
                "Unable to see any CE's in IOCQ, dump entire CQ contents");
            throw FrmwkEx(HERE, "Verified there was 1 CE, but reaping failed");
        }

        LOG_NRM("The reaped CE is...");
        iocq->LogCE(iocqMetrics.head_ptr);

        union CE ce = iocq->PeekCE(iocqMetrics.head_ptr);
        ProcessCE::Validate(ce);  // throws upon error

        if (ce.n.SQID != iosq->GetQId()) {
            iocq->Dump(
                FileSystem::PrepDumpFile(mGrpName, mTestName, "iocq", "sqId"),
                "Wrong SQID in the CE of IOCQ, dump entire CQ contents");
            throw FrmwkEx(HERE, "Invalid SQID %d in CE, expected SQID", ce.n.SQID,
                iosq->GetQId());
        }
    }
}
Example #5
0
void
CIDAcceptedIOSQ_r10b::RunCoreTest()
{
    /** \verbatim
     * Assumptions:
     * None.
     * \endverbatim
     */
    if (gCtrlrConfig->SetState(ST_DISABLE_COMPLETELY) == false)
        throw FrmwkEx(HERE);

    LOG_NRM("Create ACQ and ASQ objects which have test life time");
    SharedACQPtr acq = CAST_TO_ACQ(SharedACQPtr(new ACQ(gDutFd)))
    acq->Init(5);
    SharedASQPtr asq = CAST_TO_ASQ(SharedASQPtr(new ASQ(gDutFd)))
    asq->Init(5);

    IRQ::SetAnySchemeSpecifyNum(2);     // throws upon error

    gCtrlrConfig->SetCSS(CtrlrConfig::CSS_NVM_CMDSET);
    if (gCtrlrConfig->SetState(ST_ENABLE) == false)
        throw FrmwkEx(HERE);

    vector<SharedIOSQPtr> iosqs;
    SharedIOCQPtr iocq;
    InitTstRsrcs(asq, acq, iosqs, iocq);
    SharedWritePtr writeCmd = CreateWriteCmd();

    LOG_NRM("Learn initial unique command id assigned by dnvme");
    uint16_t curCID;
    vector <SharedIOSQPtr>::iterator iosq;
    for (iosq = iosqs.begin(); iosq != iosqs.end(); iosq++)
        (*iosq)->Send(writeCmd, curCID);

    uint16_t prevCID = curCID;
    for (uint32_t nCmds = 0; nCmds < MAX_CMDS; nCmds++) {
        for (iosq = iosqs.begin(); iosq != iosqs.end(); iosq++) {
            LOG_NRM("(nCmds, curCID, prevCID, SQ ID) = (%d, %d, %d, %d)",
                nCmds, curCID, prevCID, (*iosq)->GetQId());
            (*iosq)->Ring();
            ReapVerifyCID(*iosq, iocq, prevCID);

            (*iosq)->Send(writeCmd, curCID);
            if (curCID != (uint16_t)(prevCID + 1)) {
                (*iosq)->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName,
                    "iosq.fail." + (*iosq)->GetQId()), "Dump Entire IOSQ");
                iocq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName,
                    "iocq.fail." + iocq->GetQId()), "Dump Entire IOCQ");
                throw FrmwkEx(HERE, "curCID(%d) != (prevCID + 1)(%d)", curCID,
                    (prevCID + 1));
            }
        }
        prevCID = curCID;
    }
}
Example #6
0
void
InvalidNamspc_r10b::RunCoreTest()
{
    /** \verbatim
     * Assumptions:
     * 1) Test CreateResources_r10b has run prior.
     * \endverbatim
     */
    uint64_t inc, i;
    string work;
    uint32_t numCE;
    uint32_t isrCountB4;

    LOG_NRM("Lookup objs which were created in a prior test within group");
    SharedIOSQPtr iosq = CAST_TO_IOSQ(gRsrcMngr->GetObj(IOSQ_GROUP_ID));
    SharedIOCQPtr iocq = CAST_TO_IOCQ(gRsrcMngr->GetObj(IOCQ_GROUP_ID));

    if ((numCE = iocq->ReapInquiry(isrCountB4, true)) != 0) {
        iocq->Dump(
            FileSystem::PrepDumpFile(mGrpName, mTestName, "iocq",
                                     "notEmpty"), "Test assumption have not been met");
        throw FrmwkEx(HERE, "Require 0 CE's within CQ %d, not upheld, found %d",
                      iocq->GetQId(), numCE);
    }

    LOG_NRM("Setup write cmd's values that won't change per namspc");
    SharedMemBufferPtr writeMem = SharedMemBufferPtr(new MemBuffer());
    uint64_t lbaDataSize = 512;
    writeMem->Init(lbaDataSize);

    SharedWritePtr writeCmd = SharedWritePtr(new Write());
    send_64b_bitmask prpBitmask = (send_64b_bitmask)
                                  (MASK_PRP1_PAGE | MASK_PRP2_PAGE | MASK_PRP2_LIST);
    writeCmd->SetPrpBuffer(prpBitmask, writeMem);
    writeCmd->SetNLB(0);    // convert to 0-based value
    writeCmd->SetSLBA(0);

    LOG_NRM("For all namspc's issue cmd to an illegal namspc");
    ConstSharedIdentifyPtr idCtrlrStruct = gInformative->GetIdentifyCmdCtrlr();
    uint32_t nn = (uint32_t)idCtrlrStruct->GetValue(IDCTRLRCAP_NN);
    if (nn == 0 ) {
        throw FrmwkEx(HERE, "Required to support >= 1 namespace");
    }

    for (i = (nn + 1), inc = 1; i <= 0xffffffff; i += (2 * inc), inc += 1327) {

        LOG_NRM("Issue flush cmd with illegal namspc ID=%llu",
                (unsigned long long)i);
        writeCmd->SetNSID(i);
        work = str(boost::format("namspc%d") % i);
        SendCmdToHdw(iosq, iocq, writeCmd, work);
    }
}
void
MaxIOQMSIXManyTo1_r10b::SendCmdAndReap(SharedIOSQPtr iosq, SharedIOCQPtr iocq,
    SharedCmdPtr cmd, uint32_t anticipatedIrqs)
{
    uint16_t uniqueId;
    uint32_t numCE;
    string work;
    uint32_t isrCount;

    LOG_NRM("Send the cmd to hdw via IOSQ #%d", iosq->GetQId());
    iosq->Send(cmd, uniqueId);
    work = str(boost::format("ioqId.%d.%s") % iosq->GetQId() %
        cmd->GetName().c_str());
    iosq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iosq", work),
        "Just B4 ringing doorbell, dump IOSQ");
    iosq->Ring();

    LOG_NRM("Wait for the CE to arrive in CQ %d", iocq->GetQId());
    if (iocq->ReapInquiryWaitSpecify(DEFAULT_CMD_WAIT_ms, 1, numCE, isrCount)
        == false) {
        iocq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iocq.fail",
            work), "Dump Entire IOCQ");
        iosq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iosq.fail",
            work), "Dump Entire IOSQ");
        throw FrmwkEx(HERE, "Unable to see CEs for issued cmd");
    }

    work = str(boost::format("iocq.%d") % uniqueId);
    IO::ReapCE(iocq, 1, isrCount, mGrpName, mTestName, work);

    if (isrCount != anticipatedIrqs) {
        iocq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iocq.fail",
            work), "Dump Entire IOCQ");
        iosq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iosq.fail",
            work), "Dump Entire IOSQ");
        throw FrmwkEx(HERE, "Anticipated ISRs #%d but fired #%d",
            anticipatedIrqs, isrCount);
    }
}
Example #8
0
void
IOQRollChkDiff_r10b::VerifyQPointers(SharedIOSQPtr iosq, SharedIOCQPtr iocq)
{
    struct nvme_gen_cq iocqMetrics = iocq->GetQMetrics();
    struct nvme_gen_sq iosqMetrics = iosq->GetQMetrics();

    uint32_t expectedVal = (2 + MAX(iocq->GetNumEntries(),
        iosq->GetNumEntries())) % iocq->GetNumEntries();
    if (iocqMetrics.head_ptr != expectedVal) {
        iocq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iocq",
            "head_ptr"), "CQ Metrics Head Pointer Inconsistent");
        throw FrmwkEx(HERE, "Expected IO CQ.head_ptr = 0x%04X but actual "
            "IOCQ.head_ptr = 0x%04X", expectedVal, iocqMetrics.head_ptr);
    }

    expectedVal = (2 + MAX(iocq->GetNumEntries(), iosq->GetNumEntries())) %
        iosq->GetNumEntries();
    if (iosqMetrics.tail_ptr != expectedVal) {
        iosq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName, "iosq",
            "tail_ptr"), "SQ Metrics Tail Pointer Inconsistent");
        throw FrmwkEx(HERE, "Expected  IO SQ.tail_ptr = 0x%04X but actual "
            "IOSQ.tail_ptr  = 0x%04X", expectedVal, iosqMetrics.tail_ptr);
    }
}
Example #9
0
void
ManyCmdSubmit_r10b::RunCoreTest()
{
    /** \verbatim
     * Assumptions:
     * 1) Test CreateResources_r10b has run prior.
     *  \endverbatim
     */
    uint32_t nCmds;
    uint32_t isrCount;
    uint32_t ceRemain;
    uint32_t numReaped;
    uint32_t numCE;
    uint16_t uniqueId;

    // Lookup objs which were created in a prior test within group
    SharedASQPtr asq = CAST_TO_ASQ(gRsrcMngr->GetObj(ASQ_GROUP_ID))
    SharedACQPtr acq = CAST_TO_ACQ(gRsrcMngr->GetObj(ACQ_GROUP_ID))

    LOG_NRM("Determine the max IOQ entries supported");
    uint64_t ctrlCapReg;
    if (gRegisters->Read(CTLSPC_CAP, ctrlCapReg) == false)
        throw FrmwkEx(HERE, "Unable to determine MQES");
    uint32_t maxIOQEntries = (uint32_t)(ctrlCapReg & CAP_MQES);
    maxIOQEntries += 1;     // convert to 1-based.

    LOG_NRM("Create contig IOQ's");
    SharedIOCQPtr iocq = Queues::CreateIOCQContigToHdw(mGrpName,
        mTestName, CALC_TIMEOUT_ms(1), asq, acq, IOQ_ID, maxIOQEntries,
        false, IOCQ_CONTIG_GROUP_ID, true, 0);

    SharedIOSQPtr iosq = Queues::CreateIOSQContigToHdw(mGrpName,
        mTestName, CALC_TIMEOUT_ms(1), asq, acq, IOQ_ID, maxIOQEntries,
        false, IOSQ_CONTIG_GROUP_ID, IOQ_ID, 0);

    SharedWritePtr writeCmd = SetWriteCmd();

    uint32_t increment = 1;
    for (uint32_t x = 1; x < maxIOQEntries; x += increment) {
        LOG_NRM("Sending #%d simultaneous NVM write cmds to IOSQ", x);
        // Issue x simultaneous NVM write cmds.
        for (nCmds = 1; nCmds <= x; nCmds++)
            iosq->Send(writeCmd, uniqueId);
        iosq->Ring();

        // Variable wait time w.r.t "x" and expect all CE's to arrive in CQ.
        if (iocq->ReapInquiryWaitSpecify(CALC_TIMEOUT_ms(x),
            x, numCE, isrCount) == false) {
            iocq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName,
                "iocq.reqpinq." + writeCmd->GetName()), "Dump Entire IOCQ");
            LogCEAndCQMetrics(iocq);
            throw FrmwkEx(HERE, "Unable to see CEs for issued cmds #%d", x);
        } else if (numCE != x) {
            iocq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName,
                "iocq.reqpinq." + writeCmd->GetName()), "Dump Entire IOCQ");
            LogCEAndCQMetrics(iocq);
            throw FrmwkEx(HERE, "The IOCQ should only have #%d CE's as a result "
                "of #%d simultaneous cmds but found #%d", x, x, numCE);
        }

        SharedMemBufferPtr ceMem = SharedMemBufferPtr(new MemBuffer());
        if ((numReaped = iocq->Reap(ceRemain, ceMem, isrCount, x, true)) != x) {
            iocq->Dump(FileSystem::PrepDumpFile(mGrpName, mTestName,
                "iocq.reap." + writeCmd->GetName()), "Dump Entire IOCQ");
            LogCEAndCQMetrics(iocq);
            throw FrmwkEx(HERE, "Unable to reap on IOCQ #%d. Reaped #%d of #%d",
                IOQ_ID, numReaped, x);
        }

        // Testing every cmd takes numerous hrs, so compromise.
        if ((maxIOQEntries - 1000) % x == 0)
            increment = 100;
        else if ((x % 1000) == 0)
            increment = 1000;
        else if ((maxIOQEntries - 100) % x == 0)
            increment = 10;
        else if ((x % 100) == 0)
            increment = 100;
        else if ((maxIOQEntries - 10) % x == 0)
            increment = 1;
        else if ((x % 10) == 0)
            increment = 10;

    }

    // Delete IOSQ before the IOCQ to comply with spec.
    Queues::DeleteIOSQToHdw(mGrpName, mTestName, CALC_TIMEOUT_ms(1),
        iosq, asq, acq);
    Queues::DeleteIOCQToHdw(mGrpName, mTestName, CALC_TIMEOUT_ms(1),
        iocq, asq, acq);
}
Example #10
0
void
PartialReapMSIX_r10b::RunCoreTest()
{
    /** \verbatim
     * Assumptions:
     * 1) Requires test createResources_r10b to execute 1st
     * \endverbatim
     */
    bool capable;
    char work[25];
    uint64_t reg;
    uint16_t numIrqSupport;
    uint16_t numIrqs = 1;
    uint16_t uniqueId;
    uint32_t isrCount;
    uint32_t numCE;


    // Only allowed to execute if DUT supports MSI-X IRQ's
    if (gCtrlrConfig->IsMSIXCapable(capable, numIrqSupport) == false)
        throw FrmwkEx(HERE);
    else if (capable == false) {
        LOG_WARN("DUT does not support MSI-X IRQ's; unable to execute test");
        return;
    }

    // This requirement must be met, if not then DUT is most likely at fault
    if (gRegisters->Read(CTLSPC_CAP, reg) == false)
        throw FrmwkEx(HERE, "Unable to determine CAP.MQES");
    reg &= CAP_MQES;
    reg += 1;      // convert to 1-based
    if (reg < (uint64_t)NUM_IOQ_ENTRY) {
        LOG_WARN("Desired to support >= %d elements in IOQ's for test",
                 NUM_IOQ_ENTRY);
        return;
    }

    LOG_NRM("Setup the necessary IRQ's");
    if (gCtrlrConfig->SetState(ST_DISABLE) == false)
        throw FrmwkEx(HERE);
    if (gCtrlrConfig->SetIrqScheme(INT_MSIX, numIrqs) == false) {
        throw FrmwkEx(HERE,
                      "Unable to use %d IRQ's, but DUT reports it supports", numIrqs);
    }
    gCtrlrConfig->SetCSS(CtrlrConfig::CSS_NVM_CMDSET);
    if (gCtrlrConfig->SetState(ST_ENABLE) == false)
        throw FrmwkEx(HERE);

    // Lookup objs which were created in a prior test within group
    SharedASQPtr asq = CAST_TO_ASQ(gRsrcMngr->GetObj(ASQ_GROUP_ID))
                       SharedACQPtr acq = CAST_TO_ACQ(gRsrcMngr->GetObj(ACQ_GROUP_ID))

                                          LOG_NRM("Create the IOQ pair which is needed");
    gCtrlrConfig->SetIOCQES(gInformative->GetIdentifyCmdCtrlr()->
                            GetValue(IDCTRLRCAP_CQES) & 0xf);
    SharedIOCQPtr iocq = Queues::CreateIOCQContigToHdw(mGrpName, mTestName,
                         DEFAULT_CMD_WAIT_ms, asq, acq, IOQ_ID, NUM_IOQ_ENTRY, false, "",
                         true, 0);

    gCtrlrConfig->SetIOSQES(gInformative->GetIdentifyCmdCtrlr()->
                            GetValue(IDCTRLRCAP_SQES) & 0xf);
    SharedIOSQPtr iosq = Queues::CreateIOSQContigToHdw(mGrpName, mTestName,
                         DEFAULT_CMD_WAIT_ms, asq, acq, IOQ_ID, NUM_IOQ_ENTRY, false, "",
                         IOQ_ID, 0);

    // NOTE: We are overloading IRQ 0, it is being used by ACQ and we have
    //       created 1 IOQ's, thus to start out the ACQ will already have 2
    //       IRQs. The following algo must add this usage.
    uint32_t anticipatedIrqs = (2 + 1);

    SharedWritePtr writeCmd = CreateCmd();
    for (unsigned i = 1; i <= NUM_CMDS_ISSUE; i++) {
        LOG_NRM("Sending write cmd %d", i);
        iosq->Send(writeCmd, uniqueId);
        iosq->Ring();

        LOG_NRM("Wait for cmd to be processed");
        if (iocq->ReapInquiryWaitSpecify(DEFAULT_CMD_WAIT_ms, i, numCE,
                                         isrCount) == false) {

            iocq->Dump(
                FileSystem::PrepDumpFile(mGrpName, mTestName, "acq",
                                         "error"), "Unable to determine if CE's arrived");
            throw FrmwkEx(HERE, "Only detected %d CE's arriving", numCE);
        } else if (numCE != i) {
            iocq->Dump(
                FileSystem::PrepDumpFile(mGrpName, mTestName, "acq",
                                         "notEnough"), "Test requires seeing all CE's");
            throw FrmwkEx(HERE,
                          "The anticipated %d CE's have not arrived", i);
        } else if (isrCount != anticipatedIrqs) {
            // 1 IRQ per cmd did not occur
            iocq->Dump(
                FileSystem::PrepDumpFile(mGrpName, mTestName, "acq",
                                         "irqBad"), "Test requires seeing all correct num of IRQ's");
            throw FrmwkEx(HERE,
                          "The anticipated %d IRQ', but detected %d", anticipatedIrqs,
                          isrCount);
        }
    }

    // There is an active IRQ outstanding and another to arrive due to the
    // PBA pending bit being set. The pending bit makes the 4th IRQ arrive
    // after reaping a single cmd.
    LOG_NRM("Start reaping and validate %d IRQ's", anticipatedIrqs);
    for (int i = 0; i < NUM_CMDS_ISSUE; i++) {
        snprintf(work, sizeof(work), "cmd.%d", i);
        IO::ReapCE(iocq, 1, isrCount, mGrpName, mTestName, work);
        if (isrCount != anticipatedIrqs) {
            iocq->Dump(
                FileSystem::PrepDumpFile(mGrpName, mTestName, "acq",
                                         work), "Number of IRQ's changed while reaping");
            throw FrmwkEx(HERE,
                          "Anticipated %d IRQ's; but only see %d",
                          anticipatedIrqs, isrCount);
        }

        // Now account for a new IRQ fired due to the pending bit being handled
        // by the DUT, but delay processing so that latency in handling this
        // extra IRQ is not the cause of a test failure. Rather make the
        // absence of the IRQ be the failure, thus delaying is OK.
        if (i == 0) {
            anticipatedIrqs++;
            sleep(1);
        }
    }
}