BOOLEAN
SynchronizedTMFRoutine(
IN PVOID DeviceExtension,
IN PVOID Context
)
{
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
PSCSI_REQUEST_BLOCK Srb = (PSCSI_REQUEST_BLOCK) Context;
PSRB_EXTENSION srbExt = (PSRB_EXTENSION)Srb->SrbExtension;
PVOID va;
ULONGLONG pa;
ENTER_FN();
SET_VA_PA();
if (adaptExt->vq[0]->vq_ops->add_buf(adaptExt->vq[0],
&srbExt->sg[0],
srbExt->out, srbExt->in,
&srbExt->cmd, va, pa) >= 0){
adaptExt->vq[0]->vq_ops->kick(adaptExt->vq[0]);
return TRUE;
}
Srb->SrbStatus = SRB_STATUS_BUSY;
StorPortBusy(DeviceExtension, adaptExt->queue_depth);
EXIT_ERR();
return FALSE;
}
BOOLEAN
SynchronizedTMFRoutine(
IN PVOID DeviceExtension,
IN PVOID Context
)
{
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
PSCSI_REQUEST_BLOCK Srb = (PSCSI_REQUEST_BLOCK) Context;
PSRB_EXTENSION srbExt = SRB_EXTENSION(Srb);
PVOID va;
ULONGLONG pa;
ENTER_FN();
SET_VA_PA();
if (virtqueue_add_buf(adaptExt->vq[VIRTIO_SCSI_CONTROL_QUEUE],
&srbExt->sg[0],
srbExt->out, srbExt->in,
&srbExt->cmd, va, pa) >= 0){
virtqueue_kick(adaptExt->vq[VIRTIO_SCSI_CONTROL_QUEUE]);
return TRUE;
}
SRB_SET_SRB_STATUS(Srb, SRB_STATUS_BUSY);
StorPortBusy(DeviceExtension, adaptExt->queue_depth);
EXIT_ERR();
return FALSE;
}
BOOLEAN
SynchronizedReadWriteRoutine(
IN PVOID DeviceExtension,
IN PVOID Context
)
{
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
PSCSI_REQUEST_BLOCK Srb = (PSCSI_REQUEST_BLOCK) Context;
PRHEL_SRB_EXTENSION srbExt = (PRHEL_SRB_EXTENSION)Srb->SrbExtension;
PVOID va;
ULONGLONG pa;
SET_VA_PA();
if (adaptExt->vq->vq_ops->add_buf(adaptExt->vq,
&srbExt->vbr.sg[0],
srbExt->out, srbExt->in,
&srbExt->vbr, va, pa) >= 0){
InsertTailList(&adaptExt->list_head, &srbExt->vbr.list_entry);
adaptExt->vq->vq_ops->kick(adaptExt->vq);
return TRUE;
}
StorPortBusy(DeviceExtension, 5);
return FALSE;
}
BOOLEAN
SendSRB(
IN PVOID DeviceExtension,
IN PSRB_TYPE Srb
)
{
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
PSRB_EXTENSION srbExt = SRB_EXTENSION(Srb);
PVOID va = NULL;
ULONGLONG pa = 0;
ULONG QueueNumber = 0;
ULONG OldIrql = 0;
ULONG MessageId = 0;
BOOLEAN kick = FALSE;
STOR_LOCK_HANDLE LockHandle = { 0 };
ULONG status = STOR_STATUS_SUCCESS;
ENTER_FN();
SET_VA_PA();
if (adaptExt->num_queues > 1) {
QueueNumber = adaptExt->cpu_to_vq_map[srbExt->cpu] + VIRTIO_SCSI_REQUEST_QUEUE_0;
MessageId = QueueNumber + 1;
}
else {
QueueNumber = VIRTIO_SCSI_REQUEST_QUEUE_0;
}
VioScsiAcquireSpinLock(DeviceExtension, MessageId, &LockHandle);
if (virtqueue_add_buf(adaptExt->vq[QueueNumber],
&srbExt->sg[0],
srbExt->out, srbExt->in,
&srbExt->cmd, va, pa) >= 0){
kick = TRUE;
}
else {
RhelDbgPrint(TRACE_LEVEL_ERROR, ("%s Can not add packet to queue.\n", __FUNCTION__));
//FIXME
}
VioScsiReleaseSpinLock(DeviceExtension, MessageId, &LockHandle);
if (kick == TRUE) {
virtqueue_kick(adaptExt->vq[QueueNumber]);
}
if (adaptExt->num_queues > 1) {
if (CHECKFLAG(adaptExt->perfFlags, STOR_PERF_OPTIMIZE_FOR_COMPLETION_DURING_STARTIO)) {
// ProcessQueue(DeviceExtension, MessageId, FALSE);
}
}
EXIT_FN();
return kick;
}
BOOLEAN
SynchronizedFlushRoutine(
IN PVOID DeviceExtension,
IN PVOID Context
)
{
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
PSCSI_REQUEST_BLOCK Srb = (PSCSI_REQUEST_BLOCK) Context;
PRHEL_SRB_EXTENSION srbExt = (PRHEL_SRB_EXTENSION)Srb->SrbExtension;
ULONG fragLen;
PVOID va;
ULONGLONG pa;
SET_VA_PA();
srbExt->vbr.out_hdr.sector = 0;
srbExt->vbr.out_hdr.ioprio = 0;
srbExt->vbr.req = (struct request *)Srb;
srbExt->vbr.out_hdr.type = VIRTIO_BLK_T_FLUSH;
srbExt->out = 1;
srbExt->in = 1;
srbExt->vbr.sg[0].physAddr = ScsiPortGetPhysicalAddress(DeviceExtension, NULL, &srbExt->vbr.out_hdr, &fragLen);
srbExt->vbr.sg[0].length = sizeof(srbExt->vbr.out_hdr);
srbExt->vbr.sg[1].physAddr = ScsiPortGetPhysicalAddress(DeviceExtension, NULL, &srbExt->vbr.status, &fragLen);
srbExt->vbr.sg[1].length = sizeof(srbExt->vbr.status);
if (virtqueue_add_buf(adaptExt->vq,
&srbExt->vbr.sg[0],
srbExt->out, srbExt->in,
&srbExt->vbr, va, pa) >= 0) {
virtqueue_kick(adaptExt->vq);
return TRUE;
}
virtqueue_kick(adaptExt->vq);
#ifdef USE_STORPORT
StorPortBusy(DeviceExtension, 2);
#endif
return FALSE;
}
BOOLEAN
RhelDoReadWrite(PVOID DeviceExtension,
PSCSI_REQUEST_BLOCK Srb)
{
PCDB cdb;
ULONG fragLen;
ULONG sgElement;
ULONG BytesLeft;
PVOID DataBuffer;
PADAPTER_EXTENSION adaptExt;
PRHEL_SRB_EXTENSION srbExt;
int num_free;
PVOID va;
ULONGLONG pa;
ULONG i;
ULONG sgMaxElements;
cdb = (PCDB)&Srb->Cdb[0];
srbExt = (PRHEL_SRB_EXTENSION)Srb->SrbExtension;
adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
BytesLeft= Srb->DataTransferLength;
DataBuffer = Srb->DataBuffer;
memset(srbExt, 0, sizeof (RHEL_SRB_EXTENSION));
sgMaxElements = MAX_PHYS_SEGMENTS + 1;
for (i = 0, sgElement = 1; (i < sgMaxElements) && BytesLeft; i++, sgElement++) {
srbExt->vbr.sg[sgElement].physAddr = ScsiPortGetPhysicalAddress(DeviceExtension, Srb, DataBuffer, &fragLen);
srbExt->vbr.sg[sgElement].ulSize = fragLen;
srbExt->Xfer += fragLen;
BytesLeft -= fragLen;
DataBuffer = (PVOID)((ULONG_PTR)DataBuffer + fragLen);
}
srbExt->vbr.out_hdr.sector = RhelGetLba(DeviceExtension, cdb);
srbExt->vbr.out_hdr.ioprio = 0;
srbExt->vbr.req = (struct request *)Srb;
if (Srb->SrbFlags & SRB_FLAGS_DATA_OUT) {
srbExt->vbr.out_hdr.type = VIRTIO_BLK_T_OUT;
srbExt->out = sgElement;
srbExt->in = 1;
} else {
srbExt->vbr.out_hdr.type = VIRTIO_BLK_T_IN;
srbExt->out = 1;
srbExt->in = sgElement;
}
srbExt->vbr.sg[0].physAddr = ScsiPortGetPhysicalAddress(DeviceExtension, NULL, &srbExt->vbr.out_hdr, &fragLen);
srbExt->vbr.sg[0].ulSize = sizeof(srbExt->vbr.out_hdr);
srbExt->vbr.sg[sgElement].physAddr = ScsiPortGetPhysicalAddress(DeviceExtension, NULL, &srbExt->vbr.status, &fragLen);
srbExt->vbr.sg[sgElement].ulSize = sizeof(srbExt->vbr.status);
SET_VA_PA();
num_free = adaptExt->vq->vq_ops->add_buf(adaptExt->vq,
&srbExt->vbr.sg[0],
srbExt->out, srbExt->in,
&srbExt->vbr, va, pa);
if ( num_free >= 0) {
InsertTailList(&adaptExt->list_head, &srbExt->vbr.list_entry);
adaptExt->vq->vq_ops->kick(adaptExt->vq);
srbExt->call_next = FALSE;
if(!adaptExt->indirect && num_free < VIRTIO_MAX_SG) {
srbExt->call_next = TRUE;
} else {
ScsiPortNotification(NextLuRequest, DeviceExtension, Srb->PathId, Srb->TargetId, Srb->Lun);
}
}
return TRUE;
}
BOOLEAN
RhelDoFlush(
PVOID DeviceExtension,
PSRB_TYPE Srb,
BOOLEAN resend,
BOOLEAN bIsr
)
{
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
PSRB_EXTENSION srbExt = SRB_EXTENSION(Srb);
ULONG fragLen = 0UL;
PVOID va = NULL;
ULONGLONG pa = 0ULL;
ULONG QueueNumber = 0;
ULONG OldIrql = 0;
ULONG MessageId = 0;
BOOLEAN result = FALSE;
bool notify = FALSE;
STOR_LOCK_HANDLE LockHandle = { 0 };
ULONG status = STOR_STATUS_SUCCESS;
struct virtqueue *vq = NULL;
SET_VA_PA();
if (resend) {
MessageId = srbExt->MessageID;
QueueNumber = MessageId - 1;
} else if (adaptExt->num_queues > 1) {
STARTIO_PERFORMANCE_PARAMETERS param;
param.Size = sizeof(STARTIO_PERFORMANCE_PARAMETERS);
status = StorPortGetStartIoPerfParams(DeviceExtension, (PSCSI_REQUEST_BLOCK)Srb, ¶m);
if (status == STOR_STATUS_SUCCESS && param.MessageNumber != 0) {
MessageId = param.MessageNumber;
QueueNumber = MessageId - 1;
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("%s srb %p, cpu %d :: QueueNumber %lu, MessageNumber %lu, ChannelNumber %lu.\n", __FUNCTION__, Srb, srbExt->cpu, QueueNumber, param.MessageNumber, param.ChannelNumber));
}
else {
RhelDbgPrint(TRACE_LEVEL_ERROR, ("%s StorPortGetStartIoPerfParams failed. srb %p cpu %d status 0x%x.\n",__FUNCTION__, Srb, srbExt->cpu, status));
QueueNumber = 0;
MessageId = 1;
}
} else {
QueueNumber = 0;
MessageId = 1;
}
srbExt->MessageID = MessageId;
vq = adaptExt->vq[QueueNumber];
srbExt->vbr.out_hdr.sector = 0;
srbExt->vbr.out_hdr.ioprio = 0;
srbExt->vbr.req = (struct request *)Srb;
srbExt->vbr.out_hdr.type = VIRTIO_BLK_T_FLUSH;
srbExt->out = 1;
srbExt->in = 1;
srbExt->vbr.sg[0].physAddr = StorPortGetPhysicalAddress(DeviceExtension, NULL, &srbExt->vbr.out_hdr, &fragLen);
srbExt->vbr.sg[0].length = sizeof(srbExt->vbr.out_hdr);
srbExt->vbr.sg[1].physAddr = StorPortGetPhysicalAddress(DeviceExtension, NULL, &srbExt->vbr.status, &fragLen);
srbExt->vbr.sg[1].length = sizeof(srbExt->vbr.status);
VioStorVQLock(DeviceExtension, MessageId, &LockHandle, FALSE);
if (virtqueue_add_buf(vq,
&srbExt->vbr.sg[0],
srbExt->out, srbExt->in,
&srbExt->vbr, va, pa) >= 0) {
notify = virtqueue_kick_prepare(vq);
VioStorVQUnlock(DeviceExtension, MessageId, &LockHandle, FALSE);
result = TRUE;
#ifdef DBG
InterlockedIncrement((LONG volatile*)&adaptExt->inqueue_cnt);
#endif
}
else {
VioStorVQUnlock(DeviceExtension, MessageId, &LockHandle, FALSE);
RhelDbgPrint(TRACE_LEVEL_FATAL, ("%s Can not add packet to queue %d.\n", __FUNCTION__, QueueNumber));
StorPortBusy(DeviceExtension, 2);
}
if (notify) {
virtqueue_notify(vq);
}
return result;
}
BOOLEAN
RhelDoReadWrite(PVOID DeviceExtension,
PSRB_TYPE Srb)
{
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
PSRB_EXTENSION srbExt = SRB_EXTENSION(Srb);
PVOID va = NULL;
ULONGLONG pa = 0ULL;
ULONG QueueNumber = 0;
ULONG OldIrql = 0;
ULONG MessageId = 0;
BOOLEAN result = FALSE;
bool notify = FALSE;
STOR_LOCK_HANDLE LockHandle = { 0 };
ULONG status = STOR_STATUS_SUCCESS;
struct virtqueue *vq = NULL;
SET_VA_PA();
if (adaptExt->num_queues > 1) {
STARTIO_PERFORMANCE_PARAMETERS param;
param.Size = sizeof(STARTIO_PERFORMANCE_PARAMETERS);
status = StorPortGetStartIoPerfParams(DeviceExtension, (PSCSI_REQUEST_BLOCK)Srb, ¶m);
if (status == STOR_STATUS_SUCCESS && param.MessageNumber != 0) {
MessageId = param.MessageNumber;
QueueNumber = MessageId - 1;
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("%s srb %p, cpu %d :: QueueNumber %lu, MessageNumber %lu, ChannelNumber %lu.\n", __FUNCTION__, Srb, srbExt->cpu, QueueNumber, param.MessageNumber, param.ChannelNumber));
}
else {
RhelDbgPrint(TRACE_LEVEL_ERROR, ("%s StorPortGetStartIoPerfParams failed srb %p cpu %d status 0x%x.\n", __FUNCTION__, Srb, srbExt->cpu, status));
QueueNumber = 0;
MessageId = 1;
}
}
else {
QueueNumber = 0;
MessageId = 1;
}
srbExt->MessageID = MessageId;
vq = adaptExt->vq[QueueNumber];
RhelDbgPrint(TRACE_LEVEL_VERBOSE, ("<--->%s : QueueNumber 0x%x vq = %p\n", __FUNCTION__, QueueNumber, vq));
VioStorVQLock(DeviceExtension, MessageId, &LockHandle, FALSE);
if (virtqueue_add_buf(vq,
&srbExt->vbr.sg[0],
srbExt->out, srbExt->in,
&srbExt->vbr, va, pa) >= 0) {
notify = virtqueue_kick_prepare(vq);
VioStorVQUnlock(DeviceExtension, MessageId, &LockHandle, FALSE);
#ifdef DBG
InterlockedIncrement((LONG volatile*)&adaptExt->inqueue_cnt);
#endif
result = TRUE;
}
else {
VioStorVQUnlock(DeviceExtension, MessageId, &LockHandle, FALSE);
RhelDbgPrint(TRACE_LEVEL_FATAL, ("%s Can not add packet to queue %d.\n", __FUNCTION__, QueueNumber));
StorPortBusy(DeviceExtension, 2);
}
if (notify) {
virtqueue_notify(vq);
}
#if (NTDDI_VERSION > NTDDI_WIN7)
if (adaptExt->num_queues > 1) {
if (CHECKFLAG(adaptExt->perfFlags, STOR_PERF_OPTIMIZE_FOR_COMPLETION_DURING_STARTIO)) {
VioStorCompleteRequest(DeviceExtension, MessageId, FALSE);
}
}
#endif
return result;
}
BOOLEAN
SendSRB(
IN PVOID DeviceExtension,
IN PSCSI_REQUEST_BLOCK Srb
)
{
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
PSRB_EXTENSION srbExt = (PSRB_EXTENSION)Srb->SrbExtension;
PVOID va = NULL;
ULONGLONG pa = 0;
ULONG QueueNumber = 0;
ULONG OldIrql = 0;
ULONG MessageId = 0;
BOOLEAN kick = FALSE;
STOR_LOCK_HANDLE LockHandle = { 0 };
ULONG status = STOR_STATUS_SUCCESS;
ENTER_FN();
SET_VA_PA();
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("Srb %p issued on %d::%d QueueNumber %d\n",
Srb, srbExt->procNum.Group, srbExt->procNum.Number, QueueNumber));
if (adaptExt->num_queues > 1) {
QueueNumber = adaptExt->cpu_to_vq_map[srbExt->procNum.Number];
MessageId = QueueNumber + 1;
if (CHECKFLAG(adaptExt->perfFlags, STOR_PERF_OPTIMIZE_FOR_COMPLETION_DURING_STARTIO)) {
ProcessQueue(DeviceExtension, MessageId, FALSE);
}
// status = StorPortAcquireMSISpinLock(DeviceExtension, MessageId, &OldIrql);
if (status != STOR_STATUS_SUCCESS) {
RhelDbgPrint(TRACE_LEVEL_ERROR, ("%s StorPortAcquireMSISpinLock returned status 0x%x\n", __FUNCTION__, status));
}
}
else {
QueueNumber = VIRTIO_SCSI_REQUEST_QUEUE_0;
StorPortAcquireSpinLock(DeviceExtension, InterruptLock, NULL, &LockHandle);
}
if (virtqueue_add_buf(adaptExt->vq[QueueNumber],
&srbExt->sg[0],
srbExt->out, srbExt->in,
&srbExt->cmd, va, pa) >= 0){
kick = TRUE;
}
else {
RhelDbgPrint(TRACE_LEVEL_WARNING, ("%s Cant add packet to queue.\n", __FUNCTION__));
//FIXME
}
if (adaptExt->num_queues > 1) {
// status = StorPortReleaseMSISpinLock(DeviceExtension, MessageId, OldIrql);
if (status != STOR_STATUS_SUCCESS) {
RhelDbgPrint(TRACE_LEVEL_ERROR, ("%s StorPortReleaseMSISpinLock returned status 0x%x\n", __FUNCTION__, status));
}
}
else {
StorPortReleaseSpinLock(DeviceExtension, &LockHandle);
}
if (kick == TRUE) {
virtqueue_kick(adaptExt->vq[QueueNumber]);
}
return kick;
EXIT_FN();
}