static struct virtqueue *FindVirtualQueue(PADAPTER_EXTENSION adaptExt, ULONG index, ULONG vector)
{
struct virtqueue *vq = NULL;
if (adaptExt->uncachedExtensionVa)
{
ULONG len;
PHYSICAL_ADDRESS pa = ScsiPortGetPhysicalAddress(adaptExt, NULL, adaptExt->uncachedExtensionVa, &len);
if (pa.QuadPart)
vq = VirtIODevicePrepareQueue(&adaptExt->vdev, index, pa, adaptExt->uncachedExtensionVa, len, NULL);
}
if (vq && vector)
{
unsigned res;
ScsiPortWritePortUshort((PUSHORT)(adaptExt->vdev.addr + VIRTIO_MSI_QUEUE_VECTOR),(USHORT)vector);
res = ScsiPortReadPortUshort((PUSHORT)(adaptExt->vdev.addr + VIRTIO_MSI_QUEUE_VECTOR));
RhelDbgPrint(TRACE_LEVEL_FATAL, ("%s>> VIRTIO_MSI_QUEUE_VECTOR vector = %d, res = 0x%x\n", __FUNCTION__, vector, res));
if(res == VIRTIO_MSI_NO_VECTOR) {
VirtIODeviceDeleteQueue(vq, NULL);
vq = NULL;
RhelDbgPrint(TRACE_LEVEL_FATAL, ("%s>> Cannot create vq vector\n", __FUNCTION__));
return NULL;
}
}
return vq;
}
BOOLEAN
VirtIoInterrupt(
IN PVOID DeviceExtension
)
{
pblk_req vbr;
unsigned int len;
PADAPTER_EXTENSION adaptExt;
BOOLEAN isInterruptServiced = FALSE;
PSCSI_REQUEST_BLOCK Srb;
ULONG intReason = 0;
adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
RhelDbgPrint(TRACE_LEVEL_VERBOSE, ("%s (%d)\n", __FUNCTION__, KeGetCurrentIrql()));
intReason = VirtIODeviceISR((VirtIODevice*)DeviceExtension);
if ( intReason == 1) {
isInterruptServiced = TRUE;
while((vbr = (pblk_req)adaptExt->vq->vq_ops->get_buf(adaptExt->vq, &len)) != NULL) {
Srb = (PSCSI_REQUEST_BLOCK)vbr->req;
if (Srb) {
switch (vbr->status) {
case VIRTIO_BLK_S_OK:
Srb->SrbStatus = SRB_STATUS_SUCCESS;
break;
case VIRTIO_BLK_S_UNSUPP:
Srb->SrbStatus = SRB_STATUS_INVALID_REQUEST;
break;
default:
Srb->SrbStatus = SRB_STATUS_ERROR;
RhelDbgPrint(TRACE_LEVEL_ERROR, ("SRB_STATUS_ERROR\n"));
break;
}
}
if (vbr->out_hdr.type == VIRTIO_BLK_T_FLUSH) {
#ifdef USE_STORPORT
--adaptExt->in_fly;
#endif
CompleteSRB(DeviceExtension, Srb);
} else if (vbr->out_hdr.type == VIRTIO_BLK_T_GET_ID) {
adaptExt->sn_ok = TRUE;
} else if (Srb) {
#ifdef USE_STORPORT
--adaptExt->in_fly;
#endif
CompleteDPC(DeviceExtension, vbr, 0);
}
}
} else if (intReason == 3) {
adaptExt->rescan_geometry = TRUE;
ScsiPortNotification( BusChangeDetected, DeviceExtension, 0);
isInterruptServiced = TRUE;
}
#ifdef USE_STORPORT
if (adaptExt->in_fly > 0) {
adaptExt->vq->vq_ops->kick(adaptExt->vq);
}
#endif
RhelDbgPrint(TRACE_LEVEL_VERBOSE, ("%s isInterruptServiced = %d\n", __FUNCTION__, isInterruptServiced));
return isInterruptServiced;
}
VOID
VioStorVQUnlock(
IN PVOID DeviceExtension,
IN ULONG MessageID,
IN PSTOR_LOCK_HANDLE LockHandle,
IN BOOLEAN isr
)
{
PADAPTER_EXTENSION adaptExt;
RhelDbgPrint(TRACE_LEVEL_VERBOSE, ("--->%s MessageID = %d\n", __FUNCTION__, MessageID));
adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
if (!adaptExt->msix_enabled) {
if (!isr) {
StorPortReleaseSpinLock(DeviceExtension, LockHandle);
}
}
else {
if ((adaptExt->num_queues == 1) ||
(!CHECKFLAG(adaptExt->perfFlags, STOR_PERF_CONCURRENT_CHANNELS))) {
if (!isr) {
StorPortReleaseMSISpinLock(DeviceExtension, (adaptExt->msix_one_vector ? 0 : MessageID), LockHandle->Context.OldIrql);
}
}
else {
NT_ASSERT(MessageID > 0);
NT_ASSERT(MessageID <= adaptExt->num_queues);
StorPortReleaseSpinLock(DeviceExtension, LockHandle);
}
}
RhelDbgPrint(TRACE_LEVEL_VERBOSE, ("<---%s MessageID = %d\n", __FUNCTION__, MessageID));
}
BOOLEAN
VirtIoMSInterruptRoutine (
IN PVOID DeviceExtension,
IN ULONG MessageID
)
{
pblk_req vbr;
unsigned int len;
PADAPTER_EXTENSION adaptExt;
PSCSI_REQUEST_BLOCK Srb;
BOOLEAN isInterruptServiced = FALSE;
adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
RhelDbgPrint(TRACE_LEVEL_VERBOSE,
("<--->%s : MessageID 0x%x\n", __FUNCTION__, MessageID));
if (MessageID == 0) {
adaptExt->rescan_geometry = TRUE;
StorPortNotification( BusChangeDetected, DeviceExtension, 0);
return TRUE;
}
while((vbr = (pblk_req)adaptExt->vq->vq_ops->get_buf(adaptExt->vq, &len)) != NULL) {
Srb = (PSCSI_REQUEST_BLOCK)vbr->req;
if (Srb) {
switch (vbr->status) {
case VIRTIO_BLK_S_OK:
Srb->SrbStatus = SRB_STATUS_SUCCESS;
break;
case VIRTIO_BLK_S_UNSUPP:
Srb->SrbStatus = SRB_STATUS_INVALID_REQUEST;
break;
default:
Srb->SrbStatus = SRB_STATUS_ERROR;
RhelDbgPrint(TRACE_LEVEL_ERROR, ("SRB_STATUS_ERROR\n"));
break;
}
}
if (vbr->out_hdr.type == VIRTIO_BLK_T_FLUSH) {
--adaptExt->in_fly;
CompleteSRB(DeviceExtension, Srb);
} else if (vbr->out_hdr.type == VIRTIO_BLK_T_GET_ID) {
adaptExt->sn_ok = TRUE;
} else if (Srb) {
--adaptExt->in_fly;
CompleteDPC(DeviceExtension, vbr, MessageID);
}
isInterruptServiced = TRUE;
}
if (adaptExt->in_fly > 0) {
adaptExt->vq->vq_ops->kick(adaptExt->vq);
}
return isInterruptServiced;
}
ULONG
DriverEntry(
IN PVOID DriverObject,
IN PVOID RegistryPath
)
{
HW_INITIALIZATION_DATA hwInitData;
ULONG initResult;
InitializeDebugPrints((PDRIVER_OBJECT)DriverObject, (PUNICODE_STRING)RegistryPath);
RhelDbgPrint(TRACE_LEVEL_ERROR, ("Vioscsi driver started...built on %s %s\n", __DATE__, __TIME__));
IsCrashDumpMode = FALSE;
if (RegistryPath == NULL) {
RhelDbgPrint(TRACE_LEVEL_INFORMATION,
("DriverEntry: Crash dump mode\n"));
IsCrashDumpMode = TRUE;
}
memset(&hwInitData, 0, sizeof(HW_INITIALIZATION_DATA));
hwInitData.HwInitializationDataSize = sizeof(HW_INITIALIZATION_DATA);
hwInitData.HwFindAdapter = VioScsiFindAdapter;
hwInitData.HwInitialize = VioScsiHwInitialize;
hwInitData.HwStartIo = VioScsiStartIo;
hwInitData.HwInterrupt = VioScsiInterrupt;
hwInitData.HwResetBus = VioScsiResetBus;
hwInitData.HwAdapterControl = VioScsiAdapterControl;
hwInitData.HwBuildIo = VioScsiBuildIo;
hwInitData.NeedPhysicalAddresses = TRUE;
hwInitData.TaggedQueuing = TRUE;
hwInitData.AutoRequestSense = TRUE;
hwInitData.MultipleRequestPerLu = TRUE;
hwInitData.DeviceExtensionSize = sizeof(ADAPTER_EXTENSION);
hwInitData.SrbExtensionSize = sizeof(SRB_EXTENSION);
hwInitData.AdapterInterfaceType = PCIBus;
hwInitData.NumberOfAccessRanges = 1;
hwInitData.MapBuffers = STOR_MAP_NON_READ_WRITE_BUFFERS;
initResult = StorPortInitialize(DriverObject,
RegistryPath,
&hwInitData,
NULL);
RhelDbgPrint(TRACE_LEVEL_VERBOSE,
("Initialize returned 0x%x\n", initResult));
return initResult;
}
BOOLEAN
InitHW(
IN PVOID DeviceExtension,
IN PPORT_CONFIGURATION_INFORMATION ConfigInfo
)
{
PACCESS_RANGE accessRange;
PADAPTER_EXTENSION adaptExt;
ENTER_FN();
adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
accessRange = &(*ConfigInfo->AccessRanges)[0];
ASSERT (FALSE == accessRange->RangeInMemory) ;
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("Port Resource [%08I64X-%08I64X]\n",
accessRange->RangeStart.QuadPart,
accessRange->RangeStart.QuadPart +
accessRange->RangeLength));
if ( accessRange->RangeLength < IO_PORT_LENGTH) {
LogError(DeviceExtension,
SP_INTERNAL_ADAPTER_ERROR,
__LINE__);
RhelDbgPrint(TRACE_LEVEL_FATAL, ("Wrong access range %x bytes\n", accessRange->RangeLength));
return FALSE;
}
adaptExt->device_base = (ULONG_PTR)StorPortGetDeviceBase(DeviceExtension,
ConfigInfo->AdapterInterfaceType,
ConfigInfo->SystemIoBusNumber,
accessRange->RangeStart,
accessRange->RangeLength,
(BOOLEAN)!accessRange->RangeInMemory);
if (adaptExt->device_base == (ULONG_PTR)NULL) {
LogError(DeviceExtension,
SP_INTERNAL_ADAPTER_ERROR,
__LINE__);
RhelDbgPrint(TRACE_LEVEL_FATAL, ("Couldn't map %x for %x bytes\n",
(*ConfigInfo->AccessRanges)[0].RangeStart.LowPart,
(*ConfigInfo->AccessRanges)[0].RangeLength));
return FALSE;
}
adaptExt->pvdev = &adaptExt->vdev;
VirtIODeviceInitialize(adaptExt->pvdev, adaptExt->device_base, sizeof(VirtIODevice));
EXIT_FN();
return TRUE;
}
VOID
GetScsiConfig(
IN PVOID DeviceExtension
)
{
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
ENTER_FN();
adaptExt->features = StorPortReadPortUlong(DeviceExtension, (PULONG)(adaptExt->device_base + VIRTIO_PCI_HOST_FEATURES));
VirtIODeviceGet( adaptExt->pvdev, FIELD_OFFSET(VirtIOSCSIConfig, seg_max),
&adaptExt->scsi_config.seg_max, sizeof(adaptExt->scsi_config.seg_max));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("seg_max %lu\n", adaptExt->scsi_config.seg_max));
VirtIODeviceGet( adaptExt->pvdev, FIELD_OFFSET(VirtIOSCSIConfig, num_queues),
&adaptExt->scsi_config.num_queues, sizeof(adaptExt->scsi_config.num_queues));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("num_queues %lu\n", adaptExt->scsi_config.num_queues));
VirtIODeviceGet( adaptExt->pvdev, FIELD_OFFSET(VirtIOSCSIConfig, max_sectors),
&adaptExt->scsi_config.max_sectors, sizeof(adaptExt->scsi_config.max_sectors));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("max_sectors %lu\n", adaptExt->scsi_config.max_sectors));
VirtIODeviceGet( adaptExt->pvdev, FIELD_OFFSET(VirtIOSCSIConfig, cmd_per_lun),
&adaptExt->scsi_config.cmd_per_lun, sizeof(adaptExt->scsi_config.cmd_per_lun));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("cmd_per_lun %lu\n", adaptExt->scsi_config.cmd_per_lun));
VirtIODeviceGet( adaptExt->pvdev, FIELD_OFFSET(VirtIOSCSIConfig, event_info_size),
&adaptExt->scsi_config.event_info_size, sizeof(adaptExt->scsi_config.event_info_size));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("seg_max %lu\n", adaptExt->scsi_config.seg_max));
VirtIODeviceGet( adaptExt->pvdev, FIELD_OFFSET(VirtIOSCSIConfig, sense_size),
&adaptExt->scsi_config.sense_size, sizeof(adaptExt->scsi_config.sense_size));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("event_info_size %lu\n", adaptExt->scsi_config.event_info_size));
VirtIODeviceGet( adaptExt->pvdev, FIELD_OFFSET(VirtIOSCSIConfig, cdb_size),
&adaptExt->scsi_config.cdb_size, sizeof(adaptExt->scsi_config.cdb_size));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("cdb_size %lu\n", adaptExt->scsi_config.cdb_size));
VirtIODeviceGet( adaptExt->pvdev, FIELD_OFFSET(VirtIOSCSIConfig, max_channel),
&adaptExt->scsi_config.max_channel, sizeof(adaptExt->scsi_config.max_channel));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("max_channel %u\n", adaptExt->scsi_config.max_channel));
VirtIODeviceGet( adaptExt->pvdev, FIELD_OFFSET(VirtIOSCSIConfig, max_target),
&adaptExt->scsi_config.max_target, sizeof(adaptExt->scsi_config.max_target));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("max_target %u\n", adaptExt->scsi_config.max_target));
VirtIODeviceGet( adaptExt->pvdev, FIELD_OFFSET(VirtIOSCSIConfig, max_lun),
&adaptExt->scsi_config.max_lun, sizeof(adaptExt->scsi_config.max_lun));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("max_lun %lu\n", adaptExt->scsi_config.max_lun));
EXIT_FN();
}
ULONG
VioScsiAcquireSpinLock(
IN PVOID DeviceExtension,
IN ULONG MessageID,
IN PSTOR_LOCK_HANDLE LockHandle
)
{
PADAPTER_EXTENSION adaptExt;
ULONG status = STOR_STATUS_SUCCESS;
ENTER_FN();
adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
if (adaptExt->num_queues > 1) {
ULONG oldIrql = 0;
status = StorPortAcquireMSISpinLock(DeviceExtension, MessageID, &oldIrql);
LockHandle->Context.OldIrql = (KIRQL)oldIrql;
if (status != STOR_STATUS_SUCCESS) {
RhelDbgPrint(TRACE_LEVEL_ERROR, ("%s StorPortAcquireMSISpinLock returned status 0x%x\n", __FUNCTION__, status));
}
}
else {
StorPortAcquireSpinLock(DeviceExtension, InterruptLock, NULL, LockHandle);
}
return status;
EXIT_FN();
}
static struct virtqueue *FindVirtualQueue(PADAPTER_EXTENSION adaptExt, ULONG index, ULONG vector)
{
struct virtqueue *vq = NULL;
if (adaptExt->uncachedExtensionVa)
{
ULONG len;
PVOID ptr = (PVOID)((ULONG_PTR)adaptExt->uncachedExtensionVa + adaptExt->offset[index]);
PHYSICAL_ADDRESS pa = StorPortGetPhysicalAddress(adaptExt, NULL, ptr, &len);
if (pa.QuadPart)
{
vq = VirtIODevicePrepareQueue(&adaptExt->vdev, index, pa, ptr, len, NULL, FALSE);
}
if (vq == NULL)
{
RhelDbgPrint(TRACE_LEVEL_FATAL, ("%s>> cannot create virtual queue index = %d vector = % pa = %08I64X\n", __FUNCTION__, index, vector, pa.QuadPart));
return NULL;
}
if (vector)
{
unsigned res;
StorPortWritePortUshort(adaptExt, (PUSHORT)(adaptExt->vdev.addr + VIRTIO_MSI_QUEUE_VECTOR),(USHORT)vector);
res = StorPortReadPortUshort(adaptExt, (PUSHORT)(adaptExt->vdev.addr + VIRTIO_MSI_QUEUE_VECTOR));
RhelDbgPrint(TRACE_LEVEL_FATAL, ("%s>> VIRTIO_MSI_QUEUE_VECTOR vector = %d, res = 0x%x\n", __FUNCTION__, vector, res));
if(res == VIRTIO_MSI_NO_VECTOR)
{
VirtIODeviceDeleteQueue(vq, NULL);
vq = NULL;
RhelDbgPrint(TRACE_LEVEL_FATAL, ("%s>> Cannot create vq vector\n", __FUNCTION__));
return NULL;
}
StorPortWritePortUshort(adaptExt, (PUSHORT)(adaptExt->vdev.addr + VIRTIO_MSI_CONFIG_VECTOR),(USHORT)vector);
res = StorPortReadPortUshort(adaptExt, (PUSHORT)(adaptExt->vdev.addr + VIRTIO_MSI_CONFIG_VECTOR));
if (res != vector)
{
VirtIODeviceDeleteQueue(vq, NULL);
vq = NULL;
RhelDbgPrint(TRACE_LEVEL_FATAL, ("%s>> Cannot set config vector\n", __FUNCTION__));
return NULL;
}
}
}
return vq;
}
BOOLEAN
SendSRB(
IN PVOID DeviceExtension,
IN PSRB_TYPE Srb
)
{
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
PSRB_EXTENSION srbExt = SRB_EXTENSION(Srb);
PVOID va = NULL;
ULONGLONG pa = 0;
ULONG QueueNumber = 0;
ULONG OldIrql = 0;
ULONG MessageId = 0;
BOOLEAN kick = FALSE;
STOR_LOCK_HANDLE LockHandle = { 0 };
ULONG status = STOR_STATUS_SUCCESS;
ENTER_FN();
SET_VA_PA();
if (adaptExt->num_queues > 1) {
QueueNumber = adaptExt->cpu_to_vq_map[srbExt->cpu] + VIRTIO_SCSI_REQUEST_QUEUE_0;
MessageId = QueueNumber + 1;
}
else {
QueueNumber = VIRTIO_SCSI_REQUEST_QUEUE_0;
}
VioScsiAcquireSpinLock(DeviceExtension, MessageId, &LockHandle);
if (virtqueue_add_buf(adaptExt->vq[QueueNumber],
&srbExt->sg[0],
srbExt->out, srbExt->in,
&srbExt->cmd, va, pa) >= 0){
kick = TRUE;
}
else {
RhelDbgPrint(TRACE_LEVEL_ERROR, ("%s Can not add packet to queue.\n", __FUNCTION__));
//FIXME
}
VioScsiReleaseSpinLock(DeviceExtension, MessageId, &LockHandle);
if (kick == TRUE) {
virtqueue_kick(adaptExt->vq[QueueNumber]);
}
if (adaptExt->num_queues > 1) {
if (CHECKFLAG(adaptExt->perfFlags, STOR_PERF_OPTIMIZE_FOR_COMPLETION_DURING_STARTIO)) {
// ProcessQueue(DeviceExtension, MessageId, FALSE);
}
}
EXIT_FN();
return kick;
}
static void *mem_alloc_nonpaged_block(void *context, size_t size)
{
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)context;
PVOID ptr = (PVOID)((ULONG_PTR)adaptExt->poolAllocationVa + adaptExt->poolOffset);
if ((adaptExt->poolOffset + size) <= adaptExt->poolAllocationSize) {
adaptExt->poolOffset += size;
RtlZeroMemory(ptr, size);
return ptr;
} else {
RhelDbgPrint(TRACE_LEVEL_FATAL, ("Ran out of memory in %s(%Id)\n", __FUNCTION__, size));
return NULL;
}
}
static void *mem_alloc_contiguous_pages(void *context, size_t size)
{
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)context;
PVOID ptr = (PVOID)((ULONG_PTR)adaptExt->pageAllocationVa + adaptExt->pageOffset);
if ((adaptExt->pageOffset + size) <= adaptExt->pageAllocationSize) {
size = ROUND_TO_PAGES(size);
adaptExt->pageOffset += size;
RtlZeroMemory(ptr, size);
return ptr;
} else {
RhelDbgPrint(TRACE_LEVEL_FATAL, ("Ran out of memory in %s(%Id)\n", __FUNCTION__, size));
return NULL;
}
}
VOID
TransportReset(
IN PVOID DeviceExtension,
IN PVirtIOSCSIEvent evt
)
{
UCHAR TargetId = evt->lun[1];
UCHAR Lun = (evt->lun[2] << 8) | evt->lun[3];
switch (evt->reason)
{
case VIRTIO_SCSI_EVT_RESET_RESCAN:
StorPortNotification( BusChangeDetected, DeviceExtension, 0);
break;
case VIRTIO_SCSI_EVT_RESET_REMOVED:
StorPortNotification( BusChangeDetected, DeviceExtension, 0);
break;
default:
RhelDbgPrint(TRACE_LEVEL_VERBOSE, ("<-->Unsupport virtio scsi event reason 0x%x\n", evt->reason));
}
}
static void *pci_map_address_range(void *context, int bar, size_t offset, size_t maxlen)
{
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)context;
if (bar < PCI_TYPE0_ADDRESSES) {
PVIRTIO_BAR pBar = &adaptExt->pci_bars[bar];
if (pBar->pBase == NULL) {
#ifndef USE_STORPORT
if (!ScsiPortValidateRange(
adaptExt,
PCIBus,
adaptExt->system_io_bus_number,
pBar->BasePA,
pBar->uLength,
!!pBar->bPortSpace)) {
LogError(adaptExt,
SP_INTERNAL_ADAPTER_ERROR,
__LINE__);
RhelDbgPrint(TRACE_LEVEL_FATAL, ("Range validation failed %I64x for %x bytes\n",
pBar->BasePA.QuadPart,
pBar->uLength));
return NULL;
}
#endif
pBar->pBase = ScsiPortGetDeviceBase(
adaptExt,
PCIBus,
adaptExt->system_io_bus_number,
pBar->BasePA,
pBar->uLength,
!!pBar->bPortSpace);
}
if (pBar->pBase != NULL && offset < pBar->uLength) {
return (PUCHAR)pBar->pBase + offset;
}
}
return NULL;
}
BOOLEAN
VioScsiMSInterrupt (
IN PVOID DeviceExtension,
IN ULONG MessageID
)
{
PVirtIOSCSICmd cmd;
PVirtIOSCSIEventNode evtNode;
unsigned int len;
PADAPTER_EXTENSION adaptExt;
BOOLEAN isInterruptServiced = FALSE;
PSCSI_REQUEST_BLOCK Srb;
PSRB_EXTENSION srbExt;
ULONG intReason = 0;
adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
RhelDbgPrint(TRACE_LEVEL_VERBOSE,
("<--->%s : MessageID 0x%x\n", __FUNCTION__, MessageID));
if (MessageID == 0)
{
return TRUE;
}
if (MessageID == 1)
{
if (adaptExt->tmf_infly)
{
while((cmd = (PVirtIOSCSICmd)adaptExt->vq[0]->vq_ops->get_buf(adaptExt->vq[0], &len)) != NULL)
{
VirtIOSCSICtrlTMFResp *resp;
Srb = (PSCSI_REQUEST_BLOCK)cmd->sc;
ASSERT(Srb == &adaptExt->tmf_cmd.Srb);
resp = &cmd->resp.tmf;
switch(resp->response) {
case VIRTIO_SCSI_S_OK:
case VIRTIO_SCSI_S_FUNCTION_SUCCEEDED:
break;
default:
RhelDbgPrint(TRACE_LEVEL_ERROR, ("Unknown response %d\n", resp->response));
ASSERT(0);
break;
}
StorPortResume(DeviceExtension);
}
adaptExt->tmf_infly = FALSE;
}
return TRUE;
}
if (MessageID == 2) {
while((evtNode = (PVirtIOSCSIEventNode)adaptExt->vq[1]->vq_ops->get_buf(adaptExt->vq[1], &len)) != NULL) {
PVirtIOSCSIEvent evt = &evtNode->event;
switch (evt->event) {
case VIRTIO_SCSI_T_NO_EVENT:
break;
case VIRTIO_SCSI_T_TRANSPORT_RESET:
TransportReset(DeviceExtension, evt);
break;
case VIRTIO_SCSI_T_PARAM_CHANGE:
ParamChange(DeviceExtension, evt);
break;
default:
RhelDbgPrint(TRACE_LEVEL_ERROR, ("Unsupport virtio scsi event %x\n", evt->event));
break;
}
SynchronizedKickEventRoutine(DeviceExtension, evtNode);
}
return TRUE;
}
if (MessageID == 3)
{
while((cmd = (PVirtIOSCSICmd)adaptExt->vq[2]->vq_ops->get_buf(adaptExt->vq[2], &len)) != NULL)
{
VirtIOSCSICmdResp *resp;
Srb = (PSCSI_REQUEST_BLOCK)cmd->sc;
resp = &cmd->resp.cmd;
srbExt = (PSRB_EXTENSION)Srb->SrbExtension;
switch (resp->response)
{
case VIRTIO_SCSI_S_OK:
Srb->ScsiStatus = resp->status;
Srb->SrbStatus = (Srb->ScsiStatus == SCSISTAT_GOOD) ? SRB_STATUS_SUCCESS : SRB_STATUS_ERROR;
break;
case VIRTIO_SCSI_S_UNDERRUN:
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("VIRTIO_SCSI_S_UNDERRUN\n"));
Srb->SrbStatus = SRB_STATUS_DATA_OVERRUN;
break;
case VIRTIO_SCSI_S_ABORTED:
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("VIRTIO_SCSI_S_ABORTED\n"));
Srb->SrbStatus = SRB_STATUS_ABORTED;
break;
case VIRTIO_SCSI_S_BAD_TARGET:
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("VIRTIO_SCSI_S_BAD_TARGET\n"));
Srb->SrbStatus = SRB_STATUS_INVALID_TARGET_ID;
break;
case VIRTIO_SCSI_S_RESET:
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("VIRTIO_SCSI_S_RESET\n"));
Srb->SrbStatus = SRB_STATUS_BUS_RESET;
break;
case VIRTIO_SCSI_S_BUSY:
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("VIRTIO_SCSI_S_BUSY\n"));
Srb->SrbStatus = SRB_STATUS_BUSY;
break;
case VIRTIO_SCSI_S_TRANSPORT_FAILURE:
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("VIRTIO_SCSI_S_TRANSPORT_FAILURE\n"));
Srb->SrbStatus = SRB_STATUS_ERROR;
break;
case VIRTIO_SCSI_S_TARGET_FAILURE:
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("VIRTIO_SCSI_S_TARGET_FAILURE\n"));
Srb->SrbStatus = SRB_STATUS_ERROR;
break;
case VIRTIO_SCSI_S_NEXUS_FAILURE:
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("VIRTIO_SCSI_S_NEXUS_FAILURE\n"));
Srb->SrbStatus = SRB_STATUS_ERROR;
break;
case VIRTIO_SCSI_S_FAILURE:
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("VIRTIO_SCSI_S_FAILURE\n"));
Srb->SrbStatus = SRB_STATUS_ERROR;
break;
default:
Srb->SrbStatus = SRB_STATUS_ERROR;
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("Unknown response %d\n", resp->response));
break;
}
if (Srb->SrbStatus != SRB_STATUS_SUCCESS)
{
PSENSE_DATA pSense = (PSENSE_DATA) Srb->SenseInfoBuffer;
if (Srb->SenseInfoBufferLength >= FIELD_OFFSET(SENSE_DATA, CommandSpecificInformation)) {
memcpy(Srb->SenseInfoBuffer, resp->sense,
min(resp->sense_len, Srb->SenseInfoBufferLength));
if (Srb->SrbStatus == SRB_STATUS_ERROR) {
Srb->SrbStatus |= SRB_STATUS_AUTOSENSE_VALID;
}
}
Srb->DataTransferLength = 0;
}
else if (srbExt->Xfer && Srb->DataTransferLength > srbExt->Xfer)
{
Srb->DataTransferLength = srbExt->Xfer;
Srb->SrbStatus = SRB_STATUS_DATA_OVERRUN;
}
if (srbExt && srbExt->Xfer && Srb->DataTransferLength > srbExt->Xfer) {
Srb->DataTransferLength = srbExt->Xfer;
Srb->SrbStatus = SRB_STATUS_DATA_OVERRUN;
}
--adaptExt->in_fly;
StorPortNotification(RequestComplete,
DeviceExtension,
Srb);
if (adaptExt->in_fly > 0)
{
adaptExt->vq[2]->vq_ops->kick(adaptExt->vq[2]);
}
}
return TRUE;
}
return FALSE;
}
BOOLEAN
SendSRB(
IN PVOID DeviceExtension,
IN PSCSI_REQUEST_BLOCK Srb
)
{
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
PSRB_EXTENSION srbExt = (PSRB_EXTENSION)Srb->SrbExtension;
PVOID va = NULL;
ULONGLONG pa = 0;
ULONG QueueNumber = 0;
ULONG OldIrql = 0;
ULONG MessageId = 0;
BOOLEAN kick = FALSE;
STOR_LOCK_HANDLE LockHandle = { 0 };
ULONG status = STOR_STATUS_SUCCESS;
ENTER_FN();
SET_VA_PA();
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("Srb %p issued on %d::%d QueueNumber %d\n",
Srb, srbExt->procNum.Group, srbExt->procNum.Number, QueueNumber));
if (adaptExt->num_queues > 1) {
QueueNumber = adaptExt->cpu_to_vq_map[srbExt->procNum.Number];
MessageId = QueueNumber + 1;
if (CHECKFLAG(adaptExt->perfFlags, STOR_PERF_OPTIMIZE_FOR_COMPLETION_DURING_STARTIO)) {
ProcessQueue(DeviceExtension, MessageId, FALSE);
}
// status = StorPortAcquireMSISpinLock(DeviceExtension, MessageId, &OldIrql);
if (status != STOR_STATUS_SUCCESS) {
RhelDbgPrint(TRACE_LEVEL_ERROR, ("%s StorPortAcquireMSISpinLock returned status 0x%x\n", __FUNCTION__, status));
}
}
else {
QueueNumber = VIRTIO_SCSI_REQUEST_QUEUE_0;
StorPortAcquireSpinLock(DeviceExtension, InterruptLock, NULL, &LockHandle);
}
if (virtqueue_add_buf(adaptExt->vq[QueueNumber],
&srbExt->sg[0],
srbExt->out, srbExt->in,
&srbExt->cmd, va, pa) >= 0){
kick = TRUE;
}
else {
RhelDbgPrint(TRACE_LEVEL_WARNING, ("%s Cant add packet to queue.\n", __FUNCTION__));
//FIXME
}
if (adaptExt->num_queues > 1) {
// status = StorPortReleaseMSISpinLock(DeviceExtension, MessageId, OldIrql);
if (status != STOR_STATUS_SUCCESS) {
RhelDbgPrint(TRACE_LEVEL_ERROR, ("%s StorPortReleaseMSISpinLock returned status 0x%x\n", __FUNCTION__, status));
}
}
else {
StorPortReleaseSpinLock(DeviceExtension, &LockHandle);
}
if (kick == TRUE) {
virtqueue_kick(adaptExt->vq[QueueNumber]);
}
return kick;
EXIT_FN();
}
BOOLEAN
VirtIoMSInterruptRoutine (
IN PVOID DeviceExtension,
IN ULONG MessageID
)
{
pblk_req vbr;
unsigned int len;
PADAPTER_EXTENSION adaptExt;
PSCSI_REQUEST_BLOCK Srb;
BOOLEAN isInterruptServiced = FALSE;
PRHEL_SRB_EXTENSION srbExt;
adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
RhelDbgPrint(TRACE_LEVEL_VERBOSE,
("<--->%s : MessageID 0x%x\n", __FUNCTION__, MessageID));
if (MessageID == 0) {
RhelGetDiskGeometry(DeviceExtension);
return TRUE;
}
while((vbr = (pblk_req)adaptExt->vq->vq_ops->get_buf(adaptExt->vq, &len)) != NULL) {
Srb = (PSCSI_REQUEST_BLOCK)vbr->req;
if (Srb) {
switch (vbr->status) {
case VIRTIO_BLK_S_OK:
Srb->SrbStatus = SRB_STATUS_SUCCESS;
break;
case VIRTIO_BLK_S_UNSUPP:
Srb->SrbStatus = SRB_STATUS_INVALID_REQUEST;
break;
default:
Srb->SrbStatus = SRB_STATUS_ERROR;
RhelDbgPrint(TRACE_LEVEL_ERROR, ("SRB_STATUS_ERROR\n"));
break;
}
}
if (vbr->out_hdr.type == VIRTIO_BLK_T_FLUSH) {
--adaptExt->in_fly;
CompleteSRB(DeviceExtension, Srb);
} else if (vbr->out_hdr.type == VIRTIO_BLK_T_GET_ID) {
adaptExt->sn_ok = TRUE;
} else if (Srb) {
--adaptExt->in_fly;
srbExt = (PRHEL_SRB_EXTENSION)Srb->SrbExtension;
if (srbExt->fua == TRUE) {
RemoveEntryList(&vbr->list_entry);
Srb->SrbStatus = SRB_STATUS_PENDING;
Srb->ScsiStatus = SCSISTAT_GOOD;
if (!RhelDoFlush(DeviceExtension, Srb, FALSE)) {
Srb->SrbStatus = SRB_STATUS_ERROR;
CompleteSRB(DeviceExtension, Srb);
} else {
srbExt->fua = FALSE;
}
} else {
CompleteDPC(DeviceExtension, vbr, MessageID);
}
}
isInterruptServiced = TRUE;
}
if (adaptExt->in_fly > 0) {
adaptExt->vq->vq_ops->kick(adaptExt->vq);
}
return isInterruptServiced;
}
SCSI_ADAPTER_CONTROL_STATUS
VirtIoAdapterControl(
IN PVOID DeviceExtension,
IN SCSI_ADAPTER_CONTROL_TYPE ControlType,
IN PVOID Parameters
)
{
PSCSI_SUPPORTED_CONTROL_TYPE_LIST ControlTypeList;
ULONG AdjustedMaxControlType;
ULONG Index;
PADAPTER_EXTENSION adaptExt;
SCSI_ADAPTER_CONTROL_STATUS status = ScsiAdapterControlUnsuccessful;
BOOLEAN SupportedConrolTypes[5] = {TRUE, TRUE, TRUE, FALSE, FALSE};
adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
RhelDbgPrint(TRACE_LEVEL_VERBOSE, ("%s %d\n", __FUNCTION__, ControlType));
switch (ControlType) {
case ScsiQuerySupportedControlTypes: {
RhelDbgPrint(TRACE_LEVEL_VERBOSE, ("ScsiQuerySupportedControlTypes\n"));
ControlTypeList = (PSCSI_SUPPORTED_CONTROL_TYPE_LIST)Parameters;
AdjustedMaxControlType =
(ControlTypeList->MaxControlType < 5) ?
ControlTypeList->MaxControlType :
5;
for (Index = 0; Index < AdjustedMaxControlType; Index++) {
ControlTypeList->SupportedTypeList[Index] =
SupportedConrolTypes[Index];
}
status = ScsiAdapterControlSuccess;
break;
}
case ScsiStopAdapter: {
RhelDbgPrint(TRACE_LEVEL_VERBOSE, ("ScsiStopAdapter\n"));
RhelShutDown(DeviceExtension);
status = ScsiAdapterControlSuccess;
break;
}
case ScsiRestartAdapter: {
RhelDbgPrint(TRACE_LEVEL_VERBOSE, ("ScsiRestartAdapter\n"));
VirtIODeviceReset(&adaptExt->vdev);
WriteVirtIODeviceWord(adaptExt->vdev.addr + VIRTIO_PCI_QUEUE_SEL, (USHORT)0);
WriteVirtIODeviceRegister(adaptExt->vdev.addr + VIRTIO_PCI_QUEUE_PFN,(USHORT)0);
adaptExt->vq = NULL;
if (!VirtIoHwInitialize(DeviceExtension))
{
RhelDbgPrint(TRACE_LEVEL_FATAL, ("Cannot Initialize HW\n"));
break;
}
status = ScsiAdapterControlSuccess;
break;
}
default:
break;
}
return status;
}
BOOLEAN
VirtIoInterrupt(
IN PVOID DeviceExtension
)
{
pblk_req vbr;
unsigned int len;
PADAPTER_EXTENSION adaptExt;
BOOLEAN isInterruptServiced = FALSE;
PSCSI_REQUEST_BLOCK Srb;
ULONG intReason = 0;
PRHEL_SRB_EXTENSION srbExt;
adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
RhelDbgPrint(TRACE_LEVEL_VERBOSE, ("%s (%d)\n", __FUNCTION__, KeGetCurrentIrql()));
intReason = VirtIODeviceISR((VirtIODevice*)DeviceExtension);
if ( intReason == 1) {
isInterruptServiced = TRUE;
while((vbr = (pblk_req)adaptExt->vq->vq_ops->get_buf(adaptExt->vq, &len)) != NULL) {
Srb = (PSCSI_REQUEST_BLOCK)vbr->req;
if (Srb) {
switch (vbr->status) {
case VIRTIO_BLK_S_OK:
Srb->SrbStatus = SRB_STATUS_SUCCESS;
break;
case VIRTIO_BLK_S_UNSUPP:
Srb->SrbStatus = SRB_STATUS_INVALID_REQUEST;
break;
default:
Srb->SrbStatus = SRB_STATUS_ERROR;
RhelDbgPrint(TRACE_LEVEL_ERROR, ("SRB_STATUS_ERROR\n"));
break;
}
}
if (vbr->out_hdr.type == VIRTIO_BLK_T_FLUSH) {
#ifdef USE_STORPORT
--adaptExt->in_fly;
#endif
CompleteSRB(DeviceExtension, Srb);
} else if (vbr->out_hdr.type == VIRTIO_BLK_T_GET_ID) {
adaptExt->sn_ok = TRUE;
} else if (Srb) {
#ifdef USE_STORPORT
--adaptExt->in_fly;
#endif
srbExt = (PRHEL_SRB_EXTENSION)Srb->SrbExtension;
if (srbExt->fua) {
RemoveEntryList(&vbr->list_entry);
Srb->SrbStatus = SRB_STATUS_PENDING;
Srb->ScsiStatus = SCSISTAT_GOOD;
if (!RhelDoFlush(DeviceExtension, Srb, FALSE)) {
Srb->SrbStatus = SRB_STATUS_ERROR;
CompleteSRB(DeviceExtension, Srb);
} else {
srbExt->fua = FALSE;
}
} else {
CompleteDPC(DeviceExtension, vbr, 0);
}
}
}
} else if (intReason == 3) {
RhelGetDiskGeometry(DeviceExtension);
isInterruptServiced = TRUE;
}
#ifdef USE_STORPORT
if (adaptExt->in_fly > 0) {
adaptExt->vq->vq_ops->kick(adaptExt->vq);
}
#endif
RhelDbgPrint(TRACE_LEVEL_VERBOSE, ("%s isInterruptServiced = %d\n", __FUNCTION__, isInterruptServiced));
return isInterruptServiced;
}
BOOLEAN
VirtIoHwInitialize(
IN PVOID DeviceExtension
)
{
PADAPTER_EXTENSION adaptExt;
BOOLEAN ret = FALSE;
#ifdef MSI_SUPPORTED
MESSAGE_INTERRUPT_INFORMATION msi_info;
#endif
RhelDbgPrint(TRACE_LEVEL_VERBOSE, ("%s (%d)\n", __FUNCTION__, KeGetCurrentIrql()));
adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
adaptExt->msix_vectors = 0;
#ifdef MSI_SUPPORTED
while(StorPortGetMSIInfo(DeviceExtension, adaptExt->msix_vectors, &msi_info) == STOR_STATUS_SUCCESS) {
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("MessageId = %x\n", msi_info.MessageId));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("MessageData = %x\n", msi_info.MessageData));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("InterruptVector = %x\n", msi_info.InterruptVector));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("InterruptLevel = %x\n", msi_info.InterruptLevel));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("InterruptMode = %s\n", msi_info.InterruptMode == LevelSensitive ? "LevelSensitive" : "Latched"));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("MessageAddress = %p\n\n", msi_info.MessageAddress));
++adaptExt->msix_vectors;
}
if(!adaptExt->dump_mode && (adaptExt->msix_vectors > 1)) {
adaptExt->vq = FindVirtualQueue(adaptExt, 0, adaptExt->msix_vectors - 1);
}
#endif
if(!adaptExt->vq) {
adaptExt->vq = FindVirtualQueue(adaptExt, 0, 0);
}
if (!adaptExt->vq) {
LogError(DeviceExtension,
SP_INTERNAL_ADAPTER_ERROR,
__LINE__);
RhelDbgPrint(TRACE_LEVEL_FATAL, ("Cannot find snd virtual queue\n"));
VirtIODeviceAddStatus(&adaptExt->vdev, VIRTIO_CONFIG_S_FAILED);
return ret;
}
RhelGetDiskGeometry(DeviceExtension);
memset(&adaptExt->inquiry_data, 0, sizeof(INQUIRYDATA));
adaptExt->inquiry_data.ANSIVersion = 4;
adaptExt->inquiry_data.ResponseDataFormat = 2;
adaptExt->inquiry_data.CommandQueue = 1;
adaptExt->inquiry_data.DeviceType = DIRECT_ACCESS_DEVICE;
adaptExt->inquiry_data.Wide32Bit = 1;
adaptExt->inquiry_data.AdditionalLength = 91;
ScsiPortMoveMemory(&adaptExt->inquiry_data.VendorId, "Red Hat ", sizeof("Red Hat "));
ScsiPortMoveMemory(&adaptExt->inquiry_data.ProductId, "VirtIO", sizeof("VirtIO"));
ScsiPortMoveMemory(&adaptExt->inquiry_data.ProductRevisionLevel, "0001", sizeof("0001"));
ScsiPortMoveMemory(&adaptExt->inquiry_data.VendorSpecific, "0001", sizeof("0001"));
if(!adaptExt->dump_mode && !adaptExt->sn_ok)
{
RhelGetSerialNumber(DeviceExtension);
}
ret = TRUE;
#ifdef USE_STORPORT
if(!adaptExt->dump_mode && !adaptExt->dpc_ok)
{
ret = StorPortEnablePassiveInitialization(DeviceExtension, VirtIoPassiveInitializeRoutine);
}
#endif
if (ret) {
VirtIODeviceAddStatus(&adaptExt->vdev, VIRTIO_CONFIG_S_DRIVER_OK);
} else {
VirtIODeviceAddStatus(&adaptExt->vdev, VIRTIO_CONFIG_S_FAILED);
}
return ret;
}
ULONG
VirtIoFindAdapter(
IN PVOID DeviceExtension,
IN PVOID HwContext,
IN PVOID BusInformation,
IN PCHAR ArgumentString,
IN OUT PPORT_CONFIGURATION_INFORMATION ConfigInfo,
OUT PBOOLEAN Again
)
{
PACCESS_RANGE accessRange;
PADAPTER_EXTENSION adaptExt;
ULONG_PTR deviceBase;
ULONG allocationSize;
ULONG pageNum;
#ifdef MSI_SUPPORTED
PPCI_COMMON_CONFIG pPciConf = NULL;
UCHAR pci_cfg_buf[256];
ULONG pci_cfg_len;
#endif
UNREFERENCED_PARAMETER( HwContext );
UNREFERENCED_PARAMETER( BusInformation );
UNREFERENCED_PARAMETER( ArgumentString );
UNREFERENCED_PARAMETER( Again );
adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
adaptExt->dump_mode = IsCrashDumpMode;
ConfigInfo->Master = TRUE;
ConfigInfo->ScatterGather = TRUE;
ConfigInfo->DmaWidth = Width32Bits;
ConfigInfo->Dma32BitAddresses = TRUE;
ConfigInfo->Dma64BitAddresses = TRUE;
ConfigInfo->WmiDataProvider = FALSE;
ConfigInfo->AlignmentMask = 0x3;
#ifdef USE_STORPORT
ConfigInfo->MapBuffers = STOR_MAP_NON_READ_WRITE_BUFFERS;
ConfigInfo->SynchronizationModel = StorSynchronizeFullDuplex;
#ifdef MSI_SUPPORTED
ConfigInfo->HwMSInterruptRoutine = VirtIoMSInterruptRoutine;
ConfigInfo->InterruptSynchronizationMode=InterruptSynchronizePerMessage;
#endif
#else
ConfigInfo->MapBuffers = TRUE;
#endif
accessRange = &(*ConfigInfo->AccessRanges)[0];
ASSERT (FALSE == accessRange->RangeInMemory) ;
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("Port Resource [%08I64X-%08I64X]\n",
accessRange->RangeStart.QuadPart,
accessRange->RangeStart.QuadPart +
accessRange->RangeLength));
if ( accessRange->RangeLength < IO_PORT_LENGTH) {
LogError(DeviceExtension,
SP_INTERNAL_ADAPTER_ERROR,
__LINE__);
RhelDbgPrint(TRACE_LEVEL_FATAL, ("Wrong access range %x bytes\n", accessRange->RangeLength));
return SP_RETURN_NOT_FOUND;
}
#ifndef USE_STORPORT
if (!ScsiPortValidateRange(DeviceExtension,
ConfigInfo->AdapterInterfaceType,
ConfigInfo->SystemIoBusNumber,
accessRange->RangeStart,
accessRange->RangeLength,
(BOOLEAN)!accessRange->RangeInMemory)) {
LogError(DeviceExtension,
SP_INTERNAL_ADAPTER_ERROR,
__LINE__);
RhelDbgPrint(TRACE_LEVEL_FATAL, ("Range validation failed %x for %x bytes\n",
(*ConfigInfo->AccessRanges)[0].RangeStart.LowPart,
(*ConfigInfo->AccessRanges)[0].RangeLength));
return SP_RETURN_ERROR;
}
#endif
ConfigInfo->NumberOfBuses = 1;
ConfigInfo->MaximumNumberOfTargets = 1;
ConfigInfo->MaximumNumberOfLogicalUnits = 1;
deviceBase = (ULONG_PTR)ScsiPortGetDeviceBase(DeviceExtension,
ConfigInfo->AdapterInterfaceType,
ConfigInfo->SystemIoBusNumber,
accessRange->RangeStart,
accessRange->RangeLength,
(BOOLEAN)!accessRange->RangeInMemory);
if (deviceBase == (ULONG_PTR)NULL) {
LogError(DeviceExtension,
SP_INTERNAL_ADAPTER_ERROR,
__LINE__);
RhelDbgPrint(TRACE_LEVEL_FATAL, ("Couldn't map %x for %x bytes\n",
(*ConfigInfo->AccessRanges)[0].RangeStart.LowPart,
(*ConfigInfo->AccessRanges)[0].RangeLength));
return SP_RETURN_ERROR;
}
VirtIODeviceInitialize(&adaptExt->vdev, deviceBase, sizeof(adaptExt->vdev));
VirtIODeviceAddStatus(&adaptExt->vdev, VIRTIO_CONFIG_S_DRIVER);
adaptExt->msix_enabled = FALSE;
#ifdef MSI_SUPPORTED
pci_cfg_len = StorPortGetBusData (DeviceExtension,
PCIConfiguration,
ConfigInfo->SystemIoBusNumber,
(ULONG)ConfigInfo->SlotNumber,
(PVOID)pci_cfg_buf,
(ULONG)256);
if (pci_cfg_len == 256)
{
UCHAR CapOffset;
PPCI_MSIX_CAPABILITY pMsixCapOffset;
pPciConf = (PPCI_COMMON_CONFIG)pci_cfg_buf;
if ( (pPciConf->Status & PCI_STATUS_CAPABILITIES_LIST) == 0)
{
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("NO CAPABILITIES_LIST\n"));
}
else
{
if ( (pPciConf->HeaderType & (~PCI_MULTIFUNCTION)) == PCI_DEVICE_TYPE )
{
CapOffset = pPciConf->u.type0.CapabilitiesPtr;
while (CapOffset != 0)
{
pMsixCapOffset = (PPCI_MSIX_CAPABILITY)(pci_cfg_buf + CapOffset);
if ( pMsixCapOffset->Header.CapabilityID == PCI_CAPABILITY_ID_MSIX )
{
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("MessageControl.TableSize = %d\n", pMsixCapOffset->MessageControl.TableSize));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("MessageControl.FunctionMask = %d\n", pMsixCapOffset->MessageControl.FunctionMask));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("MessageControl.MSIXEnable = %d\n", pMsixCapOffset->MessageControl.MSIXEnable));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("MessageTable = %p\n", pMsixCapOffset->MessageTable));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("PBATable = %d\n", pMsixCapOffset->PBATable));
adaptExt->msix_enabled = (pMsixCapOffset->MessageControl.MSIXEnable == 1);
break;
}
else
{
CapOffset = pMsixCapOffset->Header.Next;
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("CapabilityID = %x, Next CapOffset = %x\n", pMsixCapOffset->Header.CapabilityID, CapOffset));
}
}
VirtIODeviceSetMSIXUsed(&adaptExt->vdev, adaptExt->msix_enabled);
}
else
{
RhelDbgPrint(TRACE_LEVEL_FATAL, ("NOT A PCI_DEVICE_TYPE\n"));
}
}
}
else
{
RhelDbgPrint(TRACE_LEVEL_FATAL, ("CANNOT READ PCI CONFIGURATION SPACE %d\n", pci_cfg_len));
}
#endif
VirtIODeviceReset(&adaptExt->vdev);
VirtIODeviceAddStatus(&adaptExt->vdev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
WriteVirtIODeviceWord(adaptExt->vdev.addr + VIRTIO_PCI_QUEUE_SEL, (USHORT)0);
if (adaptExt->dump_mode) {
WriteVirtIODeviceWord(adaptExt->vdev.addr + VIRTIO_PCI_QUEUE_PFN, (USHORT)0);
}
adaptExt->features = ReadVirtIODeviceRegister(adaptExt->vdev.addr + VIRTIO_PCI_HOST_FEATURES);
ConfigInfo->CachesData = CHECKBIT(adaptExt->features, VIRTIO_BLK_F_WCACHE) ? TRUE : FALSE;
if (ConfigInfo->CachesData) {
u32 GuestFeatures = 0;
VirtIOFeatureEnable(GuestFeatures, VIRTIO_BLK_F_WCACHE);
VirtIODeviceWriteGuestFeatures(&adaptExt->vdev, GuestFeatures);
}
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("VIRTIO_BLK_F_WCACHE = %d\n", ConfigInfo->CachesData));
VirtIODeviceQueryQueueAllocation(&adaptExt->vdev, 0, &pageNum, &allocationSize);
if(adaptExt->dump_mode) {
ConfigInfo->NumberOfPhysicalBreaks = 8;
} else {
ConfigInfo->NumberOfPhysicalBreaks = MAX_PHYS_SEGMENTS + 1;
}
ConfigInfo->MaximumTransferLength = 0x00FFFFFF;
adaptExt->queue_depth = pageNum / ConfigInfo->NumberOfPhysicalBreaks - 1;
#if (INDIRECT_SUPPORTED)
if(!adaptExt->dump_mode) {
adaptExt->indirect = CHECKBIT(adaptExt->features, VIRTIO_RING_F_INDIRECT_DESC);
}
if(adaptExt->indirect) {
adaptExt->queue_depth = pageNum;
}
#else
adaptExt->indirect = 0;
#endif
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("breaks_number = %x queue_depth = %x\n",
ConfigInfo->NumberOfPhysicalBreaks,
adaptExt->queue_depth));
adaptExt->uncachedExtensionVa = ScsiPortGetUncachedExtension(DeviceExtension, ConfigInfo, allocationSize);
if (!adaptExt->uncachedExtensionVa) {
LogError(DeviceExtension,
SP_INTERNAL_ADAPTER_ERROR,
__LINE__);
RhelDbgPrint(TRACE_LEVEL_FATAL, ("Couldn't get uncached extension\n"));
return SP_RETURN_ERROR;
}
InitializeListHead(&adaptExt->list_head);
#ifdef USE_STORPORT
InitializeListHead(&adaptExt->complete_list);
#endif
return SP_RETURN_FOUND;
}
ULONG
DriverEntry(
IN PVOID DriverObject,
IN PVOID RegistryPath
)
{
HW_INITIALIZATION_DATA hwInitData;
ULONG initResult;
#ifndef USE_STORPORT
UCHAR venId[4] = {'1', 'A', 'F', '4'};
UCHAR devId[4] = {'1', '0', '0', '1'};
#endif
InitializeDebugPrints((PDRIVER_OBJECT)DriverObject, (PUNICODE_STRING)RegistryPath);
RhelDbgPrint(TRACE_LEVEL_ERROR, ("Viostor driver started...built on %s %s\n", __DATE__, __TIME__));
IsCrashDumpMode = FALSE;
if (RegistryPath == NULL) {
RhelDbgPrint(TRACE_LEVEL_INFORMATION,
("DriverEntry: Crash dump mode\n"));
IsCrashDumpMode = TRUE;
}
memset(&hwInitData, 0, sizeof(HW_INITIALIZATION_DATA));
hwInitData.HwInitializationDataSize = sizeof(HW_INITIALIZATION_DATA);
hwInitData.HwFindAdapter = VirtIoFindAdapter;
hwInitData.HwInitialize = VirtIoHwInitialize;
hwInitData.HwStartIo = VirtIoStartIo;
hwInitData.HwInterrupt = VirtIoInterrupt;
hwInitData.HwResetBus = VirtIoResetBus;
hwInitData.HwAdapterControl = VirtIoAdapterControl;
#ifdef USE_STORPORT
hwInitData.HwBuildIo = VirtIoBuildIo;
#endif
hwInitData.NeedPhysicalAddresses = TRUE;
hwInitData.TaggedQueuing = TRUE;
hwInitData.AutoRequestSense = TRUE;
hwInitData.MultipleRequestPerLu = TRUE;
hwInitData.DeviceExtensionSize = sizeof(ADAPTER_EXTENSION);
hwInitData.SrbExtensionSize = sizeof(RHEL_SRB_EXTENSION);
hwInitData.AdapterInterfaceType = PCIBus;
#ifndef USE_STORPORT
hwInitData.VendorIdLength = 4;
hwInitData.VendorId = venId;
hwInitData.DeviceIdLength = 4;
hwInitData.DeviceId = devId;
#endif
hwInitData.NumberOfAccessRanges = 1;
#ifdef USE_STORPORT
hwInitData.MapBuffers = STOR_MAP_NON_READ_WRITE_BUFFERS;
#else
hwInitData.MapBuffers = TRUE;
#endif
initResult = ScsiPortInitialize(DriverObject,
RegistryPath,
&hwInitData,
NULL);
RhelDbgPrint(TRACE_LEVEL_VERBOSE,
("Initialize returned 0x%x\n", initResult));
return initResult;
}
BOOLEAN
VioScsiBuildIo(
IN PVOID DeviceExtension,
IN PSCSI_REQUEST_BLOCK Srb
)
{
PCDB cdb;
ULONG i;
ULONG fragLen;
ULONG sgElement;
ULONG sgMaxElements;
PADAPTER_EXTENSION adaptExt;
PSRB_EXTENSION srbExt;
PSTOR_SCATTER_GATHER_LIST sgList;
VirtIOSCSICmd *cmd;
ENTER_FN();
cdb = (PCDB)&Srb->Cdb[0];
srbExt = (PSRB_EXTENSION)Srb->SrbExtension;
adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
if( (Srb->PathId > 0) ||
(Srb->TargetId >= adaptExt->scsi_config.max_target) ||
(Srb->Lun >= adaptExt->scsi_config.max_lun) ) {
Srb->SrbStatus = SRB_STATUS_NO_DEVICE;
StorPortNotification(RequestComplete,
DeviceExtension,
Srb);
return FALSE;
}
RhelDbgPrint(TRACE_LEVEL_VERBOSE, ("<-->%s (%d::%d::%d)\n", DbgGetScsiOpStr(Srb), Srb->PathId, Srb->TargetId, Srb->Lun));
memset(srbExt, 0, sizeof(*srbExt));
cmd = &srbExt->cmd;
cmd->sc = Srb;
cmd->req.cmd.lun[0] = 1;
cmd->req.cmd.lun[1] = Srb->TargetId;
cmd->req.cmd.lun[2] = 0;
cmd->req.cmd.lun[3] = Srb->Lun;
cmd->req.cmd.tag = (unsigned long)Srb;
cmd->req.cmd.task_attr = VIRTIO_SCSI_S_SIMPLE;
cmd->req.cmd.prio = 0;
cmd->req.cmd.crn = 0;
memcpy(cmd->req.cmd.cdb, cdb, min(VIRTIO_SCSI_CDB_SIZE, Srb->CdbLength));
sgElement = 0;
srbExt->sg[sgElement].physAddr = StorPortGetPhysicalAddress(DeviceExtension, NULL, &cmd->req.cmd, &fragLen);
srbExt->sg[sgElement].ulSize = sizeof(cmd->req.cmd);
sgElement++;
sgList = StorPortGetScatterGatherList(DeviceExtension, Srb);
if (sgList)
{
sgMaxElements = sgList->NumberOfElements;
if((Srb->SrbFlags & SRB_FLAGS_DATA_OUT) == SRB_FLAGS_DATA_OUT) {
for (i = 0; i < sgMaxElements; i++, sgElement++) {
srbExt->sg[sgElement].physAddr = sgList->List[i].PhysicalAddress;
srbExt->sg[sgElement].ulSize = sgList->List[i].Length;
srbExt->Xfer += sgList->List[i].Length;
}
}
}
srbExt->out = sgElement;
srbExt->sg[sgElement].physAddr = StorPortGetPhysicalAddress(DeviceExtension, NULL, &cmd->resp.cmd, &fragLen);
srbExt->sg[sgElement].ulSize = sizeof(cmd->resp.cmd);
sgElement++;
if (sgList)
{
sgMaxElements = sgList->NumberOfElements;
if((Srb->SrbFlags & SRB_FLAGS_DATA_OUT) != SRB_FLAGS_DATA_OUT) {
for (i = 0; i < sgMaxElements; i++, sgElement++) {
srbExt->sg[sgElement].physAddr = sgList->List[i].PhysicalAddress;
srbExt->sg[sgElement].ulSize = sgList->List[i].Length;
srbExt->Xfer += sgList->List[i].Length;
}
}
}
srbExt->in = sgElement - srbExt->out;
EXIT_FN();
return TRUE;
}
ULONG
VioScsiFindAdapter(
IN PVOID DeviceExtension,
IN PVOID HwContext,
IN PVOID BusInformation,
IN PCHAR ArgumentString,
IN OUT PPORT_CONFIGURATION_INFORMATION ConfigInfo,
IN PBOOLEAN Again
)
{
PADAPTER_EXTENSION adaptExt;
ULONG allocationSize;
ULONG pageNum;
ULONG dummy;
ULONG Size;
#if (MSI_SUPPORTED == 1)
PPCI_COMMON_CONFIG pPciConf = NULL;
UCHAR pci_cfg_buf[256];
ULONG pci_cfg_len;
#endif
UNREFERENCED_PARAMETER( HwContext );
UNREFERENCED_PARAMETER( BusInformation );
UNREFERENCED_PARAMETER( ArgumentString );
UNREFERENCED_PARAMETER( Again );
ENTER_FN();
adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
memset(adaptExt, 0, sizeof(ADAPTER_EXTENSION));
adaptExt->dump_mode = IsCrashDumpMode;
ConfigInfo->Master = TRUE;
ConfigInfo->ScatterGather = TRUE;
ConfigInfo->DmaWidth = Width32Bits;
ConfigInfo->Dma32BitAddresses = TRUE;
ConfigInfo->Dma64BitAddresses = TRUE;
ConfigInfo->WmiDataProvider = FALSE;
ConfigInfo->AlignmentMask = 0x3;
ConfigInfo->MapBuffers = STOR_MAP_NON_READ_WRITE_BUFFERS;
ConfigInfo->SynchronizationModel = StorSynchronizeFullDuplex;
#if (MSI_SUPPORTED == 1)
ConfigInfo->HwMSInterruptRoutine = VioScsiMSInterrupt;
ConfigInfo->InterruptSynchronizationMode=InterruptSynchronizePerMessage;
#endif
if (!InitHW(DeviceExtension, ConfigInfo)) {
RhelDbgPrint(TRACE_LEVEL_ERROR, ("Cannot initialize HardWare\n"));
return SP_RETURN_NOT_FOUND;
}
#if (MSI_SUPPORTED == 1)
pci_cfg_len = StorPortGetBusData (DeviceExtension,
PCIConfiguration,
ConfigInfo->SystemIoBusNumber,
(ULONG)ConfigInfo->SlotNumber,
(PVOID)pci_cfg_buf,
(ULONG)256);
if (pci_cfg_len == 256)
{
UCHAR CapOffset;
PPCI_MSIX_CAPABILITY pMsixCapOffset;
pPciConf = (PPCI_COMMON_CONFIG)pci_cfg_buf;
if ( (pPciConf->Status & PCI_STATUS_CAPABILITIES_LIST) == 0)
{
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("NO CAPABILITIES_LIST\n"));
}
else
{
if ( (pPciConf->HeaderType & (~PCI_MULTIFUNCTION)) == PCI_DEVICE_TYPE )
{
CapOffset = pPciConf->u.type0.CapabilitiesPtr;
while (CapOffset != 0)
{
pMsixCapOffset = (PPCI_MSIX_CAPABILITY)(pci_cfg_buf + CapOffset);
if ( pMsixCapOffset->Header.CapabilityID == PCI_CAPABILITY_ID_MSIX )
{
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("MessageControl.TableSize = %d\n", pMsixCapOffset->MessageControl.TableSize));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("MessageControl.FunctionMask = %d\n", pMsixCapOffset->MessageControl.FunctionMask));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("MessageControl.MSIXEnable = %d\n", pMsixCapOffset->MessageControl.MSIXEnable));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("MessageTable = %p\n", pMsixCapOffset->MessageTable));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("PBATable = %d\n", pMsixCapOffset->PBATable));
adaptExt->msix_enabled = (pMsixCapOffset->MessageControl.MSIXEnable == 1);
}
else
{
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("CapabilityID = %x, Next CapOffset = %x\n", pMsixCapOffset->Header.CapabilityID, CapOffset));
}
CapOffset = pMsixCapOffset->Header.Next;
}
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("msix_enabled = %d\n", adaptExt->msix_enabled));
VirtIODeviceSetMSIXUsed(&adaptExt->vdev, adaptExt->msix_enabled);
}
else
{
RhelDbgPrint(TRACE_LEVEL_FATAL, ("NOT A PCI_DEVICE_TYPE\n"));
}
}
}
else
{
RhelDbgPrint(TRACE_LEVEL_FATAL, ("CANNOT READ PCI CONFIGURATION SPACE %d\n", pci_cfg_len));
}
#endif
GetScsiConfig(DeviceExtension);
ConfigInfo->NumberOfBuses = 1;
ConfigInfo->MaximumNumberOfTargets = (UCHAR)adaptExt->scsi_config.max_target;
ConfigInfo->MaximumNumberOfLogicalUnits = (UCHAR)adaptExt->scsi_config.max_lun;
if(adaptExt->dump_mode) {
ConfigInfo->NumberOfPhysicalBreaks = 8;
} else {
ConfigInfo->NumberOfPhysicalBreaks = min((MAX_PHYS_SEGMENTS + 1), adaptExt->scsi_config.seg_max);
}
ConfigInfo->MaximumTransferLength = 0x00FFFFFF;
VirtIODeviceReset(&adaptExt->vdev);
if (adaptExt->dump_mode) {
StorPortWritePortUshort(DeviceExtension, (PUSHORT)(adaptExt->device_base + VIRTIO_PCI_QUEUE_SEL), (USHORT)0);
StorPortWritePortUshort(DeviceExtension, (PUSHORT)(adaptExt->device_base + VIRTIO_PCI_QUEUE_PFN),(USHORT)0);
StorPortWritePortUshort(DeviceExtension, (PUSHORT)(adaptExt->device_base + VIRTIO_PCI_QUEUE_SEL), (USHORT)1);
StorPortWritePortUshort(DeviceExtension, (PUSHORT)(adaptExt->device_base + VIRTIO_PCI_QUEUE_PFN),(USHORT)0);
StorPortWritePortUshort(DeviceExtension, (PUSHORT)(adaptExt->device_base + VIRTIO_PCI_QUEUE_SEL), (USHORT)2);
StorPortWritePortUshort(DeviceExtension, (PUSHORT)(adaptExt->device_base + VIRTIO_PCI_QUEUE_PFN),(USHORT)0);
}
adaptExt->features = StorPortReadPortUlong(DeviceExtension, (PULONG)(adaptExt->device_base + VIRTIO_PCI_HOST_FEATURES));
allocationSize = 0;
adaptExt->offset[0] = 0;
VirtIODeviceQueryQueueAllocation(&adaptExt->vdev, 0, &pageNum, &Size);
allocationSize += ROUND_TO_PAGES(Size);
adaptExt->offset[1] = ROUND_TO_PAGES(Size);
VirtIODeviceQueryQueueAllocation(&adaptExt->vdev, 1, &dummy, &Size);
allocationSize += ROUND_TO_PAGES(Size);
adaptExt->offset[2] = adaptExt->offset[1] + ROUND_TO_PAGES(Size);
VirtIODeviceQueryQueueAllocation(&adaptExt->vdev, 2, &dummy, &Size);
allocationSize += ROUND_TO_PAGES(Size);
adaptExt->offset[3] = adaptExt->offset[2] + ROUND_TO_PAGES(Size);
allocationSize += ROUND_TO_PAGES(sizeof(SRB_EXTENSION));
adaptExt->offset[4] = adaptExt->offset[3] + ROUND_TO_PAGES(sizeof(SRB_EXTENSION));
allocationSize += ROUND_TO_PAGES(sizeof(VirtIOSCSIEventNode) * 8);
#if (INDIRECT_SUPPORTED == 1)
if(!adaptExt->dump_mode) {
adaptExt->indirect = CHECKBIT(adaptExt->features, VIRTIO_RING_F_INDIRECT_DESC);
}
#else
adaptExt->indirect = 0;
#endif
if(adaptExt->indirect) {
adaptExt->queue_depth = max(2, (pageNum / 4));
} else {
adaptExt->queue_depth = pageNum / ConfigInfo->NumberOfPhysicalBreaks - 1;
}
RhelDbgPrint(TRACE_LEVEL_ERROR, ("breaks_number = %x queue_depth = %x\n",
ConfigInfo->NumberOfPhysicalBreaks,
adaptExt->queue_depth));
adaptExt->uncachedExtensionVa = StorPortGetUncachedExtension(DeviceExtension, ConfigInfo, allocationSize);
if (!adaptExt->uncachedExtensionVa) {
LogError(DeviceExtension,
SP_INTERNAL_ADAPTER_ERROR,
__LINE__);
RhelDbgPrint(TRACE_LEVEL_FATAL, ("Can't get uncached extension\n"));
return SP_RETURN_ERROR;
}
return SP_RETURN_FOUND;
}
BOOLEAN
VioScsiHwInitialize(
IN PVOID DeviceExtension
)
{
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
PVOID ptr = adaptExt->uncachedExtensionVa;
ULONG i;
#if (MSI_SUPPORTED == 1)
MESSAGE_INTERRUPT_INFORMATION msi_info;
#endif
ENTER_FN();
adaptExt->msix_vectors = 0;
#if (MSI_SUPPORTED == 1)
while(StorPortGetMSIInfo(DeviceExtension, adaptExt->msix_vectors, &msi_info) == STOR_STATUS_SUCCESS) {
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("MessageId = %x\n", msi_info.MessageId));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("MessageData = %x\n", msi_info.MessageData));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("InterruptVector = %x\n", msi_info.InterruptVector));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("InterruptLevel = %x\n", msi_info.InterruptLevel));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("InterruptMode = %s\n", msi_info.InterruptMode == LevelSensitive ? "LevelSensitive" : "Latched"));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("MessageAddress = %p\n\n", msi_info.MessageAddress));
++adaptExt->msix_vectors;
}
if(!adaptExt->dump_mode && (adaptExt->msix_vectors > 1)) {
adaptExt->vq[0] = FindVirtualQueue(adaptExt, 0, 1);
}
if(!adaptExt->dump_mode && (adaptExt->msix_vectors > 2)) {
adaptExt->vq[1] = FindVirtualQueue(adaptExt, 1, 2);
}
if(!adaptExt->dump_mode && (adaptExt->msix_vectors > 3)) {
adaptExt->vq[2] = FindVirtualQueue(adaptExt, 2, 3);
}
#endif
if (!adaptExt->vq[0]) {
adaptExt->vq[0] = FindVirtualQueue(adaptExt, 0, 0);
}
if (!adaptExt->vq[0]) {
RhelDbgPrint(TRACE_LEVEL_FATAL, ("Cannot find virtual queue 0\n"));
return FALSE;
}
if (!adaptExt->vq[1]) {
adaptExt->vq[1] = FindVirtualQueue(adaptExt, 1, 0);
}
if (!adaptExt->vq[1]) {
RhelDbgPrint(TRACE_LEVEL_FATAL, ("Cannot find virtual queue 1\n"));
return FALSE;
}
if (!adaptExt->vq[2]) {
adaptExt->vq[2] = FindVirtualQueue(adaptExt, 2, 0);
}
if (!adaptExt->vq[2]) {
RhelDbgPrint(TRACE_LEVEL_FATAL, ("Cannot find virtual queue 2\n"));
return FALSE;
}
adaptExt->tmf_cmd.SrbExtension = (PSRB_EXTENSION)((ULONG_PTR)adaptExt->uncachedExtensionVa + adaptExt->offset[3]);
adaptExt->events = (PVirtIOSCSIEventNode)((ULONG_PTR)adaptExt->uncachedExtensionVa + adaptExt->offset[4]);
if (!adaptExt->dump_mode && CHECKBIT(adaptExt->features, VIRTIO_SCSI_F_HOTPLUG)) {
PVirtIOSCSIEventNode events = adaptExt->events;
for (i = 0; i < 8; i++) {
if (!KickEvent(DeviceExtension, (PVOID)(&events[i]))) {
RhelDbgPrint(TRACE_LEVEL_FATAL, ("Can't add event %d\n", i));
}
}
}
StorPortWritePortUshort(DeviceExtension,
(PUSHORT)(adaptExt->device_base + VIRTIO_PCI_GUEST_FEATURES),
(USHORT)((1 << VIRTIO_SCSI_F_HOTPLUG) | (1 << VIRTIO_SCSI_F_CHANGE)));
StorPortWritePortUchar(DeviceExtension,
(PUCHAR)(adaptExt->device_base + VIRTIO_PCI_STATUS),
(UCHAR)VIRTIO_CONFIG_S_DRIVER_OK);
EXIT_FN();
return TRUE;
}
BOOLEAN
RhelDoFlush(
PVOID DeviceExtension,
PSRB_TYPE Srb,
BOOLEAN resend,
BOOLEAN bIsr
)
{
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
PSRB_EXTENSION srbExt = SRB_EXTENSION(Srb);
ULONG fragLen = 0UL;
PVOID va = NULL;
ULONGLONG pa = 0ULL;
ULONG QueueNumber = 0;
ULONG OldIrql = 0;
ULONG MessageId = 0;
BOOLEAN result = FALSE;
bool notify = FALSE;
STOR_LOCK_HANDLE LockHandle = { 0 };
ULONG status = STOR_STATUS_SUCCESS;
struct virtqueue *vq = NULL;
SET_VA_PA();
if (resend) {
MessageId = srbExt->MessageID;
QueueNumber = MessageId - 1;
} else if (adaptExt->num_queues > 1) {
STARTIO_PERFORMANCE_PARAMETERS param;
param.Size = sizeof(STARTIO_PERFORMANCE_PARAMETERS);
status = StorPortGetStartIoPerfParams(DeviceExtension, (PSCSI_REQUEST_BLOCK)Srb, ¶m);
if (status == STOR_STATUS_SUCCESS && param.MessageNumber != 0) {
MessageId = param.MessageNumber;
QueueNumber = MessageId - 1;
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("%s srb %p, cpu %d :: QueueNumber %lu, MessageNumber %lu, ChannelNumber %lu.\n", __FUNCTION__, Srb, srbExt->cpu, QueueNumber, param.MessageNumber, param.ChannelNumber));
}
else {
RhelDbgPrint(TRACE_LEVEL_ERROR, ("%s StorPortGetStartIoPerfParams failed. srb %p cpu %d status 0x%x.\n",__FUNCTION__, Srb, srbExt->cpu, status));
QueueNumber = 0;
MessageId = 1;
}
} else {
QueueNumber = 0;
MessageId = 1;
}
srbExt->MessageID = MessageId;
vq = adaptExt->vq[QueueNumber];
srbExt->vbr.out_hdr.sector = 0;
srbExt->vbr.out_hdr.ioprio = 0;
srbExt->vbr.req = (struct request *)Srb;
srbExt->vbr.out_hdr.type = VIRTIO_BLK_T_FLUSH;
srbExt->out = 1;
srbExt->in = 1;
srbExt->vbr.sg[0].physAddr = StorPortGetPhysicalAddress(DeviceExtension, NULL, &srbExt->vbr.out_hdr, &fragLen);
srbExt->vbr.sg[0].length = sizeof(srbExt->vbr.out_hdr);
srbExt->vbr.sg[1].physAddr = StorPortGetPhysicalAddress(DeviceExtension, NULL, &srbExt->vbr.status, &fragLen);
srbExt->vbr.sg[1].length = sizeof(srbExt->vbr.status);
VioStorVQLock(DeviceExtension, MessageId, &LockHandle, FALSE);
if (virtqueue_add_buf(vq,
&srbExt->vbr.sg[0],
srbExt->out, srbExt->in,
&srbExt->vbr, va, pa) >= 0) {
notify = virtqueue_kick_prepare(vq);
VioStorVQUnlock(DeviceExtension, MessageId, &LockHandle, FALSE);
result = TRUE;
#ifdef DBG
InterlockedIncrement((LONG volatile*)&adaptExt->inqueue_cnt);
#endif
}
else {
VioStorVQUnlock(DeviceExtension, MessageId, &LockHandle, FALSE);
RhelDbgPrint(TRACE_LEVEL_FATAL, ("%s Can not add packet to queue %d.\n", __FUNCTION__, QueueNumber));
StorPortBusy(DeviceExtension, 2);
}
if (notify) {
virtqueue_notify(vq);
}
return result;
}
BOOLEAN
RhelDoReadWrite(PVOID DeviceExtension,
PSRB_TYPE Srb)
{
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
PSRB_EXTENSION srbExt = SRB_EXTENSION(Srb);
PVOID va = NULL;
ULONGLONG pa = 0ULL;
ULONG QueueNumber = 0;
ULONG OldIrql = 0;
ULONG MessageId = 0;
BOOLEAN result = FALSE;
bool notify = FALSE;
STOR_LOCK_HANDLE LockHandle = { 0 };
ULONG status = STOR_STATUS_SUCCESS;
struct virtqueue *vq = NULL;
SET_VA_PA();
if (adaptExt->num_queues > 1) {
STARTIO_PERFORMANCE_PARAMETERS param;
param.Size = sizeof(STARTIO_PERFORMANCE_PARAMETERS);
status = StorPortGetStartIoPerfParams(DeviceExtension, (PSCSI_REQUEST_BLOCK)Srb, ¶m);
if (status == STOR_STATUS_SUCCESS && param.MessageNumber != 0) {
MessageId = param.MessageNumber;
QueueNumber = MessageId - 1;
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("%s srb %p, cpu %d :: QueueNumber %lu, MessageNumber %lu, ChannelNumber %lu.\n", __FUNCTION__, Srb, srbExt->cpu, QueueNumber, param.MessageNumber, param.ChannelNumber));
}
else {
RhelDbgPrint(TRACE_LEVEL_ERROR, ("%s StorPortGetStartIoPerfParams failed srb %p cpu %d status 0x%x.\n", __FUNCTION__, Srb, srbExt->cpu, status));
QueueNumber = 0;
MessageId = 1;
}
}
else {
QueueNumber = 0;
MessageId = 1;
}
srbExt->MessageID = MessageId;
vq = adaptExt->vq[QueueNumber];
RhelDbgPrint(TRACE_LEVEL_VERBOSE, ("<--->%s : QueueNumber 0x%x vq = %p\n", __FUNCTION__, QueueNumber, vq));
VioStorVQLock(DeviceExtension, MessageId, &LockHandle, FALSE);
if (virtqueue_add_buf(vq,
&srbExt->vbr.sg[0],
srbExt->out, srbExt->in,
&srbExt->vbr, va, pa) >= 0) {
notify = virtqueue_kick_prepare(vq);
VioStorVQUnlock(DeviceExtension, MessageId, &LockHandle, FALSE);
#ifdef DBG
InterlockedIncrement((LONG volatile*)&adaptExt->inqueue_cnt);
#endif
result = TRUE;
}
else {
VioStorVQUnlock(DeviceExtension, MessageId, &LockHandle, FALSE);
RhelDbgPrint(TRACE_LEVEL_FATAL, ("%s Can not add packet to queue %d.\n", __FUNCTION__, QueueNumber));
StorPortBusy(DeviceExtension, 2);
}
if (notify) {
virtqueue_notify(vq);
}
#if (NTDDI_VERSION > NTDDI_WIN7)
if (adaptExt->num_queues > 1) {
if (CHECKFLAG(adaptExt->perfFlags, STOR_PERF_OPTIMIZE_FOR_COMPLETION_DURING_STARTIO)) {
VioStorCompleteRequest(DeviceExtension, MessageId, FALSE);
}
}
#endif
return result;
}
VOID
RhelGetDiskGeometry(
IN PVOID DeviceExtension
)
{
u64 cap;
u32 v;
struct virtio_blk_geometry vgeo;
PADAPTER_EXTENSION adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
adaptExt->features = ScsiPortReadPortUlong((PULONG)(adaptExt->vdev.addr + VIRTIO_PCI_HOST_FEATURES));
if (CHECKBIT(adaptExt->features, VIRTIO_BLK_F_BARRIER)) {
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("VIRTIO_BLK_F_BARRIER\n"));
}
if (CHECKBIT(adaptExt->features, VIRTIO_BLK_F_RO)) {
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("VIRTIO_BLK_F_RO\n"));
}
if (CHECKBIT(adaptExt->features, VIRTIO_BLK_F_SIZE_MAX)) {
VirtIODeviceGet( &adaptExt->vdev, FIELD_OFFSET(blk_config, size_max),
&v, sizeof(v));
adaptExt->info.size_max = v;
} else {
adaptExt->info.size_max = SECTOR_SIZE;
}
if (CHECKBIT(adaptExt->features, VIRTIO_BLK_F_SEG_MAX)) {
VirtIODeviceGet(&adaptExt->vdev, FIELD_OFFSET(blk_config, seg_max),
&v, sizeof(v));
adaptExt->info.seg_max = v;
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("VIRTIO_BLK_F_SEG_MAX = %d\n", adaptExt->info.seg_max));
}
if (CHECKBIT(adaptExt->features, VIRTIO_BLK_F_BLK_SIZE)) {
VirtIODeviceGet( &adaptExt->vdev, FIELD_OFFSET(blk_config, blk_size),
&v, sizeof(v));
adaptExt->info.blk_size = v;
} else {
adaptExt->info.blk_size = SECTOR_SIZE;
}
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("VIRTIO_BLK_F_BLK_SIZE = %d\n", adaptExt->info.blk_size));
if (CHECKBIT(adaptExt->features, VIRTIO_BLK_F_GEOMETRY)) {
VirtIODeviceGet( &adaptExt->vdev, FIELD_OFFSET(blk_config, geometry),
&vgeo, sizeof(vgeo));
adaptExt->info.geometry.cylinders= vgeo.cylinders;
adaptExt->info.geometry.heads = vgeo.heads;
adaptExt->info.geometry.sectors = vgeo.sectors;
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("VIRTIO_BLK_F_GEOMETRY. cylinders = %d heads = %d sectors = %d\n", adaptExt->info.geometry.cylinders, adaptExt->info.geometry.heads, adaptExt->info.geometry.sectors));
}
VirtIODeviceGet( &adaptExt->vdev, FIELD_OFFSET(blk_config, capacity),
&cap, sizeof(cap));
adaptExt->info.capacity = cap;
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("capacity = %08I64X\n", adaptExt->info.capacity));
if(CHECKBIT(adaptExt->features, VIRTIO_BLK_F_TOPOLOGY)) {
VirtIODeviceGet( &adaptExt->vdev, FIELD_OFFSET(blk_config, physical_block_exp),
&adaptExt->info.physical_block_exp, sizeof(adaptExt->info.physical_block_exp));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("physical_block_exp = %d\n", adaptExt->info.physical_block_exp));
VirtIODeviceGet( &adaptExt->vdev, FIELD_OFFSET(blk_config, alignment_offset),
&adaptExt->info.alignment_offset, sizeof(adaptExt->info.alignment_offset));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("alignment_offset = %d\n", adaptExt->info.alignment_offset));
VirtIODeviceGet( &adaptExt->vdev, FIELD_OFFSET(blk_config, min_io_size),
&adaptExt->info.min_io_size, sizeof(adaptExt->info.min_io_size));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("min_io_size = %d\n", adaptExt->info.min_io_size));
VirtIODeviceGet( &adaptExt->vdev, FIELD_OFFSET(blk_config, opt_io_size),
&adaptExt->info.opt_io_size, sizeof(adaptExt->info.opt_io_size));
RhelDbgPrint(TRACE_LEVEL_INFORMATION, ("opt_io_size = %d\n", adaptExt->info.opt_io_size));
}
}
BOOLEAN
VirtIoBuildIo(
IN PVOID DeviceExtension,
IN PSCSI_REQUEST_BLOCK Srb
)
{
PCDB cdb;
ULONG i;
ULONG dummy;
ULONG sgElement;
ULONG sgMaxElements;
PADAPTER_EXTENSION adaptExt;
PRHEL_SRB_EXTENSION srbExt;
PSTOR_SCATTER_GATHER_LIST sgList;
ULONGLONG lba;
ULONG blocks;
cdb = (PCDB)&Srb->Cdb[0];
srbExt = (PRHEL_SRB_EXTENSION)Srb->SrbExtension;
adaptExt = (PADAPTER_EXTENSION)DeviceExtension;
if(Srb->PathId || Srb->TargetId || Srb->Lun) {
Srb->SrbStatus = SRB_STATUS_NO_DEVICE;
ScsiPortNotification(RequestComplete,
DeviceExtension,
Srb);
return FALSE;
}
switch (cdb->CDB6GENERIC.OperationCode) {
case SCSIOP_READ:
case SCSIOP_WRITE:
case SCSIOP_WRITE_VERIFY:
case SCSIOP_READ6:
case SCSIOP_WRITE6:
case SCSIOP_READ12:
case SCSIOP_WRITE12:
case SCSIOP_WRITE_VERIFY12:
case SCSIOP_READ16:
case SCSIOP_WRITE16:
case SCSIOP_WRITE_VERIFY16: {
break;
}
default: {
Srb->SrbStatus = SRB_STATUS_SUCCESS;
return TRUE;
}
}
lba = RhelGetLba(DeviceExtension, cdb);
blocks = (Srb->DataTransferLength + adaptExt->info.blk_size - 1) / adaptExt->info.blk_size;
if (lba > adaptExt->lastLBA) {
RhelDbgPrint(TRACE_LEVEL_ERROR, ("SRB_STATUS_BAD_SRB_BLOCK_LENGTH lba = %llu lastLBA= %llu\n", lba, adaptExt->lastLBA));
Srb->SrbStatus = SRB_STATUS_BAD_SRB_BLOCK_LENGTH;
CompleteSRB(DeviceExtension, Srb);
return FALSE;
}
if ((lba + blocks) > adaptExt->lastLBA) {
blocks = (ULONG)(adaptExt->lastLBA + 1 - lba);
RhelDbgPrint(TRACE_LEVEL_ERROR, ("lba = %llu lastLBA= %llu blocks = %lu\n", lba, adaptExt->lastLBA, blocks));
Srb->DataTransferLength = (ULONG)(blocks * adaptExt->info.blk_size);
}
sgList = StorPortGetScatterGatherList(DeviceExtension, Srb);
sgMaxElements = min((MAX_PHYS_SEGMENTS + 1), sgList->NumberOfElements);
srbExt->Xfer = 0;
for (i = 0, sgElement = 1; i < sgMaxElements; i++, sgElement++) {
srbExt->vbr.sg[sgElement].physAddr = sgList->List[i].PhysicalAddress;
srbExt->vbr.sg[sgElement].length = sgList->List[i].Length;
srbExt->Xfer += sgList->List[i].Length;
}
srbExt->vbr.out_hdr.sector = lba;
srbExt->vbr.out_hdr.ioprio = 0;
srbExt->vbr.req = (PVOID)Srb;
srbExt->fua = (cdb->CDB10.ForceUnitAccess == 1);
if (Srb->SrbFlags & SRB_FLAGS_DATA_OUT) {
srbExt->vbr.out_hdr.type = VIRTIO_BLK_T_OUT;
srbExt->out = sgElement;
srbExt->in = 1;
} else {
srbExt->vbr.out_hdr.type = VIRTIO_BLK_T_IN;
srbExt->out = 1;
srbExt->in = sgElement;
}
srbExt->vbr.sg[0].physAddr = ScsiPortGetPhysicalAddress(DeviceExtension, NULL, &srbExt->vbr.out_hdr, &dummy);
srbExt->vbr.sg[0].length = sizeof(srbExt->vbr.out_hdr);
srbExt->vbr.sg[sgElement].physAddr = ScsiPortGetPhysicalAddress(DeviceExtension, NULL, &srbExt->vbr.status, &dummy);
srbExt->vbr.sg[sgElement].length = sizeof(srbExt->vbr.status);
return TRUE;
}