/**
* \brief globally enables the interrupts for the given device
*
* \param dev IOAT DMA device
* \param type the interrupt type to enable
*/
errval_t ioat_dma_device_irq_setup(struct ioat_dma_device *dev,
dma_irq_t type)
{
ioat_dma_intrctrl_t intcrtl = 0;
intcrtl = ioat_dma_intrctrl_intp_en_insert(intcrtl, 1);
dev->common.irq_type = type;
switch (type) {
case DMA_IRQ_MSIX:
IOATDEV_DEBUG("Initializing MSI-X interrupts \n", dev->common.id);
assert(!"NYI");
break;
case DMA_IRQ_MSI:
IOATDEV_DEBUG("Initializing MSI interrupts \n", dev->common.id);
assert(!"NYI");
break;
case DMA_IRQ_INTX:
IOATDEV_DEBUG("Initializing INTx interrupts \n", dev->common.id);
assert(!"NYI");
break;
default:
/* disabled */
intcrtl = 0;
IOATDEV_DEBUG("Disabling interrupts \n", dev->common.id);
break;
}
ioat_dma_intrctrl_wr(&dev->device, intcrtl);
return SYS_ERR_OK;
}
static void ioat_dma_device_irq_handler(void* arg)
{
errval_t err;
struct dma_device *dev = arg;
IOATDEV_DEBUG("############ MSIX INTERRUPT HAPPENED.\n", dev->id);
#if IOAT_DEBUG_INTR_ENABLED
msix_intr_happened=1;
#endif
err = ioat_dma_device_poll_channels(dev);
if (err_is_fail(err)) {
if (err_no(err) == DMA_ERR_DEVICE_IDLE) {
IOATDEV_DEBUG("WARNING: MSI-X interrupt on idle device\n", dev->id);
return;
}
USER_PANIC_ERR(err, "dma poll device returned an error\n");
}
}
static errval_t device_init_ioat_v3(struct ioat_dma_device *dev)
{
errval_t err;
IOATDEV_DEBUG("initialize Crystal Beach 3 DMA device\n", dev->common.id);
ioat_dma_dmacapability_t cap = ioat_dma_dmacapability_rd(&dev->device);
if (ioat_dma_cbver_minor_extract(dev->version) == 2) {
IOATDEV_DEBUG("disabling XOR and PQ opcodes for Crystal Beach 3.2\n",
dev->common.id);
cap = ioat_dma_dmacapability_xor_insert(cap, 0x0);
cap = ioat_dma_dmacapability_pq_insert(cap, 0x0);
} else if (ioat_dma_cbver_minor_extract(dev->version) == 3) {
IOATDEV_DEBUG("devices of Crystal Beach Version 3.3 are not supported.\n",
dev->common.id);
return DMA_ERR_DEVICE_UNSUPPORTED;
}
/* if DCA is enabled, we cannot support the RAID functions */
if (ioat_dma_dca_is_enabled()) {
IOATDEV_DEBUG("Disabling XOR and PQ while DCA is enabled\n", dev->common.id);
cap = ioat_dma_dmacapability_xor_insert(cap, 0x0);
cap = ioat_dma_dmacapability_pq_insert(cap, 0x0);
}
if (ioat_dma_dmacapability_xor_extract(cap)) {
IOATDEV_DEBUG("device supports XOR RAID.\n", dev->common.id);
dev->flags |= IOAT_DMA_DEV_F_RAID;
/*
* this may need some additional functions to prepare
* the specific transfers...
*
* max_xor = 8;
* prepare_xor, prepare_xor_val
*/
}
if (ioat_dma_dmacapability_pq_extract(cap)) {
IOATDEV_DEBUG("device supports PQ RAID.\n", dev->common.id);
dev->flags |= IOAT_DMA_DEV_F_RAID;
/*
* this may need some additional functions to prepare the
* DMA descriptors
*
* max_xor = 8;
* max_pq = 8;
* prepare_pq, perpare_pq_val
*
* also set the prepare_xor pointers...
*
*/
}
/* set the interrupt type to disabled*/
dev->common.irq_type = DMA_IRQ_DISABLED;
dev->common.type = DMA_DEV_TYPE_IOAT;
/* allocate memory for completion status writeback */
err = dma_mem_alloc(IOAT_DMA_COMPLSTATUS_SIZE, IOAT_DMA_COMPLSTATUS_FLAGS,
&dev->complstatus);
if (err_is_fail(err)) {
return err;
}
dev->common.channels.count = ioat_dma_chancnt_num_rdf(&dev->device);
dev->common.channels.c = calloc(dev->common.channels.count,
sizeof(*dev->common.channels.c));
if (dev->common.channels.c == NULL) {
dma_mem_free(&dev->complstatus);
return LIB_ERR_MALLOC_FAIL;
}
/* channel enumeration */
IOATDEV_DEBUG("channel enumeration. discovered %u channels\n", dev->common.id,
dev->common.channels.count);
uint32_t max_xfer_size = (1 << ioat_dma_xfercap_max_rdf(&dev->device));
for (uint8_t i = 0; i < dev->common.channels.count; ++i) {
struct dma_channel **chan = &dev->common.channels.c[i];
err = ioat_dma_channel_init(dev, i, max_xfer_size,
(struct ioat_dma_channel **) chan);
}
if (dev->flags & IOAT_DMA_DEV_F_DCA) {
/*TODO: DCA initialization device->dca = ioat3_dca_init(pdev, device->reg_base);*/
}
return SYS_ERR_OK;
}
static errval_t device_init_ioat_v2(struct ioat_dma_device *dev)
{
IOATDEV_DEBUG("devices of Crystal Beach Version 2.xx are currently not supported.\n",
dev->common.id);
return DMA_ERR_DEVICE_UNSUPPORTED;
}
/**
* \brief initializes a IOAT DMA device with the giving capability
*
* \param mmio capability representing the device's MMIO registers
* \param dev returns a pointer to the device structure
*
* \returns SYS_ERR_OK on success
* errval on error
*/
errval_t ioat_dma_device_init(struct capref mmio,
struct ioat_dma_device **dev)
{
errval_t err;
struct ioat_dma_device *ioat_device = calloc(1, sizeof(*ioat_device));
if (ioat_device == NULL) {
return LIB_ERR_MALLOC_FAIL;
}
#if DMA_BENCH_ENABLED
bench_init();
#endif
struct dma_device *dma_dev = &ioat_device->common;
struct frame_identity mmio_id;
err = invoke_frame_identify(mmio, &mmio_id);
if (err_is_fail(err)) {
free(ioat_device);
return err;
}
dma_dev->id = device_id++;
dma_dev->mmio.paddr = mmio_id.base;
dma_dev->mmio.bytes = (1UL << mmio_id.bits);
dma_dev->mmio.frame = mmio;
IOATDEV_DEBUG("init device with mmio range: {paddr=0x%016lx, size=%u kB}\n",
dma_dev->id, mmio_id.base, 1 << mmio_id.bits);
err = vspace_map_one_frame_attr((void**) &dma_dev->mmio.vaddr,
dma_dev->mmio.bytes, dma_dev->mmio.frame,
VREGION_FLAGS_READ_WRITE_NOCACHE,
NULL,
NULL);
if (err_is_fail(err)) {
free(ioat_device);
return err;
}
ioat_dma_initialize(&ioat_device->device, NULL, (void *) dma_dev->mmio.vaddr);
ioat_device->version = ioat_dma_cbver_rd(&ioat_device->device);
IOATDEV_DEBUG("device registers mapped at 0x%016lx. IOAT version: %u.%u\n",
dma_dev->id, dma_dev->mmio.vaddr,
ioat_dma_cbver_major_extract(ioat_device->version),
ioat_dma_cbver_minor_extract(ioat_device->version));
switch (ioat_dma_cbver_major_extract(ioat_device->version)) {
case ioat_dma_cbver_1x:
err = device_init_ioat_v1(ioat_device);
break;
case ioat_dma_cbver_2x:
err = device_init_ioat_v2(ioat_device);
break;
case ioat_dma_cbver_3x:
err = device_init_ioat_v3(ioat_device);
break;
default:
err = DMA_ERR_DEVICE_UNSUPPORTED;
}
if (err_is_fail(err)) {
vspace_unmap((void*) dma_dev->mmio.vaddr);
free(ioat_device);
return err;
}
dma_dev->f.deregister_memory = NULL;
dma_dev->f.register_memory = NULL;
dma_dev->f.poll = ioat_dma_device_poll_channels;
*dev = ioat_device;
return err;
}
/**
* \brief globally enables the interrupts for the given device
*
* \param dev IOAT DMA device
* \param type the interrupt type to enable
*/
errval_t ioat_dma_device_irq_setup(struct ioat_dma_device *dev,
dma_irq_t type)
{
errval_t err;
ioat_dma_intrctrl_t intcrtl = 0;
intcrtl = ioat_dma_intrctrl_intp_en_insert(intcrtl, 1);
dev->common.irq_type = type;
switch (type) {
case DMA_IRQ_MSIX:
/* The number of MSI-X vectors should equal the number of channels */
IOATDEV_DEBUG("MSI-X interrupt setup for device (%u, %u, %u)\n",
dev->common.id, dev->pci_addr.bus, dev->pci_addr.device,
dev->pci_addr.function);
err = pci_msix_enable_addr(&dev->pci_addr, &dev->irq_msix_count);
if (err_is_fail(err)) {
return err;
}
assert(dev->irq_msix_count > 0);
IOATDEV_DEBUG("MSI-X enabled #vecs=%d\n", dev->common.id,
dev->irq_msix_count);
err = pci_setup_inthandler(ioat_dma_device_irq_handler, dev,
&dev->irq_msix_vector);
assert(err_is_ok(err));
uint8_t dest = get_local_apic_id();
IOATDEV_DEBUG("MSI-X routing to apic=%u\n", dev->common.id,
dest);
err = pci_msix_vector_init_addr(&dev->pci_addr, 0, dest,
dev->irq_msix_vector);
assert(err_is_ok(err));
/* enable the interrupts */
intcrtl = ioat_dma_intrctrl_msix_vec_insert(intcrtl, 1);
intcrtl = ioat_dma_intrctrl_intp_en_insert(intcrtl, 1);
break;
case DMA_IRQ_MSI:
IOATDEV_DEBUG("Initializing MSI interrupts \n", dev->common.id);
assert(!"NYI");
break;
case DMA_IRQ_INTX:
IOATDEV_DEBUG("Initializing INTx interrupts \n", dev->common.id);
assert(!"NYI");
break;
default:
/* disabled */
intcrtl = 0;
IOATDEV_DEBUG("Disabling interrupts \n", dev->common.id);
break;
}
ioat_dma_intrctrl_wr(&dev->device, intcrtl);
#if IOAT_DEBUG_INTR_ENABLED
/*
* check if interrupts are working.
*/
msix_intr_happened = 0;
struct ioat_dma_channel *chan;
chan = (struct ioat_dma_channel *)dev->common.channels.c[0];
ioat_dma_request_nop_chan(chan);
err = ioat_dma_channel_issue_pending(chan);
if (err_is_fail(err)) {
return err;
}
while(msix_intr_happened == 0) {
uint64_t status = ioat_dma_channel_get_status(chan);
err = event_dispatch_non_block(get_default_waitset());
if (!ioat_dma_channel_is_active(status) && !ioat_dma_channel_is_idle(status)) {
USER_PANIC("DMA request turned channel into erroneous state.")
}
switch(err_no(err)) {
case LIB_ERR_NO_EVENT:
thread_yield();
break;
case SYS_ERR_OK:
continue;
default:
USER_PANIC_ERR(err, "dispatching event");
}
}
#endif
return SYS_ERR_OK;
}
