static int
ntb_setup_xeon(struct ntb_softc *ntb)
{
uint8_t val, connection_type;
val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1);
connection_type = val & XEON_PPD_CONN_TYPE;
switch (connection_type) {
case NTB_CONN_B2B:
ntb->conn_type = NTB_CONN_B2B;
break;
case NTB_CONN_CLASSIC:
case NTB_CONN_RP:
default:
device_printf(ntb->device, "Connection type %d not supported\n",
connection_type);
return (ENXIO);
}
if ((val & XEON_PPD_DEV_TYPE) != 0)
ntb->dev_type = NTB_DEV_DSD;
else
ntb->dev_type = NTB_DEV_USD;
ntb->reg_ofs.pdb = XEON_PDOORBELL_OFFSET;
ntb->reg_ofs.pdb_mask = XEON_PDBMSK_OFFSET;
ntb->reg_ofs.sbar2_xlat = XEON_SBAR2XLAT_OFFSET;
ntb->reg_ofs.sbar4_xlat = XEON_SBAR4XLAT_OFFSET;
ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET;
ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET;
ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET;
ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET;
if (ntb->conn_type == NTB_CONN_B2B) {
ntb->reg_ofs.sdb = XEON_B2B_DOORBELL_OFFSET;
ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET;
ntb->limits.max_spads = XEON_MAX_SPADS;
} else {
ntb->reg_ofs.sdb = XEON_SDOORBELL_OFFSET;
ntb->reg_ofs.spad_remote = XEON_SPAD_OFFSET;
ntb->limits.max_spads = XEON_MAX_COMPAT_SPADS;
}
ntb->limits.max_db_bits = XEON_MAX_DB_BITS;
ntb->limits.msix_cnt = XEON_MSIX_CNT;
ntb->bits_per_vector = XEON_DB_BITS_PER_VEC;
configure_xeon_secondary_side_bars(ntb);
/* Enable Bus Master and Memory Space on the secondary side */
ntb_reg_write(2, ntb->reg_ofs.spci_cmd,
PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
/* Enable link training */
ntb_reg_write(4, ntb->reg_ofs.lnk_cntl,
NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP);
return (0);
}
static void
db_iowrite(struct ntb_softc *ntb, uint32_t regoff, uint64_t val)
{
if (ntb->type == NTB_SOC) {
ntb_reg_write(8, regoff, val);
return;
}
KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
ntb_reg_write(2, regoff, (uint16_t)val);
}
static void
configure_xeon_secondary_side_bars(struct ntb_softc *ntb)
{
if (ntb->dev_type == NTB_DEV_USD) {
ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR);
if (HAS_FEATURE(NTB_REGS_THRU_MW))
ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
MBAR01_DSD_ADDR);
else {
ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
PBAR4XLAT_USD_ADDR);
/*
* B2B_XLAT_OFFSET is a 64-bit register but can only be
* written 32 bits at a time.
*/
ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL,
MBAR01_DSD_ADDR & 0xffffffff);
ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU,
MBAR01_DSD_ADDR >> 32);
}
ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_USD_ADDR);
ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_USD_ADDR);
ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_USD_ADDR);
} else {
static void
handle_soc_irq(void *arg)
{
struct ntb_db_cb *db_cb = arg;
struct ntb_softc *ntb = db_cb->ntb;
ntb_reg_write(8, ntb->reg_ofs.pdb, (uint64_t) 1 << db_cb->db_num);
if (db_cb->callback != NULL)
db_cb->callback(db_cb->data, db_cb->db_num);
}
/* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */
static void
handle_xeon_event_irq(void *arg)
{
struct ntb_softc *ntb = arg;
int rc;
rc = ntb_check_link_status(ntb);
if (rc != 0)
device_printf(ntb->device, "Error determining link status\n");
/* bit 15 is always the link bit */
ntb_reg_write(2, ntb->reg_ofs.pdb, 1 << ntb->limits.max_db_bits);
}
static void
configure_soc_secondary_side_bars(struct ntb_softc *ntb)
{
if (ntb->dev_type == NTB_DEV_USD) {
ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, MBAR23_DSD_ADDR);
ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, MBAR4_DSD_ADDR);
ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_USD_ADDR);
ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR4_USD_ADDR);
} else {
ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, MBAR23_USD_ADDR);
ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, MBAR4_USD_ADDR);
ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_DSD_ADDR);
ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR4_DSD_ADDR);
}
}
static void
soc_perform_link_restart(struct ntb_softc *ntb)
{
uint32_t status;
/* Driver resets the NTB ModPhy lanes - magic! */
ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0);
ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40);
ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60);
ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60);
/* Driver waits 100ms to allow the NTB ModPhy to settle */
pause("ModPhy", hz / 10);
/* Clear AER Errors, write to clear */
status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET);
status &= PCIM_AER_COR_REPLAY_ROLLOVER;
ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status);
/* Clear unexpected electrical idle event in LTSSM, write to clear */
status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET);
status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI;
ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status);
/* Clear DeSkew Buffer error, write to clear */
status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET);
status |= SOC_DESKEWSTS_DBERR;
ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status);
status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET);
status &= SOC_IBIST_ERR_OFLOW;
ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status);
/* Releases the NTB state machine to allow the link to retrain */
status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET);
status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT;
ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status);
}
static int
ntb_setup_soc(struct ntb_softc *ntb)
{
KASSERT(ntb->conn_type == NTB_CONN_B2B,
("Unsupported NTB configuration (%d)\n", ntb->conn_type));
/* Initiate PCI-E link training */
pci_write_config(ntb->device, NTB_PPD_OFFSET,
ntb->ppd | SOC_PPD_INIT_LINK, 4);
ntb->reg_ofs.ldb = SOC_PDOORBELL_OFFSET;
ntb->reg_ofs.ldb_mask = SOC_PDBMSK_OFFSET;
ntb->reg_ofs.rdb = SOC_B2B_DOORBELL_OFFSET;
ntb->reg_ofs.bar2_xlat = SOC_SBAR2XLAT_OFFSET;
ntb->reg_ofs.bar4_xlat = SOC_SBAR4XLAT_OFFSET;
ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET;
ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET;
ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET;
ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET;
ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET;
ntb->limits.max_spads = SOC_MAX_SPADS;
ntb->limits.max_db_bits = SOC_MAX_DB_BITS;
ntb->limits.msix_cnt = SOC_MSIX_CNT;
ntb->bits_per_vector = SOC_DB_BITS_PER_VEC;
/*
* FIXME - MSI-X bug on early SOC HW, remove once internal issue is
* resolved. Mask transaction layer internal parity errors.
*/
pci_write_config(ntb->device, 0xFC, 0x4, 4);
configure_soc_secondary_side_bars(ntb);
/* Enable Bus Master and Memory Space on the secondary side */
ntb_reg_write(2, ntb->reg_ofs.spci_cmd,
PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb);
return (0);
}
static void
handle_xeon_irq(void *arg)
{
struct ntb_db_cb *db_cb = arg;
struct ntb_softc *ntb = db_cb->ntb;
/*
* On Xeon, there are 16 bits in the interrupt register
* but only 4 vectors. So, 5 bits are assigned to the first 3
* vectors, with the 4th having a single bit for link
* interrupts.
*/
ntb_reg_write(2, ntb->reg_ofs.pdb,
((1 << ntb->bits_per_vector) - 1) <<
(db_cb->db_num * ntb->bits_per_vector));
if (db_cb->callback != NULL)
db_cb->callback(db_cb->data, db_cb->db_num);
}
static int
ntb_setup_xeon(struct ntb_softc *ntb)
{
ntb->reg_ofs.ldb = XEON_PDOORBELL_OFFSET;
ntb->reg_ofs.ldb_mask = XEON_PDBMSK_OFFSET;
ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET;
ntb->reg_ofs.bar2_xlat = XEON_SBAR2XLAT_OFFSET;
ntb->reg_ofs.bar4_xlat = XEON_SBAR4XLAT_OFFSET;
if (HAS_FEATURE(NTB_SPLIT_BAR))
ntb->reg_ofs.bar5_xlat = XEON_SBAR5XLAT_OFFSET;
switch (ntb->conn_type) {
case NTB_CONN_B2B:
/*
* reg_ofs.rdb and reg_ofs.spad_remote are effectively ignored
* with the NTB_REGS_THRU_MW errata mode enabled. (See
* ntb_ring_doorbell() and ntb_read/write_remote_spad().)
*/
ntb->reg_ofs.rdb = XEON_B2B_DOORBELL_OFFSET;
ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET;
ntb->limits.max_spads = XEON_MAX_SPADS;
break;
case NTB_CONN_RP:
/*
* Every Xeon today needs NTB_REGS_THRU_MW, so punt on RP for
* now.
*/
KASSERT(HAS_FEATURE(NTB_REGS_THRU_MW),
("Xeon without MW errata unimplemented"));
device_printf(ntb->device,
"NTB-RP disabled to due hardware errata.\n");
return (ENXIO);
case NTB_CONN_TRANSPARENT:
default:
device_printf(ntb->device, "Connection type %d not supported\n",
ntb->conn_type);
return (ENXIO);
}
/*
* There is a Xeon hardware errata related to writes to SDOORBELL or
* B2BDOORBELL in conjunction with inbound access to NTB MMIO space,
* which may hang the system. To workaround this use the second memory
* window to access the interrupt and scratch pad registers on the
* remote system.
*
* There is another HW errata on the limit registers -- they can only
* be written when the base register is (?)4GB aligned and < 32-bit.
* This should already be the case based on the driver defaults, but
* write the limit registers first just in case.
*/
if (HAS_FEATURE(NTB_REGS_THRU_MW)) {
/*
* Set the Limit register to 4k, the minimum size, to prevent
* an illegal access.
*
* XXX: Should this be PBAR5LMT / get_mw_size(, max_mw - 1)?
*/
ntb_reg_write(8, XEON_PBAR4LMT_OFFSET,
ntb_get_mw_size(ntb, 1) + 0x1000);
/* Reserve the last MW for mapping remote spad */
ntb->limits.max_mw--;
} else
/*
* Disable the limit register, just in case it is set to
* something silly. A 64-bit write will also clear PBAR5LMT in
* split-bar mode, and this is desired.
*/
ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0);
ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET;
ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET;
ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET;
ntb->limits.max_db_bits = XEON_MAX_DB_BITS;
ntb->limits.msix_cnt = XEON_MSIX_CNT;
ntb->bits_per_vector = XEON_DB_BITS_PER_VEC;
/*
* HW Errata on bit 14 of b2bdoorbell register. Writes will not be
* mirrored to the remote system. Shrink the number of bits by one,
* since bit 14 is the last bit.
*
* On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register
* anyway. Nor for non-B2B connection types.
*/
if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14) &&
!HAS_FEATURE(NTB_REGS_THRU_MW) &&
ntb->conn_type == NTB_CONN_B2B)
ntb->limits.max_db_bits = XEON_MAX_DB_BITS - 1;
configure_xeon_secondary_side_bars(ntb);
/* Enable Bus Master and Memory Space on the secondary side */
if (ntb->conn_type == NTB_CONN_B2B)
ntb_reg_write(2, ntb->reg_ofs.spci_cmd,
PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
/* Enable link training */
ntb_hw_link_up(ntb);
return (0);
}
static void
configure_xeon_secondary_side_bars(struct ntb_softc *ntb)
{
if (ntb->dev_type == NTB_DEV_USD) {
ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR);
if (HAS_FEATURE(NTB_REGS_THRU_MW))
ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
MBAR01_DSD_ADDR);
else
ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
PBAR4XLAT_USD_ADDR);
ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_USD_ADDR);
ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_USD_ADDR);
ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_USD_ADDR);
} else {
ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR);
if (HAS_FEATURE(NTB_REGS_THRU_MW))
ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
MBAR01_USD_ADDR);
else
ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
PBAR4XLAT_DSD_ADDR);
ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_DSD_ADDR);
ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_DSD_ADDR);
ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_DSD_ADDR);
}
}
static int
ntb_setup_soc(struct ntb_softc *ntb)
{
uint32_t val, connection_type;
val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4);
connection_type = (val & SOC_PPD_CONN_TYPE) >> 8;
switch (connection_type) {
case NTB_CONN_B2B:
ntb->conn_type = NTB_CONN_B2B;
break;
case NTB_CONN_RP:
default:
device_printf(ntb->device, "Connection type %d not supported\n",
connection_type);
return (ENXIO);
}
if ((val & SOC_PPD_DEV_TYPE) != 0)
ntb->dev_type = NTB_DEV_DSD;
else
ntb->dev_type = NTB_DEV_USD;
/* Initiate PCI-E link training */
pci_write_config(ntb->device, NTB_PPD_OFFSET, val | SOC_PPD_INIT_LINK,
4);
ntb->reg_ofs.pdb = SOC_PDOORBELL_OFFSET;
ntb->reg_ofs.pdb_mask = SOC_PDBMSK_OFFSET;
ntb->reg_ofs.sbar2_xlat = SOC_SBAR2XLAT_OFFSET;
ntb->reg_ofs.sbar4_xlat = SOC_SBAR4XLAT_OFFSET;
ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET;
ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET;
ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET;
ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET;
if (ntb->conn_type == NTB_CONN_B2B) {
ntb->reg_ofs.sdb = SOC_B2B_DOORBELL_OFFSET;
ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET;
ntb->limits.max_spads = SOC_MAX_SPADS;
} else {
ntb->reg_ofs.sdb = SOC_PDOORBELL_OFFSET;
ntb->reg_ofs.spad_remote = SOC_SPAD_OFFSET;
ntb->limits.max_spads = SOC_MAX_COMPAT_SPADS;
}
ntb->limits.max_db_bits = SOC_MAX_DB_BITS;
ntb->limits.msix_cnt = SOC_MSIX_CNT;
ntb->bits_per_vector = SOC_DB_BITS_PER_VEC;
/*
* FIXME - MSI-X bug on early SOC HW, remove once internal issue is
* resolved. Mask transaction layer internal parity errors.
*/
pci_write_config(ntb->device, 0xFC, 0x4, 4);
configure_soc_secondary_side_bars(ntb);
/* Enable Bus Master and Memory Space on the secondary side */
ntb_reg_write(2, ntb->reg_ofs.spci_cmd,
PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb);
return (0);
}
static int
ntb_setup_interrupts(struct ntb_softc *ntb)
{
void (*interrupt_handler)(void *);
void *int_arg;
bool use_msix = 0;
uint32_t num_vectors;
int i;
ntb->allocated_interrupts = 0;
/*
* On SOC, disable all interrupts. On XEON, disable all but Link
* Interrupt. The rest will be unmasked as callbacks are registered.
*/
if (ntb->type == NTB_SOC)
ntb_reg_write(8, ntb->reg_ofs.pdb_mask, ~0);
else
ntb_reg_write(2, ntb->reg_ofs.pdb_mask,
~(1 << ntb->limits.max_db_bits));
num_vectors = MIN(pci_msix_count(ntb->device),
ntb->limits.max_db_bits);
if (num_vectors >= 1) {
pci_alloc_msix(ntb->device, &num_vectors);
if (num_vectors >= 4)
use_msix = TRUE;
}
ntb_create_callbacks(ntb, num_vectors);
if (use_msix == TRUE) {
for (i = 0; i < num_vectors; i++) {
ntb->int_info[i].rid = i + 1;
ntb->int_info[i].res = bus_alloc_resource_any(
ntb->device, SYS_RES_IRQ, &ntb->int_info[i].rid,
RF_ACTIVE);
if (ntb->int_info[i].res == NULL) {
device_printf(ntb->device,
"bus_alloc_resource failed\n");
return (-1);
}
ntb->int_info[i].tag = NULL;
ntb->allocated_interrupts++;
if (ntb->type == NTB_SOC) {
interrupt_handler = handle_soc_irq;
int_arg = &ntb->db_cb[i];
} else {
if (i == num_vectors - 1) {
interrupt_handler =
handle_xeon_event_irq;
int_arg = ntb;
} else {
interrupt_handler =
handle_xeon_irq;
int_arg = &ntb->db_cb[i];
}
}
if (bus_setup_intr(ntb->device, ntb->int_info[i].res,
INTR_MPSAFE | INTR_TYPE_MISC, NULL,
interrupt_handler, int_arg,
&ntb->int_info[i].tag) != 0) {
device_printf(ntb->device,
"bus_setup_intr failed\n");
return (ENXIO);
}
}
}
else {
ntb->int_info[0].rid = 0;
ntb->int_info[0].res = bus_alloc_resource_any(ntb->device,
SYS_RES_IRQ, &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE);
interrupt_handler = ntb_handle_legacy_interrupt;
if (ntb->int_info[0].res == NULL) {
device_printf(ntb->device,
"bus_alloc_resource failed\n");
return (-1);
}
ntb->int_info[0].tag = NULL;
ntb->allocated_interrupts = 1;
if (bus_setup_intr(ntb->device, ntb->int_info[0].res,
INTR_MPSAFE | INTR_TYPE_MISC, NULL,
interrupt_handler, ntb, &ntb->int_info[0].tag) != 0) {
device_printf(ntb->device, "bus_setup_intr failed\n");
return (ENXIO);
}
}
return (0);
}
