void
ahci_intr_port(struct ahci_softc *sc, struct ahci_channel *achp)
{
u_int32_t is, tfd;
struct ata_channel *chp = &achp->ata_channel;
struct ata_xfer *xfer = chp->ch_queue->active_xfer;
int slot;
is = AHCI_READ(sc, AHCI_P_IS(chp->ch_channel));
AHCI_WRITE(sc, AHCI_P_IS(chp->ch_channel), is);
AHCIDEBUG_PRINT(("ahci_intr_port %s port %d is 0x%x CI 0x%x\n", AHCINAME(sc),
chp->ch_channel, is, AHCI_READ(sc, AHCI_P_CI(chp->ch_channel))),
DEBUG_INTR);
if (is & (AHCI_P_IX_TFES | AHCI_P_IX_HBFS | AHCI_P_IX_IFS |
AHCI_P_IX_OFS | AHCI_P_IX_UFS)) {
slot = (AHCI_READ(sc, AHCI_P_CMD(chp->ch_channel))
& AHCI_P_CMD_CCS_MASK) >> AHCI_P_CMD_CCS_SHIFT;
if ((achp->ahcic_cmds_active & (1 << slot)) == 0)
return;
/* stop channel */
ahci_channel_stop(sc, chp, 0);
if (slot != 0) {
printf("ahci_intr_port: slot %d\n", slot);
panic("ahci_intr_port");
}
if (is & AHCI_P_IX_TFES) {
tfd = AHCI_READ(sc, AHCI_P_TFD(chp->ch_channel));
chp->ch_error =
(tfd & AHCI_P_TFD_ERR_MASK) >> AHCI_P_TFD_ERR_SHIFT;
chp->ch_status = (tfd & 0xff);
} else {
void
ahci_enable_intrs(struct ahci_softc *sc)
{
/* clear interrupts */
AHCI_WRITE(sc, AHCI_IS, AHCI_READ(sc, AHCI_IS));
/* enable interrupts */
AHCI_WRITE(sc, AHCI_GHC, AHCI_READ(sc, AHCI_GHC) | AHCI_GHC_IE);
}
static void
jmahci_attach(device_t parent, device_t self, void *aux)
{
struct jmahci_attach_args *jma = aux;
const struct pci_attach_args *pa = jma->jma_pa;
struct ahci_softc *sc = device_private(self);
uint32_t ahci_cap;
aprint_naive(": AHCI disk controller\n");
aprint_normal("\n");
sc->sc_atac.atac_dev = self;
sc->sc_ahcit = jma->jma_ahcit;
sc->sc_ahcih = jma->jma_ahcih;
ahci_cap = AHCI_READ(sc, AHCI_CAP);
if (pci_dma64_available(jma->jma_pa) && (ahci_cap & AHCI_CAP_64BIT))
sc->sc_dmat = jma->jma_pa->pa_dmat64;
else
sc->sc_dmat = jma->jma_pa->pa_dmat;
if (PCI_SUBCLASS(pa->pa_class) == PCI_SUBCLASS_MASS_STORAGE_RAID)
sc->sc_atac_capflags = ATAC_CAP_RAID;
ahci_attach(sc);
if (!pmf_device_register(self, NULL, jmahci_resume))
aprint_error_dev(self, "couldn't establish power handler\n");
}
static void
awin_ahci_channel_start(struct ahci_softc *sc, struct ata_channel *chp)
{
bus_size_t dma_reg = AHCI_P_AWIN_DMA(chp->ch_channel);
uint32_t dma = AHCI_READ(sc, dma_reg);
dma &= ~0xff00;
dma |= 0x4400;
AHCI_WRITE(sc, dma_reg, dma);
}
int
ahci_reset(struct ahci_softc *sc)
{
int i;
/* reset controller */
AHCI_WRITE(sc, AHCI_GHC, AHCI_GHC_HR);
/* wait up to 1s for reset to complete */
for (i = 0; i < 1000; i++) {
delay(1000);
if ((AHCI_READ(sc, AHCI_GHC) & AHCI_GHC_HR) == 0)
break;
}
if ((AHCI_READ(sc, AHCI_GHC) & AHCI_GHC_HR)) {
aprint_error("%s: reset failed\n", AHCINAME(sc));
return -1;
}
/* enable ahci mode */
AHCI_WRITE(sc, AHCI_GHC, AHCI_GHC_AE);
return 0;
}
int
ahci_intr(void *v)
{
struct ahci_softc *sc = v;
u_int32_t is;
int i, r = 0;
while ((is = AHCI_READ(sc, AHCI_IS))) {
AHCIDEBUG_PRINT(("%s ahci_intr 0x%x\n", AHCINAME(sc), is),
DEBUG_INTR);
r = 1;
AHCI_WRITE(sc, AHCI_IS, is);
for (i = 0; i < AHCI_MAX_PORTS; i++)
if (is & (1 << i))
ahci_intr_port(sc, &sc->sc_channels[i]);
}
return r;
}
void
ahci_setup_ports(struct ahci_softc *sc)
{
u_int32_t ahci_ports;
int i, port;
ahci_ports = AHCI_READ(sc, AHCI_PI);
for (i = 0, port = 0; i < AHCI_MAX_PORTS; i++) {
if ((ahci_ports & (1 << i)) == 0)
continue;
if (port >= sc->sc_atac.atac_nchannels) {
aprint_error("%s: more ports than announced\n",
AHCINAME(sc));
break;
}
ahci_setup_port(sc, i);
}
}
void
ahci_reprobe_drives(struct ahci_softc *sc)
{
u_int32_t ahci_ports;
int i, port;
struct ahci_channel *achp;
struct ata_channel *chp;
ahci_ports = AHCI_READ(sc, AHCI_PI);
for (i = 0, port = 0; i < AHCI_MAX_PORTS; i++) {
if ((ahci_ports & (1 << i)) == 0)
continue;
if (port >= sc->sc_atac.atac_nchannels) {
aprint_error("%s: more ports than announced\n",
AHCINAME(sc));
break;
}
achp = &sc->sc_channels[i];
chp = &achp->ata_channel;
ahci_probe_drive(chp);
}
}
static void
ahci_pci_attach(device_t parent, device_t self, void *aux)
{
struct pci_attach_args *pa = aux;
struct ahci_pci_softc *psc = device_private(self);
struct ahci_softc *sc = &psc->ah_sc;
const char *intrstr;
bool ahci_cap_64bit;
bool ahci_bad_64bit;
pci_intr_handle_t intrhandle;
sc->sc_atac.atac_dev = self;
if (pci_mapreg_map(pa, AHCI_PCI_ABAR,
PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT, 0,
&sc->sc_ahcit, &sc->sc_ahcih, NULL, &sc->sc_ahcis) != 0) {
aprint_error_dev(self, "can't map ahci registers\n");
return;
}
psc->sc_pc = pa->pa_pc;
psc->sc_pcitag = pa->pa_tag;
pci_aprint_devinfo(pa, "AHCI disk controller");
if (pci_intr_map(pa, &intrhandle) != 0) {
aprint_error_dev(self, "couldn't map interrupt\n");
return;
}
intrstr = pci_intr_string(pa->pa_pc, intrhandle);
psc->sc_ih = pci_intr_establish(pa->pa_pc, intrhandle, IPL_BIO, ahci_intr, sc);
if (psc->sc_ih == NULL) {
aprint_error_dev(self, "couldn't establish interrupt\n");
return;
}
aprint_normal_dev(self, "interrupting at %s\n",
intrstr ? intrstr : "unknown interrupt");
sc->sc_dmat = pa->pa_dmat;
sc->sc_ahci_quirks = ahci_pci_has_quirk(PCI_VENDOR(pa->pa_id),
PCI_PRODUCT(pa->pa_id));
ahci_cap_64bit = (AHCI_READ(sc, AHCI_CAP) & AHCI_CAP_64BIT) != 0;
ahci_bad_64bit = ((sc->sc_ahci_quirks & AHCI_PCI_QUIRK_BAD64) != 0);
if (pci_dma64_available(pa) && ahci_cap_64bit) {
if (!ahci_bad_64bit)
sc->sc_dmat = pa->pa_dmat64;
aprint_verbose_dev(self, "64-bit DMA%s\n",
(sc->sc_dmat == pa->pa_dmat) ? " unavailable" : "");
}
if (PCI_SUBCLASS(pa->pa_class) == PCI_SUBCLASS_MASS_STORAGE_RAID) {
AHCIDEBUG_PRINT(("%s: RAID mode\n", AHCINAME(sc)), DEBUG_PROBE);
sc->sc_atac_capflags = ATAC_CAP_RAID;
} else {
AHCIDEBUG_PRINT(("%s: SATA mode\n", AHCINAME(sc)), DEBUG_PROBE);
}
ahci_attach(sc);
if (!pmf_device_register(self, NULL, ahci_pci_resume))
aprint_error_dev(self, "couldn't establish power handler\n");
}
void
ahci_attach(struct ahci_softc *sc)
{
u_int32_t ahci_cap, ahci_rev, ahci_ports;
int i, j, port;
struct ahci_channel *achp;
struct ata_channel *chp;
int error;
bus_dma_segment_t seg;
int rseg;
int dmasize;
void *cmdhp;
void *cmdtblp;
if (ahci_reset(sc) != 0)
return;
ahci_cap = AHCI_READ(sc, AHCI_CAP);
sc->sc_atac.atac_nchannels = (ahci_cap & AHCI_CAP_NPMASK) + 1;
sc->sc_ncmds = ((ahci_cap & AHCI_CAP_NCS) >> 8) + 1;
ahci_rev = AHCI_READ(sc, AHCI_VS);
aprint_normal("%s: AHCI revision ", AHCINAME(sc));
switch(ahci_rev) {
case AHCI_VS_10:
aprint_normal("1.0");
break;
case AHCI_VS_11:
aprint_normal("1.1");
break;
case AHCI_VS_12:
aprint_normal("1.2");
break;
default:
aprint_normal("0x%x", ahci_rev);
break;
}
aprint_normal(", %d ports, %d command slots, features 0x%x\n",
sc->sc_atac.atac_nchannels, sc->sc_ncmds,
ahci_cap & ~(AHCI_CAP_NPMASK|AHCI_CAP_NCS));
sc->sc_atac.atac_cap = ATAC_CAP_DATA16 | ATAC_CAP_DMA | ATAC_CAP_UDMA;
sc->sc_atac.atac_cap |= sc->sc_atac_capflags;
sc->sc_atac.atac_pio_cap = 4;
sc->sc_atac.atac_dma_cap = 2;
sc->sc_atac.atac_udma_cap = 6;
sc->sc_atac.atac_channels = sc->sc_chanarray;
sc->sc_atac.atac_probe = ahci_probe_drive;
sc->sc_atac.atac_bustype_ata = &ahci_ata_bustype;
sc->sc_atac.atac_set_modes = ahci_setup_channel;
#if NATAPIBUS > 0
sc->sc_atac.atac_atapibus_attach = ahci_atapibus_attach;
#endif
dmasize =
(AHCI_RFIS_SIZE + AHCI_CMDH_SIZE) * sc->sc_atac.atac_nchannels;
error = bus_dmamem_alloc(sc->sc_dmat, dmasize, PAGE_SIZE, 0,
&seg, 1, &rseg, BUS_DMA_NOWAIT);
if (error) {
aprint_error("%s: unable to allocate command header memory"
", error=%d\n", AHCINAME(sc), error);
return;
}
error = bus_dmamem_map(sc->sc_dmat, &seg, rseg, dmasize,
&cmdhp, BUS_DMA_NOWAIT|BUS_DMA_COHERENT);
if (error) {
aprint_error("%s: unable to map command header memory"
", error=%d\n", AHCINAME(sc), error);
return;
}
error = bus_dmamap_create(sc->sc_dmat, dmasize, 1, dmasize, 0,
BUS_DMA_NOWAIT, &sc->sc_cmd_hdrd);
if (error) {
aprint_error("%s: unable to create command header map"
", error=%d\n", AHCINAME(sc), error);
return;
}
error = bus_dmamap_load(sc->sc_dmat, sc->sc_cmd_hdrd,
cmdhp, dmasize, NULL, BUS_DMA_NOWAIT);
if (error) {
aprint_error("%s: unable to load command header map"
", error=%d\n", AHCINAME(sc), error);
return;
}
sc->sc_cmd_hdr = cmdhp;
ahci_enable_intrs(sc);
ahci_ports = AHCI_READ(sc, AHCI_PI);
for (i = 0, port = 0; i < AHCI_MAX_PORTS; i++) {
if ((ahci_ports & (1 << i)) == 0)
continue;
if (port >= sc->sc_atac.atac_nchannels) {
aprint_error("%s: more ports than announced\n",
AHCINAME(sc));
break;
}
achp = &sc->sc_channels[i];
chp = (struct ata_channel *)achp;
sc->sc_chanarray[i] = chp;
chp->ch_channel = i;
chp->ch_atac = &sc->sc_atac;
chp->ch_queue = malloc(sizeof(struct ata_queue),
M_DEVBUF, M_NOWAIT);
if (chp->ch_queue == NULL) {
aprint_error("%s port %d: can't allocate memory for "
"command queue", AHCINAME(sc), i);
break;
}
dmasize = AHCI_CMDTBL_SIZE * sc->sc_ncmds;
error = bus_dmamem_alloc(sc->sc_dmat, dmasize, PAGE_SIZE, 0,
&seg, 1, &rseg, BUS_DMA_NOWAIT);
if (error) {
aprint_error("%s: unable to allocate command table "
"memory, error=%d\n", AHCINAME(sc), error);
break;
}
error = bus_dmamem_map(sc->sc_dmat, &seg, rseg, dmasize,
&cmdtblp, BUS_DMA_NOWAIT|BUS_DMA_COHERENT);
if (error) {
aprint_error("%s: unable to map command table memory"
", error=%d\n", AHCINAME(sc), error);
break;
}
error = bus_dmamap_create(sc->sc_dmat, dmasize, 1, dmasize, 0,
BUS_DMA_NOWAIT, &achp->ahcic_cmd_tbld);
if (error) {
aprint_error("%s: unable to create command table map"
", error=%d\n", AHCINAME(sc), error);
break;
}
error = bus_dmamap_load(sc->sc_dmat, achp->ahcic_cmd_tbld,
cmdtblp, dmasize, NULL, BUS_DMA_NOWAIT);
if (error) {
aprint_error("%s: unable to load command table map"
", error=%d\n", AHCINAME(sc), error);
break;
}
achp->ahcic_cmdh = (struct ahci_cmd_header *)
((char *)cmdhp + AHCI_CMDH_SIZE * port);
achp->ahcic_bus_cmdh = sc->sc_cmd_hdrd->dm_segs[0].ds_addr +
AHCI_CMDH_SIZE * port;
achp->ahcic_rfis = (struct ahci_r_fis *)
((char *)cmdhp +
AHCI_CMDH_SIZE * sc->sc_atac.atac_nchannels +
AHCI_RFIS_SIZE * port);
achp->ahcic_bus_rfis = sc->sc_cmd_hdrd->dm_segs[0].ds_addr +
AHCI_CMDH_SIZE * sc->sc_atac.atac_nchannels +
AHCI_RFIS_SIZE * port;
AHCIDEBUG_PRINT(("port %d cmdh %p (0x%x) rfis %p (0x%x)\n", i,
achp->ahcic_cmdh, (u_int)achp->ahcic_bus_cmdh,
achp->ahcic_rfis, (u_int)achp->ahcic_bus_rfis),
DEBUG_PROBE);
for (j = 0; j < sc->sc_ncmds; j++) {
achp->ahcic_cmd_tbl[j] = (struct ahci_cmd_tbl *)
((char *)cmdtblp + AHCI_CMDTBL_SIZE * j);
achp->ahcic_bus_cmd_tbl[j] =
achp->ahcic_cmd_tbld->dm_segs[0].ds_addr +
AHCI_CMDTBL_SIZE * j;
achp->ahcic_cmdh[j].cmdh_cmdtba =
htole32(achp->ahcic_bus_cmd_tbl[j]);
achp->ahcic_cmdh[j].cmdh_cmdtbau = htole32(0);
AHCIDEBUG_PRINT(("port %d/%d tbl %p (0x%x)\n", i, j,
achp->ahcic_cmd_tbl[j],
(u_int)achp->ahcic_bus_cmd_tbl[j]), DEBUG_PROBE);
/* The xfer DMA map */
error = bus_dmamap_create(sc->sc_dmat, MAXPHYS,
AHCI_NPRD, 0x400000 /* 4MB */, 0,
BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
&achp->ahcic_datad[j]);
if (error) {
aprint_error("%s: couldn't alloc xfer DMA map, "
"error=%d\n", AHCINAME(sc), error);
goto end;
}
}
ahci_setup_port(sc, i);
chp->ch_ndrive = 1;
if (bus_space_subregion(sc->sc_ahcit, sc->sc_ahcih,
AHCI_P_SSTS(i), 1, &achp->ahcic_sstatus) != 0) {
aprint_error("%s: couldn't map channel %d "
"sata_status regs\n", AHCINAME(sc), i);
break;
}
if (bus_space_subregion(sc->sc_ahcit, sc->sc_ahcih,
AHCI_P_SCTL(i), 1, &achp->ahcic_scontrol) != 0) {
aprint_error("%s: couldn't map channel %d "
"sata_control regs\n", AHCINAME(sc), i);
break;
}
if (bus_space_subregion(sc->sc_ahcit, sc->sc_ahcih,
AHCI_P_SERR(i), 1, &achp->ahcic_serror) != 0) {
aprint_error("%s: couldn't map channel %d "
"sata_error regs\n", AHCINAME(sc), i);
break;
}
ata_channel_attach(chp);
port++;
end:
continue;
}
}