/*
* Per-port thread helper. This helper thread is responsible for
* atomically retrieving and clearing the signal mask and calling
* the machine-independant driver core.
*
* MPSAFE
*/
static
void
ahci_port_thread(void *arg)
{
struct ahci_port *ap = arg;
int mask;
/*
* Sets us up as an interrupt support thread, meaning we are
* given a higher priority and we can preempt normal threads.
*/
lwkt_set_interrupt_support_thread();
/*
* The helper thread is responsible for the initial port init,
* so all the ports can be inited in parallel.
*
* We also run the state machine which should do all probes.
* Since CAM is not attached yet we will not get out-of-order
* SCSI attachments.
*/
ahci_os_lock_port(ap);
ahci_port_init(ap);
atomic_clear_int(&ap->ap_signal, AP_SIGF_THREAD_SYNC);
wakeup(&ap->ap_signal);
ahci_port_state_machine(ap, 1);
ahci_os_unlock_port(ap);
atomic_clear_int(&ap->ap_signal, AP_SIGF_INIT);
wakeup(&ap->ap_signal);
/*
* Then loop on the helper core.
*/
mask = ap->ap_signal;
while ((mask & AP_SIGF_STOP) == 0) {
ahci_port_thread_core(ap, mask);
lockmgr(&ap->ap_sig_lock, LK_EXCLUSIVE);
if (ap->ap_signal == 0) {
lksleep(&ap->ap_thread, &ap->ap_sig_lock, 0,
"ahport", 0);
}
mask = ap->ap_signal;
atomic_clear_int(&ap->ap_signal, mask);
lockmgr(&ap->ap_sig_lock, LK_RELEASE);
}
ap->ap_thread = NULL;
}
/*
* AHCI port multiplier probe. This routine is run by the hardreset code
* if it gets past the device detect, whether or not BSY is found to be
* stuck.
*
* We MUST use CLO to properly probe whether the port multiplier exists
* or not.
*
* Return 0 on success, non-zero on failure.
*/
int
ahci_pm_port_probe(struct ahci_port *ap, int orig_error)
{
struct ahci_cmd_hdr *cmd_slot;
struct ata_port *at;
struct ahci_ccb *ccb = NULL;
u_int8_t *fis = NULL;
int error;
u_int32_t cmd;
u_int32_t fbs;
int count;
int rstcount;
int i;
int fbsmode;
int sig;
count = 2;
rstcount = 2;
retry:
fbsmode = 0;
/*
* This code is only called from hardreset, which does not
* high level command processing. The port should be stopped.
*
* Set PMA mode while the port is stopped.
*
* NOTE: On retry the port might be running, stopped, or failed.
*/
ahci_port_stop(ap, 0);
ap->ap_state = AP_S_NORMAL;
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
#if 1
if ((cmd & AHCI_PREG_CMD_PMA) == 0) {
cmd |= AHCI_PREG_CMD_PMA;
ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
}
#endif
/*
* Check to see if FBS is supported by checking the cap and the
* support bit in the port CMD register. If it looks good, try
* to write to the FBS register.
*/
if (ap->ap_sc->sc_cap & AHCI_REG_CAP_FBSS) {
const char *str1 = "";
const char *str2 = "";
const char *str3 = "";
if (cmd & AHCI_PREG_CMD_FBSCP) {
str1 = "HW indicates support";
} else if (ap->ap_sc->sc_flags & AHCI_F_FORCE_FBSS) {
cmd |= AHCI_PREG_CMD_FBSCP; /* Force on */
str1 = "HW indicates no-support, force";
} else {
str1 = "HW indicates no-support";
}
if (cmd & AHCI_PREG_CMD_FBSCP) {
fbs = ahci_pread(ap, AHCI_PREG_FBS);
ahci_pwrite(ap, AHCI_PREG_FBS, fbs | AHCI_PREG_FBS_EN);
fbs = ahci_pread(ap, AHCI_PREG_FBS);
if (fbs & AHCI_PREG_FBS_EN) {
str2 = ", enable succeeded";
fbsmode = 1;
} else {
str2 = ", enable failed";
}
/*
* Must be off during the PM probe, we will enable
* it at the end.
*/
ahci_pwrite(ap, AHCI_PREG_FBS, fbs & ~AHCI_PREG_FBS_EN);
}
kprintf("%s: FBS Cap detected: %s%s%s\n",
PORTNAME(ap), str1, str2, str3);
}
/*
* If the probe fails, cleanup any previous state.
*/
if (fbsmode == 0 && (ap->ap_flags & AP_F_FBSS_ENABLED)) {
ap->ap_flags &= ~AP_F_FBSS_ENABLED;
ahci_pwrite(ap, AHCI_PREG_FBS, 0);
}
/*
* Flush any errors and request CLO unconditionally, then start
* the port.
*/
ahci_flush_tfd(ap);
ahci_port_clo(ap);
if (ahci_port_start(ap)) {
kprintf("%s: PMPROBE cannot start port\n",
PORTNAME(ap));
error = EIO;
goto err;
}
/*
* When a PM is present and the cable or driver cycles, the PM may
* hang in BSY and require another COMRESET sequence to clean it up.
* The CLO above didn't work (for example, if there is no CLO
* support), so do another hard COMRESET to try to clear the busy
* condition.
*/
if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
if (--rstcount) {
int pmdetect;
kprintf("%s: PMPROBE BUSY, comreset and retry\n",
PORTNAME(ap));
ahci_comreset(ap, &pmdetect);
if (pmdetect)
goto retry;
}
kprintf("%s: PMPROBE BUSY, giving up\n",
PORTNAME(ap));
error = EBUSY;
goto err;
}
/*
* Use the error CCB for all commands
*
* NOTE! This CCB is used for both the first and second commands.
* The second command must use CCB slot 1 to properly load
* the signature.
*/
ccb = ahci_get_err_ccb(ap);
ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_SILENT;
ccb->ccb_xa.complete = ahci_pm_dummy_done;
ccb->ccb_xa.at = ap->ap_ata[15];
cmd_slot = ccb->ccb_cmd_hdr;
KKASSERT(ccb->ccb_slot == 1);
/*
* Prep the first H2D command with SRST feature & clear busy/reset
* flags.
*/
fis = ccb->ccb_cmd_table->cfis;
bzero(fis, sizeof(ccb->ccb_cmd_table->cfis));
fis[0] = ATA_FIS_TYPE_H2D;
fis[1] = 0x0F; /* Target 15 */
fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT;
cmd_slot->prdtl = 0;
cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_C); /* Clear busy on OK */
cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_R); /* Reset */
cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_PMP); /* port 0xF */
ccb->ccb_xa.state = ATA_S_PENDING;
if (bootverbose) {
kprintf("%s: PMPROBE PreStatus 0x%pb%i\n", PORTNAME(ap),
AHCI_PFMT_TFD_STS, ahci_pread(ap, AHCI_PREG_TFD));
}
/*
* The only way one can determine if a port multiplier is on the
* port is to probe target 15, and of course this will fail if
* there is no port multiplier.
*
* The probing has to be done whether or not a device is probed on
* target 0, because when a PM is attached target 0 represents
* slot #0 behind the PM.
*
* Port multipliers are expected to answer more quickly than normal
* devices, use a shorter timeout than normal.
*
* If there is no PM here this command can still succeed due to
* the _C_
*/
if (ahci_poll(ccb, 1000, ahci_quick_timeout) != ATA_S_COMPLETE) {
kprintf("%s: PMPROBE(1) No Port Multiplier was found.\n",
PORTNAME(ap));
if (--count) {
rstcount = 2;
ahci_put_err_ccb(ccb);
goto retry;
}
error = EBUSY;
goto err;
}
if (bootverbose) {
kprintf("%s: PMPROBE PosStatus 0x%pb%i\n", PORTNAME(ap),
AHCI_PFMT_TFD_STS, ahci_pread(ap, AHCI_PREG_TFD));
}
#if 0
/*
* REMOVED - Ignore a BSY condition between the first and second
* FIS in the PM probe. Seems to work better.
*/
if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
kprintf("%s: PMPROBE Busy after first FIS\n", PORTNAME(ap));
} else if (bootverbose) {
kprintf("%s: PMPROBE Clean after first FIS\n", PORTNAME(ap));
}
#endif
/*
* The device may have muffed up the PHY when it reset.
*/
ahci_flush_tfd(ap);
ahci_pwrite(ap, AHCI_PREG_SERR, -1);
/* ahci_pm_phy_status(ap, 15, &cmd); */
/*
* Prep second D2H command to read status and complete reset sequence
* AHCI 10.4.1 and "Serial ATA Revision 2.6". I can't find the ATA
* Rev 2.6 and it is unclear how the second FIS should be set up
* from the AHCI document.
*
* Give the device 3ms before sending the second FIS.
*
* It is unclear which other fields in the FIS are used. Just zero
* everything.
*/
ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_SILENT;
bzero(fis, sizeof(ccb->ccb_cmd_table->cfis));
fis[0] = ATA_FIS_TYPE_H2D;
fis[1] = 0x0F;
fis[15] = ATA_FIS_CONTROL_4BIT;
cmd_slot->prdtl = 0;
cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_PMP); /* port 0xF */
ccb->ccb_xa.state = ATA_S_PENDING;
/*
* The only way one can determine if a port multiplier is on the
* port is to probe target 15, and of course this will fail if
* there is no port multiplier.
*
* The probing has to be done whether or not a device is probed on
* target 0, because when a PM is attached target 0 represents
* slot #0 behind the PM.
*/
if (ahci_poll(ccb, 5000, ahci_quick_timeout) != ATA_S_COMPLETE) {
kprintf("%s: PMPROBE(2) No Port Multiplier was found.\n",
PORTNAME(ap));
if (--count) {
rstcount = 2;
ahci_put_err_ccb(ccb);
goto retry;
}
error = EBUSY;
goto err;
}
/*
* Some controllers return completion for the second FIS before
* updating the signature register. Sleep a bit to allow for it.
*/
ahci_os_sleep(500);
/*
* What? We succeeded? Yup, but for some reason the signature
* is still latched from the original detect (that saw target 0
* behind the PM), and I don't know how to clear the condition
* other then by retrying the whole reset sequence.
*/
sig = ahci_port_signature_detect(ap, NULL);
if (sig == ATA_PORT_T_PM) {
kprintf("%s: PMPROBE PM Signature detected\n",
PORTNAME(ap));
ap->ap_ata[15]->at_probe = ATA_PROBE_GOOD;
error = 0;
} else if (--count == 0) {
kprintf("%s: PMPROBE PM Signature not detected\n",
PORTNAME(ap));
error = EBUSY;
} else {
rstcount = 2;
fis[15] = 0;
ahci_put_err_ccb(ccb);
if (bootverbose) {
kprintf("%s: PMPROBE retry on count\n",
PORTNAME(ap));
}
goto retry;
}
/*
* Fall through / clean up the CCB and perform error processing.
*/
err:
if (ccb != NULL)
ahci_put_err_ccb(ccb);
if (error == 0 && ahci_pm_identify(ap)) {
ahci_os_sleep(500);
if (ahci_pm_identify(ap)) {
kprintf("%s: PM - cannot identify port multiplier\n",
PORTNAME(ap));
error = EBUSY;
} else {
kprintf("%s: PM - Had to identify twice\n",
PORTNAME(ap));
}
}
/*
* Turn on FBS mode, clear any stale error. Enforce a 1/10 second
* delay primarily for the IGN_CR quirk.
*/
if (error == 0 && fbsmode) {
ahci_port_stop(ap, 0);
ap->ap_flags |= AP_F_FBSS_ENABLED;
fbs = ahci_pread(ap, AHCI_PREG_FBS);
fbs &= ~AHCI_PREG_FBS_DEV;
fbs |= AHCI_PREG_FBS_DEC;
ahci_pwrite(ap, AHCI_PREG_FBS, fbs | AHCI_PREG_FBS_EN);
ahci_os_sleep(100);
if (ahci_port_start(ap)) {
kprintf("%s: PMPROBE failed to restart port "
"after FBS enable\n",
PORTNAME(ap));
ahci_pwrite(ap, AHCI_PREG_FBS, fbs & ~AHCI_PREG_FBS_EN);
ap->ap_flags &= ~AP_F_FBSS_ENABLED;
}
}
/*
* If we probed the PM reset the state for the targets behind
* it so they get probed by the state machine.
*/
if (error == 0) {
for (i = 0; i < AHCI_MAX_PMPORTS; ++i) {
at = ap->ap_ata[i];
at->at_probe = ATA_PROBE_NEED_INIT;
at->at_features |= ATA_PORT_F_RESCAN;
}
ap->ap_type = ATA_PORT_T_PM;
return (0);
}
/*
* If we failed turn off PMA, otherwise identify the port multiplier.
* CAM will iterate the devices.
*/
ahci_port_stop(ap, 0);
ahci_port_clo(ap);
#if 1
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
cmd &= ~AHCI_PREG_CMD_PMA;
ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
#endif
if (orig_error == 0) {
if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
kprintf("%s: PM probe: port will not come ready\n",
PORTNAME(ap));
orig_error = EBUSY;
ahci_port_init(ap);
}
}
return(orig_error);
}
/*
* AHCI port multiplier probe. This routine is run by the hardreset code
* if it gets past the device detect, whether or not BSY is found to be
* stuck.
*
* We MUST use CLO to properly probe whether the port multiplier exists
* or not.
*
* Return 0 on success, non-zero on failure.
*/
int
ahci_pm_port_probe(struct ahci_port *ap, int orig_error)
{
struct ahci_cmd_hdr *cmd_slot;
struct ata_port *at;
struct ahci_ccb *ccb = NULL;
u_int8_t *fis = NULL;
int error;
u_int32_t cmd;
u_int32_t fbs;
int count;
int i;
count = 2;
retry:
/*
* This code is only called from hardreset, which does not
* high level command processing. The port should be stopped.
*
* Set PMA mode while the port is stopped.
*
* NOTE: On retry the port might be running, stopped, or failed.
*/
ahci_port_stop(ap, 0);
ap->ap_state = AP_S_NORMAL;
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
if ((cmd & AHCI_PREG_CMD_PMA) == 0) {
cmd |= AHCI_PREG_CMD_PMA;
ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
}
/*
* Try to enable FIS-Based switching. We have to probe PREG_FBS
*
* XXX I'm still trying to find an AHCI chipset that actually
* supports FBSS so I can finish implementing support, so
* this doesn't do a whole lot right now.
*/
if (ap->ap_sc->sc_cap & AHCI_REG_CAP_FBSS) {
const char *str1 = "";
const char *str2 = "";
const char *str3 = "";
if (cmd & AHCI_PREG_CMD_FBSCP) {
str1 = "HW indicates support";
} else if (ap->ap_sc->sc_flags & AHCI_F_FORCE_FBSS) {
cmd |= AHCI_PREG_CMD_FBSCP; /* Force on */
str1 = "HW indicates no-support, force";
} else {
str1 = "HW indicates no-support";
}
if (cmd & AHCI_PREG_CMD_FBSCP) {
fbs = ahci_pread(ap, AHCI_PREG_FBS);
ahci_pwrite(ap, AHCI_PREG_FBS, fbs | AHCI_PREG_FBS_EN);
fbs = ahci_pread(ap, AHCI_PREG_FBS);
if (fbs & AHCI_PREG_FBS_EN) {
str2 = ", enable succeeded";
str3 = ", (driver support not yet implemented)";
} else {
str2 = ", enable failed";
}
ahci_pwrite(ap, AHCI_PREG_FBS, fbs & ~AHCI_PREG_FBS_EN);
}
kprintf("%s: Port multiplier: FIS-Based Sw: %s%s%s\n",
PORTNAME(ap), str1, str2, str3);
}
/*
* Flush any errors and request CLO unconditionally, then start
* the port.
*/
ahci_flush_tfd(ap);
ahci_port_clo(ap);
if (ahci_port_start(ap)) {
kprintf("%s: PMPROBE failed to start port, cannot softreset\n",
PORTNAME(ap));
error = EIO;
goto err;
}
/*
* Check whether CLO worked
*/
if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
kprintf("%s: PMPROBE CLO %s, need port reset\n",
PORTNAME(ap),
(ahci_read(ap->ap_sc, AHCI_REG_CAP) & AHCI_REG_CAP_SCLO)
? "failed" : "unsupported");
error = EBUSY;
goto err;
}
/*
* Use the error CCB for all commands
*
* NOTE! This CCB is used for both the first and second commands.
* The second command must use CCB slot 1 to properly load
* the signature.
*/
ccb = ahci_get_err_ccb(ap);
ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_SILENT;
ccb->ccb_xa.complete = ahci_pm_dummy_done;
ccb->ccb_xa.at = ap->ap_ata[15];
cmd_slot = ccb->ccb_cmd_hdr;
KKASSERT(ccb->ccb_slot == 1);
/*
* Prep the first H2D command with SRST feature & clear busy/reset
* flags.
*/
fis = ccb->ccb_cmd_table->cfis;
bzero(fis, sizeof(ccb->ccb_cmd_table->cfis));
fis[0] = ATA_FIS_TYPE_H2D;
fis[1] = 0x0F; /* Target 15 */
fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT;
cmd_slot->prdtl = 0;
cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_C); /* Clear busy on OK */
cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_R); /* Reset */
cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_PMP); /* port 0xF */
ccb->ccb_xa.state = ATA_S_PENDING;
/*
* The only way one can determine if a port multiplier is on the
* port is to probe target 15, and of course this will fail if
* there is no port multiplier.
*
* The probing has to be done whether or not a device is probed on
* target 0, because when a PM is attached target 0 represents
* slot #0 behind the PM.
*
* Port multipliers are expected to answer more quickly than normal
* devices, use a shorter timeout than normal.
*
* If there is no PM here this command can still succeed due to
* the _C_
*/
if (ahci_poll(ccb, 500, ahci_quick_timeout) != ATA_S_COMPLETE) {
kprintf("%s: PMPROBE(1) No Port Multiplier was found.\n",
PORTNAME(ap));
if (--count) {
ahci_put_err_ccb(ccb);
goto retry;
}
error = EBUSY;
goto err;
}
if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
kprintf("%s: PMPROBE Busy after first FIS\n", PORTNAME(ap));
}
/*
* The device may have muffed up the PHY when it reset.
*/
ahci_flush_tfd(ap);
ahci_pwrite(ap, AHCI_PREG_SERR, -1);
/* ahci_pm_phy_status(ap, 15, &cmd); */
/*
* Prep second D2H command to read status and complete reset sequence
* AHCI 10.4.1 and "Serial ATA Revision 2.6". I can't find the ATA
* Rev 2.6 and it is unclear how the second FIS should be set up
* from the AHCI document.
*
* Give the device 3ms before sending the second FIS.
*
* It is unclear which other fields in the FIS are used. Just zero
* everything.
*/
ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_SILENT;
bzero(fis, sizeof(ccb->ccb_cmd_table->cfis));
fis[0] = ATA_FIS_TYPE_H2D;
fis[1] = 0x0F;
fis[15] = ATA_FIS_CONTROL_4BIT;
cmd_slot->prdtl = 0;
cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_PMP); /* port 0xF */
ccb->ccb_xa.state = ATA_S_PENDING;
/*
* The only way one can determine if a port multiplier is on the
* port is to probe target 15, and of course this will fail if
* there is no port multiplier.
*
* The probing has to be done whether or not a device is probed on
* target 0, because when a PM is attached target 0 represents
* slot #0 behind the PM.
*/
if (ahci_poll(ccb, 5000, ahci_quick_timeout) != ATA_S_COMPLETE) {
kprintf("%s: PMPROBE(2) No Port Multiplier was found.\n",
PORTNAME(ap));
if (--count) {
ahci_put_err_ccb(ccb);
goto retry;
}
error = EBUSY;
goto err;
}
/*
* What? We succeeded? Yup, but for some reason the signature
* is still latched from the original detect (that saw target 0
* behind the PM), and I don't know how to clear the condition
* other then by retrying the whole reset sequence.
*/
if (--count) {
fis[15] = 0;
ahci_put_err_ccb(ccb);
goto retry;
}
/*
* Get the signature. The caller sets the ap fields.
*/
if (ahci_port_signature_detect(ap, NULL) == ATA_PORT_T_PM) {
ap->ap_ata[15]->at_probe = ATA_PROBE_GOOD;
error = 0;
} else {
error = EBUSY;
}
/*
* Fall through / clean up the CCB and perform error processing.
*/
err:
if (ccb != NULL)
ahci_put_err_ccb(ccb);
if (error == 0 && ahci_pm_identify(ap)) {
kprintf("%s: PM - cannot identify port multiplier\n",
PORTNAME(ap));
error = EBUSY;
}
/*
* If we probed the PM reset the state for the targets behind
* it so they get probed by the state machine.
*/
if (error == 0) {
for (i = 0; i < AHCI_MAX_PMPORTS; ++i) {
at = ap->ap_ata[i];
at->at_probe = ATA_PROBE_NEED_INIT;
at->at_features |= ATA_PORT_F_RESCAN;
}
ap->ap_type = ATA_PORT_T_PM;
return (0);
}
/*
* If we failed turn off PMA, otherwise identify the port multiplier.
* CAM will iterate the devices.
*/
ahci_port_stop(ap, 0);
ahci_port_clo(ap);
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
cmd &= ~AHCI_PREG_CMD_PMA;
ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
if (orig_error == 0) {
if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
kprintf("%s: PM probe: port will not come ready\n",
PORTNAME(ap));
orig_error = EBUSY;
ahci_port_init(ap);
}
}
return(orig_error);
}