int
cpu_offline(fmd_hdl_t *hdl, nvlist_t *asru, const char *uuid, int cpustate)
{
int i;
uint_t cpuid;
cma_cpu_t *cpu;
if (nvlist_lookup_uint32(asru, FM_FMRI_CPU_ID, &cpuid) != 0) {
fmd_hdl_debug(hdl, "missing '%s'\n", FM_FMRI_CPU_ID);
cma_stats.bad_flts.fmds_value.ui64++;
return (CMA_RA_FAILURE);
}
/*
* cpu offlining using ldom_fmri_retire() may be asynchronous, so we
* have to set the timer and check the cpu status later.
*/
for (i = 0; i < cma.cma_cpu_tries;
i++, (void) nanosleep(&cma.cma_cpu_delay, NULL)) {
if (cpu_cmd(hdl, asru, cpustate) != -1) {
cma_stats.cpu_flts.fmds_value.ui64++;
break;
}
}
if (i >= cma.cma_cpu_tries) {
cma_stats.cpu_fails.fmds_value.ui64++;
}
/*
* check to see if the cpu has been offline.
*/
fmd_hdl_debug(hdl, "cpu is not offline yet - sleeping\n");
/*
* Create a cpu node and add to the head of the cpu list
*/
cpu = fmd_hdl_zalloc(hdl, sizeof (cma_cpu_t), FMD_SLEEP);
(void) nvlist_dup(asru, &cpu->cpu_fmri, 0);
if (uuid != NULL)
cpu->cpu_uuid = fmd_hdl_strdup(hdl, uuid, FMD_SLEEP);
cpu->cpuid = cpuid;
cpu->cpu_next = cma.cma_cpus;
cma.cma_cpus = cpu;
if (cma.cma_cpu_timerid != 0)
fmd_timer_remove(hdl, cma.cma_cpu_timerid);
cma.cma_cpu_curdelay = cma.cma_cpu_mindelay;
cma.cma_cpu_timerid =
fmd_timer_install(hdl, NULL, NULL, cma.cma_cpu_curdelay);
return (CMA_RA_FAILURE);
}
void
cma_cpu_start_retry(fmd_hdl_t *hdl, nvlist_t *fmri, const char *uuid,
boolean_t repair)
{
cma_cpu_t *cpu;
char *scheme;
uint_t cpuid;
nvlist_t *asru = NULL;
topo_hdl_t *thp;
int err;
if (repair || nvlist_lookup_string(fmri, FM_FMRI_SCHEME, &scheme) != 0)
return;
if (strcmp(scheme, FM_FMRI_SCHEME_CPU) == 0) {
if (nvlist_lookup_uint32(fmri, FM_FMRI_CPU_ID, &cpuid) != 0)
return;
} else if (strcmp(scheme, FM_FMRI_SCHEME_HC) != 0) {
return;
} else {
/* lookup cpuid from ASRU */
thp = fmd_fmri_topo_hold(TOPO_VERSION);
if (thp != NULL) {
(void) topo_fmri_asru(thp, fmri, &asru, &err);
fmd_fmri_topo_rele(thp);
}
if (nvlist_lookup_uint32(asru, FM_FMRI_CPU_ID, &cpuid) != 0) {
nvlist_free(asru);
return;
}
}
/*
* check to see if the cpu has been offline.
*/
fmd_hdl_debug(hdl, "cpu %u is not offline yet - sleeping\n", cpuid);
/*
* Create a cpu node and add to the head of the cpu list
*/
cpu = fmd_hdl_zalloc(hdl, sizeof (cma_cpu_t), FMD_SLEEP);
(void) nvlist_dup(fmri, &cpu->cpu_fmri, 0);
if (uuid != NULL)
cpu->cpu_uuid = fmd_hdl_strdup(hdl, uuid, FMD_SLEEP);
cpu->cpuid = cpuid;
cpu->cpu_next = cma.cma_cpus;
cma.cma_cpus = cpu;
if (cma.cma_cpu_timerid != 0)
fmd_timer_remove(hdl, cma.cma_cpu_timerid);
cma.cma_cpu_curdelay = cma.cma_cpu_mindelay;
cma.cma_cpu_timerid =
fmd_timer_install(hdl, NULL, NULL, cma.cma_cpu_curdelay);
}
/*
* Solve a given ZFS case. This first checks to make sure the diagnosis is
* still valid, as well as cleaning up any pending timer associated with the
* case.
*/
static void
zfs_case_solve(fmd_hdl_t *hdl, zfs_case_t *zcp, const char *faultname,
boolean_t checkunusable)
{
nvlist_t *detector, *fault;
boolean_t serialize;
nvlist_t *fru = NULL;
fmd_hdl_debug(hdl, "solving fault '%s'", faultname);
/*
* Construct the detector from the case data. The detector is in the
* ZFS scheme, and is either the pool or the vdev, depending on whether
* this is a vdev or pool fault.
*/
detector = fmd_nvl_alloc(hdl, FMD_SLEEP);
(void) nvlist_add_uint8(detector, FM_VERSION, ZFS_SCHEME_VERSION0);
(void) nvlist_add_string(detector, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS);
(void) nvlist_add_uint64(detector, FM_FMRI_ZFS_POOL,
zcp->zc_data.zc_pool_guid);
if (zcp->zc_data.zc_vdev_guid != 0) {
(void) nvlist_add_uint64(detector, FM_FMRI_ZFS_VDEV,
zcp->zc_data.zc_vdev_guid);
}
fault = fmd_nvl_create_fault(hdl, faultname, 100, detector,
fru, detector);
fmd_case_add_suspect(hdl, zcp->zc_case, fault);
nvlist_free(fru);
fmd_case_solve(hdl, zcp->zc_case);
serialize = B_FALSE;
if (zcp->zc_data.zc_has_remove_timer) {
fmd_timer_remove(hdl, zcp->zc_remove_timer);
zcp->zc_data.zc_has_remove_timer = 0;
serialize = B_TRUE;
}
if (serialize)
zfs_case_serialize(hdl, zcp);
nvlist_free(detector);
}
static void
dl_trigger_enum(disk_lights_t *dl)
{
/*
* If we're already on the short-poll coalesce timer, then return
* immediately.
*/
if (dl->dl_triggered == B_TRUE)
return;
dl->dl_triggered = B_TRUE;
/*
* Replace existing poll timer with coalesce timer:
*/
if (dl->dl_timer != 0)
fmd_timer_remove(dl->dl_fmd, dl->dl_timer);
dl->dl_timer = fmd_timer_install(dl->dl_fmd, NULL, NULL,
dl->dl_coalesce_interval);
}
/*
* Solve a given ZFS case. This first checks to make sure the diagnosis is
* still valid, as well as cleaning up any pending timer associated with the
* case.
*/
static void
zfs_case_solve(fmd_hdl_t *hdl, zfs_case_t *zcp, const char *faultname,
boolean_t checkunusable)
{
libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
nvlist_t *detector, *fault;
boolean_t serialize;
nvlist_t *fmri, *fru;
topo_hdl_t *thp;
int err;
/*
* Construct the detector from the case data. The detector is in the
* ZFS scheme, and is either the pool or the vdev, depending on whether
* this is a vdev or pool fault.
*/
detector = fmd_nvl_alloc(hdl, FMD_SLEEP);
(void) nvlist_add_uint8(detector, FM_VERSION, ZFS_SCHEME_VERSION0);
(void) nvlist_add_string(detector, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS);
(void) nvlist_add_uint64(detector, FM_FMRI_ZFS_POOL,
zcp->zc_data.zc_pool_guid);
if (zcp->zc_data.zc_vdev_guid != 0) {
(void) nvlist_add_uint64(detector, FM_FMRI_ZFS_VDEV,
zcp->zc_data.zc_vdev_guid);
}
/*
* We also want to make sure that the detector (pool or vdev) properly
* reflects the diagnosed state, when the fault corresponds to internal
* ZFS state (i.e. not checksum or I/O error-induced). Otherwise, a
* device which was unavailable early in boot (because the driver/file
* wasn't available) and is now healthy will be mis-diagnosed.
*/
if (!fmd_nvl_fmri_present(hdl, detector) ||
(checkunusable && !fmd_nvl_fmri_unusable(hdl, detector))) {
fmd_case_close(hdl, zcp->zc_case);
nvlist_free(detector);
return;
}
fru = NULL;
if (zcp->zc_fru != NULL &&
(thp = fmd_hdl_topo_hold(hdl, TOPO_VERSION)) != NULL) {
/*
* If the vdev had an associated FRU, then get the FRU nvlist
* from the topo handle and use that in the suspect list. We
* explicitly lookup the FRU because the fmri reported from the
* kernel may not have up to date details about the disk itself
* (serial, part, etc).
*/
if (topo_fmri_str2nvl(thp, zcp->zc_fru, &fmri, &err) == 0) {
/*
* If the disk is part of the system chassis, but the
* FRU indicates a different chassis ID than our
* current system, then ignore the error. This
* indicates that the device was part of another
* cluster head, and for obvious reasons cannot be
* imported on this system.
*/
if (libzfs_fru_notself(zhdl, zcp->zc_fru)) {
fmd_case_close(hdl, zcp->zc_case);
nvlist_free(fmri);
fmd_hdl_topo_rele(hdl, thp);
nvlist_free(detector);
return;
}
/*
* If the device is no longer present on the system, or
* topo_fmri_fru() fails for other reasons, then fall
* back to the fmri specified in the vdev.
*/
if (topo_fmri_fru(thp, fmri, &fru, &err) != 0)
fru = fmd_nvl_dup(hdl, fmri, FMD_SLEEP);
nvlist_free(fmri);
}
fmd_hdl_topo_rele(hdl, thp);
}
fault = fmd_nvl_create_fault(hdl, faultname, 100, detector,
fru, detector);
fmd_case_add_suspect(hdl, zcp->zc_case, fault);
nvlist_free(fru);
fmd_case_solve(hdl, zcp->zc_case);
serialize = B_FALSE;
if (zcp->zc_data.zc_has_remove_timer) {
fmd_timer_remove(hdl, zcp->zc_remove_timer);
zcp->zc_data.zc_has_remove_timer = 0;
serialize = B_TRUE;
}
if (serialize)
zfs_case_serialize(hdl, zcp);
nvlist_free(detector);
}
/*ARGSUSED*/
int
cma_page_retire(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru,
const char *uuid, boolean_t repair)
{
cma_page_t *page;
uint64_t pageaddr;
const char *action = repair ? "unretire" : "retire";
int rc;
nvlist_t *rsrc = NULL, *asrucp = NULL, *hcsp;
(void) nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc);
if (nvlist_dup(asru, &asrucp, 0) != 0) {
fmd_hdl_debug(hdl, "page retire nvlist dup failed\n");
return (CMA_RA_FAILURE);
}
/* It should already be expanded, but we'll do it again anyway */
if (fmd_nvl_fmri_expand(hdl, asrucp) < 0) {
fmd_hdl_debug(hdl, "failed to expand page asru\n");
cma_stats.bad_flts.fmds_value.ui64++;
nvlist_free(asrucp);
return (CMA_RA_FAILURE);
}
if (!repair && !fmd_nvl_fmri_present(hdl, asrucp)) {
fmd_hdl_debug(hdl, "page retire overtaken by events\n");
cma_stats.page_nonent.fmds_value.ui64++;
nvlist_free(asrucp);
return (CMA_RA_SUCCESS);
}
/* Figure out physaddr from resource or asru */
if (rsrc == NULL ||
nvlist_lookup_nvlist(rsrc, FM_FMRI_HC_SPECIFIC, &hcsp) != 0 ||
(nvlist_lookup_uint64(hcsp, "asru-" FM_FMRI_HC_SPECIFIC_PHYSADDR,
&pageaddr) != 0 && nvlist_lookup_uint64(hcsp,
FM_FMRI_HC_SPECIFIC_PHYSADDR, &pageaddr) != 0)) {
if (nvlist_lookup_uint64(asrucp, FM_FMRI_MEM_PHYSADDR,
&pageaddr) != 0) {
fmd_hdl_debug(hdl, "mem fault missing 'physaddr'\n");
cma_stats.bad_flts.fmds_value.ui64++;
nvlist_free(asrucp);
return (CMA_RA_FAILURE);
}
}
if (repair) {
if (!cma.cma_page_dounretire) {
fmd_hdl_debug(hdl, "suppressed unretire of page %llx\n",
(u_longlong_t)pageaddr);
cma_stats.page_supp.fmds_value.ui64++;
nvlist_free(asrucp);
return (CMA_RA_SUCCESS);
}
/* If unretire via topo fails, we fall back to legacy way */
if (rsrc == NULL || (rc = fmd_nvl_fmri_unretire(hdl, rsrc)) < 0)
rc = cma_fmri_page_unretire(hdl, asrucp);
} else {
if (!cma.cma_page_doretire) {
fmd_hdl_debug(hdl, "suppressed retire of page %llx\n",
(u_longlong_t)pageaddr);
cma_stats.page_supp.fmds_value.ui64++;
nvlist_free(asrucp);
return (CMA_RA_FAILURE);
}
/* If retire via topo fails, we fall back to legacy way */
if (rsrc == NULL || (rc = fmd_nvl_fmri_retire(hdl, rsrc)) < 0)
rc = cma_fmri_page_retire(hdl, asrucp);
}
if (rc == FMD_AGENT_RETIRE_DONE) {
fmd_hdl_debug(hdl, "%sd page 0x%llx\n",
action, (u_longlong_t)pageaddr);
if (repair)
cma_stats.page_repairs.fmds_value.ui64++;
else
cma_stats.page_flts.fmds_value.ui64++;
nvlist_free(asrucp);
return (CMA_RA_SUCCESS);
} else if (repair || rc != FMD_AGENT_RETIRE_ASYNC) {
fmd_hdl_debug(hdl, "%s of page 0x%llx failed, will not "
"retry: %s\n", action, (u_longlong_t)pageaddr,
strerror(errno));
cma_stats.page_fails.fmds_value.ui64++;
nvlist_free(asrucp);
return (CMA_RA_FAILURE);
}
/*
* The page didn't immediately retire. We'll need to periodically
* check to see if it has been retired.
*/
fmd_hdl_debug(hdl, "page didn't retire - sleeping\n");
page = fmd_hdl_zalloc(hdl, sizeof (cma_page_t), FMD_SLEEP);
page->pg_addr = pageaddr;
if (rsrc != NULL)
(void) nvlist_dup(rsrc, &page->pg_rsrc, 0);
page->pg_asru = asrucp;
if (uuid != NULL)
page->pg_uuid = fmd_hdl_strdup(hdl, uuid, FMD_SLEEP);
page->pg_next = cma.cma_pages;
cma.cma_pages = page;
if (cma.cma_page_timerid != 0)
fmd_timer_remove(hdl, cma.cma_page_timerid);
cma.cma_page_curdelay = cma.cma_page_mindelay;
cma.cma_page_timerid =
fmd_timer_install(hdl, NULL, NULL, cma.cma_page_curdelay);
/* Don't free asrucp here. This FMRI will be needed for retry. */
return (CMA_RA_FAILURE);
}
