/* * Function sets desired raidz implementation. * * If we are called before init(), user preference will be saved in * user_sel_impl, and applied in later init() call. This occurs when module * parameter is specified on module load. Otherwise, directly update * zfs_vdev_raidz_impl. * * @val Name of raidz implementation to use * @param Unused. */ int vdev_raidz_impl_set(const char *val) { int err = -EINVAL; char req_name[RAIDZ_IMPL_NAME_MAX]; uint32_t impl = RAIDZ_IMPL_READ(user_sel_impl); size_t i; /* sanitize input */ i = strnlen(val, RAIDZ_IMPL_NAME_MAX); if (i == 0 || i == RAIDZ_IMPL_NAME_MAX) return (err); strlcpy(req_name, val, RAIDZ_IMPL_NAME_MAX); while (i > 0 && !!isspace(req_name[i-1])) i--; req_name[i] = '\0'; /* Check mandatory options */ for (i = 0; i < ARRAY_SIZE(math_impl_opts); i++) { if (strcmp(req_name, math_impl_opts[i].name) == 0) { impl = math_impl_opts[i].sel; err = 0; break; } } /* check all supported impl if init() was already called */ if (err != 0 && raidz_math_initialized) { /* check all supported implementations */ for (i = 0; i < raidz_supp_impl_cnt; i++) { if (strcmp(req_name, raidz_supp_impl[i]->name) == 0) { impl = i; err = 0; break; } } } if (err == 0) { if (raidz_math_initialized) atomic_swap_32(&zfs_vdev_raidz_impl, impl); else atomic_swap_32(&user_sel_impl, impl); } return (err); }
/* * Stop the thread to setup switching mode. */ void vsw_setup_switching_stop(vsw_t *vswp) { kt_did_t tid = 0; /* * Signal the setup_switching thread to stop and wait until it stops. */ mutex_enter(&vswp->sw_thr_lock); if (vswp->sw_thread != NULL) { tid = vswp->sw_thread->t_did; vswp->sw_thr_flags |= VSW_SWTHR_STOP; cv_signal(&vswp->sw_thr_cv); } mutex_exit(&vswp->sw_thr_lock); if (tid != 0) thread_join(tid); (void) atomic_swap_32(&vswp->switching_setup_done, B_FALSE); vswp->mac_open_retries = 0; }
static void fletcher_4_benchmark_impl(boolean_t native, char *data, uint64_t data_size) { struct fletcher_4_kstat *fastest_stat = &fletcher_4_stat_data[fletcher_4_supp_impls_cnt]; hrtime_t start; uint64_t run_bw, run_time_ns, best_run = 0; zio_cksum_t zc; uint32_t i, l, sel_save = IMPL_READ(fletcher_4_impl_chosen); fletcher_checksum_func_t *fletcher_4_test = native ? fletcher_4_native : fletcher_4_byteswap; for (i = 0; i < fletcher_4_supp_impls_cnt; i++) { struct fletcher_4_kstat *stat = &fletcher_4_stat_data[i]; uint64_t run_count = 0; /* temporary set an implementation */ fletcher_4_impl_chosen = i; kpreempt_disable(); start = gethrtime(); do { for (l = 0; l < 32; l++, run_count++) fletcher_4_test(data, data_size, NULL, &zc); run_time_ns = gethrtime() - start; } while (run_time_ns < FLETCHER_4_BENCH_NS); kpreempt_enable(); run_bw = data_size * run_count * NANOSEC; run_bw /= run_time_ns; /* B/s */ if (native) stat->native = run_bw; else stat->byteswap = run_bw; if (run_bw > best_run) { best_run = run_bw; if (native) { fastest_stat->native = i; FLETCHER_4_FASTEST_FN_COPY(native, fletcher_4_supp_impls[i]); } else { fastest_stat->byteswap = i; FLETCHER_4_FASTEST_FN_COPY(byteswap, fletcher_4_supp_impls[i]); } } } /* restore original selection */ atomic_swap_32(&fletcher_4_impl_chosen, sel_save); }
static void shmif_unlockbus(struct shmif_mem *busmem) { unsigned int old; membar_exit(); old = atomic_swap_32(&busmem->shm_lock, LOCK_UNLOCKED); KASSERT(old == LOCK_LOCKED); }
void BnxeTimerStart(um_device_t * pUM) { atomic_swap_32(&pUM->timerEnabled, B_TRUE); pUM->lm_dev.vars.stats.stats_collect.timer_wakeup = 0; /* reset */ pUM->timerID = timeout(BnxeTimer, (void *)pUM, drv_usectohz(BNXE_TIMER_INTERVAL)); }
void BnxeTimerStop(um_device_t * pUM) { atomic_swap_32(&pUM->timerEnabled, B_FALSE); BNXE_LOCK_ENTER_TIMER(pUM); BNXE_LOCK_EXIT_TIMER(pUM); untimeout(pUM->timerID); pUM->timerID = 0; }
/* * Interrupt another CPU. * This is useful to make the other CPU go through a trap so that * it recognizes an address space trap (AST) for preempting a thread. * * It is possible to be preempted here and be resumed on the CPU * being poked, so it isn't an error to poke the current CPU. * We could check this and still get preempted after the check, so * we don't bother. */ void poke_cpu(int cpun) { uint32_t *ptr = (uint32_t *)&cpu[cpun]->cpu_m.poke_cpu_outstanding; /* * If panicstr is set or a poke_cpu is already pending, * no need to send another one. Use atomic swap to protect * against multiple CPUs sending redundant pokes. */ if (panicstr || *ptr == B_TRUE || atomic_swap_32(ptr, B_TRUE) == B_TRUE) return; xt_one(cpun, setsoftint_tl1, poke_cpu_inum, 0); }
int fletcher_4_impl_set(const char *val) { int err = -EINVAL; uint32_t impl = IMPL_READ(fletcher_4_impl_chosen); size_t i, val_len; val_len = strlen(val); while ((val_len > 0) && !!isspace(val[val_len-1])) /* trim '\n' */ val_len--; /* check mandatory implementations */ for (i = 0; i < ARRAY_SIZE(fletcher_4_impl_selectors); i++) { const char *name = fletcher_4_impl_selectors[i].fis_name; if (val_len == strlen(name) && strncmp(val, name, val_len) == 0) { impl = fletcher_4_impl_selectors[i].fis_sel; err = 0; break; } } if (err != 0 && fletcher_4_initialized) { /* check all supported implementations */ for (i = 0; i < fletcher_4_supp_impls_cnt; i++) { const char *name = fletcher_4_supp_impls[i]->name; if (val_len == strlen(name) && strncmp(val, name, val_len) == 0) { impl = i; err = 0; break; } } } if (err == 0) { atomic_swap_32(&fletcher_4_impl_chosen, impl); membar_producer(); } return (err); }
/* * Setup the required switching mode. * Returns: * 0 on success. * EAGAIN if retry is needed. * 1 on all other failures. */ int vsw_setup_switching(vsw_t *vswp) { int rv = 1; D1(vswp, "%s: enter", __func__); /* * Select best switching mode. * This is done as this routine can be called from the timeout * handler to retry setting up a specific mode. Currently only * the function which sets up layer2/promisc mode returns EAGAIN * if the underlying network device is not available yet, causing * retries. */ if (vswp->smode & VSW_LAYER2) { rv = vsw_setup_layer2(vswp); } else if (vswp->smode & VSW_LAYER3) { rv = vsw_setup_layer3(vswp); } else { DERR(vswp, "unknown switch mode"); rv = 1; } if (rv && (rv != EAGAIN)) { cmn_err(CE_WARN, "!vsw%d: Unable to setup specified " "switching mode", vswp->instance); } else if (rv == 0) { (void) atomic_swap_32(&vswp->switching_setup_done, B_TRUE); } D2(vswp, "%s: Operating in mode %d", __func__, vswp->smode); D1(vswp, "%s: exit", __func__); return (rv); }
int ipi_intr(void *v) { struct cpu_info * const ci = curcpu(); int cpu_id = cpu_index(ci); int msr; uint32_t ipi; ci->ci_ev_ipi.ev_count++; ipi = atomic_swap_32(&ci->ci_pending_ipis, 0); if (ipi == IPI_NOMESG) return 1; if (ipi & IPI_XCALL) xc_ipi_handler(); if (ipi & IPI_GENERIC) ipi_cpu_handler(); if (ipi & IPI_SUSPEND) cpu_pause(NULL); if (ipi & IPI_HALT) { struct cpuset_info * const csi = &cpuset_info; aprint_normal("halting CPU %d\n", cpu_id); kcpuset_set(csi->cpus_halted, cpu_id); msr = (mfmsr() & ~PSL_EE) | PSL_POW; for (;;) { __asm volatile ("sync; isync"); mtmsr(msr); } } return 1; }
void vdev_raidz_math_init(void) { raidz_impl_ops_t *curr_impl; zio_t *bench_zio = NULL; raidz_map_t *bench_rm = NULL; uint64_t bench_parity; int i, c, fn; /* move supported impl into raidz_supp_impl */ for (i = 0, c = 0; i < ARRAY_SIZE(raidz_all_maths); i++) { curr_impl = (raidz_impl_ops_t *)raidz_all_maths[i]; /* initialize impl */ if (curr_impl->init) curr_impl->init(); if (curr_impl->is_supported()) raidz_supp_impl[c++] = (raidz_impl_ops_t *)curr_impl; } membar_producer(); /* complete raidz_supp_impl[] init */ raidz_supp_impl_cnt = c; /* number of supported impl */ #if !defined(_KERNEL) /* Skip benchmarking and use last implementation as fastest */ memcpy(&vdev_raidz_fastest_impl, raidz_supp_impl[raidz_supp_impl_cnt-1], sizeof (vdev_raidz_fastest_impl)); strcpy(vdev_raidz_fastest_impl.name, "fastest"); raidz_math_initialized = B_TRUE; /* Use 'cycle' math selection method for userspace */ VERIFY0(vdev_raidz_impl_set("cycle")); return; #endif /* Fake an zio and run the benchmark on a warmed up buffer */ bench_zio = kmem_zalloc(sizeof (zio_t), KM_SLEEP); bench_zio->io_offset = 0; bench_zio->io_size = BENCH_ZIO_SIZE; /* only data columns */ bench_zio->io_abd = abd_alloc_linear(BENCH_ZIO_SIZE, B_TRUE); memset(abd_to_buf(bench_zio->io_abd), 0xAA, BENCH_ZIO_SIZE); /* Benchmark parity generation methods */ for (fn = 0; fn < RAIDZ_GEN_NUM; fn++) { bench_parity = fn + 1; /* New raidz_map is needed for each generate_p/q/r */ bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT, BENCH_D_COLS + bench_parity, bench_parity); benchmark_raidz_impl(bench_rm, fn, benchmark_gen_impl); vdev_raidz_map_free(bench_rm); } /* Benchmark data reconstruction methods */ bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT, BENCH_COLS, PARITY_PQR); for (fn = 0; fn < RAIDZ_REC_NUM; fn++) benchmark_raidz_impl(bench_rm, fn, benchmark_rec_impl); vdev_raidz_map_free(bench_rm); /* cleanup the bench zio */ abd_free(bench_zio->io_abd); kmem_free(bench_zio, sizeof (zio_t)); /* install kstats for all impl */ raidz_math_kstat = kstat_create("zfs", 0, "vdev_raidz_bench", "misc", KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL); if (raidz_math_kstat != NULL) { raidz_math_kstat->ks_data = NULL; raidz_math_kstat->ks_ndata = UINT32_MAX; kstat_set_raw_ops(raidz_math_kstat, raidz_math_kstat_headers, raidz_math_kstat_data, raidz_math_kstat_addr); kstat_install(raidz_math_kstat); } /* Finish initialization */ atomic_swap_32(&zfs_vdev_raidz_impl, user_sel_impl); raidz_math_initialized = B_TRUE; }