static inline void run_gen_bench_impl(const char *impl) { int fn, ncols; uint64_t ds, iter_cnt, iter, disksize; struct rusage start, stop; double elapsed, d_bw; /* Benchmark generate functions */ for (fn = 0; fn < RAIDZ_GEN_NUM; fn++) { for (ds = MIN_CS_SHIFT; ds <= MAX_CS_SHIFT; ds++) { /* create suitable raidz_map */ ncols = rto_opts.rto_dcols + fn + 1; zio_bench.io_size = 1ULL << ds; rm_bench = vdev_raidz_map_alloc(&zio_bench, BENCH_ASHIFT, ncols, fn+1); /* estimate iteration count */ iter_cnt = GEN_BENCH_MEMORY; iter_cnt /= zio_bench.io_size; getrusage(RUSAGE_THREAD, &start); for (iter = 0; iter < iter_cnt; iter++) vdev_raidz_generate_parity(rm_bench); getrusage(RUSAGE_THREAD, &stop); elapsed = get_time_diff(&start, &stop); disksize = (1ULL << ds) / rto_opts.rto_dcols; d_bw = (double)iter_cnt * (double)disksize; d_bw /= (1024.0 * 1024.0 * elapsed); LOG(D_ALL, "%10s, %8s, %zu, %10llu, %lf, %lf, %u\n", impl, raidz_gen_name[fn], rto_opts.rto_dcols, (1ULL<<ds), d_bw, d_bw * (double)(ncols), (unsigned) iter_cnt); vdev_raidz_map_free(rm_bench); } } }
void vdev_raidz_math_init(void) { raidz_impl_ops_t *curr_impl; zio_t *bench_zio = NULL; raidz_map_t *bench_rm = NULL; uint64_t bench_parity; int i, c, fn; /* move supported impl into raidz_supp_impl */ for (i = 0, c = 0; i < ARRAY_SIZE(raidz_all_maths); i++) { curr_impl = (raidz_impl_ops_t *)raidz_all_maths[i]; /* initialize impl */ if (curr_impl->init) curr_impl->init(); if (curr_impl->is_supported()) raidz_supp_impl[c++] = (raidz_impl_ops_t *)curr_impl; } membar_producer(); /* complete raidz_supp_impl[] init */ raidz_supp_impl_cnt = c; /* number of supported impl */ #if !defined(_KERNEL) /* Skip benchmarking and use last implementation as fastest */ memcpy(&vdev_raidz_fastest_impl, raidz_supp_impl[raidz_supp_impl_cnt-1], sizeof (vdev_raidz_fastest_impl)); strcpy(vdev_raidz_fastest_impl.name, "fastest"); raidz_math_initialized = B_TRUE; /* Use 'cycle' math selection method for userspace */ VERIFY0(vdev_raidz_impl_set("cycle")); return; #endif /* Fake an zio and run the benchmark on a warmed up buffer */ bench_zio = kmem_zalloc(sizeof (zio_t), KM_SLEEP); bench_zio->io_offset = 0; bench_zio->io_size = BENCH_ZIO_SIZE; /* only data columns */ bench_zio->io_abd = abd_alloc_linear(BENCH_ZIO_SIZE, B_TRUE); memset(abd_to_buf(bench_zio->io_abd), 0xAA, BENCH_ZIO_SIZE); /* Benchmark parity generation methods */ for (fn = 0; fn < RAIDZ_GEN_NUM; fn++) { bench_parity = fn + 1; /* New raidz_map is needed for each generate_p/q/r */ bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT, BENCH_D_COLS + bench_parity, bench_parity); benchmark_raidz_impl(bench_rm, fn, benchmark_gen_impl); vdev_raidz_map_free(bench_rm); } /* Benchmark data reconstruction methods */ bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT, BENCH_COLS, PARITY_PQR); for (fn = 0; fn < RAIDZ_REC_NUM; fn++) benchmark_raidz_impl(bench_rm, fn, benchmark_rec_impl); vdev_raidz_map_free(bench_rm); /* cleanup the bench zio */ abd_free(bench_zio->io_abd); kmem_free(bench_zio, sizeof (zio_t)); /* install kstats for all impl */ raidz_math_kstat = kstat_create("zfs", 0, "vdev_raidz_bench", "misc", KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL); if (raidz_math_kstat != NULL) { raidz_math_kstat->ks_data = NULL; raidz_math_kstat->ks_ndata = UINT32_MAX; kstat_set_raw_ops(raidz_math_kstat, raidz_math_kstat_headers, raidz_math_kstat_data, raidz_math_kstat_addr); kstat_install(raidz_math_kstat); } /* Finish initialization */ atomic_swap_32(&zfs_vdev_raidz_impl, user_sel_impl); raidz_math_initialized = B_TRUE; }
static void run_rec_bench_impl(const char *impl) { int fn, ncols, nbad; uint64_t ds, iter_cnt, iter, disksize; hrtime_t start; double elapsed, d_bw; static const int tgt[7][3] = { {1, 2, 3}, /* rec_p: bad QR & D[0] */ {0, 2, 3}, /* rec_q: bad PR & D[0] */ {0, 1, 3}, /* rec_r: bad PQ & D[0] */ {2, 3, 4}, /* rec_pq: bad R & D[0][1] */ {1, 3, 4}, /* rec_pr: bad Q & D[0][1] */ {0, 3, 4}, /* rec_qr: bad P & D[0][1] */ {3, 4, 5} /* rec_pqr: bad & D[0][1][2] */ }; for (fn = 0; fn < RAIDZ_REC_NUM; fn++) { for (ds = MIN_CS_SHIFT; ds <= MAX_CS_SHIFT; ds++) { /* create suitable raidz_map */ ncols = rto_opts.rto_dcols + PARITY_PQR; zio_bench.io_size = 1ULL << ds; /* * raidz block is too short to test * the requested method */ if (zio_bench.io_size / rto_opts.rto_dcols < (1ULL << BENCH_ASHIFT)) continue; rm_bench = vdev_raidz_map_alloc(&zio_bench, BENCH_ASHIFT, ncols, PARITY_PQR); /* estimate iteration count */ iter_cnt = (REC_BENCH_MEMORY); iter_cnt /= zio_bench.io_size; /* calculate how many bad columns there are */ nbad = MIN(3, raidz_ncols(rm_bench) - raidz_parity(rm_bench)); start = gethrtime(); for (iter = 0; iter < iter_cnt; iter++) vdev_raidz_reconstruct(rm_bench, tgt[fn], nbad); elapsed = NSEC2SEC((double) (gethrtime() - start)); disksize = (1ULL << ds) / rto_opts.rto_dcols; d_bw = (double)iter_cnt * (double)(disksize); d_bw /= (1024.0 * 1024.0 * elapsed); LOG(D_ALL, "%10s, %8s, %zu, %10llu, %lf, %lf, %u\n", impl, raidz_rec_name[fn], rto_opts.rto_dcols, (1ULL<<ds), d_bw, d_bw * (double)ncols, (unsigned) iter_cnt); vdev_raidz_map_free(rm_bench); } } }
void vdev_raidz_math_init(void) { raidz_impl_ops_t *curr_impl; zio_t *bench_zio = NULL; raidz_map_t *bench_rm = NULL; uint64_t bench_parity; int i, c, fn; /* init & vdev_raidz_impl_lock */ rw_init(&vdev_raidz_impl_lock, NULL, RW_DEFAULT, NULL); /* move supported impl into raidz_supp_impl */ for (i = 0, c = 0; i < ARRAY_SIZE(raidz_all_maths); i++) { curr_impl = (raidz_impl_ops_t *) raidz_all_maths[i]; /* initialize impl */ if (curr_impl->init) curr_impl->init(); if (curr_impl->is_supported()) { /* init kstat */ init_raidz_kstat(&raidz_impl_kstats[c], curr_impl->name); raidz_supp_impl[c++] = (raidz_impl_ops_t *) curr_impl; } } raidz_supp_impl_cnt = c; /* number of supported impl */ raidz_supp_impl[c] = NULL; /* sentinel */ /* init kstat for original routines */ init_raidz_kstat(&(raidz_impl_kstats[raidz_supp_impl_cnt]), "original"); #if !defined(_KERNEL) /* * Skip benchmarking and use last implementation as fastest */ memcpy(&vdev_raidz_fastest_impl, raidz_supp_impl[raidz_supp_impl_cnt-1], sizeof (vdev_raidz_fastest_impl)); vdev_raidz_fastest_impl.name = "fastest"; raidz_math_initialized = B_TRUE; /* Use 'cycle' math selection method for userspace */ VERIFY0(vdev_raidz_impl_set("cycle")); return; #endif /* Fake an zio and run the benchmark on it */ bench_zio = kmem_zalloc(sizeof (zio_t), KM_SLEEP); bench_zio->io_offset = 0; bench_zio->io_size = BENCH_ZIO_SIZE; /* only data columns */ bench_zio->io_data = zio_data_buf_alloc(BENCH_ZIO_SIZE); VERIFY(bench_zio->io_data); /* Benchmark parity generation methods */ for (fn = 0; fn < RAIDZ_GEN_NUM; fn++) { bench_parity = fn + 1; /* New raidz_map is needed for each generate_p/q/r */ bench_rm = vdev_raidz_map_alloc(bench_zio, 9, BENCH_D_COLS + bench_parity, bench_parity); benchmark_raidz_impl(bench_rm, fn, benchmark_gen_impl); vdev_raidz_map_free(bench_rm); } /* Benchmark data reconstruction methods */ bench_rm = vdev_raidz_map_alloc(bench_zio, 9, BENCH_COLS, PARITY_PQR); for (fn = 0; fn < RAIDZ_REC_NUM; fn++) benchmark_raidz_impl(bench_rm, fn, benchmark_rec_impl); vdev_raidz_map_free(bench_rm); /* cleanup the bench zio */ zio_data_buf_free(bench_zio->io_data, BENCH_ZIO_SIZE); kmem_free(bench_zio, sizeof (zio_t)); /* install kstats for all impl */ raidz_math_kstat = kstat_create("zfs", 0, "vdev_raidz_bench", "misc", KSTAT_TYPE_NAMED, sizeof (raidz_impl_kstat_t) / sizeof (kstat_named_t) * (raidz_supp_impl_cnt + 1), KSTAT_FLAG_VIRTUAL); if (raidz_math_kstat != NULL) { raidz_math_kstat->ks_data = raidz_impl_kstats; kstat_install(raidz_math_kstat); } /* Finish initialization */ raidz_math_initialized = B_TRUE; if (!vdev_raidz_impl_user_set) VERIFY0(vdev_raidz_impl_set("fastest")); }