void
taskq_destroy(taskq_t *tq)
{
int t;
int nthreads = tq->tq_nthreads;
taskq_wait(tq);
mutex_enter(&tq->tq_lock);
tq->tq_flags &= ~TASKQ_ACTIVE;
cv_broadcast(&tq->tq_dispatch_cv);
while (tq->tq_nthreads != 0)
cv_wait(&tq->tq_wait_cv, &tq->tq_lock);
tq->tq_minalloc = 0;
while (tq->tq_nalloc != 0) {
ASSERT(tq->tq_freelist != NULL);
task_free(tq, task_alloc(tq, KM_SLEEP));
}
mutex_exit(&tq->tq_lock);
for (t = 0; t < nthreads; t++)
(void) thr_join(tq->tq_threadlist[t], NULL, NULL);
kmem_free(tq->tq_threadlist, nthreads * sizeof (thread_t));
rw_destroy(&tq->tq_threadlock);
kmem_free(tq, sizeof (taskq_t));
}
/*
* Wait for pending commit callbacks of already-synced transactions to finish
* processing.
* Calling this function from within a commit callback will deadlock.
*/
void
txg_wait_callbacks(dsl_pool_t *dp)
{
tx_state_t *tx = &dp->dp_tx;
if (tx->tx_commit_cb_taskq != NULL)
taskq_wait(tx->tx_commit_cb_taskq);
}
static int
splat_rwlock_test2(struct file *file, void *arg)
{
rw_priv_t *rwp;
taskq_t *tq;
int i, rc = 0, tq_count = 256;
rwp = (rw_priv_t *)kmalloc(sizeof(*rwp), GFP_KERNEL);
if (rwp == NULL)
return -ENOMEM;
splat_init_rw_priv(rwp, file);
/* Create several threads allowing tasks to race with each other */
tq = taskq_create(SPLAT_RWLOCK_TEST_TASKQ, num_online_cpus(),
maxclsyspri, 50, INT_MAX, TASKQ_PREPOPULATE);
if (tq == NULL) {
rc = -ENOMEM;
goto out;
}
/*
* Schedule N work items to the work queue each of which enters the
* writer rwlock, sleeps briefly, then exits the writer rwlock. On a
* multiprocessor box these work items will be handled by all available
* CPUs. The task function checks to ensure the tracked shared variable
* is always only incremented by one. Additionally, the rwlock itself
* is instrumented such that if any two processors are in the
* critical region at the same time the system will panic. If the
* rwlock is implemented right this will never happy, that's a pass.
*/
for (i = 0; i < tq_count; i++) {
if (!taskq_dispatch(tq,splat_rwlock_test2_func,rwp,TQ_SLEEP)) {
splat_vprint(file, SPLAT_RWLOCK_TEST2_NAME,
"Failed to queue task %d\n", i);
rc = -EINVAL;
}
}
taskq_wait(tq);
if (rwp->rw_rc == tq_count) {
splat_vprint(file, SPLAT_RWLOCK_TEST2_NAME, "%d racing threads "
"correctly entered/exited the rwlock %d times\n",
num_online_cpus(), rwp->rw_rc);
} else {
splat_vprint(file, SPLAT_RWLOCK_TEST2_NAME, "%d racing threads "
"only processed %d/%d w rwlock work items\n",
num_online_cpus(), rwp->rw_rc, tq_count);
rc = -EINVAL;
}
taskq_destroy(tq);
rw_destroy(&(rwp->rw_rwlock));
out:
kfree(rwp);
return rc;
}
static int
splat_taskq_test1_impl(struct file *file, void *arg, boolean_t prealloc)
{
taskq_t *tq;
taskqid_t id;
splat_taskq_arg_t tq_arg;
taskq_ent_t tqe;
taskq_init_ent(&tqe);
splat_vprint(file, SPLAT_TASKQ_TEST1_NAME,
"Taskq '%s' creating (%s dispatch)\n",
SPLAT_TASKQ_TEST1_NAME,
prealloc ? "prealloc" : "dynamic");
if ((tq = taskq_create(SPLAT_TASKQ_TEST1_NAME, 1, maxclsyspri,
50, INT_MAX, TASKQ_PREPOPULATE)) == NULL) {
splat_vprint(file, SPLAT_TASKQ_TEST1_NAME,
"Taskq '%s' create failed\n",
SPLAT_TASKQ_TEST1_NAME);
return -EINVAL;
}
tq_arg.flag = 0;
tq_arg.id = 0;
tq_arg.file = file;
tq_arg.name = SPLAT_TASKQ_TEST1_NAME;
splat_vprint(file, SPLAT_TASKQ_TEST1_NAME,
"Taskq '%s' function '%s' dispatching\n",
tq_arg.name, sym2str(splat_taskq_test13_func));
if (prealloc) {
taskq_dispatch_ent(tq, splat_taskq_test13_func,
&tq_arg, TQ_SLEEP, &tqe);
id = tqe.tqent_id;
} else {
id = taskq_dispatch(tq, splat_taskq_test13_func,
&tq_arg, TQ_SLEEP);
}
if (id == 0) {
splat_vprint(file, SPLAT_TASKQ_TEST1_NAME,
"Taskq '%s' function '%s' dispatch failed\n",
tq_arg.name, sym2str(splat_taskq_test13_func));
taskq_destroy(tq);
return -EINVAL;
}
splat_vprint(file, SPLAT_TASKQ_TEST1_NAME, "Taskq '%s' waiting\n",
tq_arg.name);
taskq_wait(tq);
splat_vprint(file, SPLAT_TASKQ_TEST1_NAME, "Taskq '%s' destroying\n",
tq_arg.name);
taskq_destroy(tq);
return (tq_arg.flag) ? 0 : -EINVAL;
}
/*
* Simple algorithm for now, grab the global lock and let all
* the clients update themselves in parallel. There is a lot of
* room for improvement here. We could eliminate some scans of
* the DAG by incrementally scanning at lower levels of the DAG
* rather than having each client start its own scan from the root.
*/
void
mdeg_notify_clients(void)
{
md_t *md_new;
mdeg_clnt_t *clnt;
int idx;
int nclnt;
rw_enter(&mdeg.rwlock, RW_READER);
mutex_enter(&mdeg.lock);
/*
* Rotate the MDs
*/
if ((md_new = md_get_handle()) == NULL) {
cmn_err(CE_WARN, "unable to retrieve new MD");
goto done;
}
if (mdeg.md_prev) {
(void) md_fini_handle(mdeg.md_prev);
}
mdeg.md_prev = mdeg.md_curr;
mdeg.md_curr = md_new;
if (mdeg.nclnts == 0) {
MDEG_DBG("mdeg_notify_clients: no clients registered\n");
goto done;
}
/* dispatch the update notification to all clients */
for (idx = 0, nclnt = 0; idx < mdeg.maxclnts; idx++) {
clnt = &mdeg.tbl[idx];
if (!clnt->valid)
continue;
MDEG_DBG("notifying client 0x%lx (%d/%d)\n", clnt->hdl,
++nclnt, mdeg.nclnts);
(void) taskq_dispatch(mdeg.taskq, mdeg_notify_client,
(void *)clnt, TQ_SLEEP);
}
/*
* Wait for all mdeg_notify_client notifications to
* finish while we are still holding mdeg.rwlock.
*/
taskq_wait(mdeg.taskq);
done:
mutex_exit(&mdeg.lock);
rw_exit(&mdeg.rwlock);
}
/*
* Use the global system task queue with a single task, wait until task
* completes, ensure task ran properly.
*/
static int
splat_taskq_test3_impl(struct file *file, void *arg, boolean_t prealloc)
{
taskqid_t id;
splat_taskq_arg_t *tq_arg;
taskq_ent_t *tqe;
int error;
tq_arg = kmem_alloc(sizeof (splat_taskq_arg_t), KM_SLEEP);
tqe = kmem_alloc(sizeof (taskq_ent_t), KM_SLEEP);
taskq_init_ent(tqe);
tq_arg->flag = 0;
tq_arg->id = 0;
tq_arg->file = file;
tq_arg->name = SPLAT_TASKQ_TEST3_NAME;
splat_vprint(file, SPLAT_TASKQ_TEST3_NAME,
"Taskq '%s' function '%s' %s dispatch\n",
tq_arg->name, sym2str(splat_taskq_test13_func),
prealloc ? "prealloc" : "dynamic");
if (prealloc) {
taskq_dispatch_ent(system_taskq, splat_taskq_test13_func,
tq_arg, TQ_SLEEP, tqe);
id = tqe->tqent_id;
} else {
id = taskq_dispatch(system_taskq, splat_taskq_test13_func,
tq_arg, TQ_SLEEP);
}
if (id == 0) {
splat_vprint(file, SPLAT_TASKQ_TEST3_NAME,
"Taskq '%s' function '%s' dispatch failed\n",
tq_arg->name, sym2str(splat_taskq_test13_func));
kmem_free(tqe, sizeof (taskq_ent_t));
kmem_free(tq_arg, sizeof (splat_taskq_arg_t));
return -EINVAL;
}
splat_vprint(file, SPLAT_TASKQ_TEST3_NAME, "Taskq '%s' waiting\n",
tq_arg->name);
taskq_wait(system_taskq);
error = (tq_arg->flag) ? 0 : -EINVAL;
kmem_free(tqe, sizeof (taskq_ent_t));
kmem_free(tq_arg, sizeof (splat_taskq_arg_t));
return (error);
}
/*
* Use the global system task queue with a single task, wait until task
* completes, ensure task ran properly.
*/
static int
splat_taskq_test3_impl(struct file *file, void *arg, boolean_t prealloc)
{
taskqid_t id;
splat_taskq_arg_t tq_arg;
taskq_ent_t tqe;
taskq_init_ent(&tqe);
tq_arg.flag = 0;
tq_arg.id = 0;
tq_arg.file = file;
tq_arg.name = SPLAT_TASKQ_TEST3_NAME;
splat_vprint(file, SPLAT_TASKQ_TEST3_NAME,
"Taskq '%s' function '%s' %s dispatch\n",
tq_arg.name, sym2str(splat_taskq_test13_func),
prealloc ? "prealloc" : "dynamic");
if (prealloc) {
taskq_dispatch_ent(system_taskq, splat_taskq_test13_func,
&tq_arg, TQ_SLEEP, &tqe);
id = tqe.tqent_id;
} else {
id = taskq_dispatch(system_taskq, splat_taskq_test13_func,
&tq_arg, TQ_SLEEP);
}
if (id == 0) {
splat_vprint(file, SPLAT_TASKQ_TEST3_NAME,
"Taskq '%s' function '%s' dispatch failed\n",
tq_arg.name, sym2str(splat_taskq_test13_func));
return -EINVAL;
}
splat_vprint(file, SPLAT_TASKQ_TEST3_NAME, "Taskq '%s' waiting\n",
tq_arg.name);
taskq_wait(system_taskq);
return (tq_arg.flag) ? 0 : -EINVAL;
}
void
taskq_destroy(taskq_t *tq)
{
int t;
int nthreads = tq->tq_nthreads;
taskq_wait(tq);
mxlock(&tq->tq_lock);
tq->tq_flags &= ~TASKQ_ACTIVE;
condbcast(&tq->tq_dispatch_cv);
while (tq->tq_nthreads != 0)
condwait(&tq->tq_wait_cv, &tq->tq_lock);
tq->tq_minalloc = 0;
while (tq->tq_nalloc != 0) {
assert(tq->tq_freelist != NULL);
task_free(tq, task_alloc(tq, UMEM_NOFAIL));
}
mxunlock(&tq->tq_lock);
for (t = 0; t < nthreads; t++)
pthread_join(tq->tq_threadlist[t], NULL);
umem_free(tq->tq_threadlist, nthreads * sizeof (pthread_t));
rwdestroy(&tq->tq_threadlock);
mxdestroy(&tq->tq_lock);
conddestroy(&tq->tq_dispatch_cv);
conddestroy(&tq->tq_wait_cv);
conddestroy(&tq->tq_maxalloc_cv);
umem_free(tq, sizeof (taskq_t));
}
/*
* Teardown the zfs_sb_t.
*
* Note, if 'unmounting' if FALSE, we return with the 'z_teardown_lock'
* and 'z_teardown_inactive_lock' held.
*/
int
zfs_sb_teardown(zfs_sb_t *zsb, boolean_t unmounting)
{
znode_t *zp;
rrw_enter(&zsb->z_teardown_lock, RW_WRITER, FTAG);
if (!unmounting) {
/*
* We purge the parent filesystem's super block as the
* parent filesystem and all of its snapshots have their
* inode's super block set to the parent's filesystem's
* super block. Note, 'z_parent' is self referential
* for non-snapshots.
*/
shrink_dcache_sb(zsb->z_parent->z_sb);
}
/*
* If someone has not already unmounted this file system,
* drain the iput_taskq to ensure all active references to the
* zfs_sb_t have been handled only then can it be safely destroyed.
*/
if (zsb->z_os)
taskq_wait(dsl_pool_iput_taskq(dmu_objset_pool(zsb->z_os)));
/*
* Close the zil. NB: Can't close the zil while zfs_inactive
* threads are blocked as zil_close can call zfs_inactive.
*/
if (zsb->z_log) {
zil_close(zsb->z_log);
zsb->z_log = NULL;
}
rw_enter(&zsb->z_teardown_inactive_lock, RW_WRITER);
/*
* If we are not unmounting (ie: online recv) and someone already
* unmounted this file system while we were doing the switcheroo,
* or a reopen of z_os failed then just bail out now.
*/
if (!unmounting && (zsb->z_unmounted || zsb->z_os == NULL)) {
rw_exit(&zsb->z_teardown_inactive_lock);
rrw_exit(&zsb->z_teardown_lock, FTAG);
return (EIO);
}
/*
* At this point there are no VFS ops active, and any new VFS ops
* will fail with EIO since we have z_teardown_lock for writer (only
* relevant for forced unmount).
*
* Release all holds on dbufs.
*/
mutex_enter(&zsb->z_znodes_lock);
for (zp = list_head(&zsb->z_all_znodes); zp != NULL;
zp = list_next(&zsb->z_all_znodes, zp)) {
if (zp->z_sa_hdl) {
ASSERT(atomic_read(&ZTOI(zp)->i_count) > 0);
zfs_znode_dmu_fini(zp);
}
}
mutex_exit(&zsb->z_znodes_lock);
/*
* If we are unmounting, set the unmounted flag and let new VFS ops
* unblock. zfs_inactive will have the unmounted behavior, and all
* other VFS ops will fail with EIO.
*/
if (unmounting) {
zsb->z_unmounted = B_TRUE;
rrw_exit(&zsb->z_teardown_lock, FTAG);
rw_exit(&zsb->z_teardown_inactive_lock);
}
/*
* z_os will be NULL if there was an error in attempting to reopen
* zsb, so just return as the properties had already been
*
* unregistered and cached data had been evicted before.
*/
if (zsb->z_os == NULL)
return (0);
/*
* Unregister properties.
*/
zfs_unregister_callbacks(zsb);
/*
* Evict cached data
*/
if (dsl_dataset_is_dirty(dmu_objset_ds(zsb->z_os)) &&
!zfs_is_readonly(zsb))
txg_wait_synced(dmu_objset_pool(zsb->z_os), 0);
dmu_objset_evict_dbufs(zsb->z_os);
return (0);
}
static int
splat_taskq_test4_common(struct file *file, void *arg, int minalloc,
int maxalloc, int nr_tasks, boolean_t prealloc)
{
taskq_t *tq;
taskqid_t id;
splat_taskq_arg_t tq_arg;
taskq_ent_t *tqes;
int i, j, rc = 0;
tqes = kmalloc(sizeof(*tqes) * nr_tasks, GFP_KERNEL);
if (tqes == NULL)
return -ENOMEM;
splat_vprint(file, SPLAT_TASKQ_TEST4_NAME,
"Taskq '%s' creating (%s dispatch) (%d/%d/%d)\n",
SPLAT_TASKQ_TEST4_NAME,
prealloc ? "prealloc" : "dynamic",
minalloc, maxalloc, nr_tasks);
if ((tq = taskq_create(SPLAT_TASKQ_TEST4_NAME, 1, maxclsyspri,
minalloc, maxalloc, TASKQ_PREPOPULATE)) == NULL) {
splat_vprint(file, SPLAT_TASKQ_TEST4_NAME,
"Taskq '%s' create failed\n",
SPLAT_TASKQ_TEST4_NAME);
rc = -EINVAL;
goto out_free;
}
tq_arg.file = file;
tq_arg.name = SPLAT_TASKQ_TEST4_NAME;
for (i = 1; i <= nr_tasks; i *= 2) {
atomic_set(&tq_arg.count, 0);
splat_vprint(file, SPLAT_TASKQ_TEST4_NAME,
"Taskq '%s' function '%s' dispatched %d times\n",
tq_arg.name, sym2str(splat_taskq_test4_func), i);
for (j = 0; j < i; j++) {
taskq_init_ent(&tqes[j]);
if (prealloc) {
taskq_dispatch_ent(tq, splat_taskq_test4_func,
&tq_arg, TQ_SLEEP, &tqes[j]);
id = tqes[j].tqent_id;
} else {
id = taskq_dispatch(tq, splat_taskq_test4_func,
&tq_arg, TQ_SLEEP);
}
if (id == 0) {
splat_vprint(file, SPLAT_TASKQ_TEST4_NAME,
"Taskq '%s' function '%s' dispatch "
"%d failed\n", tq_arg.name,
sym2str(splat_taskq_test4_func), j);
rc = -EINVAL;
goto out;
}
}
splat_vprint(file, SPLAT_TASKQ_TEST4_NAME, "Taskq '%s' "
"waiting for %d dispatches\n", tq_arg.name, i);
taskq_wait(tq);
splat_vprint(file, SPLAT_TASKQ_TEST4_NAME, "Taskq '%s' "
"%d/%d dispatches finished\n", tq_arg.name,
atomic_read(&tq_arg.count), i);
if (atomic_read(&tq_arg.count) != i) {
rc = -ERANGE;
goto out;
}
}
out:
splat_vprint(file, SPLAT_TASKQ_TEST4_NAME, "Taskq '%s' destroying\n",
tq_arg.name);
taskq_destroy(tq);
out_free:
kfree(tqes);
return rc;
}
static int
splat_taskq_test2_impl(struct file *file, void *arg, boolean_t prealloc) {
taskq_t *tq[TEST2_TASKQS] = { NULL };
taskqid_t id;
splat_taskq_arg_t tq_args[TEST2_TASKQS];
taskq_ent_t *func1_tqes = NULL;
taskq_ent_t *func2_tqes = NULL;
int i, rc = 0;
func1_tqes = kmalloc(sizeof(*func1_tqes) * TEST2_TASKQS, GFP_KERNEL);
if (func1_tqes == NULL) {
rc = -ENOMEM;
goto out;
}
func2_tqes = kmalloc(sizeof(*func2_tqes) * TEST2_TASKQS, GFP_KERNEL);
if (func2_tqes == NULL) {
rc = -ENOMEM;
goto out;
}
for (i = 0; i < TEST2_TASKQS; i++) {
taskq_init_ent(&func1_tqes[i]);
taskq_init_ent(&func2_tqes[i]);
splat_vprint(file, SPLAT_TASKQ_TEST2_NAME,
"Taskq '%s/%d' creating (%s dispatch)\n",
SPLAT_TASKQ_TEST2_NAME, i,
prealloc ? "prealloc" : "dynamic");
if ((tq[i] = taskq_create(SPLAT_TASKQ_TEST2_NAME,
TEST2_THREADS_PER_TASKQ,
maxclsyspri, 50, INT_MAX,
TASKQ_PREPOPULATE)) == NULL) {
splat_vprint(file, SPLAT_TASKQ_TEST2_NAME,
"Taskq '%s/%d' create failed\n",
SPLAT_TASKQ_TEST2_NAME, i);
rc = -EINVAL;
break;
}
tq_args[i].flag = i;
tq_args[i].id = i;
tq_args[i].file = file;
tq_args[i].name = SPLAT_TASKQ_TEST2_NAME;
splat_vprint(file, SPLAT_TASKQ_TEST2_NAME,
"Taskq '%s/%d' function '%s' dispatching\n",
tq_args[i].name, tq_args[i].id,
sym2str(splat_taskq_test2_func1));
if (prealloc) {
taskq_dispatch_ent(tq[i], splat_taskq_test2_func1,
&tq_args[i], TQ_SLEEP, &func1_tqes[i]);
id = func1_tqes[i].tqent_id;
} else {
id = taskq_dispatch(tq[i], splat_taskq_test2_func1,
&tq_args[i], TQ_SLEEP);
}
if (id == 0) {
splat_vprint(file, SPLAT_TASKQ_TEST2_NAME,
"Taskq '%s/%d' function '%s' dispatch "
"failed\n", tq_args[i].name, tq_args[i].id,
sym2str(splat_taskq_test2_func1));
rc = -EINVAL;
break;
}
splat_vprint(file, SPLAT_TASKQ_TEST2_NAME,
"Taskq '%s/%d' function '%s' dispatching\n",
tq_args[i].name, tq_args[i].id,
sym2str(splat_taskq_test2_func2));
if (prealloc) {
taskq_dispatch_ent(tq[i], splat_taskq_test2_func2,
&tq_args[i], TQ_SLEEP, &func2_tqes[i]);
id = func2_tqes[i].tqent_id;
} else {
id = taskq_dispatch(tq[i], splat_taskq_test2_func2,
&tq_args[i], TQ_SLEEP);
}
if (id == 0) {
splat_vprint(file, SPLAT_TASKQ_TEST2_NAME, "Taskq "
"'%s/%d' function '%s' dispatch failed\n",
tq_args[i].name, tq_args[i].id,
sym2str(splat_taskq_test2_func2));
rc = -EINVAL;
break;
}
}
/* When rc is set we're effectively just doing cleanup here, so
* ignore new errors in that case. They just cause noise. */
for (i = 0; i < TEST2_TASKQS; i++) {
if (tq[i] != NULL) {
splat_vprint(file, SPLAT_TASKQ_TEST2_NAME,
"Taskq '%s/%d' waiting\n",
tq_args[i].name, tq_args[i].id);
taskq_wait(tq[i]);
splat_vprint(file, SPLAT_TASKQ_TEST2_NAME,
"Taskq '%s/%d; destroying\n",
tq_args[i].name, tq_args[i].id);
taskq_destroy(tq[i]);
if (!rc && tq_args[i].flag != ((i * 2) + 1)) {
splat_vprint(file, SPLAT_TASKQ_TEST2_NAME,
"Taskq '%s/%d' processed tasks "
"out of order; %d != %d\n",
tq_args[i].name, tq_args[i].id,
tq_args[i].flag, i * 2 + 1);
rc = -EINVAL;
} else {
splat_vprint(file, SPLAT_TASKQ_TEST2_NAME,
"Taskq '%s/%d' processed tasks "
"in the correct order; %d == %d\n",
tq_args[i].name, tq_args[i].id,
tq_args[i].flag, i * 2 + 1);
}
}
}
out:
if (func1_tqes)
kfree(func1_tqes);
if (func2_tqes)
kfree(func2_tqes);
return rc;
}
static int
splat_taskq_test8_common(struct file *file, void *arg, int minalloc,
int maxalloc)
{
taskq_t *tq;
taskqid_t id;
splat_taskq_arg_t tq_arg;
taskq_ent_t **tqes;
int i, j, rc = 0;
tqes = vmalloc(sizeof(*tqes) * TEST8_NUM_TASKS);
if (tqes == NULL)
return -ENOMEM;
memset(tqes, 0, sizeof(*tqes) * TEST8_NUM_TASKS);
splat_vprint(file, SPLAT_TASKQ_TEST8_NAME,
"Taskq '%s' creating (%d/%d/%d)\n",
SPLAT_TASKQ_TEST8_NAME,
minalloc, maxalloc, TEST8_NUM_TASKS);
if ((tq = taskq_create(SPLAT_TASKQ_TEST8_NAME, TEST8_THREADS_PER_TASKQ,
maxclsyspri, minalloc, maxalloc,
TASKQ_PREPOPULATE)) == NULL) {
splat_vprint(file, SPLAT_TASKQ_TEST8_NAME,
"Taskq '%s' create failed\n",
SPLAT_TASKQ_TEST8_NAME);
rc = -EINVAL;
goto out_free;
}
tq_arg.file = file;
tq_arg.name = SPLAT_TASKQ_TEST8_NAME;
atomic_set(&tq_arg.count, 0);
for (i = 0; i < TEST8_NUM_TASKS; i++) {
tqes[i] = kmalloc(sizeof(taskq_ent_t), GFP_KERNEL);
if (tqes[i] == NULL) {
rc = -ENOMEM;
goto out;
}
taskq_init_ent(tqes[i]);
taskq_dispatch_ent(tq, splat_taskq_test8_func,
&tq_arg, TQ_SLEEP, tqes[i]);
id = tqes[i]->tqent_id;
if (id == 0) {
splat_vprint(file, SPLAT_TASKQ_TEST8_NAME,
"Taskq '%s' function '%s' dispatch "
"%d failed\n", tq_arg.name,
sym2str(splat_taskq_test8_func), i);
rc = -EINVAL;
goto out;
}
}
splat_vprint(file, SPLAT_TASKQ_TEST8_NAME, "Taskq '%s' "
"waiting for %d dispatches\n", tq_arg.name,
TEST8_NUM_TASKS);
taskq_wait(tq);
splat_vprint(file, SPLAT_TASKQ_TEST8_NAME, "Taskq '%s' "
"%d/%d dispatches finished\n", tq_arg.name,
atomic_read(&tq_arg.count), TEST8_NUM_TASKS);
if (atomic_read(&tq_arg.count) != TEST8_NUM_TASKS)
rc = -ERANGE;
out:
splat_vprint(file, SPLAT_TASKQ_TEST8_NAME, "Taskq '%s' destroying\n",
tq_arg.name);
taskq_destroy(tq);
out_free:
for (j = 0; j < TEST8_NUM_TASKS && tqes[j] != NULL; j++)
kfree(tqes[j]);
vfree(tqes);
return rc;
}
static int
splat_mutex_test3(struct file *file, void *arg)
{
mutex_priv_t mp;
taskq_t *tq;
int rc = 0;
mp.mp_magic = SPLAT_MUTEX_TEST_MAGIC;
mp.mp_file = file;
mutex_init(&mp.mp_mtx, SPLAT_MUTEX_TEST_NAME, MUTEX_DEFAULT, NULL);
if ((tq = taskq_create(SPLAT_MUTEX_TEST_NAME, 1, defclsyspri,
50, INT_MAX, TASKQ_PREPOPULATE)) == NULL) {
splat_vprint(file, SPLAT_MUTEX_TEST3_NAME, "Taskq '%s' "
"create failed\n", SPLAT_MUTEX_TEST3_NAME);
return -EINVAL;
}
mutex_enter(&mp.mp_mtx);
/* Mutex should be owned by current */
if (!mutex_owned(&mp.mp_mtx)) {
splat_vprint(file, SPLAT_MUTEX_TEST3_NAME, "Unowned mutex "
"should be owned by pid %d\n", current->pid);
rc = -EINVAL;
goto out_exit;
}
if (taskq_dispatch(tq, splat_mutex_owned, &mp, TQ_SLEEP) == 0) {
splat_vprint(file, SPLAT_MUTEX_TEST3_NAME, "Failed to "
"dispatch function '%s' to taskq\n",
sym2str(splat_mutex_owned));
rc = -EINVAL;
goto out_exit;
}
taskq_wait(tq);
/* Mutex should not be owned which checked from a different thread */
if (mp.mp_rc || mp.mp_rc2) {
splat_vprint(file, SPLAT_MUTEX_TEST3_NAME, "Mutex owned by "
"pid %d not by taskq\n", current->pid);
rc = -EINVAL;
goto out_exit;
}
mutex_exit(&mp.mp_mtx);
/* Mutex should not be owned by current */
if (mutex_owned(&mp.mp_mtx)) {
splat_vprint(file, SPLAT_MUTEX_TEST3_NAME, "Mutex owned by "
"pid %d it should be unowned\b", current->pid);
rc = -EINVAL;
goto out;
}
if (taskq_dispatch(tq, splat_mutex_owned, &mp, TQ_SLEEP) == 0) {
splat_vprint(file, SPLAT_MUTEX_TEST3_NAME, "Failed to "
"dispatch function '%s' to taskq\n",
sym2str(splat_mutex_owned));
rc = -EINVAL;
goto out;
}
taskq_wait(tq);
/* Mutex should be owned by no one */
if (mp.mp_rc || mp.mp_rc2) {
splat_vprint(file, SPLAT_MUTEX_TEST3_NAME, "Mutex owned by "
"no one, %d/%d disagrees\n", mp.mp_rc, mp.mp_rc2);
rc = -EINVAL;
goto out;
}
splat_vprint(file, SPLAT_MUTEX_TEST3_NAME, "%s",
"Correct mutex_owned() behavior\n");
goto out;
out_exit:
mutex_exit(&mp.mp_mtx);
out:
mutex_destroy(&mp.mp_mtx);
taskq_destroy(tq);
return rc;
}
static int
splat_taskq_throughput(struct file *file, void *arg, const char *name,
int nthreads, int minalloc, int maxalloc, int flags, int tasks,
struct timespec *delta)
{
taskq_t *tq;
taskqid_t id;
splat_taskq_arg_t tq_arg;
taskq_ent_t **tqes;
atomic_t count;
struct timespec start, stop;
int i, j, rc = 0;
tqes = vmalloc(sizeof (*tqes) * tasks);
if (tqes == NULL)
return (-ENOMEM);
memset(tqes, 0, sizeof (*tqes) * tasks);
splat_vprint(file, name, "Taskq '%s' creating (%d/%d/%d/%d)\n",
name, nthreads, minalloc, maxalloc, tasks);
if ((tq = taskq_create(name, nthreads, maxclsyspri,
minalloc, maxalloc, flags)) == NULL) {
splat_vprint(file, name, "Taskq '%s' create failed\n", name);
rc = -EINVAL;
goto out_free;
}
tq_arg.file = file;
tq_arg.name = name;
tq_arg.count = &count;
atomic_set(tq_arg.count, 0);
getnstimeofday(&start);
for (i = 0; i < tasks; i++) {
tqes[i] = kmalloc(sizeof (taskq_ent_t), GFP_KERNEL);
if (tqes[i] == NULL) {
rc = -ENOMEM;
goto out;
}
taskq_init_ent(tqes[i]);
taskq_dispatch_ent(tq, splat_taskq_throughput_func,
&tq_arg, TQ_SLEEP, tqes[i]);
id = tqes[i]->tqent_id;
if (id == 0) {
splat_vprint(file, name, "Taskq '%s' function '%s' "
"dispatch %d failed\n", tq_arg.name,
sym2str(splat_taskq_throughput_func), i);
rc = -EINVAL;
goto out;
}
}
splat_vprint(file, name, "Taskq '%s' waiting for %d dispatches\n",
tq_arg.name, tasks);
taskq_wait(tq);
if (delta != NULL) {
getnstimeofday(&stop);
*delta = timespec_sub(stop, start);
}
splat_vprint(file, name, "Taskq '%s' %d/%d dispatches finished\n",
tq_arg.name, atomic_read(tq_arg.count), tasks);
if (atomic_read(tq_arg.count) != tasks)
rc = -ERANGE;
out:
splat_vprint(file, name, "Taskq '%s' destroying\n", tq_arg.name);
taskq_destroy(tq);
out_free:
for (j = 0; j < tasks && tqes[j] != NULL; j++)
kfree(tqes[j]);
vfree(tqes);
return (rc);
}
static int
splat_mutex_test2(struct file *file, void *arg)
{
mutex_priv_t *mp;
taskq_t *tq;
int i, rc = 0;
mp = (mutex_priv_t *)kmalloc(sizeof(*mp), GFP_KERNEL);
if (mp == NULL)
return -ENOMEM;
/* Create several threads allowing tasks to race with each other */
tq = taskq_create(SPLAT_MUTEX_TEST_TASKQ, num_online_cpus(),
defclsyspri, 50, INT_MAX, TASKQ_PREPOPULATE);
if (tq == NULL) {
rc = -ENOMEM;
goto out;
}
mp->mp_magic = SPLAT_MUTEX_TEST_MAGIC;
mp->mp_file = file;
mutex_init(&(mp->mp_mtx), SPLAT_MUTEX_TEST_NAME, MUTEX_DEFAULT, NULL);
mp->mp_rc = 0;
/*
* Schedule N work items to the work queue each of which enters the
* mutex, sleeps briefly, then exits the mutex. On a multiprocessor
* box these work items will be handled by all available CPUs. The
* task function checks to ensure the tracked shared variable is
* always only incremented by one. Additionally, the mutex itself
* is instrumented such that if any two processors are in the
* critical region at the same time the system will panic. If the
* mutex is implemented right this will never happy, that's a pass.
*/
for (i = 0; i < SPLAT_MUTEX_TEST_COUNT; i++) {
if (!taskq_dispatch(tq, splat_mutex_test2_func, mp, TQ_SLEEP)) {
splat_vprint(file, SPLAT_MUTEX_TEST2_NAME,
"Failed to queue task %d\n", i);
rc = -EINVAL;
}
}
taskq_wait(tq);
if (mp->mp_rc == SPLAT_MUTEX_TEST_COUNT) {
splat_vprint(file, SPLAT_MUTEX_TEST2_NAME, "%d racing threads "
"correctly entered/exited the mutex %d times\n",
num_online_cpus(), mp->mp_rc);
} else {
splat_vprint(file, SPLAT_MUTEX_TEST2_NAME, "%d racing threads "
"only processed %d/%d mutex work items\n",
num_online_cpus(),mp->mp_rc,SPLAT_MUTEX_TEST_COUNT);
rc = -EINVAL;
}
taskq_destroy(tq);
mutex_destroy(&(mp->mp_mtx));
out:
kfree(mp);
return rc;
}
static int
splat_taskq_test9(struct file *file, void *arg)
{
taskq_t *tq;
atomic_t count;
int i, rc = 0;
int minalloc = 1;
int maxalloc = 10;
int nr_tasks = 100;
splat_vprint(file, SPLAT_TASKQ_TEST9_NAME,
"Taskq '%s' creating (%s dispatch) (%d/%d/%d)\n",
SPLAT_TASKQ_TEST9_NAME, "delay", minalloc, maxalloc, nr_tasks);
if ((tq = taskq_create(SPLAT_TASKQ_TEST9_NAME, 3, maxclsyspri,
minalloc, maxalloc, TASKQ_PREPOPULATE)) == NULL) {
splat_vprint(file, SPLAT_TASKQ_TEST9_NAME,
"Taskq '%s' create failed\n", SPLAT_TASKQ_TEST9_NAME);
return -EINVAL;
}
atomic_set(&count, 0);
for (i = 1; i <= nr_tasks; i++) {
splat_taskq_arg_t *tq_arg;
taskqid_t id;
uint32_t rnd;
/* A random timeout in jiffies of at most 5 seconds */
get_random_bytes((void *)&rnd, 4);
rnd = rnd % (5 * HZ);
tq_arg = kmem_alloc(sizeof(splat_taskq_arg_t), KM_SLEEP);
tq_arg->file = file;
tq_arg->name = SPLAT_TASKQ_TEST9_NAME;
tq_arg->expire = ddi_get_lbolt() + rnd;
tq_arg->count = &count;
splat_vprint(file, SPLAT_TASKQ_TEST9_NAME,
"Taskq '%s' delay dispatch %u jiffies\n",
SPLAT_TASKQ_TEST9_NAME, rnd);
id = taskq_dispatch_delay(tq, splat_taskq_test9_func,
tq_arg, TQ_SLEEP, ddi_get_lbolt() + rnd);
if (id == 0) {
splat_vprint(file, SPLAT_TASKQ_TEST9_NAME,
"Taskq '%s' delay dispatch failed\n",
SPLAT_TASKQ_TEST9_NAME);
kmem_free(tq_arg, sizeof(splat_taskq_arg_t));
taskq_wait(tq);
rc = -EINVAL;
goto out;
}
}
splat_vprint(file, SPLAT_TASKQ_TEST9_NAME, "Taskq '%s' waiting for "
"%d delay dispatches\n", SPLAT_TASKQ_TEST9_NAME, nr_tasks);
taskq_wait(tq);
if (atomic_read(&count) != nr_tasks)
rc = -ERANGE;
splat_vprint(file, SPLAT_TASKQ_TEST9_NAME, "Taskq '%s' %d/%d delay "
"dispatches finished on time\n", SPLAT_TASKQ_TEST9_NAME,
atomic_read(&count), nr_tasks);
splat_vprint(file, SPLAT_TASKQ_TEST9_NAME, "Taskq '%s' destroying\n",
SPLAT_TASKQ_TEST9_NAME);
out:
taskq_destroy(tq);
return rc;
}
void
taskq_wait_id(taskq_t *tq, taskqid_t id)
{
taskq_wait(tq);
}
/*
* taskq_destroy().
*
* Assumes: by the time taskq_destroy is called no one will use this task queue
* in any way and no one will try to dispatch entries in it.
*/
void
taskq_destroy(taskq_t *tq)
{
taskq_bucket_t *b = tq->tq_buckets;
int bid = 0;
ASSERT(! (tq->tq_flags & TASKQ_CPR_SAFE));
/*
* Wait for any pending entries to complete.
*/
taskq_wait(tq);
mutex_enter(&tq->tq_lock);
ASSERT((tq->tq_task.tqent_next == &tq->tq_task) &&
(tq->tq_active == 0));
if ((tq->tq_nthreads > 1) && (tq->tq_threadlist != NULL))
kmem_free(tq->tq_threadlist, sizeof (kthread_t *) *
tq->tq_nthreads);
tq->tq_flags &= ~TASKQ_ACTIVE;
cv_broadcast(&tq->tq_dispatch_cv);
while (tq->tq_nthreads != 0)
cv_wait(&tq->tq_wait_cv, &tq->tq_lock);
tq->tq_minalloc = 0;
while (tq->tq_nalloc != 0)
taskq_ent_free(tq, taskq_ent_alloc(tq, TQ_SLEEP));
mutex_exit(&tq->tq_lock);
/*
* Mark each bucket as closing and wakeup all sleeping threads.
*/
for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) {
taskq_ent_t *tqe;
mutex_enter(&b->tqbucket_lock);
b->tqbucket_flags |= TQBUCKET_CLOSE;
/* Wakeup all sleeping threads */
for (tqe = b->tqbucket_freelist.tqent_next;
tqe != &b->tqbucket_freelist; tqe = tqe->tqent_next)
cv_signal(&tqe->tqent_cv);
ASSERT(b->tqbucket_nalloc == 0);
/*
* At this point we waited for all pending jobs to complete (in
* both the task queue and the bucket and no new jobs should
* arrive. Wait for all threads to die.
*/
while (b->tqbucket_nfree > 0)
cv_wait(&b->tqbucket_cv, &b->tqbucket_lock);
mutex_exit(&b->tqbucket_lock);
mutex_destroy(&b->tqbucket_lock);
cv_destroy(&b->tqbucket_cv);
}
if (tq->tq_buckets != NULL) {
ASSERT(tq->tq_flags & TASKQ_DYNAMIC);
kmem_free(tq->tq_buckets,
sizeof (taskq_bucket_t) * tq->tq_nbuckets);
/* Cleanup fields before returning tq to the cache */
tq->tq_buckets = NULL;
tq->tq_tcreates = 0;
tq->tq_tdeaths = 0;
} else {
ASSERT(!(tq->tq_flags & TASKQ_DYNAMIC));
}
tq->tq_totaltime = 0;
tq->tq_tasks = 0;
tq->tq_maxtasks = 0;
tq->tq_executed = 0;
kmem_cache_free(taskq_cache, tq);
}
void
taskq_wait_outstanding(taskq_t *tq, taskqid_t id)
{
taskq_wait(tq);
}
static int
splat_taskq_test10(struct file *file, void *arg)
{
taskq_t *tq;
splat_taskq_arg_t **tqas;
atomic_t count;
int i, j, rc = 0;
int minalloc = 1;
int maxalloc = 10;
int nr_tasks = 100;
int canceled = 0;
int completed = 0;
int blocked = 0;
clock_t start, cancel;
tqas = vmalloc(sizeof(*tqas) * nr_tasks);
if (tqas == NULL)
return -ENOMEM;
memset(tqas, 0, sizeof(*tqas) * nr_tasks);
splat_vprint(file, SPLAT_TASKQ_TEST10_NAME,
"Taskq '%s' creating (%s dispatch) (%d/%d/%d)\n",
SPLAT_TASKQ_TEST10_NAME, "delay", minalloc, maxalloc, nr_tasks);
if ((tq = taskq_create(SPLAT_TASKQ_TEST10_NAME, 3, maxclsyspri,
minalloc, maxalloc, TASKQ_PREPOPULATE)) == NULL) {
splat_vprint(file, SPLAT_TASKQ_TEST10_NAME,
"Taskq '%s' create failed\n", SPLAT_TASKQ_TEST10_NAME);
rc = -EINVAL;
goto out_free;
}
atomic_set(&count, 0);
for (i = 0; i < nr_tasks; i++) {
splat_taskq_arg_t *tq_arg;
uint32_t rnd;
/* A random timeout in jiffies of at most 5 seconds */
get_random_bytes((void *)&rnd, 4);
rnd = rnd % (5 * HZ);
tq_arg = kmem_alloc(sizeof(splat_taskq_arg_t), KM_SLEEP);
tq_arg->file = file;
tq_arg->name = SPLAT_TASKQ_TEST10_NAME;
tq_arg->count = &count;
tqas[i] = tq_arg;
/*
* Dispatch every 1/3 one immediately to mix it up, the cancel
* code is inherently racy and we want to try and provoke any
* subtle concurrently issues.
*/
if ((i % 3) == 0) {
tq_arg->expire = ddi_get_lbolt();
tq_arg->id = taskq_dispatch(tq, splat_taskq_test10_func,
tq_arg, TQ_SLEEP);
} else {
tq_arg->expire = ddi_get_lbolt() + rnd;
tq_arg->id = taskq_dispatch_delay(tq,
splat_taskq_test10_func,
tq_arg, TQ_SLEEP, ddi_get_lbolt() + rnd);
}
if (tq_arg->id == 0) {
splat_vprint(file, SPLAT_TASKQ_TEST10_NAME,
"Taskq '%s' dispatch failed\n",
SPLAT_TASKQ_TEST10_NAME);
kmem_free(tq_arg, sizeof(splat_taskq_arg_t));
taskq_wait(tq);
rc = -EINVAL;
goto out;
} else {
splat_vprint(file, SPLAT_TASKQ_TEST10_NAME,
"Taskq '%s' dispatch %lu in %lu jiffies\n",
SPLAT_TASKQ_TEST10_NAME, (unsigned long)tq_arg->id,
!(i % 3) ? 0 : tq_arg->expire - ddi_get_lbolt());
}
}
/*
* Start randomly canceling tasks for the duration of the test. We
* happen to know the valid task id's will be in the range 1..nr_tasks
* because the taskq is private and was just created. However, we
* have no idea of a particular task has already executed or not.
*/
splat_vprint(file, SPLAT_TASKQ_TEST10_NAME, "Taskq '%s' randomly "
"canceling task ids\n", SPLAT_TASKQ_TEST10_NAME);
start = ddi_get_lbolt();
i = 0;
while (ddi_time_before(ddi_get_lbolt(), start + 5 * HZ)) {
taskqid_t id;
uint32_t rnd;
i++;
cancel = ddi_get_lbolt();
get_random_bytes((void *)&rnd, 4);
id = 1 + (rnd % nr_tasks);
rc = taskq_cancel_id(tq, id);
/*
* Keep track of the results of the random cancels.
*/
if (rc == 0) {
canceled++;
} else if (rc == ENOENT) {
completed++;
} else if (rc == EBUSY) {
blocked++;
} else {
rc = -EINVAL;
break;
}
/*
* Verify we never get blocked to long in taskq_cancel_id().
* The worst case is 10ms if we happen to cancel the task
* which is currently executing. We allow a factor of 2x.
*/
if (ddi_get_lbolt() - cancel > HZ / 50) {
splat_vprint(file, SPLAT_TASKQ_TEST10_NAME,
"Taskq '%s' cancel for %lu took %lu\n",
SPLAT_TASKQ_TEST10_NAME, (unsigned long)id,
ddi_get_lbolt() - cancel);
rc = -ETIMEDOUT;
break;
}
get_random_bytes((void *)&rnd, 4);
msleep(1 + (rnd % 100));
rc = 0;
}
taskq_wait(tq);
/*
* Cross check the results of taskq_cancel_id() with the number of
* times the dispatched function actually ran successfully.
*/
if ((rc == 0) && (nr_tasks - canceled != atomic_read(&count)))
rc = -EDOM;
splat_vprint(file, SPLAT_TASKQ_TEST10_NAME, "Taskq '%s' %d attempts, "
"%d canceled, %d completed, %d blocked, %d/%d tasks run\n",
SPLAT_TASKQ_TEST10_NAME, i, canceled, completed, blocked,
atomic_read(&count), nr_tasks);
splat_vprint(file, SPLAT_TASKQ_TEST10_NAME, "Taskq '%s' destroying %d\n",
SPLAT_TASKQ_TEST10_NAME, rc);
out:
taskq_destroy(tq);
out_free:
for (j = 0; j < nr_tasks && tqas[j] != NULL; j++)
kmem_free(tqas[j], sizeof(splat_taskq_arg_t));
vfree(tqas);
return rc;
}
/*
* Given a list of directories to search, find all pools stored on disk. This
* includes partial pools which are not available to import. If no args are
* given (argc is 0), then the default directory (/dev/dsk) is searched.
* poolname or guid (but not both) are provided by the caller when trying
* to import a specific pool.
*/
static nvlist_t *
zpool_find_import_impl(libzfs_handle_t *hdl, importargs_t *iarg)
{
int i, dirs = iarg->paths;
struct dirent *dp;
char path[MAXPATHLEN];
char *end, **dir = iarg->path;
size_t pathleft;
nvlist_t *ret = NULL;
pool_list_t pools = { 0 };
pool_entry_t *pe, *penext;
vdev_entry_t *ve, *venext;
config_entry_t *ce, *cenext;
name_entry_t *ne, *nenext;
avl_tree_t slice_cache;
rdsk_node_t *slice;
void *cookie;
verify(iarg->poolname == NULL || iarg->guid == 0);
if (dirs == 0) {
#ifdef HAVE_LIBBLKID
/* Use libblkid to scan all device for their type */
if (zpool_find_import_blkid(hdl, &pools) == 0)
goto skip_scanning;
(void) zfs_error_fmt(hdl, EZFS_BADCACHE,
dgettext(TEXT_DOMAIN, "blkid failure falling back "
"to manual probing"));
#endif /* HAVE_LIBBLKID */
dir = zpool_default_import_path;
dirs = DEFAULT_IMPORT_PATH_SIZE;
}
/*
* Go through and read the label configuration information from every
* possible device, organizing the information according to pool GUID
* and toplevel GUID.
*/
for (i = 0; i < dirs; i++) {
taskq_t *t;
char rdsk[MAXPATHLEN];
int dfd;
boolean_t config_failed = B_FALSE;
DIR *dirp;
/* use realpath to normalize the path */
if (realpath(dir[i], path) == 0) {
/* it is safe to skip missing search paths */
if (errno == ENOENT)
continue;
zfs_error_aux(hdl, strerror(errno));
(void) zfs_error_fmt(hdl, EZFS_BADPATH,
dgettext(TEXT_DOMAIN, "cannot open '%s'"), dir[i]);
goto error;
}
end = &path[strlen(path)];
*end++ = '/';
*end = 0;
pathleft = &path[sizeof (path)] - end;
/*
* Using raw devices instead of block devices when we're
* reading the labels skips a bunch of slow operations during
* close(2) processing, so we replace /dev/dsk with /dev/rdsk.
*/
if (strcmp(path, ZFS_DISK_ROOTD) == 0)
(void) strlcpy(rdsk, ZFS_RDISK_ROOTD, sizeof (rdsk));
else
(void) strlcpy(rdsk, path, sizeof (rdsk));
if ((dfd = open(rdsk, O_RDONLY)) < 0 ||
(dirp = fdopendir(dfd)) == NULL) {
if (dfd >= 0)
(void) close(dfd);
zfs_error_aux(hdl, strerror(errno));
(void) zfs_error_fmt(hdl, EZFS_BADPATH,
dgettext(TEXT_DOMAIN, "cannot open '%s'"),
rdsk);
goto error;
}
avl_create(&slice_cache, slice_cache_compare,
sizeof (rdsk_node_t), offsetof(rdsk_node_t, rn_node));
/*
* This is not MT-safe, but we have no MT consumers of libzfs
*/
while ((dp = readdir(dirp)) != NULL) {
const char *name = dp->d_name;
if (name[0] == '.' &&
(name[1] == 0 || (name[1] == '.' && name[2] == 0)))
continue;
slice = zfs_alloc(hdl, sizeof (rdsk_node_t));
slice->rn_name = zfs_strdup(hdl, name);
slice->rn_avl = &slice_cache;
slice->rn_dfd = dfd;
slice->rn_hdl = hdl;
slice->rn_nozpool = B_FALSE;
avl_add(&slice_cache, slice);
}
/*
* create a thread pool to do all of this in parallel;
* rn_nozpool is not protected, so this is racy in that
* multiple tasks could decide that the same slice can
* not hold a zpool, which is benign. Also choose
* double the number of processors; we hold a lot of
* locks in the kernel, so going beyond this doesn't
* buy us much.
*/
t = taskq_create("z_import", 2 * max_ncpus, defclsyspri,
2 * max_ncpus, INT_MAX, TASKQ_PREPOPULATE);
for (slice = avl_first(&slice_cache); slice;
(slice = avl_walk(&slice_cache, slice,
AVL_AFTER)))
(void) taskq_dispatch(t, zpool_open_func, slice,
TQ_SLEEP);
taskq_wait(t);
taskq_destroy(t);
cookie = NULL;
while ((slice = avl_destroy_nodes(&slice_cache,
&cookie)) != NULL) {
if (slice->rn_config != NULL && !config_failed) {
nvlist_t *config = slice->rn_config;
boolean_t matched = B_TRUE;
if (iarg->poolname != NULL) {
char *pname;
matched = nvlist_lookup_string(config,
ZPOOL_CONFIG_POOL_NAME,
&pname) == 0 &&
strcmp(iarg->poolname, pname) == 0;
} else if (iarg->guid != 0) {
uint64_t this_guid;
matched = nvlist_lookup_uint64(config,
ZPOOL_CONFIG_POOL_GUID,
&this_guid) == 0 &&
iarg->guid == this_guid;
}
if (!matched) {
nvlist_free(config);
} else {
/*
* use the non-raw path for the config
*/
(void) strlcpy(end, slice->rn_name,
pathleft);
if (add_config(hdl, &pools, path, i+1,
slice->rn_num_labels, config) != 0)
config_failed = B_TRUE;
}
}
free(slice->rn_name);
free(slice);
}
avl_destroy(&slice_cache);
(void) closedir(dirp);
if (config_failed)
goto error;
}
#ifdef HAVE_LIBBLKID
skip_scanning:
#endif
ret = get_configs(hdl, &pools, iarg->can_be_active, iarg->policy);
error:
for (pe = pools.pools; pe != NULL; pe = penext) {
penext = pe->pe_next;
for (ve = pe->pe_vdevs; ve != NULL; ve = venext) {
venext = ve->ve_next;
for (ce = ve->ve_configs; ce != NULL; ce = cenext) {
cenext = ce->ce_next;
if (ce->ce_config)
nvlist_free(ce->ce_config);
free(ce);
}
free(ve);
}
free(pe);
}
for (ne = pools.names; ne != NULL; ne = nenext) {
nenext = ne->ne_next;
free(ne->ne_name);
free(ne);
}
return (ret);
}
/*
* Routine called by both ioctl and k-api. The consumer should
* bundle the parameters into a kcf_req_params_t structure. A bunch
* of macros are available in ops_impl.h for this bundling. They are:
*
* KCF_WRAP_DIGEST_OPS_PARAMS()
* KCF_WRAP_MAC_OPS_PARAMS()
* KCF_WRAP_ENCRYPT_OPS_PARAMS()
* KCF_WRAP_DECRYPT_OPS_PARAMS() ... etc.
*
* It is the caller's responsibility to free the ctx argument when
* appropriate. See the KCF_CONTEXT_COND_RELEASE macro for details.
*/
int
kcf_submit_request(kcf_provider_desc_t *pd, crypto_ctx_t *ctx,
crypto_call_req_t *crq, kcf_req_params_t *params, boolean_t cont)
{
int error = CRYPTO_SUCCESS;
kcf_areq_node_t *areq;
kcf_sreq_node_t *sreq;
kcf_context_t *kcf_ctx;
taskq_t *taskq = pd->pd_sched_info.ks_taskq;
kcf_ctx = ctx ? (kcf_context_t *)ctx->cc_framework_private : NULL;
/* Synchronous cases */
if (crq == NULL) {
switch (pd->pd_prov_type) {
case CRYPTO_SW_PROVIDER:
error = common_submit_request(pd, ctx, params,
KCF_RHNDL(KM_SLEEP));
break;
case CRYPTO_HW_PROVIDER:
/*
* Special case for CRYPTO_SYNCHRONOUS providers that
* never return a CRYPTO_QUEUED error. We skip any
* request allocation and call the SPI directly.
*/
if ((pd->pd_flags & CRYPTO_SYNCHRONOUS) &&
EMPTY_TASKQ(taskq)) {
KCF_PROV_IREFHOLD(pd);
if (pd->pd_state == KCF_PROV_READY) {
error = common_submit_request(pd, ctx,
params, KCF_RHNDL(KM_SLEEP));
KCF_PROV_IREFRELE(pd);
ASSERT(error != CRYPTO_QUEUED);
break;
}
KCF_PROV_IREFRELE(pd);
}
sreq = kmem_cache_alloc(kcf_sreq_cache, KM_SLEEP);
sreq->sn_state = REQ_ALLOCATED;
sreq->sn_rv = CRYPTO_FAILED;
sreq->sn_params = params;
/*
* Note that we do not need to hold the context
* for synchronous case as the context will never
* become invalid underneath us. We do not need to hold
* the provider here either as the caller has a hold.
*/
sreq->sn_context = kcf_ctx;
ASSERT(KCF_PROV_REFHELD(pd));
sreq->sn_provider = pd;
ASSERT(taskq != NULL);
/*
* Call the SPI directly if the taskq is empty and the
* provider is not busy, else dispatch to the taskq.
* Calling directly is fine as this is the synchronous
* case. This is unlike the asynchronous case where we
* must always dispatch to the taskq.
*/
if (EMPTY_TASKQ(taskq) &&
pd->pd_state == KCF_PROV_READY) {
process_req_hwp(sreq);
} else {
/*
* We can not tell from taskq_dispatch() return
* value if we exceeded maxalloc. Hence the
* check here. Since we are allowed to wait in
* the synchronous case, we wait for the taskq
* to become empty.
*/
if (taskq->tq_nalloc >= crypto_taskq_maxalloc) {
taskq_wait(taskq);
}
(void) taskq_dispatch(taskq, process_req_hwp,
sreq, TQ_SLEEP);
}
/*
* Wait for the notification to arrive,
* if the operation is not done yet.
* Bug# 4722589 will make the wait a cv_wait_sig().
*/
mutex_enter(&sreq->sn_lock);
while (sreq->sn_state < REQ_DONE)
cv_wait(&sreq->sn_cv, &sreq->sn_lock);
mutex_exit(&sreq->sn_lock);
error = sreq->sn_rv;
kmem_cache_free(kcf_sreq_cache, sreq);
break;
default:
error = CRYPTO_FAILED;
break;
}
} else { /* Asynchronous cases */
switch (pd->pd_prov_type) {
case CRYPTO_SW_PROVIDER:
if (!(crq->cr_flag & CRYPTO_ALWAYS_QUEUE)) {
/*
* This case has less overhead since there is
* no switching of context.
*/
error = common_submit_request(pd, ctx, params,
KCF_RHNDL(KM_NOSLEEP));
} else {
/*
* CRYPTO_ALWAYS_QUEUE is set. We need to
* queue the request and return.
*/
areq = kcf_areqnode_alloc(pd, kcf_ctx, crq,
params, cont);
if (areq == NULL)
error = CRYPTO_HOST_MEMORY;
else {
if (!(crq->cr_flag
& CRYPTO_SKIP_REQID)) {
/*
* Set the request handle. This handle
* is used for any crypto_cancel_req(9f)
* calls from the consumer. We have to
* do this before dispatching the
* request.
*/
crq->cr_reqid = kcf_reqid_insert(areq);
}
error = kcf_disp_sw_request(areq);
/*
* There is an error processing this
* request. Remove the handle and
* release the request structure.
*/
if (error != CRYPTO_QUEUED) {
if (!(crq->cr_flag
& CRYPTO_SKIP_REQID))
kcf_reqid_delete(areq);
KCF_AREQ_REFRELE(areq);
}
}
}
break;
case CRYPTO_HW_PROVIDER:
/*
* We need to queue the request and return.
*/
areq = kcf_areqnode_alloc(pd, kcf_ctx, crq, params,
cont);
if (areq == NULL) {
error = CRYPTO_HOST_MEMORY;
goto done;
}
ASSERT(taskq != NULL);
/*
* We can not tell from taskq_dispatch() return
* value if we exceeded maxalloc. Hence the check
* here.
*/
if (taskq->tq_nalloc >= crypto_taskq_maxalloc) {
error = CRYPTO_BUSY;
KCF_AREQ_REFRELE(areq);
goto done;
}
if (!(crq->cr_flag & CRYPTO_SKIP_REQID)) {
/*
* Set the request handle. This handle is used
* for any crypto_cancel_req(9f) calls from the
* consumer. We have to do this before dispatching
* the request.
*/
crq->cr_reqid = kcf_reqid_insert(areq);
}
if (taskq_dispatch(taskq,
process_req_hwp, areq, TQ_NOSLEEP) ==
TASKQID_INVALID) {
error = CRYPTO_HOST_MEMORY;
if (!(crq->cr_flag & CRYPTO_SKIP_REQID))
kcf_reqid_delete(areq);
KCF_AREQ_REFRELE(areq);
} else {
error = CRYPTO_QUEUED;
}
break;
default:
error = CRYPTO_FAILED;
break;
}
}
done:
return (error);
}
