uint64_t
dmu_object_alloc(objset_t *os, dmu_object_type_t ot, int blocksize,
dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx)
{
uint64_t object;
uint64_t L2_dnode_count = DNODES_PER_BLOCK <<
(DMU_META_DNODE(os)->dn_indblkshift - SPA_BLKPTRSHIFT);
dnode_t *dn = NULL;
int restarted = B_FALSE;
mutex_enter(&os->os_obj_lock);
for (;;) {
object = os->os_obj_next;
/*
* Each time we polish off an L2 bp worth of dnodes
* (2^13 objects), move to another L2 bp that's still
* reasonably sparse (at most 1/4 full). Look from the
* beginning once, but after that keep looking from here.
* If we can't find one, just keep going from here.
*
* Note that dmu_traverse depends on the behavior that we use
* multiple blocks of the dnode object before going back to
* reuse objects. Any change to this algorithm should preserve
* that property or find another solution to the issues
* described in traverse_visitbp.
*/
if (P2PHASE(object, L2_dnode_count) == 0) {
uint64_t offset = restarted ? object << DNODE_SHIFT : 0;
int error = dnode_next_offset(DMU_META_DNODE(os),
DNODE_FIND_HOLE,
&offset, 2, DNODES_PER_BLOCK >> 2, 0);
restarted = B_TRUE;
if (error == 0)
object = offset >> DNODE_SHIFT;
}
os->os_obj_next = ++object;
/*
* XXX We should check for an i/o error here and return
* up to our caller. Actually we should pre-read it in
* dmu_tx_assign(), but there is currently no mechanism
* to do so.
*/
(void) dnode_hold_impl(os, object, DNODE_MUST_BE_FREE,
FTAG, &dn);
if (dn)
break;
if (dmu_object_next(os, &object, B_TRUE, 0) == 0)
os->os_obj_next = object - 1;
}
/* called from dsl for meta-objset */
objset_t *
dmu_objset_create_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
dmu_objset_type_t type, dmu_tx_t *tx)
{
objset_t *os;
dnode_t *mdn;
ASSERT(dmu_tx_is_syncing(tx));
if (ds != NULL)
VERIFY0(dmu_objset_from_ds(ds, &os));
else
VERIFY0(dmu_objset_open_impl(spa, NULL, bp, &os));
mdn = DMU_META_DNODE(os);
dnode_allocate(mdn, DMU_OT_DNODE, 1 << DNODE_BLOCK_SHIFT,
DN_MAX_INDBLKSHIFT, DMU_OT_NONE, 0, tx);
/*
* We don't want to have to increase the meta-dnode's nlevels
* later, because then we could do it in quescing context while
* we are also accessing it in open context.
*
* This precaution is not necessary for the MOS (ds == NULL),
* because the MOS is only updated in syncing context.
* This is most fortunate: the MOS is the only objset that
* needs to be synced multiple times as spa_sync() iterates
* to convergence, so minimizing its dn_nlevels matters.
*/
if (ds != NULL) {
int levels = 1;
/*
* Determine the number of levels necessary for the meta-dnode
* to contain DN_MAX_OBJECT dnodes.
*/
while ((uint64_t)mdn->dn_nblkptr << (mdn->dn_datablkshift +
(levels - 1) * (mdn->dn_indblkshift - SPA_BLKPTRSHIFT)) <
DN_MAX_OBJECT * sizeof (dnode_phys_t))
levels++;
mdn->dn_next_nlevels[tx->tx_txg & TXG_MASK] =
mdn->dn_nlevels = levels;
}
ASSERT(type != DMU_OST_NONE);
ASSERT(type != DMU_OST_ANY);
ASSERT(type < DMU_OST_NUMTYPES);
os->os_phys->os_type = type;
if (dmu_objset_userused_enabled(os)) {
os->os_phys->os_flags |= OBJSET_FLAG_USERACCOUNTING_COMPLETE;
os->os_flags = os->os_phys->os_flags;
}
dsl_dataset_dirty(ds, tx);
return (os);
}
int
dmu_objset_evict_dbufs(objset_t *os)
{
dnode_t *dn;
mutex_enter(&os->os_lock);
/* process the mdn last, since the other dnodes have holds on it */
list_remove(&os->os_dnodes, DMU_META_DNODE(os));
list_insert_tail(&os->os_dnodes, DMU_META_DNODE(os));
/*
* Find the first dnode with holds. We have to do this dance
* because dnode_add_ref() only works if you already have a
* hold. If there are no holds then it has no dbufs so OK to
* skip.
*/
for (dn = list_head(&os->os_dnodes);
dn && !dnode_add_ref(dn, FTAG);
dn = list_next(&os->os_dnodes, dn))
continue;
while (dn) {
dnode_t *next_dn = dn;
do {
next_dn = list_next(&os->os_dnodes, next_dn);
} while (next_dn && !dnode_add_ref(next_dn, FTAG));
mutex_exit(&os->os_lock);
dnode_evict_dbufs(dn);
dnode_rele(dn, FTAG);
mutex_enter(&os->os_lock);
dn = next_dn;
}
dn = list_head(&os->os_dnodes);
mutex_exit(&os->os_lock);
return (dn != DMU_META_DNODE(os));
}
static uint64_t
dmu_object_alloc_impl(objset_t *os, dmu_object_type_t ot, int blocksize,
int indirect_blockshift, dmu_object_type_t bonustype, int bonuslen,
int dnodesize, dnode_t **allocated_dnode, void *tag, dmu_tx_t *tx)
{
uint64_t object;
uint64_t L1_dnode_count = DNODES_PER_BLOCK <<
(DMU_META_DNODE(os)->dn_indblkshift - SPA_BLKPTRSHIFT);
dnode_t *dn = NULL;
int dn_slots = dnodesize >> DNODE_SHIFT;
boolean_t restarted = B_FALSE;
uint64_t *cpuobj = NULL;
int dnodes_per_chunk = 1 << dmu_object_alloc_chunk_shift;
int error;
kpreempt_disable();
cpuobj = &os->os_obj_next_percpu[CPU_SEQID %
os->os_obj_next_percpu_len];
kpreempt_enable();
if (dn_slots == 0) {
dn_slots = DNODE_MIN_SLOTS;
} else {
ASSERT3S(dn_slots, >=, DNODE_MIN_SLOTS);
ASSERT3S(dn_slots, <=, DNODE_MAX_SLOTS);
}
/*
* The "chunk" of dnodes that is assigned to a CPU-specific
* allocator needs to be at least one block's worth, to avoid
* lock contention on the dbuf. It can be at most one L1 block's
* worth, so that the "rescan after polishing off a L1's worth"
* logic below will be sure to kick in.
*/
if (dnodes_per_chunk < DNODES_PER_BLOCK)
dnodes_per_chunk = DNODES_PER_BLOCK;
if (dnodes_per_chunk > L1_dnode_count)
dnodes_per_chunk = L1_dnode_count;
/*
* The caller requested the dnode be returned as a performance
* optimization in order to avoid releasing the hold only to
* immediately reacquire it. Since they caller is responsible
* for releasing the hold they must provide the tag.
*/
if (allocated_dnode != NULL) {
ASSERT3P(tag, !=, NULL);
} else {
static void
dmu_objset_dirty(objset_t *os)
{
// FIXME
#if 1
//arc_make_writable(os->os_phys_buf);
if (os->os_phys_buf->b_data != os->os_phys) {
os->os_phys = os->os_phys_buf->b_data;
DMU_META_DNODE(os)->dn_phys = &os->os_phys->os_meta_dnode;
if (DMU_USERUSED_DNODE(os)) {
DMU_USERUSED_DNODE(os)->dn_phys =
&os->os_phys->os_userused_dnode;
DMU_GROUPUSED_DNODE(os)->dn_phys =
&os->os_phys->os_groupused_dnode;
}
}
#endif
}
void
dmu_objset_evict_dbufs(objset_t *os)
{
dnode_t dn_marker;
dnode_t *dn;
mutex_enter(&os->os_lock);
dn = list_head(&os->os_dnodes);
while (dn != NULL) {
/*
* Skip dnodes without holds. We have to do this dance
* because dnode_add_ref() only works if there is already a
* hold. If the dnode has no holds, then it has no dbufs.
*/
if (dnode_add_ref(dn, FTAG)) {
list_insert_after(&os->os_dnodes, dn, &dn_marker);
mutex_exit(&os->os_lock);
dnode_evict_dbufs(dn);
dnode_rele(dn, FTAG);
mutex_enter(&os->os_lock);
dn = list_next(&os->os_dnodes, &dn_marker);
list_remove(&os->os_dnodes, &dn_marker);
} else {
dn = list_next(&os->os_dnodes, dn);
}
}
mutex_exit(&os->os_lock);
if (DMU_USERUSED_DNODE(os) != NULL) {
dnode_evict_dbufs(DMU_GROUPUSED_DNODE(os));
dnode_evict_dbufs(DMU_USERUSED_DNODE(os));
}
dnode_evict_dbufs(DMU_META_DNODE(os));
}
/* called from dsl for meta-objset */
objset_t *
dmu_objset_create_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
dmu_objset_type_t type, dsl_crypto_ctx_t *dcc, dmu_tx_t *tx)
{
objset_t *os;
dnode_t *mdn;
ASSERT(dmu_tx_is_syncing(tx));
if (ds != NULL)
VERIFY(0 == dmu_objset_from_ds(ds, &os));
else
VERIFY(0 == dmu_objset_open_impl(spa, NULL, bp, &os));
mdn = DMU_META_DNODE(os);
dnode_allocate(mdn, DMU_OT_DNODE, 1 << DNODE_BLOCK_SHIFT,
DN_MAX_INDBLKSHIFT, DMU_OT_NONE, 0, tx);
/*
* We don't want to have to increase the meta-dnode's nlevels
* later, because then we could do it in quescing context while
* we are also accessing it in open context.
*
* This precaution is not necessary for the MOS (ds == NULL),
* because the MOS is only updated in syncing context.
* This is most fortunate: the MOS is the only objset that
* needs to be synced multiple times as spa_sync() iterates
* to convergence, so minimizing its dn_nlevels matters.
*/
if (ds != NULL) {
int levels = 1;
/*
* Determine the number of levels necessary for the meta-dnode
* to contain DN_MAX_OBJECT dnodes.
*/
while ((uint64_t)mdn->dn_nblkptr << (mdn->dn_datablkshift +
(levels - 1) * (mdn->dn_indblkshift - SPA_BLKPTRSHIFT)) <
DN_MAX_OBJECT * sizeof (dnode_phys_t))
levels++;
mdn->dn_next_nlevels[tx->tx_txg & TXG_MASK] =
mdn->dn_nlevels = levels;
}
ASSERT(type != DMU_OST_NONE);
ASSERT(type != DMU_OST_ANY);
ASSERT(type < DMU_OST_NUMTYPES);
/*
* Note: although we should not be dirtying the objset outside of
* sync context, the os_type is used directly from the os_phys
* in dsl_scan so it may not be save to put os_type in the in-core
* objset_t and set it in the phys in sync context (as with os_flags)
*/
dmu_objset_dirty(os);
os->os_phys->os_type = type;
if (dmu_objset_userused_enabled(os)) {
os->os_phys->os_flags |= OBJSET_FLAG_USERACCOUNTING_COMPLETE;
// FIXME - removed in newer.
// os->os_flags = os->os_phys->os_flags;
}
if (dcc != NULL && dcc->dcc_crypt != ZIO_CRYPT_INHERIT) {
os->os_crypt = zio_crypt_select(dcc->dcc_crypt,
ZIO_CRYPT_ON_VALUE);
}
dsl_dataset_dirty(ds, tx);
return (os);
}
int
dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
objset_t **osp)
{
objset_t *os;
int i, err = 0;
ASSERT(ds == NULL || MUTEX_HELD(&ds->ds_opening_lock));
os = kmem_zalloc(sizeof (objset_t), KM_PUSHPAGE);
os->os_dsl_dataset = ds;
os->os_spa = spa;
os->os_rootbp = bp;
if (!BP_IS_HOLE(os->os_rootbp)) {
uint32_t aflags = ARC_WAIT;
zbookmark_t zb;
SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET,
ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
if (DMU_OS_IS_L2CACHEABLE(os))
aflags |= ARC_L2CACHE;
dprintf_bp(os->os_rootbp, "reading %s", "");
/*
* XXX when bprewrite scrub can change the bp,
* and this is called from dmu_objset_open_ds_os, the bp
* could change, and we'll need a lock.
*/
err = dsl_read_nolock(NULL, spa, os->os_rootbp,
arc_getbuf_func, &os->os_phys_buf,
ZIO_PRIORITY_SYNC_READ, ZIO_FLAG_CANFAIL, &aflags, &zb);
if (err) {
kmem_free(os, sizeof (objset_t));
/* convert checksum errors into IO errors */
if (err == ECKSUM)
err = EIO;
return (err);
}
/* Increase the blocksize if we are permitted. */
if (spa_version(spa) >= SPA_VERSION_USERSPACE &&
arc_buf_size(os->os_phys_buf) < sizeof (objset_phys_t)) {
arc_buf_t *buf = arc_buf_alloc(spa,
sizeof (objset_phys_t), &os->os_phys_buf,
ARC_BUFC_METADATA);
bzero(buf->b_data, sizeof (objset_phys_t));
bcopy(os->os_phys_buf->b_data, buf->b_data,
arc_buf_size(os->os_phys_buf));
(void) arc_buf_remove_ref(os->os_phys_buf,
&os->os_phys_buf);
os->os_phys_buf = buf;
}
os->os_phys = os->os_phys_buf->b_data;
os->os_flags = os->os_phys->os_flags;
} else {
int size = spa_version(spa) >= SPA_VERSION_USERSPACE ?
sizeof (objset_phys_t) : OBJSET_OLD_PHYS_SIZE;
os->os_phys_buf = arc_buf_alloc(spa, size,
&os->os_phys_buf, ARC_BUFC_METADATA);
os->os_phys = os->os_phys_buf->b_data;
bzero(os->os_phys, size);
}
/*
* Note: the changed_cb will be called once before the register
* func returns, thus changing the checksum/compression from the
* default (fletcher2/off). Snapshots don't need to know about
* checksum/compression/copies. But they do need to know about
* encryption so that clones from the snaphost inherit the
* same encryption property regardless of where in the namespace
* they get created.
*/
if (ds) {
err = dsl_prop_register(ds, "primarycache",
primary_cache_changed_cb, os);
if (err == 0)
err = dsl_prop_register(ds, "secondarycache",
secondary_cache_changed_cb, os);
if (err == 0)
err = dsl_prop_register(ds, "encryption",
crypt_changed_cb, os);
if (!dsl_dataset_is_snapshot(ds)) {
if (err == 0)
err = dsl_prop_register(ds, "checksum",
checksum_changed_cb, os);
if (err == 0)
err = dsl_prop_register(ds, "compression",
compression_changed_cb, os);
if (err == 0)
err = dsl_prop_register(ds, "copies",
copies_changed_cb, os);
if (err == 0)
err = dsl_prop_register(ds, "dedup",
dedup_changed_cb, os);
if (err == 0)
err = dsl_prop_register(ds, "logbias",
logbias_changed_cb, os);
if (err == 0)
err = dsl_prop_register(ds, "sync",
sync_changed_cb, os);
}
if (err) {
VERIFY(arc_buf_remove_ref(os->os_phys_buf,
&os->os_phys_buf) == 1);
kmem_free(os, sizeof (objset_t));
return (err);
}
} else if (ds == NULL) {
/*
* It's the meta-objset.
* Encryption is off for ZFS metadata but on for ZPL metadata
* and file/zvol contents.
*/
os->os_checksum = ZIO_CHECKSUM_FLETCHER_4;
os->os_compress = ZIO_COMPRESS_LZJB;
os->os_copies = spa_max_replication(spa);
os->os_dedup_checksum = ZIO_CHECKSUM_OFF;
os->os_dedup_verify = 0;
os->os_logbias = 0;
os->os_sync = 0;
os->os_primary_cache = ZFS_CACHE_ALL;
os->os_secondary_cache = ZFS_CACHE_ALL;
os->os_crypt = ZIO_CRYPT_OFF;
}
if (ds == NULL || !dsl_dataset_is_snapshot(ds))
os->os_zil_header = os->os_phys->os_zil_header;
os->os_zil = zil_alloc(os, &os->os_zil_header);
for (i = 0; i < TXG_SIZE; i++) {
list_create(&os->os_dirty_dnodes[i], sizeof (dnode_t),
offsetof(dnode_t, dn_dirty_link[i]));
list_create(&os->os_free_dnodes[i], sizeof (dnode_t),
offsetof(dnode_t, dn_dirty_link[i]));
}
list_create(&os->os_dnodes, sizeof (dnode_t),
offsetof(dnode_t, dn_link));
list_create(&os->os_downgraded_dbufs, sizeof (dmu_buf_impl_t),
offsetof(dmu_buf_impl_t, db_link));
mutex_init(&os->os_lock, NULL, MUTEX_DEFAULT, NULL);
mutex_init(&os->os_obj_lock, NULL, MUTEX_DEFAULT, NULL);
mutex_init(&os->os_user_ptr_lock, NULL, MUTEX_DEFAULT, NULL);
DMU_META_DNODE(os) = dnode_special_open(os,
&os->os_phys->os_meta_dnode, DMU_META_DNODE_OBJECT,
&os->os_meta_dnode);
if (arc_buf_size(os->os_phys_buf) >= sizeof (objset_phys_t)) {
DMU_USERUSED_DNODE(os) = dnode_special_open(os,
&os->os_phys->os_userused_dnode, DMU_USERUSED_OBJECT,
&os->os_userused_dnode);
DMU_GROUPUSED_DNODE(os) = dnode_special_open(os,
&os->os_phys->os_groupused_dnode, DMU_GROUPUSED_OBJECT,
&os->os_groupused_dnode);
}
/*
* We should be the only thread trying to do this because we
* have ds_opening_lock
*/
if (ds) {
mutex_enter(&ds->ds_lock);
ASSERT(ds->ds_objset == NULL);
ds->ds_objset = os;
mutex_exit(&ds->ds_lock);
}
*osp = os;
return (0);
}
/* called from dsl */
void
dmu_objset_sync(objset_t *os, zio_t *pio, dmu_tx_t *tx)
{
int txgoff;
zbookmark_t zb;
zio_prop_t zp;
zio_t *zio;
list_t *list;
list_t *newlist = NULL;
dbuf_dirty_record_t *dr;
dprintf_ds(os->os_dsl_dataset, "txg=%llu\n", tx->tx_txg);
ASSERT(dmu_tx_is_syncing(tx));
/* XXX the write_done callback should really give us the tx... */
os->os_synctx = tx;
if (os->os_dsl_dataset == NULL) {
/*
* This is the MOS. If we have upgraded,
* spa_max_replication() could change, so reset
* os_copies here.
*/
os->os_copies = spa_max_replication(os->os_spa);
}
/*
* Create the root block IO
*/
SET_BOOKMARK(&zb, os->os_dsl_dataset ?
os->os_dsl_dataset->ds_object : DMU_META_OBJSET,
ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
arc_release(os->os_phys_buf, &os->os_phys_buf);
dmu_write_policy(os, NULL, 0, 0, &zp);
zio = arc_write(pio, os->os_spa, tx->tx_txg,
os->os_rootbp, os->os_phys_buf, DMU_OS_IS_L2CACHEABLE(os), &zp,
dmu_objset_write_ready, dmu_objset_write_done, os,
ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb);
/*
* Sync special dnodes - the parent IO for the sync is the root block
*/
DMU_META_DNODE(os)->dn_zio = zio;
dnode_sync(DMU_META_DNODE(os), tx);
os->os_phys->os_flags = os->os_flags;
if (DMU_USERUSED_DNODE(os) &&
DMU_USERUSED_DNODE(os)->dn_type != DMU_OT_NONE) {
DMU_USERUSED_DNODE(os)->dn_zio = zio;
dnode_sync(DMU_USERUSED_DNODE(os), tx);
DMU_GROUPUSED_DNODE(os)->dn_zio = zio;
dnode_sync(DMU_GROUPUSED_DNODE(os), tx);
}
txgoff = tx->tx_txg & TXG_MASK;
if (dmu_objset_userused_enabled(os)) {
newlist = &os->os_synced_dnodes;
/*
* We must create the list here because it uses the
* dn_dirty_link[] of this txg.
*/
list_create(newlist, sizeof (dnode_t),
offsetof(dnode_t, dn_dirty_link[txgoff]));
}
dmu_objset_sync_dnodes(&os->os_free_dnodes[txgoff], newlist, tx);
dmu_objset_sync_dnodes(&os->os_dirty_dnodes[txgoff], newlist, tx);
list = &DMU_META_DNODE(os)->dn_dirty_records[txgoff];
while (dr = list_head(list)) {
ASSERT0(dr->dr_dbuf->db_level);
list_remove(list, dr);
if (dr->dr_zio)
zio_nowait(dr->dr_zio);
}
/*
* Free intent log blocks up to this tx.
*/
zil_sync(os->os_zil, tx);
os->os_phys->os_zil_header = os->os_zil_header;
zio_nowait(zio);
}
int
dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
objset_t **osp)
{
objset_t *os;
int i, err;
ASSERT(ds == NULL || MUTEX_HELD(&ds->ds_opening_lock));
os = kmem_zalloc(sizeof (objset_t), KM_SLEEP);
os->os_dsl_dataset = ds;
os->os_spa = spa;
os->os_rootbp = bp;
if (!BP_IS_HOLE(os->os_rootbp)) {
uint32_t aflags = ARC_WAIT;
zbookmark_t zb;
SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET,
ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
if (DMU_OS_IS_L2CACHEABLE(os))
aflags |= ARC_L2CACHE;
dprintf_bp(os->os_rootbp, "reading %s", "");
err = arc_read(NULL, spa, os->os_rootbp,
arc_getbuf_func, &os->os_phys_buf,
ZIO_PRIORITY_SYNC_READ, ZIO_FLAG_CANFAIL, &aflags, &zb);
if (err != 0) {
kmem_free(os, sizeof (objset_t));
/* convert checksum errors into IO errors */
if (err == ECKSUM)
err = SET_ERROR(EIO);
return (err);
}
/* Increase the blocksize if we are permitted. */
if (spa_version(spa) >= SPA_VERSION_USERSPACE &&
arc_buf_size(os->os_phys_buf) < sizeof (objset_phys_t)) {
arc_buf_t *buf = arc_buf_alloc(spa,
sizeof (objset_phys_t), &os->os_phys_buf,
ARC_BUFC_METADATA);
bzero(buf->b_data, sizeof (objset_phys_t));
bcopy(os->os_phys_buf->b_data, buf->b_data,
arc_buf_size(os->os_phys_buf));
(void) arc_buf_remove_ref(os->os_phys_buf,
&os->os_phys_buf);
os->os_phys_buf = buf;
}
os->os_phys = os->os_phys_buf->b_data;
os->os_flags = os->os_phys->os_flags;
} else {
int size = spa_version(spa) >= SPA_VERSION_USERSPACE ?
sizeof (objset_phys_t) : OBJSET_OLD_PHYS_SIZE;
os->os_phys_buf = arc_buf_alloc(spa, size,
&os->os_phys_buf, ARC_BUFC_METADATA);
os->os_phys = os->os_phys_buf->b_data;
bzero(os->os_phys, size);
}
/*
* Note: the changed_cb will be called once before the register
* func returns, thus changing the checksum/compression from the
* default (fletcher2/off). Snapshots don't need to know about
* checksum/compression/copies.
*/
if (ds) {
err = dsl_prop_register(ds,
zfs_prop_to_name(ZFS_PROP_PRIMARYCACHE),
primary_cache_changed_cb, os);
if (err == 0) {
err = dsl_prop_register(ds,
zfs_prop_to_name(ZFS_PROP_SECONDARYCACHE),
secondary_cache_changed_cb, os);
}
if (!dsl_dataset_is_snapshot(ds)) {
if (err == 0) {
err = dsl_prop_register(ds,
zfs_prop_to_name(ZFS_PROP_CHECKSUM),
checksum_changed_cb, os);
}
if (err == 0) {
err = dsl_prop_register(ds,
zfs_prop_to_name(ZFS_PROP_COMPRESSION),
compression_changed_cb, os);
}
if (err == 0) {
err = dsl_prop_register(ds,
zfs_prop_to_name(ZFS_PROP_COPIES),
copies_changed_cb, os);
}
if (err == 0) {
err = dsl_prop_register(ds,
zfs_prop_to_name(ZFS_PROP_DEDUP),
dedup_changed_cb, os);
}
if (err == 0) {
err = dsl_prop_register(ds,
zfs_prop_to_name(ZFS_PROP_LOGBIAS),
logbias_changed_cb, os);
}
if (err == 0) {
err = dsl_prop_register(ds,
zfs_prop_to_name(ZFS_PROP_SYNC),
sync_changed_cb, os);
}
}
if (err != 0) {
VERIFY(arc_buf_remove_ref(os->os_phys_buf,
&os->os_phys_buf));
kmem_free(os, sizeof (objset_t));
return (err);
}
} else if (ds == NULL) {
/* It's the meta-objset. */
os->os_checksum = ZIO_CHECKSUM_FLETCHER_4;
os->os_compress = ZIO_COMPRESS_LZJB;
os->os_copies = spa_max_replication(spa);
os->os_dedup_checksum = ZIO_CHECKSUM_OFF;
os->os_dedup_verify = 0;
os->os_logbias = 0;
os->os_sync = 0;
os->os_primary_cache = ZFS_CACHE_ALL;
os->os_secondary_cache = ZFS_CACHE_ALL;
}
if (ds == NULL || !dsl_dataset_is_snapshot(ds))
os->os_zil_header = os->os_phys->os_zil_header;
os->os_zil = zil_alloc(os, &os->os_zil_header);
for (i = 0; i < TXG_SIZE; i++) {
list_create(&os->os_dirty_dnodes[i], sizeof (dnode_t),
offsetof(dnode_t, dn_dirty_link[i]));
list_create(&os->os_free_dnodes[i], sizeof (dnode_t),
offsetof(dnode_t, dn_dirty_link[i]));
}
list_create(&os->os_dnodes, sizeof (dnode_t),
offsetof(dnode_t, dn_link));
list_create(&os->os_downgraded_dbufs, sizeof (dmu_buf_impl_t),
offsetof(dmu_buf_impl_t, db_link));
mutex_init(&os->os_lock, NULL, MUTEX_DEFAULT, NULL);
mutex_init(&os->os_obj_lock, NULL, MUTEX_DEFAULT, NULL);
mutex_init(&os->os_user_ptr_lock, NULL, MUTEX_DEFAULT, NULL);
DMU_META_DNODE(os) = dnode_special_open(os,
&os->os_phys->os_meta_dnode, DMU_META_DNODE_OBJECT,
&os->os_meta_dnode);
if (arc_buf_size(os->os_phys_buf) >= sizeof (objset_phys_t)) {
DMU_USERUSED_DNODE(os) = dnode_special_open(os,
&os->os_phys->os_userused_dnode, DMU_USERUSED_OBJECT,
&os->os_userused_dnode);
DMU_GROUPUSED_DNODE(os) = dnode_special_open(os,
&os->os_phys->os_groupused_dnode, DMU_GROUPUSED_OBJECT,
&os->os_groupused_dnode);
}
/*
* We should be the only thread trying to do this because we
* have ds_opening_lock
*/
if (ds) {
mutex_enter(&ds->ds_lock);
ASSERT(ds->ds_objset == NULL);
ds->ds_objset = os;
mutex_exit(&ds->ds_lock);
}
*osp = os;
return (0);
}
/*
* Calculate the real range based on the type, level, and range given.
*/
static int
calculate_range(const char *dataset, err_type_t type, int level, char *range,
zinject_record_t *record)
{
objset_t *os = NULL;
dnode_t *dn = NULL;
int err;
int ret = -1;
/*
* Determine the numeric range from the string.
*/
if (range == NULL) {
/*
* If range is unspecified, set the range to [0,-1], which
* indicates that the whole object should be treated as an
* error.
*/
record->zi_start = 0;
record->zi_end = -1ULL;
} else {
char *end;
/* XXX add support for suffixes */
record->zi_start = strtoull(range, &end, 10);
if (*end == '\0')
record->zi_end = record->zi_start + 1;
else if (*end == ',')
record->zi_end = strtoull(end + 1, &end, 10);
if (*end != '\0') {
(void) fprintf(stderr, "invalid range '%s': must be "
"a numeric range of the form 'start[,end]'\n",
range);
goto out;
}
}
switch (type) {
case TYPE_DATA:
break;
case TYPE_DNODE:
/*
* If this is a request to inject faults into the dnode, then we
* must translate the current (objset,object) pair into an
* offset within the metadnode for the objset. Specifying any
* kind of range with type 'dnode' is illegal.
*/
if (range != NULL) {
(void) fprintf(stderr, "range cannot be specified when "
"type is 'dnode'\n");
goto out;
}
record->zi_start = record->zi_object * sizeof (dnode_phys_t);
record->zi_end = record->zi_start + sizeof (dnode_phys_t);
record->zi_object = 0;
break;
}
/*
* Get the dnode associated with object, so we can calculate the block
* size.
*/
if ((err = dmu_objset_own(dataset, DMU_OST_ANY,
B_TRUE, FTAG, &os)) != 0) {
(void) fprintf(stderr, "cannot open dataset '%s': %s\n",
dataset, strerror(err));
goto out;
}
if (record->zi_object == 0) {
dn = DMU_META_DNODE(os);
} else {
err = dnode_hold(os, record->zi_object, FTAG, &dn);
if (err != 0) {
(void) fprintf(stderr, "failed to hold dnode "
"for object %llu\n",
(u_longlong_t)record->zi_object);
goto out;
}
}
ziprintf("data shift: %d\n", (int)dn->dn_datablkshift);
ziprintf(" ind shift: %d\n", (int)dn->dn_indblkshift);
/*
* Translate range into block IDs.
*/
if (record->zi_start != 0 || record->zi_end != -1ULL) {
record->zi_start >>= dn->dn_datablkshift;
record->zi_end >>= dn->dn_datablkshift;
}
uint64_t
dmu_object_alloc_dnsize(objset_t *os, dmu_object_type_t ot, int blocksize,
dmu_object_type_t bonustype, int bonuslen, int dnodesize, dmu_tx_t *tx)
{
uint64_t object;
uint64_t L1_dnode_count = DNODES_PER_BLOCK <<
(DMU_META_DNODE(os)->dn_indblkshift - SPA_BLKPTRSHIFT);
dnode_t *dn = NULL;
int dn_slots = dnodesize >> DNODE_SHIFT;
boolean_t restarted = B_FALSE;
uint64_t *cpuobj = NULL;
int dnodes_per_chunk = 1 << dmu_object_alloc_chunk_shift;
kpreempt_disable();
cpuobj = &os->os_obj_next_percpu[CPU_SEQID %
os->os_obj_next_percpu_len];
kpreempt_enable();
if (dn_slots == 0) {
dn_slots = DNODE_MIN_SLOTS;
} else {
ASSERT3S(dn_slots, >=, DNODE_MIN_SLOTS);
ASSERT3S(dn_slots, <=, DNODE_MAX_SLOTS);
}
/*
* The "chunk" of dnodes that is assigned to a CPU-specific
* allocator needs to be at least one block's worth, to avoid
* lock contention on the dbuf. It can be at most one L1 block's
* worth, so that the "rescan after polishing off a L1's worth"
* logic below will be sure to kick in.
*/
if (dnodes_per_chunk < DNODES_PER_BLOCK)
dnodes_per_chunk = DNODES_PER_BLOCK;
if (dnodes_per_chunk > L1_dnode_count)
dnodes_per_chunk = L1_dnode_count;
object = *cpuobj;
for (;;) {
/*
* If we finished a chunk of dnodes, get a new one from
* the global allocator.
*/
if (P2PHASE(object, dnodes_per_chunk) == 0) {
mutex_enter(&os->os_obj_lock);
ASSERT0(P2PHASE(os->os_obj_next_chunk,
dnodes_per_chunk));
object = os->os_obj_next_chunk;
/*
* Each time we polish off a L1 bp worth of dnodes
* (2^12 objects), move to another L1 bp that's
* still reasonably sparse (at most 1/4 full). Look
* from the beginning at most once per txg. If we
* still can't allocate from that L1 block, search
* for an empty L0 block, which will quickly skip
* to the end of the metadnode if no nearby L0
* blocks are empty. This fallback avoids a
* pathology where full dnode blocks containing
* large dnodes appear sparse because they have a
* low blk_fill, leading to many failed allocation
* attempts. In the long term a better mechanism to
* search for sparse metadnode regions, such as
* spacemaps, could be implemented.
*
* os_scan_dnodes is set during txg sync if enough
* objects have been freed since the previous
* rescan to justify backfilling again.
*
* Note that dmu_traverse depends on the behavior
* that we use multiple blocks of the dnode object
* before going back to reuse objects. Any change
* to this algorithm should preserve that property
* or find another solution to the issues described
* in traverse_visitbp.
*/
if (P2PHASE(object, L1_dnode_count) == 0) {
uint64_t offset;
uint64_t blkfill;
int minlvl;
int error;
if (os->os_rescan_dnodes) {
offset = 0;
os->os_rescan_dnodes = B_FALSE;
} else {
offset = object << DNODE_SHIFT;
}
blkfill = restarted ? 1 : DNODES_PER_BLOCK >> 2;
minlvl = restarted ? 1 : 2;
restarted = B_TRUE;
error = dnode_next_offset(DMU_META_DNODE(os),
DNODE_FIND_HOLE, &offset, minlvl,
blkfill, 0);
if (error == 0) {
object = offset >> DNODE_SHIFT;
}
}
uint64_t
dmu_object_alloc_dnsize(objset_t *os, dmu_object_type_t ot, int blocksize,
dmu_object_type_t bonustype, int bonuslen, int dnodesize, dmu_tx_t *tx)
{
uint64_t object;
uint64_t L1_dnode_count = DNODES_PER_BLOCK <<
(DMU_META_DNODE(os)->dn_indblkshift - SPA_BLKPTRSHIFT);
dnode_t *dn = NULL;
int dn_slots = dnodesize >> DNODE_SHIFT;
boolean_t restarted = B_FALSE;
if (dn_slots == 0) {
dn_slots = DNODE_MIN_SLOTS;
} else {
ASSERT3S(dn_slots, >=, DNODE_MIN_SLOTS);
ASSERT3S(dn_slots, <=, DNODE_MAX_SLOTS);
}
mutex_enter(&os->os_obj_lock);
for (;;) {
object = os->os_obj_next;
/*
* Each time we polish off a L1 bp worth of dnodes (2^12
* objects), move to another L1 bp that's still
* reasonably sparse (at most 1/4 full). Look from the
* beginning at most once per txg. If we still can't
* allocate from that L1 block, search for an empty L0
* block, which will quickly skip to the end of the
* metadnode if the no nearby L0 blocks are empty. This
* fallback avoids a pathology where full dnode blocks
* containing large dnodes appear sparse because they
* have a low blk_fill, leading to many failed
* allocation attempts. In the long term a better
* mechanism to search for sparse metadnode regions,
* such as spacemaps, could be implemented.
*
* os_scan_dnodes is set during txg sync if enough objects
* have been freed since the previous rescan to justify
* backfilling again.
*
* Note that dmu_traverse depends on the behavior that we use
* multiple blocks of the dnode object before going back to
* reuse objects. Any change to this algorithm should preserve
* that property or find another solution to the issues
* described in traverse_visitbp.
*/
if (P2PHASE(object, L1_dnode_count) == 0) {
uint64_t offset;
uint64_t blkfill;
int minlvl;
int error;
if (os->os_rescan_dnodes) {
offset = 0;
os->os_rescan_dnodes = B_FALSE;
} else {
offset = object << DNODE_SHIFT;
}
blkfill = restarted ? 1 : DNODES_PER_BLOCK >> 2;
minlvl = restarted ? 1 : 2;
restarted = B_TRUE;
error = dnode_next_offset(DMU_META_DNODE(os),
DNODE_FIND_HOLE, &offset, minlvl, blkfill, 0);
if (error == 0)
object = offset >> DNODE_SHIFT;
}
os->os_obj_next = object + dn_slots;
/*
* XXX We should check for an i/o error here and return
* up to our caller. Actually we should pre-read it in
* dmu_tx_assign(), but there is currently no mechanism
* to do so.
*/
(void) dnode_hold_impl(os, object, DNODE_MUST_BE_FREE, dn_slots,
FTAG, &dn);
if (dn)
break;
if (dmu_object_next(os, &object, B_TRUE, 0) == 0)
os->os_obj_next = object;
else
/*
* Skip to next known valid starting point for a dnode.
*/
os->os_obj_next = P2ROUNDUP(object + 1,
DNODES_PER_BLOCK);
}
