*/
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
*/
#include <sys/dmu.h>
#endif
const dmu_object_type_info_t dmu_ot[DMU_OT_NUMTYPES] = {
- { byteswap_uint8_array, TRUE, "unallocated" },
- { zap_byteswap, TRUE, "object directory" },
- { byteswap_uint64_array, TRUE, "object array" },
- { byteswap_uint8_array, TRUE, "packed nvlist" },
- { byteswap_uint64_array, TRUE, "packed nvlist size" },
- { byteswap_uint64_array, TRUE, "bpobj" },
- { byteswap_uint64_array, TRUE, "bpobj header" },
- { byteswap_uint64_array, TRUE, "SPA space map header" },
- { byteswap_uint64_array, TRUE, "SPA space map" },
- { byteswap_uint64_array, TRUE, "ZIL intent log" },
- { dnode_buf_byteswap, TRUE, "DMU dnode" },
- { dmu_objset_byteswap, TRUE, "DMU objset" },
- { byteswap_uint64_array, TRUE, "DSL directory" },
- { zap_byteswap, TRUE, "DSL directory child map"},
- { zap_byteswap, TRUE, "DSL dataset snap map" },
- { zap_byteswap, TRUE, "DSL props" },
- { byteswap_uint64_array, TRUE, "DSL dataset" },
- { zfs_znode_byteswap, TRUE, "ZFS znode" },
- { zfs_oldacl_byteswap, TRUE, "ZFS V0 ACL" },
- { byteswap_uint8_array, FALSE, "ZFS plain file" },
- { zap_byteswap, TRUE, "ZFS directory" },
- { zap_byteswap, TRUE, "ZFS master node" },
- { zap_byteswap, TRUE, "ZFS delete queue" },
- { byteswap_uint8_array, FALSE, "zvol object" },
- { zap_byteswap, TRUE, "zvol prop" },
- { byteswap_uint8_array, FALSE, "other uint8[]" },
- { byteswap_uint64_array, FALSE, "other uint64[]" },
- { zap_byteswap, TRUE, "other ZAP" },
- { zap_byteswap, TRUE, "persistent error log" },
- { byteswap_uint8_array, TRUE, "SPA history" },
- { byteswap_uint64_array, TRUE, "SPA history offsets" },
- { zap_byteswap, TRUE, "Pool properties" },
- { zap_byteswap, TRUE, "DSL permissions" },
- { zfs_acl_byteswap, TRUE, "ZFS ACL" },
- { byteswap_uint8_array, TRUE, "ZFS SYSACL" },
- { byteswap_uint8_array, TRUE, "FUID table" },
- { byteswap_uint64_array, TRUE, "FUID table size" },
- { zap_byteswap, TRUE, "DSL dataset next clones"},
- { zap_byteswap, TRUE, "scan work queue" },
- { zap_byteswap, TRUE, "ZFS user/group used" },
- { zap_byteswap, TRUE, "ZFS user/group quota" },
- { zap_byteswap, TRUE, "snapshot refcount tags"},
- { zap_byteswap, TRUE, "DDT ZAP algorithm" },
- { zap_byteswap, TRUE, "DDT statistics" },
- { byteswap_uint8_array, TRUE, "System attributes" },
- { zap_byteswap, TRUE, "SA master node" },
- { zap_byteswap, TRUE, "SA attr registration" },
- { zap_byteswap, TRUE, "SA attr layouts" },
- { zap_byteswap, TRUE, "scan translations" },
- { byteswap_uint8_array, FALSE, "deduplicated block" },
- { zap_byteswap, TRUE, "DSL deadlist map" },
- { byteswap_uint64_array, TRUE, "DSL deadlist map hdr" },
- { zap_byteswap, TRUE, "DSL dir clones" },
- { byteswap_uint64_array, TRUE, "bpobj subobj" },
+ { DMU_BSWAP_UINT8, TRUE, "unallocated" },
+ { DMU_BSWAP_ZAP, TRUE, "object directory" },
+ { DMU_BSWAP_UINT64, TRUE, "object array" },
+ { DMU_BSWAP_UINT8, TRUE, "packed nvlist" },
+ { DMU_BSWAP_UINT64, TRUE, "packed nvlist size" },
+ { DMU_BSWAP_UINT64, TRUE, "bpobj" },
+ { DMU_BSWAP_UINT64, TRUE, "bpobj header" },
+ { DMU_BSWAP_UINT64, TRUE, "SPA space map header" },
+ { DMU_BSWAP_UINT64, TRUE, "SPA space map" },
+ { DMU_BSWAP_UINT64, TRUE, "ZIL intent log" },
+ { DMU_BSWAP_DNODE, TRUE, "DMU dnode" },
+ { DMU_BSWAP_OBJSET, TRUE, "DMU objset" },
+ { DMU_BSWAP_UINT64, TRUE, "DSL directory" },
+ { DMU_BSWAP_ZAP, TRUE, "DSL directory child map"},
+ { DMU_BSWAP_ZAP, TRUE, "DSL dataset snap map" },
+ { DMU_BSWAP_ZAP, TRUE, "DSL props" },
+ { DMU_BSWAP_UINT64, TRUE, "DSL dataset" },
+ { DMU_BSWAP_ZNODE, TRUE, "ZFS znode" },
+ { DMU_BSWAP_OLDACL, TRUE, "ZFS V0 ACL" },
+ { DMU_BSWAP_UINT8, FALSE, "ZFS plain file" },
+ { DMU_BSWAP_ZAP, TRUE, "ZFS directory" },
+ { DMU_BSWAP_ZAP, TRUE, "ZFS master node" },
+ { DMU_BSWAP_ZAP, TRUE, "ZFS delete queue" },
+ { DMU_BSWAP_UINT8, FALSE, "zvol object" },
+ { DMU_BSWAP_ZAP, TRUE, "zvol prop" },
+ { DMU_BSWAP_UINT8, FALSE, "other uint8[]" },
+ { DMU_BSWAP_UINT64, FALSE, "other uint64[]" },
+ { DMU_BSWAP_ZAP, TRUE, "other ZAP" },
+ { DMU_BSWAP_ZAP, TRUE, "persistent error log" },
+ { DMU_BSWAP_UINT8, TRUE, "SPA history" },
+ { DMU_BSWAP_UINT64, TRUE, "SPA history offsets" },
+ { DMU_BSWAP_ZAP, TRUE, "Pool properties" },
+ { DMU_BSWAP_ZAP, TRUE, "DSL permissions" },
+ { DMU_BSWAP_ACL, TRUE, "ZFS ACL" },
+ { DMU_BSWAP_UINT8, TRUE, "ZFS SYSACL" },
+ { DMU_BSWAP_UINT8, TRUE, "FUID table" },
+ { DMU_BSWAP_UINT64, TRUE, "FUID table size" },
+ { DMU_BSWAP_ZAP, TRUE, "DSL dataset next clones"},
+ { DMU_BSWAP_ZAP, TRUE, "scan work queue" },
+ { DMU_BSWAP_ZAP, TRUE, "ZFS user/group used" },
+ { DMU_BSWAP_ZAP, TRUE, "ZFS user/group quota" },
+ { DMU_BSWAP_ZAP, TRUE, "snapshot refcount tags"},
+ { DMU_BSWAP_ZAP, TRUE, "DDT ZAP algorithm" },
+ { DMU_BSWAP_ZAP, TRUE, "DDT statistics" },
+ { DMU_BSWAP_UINT8, TRUE, "System attributes" },
+ { DMU_BSWAP_ZAP, TRUE, "SA master node" },
+ { DMU_BSWAP_ZAP, TRUE, "SA attr registration" },
+ { DMU_BSWAP_ZAP, TRUE, "SA attr layouts" },
+ { DMU_BSWAP_ZAP, TRUE, "scan translations" },
+ { DMU_BSWAP_UINT8, FALSE, "deduplicated block" },
+ { DMU_BSWAP_ZAP, TRUE, "DSL deadlist map" },
+ { DMU_BSWAP_UINT64, TRUE, "DSL deadlist map hdr" },
+ { DMU_BSWAP_ZAP, TRUE, "DSL dir clones" },
+ { DMU_BSWAP_UINT64, TRUE, "bpobj subobj" }
+};
+
+const dmu_object_byteswap_info_t dmu_ot_byteswap[DMU_BSWAP_NUMFUNCS] = {
+ { byteswap_uint8_array, "uint8" },
+ { byteswap_uint16_array, "uint16" },
+ { byteswap_uint32_array, "uint32" },
+ { byteswap_uint64_array, "uint64" },
+ { zap_byteswap, "zap" },
+ { dnode_buf_byteswap, "dnode" },
+ { dmu_objset_byteswap, "objset" },
+ { zfs_znode_byteswap, "znode" },
+ { zfs_oldacl_byteswap, "oldacl" },
+ { zfs_acl_byteswap, "acl" }
};
int
}
dnode_rele(dn, FTAG);
- *dbp = &db->db;
+ *dbp = &db->db; /* NULL db plus first field offset is NULL */
return (err);
}
}
int
-dmu_set_bonus(dmu_buf_t *db, int newsize, dmu_tx_t *tx)
+dmu_set_bonus(dmu_buf_t *db_fake, int newsize, dmu_tx_t *tx)
{
- dnode_t *dn = ((dmu_buf_impl_t *)db)->db_dnode;
+ dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
+ dnode_t *dn;
+ int error;
- if (dn->dn_bonus != (dmu_buf_impl_t *)db)
- return (EINVAL);
- if (newsize < 0 || newsize > db->db_size)
- return (EINVAL);
- dnode_setbonuslen(dn, newsize, tx);
- return (0);
+ DB_DNODE_ENTER(db);
+ dn = DB_DNODE(db);
+
+ if (dn->dn_bonus != db) {
+ error = EINVAL;
+ } else if (newsize < 0 || newsize > db_fake->db_size) {
+ error = EINVAL;
+ } else {
+ dnode_setbonuslen(dn, newsize, tx);
+ error = 0;
+ }
+
+ DB_DNODE_EXIT(db);
+ return (error);
}
int
-dmu_set_bonustype(dmu_buf_t *db, dmu_object_type_t type, dmu_tx_t *tx)
+dmu_set_bonustype(dmu_buf_t *db_fake, dmu_object_type_t type, dmu_tx_t *tx)
{
- dnode_t *dn = ((dmu_buf_impl_t *)db)->db_dnode;
+ dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
+ dnode_t *dn;
+ int error;
- if (type > DMU_OT_NUMTYPES)
- return (EINVAL);
+ DB_DNODE_ENTER(db);
+ dn = DB_DNODE(db);
- if (dn->dn_bonus != (dmu_buf_impl_t *)db)
- return (EINVAL);
+ if (!DMU_OT_IS_VALID(type)) {
+ error = EINVAL;
+ } else if (dn->dn_bonus != db) {
+ error = EINVAL;
+ } else {
+ dnode_setbonus_type(dn, type, tx);
+ error = 0;
+ }
- dnode_setbonus_type(dn, type, tx);
- return (0);
+ DB_DNODE_EXIT(db);
+ return (error);
+}
+
+dmu_object_type_t
+dmu_get_bonustype(dmu_buf_t *db_fake)
+{
+ dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
+ dnode_t *dn;
+ dmu_object_type_t type;
+
+ DB_DNODE_ENTER(db);
+ dn = DB_DNODE(db);
+ type = dn->dn_bonustype;
+ DB_DNODE_EXIT(db);
+
+ return (type);
}
int
dbuf_create_bonus(dn);
}
db = dn->dn_bonus;
- rw_exit(&dn->dn_struct_rwlock);
/* as long as the bonus buf is held, the dnode will be held */
- if (refcount_add(&db->db_holds, tag) == 1)
+ if (refcount_add(&db->db_holds, tag) == 1) {
VERIFY(dnode_add_ref(dn, db));
+ (void) atomic_inc_32_nv(&dn->dn_dbufs_count);
+ }
+
+ /*
+ * Wait to drop dn_struct_rwlock until after adding the bonus dbuf's
+ * hold and incrementing the dbuf count to ensure that dnode_move() sees
+ * a dnode hold for every dbuf.
+ */
+ rw_exit(&dn->dn_struct_rwlock);
dnode_rele(dn, FTAG);
rw_exit(&dn->dn_struct_rwlock);
ASSERT(db != NULL);
- err = dbuf_read(db, NULL, DB_RF_MUST_SUCCEED | flags);
- *dbp = &db->db;
+ err = dbuf_read(db, NULL, flags);
+ if (err == 0)
+ *dbp = &db->db;
+ else
+ dbuf_rele(db, tag);
return (err);
}
int
dmu_spill_hold_existing(dmu_buf_t *bonus, void *tag, dmu_buf_t **dbp)
{
- dnode_t *dn = ((dmu_buf_impl_t *)bonus)->db_dnode;
+ dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus;
+ dnode_t *dn;
int err;
- if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_SA)
- return (EINVAL);
- rw_enter(&dn->dn_struct_rwlock, RW_READER);
+ DB_DNODE_ENTER(db);
+ dn = DB_DNODE(db);
+
+ if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_SA) {
+ err = EINVAL;
+ } else {
+ rw_enter(&dn->dn_struct_rwlock, RW_READER);
+
+ if (!dn->dn_have_spill) {
+ err = ENOENT;
+ } else {
+ err = dmu_spill_hold_by_dnode(dn,
+ DB_RF_HAVESTRUCT | DB_RF_CANFAIL, tag, dbp);
+ }
- if (!dn->dn_have_spill) {
rw_exit(&dn->dn_struct_rwlock);
- return (ENOENT);
}
- err = dmu_spill_hold_by_dnode(dn, DB_RF_HAVESTRUCT, tag, dbp);
- rw_exit(&dn->dn_struct_rwlock);
+
+ DB_DNODE_EXIT(db);
return (err);
}
int
dmu_spill_hold_by_bonus(dmu_buf_t *bonus, void *tag, dmu_buf_t **dbp)
{
- return (dmu_spill_hold_by_dnode(((dmu_buf_impl_t *)bonus)->db_dnode,
- 0, tag, dbp));
+ dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus;
+ dnode_t *dn;
+ int err;
+
+ DB_DNODE_ENTER(db);
+ dn = DB_DNODE(db);
+ err = dmu_spill_hold_by_dnode(dn, DB_RF_CANFAIL, tag, dbp);
+ DB_DNODE_EXIT(db);
+
+ return (err);
}
/*
uint32_t dbuf_flags;
int err;
zio_t *zio;
- hrtime_t start;
+ hrtime_t start = 0;
ASSERT(length <= DMU_MAX_ACCESS);
}
nblks = 1;
}
- dbp = kmem_zalloc(sizeof (dmu_buf_t *) * nblks, KM_SLEEP);
+ dbp = kmem_zalloc(sizeof (dmu_buf_t *) * nblks, KM_PUSHPAGE | KM_NODEBUG);
if (dn->dn_objset->os_dsl_dataset)
dp = dn->dn_objset->os_dsl_dataset->ds_dir->dd_pool;
}
int
-dmu_buf_hold_array_by_bonus(dmu_buf_t *db, uint64_t offset,
+dmu_buf_hold_array_by_bonus(dmu_buf_t *db_fake, uint64_t offset,
uint64_t length, int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp)
{
- dnode_t *dn = ((dmu_buf_impl_t *)db)->db_dnode;
+ dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
+ dnode_t *dn;
int err;
+ DB_DNODE_ENTER(db);
+ dn = DB_DNODE(db);
err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag,
numbufsp, dbpp, DMU_READ_PREFETCH);
+ DB_DNODE_EXIT(db);
return (err);
}
return;
if (len == 0) { /* they're interested in the bonus buffer */
- dn = os->os_meta_dnode;
+ dn = DMU_META_DNODE(os);
if (object == 0 || object >= DN_MAX_OBJECT)
return;
else
dmu_buf_will_dirty(db, tx);
- bcopy(buf, (char *)db->db_data + bufoff, tocpy);
+ (void) memcpy((char *)db->db_data + bufoff, buf, tocpy);
if (tocpy == db->db_size)
dmu_buf_fill_done(db, tx);
*/
kstat_t *xuio_ksp = NULL;
+typedef struct xuio_stats {
+ /* loaned yet not returned arc_buf */
+ kstat_named_t xuiostat_onloan_rbuf;
+ kstat_named_t xuiostat_onloan_wbuf;
+ /* whether a copy is made when loaning out a read buffer */
+ kstat_named_t xuiostat_rbuf_copied;
+ kstat_named_t xuiostat_rbuf_nocopy;
+ /* whether a copy is made when assigning a write buffer */
+ kstat_named_t xuiostat_wbuf_copied;
+ kstat_named_t xuiostat_wbuf_nocopy;
+} xuio_stats_t;
+
+static xuio_stats_t xuio_stats = {
+ { "onloan_read_buf", KSTAT_DATA_UINT64 },
+ { "onloan_write_buf", KSTAT_DATA_UINT64 },
+ { "read_buf_copied", KSTAT_DATA_UINT64 },
+ { "read_buf_nocopy", KSTAT_DATA_UINT64 },
+ { "write_buf_copied", KSTAT_DATA_UINT64 },
+ { "write_buf_nocopy", KSTAT_DATA_UINT64 }
+};
+
+#define XUIOSTAT_INCR(stat, val) \
+ atomic_add_64(&xuio_stats.stat.value.ui64, (val))
+#define XUIOSTAT_BUMP(stat) XUIOSTAT_INCR(stat, 1)
+
int
dmu_xuio_init(xuio_t *xuio, int nblk)
{
uio_t *uio = &xuio->xu_uio;
uio->uio_iovcnt = nblk;
- uio->uio_iov = kmem_zalloc(nblk * sizeof (iovec_t), KM_SLEEP);
+ uio->uio_iov = kmem_zalloc(nblk * sizeof (iovec_t), KM_PUSHPAGE);
- priv = kmem_zalloc(sizeof (dmu_xuio_t), KM_SLEEP);
+ priv = kmem_zalloc(sizeof (dmu_xuio_t), KM_PUSHPAGE);
priv->cnt = nblk;
- priv->bufs = kmem_zalloc(nblk * sizeof (arc_buf_t *), KM_SLEEP);
+ priv->bufs = kmem_zalloc(nblk * sizeof (arc_buf_t *), KM_PUSHPAGE);
priv->iovp = uio->uio_iov;
XUIO_XUZC_PRIV(xuio) = priv;
}
#ifdef _KERNEL
+
+/*
+ * Copy up to size bytes between arg_buf and req based on the data direction
+ * described by the req. If an entire req's data cannot be transfered the
+ * req's is updated such that it's current index and bv offsets correctly
+ * reference any residual data which could not be copied. The return value
+ * is the number of bytes successfully copied to arg_buf.
+ */
+static int
+dmu_req_copy(void *arg_buf, int size, int *offset, struct request *req)
+{
+ struct bio_vec *bv;
+ struct req_iterator iter;
+ char *bv_buf;
+ int tocpy;
+
+ *offset = 0;
+ rq_for_each_segment(bv, req, iter) {
+
+ /* Fully consumed the passed arg_buf */
+ ASSERT3S(*offset, <=, size);
+ if (size == *offset)
+ break;
+
+ /* Skip fully consumed bv's */
+ if (bv->bv_len == 0)
+ continue;
+
+ tocpy = MIN(bv->bv_len, size - *offset);
+ ASSERT3S(tocpy, >=, 0);
+
+ bv_buf = page_address(bv->bv_page) + bv->bv_offset;
+ ASSERT3P(bv_buf, !=, NULL);
+
+ if (rq_data_dir(req) == WRITE)
+ memcpy(arg_buf + *offset, bv_buf, tocpy);
+ else
+ memcpy(bv_buf, arg_buf + *offset, tocpy);
+
+ *offset += tocpy;
+ bv->bv_offset += tocpy;
+ bv->bv_len -= tocpy;
+ }
+
+ return 0;
+}
+
+static void
+dmu_bio_put(struct bio *bio)
+{
+ struct bio *bio_next;
+
+ while (bio) {
+ bio_next = bio->bi_next;
+ bio_put(bio);
+ bio = bio_next;
+ }
+}
+
+static int
+dmu_bio_clone(struct bio *bio, struct bio **bio_copy)
+{
+ struct bio *bio_root = NULL;
+ struct bio *bio_last = NULL;
+ struct bio *bio_new;
+
+ if (bio == NULL)
+ return EINVAL;
+
+ while (bio) {
+ bio_new = bio_clone(bio, GFP_NOIO);
+ if (bio_new == NULL) {
+ dmu_bio_put(bio_root);
+ return ENOMEM;
+ }
+
+ if (bio_last) {
+ bio_last->bi_next = bio_new;
+ bio_last = bio_new;
+ } else {
+ bio_root = bio_new;
+ bio_last = bio_new;
+ }
+
+ bio = bio->bi_next;
+ }
+
+ *bio_copy = bio_root;
+
+ return 0;
+}
+
+int
+dmu_read_req(objset_t *os, uint64_t object, struct request *req)
+{
+ uint64_t size = blk_rq_bytes(req);
+ uint64_t offset = blk_rq_pos(req) << 9;
+ struct bio *bio_saved = req->bio;
+ dmu_buf_t **dbp;
+ int numbufs, i, err;
+
+ /*
+ * NB: we could do this block-at-a-time, but it's nice
+ * to be reading in parallel.
+ */
+ err = dmu_buf_hold_array(os, object, offset, size, TRUE, FTAG,
+ &numbufs, &dbp);
+ if (err)
+ return (err);
+
+ /*
+ * Clone the bio list so the bv->bv_offset and bv->bv_len members
+ * can be safely modified. The original bio list is relinked in to
+ * the request when the function exits. This is required because
+ * some file systems blindly assume that these values will remain
+ * constant between bio_submit() and the IO completion callback.
+ */
+ err = dmu_bio_clone(bio_saved, &req->bio);
+ if (err)
+ goto error;
+
+ for (i = 0; i < numbufs; i++) {
+ int tocpy, didcpy, bufoff;
+ dmu_buf_t *db = dbp[i];
+
+ bufoff = offset - db->db_offset;
+ ASSERT3S(bufoff, >=, 0);
+
+ tocpy = (int)MIN(db->db_size - bufoff, size);
+ if (tocpy == 0)
+ break;
+
+ err = dmu_req_copy(db->db_data + bufoff, tocpy, &didcpy, req);
+
+ if (didcpy < tocpy)
+ err = EIO;
+
+ if (err)
+ break;
+
+ size -= tocpy;
+ offset += didcpy;
+ err = 0;
+ }
+
+ dmu_bio_put(req->bio);
+ req->bio = bio_saved;
+error:
+ dmu_buf_rele_array(dbp, numbufs, FTAG);
+
+ return (err);
+}
+
+int
+dmu_write_req(objset_t *os, uint64_t object, struct request *req, dmu_tx_t *tx)
+{
+ uint64_t size = blk_rq_bytes(req);
+ uint64_t offset = blk_rq_pos(req) << 9;
+ struct bio *bio_saved = req->bio;
+ dmu_buf_t **dbp;
+ int numbufs;
+ int err = 0;
+ int i;
+
+ if (size == 0)
+ return (0);
+
+ err = dmu_buf_hold_array(os, object, offset, size, FALSE, FTAG,
+ &numbufs, &dbp);
+ if (err)
+ return (err);
+
+ /*
+ * Clone the bio list so the bv->bv_offset and bv->bv_len members
+ * can be safely modified. The original bio list is relinked in to
+ * the request when the function exits. This is required because
+ * some file systems blindly assume that these values will remain
+ * constant between bio_submit() and the IO completion callback.
+ */
+ err = dmu_bio_clone(bio_saved, &req->bio);
+ if (err)
+ goto error;
+
+ for (i = 0; i < numbufs; i++) {
+ int tocpy, didcpy, bufoff;
+ dmu_buf_t *db = dbp[i];
+
+ bufoff = offset - db->db_offset;
+ ASSERT3S(bufoff, >=, 0);
+
+ tocpy = (int)MIN(db->db_size - bufoff, size);
+ if (tocpy == 0)
+ break;
+
+ ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
+
+ if (tocpy == db->db_size)
+ dmu_buf_will_fill(db, tx);
+ else
+ dmu_buf_will_dirty(db, tx);
+
+ err = dmu_req_copy(db->db_data + bufoff, tocpy, &didcpy, req);
+
+ if (tocpy == db->db_size)
+ dmu_buf_fill_done(db, tx);
+
+ if (didcpy < tocpy)
+ err = EIO;
+
+ if (err)
+ break;
+
+ size -= tocpy;
+ offset += didcpy;
+ err = 0;
+ }
+
+ dmu_bio_put(req->bio);
+ req->bio = bio_saved;
+error:
+ dmu_buf_rele_array(dbp, numbufs, FTAG);
+
+ return (err);
+}
+
int
dmu_read_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size)
{
if (err)
return (err);
- if (uio->uio_extflg == UIO_XUIO)
- xuio = (xuio_t *)uio;
-
for (i = 0; i < numbufs; i++) {
int tocpy;
int bufoff;
dmu_buf_will_dirty(db, tx);
/*
- * XXX uiomove could block forever (eg. nfs-backed
+ * XXX uiomove could block forever (eg.nfs-backed
* pages). There needs to be a uiolockdown() function
* to lock the pages in memory, so that uiomove won't
* block.
dmu_write_uio_dbuf(dmu_buf_t *zdb, uio_t *uio, uint64_t size,
dmu_tx_t *tx)
{
- if (size == 0)
- return (0);
-
- return (dmu_write_uio_dnode(((dmu_buf_impl_t *)zdb)->db_dnode,
- uio, size, tx));
-}
-
-int
-dmu_write_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size,
- dmu_tx_t *tx)
-{
+ dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb;
dnode_t *dn;
int err;
if (size == 0)
return (0);
- err = dnode_hold(os, object, FTAG, &dn);
- if (err)
- return (err);
-
+ DB_DNODE_ENTER(db);
+ dn = DB_DNODE(db);
err = dmu_write_uio_dnode(dn, uio, size, tx);
-
- dnode_rele(dn, FTAG);
+ DB_DNODE_EXIT(db);
return (err);
}
int
-dmu_write_pages(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
- page_t *pp, dmu_tx_t *tx)
+dmu_write_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size,
+ dmu_tx_t *tx)
{
- dmu_buf_t **dbp;
- int numbufs, i;
+ dnode_t *dn;
int err;
if (size == 0)
return (0);
- err = dmu_buf_hold_array(os, object, offset, size,
- FALSE, FTAG, &numbufs, &dbp);
+ err = dnode_hold(os, object, FTAG, &dn);
if (err)
return (err);
- for (i = 0; i < numbufs; i++) {
- int tocpy, copied, thiscpy;
- int bufoff;
- dmu_buf_t *db = dbp[i];
- caddr_t va;
-
- ASSERT(size > 0);
- ASSERT3U(db->db_size, >=, PAGESIZE);
-
- bufoff = offset - db->db_offset;
- tocpy = (int)MIN(db->db_size - bufoff, size);
-
- ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
-
- if (tocpy == db->db_size)
- dmu_buf_will_fill(db, tx);
- else
- dmu_buf_will_dirty(db, tx);
-
- for (copied = 0; copied < tocpy; copied += PAGESIZE) {
- ASSERT3U(pp->p_offset, ==, db->db_offset + bufoff);
- thiscpy = MIN(PAGESIZE, tocpy - copied);
- va = zfs_map_page(pp, S_READ);
- bcopy(va, (char *)db->db_data + bufoff, thiscpy);
- zfs_unmap_page(pp, va);
- pp = pp->p_next;
- bufoff += PAGESIZE;
- }
+ err = dmu_write_uio_dnode(dn, uio, size, tx);
- if (tocpy == db->db_size)
- dmu_buf_fill_done(db, tx);
+ dnode_rele(dn, FTAG);
- offset += tocpy;
- size -= tocpy;
- }
- dmu_buf_rele_array(dbp, numbufs, FTAG);
return (err);
}
-#endif
+#endif /* _KERNEL */
/*
* Allocate a loaned anonymous arc buffer.
arc_buf_t *
dmu_request_arcbuf(dmu_buf_t *handle, int size)
{
- dnode_t *dn = ((dmu_buf_impl_t *)handle)->db_dnode;
+ dmu_buf_impl_t *db = (dmu_buf_impl_t *)handle;
+ spa_t *spa;
- return (arc_loan_buf(dn->dn_objset->os_spa, size));
+ DB_GET_SPA(&spa, db);
+ return (arc_loan_buf(spa, size));
}
/*
dmu_assign_arcbuf(dmu_buf_t *handle, uint64_t offset, arc_buf_t *buf,
dmu_tx_t *tx)
{
- dnode_t *dn = ((dmu_buf_impl_t *)handle)->db_dnode;
+ dmu_buf_impl_t *dbuf = (dmu_buf_impl_t *)handle;
+ dnode_t *dn;
dmu_buf_impl_t *db;
uint32_t blksz = (uint32_t)arc_buf_size(buf);
uint64_t blkid;
+ DB_DNODE_ENTER(dbuf);
+ dn = DB_DNODE(dbuf);
rw_enter(&dn->dn_struct_rwlock, RW_READER);
blkid = dbuf_whichblock(dn, offset);
VERIFY((db = dbuf_hold(dn, blkid, FTAG)) != NULL);
rw_exit(&dn->dn_struct_rwlock);
+ DB_DNODE_EXIT(dbuf);
if (offset == db->db.db_offset && blksz == db->db.db_size) {
dbuf_assign_arcbuf(db, buf, tx);
dbuf_rele(db, FTAG);
} else {
+ objset_t *os;
+ uint64_t object;
+
+ DB_DNODE_ENTER(dbuf);
+ dn = DB_DNODE(dbuf);
+ os = dn->dn_objset;
+ object = dn->dn_object;
+ DB_DNODE_EXIT(dbuf);
+
dbuf_rele(db, FTAG);
- dmu_write(dn->dn_objset, dn->dn_object, offset, blksz,
- buf->b_data, tx);
+ dmu_write(os, object, offset, blksz, buf->b_data, tx);
dmu_return_arcbuf(buf);
XUIOSTAT_BUMP(xuiostat_wbuf_copied);
}
{
dmu_sync_arg_t *dsa = varg;
dmu_buf_t *db = dsa->dsa_zgd->zgd_db;
- dnode_t *dn = ((dmu_buf_impl_t *)db)->db_dnode;
blkptr_t *bp = zio->io_bp;
if (zio->io_error == 0) {
*/
BP_SET_LSIZE(bp, db->db_size);
} else {
- ASSERT(BP_GET_TYPE(bp) == dn->dn_type);
ASSERT(BP_GET_LEVEL(bp) == 0);
bp->blk_fill = 1;
}
return (EIO); /* Make zl_get_data do txg_waited_synced() */
}
- dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP);
+ dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_PUSHPAGE);
dsa->dsa_dr = NULL;
dsa->dsa_done = done;
dsa->dsa_zgd = zgd;
zio_nowait(zio_write(pio, os->os_spa, dmu_tx_get_txg(tx), zgd->zgd_bp,
zgd->zgd_db->db_data, zgd->zgd_db->db_size, zp,
dmu_sync_late_arrival_ready, dmu_sync_late_arrival_done, dsa,
- ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, zb));
+ ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL | ZIO_FLAG_FASTWRITE, zb));
return (0);
}
dmu_sync_arg_t *dsa;
zbookmark_t zb;
zio_prop_t zp;
+ dnode_t *dn;
ASSERT(pio != NULL);
ASSERT(BP_IS_HOLE(bp));
SET_BOOKMARK(&zb, ds->ds_object,
db->db.db_object, db->db_level, db->db_blkid);
- dmu_write_policy(os, db->db_dnode, db->db_level, WP_DMU_SYNC, &zp);
+ DB_DNODE_ENTER(db);
+ dn = DB_DNODE(db);
+ dmu_write_policy(os, dn, db->db_level, WP_DMU_SYNC, &zp);
+ DB_DNODE_EXIT(db);
/*
* If we're frozen (running ziltest), we always need to generate a bp.
dr->dt.dl.dr_override_state = DR_IN_DMU_SYNC;
mutex_exit(&db->db_mtx);
- dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP);
+ dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_PUSHPAGE);
dsa->dsa_dr = dr;
dsa->dsa_done = done;
dsa->dsa_zgd = zgd;
dsa->dsa_tx = NULL;
zio_nowait(arc_write(pio, os->os_spa, txg,
- bp, dr->dt.dl.dr_data, DBUF_IS_L2CACHEABLE(db), &zp,
- dmu_sync_ready, dmu_sync_done, dsa,
- ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, &zb));
+ bp, dr->dt.dl.dr_data, DBUF_IS_L2CACHEABLE(db),
+ DBUF_IS_L2COMPRESSIBLE(db), &zp, dmu_sync_ready, dmu_sync_done,
+ dsa, ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL | ZIO_FLAG_FASTWRITE, &zb));
return (0);
}
dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp, zio_prop_t *zp)
{
dmu_object_type_t type = dn ? dn->dn_type : DMU_OT_OBJSET;
- boolean_t ismd = (level > 0 || dmu_ot[type].ot_metadata);
+ boolean_t ismd = (level > 0 || DMU_OT_IS_METADATA(type) ||
+ (wp & WP_SPILL));
enum zio_checksum checksum = os->os_checksum;
enum zio_compress compress = os->os_compress;
enum zio_checksum dedup_checksum = os->os_dedup_checksum;
dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi)
{
dnode_phys_t *dnp;
+ int i;
rw_enter(&dn->dn_struct_rwlock, RW_READER);
mutex_enter(&dn->dn_mtx);
doi->doi_physical_blocks_512 = (DN_USED_BYTES(dnp) + 256) >> 9;
doi->doi_max_offset = (dnp->dn_maxblkid + 1) * dn->dn_datablksz;
doi->doi_fill_count = 0;
- for (int i = 0; i < dnp->dn_nblkptr; i++)
+ for (i = 0; i < dnp->dn_nblkptr; i++)
doi->doi_fill_count += dnp->dn_blkptr[i].blk_fill;
mutex_exit(&dn->dn_mtx);
* As above, but faster; can be used when you have a held dbuf in hand.
*/
void
-dmu_object_info_from_db(dmu_buf_t *db, dmu_object_info_t *doi)
+dmu_object_info_from_db(dmu_buf_t *db_fake, dmu_object_info_t *doi)
{
- dmu_object_info_from_dnode(((dmu_buf_impl_t *)db)->db_dnode, doi);
+ dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
+
+ DB_DNODE_ENTER(db);
+ dmu_object_info_from_dnode(DB_DNODE(db), doi);
+ DB_DNODE_EXIT(db);
}
/*
* This is specifically optimized for zfs_getattr().
*/
void
-dmu_object_size_from_db(dmu_buf_t *db, uint32_t *blksize, u_longlong_t *nblk512)
+dmu_object_size_from_db(dmu_buf_t *db_fake, uint32_t *blksize,
+ u_longlong_t *nblk512)
{
- dnode_t *dn = ((dmu_buf_impl_t *)db)->db_dnode;
+ dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
+ dnode_t *dn;
+
+ DB_DNODE_ENTER(db);
+ dn = DB_DNODE(db);
*blksize = dn->dn_datablksz;
/* add 1 for dnode space */
*nblk512 = ((DN_USED_BYTES(dn->dn_phys) + SPA_MINBLOCKSIZE/2) >>
SPA_MINBLOCKSHIFT) + 1;
+ DB_DNODE_EXIT(db);
}
void
dmu_init(void)
{
zfs_dbgmsg_init();
- dbuf_init();
+ sa_cache_init();
+ xuio_stat_init();
+ dmu_objset_init();
dnode_init();
+ dbuf_init();
zfetch_init();
- arc_init();
+ dmu_tx_init();
l2arc_init();
- xuio_stat_init();
- sa_cache_init();
+ arc_init();
}
void
dmu_fini(void)
{
arc_fini();
+ l2arc_fini();
+ dmu_tx_fini();
zfetch_fini();
- dnode_fini();
dbuf_fini();
- l2arc_fini();
+ dnode_fini();
+ dmu_objset_fini();
xuio_stat_fini();
sa_cache_fini();
zfs_dbgmsg_fini();
}
+
+#if defined(_KERNEL) && defined(HAVE_SPL)
+EXPORT_SYMBOL(dmu_bonus_hold);
+EXPORT_SYMBOL(dmu_buf_hold_array_by_bonus);
+EXPORT_SYMBOL(dmu_buf_rele_array);
+EXPORT_SYMBOL(dmu_prefetch);
+EXPORT_SYMBOL(dmu_free_range);
+EXPORT_SYMBOL(dmu_free_long_range);
+EXPORT_SYMBOL(dmu_free_object);
+EXPORT_SYMBOL(dmu_read);
+EXPORT_SYMBOL(dmu_write);
+EXPORT_SYMBOL(dmu_prealloc);
+EXPORT_SYMBOL(dmu_object_info);
+EXPORT_SYMBOL(dmu_object_info_from_dnode);
+EXPORT_SYMBOL(dmu_object_info_from_db);
+EXPORT_SYMBOL(dmu_object_size_from_db);
+EXPORT_SYMBOL(dmu_object_set_blocksize);
+EXPORT_SYMBOL(dmu_object_set_checksum);
+EXPORT_SYMBOL(dmu_object_set_compress);
+EXPORT_SYMBOL(dmu_write_policy);
+EXPORT_SYMBOL(dmu_sync);
+EXPORT_SYMBOL(dmu_request_arcbuf);
+EXPORT_SYMBOL(dmu_return_arcbuf);
+EXPORT_SYMBOL(dmu_assign_arcbuf);
+EXPORT_SYMBOL(dmu_buf_hold);
+EXPORT_SYMBOL(dmu_ot);
+
+module_param(zfs_mdcomp_disable, int, 0644);
+MODULE_PARM_DESC(zfs_mdcomp_disable, "Disable meta data compression");
+#endif
git://git.camperquake.de / zfs.git / blobdiff