*
* CDDL HEADER END
*/
+
/*
* Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012 by Delphix. All rights reserved.
*/
#include <sys/zfs_context.h>
sa_data_op_t action, sa_data_locator_t *locator, void *datastart,
uint16_t buflen, dmu_tx_t *tx);
-arc_byteswap_func_t *sa_bswap_table[] = {
+arc_byteswap_func_t sa_bswap_table[] = {
byteswap_uint64_array,
byteswap_uint32_array,
byteswap_uint16_array,
static int sa_legacy_attr_count = 16;
static kmem_cache_t *sa_cache = NULL;
+static kmem_cache_t *spill_cache = NULL;
/*ARGSUSED*/
static int
sa_cache = kmem_cache_create("sa_cache",
sizeof (sa_handle_t), 0, sa_cache_constructor,
sa_cache_destructor, NULL, NULL, NULL, 0);
+ spill_cache = kmem_cache_create("spill_cache",
+ SPA_MAXBLOCKSIZE, 0, NULL, NULL, NULL, NULL, NULL, 0);
}
void
{
if (sa_cache)
kmem_cache_destroy(sa_cache);
+
+ if (spill_cache)
+ kmem_cache_destroy(spill_cache);
+}
+
+void *
+sa_spill_alloc(int flags)
+{
+ return kmem_cache_alloc(spill_cache, flags);
+}
+
+void
+sa_spill_free(void *obj)
+{
+ kmem_cache_free(spill_cache, obj);
}
static int
avl_index_t loc;
ASSERT(MUTEX_HELD(&sa->sa_lock));
- tb = kmem_zalloc(sizeof (sa_lot_t), KM_SLEEP);
+ tb = kmem_zalloc(sizeof (sa_lot_t), KM_PUSHPAGE);
tb->lot_attr_count = attr_count;
tb->lot_attrs = kmem_alloc(sizeof (sa_attr_type_t) * attr_count,
- KM_SLEEP);
+ KM_PUSHPAGE);
bcopy(attrs, tb->lot_attrs, sizeof (sa_attr_type_t) * attr_count);
tb->lot_num = lot_num;
tb->lot_hash = hash;
char attr_name[8];
if (sa->sa_layout_attr_obj == 0) {
- sa->sa_layout_attr_obj = zap_create(os,
- DMU_OT_SA_ATTR_LAYOUTS, DMU_OT_NONE, 0, tx);
- VERIFY(zap_add(os, sa->sa_master_obj, SA_LAYOUTS, 8, 1,
- &sa->sa_layout_attr_obj, tx) == 0);
+ sa->sa_layout_attr_obj = zap_create_link(os,
+ DMU_OT_SA_ATTR_LAYOUTS,
+ sa->sa_master_obj, SA_LAYOUTS, tx);
}
(void) snprintf(attr_name, sizeof (attr_name),
{
int var_size = 0;
int i;
+ int j = -1;
int full_space;
int hdrsize;
boolean_t done = B_FALSE;
sizeof (sa_hdr_phys_t);
full_space = (buftype == SA_BONUS) ? DN_MAX_BONUSLEN : db->db_size;
+ ASSERT(IS_P2ALIGNED(full_space, 8));
for (i = 0; i != attr_count; i++) {
boolean_t is_var_sz;
+ *total = P2ROUNDUP(*total, 8);
*total += attr_desc[i].sa_length;
if (done)
goto next;
if (is_var_sz && var_size > 1) {
if (P2ROUNDUP(hdrsize + sizeof (uint16_t), 8) +
*total < full_space) {
+ /*
+ * Account for header space used by array of
+ * optional sizes of variable-length attributes.
+ * Record the index in case this increase needs
+ * to be reversed due to spill-over.
+ */
hdrsize += sizeof (uint16_t);
+ j = i;
} else {
done = B_TRUE;
*index = i;
* and spill buffer.
*/
if (buftype == SA_BONUS && *index == -1 &&
- P2ROUNDUP(*total + hdrsize, 8) >
+ (*total + P2ROUNDUP(hdrsize, 8)) >
(full_space - sizeof (blkptr_t))) {
*index = i;
done = B_TRUE;
}
next:
- if (P2ROUNDUP(*total + hdrsize, 8) > full_space &&
+ if ((*total + P2ROUNDUP(hdrsize, 8)) > full_space &&
buftype == SA_BONUS)
*will_spill = B_TRUE;
}
+ /*
+ * j holds the index of the last variable-sized attribute for
+ * which hdrsize was increased. Reverse the increase if that
+ * attribute will be relocated to the spill block.
+ */
+ if (*will_spill && j == *index)
+ hdrsize -= sizeof (uint16_t);
+
hdrsize = P2ROUNDUP(hdrsize, 8);
return (hdrsize);
}
int buf_space;
sa_attr_type_t *attrs, *attrs_start;
int i, lot_count;
- int hdrsize, spillhdrsize;
+ int hdrsize, spillhdrsize = 0;
int used;
dmu_object_type_t bonustype;
sa_lot_t *lot;
buf_space = hdl->sa_bonus->db_size - hdrsize;
attrs_start = attrs = kmem_alloc(sizeof (sa_attr_type_t) * attr_count,
- KM_SLEEP);
+ KM_PUSHPAGE);
lot_count = 0;
for (i = 0, len_idx = 0, hash = -1ULL; i != attr_count; i++) {
uint16_t length;
+ ASSERT(IS_P2ALIGNED(data_start, 8));
+ ASSERT(IS_P2ALIGNED(buf_space, 8));
attrs[i] = attr_desc[i].sa_attr;
length = SA_REGISTERED_LEN(sa, attrs[i]);
if (length == 0)
length = attr_desc[i].sa_length;
if (buf_space < length) { /* switch to spill buffer */
+ VERIFY(spilling);
VERIFY(bonustype == DMU_OT_SA);
if (buftype == SA_BONUS && !sa->sa_force_spill) {
sa_find_layout(hdl->sa_os, hash, attrs_start,
{
sa_os_t *sa = os->os_sa;
uint64_t sa_attr_count = 0;
- uint64_t sa_reg_count;
+ uint64_t sa_reg_count = 0;
int error = 0;
uint64_t attr_value;
sa_attr_table_t *tb;
dmu_objset_type_t ostype = dmu_objset_type(os);
sa->sa_user_table =
- kmem_zalloc(count * sizeof (sa_attr_type_t), KM_SLEEP);
+ kmem_zalloc(count * sizeof (sa_attr_type_t), KM_PUSHPAGE);
sa->sa_user_table_sz = count * sizeof (sa_attr_type_t);
if (sa->sa_reg_attr_obj != 0) {
sa->sa_num_attrs = sa_attr_count;
tb = sa->sa_attr_table =
- kmem_zalloc(sizeof (sa_attr_table_t) * sa_attr_count, KM_SLEEP);
+ kmem_zalloc(sizeof (sa_attr_table_t) * sa_attr_count, KM_PUSHPAGE);
/*
* Attribute table is constructed from requested attribute list,
continue;
}
tb[ATTR_NUM(value)].sa_name =
- kmem_zalloc(strlen(za.za_name) +1, KM_SLEEP);
+ kmem_zalloc(strlen(za.za_name) +1, KM_PUSHPAGE);
(void) strlcpy(tb[ATTR_NUM(value)].sa_name, za.za_name,
strlen(za.za_name) +1);
}
tb[i].sa_registered = B_FALSE;
tb[i].sa_name =
kmem_zalloc(strlen(sa_legacy_attrs[i].sa_name) +1,
- KM_SLEEP);
+ KM_PUSHPAGE);
(void) strlcpy(tb[i].sa_name,
sa_legacy_attrs[i].sa_name,
strlen(sa_legacy_attrs[i].sa_name) + 1);
tb[attr_id].sa_byteswap = reg_attrs[i].sa_byteswap;
tb[attr_id].sa_attr = attr_id;
tb[attr_id].sa_name =
- kmem_zalloc(strlen(reg_attrs[i].sa_name) + 1, KM_SLEEP);
+ kmem_zalloc(strlen(reg_attrs[i].sa_name) + 1, KM_PUSHPAGE);
(void) strlcpy(tb[attr_id].sa_name, reg_attrs[i].sa_name,
strlen(reg_attrs[i].sa_name) + 1);
}
return (0);
}
- sa = kmem_zalloc(sizeof (sa_os_t), KM_SLEEP);
+ sa = kmem_zalloc(sizeof (sa_os_t), KM_PUSHPAGE);
mutex_init(&sa->sa_lock, NULL, MUTEX_DEFAULT, NULL);
sa->sa_master_obj = sa_obj;
uint64_t lot_num;
lot_attrs = kmem_zalloc(sizeof (sa_attr_type_t) *
- za.za_num_integers, KM_SLEEP);
+ za.za_num_integers, KM_PUSHPAGE);
if ((error = (zap_lookup(os, sa->sa_layout_attr_obj,
za.za_name, 2, za.za_num_integers,
sa_free_attr_table(sa);
cookie = NULL;
- while (layout = avl_destroy_nodes(&sa->sa_layout_hash_tree, &cookie)) {
+ while ((layout = avl_destroy_nodes(&sa->sa_layout_hash_tree, &cookie))){
sa_idx_tab_t *tab;
- while (tab = list_head(&layout->lot_idx_tab)) {
+ while ((tab = list_head(&layout->lot_idx_tab))) {
ASSERT(refcount_count(&tab->sa_refcount));
sa_idx_tab_rele(os, tab);
}
}
cookie = NULL;
- while (layout = avl_destroy_nodes(&sa->sa_layout_num_tree, &cookie)) {
+ while ((layout = avl_destroy_nodes(&sa->sa_layout_num_tree, &cookie))){
kmem_free(layout->lot_attrs,
sizeof (sa_attr_type_t) * layout->lot_attr_count);
kmem_free(layout, sizeof (sa_lot_t));
{
sa_hdr_phys_t *sa_hdr_phys = SA_GET_HDR(hdl, buftype);
dmu_buf_impl_t *db;
- sa_os_t *sa = hdl->sa_os->os_sa;
int num_lengths = 1;
int i;
+ ASSERTV(sa_os_t *sa = hdl->sa_os->os_sa);
ASSERT(MUTEX_HELD(&sa->sa_lock));
if (sa_hdr_phys->sa_magic == SA_MAGIC)
static void
sa_idx_tab_hold(objset_t *os, sa_idx_tab_t *idx_tab)
{
- sa_os_t *sa = os->os_sa;
+ ASSERTV(sa_os_t *sa = os->os_sa);
ASSERT(MUTEX_HELD(&sa->sa_lock));
(void) refcount_add(&idx_tab->sa_refcount, NULL);
}
void
+sa_spill_rele(sa_handle_t *hdl)
+{
+ mutex_enter(&hdl->sa_lock);
+ if (hdl->sa_spill) {
+ sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);
+ dmu_buf_rele(hdl->sa_spill, NULL);
+ hdl->sa_spill = NULL;
+ hdl->sa_spill_tab = NULL;
+ }
+ mutex_exit(&hdl->sa_lock);
+}
+
+void
sa_handle_destroy(sa_handle_t *hdl)
{
mutex_enter(&hdl->sa_lock);
sa_handle_type_t hdl_type, sa_handle_t **handlepp)
{
int error = 0;
- dmu_object_info_t doi;
sa_handle_t *handle;
-
#ifdef ZFS_DEBUG
+ dmu_object_info_t doi;
+
dmu_object_info_from_db(db, &doi);
ASSERT(doi.doi_bonus_type == DMU_OT_SA ||
doi.doi_bonus_type == DMU_OT_ZNODE);
dmu_buf_t *db;
int error;
- if (error = dmu_bonus_hold(objset, objid, NULL, &db))
+ if ((error = dmu_bonus_hold(objset, objid, NULL, &db)))
return (error);
return (sa_handle_get_from_db(objset, db, userp, hdl_type,
}
mutex_exit(&hdl->sa_lock);
return (error);
-
}
#endif
}
/* No such luck, create a new entry */
- idx_tab = kmem_zalloc(sizeof (sa_idx_tab_t), KM_SLEEP);
+ idx_tab = kmem_zalloc(sizeof (sa_idx_tab_t), KM_PUSHPAGE);
idx_tab->sa_idx_tab =
- kmem_zalloc(sizeof (uint32_t) * sa->sa_num_attrs, KM_SLEEP);
+ kmem_zalloc(sizeof (uint32_t) * sa->sa_num_attrs, KM_PUSHPAGE);
idx_tab->sa_layout = tb;
refcount_create(&idx_tab->sa_refcount);
if (tb->lot_var_sizes)
idx_tab->sa_variable_lengths = kmem_alloc(sizeof (uint16_t) *
- tb->lot_var_sizes, KM_SLEEP);
+ tb->lot_var_sizes, KM_PUSHPAGE);
sa_attr_iter(os, hdr, bonustype, sa_build_idx_tab,
tb, idx_tab);
}
if (sa->sa_reg_attr_obj == 0) {
- sa->sa_reg_attr_obj = zap_create(hdl->sa_os,
- DMU_OT_SA_ATTR_REGISTRATION, DMU_OT_NONE, 0, tx);
- VERIFY(zap_add(hdl->sa_os, sa->sa_master_obj,
- SA_REGISTRY, 8, 1, &sa->sa_reg_attr_obj, tx) == 0);
+ sa->sa_reg_attr_obj = zap_create_link(hdl->sa_os,
+ DMU_OT_SA_ATTR_REGISTRATION,
+ sa->sa_master_obj, SA_REGISTRY, tx);
}
for (i = 0; i != sa->sa_num_attrs; i++) {
if (sa->sa_attr_table[i].sa_registered)
sa_bulk_attr_t *attr_desc;
void *old_data[2];
int bonus_attr_count = 0;
- int bonus_data_size, spill_data_size;
+ int bonus_data_size = 0;
int spill_attr_count = 0;
int error;
uint16_t length;
/* Bring spill buffer online if it isn't currently */
if ((error = sa_get_spill(hdl)) == 0) {
- spill_data_size = hdl->sa_spill->db_size;
- old_data[1] = kmem_alloc(spill_data_size, KM_SLEEP);
+ ASSERT3U(hdl->sa_spill->db_size, <=, SPA_MAXBLOCKSIZE);
+ old_data[1] = sa_spill_alloc(KM_SLEEP);
bcopy(hdl->sa_spill->db_data, old_data[1],
hdl->sa_spill->db_size);
spill_attr_count =
if (old_data[0])
kmem_free(old_data[0], bonus_data_size);
if (old_data[1])
- kmem_free(old_data[1], spill_data_size);
+ sa_spill_free(old_data[1]);
kmem_free(attr_desc, sizeof (sa_bulk_attr_t) * attr_count);
return (error);
int error;
sa_os_t *sa = hdl->sa_os->os_sa;
dmu_object_type_t bonustype;
-
- bonustype = SA_BONUSTYPE_FROM_DB(SA_GET_DB(hdl, SA_BONUS));
+ dmu_buf_t *saved_spill;
ASSERT(hdl);
ASSERT(MUTEX_HELD(&hdl->sa_lock));
+ bonustype = SA_BONUSTYPE_FROM_DB(SA_GET_DB(hdl, SA_BONUS));
+ saved_spill = hdl->sa_spill;
+
/* sync out registration table if necessary */
if (sa->sa_need_attr_registration)
sa_attr_register_sync(hdl, tx);
if (error == 0 && !IS_SA_BONUSTYPE(bonustype) && sa->sa_update_cb)
sa->sa_update_cb(hdl, tx);
+ /*
+ * If saved_spill is NULL and current sa_spill is not NULL that
+ * means we increased the refcount of the spill buffer through
+ * sa_get_spill() or dmu_spill_hold_by_dnode(). Therefore we
+ * must release the hold before calling dmu_tx_commit() to avoid
+ * making a copy of this buffer in dbuf_sync_leaf() due to the
+ * reference count now being greater than 1.
+ */
+ if (!saved_spill && hdl->sa_spill) {
+ if (hdl->sa_spill_tab) {
+ sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);
+ hdl->sa_spill_tab = NULL;
+ }
+
+ dmu_buf_rele((dmu_buf_t *)hdl->sa_spill, NULL);
+ hdl->sa_spill = NULL;
+ }
+
return (error);
}
ASSERT(hdl);
mutex_exit(&hdl->sa_lock);
}
+
+#ifdef _KERNEL
+EXPORT_SYMBOL(sa_handle_get);
+EXPORT_SYMBOL(sa_handle_get_from_db);
+EXPORT_SYMBOL(sa_handle_destroy);
+EXPORT_SYMBOL(sa_buf_hold);
+EXPORT_SYMBOL(sa_buf_rele);
+EXPORT_SYMBOL(sa_spill_rele);
+EXPORT_SYMBOL(sa_lookup);
+EXPORT_SYMBOL(sa_update);
+EXPORT_SYMBOL(sa_remove);
+EXPORT_SYMBOL(sa_bulk_lookup);
+EXPORT_SYMBOL(sa_bulk_lookup_locked);
+EXPORT_SYMBOL(sa_bulk_update);
+EXPORT_SYMBOL(sa_size);
+EXPORT_SYMBOL(sa_update_from_cb);
+EXPORT_SYMBOL(sa_object_info);
+EXPORT_SYMBOL(sa_object_size);
+EXPORT_SYMBOL(sa_update_user);
+EXPORT_SYMBOL(sa_get_userdata);
+EXPORT_SYMBOL(sa_set_userp);
+EXPORT_SYMBOL(sa_get_db);
+EXPORT_SYMBOL(sa_handle_object);
+EXPORT_SYMBOL(sa_register_update_callback);
+EXPORT_SYMBOL(sa_setup);
+EXPORT_SYMBOL(sa_replace_all_by_template);
+EXPORT_SYMBOL(sa_replace_all_by_template_locked);
+EXPORT_SYMBOL(sa_enabled);
+EXPORT_SYMBOL(sa_cache_init);
+EXPORT_SYMBOL(sa_cache_fini);
+EXPORT_SYMBOL(sa_spill_alloc);
+EXPORT_SYMBOL(sa_spill_free);
+EXPORT_SYMBOL(sa_set_sa_object);
+EXPORT_SYMBOL(sa_hdrsize);
+EXPORT_SYMBOL(sa_handle_lock);
+EXPORT_SYMBOL(sa_handle_unlock);
+EXPORT_SYMBOL(sa_lookup_uio);
+#endif /* _KERNEL */
git://git.camperquake.de / zfs.git / blobdiff