Use abd in dbuf and arc_buf to allow buffer sharing

cmd/zdb/zdb.c

@@ -2962,10 +2962,10 @@ visit_indirect(spa_t *spa, const dnode_phys_t *dnp,
 		    ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL, &flags, zb);
 		if (err)
 			return (err);
-		ASSERT(buf->b_data);
+		ASSERT(buf->b_abd);
 
 		/* recursively visit blocks below this */
-		cbp = buf->b_data;
+		cbp = abd_to_buf(buf->b_abd);
 		for (i = 0; i < epb; i++, cbp++) {
 			zbookmark_phys_t czb;
 
@@ -3223,7 +3223,7 @@ dump_bptree(objset_t *os, uint64_t obj, const char *name)
 		return;
 
 	VERIFY3U(0, ==, dmu_bonus_hold(os, obj, FTAG, &db));
-	bt = db->db_data;
+	bt = abd_to_buf(db->db_abd);
 	zdb_nicenum(bt->bt_bytes, bytes, sizeof (bytes));
 	(void) printf("\n    %s: %llu datasets, %s\n",
 	    name, (unsigned long long)(bt->bt_end - bt->bt_begin), bytes);
@@ -4317,7 +4317,7 @@ dump_object(objset_t *os, uint64_t object, int verbosity,
 			if (error)
 				fatal("dmu_bonus_hold(%llu) failed, errno %u",
 				    object, error);
-			bonus = db->db_data;
+			bonus = abd_to_buf(db->db_abd);
 			bsize = db->db_size;
 			dn = DB_DNODE((dmu_buf_impl_t *)db);
 		}
@@ -4917,7 +4917,7 @@ dump_config(spa_t *spa)
 	    spa->spa_config_object, FTAG, &db);
 
 	if (error == 0) {
-		nvsize = *(uint64_t *)db->db_data;
+		nvsize = *(uint64_t *)abd_to_buf(db->db_abd);
 		dmu_buf_rele(db, FTAG);
 
 		(void) printf("\nMOS Configuration:\n");

cmd/ztest.c

@@ -1883,7 +1883,8 @@ ztest_bt_bonus(dmu_buf_t *db)
 	dmu_object_info_from_db(db, &doi);
 	ASSERT3U(doi.doi_bonus_size, <=, db->db_size);
 	ASSERT3U(doi.doi_bonus_size, >=, sizeof (*bt));
-	bt = (void *)((char *)db->db_data + doi.doi_bonus_size - sizeof (*bt));
+	bt = (void *)((char *)abd_to_buf(db->db_abd) + doi.doi_bonus_size -
+	    sizeof (*bt));
 
 	return (bt);
 }
@@ -1907,12 +1908,13 @@ ztest_fill_unused_bonus(dmu_buf_t *db, void *end, uint64_t obj,
     objset_t *os, uint64_t gen)
 {
 	uint64_t *bonusp;
+	uint64_t *start = abd_to_buf(db->db_abd);
 
-	ASSERT(IS_P2ALIGNED((char *)end - (char *)db->db_data, 8));
+	ASSERT(IS_P2ALIGNED((char *)end - (char *)start, 8));
 
-	for (bonusp = db->db_data; bonusp < (uint64_t *)end; bonusp++) {
+	for (bonusp = start; bonusp < (uint64_t *)end; bonusp++) {
 		uint64_t token = ZTEST_BONUS_FILL_TOKEN(obj, dmu_objset_id(os),
-		    gen, bonusp - (uint64_t *)db->db_data);
+		    gen, bonusp - start);
 		*bonusp = token;
 	}
 }
@@ -1926,10 +1928,11 @@ ztest_verify_unused_bonus(dmu_buf_t *db, void *end, uint64_t obj,
     objset_t *os, uint64_t gen)
 {
 	uint64_t *bonusp;
+	uint64_t *start = abd_to_buf(db->db_abd);
 
-	for (bonusp = db->db_data; bonusp < (uint64_t *)end; bonusp++) {
+	for (bonusp = start; bonusp < (uint64_t *)end; bonusp++) {
 		uint64_t token = ZTEST_BONUS_FILL_TOKEN(obj, dmu_objset_id(os),
-		    gen, bonusp - (uint64_t *)db->db_data);
+		    gen, bonusp - start);
 		VERIFY3U(*bonusp, ==, token);
 	}
 }
@@ -2312,7 +2315,7 @@ ztest_replay_write(void *arg1, void *arg2, boolean_t byteswap)
 		dmu_write(os, lr->lr_foid, offset, length, data, tx,
 		    DMU_READ_PREFETCH);
 	} else {
-		memcpy(abuf->b_data, data, length);
+		abd_copy_from_buf(abuf->b_abd, data, length);
 		VERIFY0(dmu_assign_arcbuf_by_dbuf(db, offset, abuf, tx, 0));
 	}
 
@@ -5529,14 +5532,15 @@ ztest_dmu_read_write_zcopy(ztest_ds_t *zd, uint64_t id)
 		for (off = bigoff, j = 0; j < s; j++, off += chunksize) {
 			dmu_buf_t *dbt;
 			if (i != 5 || chunksize < (SPA_MINBLOCKSIZE * 2)) {
-				memcpy(bigbuf_arcbufs[j]->b_data,
+				abd_copy_from_buf(bigbuf_arcbufs[j]->b_abd,
 				    (caddr_t)bigbuf + (off - bigoff),
 				    chunksize);
 			} else {
-				memcpy(bigbuf_arcbufs[2 * j]->b_data,
+				abd_copy_from_buf(bigbuf_arcbufs[2 * j]->b_abd,
 				    (caddr_t)bigbuf + (off - bigoff),
 				    chunksize / 2);
-				memcpy(bigbuf_arcbufs[2 * j + 1]->b_data,
+				abd_copy_from_buf(
+				    bigbuf_arcbufs[2 * j + 1]->b_abd,
 				    (caddr_t)bigbuf + (off - bigoff) +
 				    chunksize / 2,
 				    chunksize / 2);

include/os/linux/spl/sys/uio.h

@@ -98,6 +98,8 @@ typedef struct zfs_uio {
 #define	zfs_uio_soffset(u)		(u)->uio_soffset
 #define	zfs_uio_rlimit_fsize(z, u)	(0)
 #define	zfs_uio_fault_move(p, n, rw, u)	zfs_uiomove((p), (n), (rw), (u))
+#define	zfs_uio_fault_move_abd(p, o, n, rw, u)	\
+	zfs_uiomove_abd((p), (o), (n), (rw), (u))
 
 extern int zfs_uio_prefaultpages(ssize_t, zfs_uio_t *);
 
@@ -210,4 +212,11 @@ zfs_uio_iov_iter_init(zfs_uio_t *uio, struct iov_iter *iter, offset_t offset,
 	(zfs_uio_iov_iter_type((iter)) == ITER_IOVEC)
 #endif
 
+static inline boolean_t
+uio_is_user(zfs_uio_t *uio)
+{
+	return (uio->uio_segflg == UIO_ITER &&
+	    zfs_user_backed_iov_iter(uio->uio_iter));
+}
+
 #endif /* SPL_UIO_H */

include/sys/abd.h

@@ -118,6 +118,8 @@ void abd_release_ownership_of_buf(abd_t *);
  */
 
 int abd_iterate_func(abd_t *, size_t, size_t, abd_iter_func_t *, void *);
+int abd_iterate_func_impl(abd_t *, size_t, size_t, abd_iter_func_t *, void *,
+    boolean_t);
 int abd_iterate_func2(abd_t *, abd_t *, size_t, size_t, size_t,
     abd_iter_func2_t *, void *);
 void abd_copy_off(abd_t *, abd_t *, size_t, size_t, size_t);

include/sys/abd_impl.h

@@ -108,6 +108,13 @@ void abd_iter_advance(struct abd_iter *, size_t);
 void abd_iter_map(struct abd_iter *);
 void abd_iter_unmap(struct abd_iter *);
 void abd_iter_page(struct abd_iter *);
+#if defined(__linux__) && defined(_KERNEL)
+void abd_iter_map_impl(struct abd_iter *, boolean_t);
+void abd_iter_unmap_impl(struct abd_iter *, boolean_t);
+#else
+#define	abd_iter_map_impl(aiter, x)	((void)x, abd_iter_map(aiter))
+#define	abd_iter_unmap_impl(aiter, x)	((void)x, abd_iter_unmap(aiter))
+#endif
 
 /*
  * Helper macros

include/sys/arc.h

@@ -201,7 +201,7 @@ typedef enum arc_buf_flags {
 struct arc_buf {
 	arc_buf_hdr_t		*b_hdr;
 	arc_buf_t		*b_next;
-	void			*b_data;
+	abd_t			*b_abd;
 	arc_buf_flags_t		b_flags;
 };
 

include/sys/dbuf.h

@@ -200,7 +200,7 @@ typedef struct dbuf_dirty_record {
 typedef struct dmu_buf_impl {
 	/*
 	 * The following members are immutable, with the exception of
-	 * db.db_data, which is protected by db_mtx.
+	 * db.db_abd, which is protected by db_mtx.
 	 */
 
 	/* the publicly visible structure */
@@ -347,7 +347,8 @@ uint64_t dbuf_whichblock(const struct dnode *di, const int64_t level,
     const uint64_t offset);
 
 void dbuf_create_bonus(struct dnode *dn);
-int dbuf_spill_set_blksz(dmu_buf_t *db, uint64_t blksz, dmu_tx_t *tx);
+int dbuf_spill_set_blksz(dmu_buf_t *db, uint64_t blksz, boolean_t copy,
+    dmu_tx_t *tx);
 
 void dbuf_rm_spill(struct dnode *dn, dmu_tx_t *tx);
 
@@ -414,7 +415,7 @@ void dbuf_free_range(struct dnode *dn, uint64_t start, uint64_t end,
 void dbuf_evict_range(struct dnode *dn, uint64_t start_blkid,
     uint64_t end_blkid);
 
-void dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx);
+void dbuf_new_size(dmu_buf_impl_t *db, int size, boolean_t copy, dmu_tx_t *tx);
 
 void dbuf_stats_init(dbuf_hash_table_t *hash);
 void dbuf_stats_destroy(void);

include/sys/dmu.h

@@ -314,6 +314,14 @@ void zap_byteswap(void *buf, size_t size);
 void zfs_oldacl_byteswap(void *buf, size_t size);
 void zfs_acl_byteswap(void *buf, size_t size);
 void zfs_znode_byteswap(void *buf, size_t size);
+void abd_byteswap_uint64_array(abd_t *abd, size_t size);
+void abd_byteswap_uint32_array(abd_t *abd, size_t size);
+void abd_byteswap_uint16_array(abd_t *abd, size_t size);
+void abd_byteswap_uint8_array(abd_t *abd, size_t size);
+void abd_zap_byteswap(abd_t *abd, size_t size);
+void abd_zfs_oldacl_byteswap(abd_t *abd, size_t size);
+void abd_zfs_acl_byteswap(abd_t *abd, size_t size);
+void abd_zfs_znode_byteswap(abd_t *abd, size_t size);
 
 #define	DS_FIND_SNAPSHOTS	(1<<0)
 #define	DS_FIND_CHILDREN	(1<<1)
@@ -366,14 +374,15 @@ int dmu_objset_snapshot_one(const char *fsname, const char *snapname);
 int dmu_objset_find(const char *name, int func(const char *, void *), void *arg,
     int flags);
 void dmu_objset_byteswap(void *buf, size_t size);
+void abd_dmu_objset_byteswap(abd_t *abd, size_t size);
 int dsl_dataset_rename_snapshot(const char *fsname,
     const char *oldsnapname, const char *newsnapname, boolean_t recursive);
 
 typedef struct dmu_buf {
 	uint64_t db_object;		/* object that this buffer is part of */
 	uint64_t db_offset;		/* byte offset in this object */
 	uint64_t db_size;		/* size of buffer in bytes */
-	void *db_data;			/* data in buffer */
+	abd_t *db_abd;			/* data, must be linear for metadata */
 } dmu_buf_t;
 
 /*
@@ -617,7 +626,7 @@ int dmu_spill_hold_existing(dmu_buf_t *bonus, const void *tag, dmu_buf_t **dbp);
  *
  * You must call dmu_buf_read, dmu_buf_will_dirty, or dmu_buf_will_fill
  * on the returned buffer before reading or writing the buffer's
- * db_data.  The comments for those routines describe what particular
+ * db_abd.  The comments for those routines describe what particular
  * operations are valid after calling them.
  *
  * The object number must be a valid, allocated object number.
@@ -818,7 +827,7 @@ void dmu_buf_user_evict_wait(void);
 struct blkptr *dmu_buf_get_blkptr(dmu_buf_t *db);
 
 /*
- * Indicate that you are going to modify the buffer's data (db_data).
+ * Indicate that you are going to modify the buffer's data (db_abd).
  *
  * The transaction (tx) must be assigned to a txg (ie. you've called
  * dmu_tx_assign()).  The buffer's object must be held in the tx
@@ -926,6 +935,8 @@ int dmu_read_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size, void *buf,
     dmu_flags_t flags);
 void dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
     const void *buf, dmu_tx_t *tx, dmu_flags_t flags);
+void dmu_write_abd(dnode_t *dn, uint64_t offset, uint64_t size,
+    abd_t *abd, dmu_tx_t *tx, dmu_flags_t flags);
 int dmu_write_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size,
     const void *buf, dmu_tx_t *tx, dmu_flags_t flags);
 void dmu_prealloc(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
@@ -991,6 +1002,7 @@ typedef struct dmu_object_info {
 } dmu_object_info_t;
 
 typedef void (*const arc_byteswap_func_t)(void *buf, size_t size);
+typedef void (*const abd_byteswap_func_t)(abd_t *abd, size_t size);
 
 typedef struct dmu_object_type_info {
 	dmu_object_byteswap_t	ot_byteswap;
@@ -1002,6 +1014,7 @@ typedef struct dmu_object_type_info {
 
 typedef const struct dmu_object_byteswap_info {
 	arc_byteswap_func_t	 ob_func;
+	abd_byteswap_func_t	 ob_abd_func;
 	const char		*ob_name;
 } dmu_object_byteswap_info_t;
 

include/sys/dmu_impl.h

@@ -269,7 +269,7 @@ void dmu_object_free_zapified(objset_t *, uint64_t, dmu_tx_t *);
 
 int dmu_write_direct(zio_t *, dmu_buf_impl_t *, abd_t *, dmu_tx_t *);
 int dmu_read_abd(dnode_t *, uint64_t, uint64_t, abd_t *, dmu_flags_t);
-int dmu_write_abd(dnode_t *, uint64_t, uint64_t, abd_t *, dmu_flags_t,
+int dmu_write_abd_direct(dnode_t *, uint64_t, uint64_t, abd_t *, dmu_flags_t,
     dmu_tx_t *);
 #if defined(_KERNEL)
 int dmu_read_uio_direct(dnode_t *, zfs_uio_t *, uint64_t, dmu_flags_t);

include/sys/dnode.h

@@ -290,7 +290,7 @@ struct dnode {
 	uint64_t dn_object;
 	struct dmu_buf_impl *dn_dbuf;
 	struct dnode_handle *dn_handle;
-	dnode_phys_t *dn_phys; /* pointer into dn->dn_dbuf->db.db_data */
+	dnode_phys_t *dn_phys; /* pointer into dn->dn_dbuf->db.db_abd */
 
 	/*
 	 * Copies of stuff in dn_phys.  They're valid in the open
@@ -441,6 +441,7 @@ void dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize,
 void dnode_free(dnode_t *dn, dmu_tx_t *tx);
 void dnode_byteswap(dnode_phys_t *dnp);
 void dnode_buf_byteswap(void *buf, size_t size);
+void abd_dnode_buf_byteswap(abd_t *abd, size_t size);
 void dnode_verify(dnode_t *dn);
 int dnode_set_nlevels(dnode_t *dn, int nlevels, dmu_tx_t *tx);
 int dnode_set_blksz(dnode_t *dn, uint64_t size, int ibs, dmu_tx_t *tx);

include/sys/dsl_dataset.h

@@ -282,7 +282,7 @@ typedef struct dsl_dataset {
 static inline dsl_dataset_phys_t *
 dsl_dataset_phys(dsl_dataset_t *ds)
 {
-	return ((dsl_dataset_phys_t *)ds->ds_dbuf->db_data);
+	return ((dsl_dataset_phys_t *)abd_to_buf(ds->ds_dbuf->db_abd));
 }
 
 typedef struct dsl_dataset_clone_arg_t {

include/sys/dsl_dir.h

@@ -135,7 +135,7 @@ struct dsl_dir {
 static inline dsl_dir_phys_t *
 dsl_dir_phys(dsl_dir_t *dd)
 {
-	return (dd->dd_dbuf->db_data);
+	return (abd_to_buf(dd->dd_dbuf->db_abd));
 }
 
 void dsl_dir_rele(dsl_dir_t *dd, const void *tag);

include/sys/sa_impl.h

@@ -221,11 +221,10 @@ struct sa_handle {
 };
 
 #define	SA_GET_DB(hdl, type)	\
-	(dmu_buf_impl_t *)((type == SA_BONUS) ? hdl->sa_bonus : hdl->sa_spill)
+	((dmu_buf_impl_t *)((type == SA_BONUS) ? hdl->sa_bonus : hdl->sa_spill))
 
 #define	SA_GET_HDR(hdl, type) \
-	((sa_hdr_phys_t *)((dmu_buf_impl_t *)(SA_GET_DB(hdl, \
-	type))->db.db_data))
+	((sa_hdr_phys_t *)abd_to_buf(SA_GET_DB(hdl, type)->db.db_abd))
 
 #define	SA_IDX_TAB_GET(hdl, type) \
 	(type == SA_BONUS ? hdl->sa_bonus_tab : hdl->sa_spill_tab)

include/sys/uio_impl.h

@@ -43,8 +43,14 @@
 #include <sys/uio.h>
 #include <sys/sysmacros.h>
 
+struct abd;
+
 extern int zfs_uiomove(void *, size_t, zfs_uio_rw_t, zfs_uio_t *);
 extern int zfs_uiocopy(void *, size_t, zfs_uio_rw_t, zfs_uio_t *, size_t *);
+extern int zfs_uiomove_abd(struct abd *, size_t, size_t, zfs_uio_rw_t,
+    zfs_uio_t *);
+extern int zfs_uiocopy_abd(struct abd *, size_t, size_t, zfs_uio_rw_t,
+    zfs_uio_t *, size_t *);
 extern void zfs_uioskip(zfs_uio_t *, size_t);
 extern void zfs_uio_free_dio_pages(zfs_uio_t *, zfs_uio_rw_t);
 extern int zfs_uio_get_dio_pages_alloc(zfs_uio_t *, zfs_uio_rw_t);

include/sys/zap_impl.h

@@ -177,13 +177,13 @@ typedef struct zap {
 static inline zap_phys_t *
 zap_f_phys(zap_t *zap)
 {
-	return (zap->zap_dbuf->db_data);
+	return (abd_to_buf(zap->zap_dbuf->db_abd));
 }
 
 static inline mzap_phys_t *
 zap_m_phys(zap_t *zap)
 {
-	return (zap->zap_dbuf->db_data);
+	return (abd_to_buf(zap->zap_dbuf->db_abd));
 }
 
 /*

include/sys/zap_leaf.h

@@ -171,7 +171,7 @@ typedef struct zap_leaf {
 static inline zap_leaf_phys_t *
 zap_leaf_phys(zap_leaf_t *l)
 {
-	return (l->l_dbuf->db_data);
+	return (abd_to_buf(l->l_dbuf->db_abd));
 }
 
 typedef struct zap_entry_handle {

include/sys/zio_checksum.h

@@ -136,6 +136,9 @@ _SYS_ZIO_CHECKSUM_H zio_abd_checksum_func_t fletcher_4_abd_ops;
 extern zio_checksum_t abd_fletcher_4_native;
 extern zio_checksum_t abd_fletcher_4_byteswap;
 
+/* Fletcher 2 */
+extern zio_checksum_t abd_fletcher_2_native;
+
 extern int zio_checksum_equal(spa_t *, blkptr_t *, enum zio_checksum,
     void *, uint64_t, uint64_t, zio_bad_cksum_t *);
 extern void zio_checksum_compute(zio_t *, enum zio_checksum,