10452 ZoL: merge in large dnode feature fixes

Portions contributed by: Ned Bass <bass6@llnl.gov>
Portions contributed by: Tom Caputi <tcaputi@datto.com>
Reviewed by: Giuseppe Di Natale <guss80@gmail.com>
Reviewed by: Alek Pinchuk <apinchuk@datto.com>
Reviewed by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed by: Paul Dagnelie <pcd@delphix.com>
Reviewed by: George Melikov <mail@gmelikov.ru>
Reviewed by: Giuseppe Di Natale <dinatale2@llnl.gov>
Reviewed by: Toomas Soome <toomas@me.com>
Reviewed by: Andy Fiddaman <andy@omniosce.org>
Reviewed by: Gergő Mihály Doma <domag02@gmail.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Approved by: Joshua M. Clulow <josh@sysmgr.org>

15 files changed, 249 insertions, 10 deletions

diff --git a/usr/src/pkg/manifests/system-test-zfstest.mf b/usr/src/pkg/manifests/system-test-zfstest.mf
index c4d70e7980..3b5b21cd90 100644
--- a/usr/src/pkg/manifests/system-test-zfstest.mf
+++ b/usr/src/pkg/manifests/system-test-zfstest.mf
@@ -2548,6 +2548,9 @@ file path=opt/zfs-tests/tests/functional/rsend/send-c_volume mode=0555
 file path=opt/zfs-tests/tests/functional/rsend/send-c_zstreamdump mode=0555
 file path=opt/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize \
     mode=0555
+file path=opt/zfs-tests/tests/functional/rsend/send_freeobjects mode=0555
+file path=opt/zfs-tests/tests/functional/rsend/send_realloc_dnode_size \
+    mode=0555
 file path=opt/zfs-tests/tests/functional/rsend/setup mode=0555
 file path=opt/zfs-tests/tests/functional/scrub_mirror/cleanup mode=0555
 file path=opt/zfs-tests/tests/functional/scrub_mirror/default.cfg mode=0444
diff --git a/usr/src/test/zfs-tests/tests/functional/large_dnode/cleanup.ksh b/usr/src/test/zfs-tests/tests/functional/large_dnode/cleanup.ksh
index 61caf39100..60e481d998 100755
--- a/usr/src/test/zfs-tests/tests/functional/large_dnode/cleanup.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/large_dnode/cleanup.ksh
@@ -20,6 +20,11 @@
 # CDDL HEADER END
 #
 
+#
+# Copyright (c) 2016 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
 . $STF_SUITE/include/libtest.shlib
 
 default_cleanup
diff --git a/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_001_pos.ksh b/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_001_pos.ksh
index c07f4e8d74..d3530292e8 100755
--- a/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_001_pos.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_001_pos.ksh
@@ -20,6 +20,11 @@
 # CDDL HEADER END
 #
 
+#
+# Copyright (c) 2016 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
 . $STF_SUITE/include/libtest.shlib
 
 #
diff --git a/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_002_pos.ksh b/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_002_pos.ksh
index 1dd8d888c3..c2b32ad662 100755
--- a/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_002_pos.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_002_pos.ksh
@@ -20,6 +20,11 @@
 # CDDL HEADER END
 #
 
+#
+# Copyright (c) 2016 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
 . $STF_SUITE/include/libtest.shlib
 
 #
diff --git a/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_003_pos.ksh b/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_003_pos.ksh
index a938c7de7c..20989e1d77 100755
--- a/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_003_pos.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_003_pos.ksh
@@ -20,6 +20,11 @@
 # CDDL HEADER END
 #
 
+#
+# Copyright (c) 2016 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
 . $STF_SUITE/include/libtest.shlib
 
 verify_runnable "both"
diff --git a/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_004_neg.ksh b/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_004_neg.ksh
index 1006ae6af5..3fa1cabe06 100755
--- a/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_004_neg.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_004_neg.ksh
@@ -21,6 +21,11 @@
 #
 
 #
+# Copyright (c) 2016 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
+#
 # Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
diff --git a/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_005_pos.ksh b/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_005_pos.ksh
index 13f1288e4e..a2d92673b1 100755
--- a/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_005_pos.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_005_pos.ksh
@@ -20,6 +20,11 @@
 # CDDL HEADER END
 #
 
+#
+# Copyright (c) 2016 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
 . $STF_SUITE/include/libtest.shlib
 
 verify_runnable "both"
diff --git a/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_006_pos.ksh b/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_006_pos.ksh
index 68fc5e3040..38b4ac52e5 100755
--- a/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_006_pos.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_006_pos.ksh
@@ -21,6 +21,11 @@
 #
 
 #
+# Copyright (c) 2016 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
+#
 # Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
diff --git a/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_007_neg.ksh b/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_007_neg.ksh
index fb4747839f..59364574b1 100755
--- a/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_007_neg.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/large_dnode/large_dnode_007_neg.ksh
@@ -21,6 +21,11 @@
 #
 
 #
+# Copyright (c) 2016 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
+#
 # Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
diff --git a/usr/src/test/zfs-tests/tests/functional/large_dnode/setup.ksh b/usr/src/test/zfs-tests/tests/functional/large_dnode/setup.ksh
index d9b1a6ee85..a9425cca98 100755
--- a/usr/src/test/zfs-tests/tests/functional/large_dnode/setup.ksh
+++ b/usr/src/test/zfs-tests/tests/functional/large_dnode/setup.ksh
@@ -20,6 +20,11 @@
 # CDDL HEADER END
 #
 
+#
+# Copyright (c) 2016 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
 . $STF_SUITE/include/libtest.shlib
 
 DISK=${DISKS%% *}
diff --git a/usr/src/test/zfs-tests/tests/functional/rsend/send_freeobjects.ksh b/usr/src/test/zfs-tests/tests/functional/rsend/send_freeobjects.ksh
new file mode 100755
index 0000000000..6533352a9a
--- /dev/null
+++ b/usr/src/test/zfs-tests/tests/functional/rsend/send_freeobjects.ksh
@@ -0,0 +1,81 @@
+#!/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# Description:
+# Verify FREEOBJECTS record frees sequential objects (See
+# https://github.com/zfsonlinux/zfs/issues/6694)
+#
+# Strategy:
+# 1. Create three files with sequential object numbers, f1 f2 and f3
+# 2. Delete f2
+# 3. Take snapshot A
+# 4. Delete f3
+# 5. Take snapshot B
+# 6. Receive a full send of A
+# 7. Receive an incremental send of B
+# 8. Fail test if f3 exists on received snapshot B
+#
+
+verify_runnable "both"
+
+log_assert "Verify FREEOBJECTS record frees sequential objects"
+
+sendds=sendfo
+recvds=recvfo
+f1=/$POOL/$sendds/f1
+f2=/$POOL/$sendds/f2
+f3=/$POOL/$sendds/f3
+
+#
+# We need to set xattr=sa and dnodesize=legacy to guarantee sequential
+# object numbers for this test. Otherwise, if we used directory-based
+# xattrs, SELinux extended attributes might consume intervening object
+# numbers.
+#
+log_must zfs create -o xattr=sa -o dnodesize=legacy $POOL/$sendds
+
+tries=100
+for ((i=0; i<$tries; i++)); do
+	touch $f1 $f2 $f3
+	o1=$(ls -li $f1 | awk '{print $1}')
+	o2=$(ls -li $f2 | awk '{print $1}')
+	o3=$(ls -li $f3 | awk '{print $1}')
+
+	if [[ $o2 -ne $(( $o1 + 1 )) ]] || [[ $o3 -ne $(( $o2 + 1 )) ]]; then
+		rm -f $f1 $f2 $f3
+	else
+		break
+	fi
+done
+
+if [[ $i -eq $tries ]]; then
+	log_fail "Failed to create three sequential objects"
+fi
+
+log_must rm $f2
+log_must zfs snap $POOL/$sendds@A
+log_must rm $f3
+log_must zfs snap $POOL/$sendds@B
+log_must eval "zfs send $POOL/$sendds@A | zfs recv $POOL/$recvds"
+log_must eval "zfs send -i $POOL/$sendds@A $POOL/$sendds@B |" \
+	"zfs recv $POOL/$recvds"
+log_mustnot zdb $POOL/$recvds@B $o3
+log_pass "Verify FREEOBJECTS record frees sequential objects"
diff --git a/usr/src/test/zfs-tests/tests/functional/rsend/send_realloc_dnode_size.ksh b/usr/src/test/zfs-tests/tests/functional/rsend/send_realloc_dnode_size.ksh
new file mode 100755
index 0000000000..12a72fa092
--- /dev/null
+++ b/usr/src/test/zfs-tests/tests/functional/rsend/send_realloc_dnode_size.ksh
@@ -0,0 +1,111 @@
+#!/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+# Copyright (c) 2018 Datto Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# Description:
+# Verify incremental receive properly handles objects with changed
+# dnode slot count.
+#
+# Strategy:
+# 1. Populate a dataset with 1k byte dnodes and snapshot
+# 2. Remove objects, set dnodesize=legacy, and remount dataset so new objects
+#    get recycled numbers and formerly "interior" dnode slots get assigned
+#    to new objects
+# 3. Remove objects, set dnodesize=2k, and remount dataset so new objects
+#    overlap with recently recycled and formerly "normal" dnode slots get
+#    assigned to new objects
+# 4. Create an empty file and add xattrs to it to exercise reclaiming a
+#    dnode that requires more than 1 slot for its bonus buffer (Zol #7433)
+# 5. Generate initial and incremental streams
+# 6. Verify initial and incremental streams can be received
+#
+
+verify_runnable "both"
+
+log_assert "Verify incremental receive handles objects with changed dnode size"
+
+function cleanup
+{
+	rm -f $BACKDIR/fs-dn-legacy
+	rm -f $BACKDIR/fs-dn-1k
+	rm -f $BACKDIR/fs-dn-2k
+	rm -f $BACKDIR/fs-attr
+
+	if datasetexists $POOL/fs ; then
+		log_must zfs destroy -rR $POOL/fs
+	fi
+
+	if datasetexists $POOL/newfs ; then
+		log_must zfs destroy -rR $POOL/newfs
+	fi
+}
+
+log_onexit cleanup
+
+# 1. Populate a dataset with 1k byte dnodes and snapshot
+log_must zfs create -o dnodesize=1k $POOL/fs
+log_must mk_files 200 262144 0 $POOL/fs
+log_must zfs snapshot $POOL/fs@a
+
+# 2. Remove objects, set dnodesize=legacy, and remount dataset so new objects
+#    get recycled numbers and formerly "interior" dnode slots get assigned
+#    to new objects
+rm /$POOL/fs/*
+
+log_must zfs unmount $POOL/fs
+log_must zfs set dnodesize=legacy $POOL/fs
+log_must zfs mount $POOL/fs
+
+log_must mk_files 200 262144 0 $POOL/fs
+log_must zfs snapshot $POOL/fs@b
+
+# 3. Remove objects, set dnodesize=2k, and remount dataset so new objects
+#    overlap with recently recycled and formerly "normal" dnode slots get
+#    assigned to new objects
+rm /$POOL/fs/*
+
+log_must zfs unmount $POOL/fs
+log_must zfs set dnodesize=2k $POOL/fs
+log_must zfs mount $POOL/fs
+
+log_must touch /$POOL/fs/attrs
+mk_files 200 262144 0 $POOL/fs
+log_must zfs snapshot $POOL/fs@c
+
+# 4. Create an empty file and add xattrs to it to exercise reclaiming a
+#    dnode that requires more than 1 slot for its bonus buffer (Zol #7433)
+log_must zfs set compression=on xattr=sa $POOL/fs
+log_must eval "python -c 'print \"a\" * 512' | attr -s bigval /$POOL/fs/attrs"
+log_must zfs snapshot $POOL/fs@d
+
+# 5. Generate initial and incremental streams
+log_must eval "zfs send $POOL/fs@a > $BACKDIR/fs-dn-1k"
+log_must eval "zfs send -i $POOL/fs@a $POOL/fs@b > $BACKDIR/fs-dn-legacy"
+log_must eval "zfs send -i $POOL/fs@b $POOL/fs@c > $BACKDIR/fs-dn-2k"
+log_must eval "zfs send -i $POOL/fs@c $POOL/fs@d > $BACKDIR/fs-attr"
+
+# 6. Verify initial and incremental streams can be received
+log_must eval "zfs recv $POOL/newfs < $BACKDIR/fs-dn-1k"
+log_must eval "zfs recv $POOL/newfs < $BACKDIR/fs-dn-legacy"
+log_must eval "zfs recv $POOL/newfs < $BACKDIR/fs-dn-2k"
+log_must eval "zfs recv $POOL/newfs < $BACKDIR/fs-attr"
+
+log_pass "Verify incremental receive handles objects with changed dnode size"
diff --git a/usr/src/uts/common/fs/zfs/dmu_object.c b/usr/src/uts/common/fs/zfs/dmu_object.c
index 9895cf4776..f835987e7d 100644
--- a/usr/src/uts/common/fs/zfs/dmu_object.c
+++ b/usr/src/uts/common/fs/zfs/dmu_object.c
@@ -263,7 +263,7 @@ dmu_object_reclaim(objset_t *os, uint64_t object, dmu_object_type_t ot,
     int blocksize, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx)
 {
 	return (dmu_object_reclaim_dnsize(os, object, ot, blocksize, bonustype,
-	    bonuslen, 0, tx));
+	    bonuslen, DNODE_MIN_SIZE, tx));
 }
 
 int
diff --git a/usr/src/uts/common/fs/zfs/dmu_send.c b/usr/src/uts/common/fs/zfs/dmu_send.c
index 8a7e1c56f9..669435aa05 100644
--- a/usr/src/uts/common/fs/zfs/dmu_send.c
+++ b/usr/src/uts/common/fs/zfs/dmu_send.c
@@ -2205,11 +2205,13 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro,
 	} else if (drro->drr_type != doi.doi_type ||
 	    drro->drr_blksz != doi.doi_data_block_size ||
 	    drro->drr_bonustype != doi.doi_bonus_type ||
-	    drro->drr_bonuslen != doi.doi_bonus_size) {
+	    drro->drr_bonuslen != doi.doi_bonus_size ||
+	    drro->drr_dn_slots != (doi.doi_dnodesize >> DNODE_SHIFT)) {
 		/* currently allocated, but with different properties */
-		err = dmu_object_reclaim(rwa->os, drro->drr_object,
+		err = dmu_object_reclaim_dnsize(rwa->os, drro->drr_object,
 		    drro->drr_type, drro->drr_blksz,
-		    drro->drr_bonustype, drro->drr_bonuslen, tx);
+		    drro->drr_bonustype, drro->drr_bonuslen,
+		    drro->drr_dn_slots << DNODE_SHIFT, tx);
 	}
 	if (err != 0) {
 		dmu_tx_commit(tx);
@@ -2259,12 +2261,10 @@ receive_freeobjects(struct receive_writer_arg *rwa,
 		int err;
 
 		err = dmu_object_info(rwa->os, obj, NULL);
-		if (err == ENOENT) {
-			obj++;
+		if (err == ENOENT)
 			continue;
-		} else if (err != 0) {
+		else if (err != 0)
 			return (err);
-		}
 
 		err = dmu_free_long_object(rwa->os, obj);
 		if (err != 0)
diff --git a/usr/src/uts/common/fs/zfs/dnode.c b/usr/src/uts/common/fs/zfs/dnode.c
index 972aebf45c..73b97c2449 100644
--- a/usr/src/uts/common/fs/zfs/dnode.c
+++ b/usr/src/uts/common/fs/zfs/dnode.c
@@ -683,8 +683,7 @@ dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize,
 	ASSERT(DMU_OT_IS_VALID(bonustype));
 	ASSERT3U(bonuslen, <=,
 	    DN_BONUS_SIZE(spa_maxdnodesize(dmu_objset_spa(dn->dn_objset))));
-
-	dn_slots = dn_slots > 0 ? dn_slots : DNODE_MIN_SLOTS;
+	ASSERT3U(bonuslen, <=, DN_BONUS_SIZE(dn_slots << DNODE_SHIFT));
 
 	dnode_free_interior_slots(dn);
 	DNODE_STAT_BUMP(dnode_reallocate);